diff --git a/WORKSPACE b/WORKSPACE index 1d7ced97..5341b094 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -73,12 +73,9 @@ http_archive( http_archive( name = "zlib", build_file = "@//third_party:zlib.BUILD", - sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", - strip_prefix = "zlib-1.2.11", - urls = [ - "http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz", - "http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15 - ], + sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", + strip_prefix = "zlib-1.2.13", + url = "http://zlib.net/fossils/zlib-1.2.13.tar.gz", patches = [ "@//third_party:zlib.diff", ], @@ -157,22 +154,22 @@ http_archive( # 2020-08-21 http_archive( name = "com_github_glog_glog", - strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6", - sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab", + strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372", + sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb", urls = [ - "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip", + "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip", ], ) http_archive( name = "com_github_glog_glog_no_gflags", - strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6", - sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab", + strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372", + sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb", build_file = "@//third_party:glog_no_gflags.BUILD", urls = [ - "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip", + "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip", ], patches = [ - "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff", + "@//third_party:com_github_glog_glog.diff", ], patch_args = [ "-p1", @@ -485,9 +482,10 @@ http_archive( ) # TensorFlow repo should always go after the other external dependencies. -# TF on 2023-05-26. -_TENSORFLOW_GIT_COMMIT = "67d5c561981edc45daf3f9d73ddd1a77963733ca" -_TENSORFLOW_SHA256 = "0c8326285e9cb695313e194b97d388eea70bf8bf5b13e8f0962ca8eed5179ece" +# TF on 2023-07-26. +_TENSORFLOW_GIT_COMMIT = "e92261fd4cec0b726692081c4d2966b75abf31dd" +# curl -L https://github.com/tensorflow/tensorflow/archive/.tar.gz | shasum -a 256 +_TENSORFLOW_SHA256 = "478a229bd4ec70a5b568ac23b5ea013d9fca46a47d6c43e30365a0412b9febf4" http_archive( name = "org_tensorflow", urls = [ @@ -495,6 +493,7 @@ http_archive( ], patches = [ "@//third_party:org_tensorflow_compatibility_fixes.diff", + "@//third_party:org_tensorflow_system_python.diff", # Diff is generated with a script, don't update it manually. "@//third_party:org_tensorflow_custom_ops.diff", ], diff --git a/docs/getting_started/hello_world_cpp.md b/docs/getting_started/hello_world_cpp.md index 7c8f9be3..f0c7ff0f 100644 --- a/docs/getting_started/hello_world_cpp.md +++ b/docs/getting_started/hello_world_cpp.md @@ -50,7 +50,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.* 3. The [`hello world`] example uses a simple MediaPipe graph in the `PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto. - ```C++ + ```c++ absl::Status PrintHelloWorld() { // Configures a simple graph, which concatenates 2 PassThroughCalculators. CalculatorGraphConfig config = ParseTextProtoOrDie(R"( @@ -126,7 +126,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.* ```c++ mediapipe::Packet packet; while (poller.Next(&packet)) { - LOG(INFO) << packet.Get(); + ABSL_LOG(INFO) << packet.Get(); } ``` diff --git a/docs/getting_started/hello_world_ios.md b/docs/getting_started/hello_world_ios.md index 4be09764..118b9a05 100644 --- a/docs/getting_started/hello_world_ios.md +++ b/docs/getting_started/hello_world_ios.md @@ -138,7 +138,7 @@ Create a `BUILD` file in the `$APPLICATION_PATH` and add the following build rules: ``` -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" load( "@build_bazel_rules_apple//apple:ios.bzl", diff --git a/mediapipe/BUILD b/mediapipe/BUILD index fd0cbab3..432ed18f 100644 --- a/mediapipe/BUILD +++ b/mediapipe/BUILD @@ -14,81 +14,155 @@ licenses(["notice"]) # Apache 2.0 -# Note: yes, these need to use "//external:android/crosstool", not -# @androidndk//:default_crosstool. +load("@mediapipe//mediapipe:platforms.bzl", "config_setting_and_platform") +# Generic Android config_setting( name = "android", - values = {"crosstool_top": "//external:android/crosstool"}, + constraint_values = [ + "@platforms//os:android", + ], visibility = ["//visibility:public"], ) -config_setting( +# Android x86 32-bit. +config_setting_and_platform( name = "android_x86", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "x86", - }, + constraint_values = [ + "@platforms//os:android", + "@platforms//cpu:x86_32", + ], visibility = ["//visibility:public"], ) -config_setting( +# Android x86 64-bit. +config_setting_and_platform( name = "android_x86_64", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "x86_64", - }, + constraint_values = [ + "@platforms//os:android", + "@platforms//cpu:x86_64", + ], visibility = ["//visibility:public"], ) -config_setting( - name = "android_armeabi", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "armeabi", - }, - visibility = ["//visibility:public"], -) - -config_setting( +# Android ARMv7. +config_setting_and_platform( name = "android_arm", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "armeabi-v7a", - }, + constraint_values = [ + "@platforms//os:android", + "@platforms//cpu:armv7", + ], visibility = ["//visibility:public"], ) -config_setting( +# Android ARM64. +config_setting_and_platform( name = "android_arm64", - values = { - "crosstool_top": "//external:android/crosstool", - "cpu": "arm64-v8a", - }, + constraint_values = [ + "@platforms//os:android", + "@platforms//cpu:arm64", + ], visibility = ["//visibility:public"], ) -# Note: this cannot just match "apple_platform_type": "macos" because that option -# defaults to "macos" even when building on Linux! -alias( +# Generic MacOS. +config_setting( name = "macos", - actual = select({ - ":macos_i386": ":macos_i386", - ":macos_x86_64": ":macos_x86_64", - ":macos_arm64": ":macos_arm64", - "//conditions:default": ":macos_i386", # Arbitrarily chosen from above. - }), + constraint_values = [ + "@platforms//os:macos", + ], visibility = ["//visibility:public"], ) -# Note: this also matches on crosstool_top so that it does not produce ambiguous -# selectors when used together with "android". +# MacOS x86 64-bit. +config_setting_and_platform( + name = "macos_x86_64", + constraint_values = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + visibility = ["//visibility:public"], +) + +# MacOS ARM64. +config_setting_and_platform( + name = "macos_arm64", + constraint_values = [ + "@platforms//os:macos", + "@platforms//cpu:arm64", + ], + visibility = ["//visibility:public"], +) + +# Generic iOS. config_setting( name = "ios", - values = { - "crosstool_top": "@bazel_tools//tools/cpp:toolchain", - "apple_platform_type": "ios", - }, + constraint_values = [ + "@platforms//os:ios", + ], + visibility = ["//visibility:public"], +) + +# iOS device ARM32. +config_setting_and_platform( + name = "ios_armv7", + constraint_values = [ + "@platforms//os:ios", + "@platforms//cpu:arm", + ], + visibility = ["//visibility:public"], +) + +# iOS device ARM64. +config_setting_and_platform( + name = "ios_arm64", + constraint_values = [ + "@platforms//os:ios", + "@platforms//cpu:arm64", + ], + visibility = ["//visibility:public"], +) + +# iOS device ARM64E. +config_setting_and_platform( + name = "ios_arm64e", + constraint_values = [ + "@platforms//os:ios", + "@platforms//cpu:arm64e", + ], + visibility = ["//visibility:public"], +) + +# iOS simulator x86 32-bit. +config_setting_and_platform( + name = "ios_i386", + constraint_values = [ + "@platforms//os:ios", + "@platforms//cpu:x86_32", + "@build_bazel_apple_support//constraints:simulator", + ], + visibility = ["//visibility:public"], +) + +# iOS simulator x86 64-bit. +config_setting_and_platform( + name = "ios_x86_64", + constraint_values = [ + "@platforms//os:ios", + "@platforms//cpu:x86_64", + "@build_bazel_apple_support//constraints:simulator", + ], + visibility = ["//visibility:public"], +) + +# iOS simulator ARM64. +config_setting_and_platform( + name = "ios_sim_arm64", + constraint_values = [ + "@platforms//os:ios", + "@platforms//cpu:arm64", + "@build_bazel_apple_support//constraints:simulator", + ], visibility = ["//visibility:public"], ) @@ -102,52 +176,24 @@ alias( visibility = ["//visibility:public"], ) -config_setting( - name = "macos_i386", - values = { - "apple_platform_type": "macos", - "cpu": "darwin", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "macos_x86_64", - values = { - "apple_platform_type": "macos", - "cpu": "darwin_x86_64", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "macos_arm64", - values = { - "apple_platform_type": "macos", - "cpu": "darwin_arm64", - }, - visibility = ["//visibility:public"], -) - -[ - config_setting( - name = arch, - values = {"cpu": arch}, - visibility = ["//visibility:public"], - ) - for arch in [ - "ios_i386", - "ios_x86_64", - "ios_armv7", - "ios_arm64", - "ios_arm64e", - "ios_sim_arm64", - ] -] - -config_setting( +# Windows 64-bit. +config_setting_and_platform( name = "windows", - values = {"cpu": "x64_windows"}, + constraint_values = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + visibility = ["//visibility:public"], +) + +# Linux 64-bit. +config_setting_and_platform( + name = "linux", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + visibility = ["//visibility:public"], ) exports_files( diff --git a/mediapipe/calculators/audio/BUILD b/mediapipe/calculators/audio/BUILD index 4a8f0f59..c12583e5 100644 --- a/mediapipe/calculators/audio/BUILD +++ b/mediapipe/calculators/audio/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Placeholder: load py_proto_library load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") licenses(["notice"]) @@ -145,6 +146,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", "//mediapipe/util:time_series_util", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_audio_tools//audio/dsp/mfcc", "@eigen_archive//:eigen3", @@ -163,8 +165,9 @@ cc_library( "//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/util:time_series_util", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_audio_tools//audio/dsp:resampler", "@com_google_audio_tools//audio/dsp:resampler_q", @@ -185,6 +188,7 @@ cc_library( "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:status", "//mediapipe/util:time_series_util", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) @@ -219,13 +223,12 @@ cc_library( deps = [ ":time_series_framer_calculator_cc_proto", "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:timestamp", "//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:time_series_header_cc_proto", - "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", "//mediapipe/util:time_series_util", + "@com_google_absl//absl/log:absl_check", "@com_google_audio_tools//audio/dsp:window_functions", "@eigen_archive//:eigen3", ], @@ -296,6 +299,7 @@ cc_test( "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:status", "//mediapipe/util:time_series_test_util", + "@com_google_absl//absl/log:absl_log", "@com_google_audio_tools//audio/dsp:number_util", "@eigen_archive//:eigen3", ], @@ -319,6 +323,21 @@ cc_test( ], ) +cc_binary( + name = "time_series_framer_calculator_benchmark", + srcs = ["time_series_framer_calculator_benchmark.cc"], + deps = [ + ":time_series_framer_calculator", + ":time_series_framer_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:time_series_header_cc_proto", + "@com_google_absl//absl/log:absl_check", + "@com_google_benchmark//:benchmark", + ], +) + cc_test( name = "time_series_framer_calculator_test", srcs = ["time_series_framer_calculator_test.cc"], @@ -333,6 +352,7 @@ cc_test( "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:status", "//mediapipe/util:time_series_test_util", + "@com_google_absl//absl/log:absl_log", "@com_google_audio_tools//audio/dsp:window_functions", "@eigen_archive//:eigen3", ], diff --git a/mediapipe/calculators/audio/mfcc_mel_calculators.cc b/mediapipe/calculators/audio/mfcc_mel_calculators.cc index a63b9d6e..ec936c84 100644 --- a/mediapipe/calculators/audio/mfcc_mel_calculators.cc +++ b/mediapipe/calculators/audio/mfcc_mel_calculators.cc @@ -23,6 +23,7 @@ #include #include "Eigen/Core" +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" @@ -138,7 +139,7 @@ absl::Status FramewiseTransformCalculatorBase::Process(CalculatorContext* cc) { TransformFrame(input_frame, &output_frame); // Copy output from vector to Eigen::Vector. - CHECK_EQ(output_frame.size(), num_output_channels_); + ABSL_CHECK_EQ(output_frame.size(), num_output_channels_); Eigen::Map output_frame_map(&output_frame[0], output_frame.size(), 1); output->col(frame) = output_frame_map.cast(); diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator.cc b/mediapipe/calculators/audio/rational_factor_resample_calculator.cc index 1a4210c3..e01bf526 100644 --- a/mediapipe/calculators/audio/rational_factor_resample_calculator.cc +++ b/mediapipe/calculators/audio/rational_factor_resample_calculator.cc @@ -16,6 +16,8 @@ #include "mediapipe/calculators/audio/rational_factor_resample_calculator.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "audio/dsp/resampler_q.h" using audio_dsp::Resampler; @@ -45,9 +47,9 @@ void CopyVectorToChannel(const std::vector& vec, Matrix* matrix, if (matrix->cols() == 0) { matrix->resize(matrix->rows(), vec.size()); } else { - CHECK_EQ(vec.size(), matrix->cols()); + ABSL_CHECK_EQ(vec.size(), matrix->cols()); } - CHECK_LT(channel, matrix->rows()); + ABSL_CHECK_LT(channel, matrix->rows()); matrix->row(channel) = Eigen::Map(vec.data(), vec.size()); } @@ -77,7 +79,7 @@ absl::Status RationalFactorResampleCalculator::Open(CalculatorContext* cc) { r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_, resample_options); if (!r) { - LOG(ERROR) << "Failed to initialize resampler."; + ABSL_LOG(ERROR) << "Failed to initialize resampler."; return absl::UnknownError("Failed to initialize resampler."); } } diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator.h b/mediapipe/calculators/audio/rational_factor_resample_calculator.h index 325886dc..2c9df30b 100644 --- a/mediapipe/calculators/audio/rational_factor_resample_calculator.h +++ b/mediapipe/calculators/audio/rational_factor_resample_calculator.h @@ -27,7 +27,6 @@ #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/time_series_header.pb.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/util/time_series_util.h" namespace mediapipe { diff --git a/mediapipe/calculators/audio/spectrogram_calculator.cc b/mediapipe/calculators/audio/spectrogram_calculator.cc index fbdbbab0..7f6528ec 100644 --- a/mediapipe/calculators/audio/spectrogram_calculator.cc +++ b/mediapipe/calculators/audio/spectrogram_calculator.cc @@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator); // Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0). const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518; +namespace { +std::unique_ptr MakeWindowFun( + const SpectrogramCalculatorOptions::WindowType window_type) { + switch (window_type) { + // The cosine window and square root of Hann are equivalent. + case SpectrogramCalculatorOptions::COSINE: + case SpectrogramCalculatorOptions::SQRT_HANN: + return std::make_unique(); + case SpectrogramCalculatorOptions::HANN: + return std::make_unique(); + case SpectrogramCalculatorOptions::HAMMING: + return std::make_unique(); + } + return nullptr; +} +} // namespace + absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) { SpectrogramCalculatorOptions spectrogram_options = cc->Options(); @@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) { output_scale_ = spectrogram_options.output_scale(); - std::vector window; - switch (spectrogram_options.window_type()) { - case SpectrogramCalculatorOptions::COSINE: - audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_, - &window); - break; - case SpectrogramCalculatorOptions::HANN: - audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_, - &window); - break; - case SpectrogramCalculatorOptions::HAMMING: - audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_, - &window); - break; - case SpectrogramCalculatorOptions::SQRT_HANN: { - audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_, - &window); - absl::c_transform(window, window.begin(), - [](double x) { return std::sqrt(x); }); - break; - } + auto window_fun = MakeWindowFun(spectrogram_options.window_type()); + if (window_fun == nullptr) { + return absl::Status(absl::StatusCode::kInvalidArgument, + absl::StrCat("Invalid window type ", + spectrogram_options.window_type())); } + std::vector window; + window_fun->GetPeriodicSamples(frame_duration_samples_, &window); // Propagate settings down to the actual Spectrogram object. spectrogram_generators_.clear(); diff --git a/mediapipe/calculators/audio/spectrogram_calculator.proto b/mediapipe/calculators/audio/spectrogram_calculator.proto index ddfca1d1..d8bca3f7 100644 --- a/mediapipe/calculators/audio/spectrogram_calculator.proto +++ b/mediapipe/calculators/audio/spectrogram_calculator.proto @@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions { HANN = 0; HAMMING = 1; COSINE = 2; - SQRT_HANN = 4; + SQRT_HANN = 4; // Alias of COSINE. } optional WindowType window_type = 6 [default = HANN]; diff --git a/mediapipe/calculators/audio/spectrogram_calculator_test.cc b/mediapipe/calculators/audio/spectrogram_calculator_test.cc index b35f3058..14cd74a3 100644 --- a/mediapipe/calculators/audio/spectrogram_calculator_test.cc +++ b/mediapipe/calculators/audio/spectrogram_calculator_test.cc @@ -22,6 +22,7 @@ #include #include "Eigen/Core" +#include "absl/log/absl_log.h" #include "audio/dsp/number_util.h" #include "mediapipe/calculators/audio/spectrogram_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -882,11 +883,11 @@ void BM_ProcessDC(benchmark::State& state) { const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0); const Matrix& output_matrix = output.packets[0].Get(); - LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x" - << output_matrix.cols(); - LOG(INFO) << "First values=" << output_matrix(0, 0) << ", " - << output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", " - << output_matrix(3, 0); + ABSL_LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x" + << output_matrix.cols(); + ABSL_LOG(INFO) << "First values=" << output_matrix(0, 0) << ", " + << output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", " + << output_matrix(3, 0); } BENCHMARK(BM_ProcessDC); diff --git a/mediapipe/calculators/audio/stabilized_log_calculator.cc b/mediapipe/calculators/audio/stabilized_log_calculator.cc index 0c697a19..a7de6a37 100644 --- a/mediapipe/calculators/audio/stabilized_log_calculator.cc +++ b/mediapipe/calculators/audio/stabilized_log_calculator.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/matrix.h" @@ -59,7 +60,7 @@ class StabilizedLogCalculator : public CalculatorBase { output_scale_ = stabilized_log_calculator_options.output_scale(); check_nonnegativity_ = stabilized_log_calculator_options.check_nonnegativity(); - CHECK_GE(stabilizer_, 0.0) + ABSL_CHECK_GE(stabilizer_, 0.0) << "stabilizer must be >= 0.0, received a value of " << stabilizer_; // If the input packets have a header, propagate the header to the output. diff --git a/mediapipe/calculators/audio/time_series_framer_calculator.cc b/mediapipe/calculators/audio/time_series_framer_calculator.cc index a200b898..d8cda514 100644 --- a/mediapipe/calculators/audio/time_series_framer_calculator.cc +++ b/mediapipe/calculators/audio/time_series_framer_calculator.cc @@ -15,19 +15,17 @@ // Defines TimeSeriesFramerCalculator. #include -#include -#include -#include +#include #include "Eigen/Core" +#include "absl/log/absl_check.h" #include "audio/dsp/window_functions.h" #include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/timestamp.h" #include "mediapipe/util/time_series_util.h" namespace mediapipe { @@ -88,11 +86,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase { absl::Status Close(CalculatorContext* cc) override; private: - // Adds input data to the internal buffer. - void EnqueueInput(CalculatorContext* cc); - // Constructs and emits framed output packets. - void FrameOutput(CalculatorContext* cc); - Timestamp CurrentOutputTimestamp() { if (use_local_timestamp_) { return current_timestamp_; @@ -106,21 +99,13 @@ class TimeSeriesFramerCalculator : public CalculatorBase { Timestamp::kTimestampUnitsPerSecond); } - // Returns the timestamp of a sample on a base, which is usually the time - // stamp of a packet. - Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base, - int64_t number_of_samples) { - return timestamp_base + round(number_of_samples / sample_rate_ * - Timestamp::kTimestampUnitsPerSecond); - } - // The number of input samples to advance after the current output frame is // emitted. int next_frame_step_samples() const { // All numbers are in input samples. const int64_t current_output_frame_start = static_cast( round(cumulative_output_frames_ * average_frame_step_samples_)); - CHECK_EQ(current_output_frame_start, cumulative_completed_samples_); + ABSL_CHECK_EQ(current_output_frame_start, cumulative_completed_samples_); const int64_t next_output_frame_start = static_cast( round((cumulative_output_frames_ + 1) * average_frame_step_samples_)); return next_output_frame_start - current_output_frame_start; @@ -142,61 +127,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase { Timestamp initial_input_timestamp_; // The current timestamp is updated along with the incoming packets. Timestamp current_timestamp_; - int num_channels_; - // Each entry in this deque consists of a single sample, i.e. a - // single column vector, and its timestamp. - std::deque> sample_buffer_; + // Samples are buffered in a vector of sample blocks. + class SampleBlockBuffer { + public: + // Initializes the buffer. + void Init(double sample_rate, int num_channels) { + ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate; + num_channels_ = num_channels; + num_samples_ = 0; + first_block_offset_ = 0; + } + + // Number of channels, equal to the number of rows in each Matrix. + int num_channels() const { return num_channels_; } + // Total number of available samples over all blocks. + int num_samples() const { return num_samples_; } + + // Pushes a new block of samples on the back of the buffer with `timestamp` + // being the input timestamp of the packet containing the Matrix. + void Push(const Matrix& samples, Timestamp timestamp); + // Copies `count` samples from the front of the buffer. If there are fewer + // samples than this, the result is zero padded to have `count` samples. + // The timestamp of the last copied sample is written to *last_timestamp. + // This output is used below to update `current_timestamp_`, which is only + // used when `use_local_timestamp` is true. + Matrix CopySamples(int count, Timestamp* last_timestamp) const; + // Drops `count` samples from the front of the buffer. If `count` exceeds + // `num_samples()`, the buffer is emptied. Returns how many samples were + // dropped. + int DropSamples(int count); + + private: + struct Block { + // Matrix of num_channels rows by num_samples columns, a block of possibly + // multiple samples. + Matrix samples; + // Timestamp of the first sample in the Block. This comes from the input + // packet's timestamp that contains this Matrix. + Timestamp timestamp; + + Block() : timestamp(Timestamp::Unstarted()) {} + Block(const Matrix& samples, Timestamp timestamp) + : samples(samples), timestamp(timestamp) {} + int num_samples() const { return samples.cols(); } + }; + std::vector blocks_; + // Number of timestamp units per sample. Used to compute timestamps as + // nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n). + double ts_units_per_sample_; + // Number of rows in each Matrix. + int num_channels_; + // The total number of samples over all blocks, equal to + // (sum_i blocks_[i].num_samples()) - first_block_offset_. + int num_samples_; + // The number of samples in the first block that have been discarded. This + // way we can cheaply represent "partially discarding" a block. + int first_block_offset_; + } sample_buffer_; bool use_window_; - Matrix window_; + Eigen::RowVectorXf window_; bool use_local_timestamp_; }; REGISTER_CALCULATOR(TimeSeriesFramerCalculator); -void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) { - const Matrix& input_frame = cc->Inputs().Index(0).Get(); - - for (int i = 0; i < input_frame.cols(); ++i) { - sample_buffer_.emplace_back(std::make_pair( - input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i))); - } +void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples, + Timestamp timestamp) { + num_samples_ += samples.cols(); + blocks_.emplace_back(samples, timestamp); } -void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) { - while (sample_buffer_.size() >= +Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples( + int count, Timestamp* last_timestamp) const { + Matrix copied(num_channels_, count); + + if (!blocks_.empty()) { + int num_copied = 0; + // First block has an offset for samples that have been discarded. + int offset = first_block_offset_; + int n; + Timestamp last_block_ts; + int last_sample_index; + + for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) { + n = std::min(it->num_samples() - offset, count); + // Copy `n` samples from the next block. + copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n); + count -= n; + num_copied += n; + last_block_ts = it->timestamp; + last_sample_index = offset + n - 1; + offset = 0; // No samples have been discarded in subsequent blocks. + } + + // Compute the timestamp of the last copied sample. + *last_timestamp = + last_block_ts + std::round(ts_units_per_sample_ * last_sample_index); + } + + if (count > 0) { + copied.rightCols(count).setZero(); // Zero pad if needed. + } + + return copied; +} + +int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) { + if (blocks_.empty()) { + return 0; + } + + auto block_it = blocks_.begin(); + if (first_block_offset_ + count < block_it->num_samples()) { + // `count` is less than the remaining samples in the first block. + first_block_offset_ += count; + num_samples_ -= count; + return count; + } + + int num_samples_dropped = block_it->num_samples() - first_block_offset_; + count -= num_samples_dropped; + first_block_offset_ = 0; + + for (++block_it; block_it != blocks_.end(); ++block_it) { + if (block_it->num_samples() > count) { + break; + } + num_samples_dropped += block_it->num_samples(); + count -= block_it->num_samples(); + } + + blocks_.erase(blocks_.begin(), block_it); // Drop whole blocks. + if (!blocks_.empty()) { + first_block_offset_ = count; // Drop part of the next block. + num_samples_dropped += count; + } + + num_samples_ -= num_samples_dropped; + return num_samples_dropped; +} + +absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) { + if (initial_input_timestamp_ == Timestamp::Unstarted()) { + initial_input_timestamp_ = cc->InputTimestamp(); + current_timestamp_ = initial_input_timestamp_; + } + + // Add input data to the internal buffer. + sample_buffer_.Push(cc->Inputs().Index(0).Get(), + cc->InputTimestamp()); + + // Construct and emit framed output packets. + while (sample_buffer_.num_samples() >= frame_duration_samples_ + samples_still_to_drop_) { - while (samples_still_to_drop_ > 0) { - sample_buffer_.pop_front(); - --samples_still_to_drop_; - } + sample_buffer_.DropSamples(samples_still_to_drop_); + Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_, + ¤t_timestamp_); const int frame_step_samples = next_frame_step_samples(); - std::unique_ptr output_frame( - new Matrix(num_channels_, frame_duration_samples_)); - for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_); - ++i) { - output_frame->col(i) = sample_buffer_.front().first; - current_timestamp_ = sample_buffer_.front().second; - sample_buffer_.pop_front(); - } - const int frame_overlap_samples = - frame_duration_samples_ - frame_step_samples; - if (frame_overlap_samples > 0) { - for (int i = 0; i < frame_overlap_samples; ++i) { - output_frame->col(i + frame_step_samples) = sample_buffer_[i].first; - current_timestamp_ = sample_buffer_[i].second; - } - } else { - samples_still_to_drop_ = -frame_overlap_samples; - } + samples_still_to_drop_ = frame_step_samples; if (use_window_) { - *output_frame = (output_frame->array() * window_.array()).matrix(); + // Apply the window to each row of output_frame. + output_frame.array().rowwise() *= window_.array(); } - cc->Outputs().Index(0).Add(output_frame.release(), - CurrentOutputTimestamp()); + cc->Outputs().Index(0).AddPacket(MakePacket(std::move(output_frame)) + .At(CurrentOutputTimestamp())); ++cumulative_output_frames_; cumulative_completed_samples_ += frame_step_samples; } @@ -206,35 +304,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) { // fact to enable packet queueing optimizations. cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp()); } -} - -absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) { - if (initial_input_timestamp_ == Timestamp::Unstarted()) { - initial_input_timestamp_ = cc->InputTimestamp(); - current_timestamp_ = initial_input_timestamp_; - } - - EnqueueInput(cc); - FrameOutput(cc); return absl::OkStatus(); } absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) { - while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) { - sample_buffer_.pop_front(); - --samples_still_to_drop_; - } - if (!sample_buffer_.empty() && pad_final_packet_) { - std::unique_ptr output_frame(new Matrix); - output_frame->setZero(num_channels_, frame_duration_samples_); - for (int i = 0; i < sample_buffer_.size(); ++i) { - output_frame->col(i) = sample_buffer_[i].first; - current_timestamp_ = sample_buffer_[i].second; - } + sample_buffer_.DropSamples(samples_still_to_drop_); - cc->Outputs().Index(0).Add(output_frame.release(), - CurrentOutputTimestamp()); + if (sample_buffer_.num_samples() > 0 && pad_final_packet_) { + Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_, + ¤t_timestamp_); + cc->Outputs().Index(0).AddPacket(MakePacket(std::move(output_frame)) + .At(CurrentOutputTimestamp())); } return absl::OkStatus(); @@ -258,7 +339,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) { cc->Inputs().Index(0).Header(), &input_header)); sample_rate_ = input_header.sample_rate(); - num_channels_ = input_header.num_channels(); + sample_buffer_.Init(sample_rate_, input_header.num_channels()); frame_duration_samples_ = time_series_util::SecondsToSamples( framer_options.frame_duration_seconds(), sample_rate_); RET_CHECK_GT(frame_duration_samples_, 0) @@ -312,9 +393,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) { } if (use_window_) { - window_ = Matrix::Ones(num_channels_, 1) * - Eigen::Map(window_vector.data(), 1, - frame_duration_samples_) + window_ = Eigen::Map(window_vector.data(), + frame_duration_samples_) .cast(); } use_local_timestamp_ = framer_options.use_local_timestamp(); diff --git a/mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc b/mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc new file mode 100644 index 00000000..6eada1ad --- /dev/null +++ b/mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc @@ -0,0 +1,93 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Benchmark for TimeSeriesFramerCalculator. +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "benchmark/benchmark.h" +#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/time_series_header.pb.h" +#include "mediapipe/framework/packet.h" + +using ::mediapipe::Matrix; + +void BM_TimeSeriesFramerCalculator(benchmark::State& state) { + constexpr float kSampleRate = 32000.0; + constexpr int kNumChannels = 2; + constexpr int kFrameDurationSeconds = 5.0; + std::mt19937 rng(0 /*seed*/); + // Input around a half second's worth of samples at a time. + std::uniform_int_distribution input_size_dist(15000, 17000); + // Generate a pool of random blocks of samples up front. + std::vector sample_pool; + sample_pool.reserve(20); + for (int i = 0; i < 20; ++i) { + sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng))); + } + std::uniform_int_distribution pool_index_dist(0, sample_pool.size() - 1); + + mediapipe::CalculatorGraphConfig config; + config.add_input_stream("input"); + config.add_output_stream("output"); + auto* node = config.add_node(); + node->set_calculator("TimeSeriesFramerCalculator"); + node->add_input_stream("input"); + node->add_output_stream("output"); + mediapipe::TimeSeriesFramerCalculatorOptions* options = + node->mutable_options()->MutableExtension( + mediapipe::TimeSeriesFramerCalculatorOptions::ext); + options->set_frame_duration_seconds(kFrameDurationSeconds); + + for (auto _ : state) { + state.PauseTiming(); // Pause benchmark timing. + + // Prepare input packets of random blocks of samples. + std::vector input_packets; + input_packets.reserve(32); + float t = 0; + for (int i = 0; i < 32; ++i) { + auto samples = + std::make_unique(sample_pool[pool_index_dist(rng)]); + const int num_samples = samples->cols(); + input_packets.push_back(mediapipe::Adopt(samples.release()) + .At(mediapipe::Timestamp::FromSeconds(t))); + t += num_samples / kSampleRate; + } + // Initialize graph. + mediapipe::CalculatorGraph graph; + ABSL_CHECK_OK(graph.Initialize(config)); + // Prepare input header. + auto header = std::make_unique(); + header->set_sample_rate(kSampleRate); + header->set_num_channels(kNumChannels); + + state.ResumeTiming(); // Resume benchmark timing. + + ABSL_CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}})); + for (auto& packet : input_packets) { + ABSL_CHECK_OK(graph.AddPacketToInputStream("input", packet)); + } + ABSL_CHECK(!graph.HasError()); + ABSL_CHECK_OK(graph.CloseAllInputStreams()); + ABSL_CHECK_OK(graph.WaitUntilIdle()); + } +} +BENCHMARK(BM_TimeSeriesFramerCalculator); + +BENCHMARK_MAIN(); diff --git a/mediapipe/calculators/audio/time_series_framer_calculator_test.cc b/mediapipe/calculators/audio/time_series_framer_calculator_test.cc index 72e9c88f..fe42ecb1 100644 --- a/mediapipe/calculators/audio/time_series_framer_calculator_test.cc +++ b/mediapipe/calculators/audio/time_series_framer_calculator_test.cc @@ -19,6 +19,7 @@ #include #include "Eigen/Core" +#include "absl/log/absl_log.h" #include "audio/dsp/window_functions.h" #include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -186,11 +187,12 @@ class TimeSeriesFramerCalculatorTest const int num_unique_output_samples = round((output().packets.size() - 1) * frame_step_samples) + frame_duration_samples; - LOG(INFO) << "packets.size()=" << output().packets.size() - << " frame_duration_samples=" << frame_duration_samples - << " frame_step_samples=" << frame_step_samples - << " num_input_samples_=" << num_input_samples_ - << " num_unique_output_samples=" << num_unique_output_samples; + ABSL_LOG(INFO) << "packets.size()=" << output().packets.size() + << " frame_duration_samples=" << frame_duration_samples + << " frame_step_samples=" << frame_step_samples + << " num_input_samples_=" << num_input_samples_ + << " num_unique_output_samples=" + << num_unique_output_samples; const int num_padding_samples = num_unique_output_samples - num_input_samples_; if (options_.pad_final_packet()) { diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index d3e63e38..02efc84e 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -117,6 +117,7 @@ mediapipe_proto_library( "//mediapipe/framework:calculator_proto", "//mediapipe/framework/formats:classification_proto", "//mediapipe/framework/formats:landmark_proto", + "//mediapipe/framework/formats:matrix_data_proto", "//mediapipe/framework/formats:time_series_header_proto", ], ) @@ -289,6 +290,7 @@ cc_library( "//mediapipe/framework/api2:node", "//mediapipe/framework/api2:port", "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:integral_types", @@ -379,17 +381,6 @@ cc_library( alwayslink = 1, ) -cc_library( - name = "clip_detection_vector_size_calculator", - srcs = ["clip_detection_vector_size_calculator.cc"], - deps = [ - ":clip_vector_size_calculator", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:detection_cc_proto", - ], - alwayslink = 1, -) - cc_test( name = "clip_vector_size_calculator_test", srcs = ["clip_vector_size_calculator_test.cc"], @@ -591,6 +582,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:options_util", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) @@ -606,6 +598,7 @@ cc_test( "//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:integral_types", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -638,6 +631,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) @@ -785,10 +779,11 @@ cc_library( "//mediapipe/framework/deps:random", "//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:options_util", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -844,6 +839,7 @@ cc_test( "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:logging", "//mediapipe/framework/tool:validate_type", + "@com_google_absl//absl/log:absl_check", "@eigen_archive//:eigen3", ], ) @@ -1031,6 +1027,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/api2:node", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) @@ -1069,6 +1066,7 @@ cc_test( "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", + "@com_google_absl//absl/log:absl_log", ], ) @@ -1115,6 +1113,7 @@ cc_library( "//mediapipe/framework/api2:node", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) @@ -1167,6 +1166,7 @@ cc_library( "//mediapipe/framework:collection_item_id", "//mediapipe/framework/formats:classification_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:matrix_data_cc_proto", "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:ret_check", diff --git a/mediapipe/calculators/core/begin_loop_calculator.cc b/mediapipe/calculators/core/begin_loop_calculator.cc index 7da90989..d030bbbd 100644 --- a/mediapipe/calculators/core/begin_loop_calculator.cc +++ b/mediapipe/calculators/core/begin_loop_calculator.cc @@ -76,4 +76,9 @@ REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator); // A calculator to process std::vector. typedef BeginLoopCalculator> BeginLoopImageCalculator; REGISTER_CALCULATOR(BeginLoopImageCalculator); + +// A calculator to process std::vector. +typedef BeginLoopCalculator> BeginLoopFloatCalculator; +REGISTER_CALCULATOR(BeginLoopFloatCalculator); + } // namespace mediapipe diff --git a/mediapipe/calculators/core/concatenate_vector_calculator.cc b/mediapipe/calculators/core/concatenate_vector_calculator.cc index 4d0d6620..53b3debf 100644 --- a/mediapipe/calculators/core/concatenate_vector_calculator.cc +++ b/mediapipe/calculators/core/concatenate_vector_calculator.cc @@ -17,6 +17,7 @@ #include #include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port/integral_types.h" @@ -104,4 +105,7 @@ typedef ConcatenateVectorCalculator ConcatenateRenderDataVectorCalculator; MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator); +typedef ConcatenateVectorCalculator + ConcatenateImageVectorCalculator; +MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator); } // namespace mediapipe diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.cc b/mediapipe/calculators/core/constant_side_packet_calculator.cc index 509f7e9d..8762c987 100644 --- a/mediapipe/calculators/core/constant_side_packet_calculator.cc +++ b/mediapipe/calculators/core/constant_side_packet_calculator.cc @@ -19,6 +19,7 @@ #include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix_data.pb.h" #include "mediapipe/framework/formats/time_series_header.pb.h" #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/integral_types.h" @@ -85,8 +86,12 @@ class ConstantSidePacketCalculator : public CalculatorBase { packet.Set(); } else if (packet_options.has_double_value()) { packet.Set(); + } else if (packet_options.has_matrix_data_value()) { + packet.Set(); } else if (packet_options.has_time_series_header_value()) { packet.Set(); + } else if (packet_options.has_int64_value()) { + packet.Set(); } else { return absl::InvalidArgumentError( "None of supported values were specified in options."); @@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase { MakePacket(packet_options.landmark_list_value())); } else if (packet_options.has_double_value()) { packet.Set(MakePacket(packet_options.double_value())); + } else if (packet_options.has_matrix_data_value()) { + packet.Set(MakePacket(packet_options.matrix_data_value())); } else if (packet_options.has_time_series_header_value()) { packet.Set(MakePacket( packet_options.time_series_header_value())); + } else if (packet_options.has_int64_value()) { + packet.Set(MakePacket(packet_options.int64_value())); } else { return absl::InvalidArgumentError( "None of supported values were specified in options."); diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.proto b/mediapipe/calculators/core/constant_side_packet_calculator.proto index 78a773a6..0d53175f 100644 --- a/mediapipe/calculators/core/constant_side_packet_calculator.proto +++ b/mediapipe/calculators/core/constant_side_packet_calculator.proto @@ -19,6 +19,7 @@ package mediapipe; import "mediapipe/framework/calculator.proto"; import "mediapipe/framework/formats/classification.proto"; import "mediapipe/framework/formats/landmark.proto"; +import "mediapipe/framework/formats/matrix_data.proto"; import "mediapipe/framework/formats/time_series_header.proto"; message ConstantSidePacketCalculatorOptions { @@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions { message ConstantSidePacket { oneof value { int32 int_value = 1; + uint64 uint64_value = 5; + int64 int64_value = 11; float float_value = 2; + double double_value = 9; bool bool_value = 3; string string_value = 4; - uint64 uint64_value = 5; ClassificationList classification_list_value = 6; LandmarkList landmark_list_value = 7; - double double_value = 9; TimeSeriesHeader time_series_header_value = 10; + MatrixData matrix_data_value = 12; } } diff --git a/mediapipe/calculators/core/constant_side_packet_calculator_test.cc b/mediapipe/calculators/core/constant_side_packet_calculator_test.cc index a7ff808f..6e8c0ec3 100644 --- a/mediapipe/calculators/core/constant_side_packet_calculator_test.cc +++ b/mediapipe/calculators/core/constant_side_packet_calculator_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include "absl/strings/string_view.h" @@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) { DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f); DoTestSingleSidePacket("{ bool_value: true }", true); DoTestSingleSidePacket(R"({ string_value: "str" })", "str"); + DoTestSingleSidePacket("{ int64_value: 63 }", 63); } TEST(ConstantSidePacketCalculatorTest, MultiplePackets) { diff --git a/mediapipe/calculators/core/end_loop_calculator.cc b/mediapipe/calculators/core/end_loop_calculator.cc index 752580cf..94f7ee22 100644 --- a/mediapipe/calculators/core/end_loop_calculator.cc +++ b/mediapipe/calculators/core/end_loop_calculator.cc @@ -14,6 +14,8 @@ #include "mediapipe/calculators/core/end_loop_calculator.h" +#include +#include #include #include "mediapipe/framework/formats/classification.pb.h" @@ -84,4 +86,8 @@ typedef EndLoopCalculator>> EndLoopAffineMatrixCalculator; REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator); +typedef EndLoopCalculator>> + EndLoopImageSizeCalculator; +REGISTER_CALCULATOR(EndLoopImageSizeCalculator); + } // namespace mediapipe diff --git a/mediapipe/calculators/core/gate_calculator_test.cc b/mediapipe/calculators/core/gate_calculator_test.cc index 8875bd7e..0c49f144 100644 --- a/mediapipe/calculators/core/gate_calculator_test.cc +++ b/mediapipe/calculators/core/gate_calculator_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/port/gtest.h" @@ -356,18 +357,18 @@ TEST_F(GateCalculatorTest, AllowWithStateChangeNoDataStreams) { RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true); constexpr int64_t kTimestampValue3 = 45; RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false); - LOG(INFO) << "a"; + ABSL_LOG(INFO) << "a"; const std::vector& output = runner()->Outputs().Get("STATE_CHANGE", 0).packets; - LOG(INFO) << "s"; + ABSL_LOG(INFO) << "s"; ASSERT_EQ(2, output.size()); - LOG(INFO) << "d"; + ABSL_LOG(INFO) << "d"; EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value()); EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value()); - LOG(INFO) << "f"; + ABSL_LOG(INFO) << "f"; EXPECT_EQ(true, output[0].Get()); // Allow. EXPECT_EQ(false, output[1].Get()); // Disallow. - LOG(INFO) << "g"; + ABSL_LOG(INFO) << "g"; } TEST_F(GateCalculatorTest, DisallowWithStateChange) { diff --git a/mediapipe/calculators/core/immediate_mux_calculator.cc b/mediapipe/calculators/core/immediate_mux_calculator.cc index 0e51cda5..05de05e4 100644 --- a/mediapipe/calculators/core/immediate_mux_calculator.cc +++ b/mediapipe/calculators/core/immediate_mux_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -78,7 +79,7 @@ absl::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) { if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) { cc->Outputs().Index(0).AddPacket(packet); } else { - LOG_FIRST_N(WARNING, 5) + ABSL_LOG_FIRST_N(WARNING, 5) << "Dropping a packet with timestamp " << packet.Timestamp(); } if (cc->Outputs().NumEntries() >= 2) { diff --git a/mediapipe/calculators/core/matrix_multiply_calculator_test.cc b/mediapipe/calculators/core/matrix_multiply_calculator_test.cc index e62ca807..60976577 100644 --- a/mediapipe/calculators/core/matrix_multiply_calculator_test.cc +++ b/mediapipe/calculators/core/matrix_multiply_calculator_test.cc @@ -16,6 +16,7 @@ #include #include "Eigen/Core" +#include "absl/log/absl_check.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/formats/matrix.h" @@ -209,7 +210,7 @@ TEST(MatrixMultiplyCalculatorTest, Multiply) { MatrixFromTextProto(kSamplesText, &samples); Matrix expected; MatrixFromTextProto(kExpectedText, &expected); - CHECK_EQ(samples.cols(), expected.cols()); + ABSL_CHECK_EQ(samples.cols(), expected.cols()); for (int i = 0; i < samples.cols(); ++i) { // Take a column from samples and produce a packet with just that diff --git a/mediapipe/calculators/core/merge_calculator.cc b/mediapipe/calculators/core/merge_calculator.cc index a283842a..43fc3b87 100644 --- a/mediapipe/calculators/core/merge_calculator.cc +++ b/mediapipe/calculators/core/merge_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_log.h" #include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/ret_check.h" @@ -53,7 +54,7 @@ class MergeCalculator : public Node { static absl::Status UpdateContract(CalculatorContract* cc) { RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream"; if (kIn(cc).Count() == 1) { - LOG(WARNING) + ABSL_LOG(WARNING) << "MergeCalculator expects multiple input streams to merge but is " "receiving only one. Make sure the calculator is configured " "correctly or consider removing this calculator to reduce " @@ -72,8 +73,8 @@ class MergeCalculator : public Node { } } - LOG(WARNING) << "Empty input packets at timestamp " - << cc->InputTimestamp().Value(); + ABSL_LOG(WARNING) << "Empty input packets at timestamp " + << cc->InputTimestamp().Value(); return absl::OkStatus(); } diff --git a/mediapipe/calculators/core/packet_resampler_calculator.cc b/mediapipe/calculators/core/packet_resampler_calculator.cc index 60ec4053..81a68f03 100644 --- a/mediapipe/calculators/core/packet_resampler_calculator.cc +++ b/mediapipe/calculators/core/packet_resampler_calculator.cc @@ -16,6 +16,9 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" + namespace { // Reflect an integer against the lower and upper bound of an interval. int64_t ReflectBetween(int64_t ts, int64_t ts_min, int64_t ts_max) { @@ -177,7 +180,7 @@ PacketResamplerCalculator::GetSamplingStrategy( const PacketResamplerCalculatorOptions& options) { if (options.reproducible_sampling()) { if (!options.jitter_with_reflection()) { - LOG(WARNING) + ABSL_LOG(WARNING) << "reproducible_sampling enabled w/ jitter_with_reflection " "disabled. " << "reproducible_sampling always uses jitter with reflection, " @@ -200,15 +203,15 @@ PacketResamplerCalculator::GetSamplingStrategy( Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp( int64_t index) const { - CHECK_EQ(jitter_, 0.0); - CHECK_NE(first_timestamp_, Timestamp::Unset()); + ABSL_CHECK_EQ(jitter_, 0.0); + ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset()); return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_); } int64_t PacketResamplerCalculator::TimestampToPeriodIndex( Timestamp timestamp) const { - CHECK_EQ(jitter_, 0.0); - CHECK_NE(first_timestamp_, Timestamp::Unset()); + ABSL_CHECK_EQ(jitter_, 0.0); + ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset()); return MathUtil::SafeRound( (timestamp - first_timestamp_).Seconds() * frame_rate_); } @@ -229,13 +232,15 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) { if (resampler_options.output_header() != PacketResamplerCalculatorOptions::NONE) { - LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not " - "the actual value."; + ABSL_LOG(WARNING) + << "VideoHeader::frame_rate holds the target value and not " + "the actual value."; } if (calculator_->flush_last_packet_) { - LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is " - "ignored, because we are adding jitter."; + ABSL_LOG(WARNING) + << "PacketResamplerCalculatorOptions.flush_last_packet is " + "ignored, because we are adding jitter."; } const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get(); @@ -254,7 +259,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) { } absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) { if (!packet_reservoir_->IsEmpty()) { - LOG(INFO) << "Emitting pack from reservoir."; + ABSL_LOG(INFO) << "Emitting pack from reservoir."; calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample()); } return absl::OkStatus(); @@ -285,7 +290,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Process( if (calculator_->frame_time_usec_ < (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) { - LOG_FIRST_N(WARNING, 2) + ABSL_LOG_FIRST_N(WARNING, 2) << "Adding jitter is not very useful when upsampling."; } @@ -340,8 +345,8 @@ void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() { next_output_timestamp_ = Timestamp(ReflectBetween( next_output_timestamp_.Value(), next_output_timestamp_min_.Value(), next_output_timestamp_max_.Value())); - CHECK_GE(next_output_timestamp_, next_output_timestamp_min_); - CHECK_LT(next_output_timestamp_, next_output_timestamp_max_); + ABSL_CHECK_GE(next_output_timestamp_, next_output_timestamp_min_); + ABSL_CHECK_LT(next_output_timestamp_, next_output_timestamp_max_); } absl::Status ReproducibleJitterWithReflectionStrategy::Open( @@ -352,13 +357,15 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Open( if (resampler_options.output_header() != PacketResamplerCalculatorOptions::NONE) { - LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not " - "the actual value."; + ABSL_LOG(WARNING) + << "VideoHeader::frame_rate holds the target value and not " + "the actual value."; } if (calculator_->flush_last_packet_) { - LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is " - "ignored, because we are adding jitter."; + ABSL_LOG(WARNING) + << "PacketResamplerCalculatorOptions.flush_last_packet is " + "ignored, because we are adding jitter."; } const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get(); @@ -411,7 +418,7 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Process( // Note, if the stream is upsampling, this could lead to the same packet // being emitted twice. Upsampling and jitter doesn't make much sense // but does technically work. - LOG_FIRST_N(WARNING, 2) + ABSL_LOG_FIRST_N(WARNING, 2) << "Adding jitter is not very useful when upsampling."; } @@ -499,13 +506,15 @@ absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) { if (resampler_options.output_header() != PacketResamplerCalculatorOptions::NONE) { - LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not " - "the actual value."; + ABSL_LOG(WARNING) + << "VideoHeader::frame_rate holds the target value and not " + "the actual value."; } if (calculator_->flush_last_packet_) { - LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is " - "ignored, because we are adding jitter."; + ABSL_LOG(WARNING) + << "PacketResamplerCalculatorOptions.flush_last_packet is " + "ignored, because we are adding jitter."; } const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get(); @@ -555,7 +564,7 @@ absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) { if (calculator_->frame_time_usec_ < (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) { - LOG_FIRST_N(WARNING, 2) + ABSL_LOG_FIRST_N(WARNING, 2) << "Adding jitter is not very useful when upsampling."; } diff --git a/mediapipe/calculators/core/packet_resampler_calculator.h b/mediapipe/calculators/core/packet_resampler_calculator.h index fbecdb0e..f26dc2ca 100644 --- a/mediapipe/calculators/core/packet_resampler_calculator.h +++ b/mediapipe/calculators/core/packet_resampler_calculator.h @@ -13,7 +13,6 @@ #include "mediapipe/framework/deps/random_base.h" #include "mediapipe/framework/formats/video_stream_header.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_macros.h" diff --git a/mediapipe/calculators/core/packet_thinner_calculator.cc b/mediapipe/calculators/core/packet_thinner_calculator.cc index 35cd966e..0bc5cc16 100644 --- a/mediapipe/calculators/core/packet_thinner_calculator.cc +++ b/mediapipe/calculators/core/packet_thinner_calculator.cc @@ -17,6 +17,7 @@ #include // for ceil #include +#include "absl/log/absl_check.h" #include "mediapipe/calculators/core/packet_thinner_calculator.pb.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_framework.h" @@ -160,8 +161,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) { thinner_type_ = options.thinner_type(); // This check enables us to assume only two thinner types exist in Process() - CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC || - thinner_type_ == PacketThinnerCalculatorOptions::SYNC) + ABSL_CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC || + thinner_type_ == PacketThinnerCalculatorOptions::SYNC) << "Unsupported thinner type."; if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) { @@ -177,7 +178,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) { } else { period_ = TimestampDiff(options.period()); } - CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive."; + ABSL_CHECK_LT(TimestampDiff(0), period_) + << "Specified period must be positive."; if (options.has_start_time()) { start_time_ = Timestamp(options.start_time()); @@ -189,7 +191,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) { end_time_ = options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max(); - CHECK_LT(start_time_, end_time_) + ABSL_CHECK_LT(start_time_, end_time_) << "Invalid PacketThinner: start_time must be earlier than end_time"; sync_output_timestamps_ = options.sync_output_timestamps(); @@ -232,7 +234,7 @@ absl::Status PacketThinnerCalculator::Close(CalculatorContext* cc) { // Emit any saved packets before quitting. if (!saved_packet_.IsEmpty()) { // Only sync thinner should have saved packets. - CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_); + ABSL_CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_); if (sync_output_timestamps_) { cc->Outputs().Index(0).AddPacket( saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp()))); @@ -269,7 +271,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess( const Timestamp saved_sync = NearestSyncTimestamp(saved); const Timestamp now = cc->InputTimestamp(); const Timestamp now_sync = NearestSyncTimestamp(now); - CHECK_LE(saved_sync, now_sync); + ABSL_CHECK_LE(saved_sync, now_sync); if (saved_sync == now_sync) { // Saved Packet is in same interval as current packet. // Replace saved packet with current if it is at least as @@ -295,7 +297,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess( } Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const { - CHECK_NE(start_time_, Timestamp::Unset()) + ABSL_CHECK_NE(start_time_, Timestamp::Unset()) << "Method only valid for sync thinner calculator."; // Computation is done using int64 arithmetic. No easy way to avoid @@ -303,12 +305,12 @@ Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const { const int64_t now64 = now.Value(); const int64_t start64 = start_time_.Value(); const int64_t period64 = period_.Value(); - CHECK_LE(0, period64); + ABSL_CHECK_LE(0, period64); // Round now64 to its closest interval (units of period64). int64_t sync64 = (now64 - start64 + period64 / 2) / period64 * period64 + start64; - CHECK_LE(abs(now64 - sync64), period64 / 2) + ABSL_CHECK_LE(abs(now64 - sync64), period64 / 2) << "start64: " << start64 << "; now64: " << now64 << "; sync64: " << sync64; diff --git a/mediapipe/calculators/core/packet_thinner_calculator_test.cc b/mediapipe/calculators/core/packet_thinner_calculator_test.cc index 09de0ca7..69c00839 100644 --- a/mediapipe/calculators/core/packet_thinner_calculator_test.cc +++ b/mediapipe/calculators/core/packet_thinner_calculator_test.cc @@ -16,6 +16,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "mediapipe/calculators/core/packet_thinner_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -70,7 +71,7 @@ class SimpleRunner : public CalculatorRunner { } double GetFrameRate() const { - CHECK(!Outputs().Index(0).header.IsEmpty()); + ABSL_CHECK(!Outputs().Index(0).header.IsEmpty()); return Outputs().Index(0).header.Get().frame_rate; } }; diff --git a/mediapipe/calculators/core/previous_loopback_calculator.cc b/mediapipe/calculators/core/previous_loopback_calculator.cc index d67e6c06..36ee0f2d 100644 --- a/mediapipe/calculators/core/previous_loopback_calculator.cc +++ b/mediapipe/calculators/core/previous_loopback_calculator.cc @@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node { // However, LOOP packet is empty. kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1); } else { - kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp)); + // Avoids sending leftovers to a stream that's already closed. + if (!kPrevLoop(cc).IsClosed()) { + kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp)); + } } loop_packets_.pop_front(); main_packet_specs_.pop_front(); diff --git a/mediapipe/calculators/core/sequence_shift_calculator.cc b/mediapipe/calculators/core/sequence_shift_calculator.cc index 026048b7..5b2a73fd 100644 --- a/mediapipe/calculators/core/sequence_shift_calculator.cc +++ b/mediapipe/calculators/core/sequence_shift_calculator.cc @@ -14,6 +14,7 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/calculators/core/sequence_shift_calculator.pb.h" #include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/calculator_framework.h" @@ -101,7 +102,7 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) { kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp())); packet_cache_.pop_front(); } else if (emit_empty_packets_before_first_packet_) { - LOG(FATAL) << "Not supported yet"; + ABSL_LOG(FATAL) << "Not supported yet"; } // Store current packet for later output. packet_cache_.push_back(kIn(cc).packet()); diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index 20e5ebda..18d4e2fe 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -97,6 +97,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) @@ -125,6 +126,7 @@ cc_library( "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) @@ -135,7 +137,6 @@ cc_library( deps = [ "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:status", ], @@ -152,11 +153,11 @@ cc_library( "//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_log", ] + select({ "//mediapipe/gpu:disable_gpu": [], "//conditions:default": [ @@ -203,6 +204,7 @@ cc_library( "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ] + select({ "//mediapipe/gpu:disable_gpu": [], @@ -301,6 +303,7 @@ cc_library( "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ] + select({ "//mediapipe/gpu:disable_gpu": [], "//conditions:default": [ @@ -397,6 +400,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -421,6 +425,8 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/util:image_frame_util", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@libyuv", ], @@ -626,9 +632,9 @@ cc_library( "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_log", ] + select({ "//mediapipe/gpu:disable_gpu": [], "//conditions:default": [ @@ -666,6 +672,7 @@ cc_test( "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:parse_text_proto", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/calculators/image/affine_transformation_runner_gl.cc b/mediapipe/calculators/image/affine_transformation_runner_gl.cc index 00641691..ee40de66 100644 --- a/mediapipe/calculators/image/affine_transformation_runner_gl.cc +++ b/mediapipe/calculators/image/affine_transformation_runner_gl.cc @@ -384,6 +384,8 @@ class GlTextureWarpAffineRunner glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, 0); + glFlush(); + return absl::OkStatus(); } diff --git a/mediapipe/calculators/image/bilateral_filter_calculator.cc b/mediapipe/calculators/image/bilateral_filter_calculator.cc index 6bb43dc0..3d364ad9 100644 --- a/mediapipe/calculators/image/bilateral_filter_calculator.cc +++ b/mediapipe/calculators/image/bilateral_filter_calculator.cc @@ -15,6 +15,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_replace.h" #include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -112,7 +113,7 @@ class BilateralFilterCalculator : public CalculatorBase { REGISTER_CALCULATOR(BilateralFilterCalculator); absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) { - CHECK_GE(cc->Inputs().NumEntries(), 1); + RET_CHECK_GE(cc->Inputs().NumEntries(), 1); if (cc->Inputs().HasTag(kInputFrameTag) && cc->Inputs().HasTag(kInputFrameTagGpu)) { @@ -183,8 +184,8 @@ absl::Status BilateralFilterCalculator::Open(CalculatorContext* cc) { sigma_color_ = options_.sigma_color(); sigma_space_ = options_.sigma_space(); - CHECK_GE(sigma_color_, 0.0); - CHECK_GE(sigma_space_, 0.0); + ABSL_CHECK_GE(sigma_color_, 0.0); + ABSL_CHECK_GE(sigma_space_, 0.0); if (!use_gpu_) sigma_color_ *= 255.0; if (use_gpu_) { diff --git a/mediapipe/calculators/image/color_convert_calculator.cc b/mediapipe/calculators/image/color_convert_calculator.cc index 4781f1ea..f8f01836 100644 --- a/mediapipe/calculators/image/color_convert_calculator.cc +++ b/mediapipe/calculators/image/color_convert_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_check.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -25,8 +26,8 @@ namespace mediapipe { namespace { void SetColorChannel(int channel, uint8 value, cv::Mat* mat) { - CHECK(mat->depth() == CV_8U); - CHECK(channel < mat->channels()); + ABSL_CHECK(mat->depth() == CV_8U); + ABSL_CHECK(channel < mat->channels()); const int step = mat->channels(); for (int r = 0; r < mat->rows; ++r) { uint8* row_ptr = mat->ptr(r); diff --git a/mediapipe/calculators/image/image_cropping_calculator.cc b/mediapipe/calculators/image/image_cropping_calculator.cc index 6776da7c..9eb3e680 100644 --- a/mediapipe/calculators/image/image_cropping_calculator.cc +++ b/mediapipe/calculators/image/image_cropping_calculator.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/rect.pb.h" @@ -202,8 +203,9 @@ absl::Status ImageCroppingCalculator::ValidateBorderModeForGPU( switch (options.border_mode()) { case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO: - LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU " - << "implementation and will fall back into BORDER_REPLICATE"; + ABSL_LOG(WARNING) + << "BORDER_ZERO mode is not supported by GPU " + << "implementation and will fall back into BORDER_REPLICATE"; break; case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE: break; diff --git a/mediapipe/calculators/image/opencv_image_encoder_calculator.cc b/mediapipe/calculators/image/opencv_image_encoder_calculator.cc index 93ec9435..0308b9b8 100644 --- a/mediapipe/calculators/image/opencv_image_encoder_calculator.cc +++ b/mediapipe/calculators/image/opencv_image_encoder_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_check.h" #include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -61,7 +62,7 @@ absl::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) { absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) { const ImageFrame& image_frame = cc->Inputs().Index(0).Get(); - CHECK_EQ(1, image_frame.ByteDepth()); + ABSL_CHECK_EQ(1, image_frame.ByteDepth()); std::unique_ptr encoded_result = absl::make_unique(); diff --git a/mediapipe/calculators/image/scale_image_calculator.cc b/mediapipe/calculators/image/scale_image_calculator.cc index d8a3cb93..1d4f980f 100644 --- a/mediapipe/calculators/image/scale_image_calculator.cc +++ b/mediapipe/calculators/image/scale_image_calculator.cc @@ -18,6 +18,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/substitute.h" #include "libyuv/scale.h" @@ -293,7 +295,7 @@ absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) { header->width = output_width_; header->height = output_height_; header->format = output_format_; - LOG(INFO) << "OUTPUTTING HEADER on stream"; + ABSL_LOG(INFO) << "OUTPUTTING HEADER on stream"; cc->Outputs() .Tag("VIDEO_HEADER") .Add(header.release(), Timestamp::PreStream()); @@ -393,10 +395,11 @@ absl::Status ScaleImageCalculator::Open(CalculatorContext* cc) { .SetHeader(Adopt(output_header.release())); has_header_ = true; } else { - LOG(WARNING) << "Stream had a VideoHeader which didn't have sufficient " - "information. " - "Dropping VideoHeader and trying to deduce needed " - "information."; + ABSL_LOG(WARNING) + << "Stream had a VideoHeader which didn't have sufficient " + "information. " + "Dropping VideoHeader and trying to deduce needed " + "information."; input_width_ = 0; input_height_ = 0; if (!options_.has_input_format()) { @@ -507,7 +510,7 @@ absl::Status ScaleImageCalculator::ValidateImageFrame( absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc, const YUVImage& yuv_image) { - CHECK_EQ(input_format_, ImageFormat::YCBCR420P); + ABSL_CHECK_EQ(input_format_, ImageFormat::YCBCR420P); if (!has_header_) { if (input_width_ != yuv_image.width() || input_height_ != yuv_image.height()) { diff --git a/mediapipe/calculators/image/scale_image_utils.cc b/mediapipe/calculators/image/scale_image_utils.cc index 86a53ffc..77b7c0ec 100644 --- a/mediapipe/calculators/image/scale_image_utils.cc +++ b/mediapipe/calculators/image/scale_image_utils.cc @@ -18,6 +18,7 @@ #include +#include "absl/log/absl_check.h" #include "absl/strings/str_split.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" @@ -40,10 +41,10 @@ absl::Status FindCropDimensions(int input_width, int input_height, // const std::string& max_aspect_ratio, // int* crop_width, int* crop_height, // int* col_start, int* row_start) { - CHECK(crop_width); - CHECK(crop_height); - CHECK(col_start); - CHECK(row_start); + ABSL_CHECK(crop_width); + ABSL_CHECK(crop_height); + ABSL_CHECK(col_start); + ABSL_CHECK(row_start); double min_aspect_ratio_q = 0.0; double max_aspect_ratio_q = 0.0; @@ -83,8 +84,8 @@ absl::Status FindCropDimensions(int input_width, int input_height, // } } - CHECK_LE(*crop_width, input_width); - CHECK_LE(*crop_height, input_height); + ABSL_CHECK_LE(*crop_width, input_width); + ABSL_CHECK_LE(*crop_height, input_height); return absl::OkStatus(); } @@ -96,8 +97,8 @@ absl::Status FindOutputDimensions(int input_width, // bool preserve_aspect_ratio, // int scale_to_multiple_of, // int* output_width, int* output_height) { - CHECK(output_width); - CHECK(output_height); + ABSL_CHECK(output_width); + ABSL_CHECK(output_height); if (target_max_area > 0 && input_width * input_height > target_max_area) { preserve_aspect_ratio = true; diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc index 81732f90..1194412a 100644 --- a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc +++ b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc @@ -15,13 +15,13 @@ #include #include +#include "absl/log/absl_log.h" #include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/vector.h" @@ -110,7 +110,7 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator); absl::Status SegmentationSmoothingCalculator::GetContract( CalculatorContract* cc) { - CHECK_GE(cc->Inputs().NumEntries(), 1); + RET_CHECK_GE(cc->Inputs().NumEntries(), 1); cc->Inputs().Tag(kCurrentMaskTag).Set(); cc->Inputs().Tag(kPreviousMaskTag).Set(); @@ -273,7 +273,7 @@ absl::Status SegmentationSmoothingCalculator::RenderGpu(CalculatorContext* cc) { const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get(); if (previous_frame.format() != current_frame.format()) { - LOG(ERROR) << "Warning: mixing input format types. "; + ABSL_LOG(ERROR) << "Warning: mixing input format types. "; } auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame); diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc b/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc index eeb812cb..0f5152fc 100644 --- a/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc +++ b/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc @@ -14,6 +14,7 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" @@ -169,7 +170,7 @@ void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) { } } } else { - LOG(ERROR) << "invalid ratio"; + ABSL_LOG(ERROR) << "invalid ratio"; } } diff --git a/mediapipe/calculators/image/set_alpha_calculator.cc b/mediapipe/calculators/image/set_alpha_calculator.cc index e20621e8..d451cd21 100644 --- a/mediapipe/calculators/image/set_alpha_calculator.cc +++ b/mediapipe/calculators/image/set_alpha_calculator.cc @@ -14,13 +14,13 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/calculators/image/set_alpha_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/status.h" @@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase { REGISTER_CALCULATOR(SetAlphaCalculator); absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) { - CHECK_GE(cc->Inputs().NumEntries(), 1); + RET_CHECK_GE(cc->Inputs().NumEntries(), 1); bool use_gpu = false; @@ -268,7 +268,7 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) { const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get(); const cv::Mat input_mat = formats::MatView(&input_frame); if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) { - LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported"; + ABSL_LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported"; } // Setup destination image @@ -328,7 +328,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) { cc->Inputs().Tag(kInputFrameTagGpu).Get(); if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 || input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) { - LOG(ERROR) << "Only RGB or RGBA input image supported"; + ABSL_LOG(ERROR) << "Only RGB or RGBA input image supported"; } auto input_texture = gpu_helper_.CreateSourceTexture(input_frame); diff --git a/mediapipe/calculators/image/yuv_to_image_calculator.cc b/mediapipe/calculators/image/yuv_to_image_calculator.cc index e84eee74..6a82877c 100644 --- a/mediapipe/calculators/image/yuv_to_image_calculator.cc +++ b/mediapipe/calculators/image/yuv_to_image_calculator.cc @@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) { buf[0] = (fourcc >> 24) & 0xff; buf[1] = (fourcc >> 16) & 0xff; buf[2] = (fourcc >> 8) & 0xff; - buf[3] = (fourcc)&0xff; + buf[3] = (fourcc) & 0xff; buf[4] = 0; return std::string(buf); } diff --git a/mediapipe/calculators/internal/BUILD b/mediapipe/calculators/internal/BUILD index a92a2f25..a5d82e13 100644 --- a/mediapipe/calculators/internal/BUILD +++ b/mediapipe/calculators/internal/BUILD @@ -31,12 +31,14 @@ mediapipe_proto_library( cc_library( name = "callback_packet_calculator", srcs = ["callback_packet_calculator.cc"], + hdrs = ["callback_packet_calculator.h"], visibility = ["//mediapipe/framework:__subpackages__"], deps = [ ":callback_packet_calculator_cc_proto", "//mediapipe/framework:calculator_base", "//mediapipe/framework:calculator_registry", "//mediapipe/framework:output_side_packet", + "@com_google_absl//absl/status", ], alwayslink = 1, ) diff --git a/mediapipe/calculators/internal/callback_packet_calculator.cc b/mediapipe/calculators/internal/callback_packet_calculator.cc index cc153483..aa86c061 100644 --- a/mediapipe/calculators/internal/callback_packet_calculator.cc +++ b/mediapipe/calculators/internal/callback_packet_calculator.cc @@ -11,10 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/calculators/internal/callback_packet_calculator.h" #include #include +#include "absl/status/status.h" #include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/calculator_registry.h" @@ -39,64 +41,55 @@ void DumpPostStreamPacket(Packet* post_stream_packet, const Packet& packet) { *post_stream_packet = packet; } } + } // namespace -// Creates a callback which takes a packet and stores it either in a -// vector of packets or stores only the packet at PostStream timestamp. -// The kind of callback is controlled by an option. The callback is -// a std::function and is directly usable by CallbackCalculator. -// Since the options for the packet generator include a serialized pointer -// value, the resulting callback is only valid on the original machine -// while that pointer is still alive. -class CallbackPacketCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = cc->Options(); - switch (options.type()) { - case CallbackPacketCalculatorOptions::VECTOR_PACKET: - case CallbackPacketCalculatorOptions::POST_STREAM_PACKET: - cc->OutputSidePackets() - .Index(0) - .Set>(); - break; - default: - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Invalid type of callback to produce."; - } - return absl::OkStatus(); - } - - absl::Status Open(CalculatorContext* cc) override { - const auto& options = cc->Options(); - void* ptr; - if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) { +absl::Status CallbackPacketCalculator::GetContract(CalculatorContract* cc) { + const auto& options = cc->Options(); + switch (options.type()) { + case CallbackPacketCalculatorOptions::VECTOR_PACKET: + case CallbackPacketCalculatorOptions::POST_STREAM_PACKET: + cc->OutputSidePackets() + .Index(0) + .Set>(); + break; + default: return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Stored pointer value in options is invalid."; - } - switch (options.type()) { - case CallbackPacketCalculatorOptions::VECTOR_PACKET: - cc->OutputSidePackets().Index(0).Set( - MakePacket>(std::bind( - &DumpToVector, reinterpret_cast*>(ptr), - std::placeholders::_1))); - break; - case CallbackPacketCalculatorOptions::POST_STREAM_PACKET: - cc->OutputSidePackets().Index(0).Set( - MakePacket>( - std::bind(&DumpPostStreamPacket, reinterpret_cast(ptr), - std::placeholders::_1))); - break; - default: - return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) - << "Invalid type to dump into."; - } - return absl::OkStatus(); + << "Invalid type of callback to produce."; } + return absl::OkStatus(); +} - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); +absl::Status CallbackPacketCalculator::Open(CalculatorContext* cc) { + const auto& options = cc->Options(); + void* ptr; + if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) { + return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) + << "Stored pointer value in options is invalid."; } -}; + switch (options.type()) { + case CallbackPacketCalculatorOptions::VECTOR_PACKET: + cc->OutputSidePackets().Index(0).Set( + MakePacket>(std::bind( + &DumpToVector, reinterpret_cast*>(ptr), + std::placeholders::_1))); + break; + case CallbackPacketCalculatorOptions::POST_STREAM_PACKET: + cc->OutputSidePackets().Index(0).Set( + MakePacket>( + std::bind(&DumpPostStreamPacket, reinterpret_cast(ptr), + std::placeholders::_1))); + break; + default: + return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) + << "Invalid type to dump into."; + } + return absl::OkStatus(); +} + +absl::Status CallbackPacketCalculator::Process(CalculatorContext* cc) { + return absl::OkStatus(); +} REGISTER_CALCULATOR(CallbackPacketCalculator); diff --git a/mediapipe/calculators/internal/callback_packet_calculator.h b/mediapipe/calculators/internal/callback_packet_calculator.h new file mode 100644 index 00000000..e0b170e3 --- /dev/null +++ b/mediapipe/calculators/internal/callback_packet_calculator.h @@ -0,0 +1,39 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_ +#define MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_ + +#include "absl/status/status.h" +#include "mediapipe/framework/calculator_base.h" + +namespace mediapipe { + +// Creates a callback which takes a packet and stores it either in a +// vector of packets or stores only the packet at PostStream timestamp. +// The kind of callback is controlled by an option. The callback is +// a std::function and is directly usable by CallbackCalculator. +// Since the options for the packet generator include a serialized pointer +// value, the resulting callback is only valid on the original machine +// while that pointer is still alive. +class CallbackPacketCalculator : public CalculatorBase { + public: + static absl::Status GetContract(CalculatorContract* cc); + absl::Status Open(CalculatorContext* cc) override; + absl::Status Process(CalculatorContext* cc) override; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_ diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 2ad98f28..017ab4f3 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -87,6 +87,7 @@ cc_library( "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/util:time_series_util", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -181,6 +182,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/api2:node", "//mediapipe/framework/formats:tensor", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", ], alwayslink = 1, @@ -198,6 +200,7 @@ cc_test( "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", + "@com_google_absl//absl/log:absl_check", "@org_tensorflow//tensorflow/lite/c:common", ], ) @@ -228,7 +231,6 @@ cc_library( "//mediapipe/tasks/metadata:metadata_schema_cc", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -280,7 +282,6 @@ cc_library( "//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils", "//mediapipe/tasks/metadata:metadata_schema_cc", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", ], alwayslink = 1, ) @@ -476,6 +477,7 @@ cc_library( "//mediapipe/gpu:gpu_buffer", "//mediapipe/objc:mediapipe_framework_ios", "//mediapipe/util/tflite:config", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings:str_format", "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate", @@ -622,6 +624,7 @@ mediapipe_proto_library( deps = [ "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", + "//mediapipe/gpu:gpu_origin_proto", ], ) @@ -651,7 +654,13 @@ cc_library( "//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "//mediapipe/framework/port:statusor", + "//mediapipe/gpu:gpu_buffer_format", + "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/strings:str_format", ] + select({ "//mediapipe/gpu:disable_gpu": [], "//conditions:default": ["tensor_converter_calculator_gpu_deps"], @@ -701,6 +710,7 @@ cc_test( "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/tool:validate_type", "@com_google_absl//absl/memory", @@ -739,6 +749,8 @@ cc_library( "//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats/object_detection:anchor_cc_proto", "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", ] + selects.with_or({ @@ -795,6 +807,7 @@ cc_library( "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) @@ -987,6 +1000,8 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/gpu:gpu_origin_cc_proto", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ] + select({ "//mediapipe/gpu:disable_gpu": [], "//conditions:default": [":image_to_tensor_calculator_gpu_deps"], @@ -1079,6 +1094,7 @@ cc_test( "//mediapipe/framework/port:parse_text_proto", "//mediapipe/util:image_test_utils", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", @@ -1206,6 +1222,7 @@ cc_library( "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:shader_util", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], }), diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc b/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc index 47617b37..eaf593a6 100644 --- a/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc @@ -20,6 +20,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" @@ -282,18 +283,23 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) { if (options.has_volume_gain_db()) { gain_ = pow(10, options.volume_gain_db() / 20.0); } - RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^ - !kAudioIn(cc).Header().IsEmpty()) - << "Must either specify the time series header of the \"AUDIO\" stream " - "or have the \"SAMPLE_RATE\" stream connected."; - if (!kAudioIn(cc).Header().IsEmpty()) { - mediapipe::TimeSeriesHeader input_header; - MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid( - kAudioIn(cc).Header(), &input_header)); - if (stream_mode_) { - MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate())); - } else { - source_sample_rate_ = input_header.sample_rate(); + if (options.has_source_sample_rate()) { + source_sample_rate_ = options.source_sample_rate(); + } else { + RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^ + !kAudioIn(cc).Header().IsEmpty()) + << "Must either specify the time series header of the \"AUDIO\" stream " + "or have the \"SAMPLE_RATE\" stream connected."; + if (!kAudioIn(cc).Header().IsEmpty()) { + mediapipe::TimeSeriesHeader input_header; + MP_RETURN_IF_ERROR( + mediapipe::time_series_util::FillTimeSeriesHeaderIfValid( + kAudioIn(cc).Header(), &input_header)); + if (stream_mode_) { + MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate())); + } else { + source_sample_rate_ = input_header.sample_rate(); + } } } AppendZerosToSampleBuffer(padding_samples_before_); @@ -343,7 +349,7 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) { return absl::InvalidArgumentError( "The audio data should be stored in column-major."); } - CHECK(channels_match || mono_output); + ABSL_CHECK(channels_match || mono_output); const Matrix& input = channels_match ? input_frame // Mono mixdown. : input_frame.colwise().mean(); @@ -452,7 +458,7 @@ absl::Status AudioToTensorCalculator::SetupStreamingResampler( } void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) { - CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`. + ABSL_CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`. if (num_samples == 0) { return; } diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto b/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto index 5b7d61bc..948c82a3 100644 --- a/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto +++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto @@ -85,4 +85,7 @@ message AudioToTensorCalculatorOptions { // The volume gain, measured in dB. // Scale the input audio amplitude by 10^(volume_gain_db/20). optional double volume_gain_db = 12; + + // The source number of samples per second (hertz) of the input audio buffers. + optional double source_sample_rate = 13; } diff --git a/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc b/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc index b5612280..12db1493 100644 --- a/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc +++ b/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc @@ -22,7 +22,6 @@ #include "absl/container/flat_hash_set.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "absl/strings/ascii.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" @@ -244,7 +243,8 @@ std::vector BertPreprocessorCalculator::GenerateInputTensors( input_tensors.reserve(kNumInputTensorsForBert); for (int i = 0; i < kNumInputTensorsForBert; ++i) { input_tensors.push_back( - {Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})}); + {Tensor::ElementType::kInt32, + Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)}); } std::memcpy(input_tensors[input_ids_tensor_index_] .GetCpuWriteView() diff --git a/mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc b/mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc index 5797cc31..6c5e5cc4 100644 --- a/mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc +++ b/mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -65,7 +66,7 @@ template Tensor MakeTensor(std::initializer_list shape, std::initializer_list values) { Tensor tensor(TensorElementType::value, shape); - CHECK_EQ(values.size(), tensor.shape().num_elements()) + ABSL_CHECK_EQ(values.size(), tensor.shape().num_elements()) << "The size of `values` is incompatible with `shape`"; absl::c_copy(values, tensor.GetCpuWriteView().buffer()); return tensor; diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc index 344e12da..26fb1d86 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc @@ -16,6 +16,7 @@ #include #include +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h" #include "mediapipe/calculators/tensor/image_to_tensor_converter.h" #include "mediapipe/calculators/tensor/image_to_tensor_utils.h" @@ -284,9 +285,9 @@ class ImageToTensorCalculator : public Node { cc, GetBorderMode(options_.border_mode()), GetOutputTensorType(/*uses_gpu=*/false, params_))); #else - LOG(FATAL) << "Cannot create image to tensor CPU converter since " - "MEDIAPIPE_DISABLE_OPENCV is defined and " - "MEDIAPIPE_ENABLE_HALIDE is not defined."; + ABSL_LOG(FATAL) << "Cannot create image to tensor CPU converter since " + "MEDIAPIPE_DISABLE_OPENCV is defined and " + "MEDIAPIPE_ENABLE_HALIDE is not defined."; #endif // !MEDIAPIPE_DISABLE_HALIDE } } diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc index 409b8623..7017c1e3 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc @@ -18,6 +18,7 @@ #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/strings/str_format.h" #include "absl/strings/substitute.h" @@ -205,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) { } else if (image_channels == 1) { return ImageFormat::GRAY8; } - CHECK(false) << "Unsupported input image channles: " << image_channels; + ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels; } Packet MakeImageFramePacket(cv::Mat input) { diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc index 165df897..465e7e0b 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc @@ -22,6 +22,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "mediapipe/calculators/tensor/image_to_tensor_converter.h" #include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h" @@ -259,7 +260,7 @@ class GlProcessor : public ImageToTensorConverter { // error. So in that case, we'll grab the transpose of our original matrix // and send that instead. const auto gl_context = mediapipe::GlContext::GetCurrent(); - LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread."; + ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread."; if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) { GetTransposedRotatedSubRectToRectTransformMatrix( sub_rect, texture.width(), texture.height(), flip_horizontaly, diff --git a/mediapipe/calculators/tensor/inference_calculator.proto b/mediapipe/calculators/tensor/inference_calculator.proto index 78a0039b..82f4ec80 100644 --- a/mediapipe/calculators/tensor/inference_calculator.proto +++ b/mediapipe/calculators/tensor/inference_calculator.proto @@ -88,6 +88,20 @@ message InferenceCalculatorOptions { // serialized model is invalid or missing. optional string serialized_model_dir = 7; + enum CacheWritingBehavior { + // Do not write any caches. + NO_WRITE = 0; + + // Try to write caches, log on failure. + TRY_WRITE = 1; + + // Write caches or return an error if write fails. + WRITE_OR_ERROR = 2; + } + // Specifies how GPU caches are written to disk. + optional CacheWritingBehavior cache_writing_behavior = 10 + [default = WRITE_OR_ERROR]; + // Unique token identifying the model. Used in conjunction with // "serialized_model_dir". It is the caller's responsibility to ensure // there is no clash of the tokens. diff --git a/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc b/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc index 8aee4618..77e6eeaf 100644 --- a/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc +++ b/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -26,6 +27,7 @@ #include "mediapipe/util/tflite/tflite_gpu_runner.h" #if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS) +#include "absl/log/absl_log.h" #include "mediapipe/framework/deps/file_path.h" #include "mediapipe/util/android/file/base/file.h" #include "mediapipe/util/android/file/base/filesystem.h" @@ -68,13 +70,21 @@ class InferenceCalculatorGlAdvancedImpl const mediapipe::InferenceCalculatorOptions::Delegate::Gpu& gpu_delegate_options); absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const; - absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const; + // Writes caches to disk based on |cache_writing_behavior_|. + absl::Status SaveGpuCachesBasedOnBehavior( + tflite::gpu::TFLiteGPURunner* gpu_runner) const; + bool UseSerializedModel() const { return use_serialized_model_; } private: + // Writes caches to disk, returns error on failure. + absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const; + bool use_kernel_caching_ = false; std::string cached_kernel_filename_; bool use_serialized_model_ = false; std::string serialized_model_path_; + mediapipe::InferenceCalculatorOptions::Delegate::Gpu::CacheWritingBehavior + cache_writing_behavior_; }; // Helper class that wraps everything related to GPU inference acceleration. @@ -150,8 +160,6 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process( } absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() { - MP_RETURN_IF_ERROR( - on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get())); return gpu_helper_.RunInGlContext([this]() -> absl::Status { tflite_gpu_runner_.reset(); return absl::OkStatus(); @@ -226,9 +234,15 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner( tflite_gpu_runner_->GetOutputShapes()[i].c}; } + if (on_disk_cache_helper_.UseSerializedModel()) { + tflite_gpu_runner_->ForceOpenCLInitFromSerializedModel(); + } + MP_RETURN_IF_ERROR( on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get())); - return tflite_gpu_runner_->Build(); + MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build()); + return on_disk_cache_helper_.SaveGpuCachesBasedOnBehavior( + tflite_gpu_runner_.get()); } #if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS) @@ -257,9 +271,36 @@ absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init( mediapipe::file::JoinPath(gpu_delegate_options.serialized_model_dir(), gpu_delegate_options.model_token()); } + cache_writing_behavior_ = gpu_delegate_options.has_cache_writing_behavior() + ? gpu_delegate_options.cache_writing_behavior() + : mediapipe::InferenceCalculatorOptions:: + Delegate::Gpu::WRITE_OR_ERROR; return absl::OkStatus(); } +absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper:: + SaveGpuCachesBasedOnBehavior( + tflite::gpu::TFLiteGPURunner* gpu_runner) const { + switch (cache_writing_behavior_) { + case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::NO_WRITE: + return absl::OkStatus(); + case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::TRY_WRITE: { + auto status = SaveGpuCaches(gpu_runner); + if (!status.ok()) { + ABSL_LOG_FIRST_N(WARNING, 1) << "Failed to save gpu caches: " << status; + } + return absl::OkStatus(); + } + case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::WRITE_OR_ERROR: + return SaveGpuCaches(gpu_runner); + default: + ABSL_LOG_FIRST_N(ERROR, 1) + << "Unknown cache writing behavior: " + << static_cast(cache_writing_behavior_); + return absl::InvalidArgumentError("Unknown cache writing behavior."); + } +} + absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches( tflite::gpu::TFLiteGPURunner* gpu_runner) const { diff --git a/mediapipe/calculators/tensor/inference_calculator_metal.cc b/mediapipe/calculators/tensor/inference_calculator_metal.cc index fba18a81..253091a8 100644 --- a/mediapipe/calculators/tensor/inference_calculator_metal.cc +++ b/mediapipe/calculators/tensor/inference_calculator_metal.cc @@ -21,6 +21,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_format.h" #include "mediapipe/calculators/tensor/inference_calculator.h" @@ -74,7 +75,7 @@ tflite::gpu::BHWC BhwcFromTensorShape(const Tensor::Shape& shape) { break; default: // Handles 0 and >4. - LOG(FATAL) + ABSL_LOG(FATAL) << "Dimensions size must be in range [1,4] for GPU inference, but " << shape.dims.size() << " is provided"; } diff --git a/mediapipe/calculators/tensor/inference_calculator_test.cc b/mediapipe/calculators/tensor/inference_calculator_test.cc index 3662af39..2e75bb97 100644 --- a/mediapipe/calculators/tensor/inference_calculator_test.cc +++ b/mediapipe/calculators/tensor/inference_calculator_test.cc @@ -16,7 +16,7 @@ #include #include -#include "absl/log/check.h" +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" diff --git a/mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc b/mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc index a2b8a928..b727f179 100644 --- a/mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc +++ b/mediapipe/calculators/tensor/inference_interpreter_delegate_runner.cc @@ -96,6 +96,19 @@ absl::StatusOr> InferenceInterpreterDelegateRunner::Run( CalculatorContext* cc, const std::vector& input_tensors) { // Read CPU input into tensors. RET_CHECK_EQ(interpreter_->inputs().size(), input_tensors.size()); + + // If the input tensors have dynamic shape, then the tensors need to be + // resized and reallocated before we can copy the tensor values. + bool resized_tensor_shapes = false; + for (int i = 0; i < input_tensors.size(); ++i) { + if (input_tensors[i].shape().is_dynamic) { + interpreter_->ResizeInputTensorStrict(i, input_tensors[i].shape().dims); + resized_tensor_shapes = true; + } + } + // Reallocation is needed for memory sanity. + if (resized_tensor_shapes) interpreter_->AllocateTensors(); + for (int i = 0; i < input_tensors.size(); ++i) { const TfLiteType input_tensor_type = interpreter_->tensor(interpreter_->inputs()[i])->type; diff --git a/mediapipe/calculators/tensor/regex_preprocessor_calculator.cc b/mediapipe/calculators/tensor/regex_preprocessor_calculator.cc index 92a5f026..8276462f 100644 --- a/mediapipe/calculators/tensor/regex_preprocessor_calculator.cc +++ b/mediapipe/calculators/tensor/regex_preprocessor_calculator.cc @@ -20,7 +20,6 @@ #include #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "mediapipe/calculators/tensor/regex_preprocessor_calculator.pb.h" #include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/api2/port.h" @@ -161,7 +160,7 @@ absl::Status RegexPreprocessorCalculator::Process(CalculatorContext* cc) { // not found in the tokenizer vocab. std::vector result; result.push_back( - {Tensor::ElementType::kInt32, Tensor::Shape({max_seq_len_})}); + {Tensor::ElementType::kInt32, Tensor::Shape({1, max_seq_len_})}); std::memcpy(result[0].GetCpuWriteView().buffer(), input_tokens.data(), input_tokens.size() * sizeof(int32_t)); kTensorsOut(cc).Send(std::move(result)); diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.cc b/mediapipe/calculators/tensor/tensor_converter_calculator.cc index c1bd9296..f624ed56 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.cc +++ b/mediapipe/calculators/tensor/tensor_converter_calculator.cc @@ -12,9 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_format.h" #include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" @@ -22,7 +27,8 @@ #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/resource_util.h" +#include "mediapipe/gpu/gpu_buffer_format.h" +#include "mediapipe/gpu/gpu_origin.pb.h" #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gpu_buffer.h" @@ -43,12 +49,36 @@ #endif // !MEDIAPIPE_DISABLE_GPU namespace { + constexpr int kWorkgroupSize = 8; // Block size for GPU shader. // Commonly used to compute the number of blocks to launch in a kernel. int NumGroups(const int size, const int group_size) { // NOLINT return (size + group_size - 1) / group_size; } +absl::StatusOr ShouldFlipVertically( + const mediapipe::TensorConverterCalculatorOptions& options) { + if (!options.has_gpu_origin()) { + return options.flip_vertically(); + } + + switch (options.gpu_origin()) { + case mediapipe::GpuOrigin::TOP_LEFT: + return false; + case mediapipe::GpuOrigin::DEFAULT: + case mediapipe::GpuOrigin::CONVENTIONAL: + // TOP_LEFT on Metal, BOTTOM_LEFT on OpenGL. +#ifdef __APPLE__ + return false; +#else + return true; +#endif + } + + return absl::InvalidArgumentError( + absl::StrFormat("Unhandled GPU origin %i", options.gpu_origin())); +} + typedef Eigen::Matrix RowMajorMatrixXf; typedef Eigen::Matrix @@ -58,6 +88,7 @@ constexpr char kImageFrameTag[] = "IMAGE"; constexpr char kGpuBufferTag[] = "IMAGE_GPU"; constexpr char kTensorsTag[] = "TENSORS"; constexpr char kMatrixTag[] = "MATRIX"; + } // namespace namespace mediapipe { @@ -378,16 +409,27 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) { // Get input image sizes. const auto& input = cc->Inputs().Tag(kGpuBufferTag).Get(); - mediapipe::ImageFormat::Format format = - mediapipe::ImageFormatForGpuBufferFormat(input.format()); + mediapipe::GpuBufferFormat format = input.format(); const bool include_alpha = (max_num_channels_ == 4); const bool single_channel = (max_num_channels_ == 1); - if (!(format == mediapipe::ImageFormat::GRAY8 || - format == mediapipe::ImageFormat::SRGB || - format == mediapipe::ImageFormat::SRGBA)) - RET_CHECK_FAIL() << "Unsupported GPU input format."; - if (include_alpha && (format != mediapipe::ImageFormat::SRGBA)) - RET_CHECK_FAIL() << "Num input channels is less than desired output."; + + RET_CHECK(format == mediapipe::GpuBufferFormat::kBGRA32 || + format == mediapipe::GpuBufferFormat::kRGB24 || + format == mediapipe::GpuBufferFormat::kRGBA32 || + format == mediapipe::GpuBufferFormat::kRGBAFloat128 || + format == mediapipe::GpuBufferFormat::kRGBAHalf64 || + format == mediapipe::GpuBufferFormat::kGrayFloat32 || + format == mediapipe::GpuBufferFormat::kGrayHalf16 || + format == mediapipe::GpuBufferFormat::kOneComponent8) + << "Unsupported GPU input format: " << static_cast(format); + if (include_alpha) { + RET_CHECK(format == mediapipe::GpuBufferFormat::kBGRA32 || + format == mediapipe::GpuBufferFormat::kRGBA32 || + format == mediapipe::GpuBufferFormat::kRGBAFloat128 || + format == mediapipe::GpuBufferFormat::kRGBAHalf64) + << "Num input channels is less than desired output, input format: " + << static_cast(format); + } #if MEDIAPIPE_METAL_ENABLED id device = gpu_helper_.mtlDevice; @@ -582,7 +624,7 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc) { if (options.has_output_tensor_float_range()) { output_range_.emplace(options.output_tensor_float_range().min(), options.output_tensor_float_range().max()); - CHECK_GT(output_range_->second, output_range_->first); + ABSL_CHECK_GT(output_range_->second, output_range_->first); } // Custom div and sub values. @@ -593,16 +635,16 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc) { } // Get y-flip mode. - flip_vertically_ = options.flip_vertically(); + ASSIGN_OR_RETURN(flip_vertically_, ShouldFlipVertically(options)); // Get row_major_matrix mode. row_major_matrix_ = options.row_major_matrix(); // Get desired way to handle input channels. max_num_channels_ = options.max_num_channels(); - CHECK_GE(max_num_channels_, 1); - CHECK_LE(max_num_channels_, 4); - CHECK_NE(max_num_channels_, 2); + ABSL_CHECK_GE(max_num_channels_, 1); + ABSL_CHECK_LE(max_num_channels_, 4); + ABSL_CHECK_NE(max_num_channels_, 2); return absl::OkStatus(); } diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.proto b/mediapipe/calculators/tensor/tensor_converter_calculator.proto index 97c2154a..194dd417 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.proto +++ b/mediapipe/calculators/tensor/tensor_converter_calculator.proto @@ -3,6 +3,7 @@ syntax = "proto2"; package mediapipe; import "mediapipe/framework/calculator.proto"; +import "mediapipe/gpu/gpu_origin.proto"; // Full Example: // @@ -43,8 +44,14 @@ message TensorConverterCalculatorOptions { // with a coordinate system where the origin is at the bottom-left corner // (e.g., in OpenGL) whereas the ML model expects an image with a top-left // origin. + // Prefer gpu_origin over this field. optional bool flip_vertically = 2 [default = false]; + // Determines when the input image should be flipped vertically. + // See GpuOrigin.Mode for more information. + // If unset, falls back to flip_vertically for backwards compatibility. + optional GpuOrigin.Mode gpu_origin = 10; + // Controls how many channels of the input image get passed through to the // tensor. Valid values are 1,3,4 only. Ignored for iOS GPU. optional int32 max_num_channels = 3 [default = 3]; diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc b/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc index 2cfbd3d1..b3df0152 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc +++ b/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include +#include #include #include "absl/memory/memory.h" @@ -24,8 +27,10 @@ #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/status_matchers.h" // NOLINT #include "mediapipe/framework/tool/validate_type.h" @@ -40,7 +45,6 @@ constexpr char kTransposeOptionsString[] = } // namespace using RandomEngine = std::mt19937_64; -using testing::Eq; const uint32_t kSeed = 1234; const int kNumSizes = 8; const int sizes[kNumSizes][2] = {{1, 1}, {12, 1}, {1, 9}, {2, 2}, @@ -110,12 +114,12 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) { // Wait until the calculator done processing. MP_ASSERT_OK(graph_->WaitUntilIdle()); - EXPECT_EQ(1, output_packets.size()); + ASSERT_EQ(output_packets.size(), 1); // Get and process results. const std::vector& tensor_vec = output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); + ASSERT_EQ(tensor_vec.size(), 1); const Tensor* tensor = &tensor_vec[0]; EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); @@ -127,7 +131,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) { auto tensor_buffer = view.buffer(); for (int i = 0; i < num_rows * num_columns; ++i) { const float expected = uniform_dist(random); - EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i; + EXPECT_FLOAT_EQ(tensor_buffer[i], expected) << "at i = " << i; } // Fully close graph at end, otherwise calculator+tensors are destroyed @@ -172,12 +176,12 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) { // Wait until the calculator done processing. MP_ASSERT_OK(graph_->WaitUntilIdle()); - EXPECT_EQ(1, output_packets.size()); + ASSERT_EQ(output_packets.size(), 1); // Get and process results. const std::vector& tensor_vec = output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); + ASSERT_EQ(tensor_vec.size(), 1); const Tensor* tensor = &tensor_vec[0]; EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); @@ -189,7 +193,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) { auto tensor_buffer = view.buffer(); for (int i = 0; i < num_rows * num_columns; ++i) { const float expected = uniform_dist(random); - EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i; + EXPECT_EQ(tensor_buffer[i], expected) << "at i = " << i; } // Fully close graph at end, otherwise calculator+tensors are destroyed @@ -239,12 +243,12 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) { // Get and process results. const std::vector& tensor_vec = output_packets[0].Get>(); - EXPECT_EQ(1, tensor_vec.size()); + ASSERT_EQ(tensor_vec.size(), 1); const Tensor* tensor = &tensor_vec[0]; EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); auto view = tensor->GetCpuReadView(); - EXPECT_FLOAT_EQ(67.0f, *view.buffer()); + EXPECT_FLOAT_EQ(*view.buffer(), 67.0f); // Fully close graph at end, otherwise calculator+tensors are destroyed // after calling WaitUntilDone(). @@ -259,25 +263,22 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) { for (std::pair range : range_values) { CalculatorGraph graph; CalculatorGraphConfig graph_config = - mediapipe::ParseTextProtoOrDie( - absl::Substitute(R"( - input_stream: "input_image" - node { - calculator: "TensorConverterCalculator" - input_stream: "IMAGE:input_image" - output_stream: "TENSORS:tensor" - options { - [mediapipe.TensorConverterCalculatorOptions.ext] { - output_tensor_float_range { - min: $0 - max: $1 + mediapipe::ParseTextProtoOrDie(absl::Substitute( + R"pb( + input_stream: "input_image" + node { + calculator: "TensorConverterCalculator" + input_stream: "IMAGE:input_image" + output_stream: "TENSORS:tensor" + options { + [mediapipe.TensorConverterCalculatorOptions.ext] { + output_tensor_float_range { min: $0 max: $1 } + } + } } - } - } - } - )", - /*$0=*/range.first, - /*$1=*/range.second)); + )pb", + /*$0=*/range.first, + /*$1=*/range.second)); std::vector output_packets; tool::AddVectorSink("tensor", &graph_config, &output_packets); @@ -292,26 +293,23 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) { // Wait until the calculator finishes processing. MP_ASSERT_OK(graph.WaitUntilIdle()); - EXPECT_THAT(output_packets.size(), Eq(1)); + ASSERT_EQ(output_packets.size(), 1); // Get and process results. const std::vector& tensor_vec = output_packets[0].Get>(); - EXPECT_THAT(tensor_vec.size(), Eq(1)); + ASSERT_EQ(tensor_vec.size(), 1); const Tensor* tensor = &tensor_vec[0]; // Calculate the expected normalized value: - float normalized_value = + float expected_value = range.first + (200 * (range.second - range.first)) / 255.0; - EXPECT_THAT(tensor->element_type(), Eq(Tensor::ElementType::kFloat32)); + EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32); auto view = tensor->GetCpuReadView(); - float dataf = *view.buffer(); - EXPECT_THAT( - normalized_value, - testing::FloatNear(dataf, 2.0f * std::abs(dataf) * - std::numeric_limits::epsilon())); + float actual_value = *view.buffer(); + EXPECT_FLOAT_EQ(actual_value, expected_value); // Fully close graph at end, otherwise calculator+tensors are destroyed // after calling WaitUntilDone(). @@ -320,4 +318,113 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) { } } +TEST_F(TensorConverterCalculatorTest, FlipVertically) { + CalculatorGraph graph; + CalculatorGraphConfig graph_config = + mediapipe::ParseTextProtoOrDie(R"pb( + input_stream: "input_image" + node { + calculator: "TensorConverterCalculator" + input_stream: "IMAGE:input_image" + output_stream: "TENSORS:tensor" + options { + [mediapipe.TensorConverterCalculatorOptions.ext] { + flip_vertically: true + output_tensor_float_range { min: 0 max: 255 } + } + } + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("tensor", &graph_config, &output_packets); + + // Run the graph. + MP_ASSERT_OK(graph.Initialize(graph_config)); + MP_ASSERT_OK(graph.StartRun({})); + auto input_image = absl::make_unique(ImageFormat::GRAY8, 1, 2); + cv::Mat mat = mediapipe::formats::MatView(input_image.get()); + constexpr uint8_t kY0Value = 100; + constexpr uint8_t kY1Value = 200; + mat.at(0, 0) = kY0Value; + mat.at(1, 0) = kY1Value; // Note: y, x! + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input_image", Adopt(input_image.release()).At(Timestamp(0)))); + + // Wait until the calculator finishes processing. + MP_ASSERT_OK(graph.WaitUntilIdle()); + ASSERT_EQ(output_packets.size(), 1); + + // Get and process results. + const std::vector& tensor_vec = + output_packets[0].Get>(); + ASSERT_EQ(tensor_vec.size(), 1); + + const Tensor* tensor = &tensor_vec[0]; + + EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32); + const float* dataf = tensor->GetCpuReadView().buffer(); + EXPECT_EQ(static_cast(roundf(dataf[0])), kY1Value); // Y0, Y1 flipped! + EXPECT_EQ(static_cast(roundf(dataf[1])), kY0Value); + + // Fully close graph at end, otherwise calculator+tensors are destroyed + // after calling WaitUntilDone(). + MP_ASSERT_OK(graph.CloseInputStream("input_image")); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + +TEST_F(TensorConverterCalculatorTest, GpuOriginOverridesFlipVertically) { + CalculatorGraph graph; + CalculatorGraphConfig graph_config = + mediapipe::ParseTextProtoOrDie(R"pb( + input_stream: "input_image" + node { + calculator: "TensorConverterCalculator" + input_stream: "IMAGE:input_image" + output_stream: "TENSORS:tensor" + options { + [mediapipe.TensorConverterCalculatorOptions.ext] { + flip_vertically: true + gpu_origin: TOP_LEFT + output_tensor_float_range { min: 0 max: 255 } + } + } + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("tensor", &graph_config, &output_packets); + + // Run the graph. + MP_ASSERT_OK(graph.Initialize(graph_config)); + MP_ASSERT_OK(graph.StartRun({})); + auto input_image = absl::make_unique(ImageFormat::GRAY8, 1, 2); + cv::Mat mat = mediapipe::formats::MatView(input_image.get()); + constexpr uint8_t kY0Value = 100; + constexpr uint8_t kY1Value = 200; + mat.at(0, 0) = kY0Value; + mat.at(1, 0) = kY1Value; // Note: y, x! + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input_image", Adopt(input_image.release()).At(Timestamp(0)))); + + // Wait until the calculator finishes processing. + MP_ASSERT_OK(graph.WaitUntilIdle()); + ASSERT_EQ(output_packets.size(), 1); + + // Get and process results. + const std::vector& tensor_vec = + output_packets[0].Get>(); + ASSERT_EQ(tensor_vec.size(), 1); + + const Tensor* tensor = &tensor_vec[0]; + + EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32); + const float* dataf = tensor->GetCpuReadView().buffer(); + EXPECT_EQ(static_cast(roundf(dataf[0])), kY0Value); // Not flipped! + EXPECT_EQ(static_cast(roundf(dataf[1])), kY1Value); + + // Fully close graph at end, otherwise calculator+tensors are destroyed + // after calling WaitUntilDone(). + MP_ASSERT_OK(graph.CloseInputStream("input_image")); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc index c8dd0e2a..6d42226b 100644 --- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc @@ -15,6 +15,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" #include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h" @@ -83,7 +84,7 @@ void ConvertRawValuesToAnchors(const float* raw_anchors, int num_boxes, void ConvertAnchorsToRawValues(const std::vector& anchors, int num_boxes, float* raw_anchors) { - CHECK_EQ(anchors.size(), num_boxes); + ABSL_CHECK_EQ(anchors.size(), num_boxes); int box = 0; for (const auto& anchor : anchors) { raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center(); @@ -256,6 +257,7 @@ class TensorsToDetectionsCalculator : public Node { bool gpu_inited_ = false; bool gpu_input_ = false; + bool gpu_has_enough_work_groups_ = true; bool anchors_init_ = false; }; MEDIAPIPE_REGISTER_NODE(TensorsToDetectionsCalculator); @@ -291,7 +293,7 @@ absl::Status TensorsToDetectionsCalculator::Open(CalculatorContext* cc) { absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) { auto output_detections = absl::make_unique>(); bool gpu_processing = false; - if (CanUseGpu()) { + if (CanUseGpu() && gpu_has_enough_work_groups_) { // Use GPU processing only if at least one input tensor is already on GPU // (to avoid CPU->GPU overhead). for (const auto& tensor : *kInTensors(cc)) { @@ -321,11 +323,20 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) { RET_CHECK(!has_custom_box_indices_); } - if (gpu_processing) { - if (!gpu_inited_) { - MP_RETURN_IF_ERROR(GpuInit(cc)); + if (gpu_processing && !gpu_inited_) { + auto status = GpuInit(cc); + if (status.ok()) { gpu_inited_ = true; + } else if (status.code() == absl::StatusCode::kFailedPrecondition) { + // For initialization error because of hardware limitation, fallback to + // CPU processing. + ABSL_LOG(WARNING) << status.message(); + } else { + // For other error, let the error propagates. + return status; } + } + if (gpu_processing && gpu_inited_) { MP_RETURN_IF_ERROR(ProcessGPU(cc, output_detections.get())); } else { MP_RETURN_IF_ERROR(ProcessCPU(cc, output_detections.get())); @@ -346,17 +357,41 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU( // TODO: Add flexible input tensor size handling. auto raw_box_tensor = &input_tensors[tensor_mapping_.detections_tensor_index()]; - RET_CHECK_EQ(raw_box_tensor->shape().dims.size(), 3); - RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1); RET_CHECK_GT(num_boxes_, 0) << "Please set num_boxes in calculator options"; - RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_); - RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_); + if (raw_box_tensor->shape().dims.size() == 3) { + // The tensors from CPU inference has dim 3. + RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1); + RET_CHECK_EQ(raw_box_tensor->shape().dims[1], num_boxes_); + RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_coords_); + } else if (raw_box_tensor->shape().dims.size() == 4) { + // The tensors from GPU inference has dim 4. For gpu-cpu fallback support, + // we allow tensors with 4 dims. + RET_CHECK_EQ(raw_box_tensor->shape().dims[0], 1); + RET_CHECK_EQ(raw_box_tensor->shape().dims[1], 1); + RET_CHECK_EQ(raw_box_tensor->shape().dims[2], num_boxes_); + RET_CHECK_EQ(raw_box_tensor->shape().dims[3], num_coords_); + } else { + return absl::InvalidArgumentError( + "The dimensions of box Tensor must be 3 or 4."); + } auto raw_score_tensor = &input_tensors[tensor_mapping_.scores_tensor_index()]; - RET_CHECK_EQ(raw_score_tensor->shape().dims.size(), 3); - RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1); - RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_); - RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_); + if (raw_score_tensor->shape().dims.size() == 3) { + // The tensors from CPU inference has dim 3. + RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1); + RET_CHECK_EQ(raw_score_tensor->shape().dims[1], num_boxes_); + RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_classes_); + } else if (raw_score_tensor->shape().dims.size() == 4) { + // The tensors from GPU inference has dim 4. For gpu-cpu fallback support, + // we allow tensors with 4 dims. + RET_CHECK_EQ(raw_score_tensor->shape().dims[0], 1); + RET_CHECK_EQ(raw_score_tensor->shape().dims[1], 1); + RET_CHECK_EQ(raw_score_tensor->shape().dims[2], num_boxes_); + RET_CHECK_EQ(raw_score_tensor->shape().dims[3], num_classes_); + } else { + return absl::InvalidArgumentError( + "The dimensions of score Tensor must be 3 or 4."); + } auto raw_box_view = raw_box_tensor->GetCpuReadView(); auto raw_boxes = raw_box_view.buffer(); auto raw_scores_view = raw_score_tensor->GetCpuReadView(); @@ -634,7 +669,7 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU( output_detections)); #else - LOG(ERROR) << "GPU input on non-Android not supported yet."; + ABSL_LOG(ERROR) << "GPU input on non-Android not supported yet."; #endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) return absl::OkStatus(); } @@ -669,18 +704,18 @@ absl::Status TensorsToDetectionsCalculator::LoadOptions(CalculatorContext* cc) { num_boxes_ = options_.num_boxes(); num_coords_ = options_.num_coords(); box_output_format_ = GetBoxFormat(options_); - CHECK_NE(options_.max_results(), 0) + ABSL_CHECK_NE(options_.max_results(), 0) << "The maximum number of the top-scored detection results must be " "non-zero."; max_results_ = options_.max_results(); // Currently only support 2D when num_values_per_keypoint equals to 2. - CHECK_EQ(options_.num_values_per_keypoint(), 2); + ABSL_CHECK_EQ(options_.num_values_per_keypoint(), 2); // Check if the output size is equal to the requested boxes and keypoints. - CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + - kNumCoordsPerBox, - num_coords_); + ABSL_CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + + kNumCoordsPerBox, + num_coords_); if (kSideInIgnoreClasses(cc).IsConnected()) { RET_CHECK(!kSideInIgnoreClasses(cc).IsEmpty()); @@ -1111,15 +1146,21 @@ void main() { int max_wg_size; // typically <= 1024 glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1, &max_wg_size); // y-dim - CHECK_LT(num_classes_, max_wg_size) - << "# classes must be < " << max_wg_size; + gpu_has_enough_work_groups_ = num_classes_ < max_wg_size; + if (!gpu_has_enough_work_groups_) { + return absl::FailedPreconditionError(absl::StrFormat( + "Hardware limitation: Processing will be done on CPU, because " + "num_classes %d exceeds the max work_group size %d.", + num_classes_, max_wg_size)); + } // TODO support better filtering. if (class_index_set_.is_allowlist) { - CHECK_EQ(class_index_set_.values.size(), - IsClassIndexAllowed(0) ? num_classes_ : num_classes_ - 1) + ABSL_CHECK_EQ(class_index_set_.values.size(), + IsClassIndexAllowed(0) ? num_classes_ : num_classes_ - 1) << "Only all classes >= class 0 or >= class 1"; } else { - CHECK_EQ(class_index_set_.values.size(), IsClassIndexAllowed(0) ? 0 : 1) + ABSL_CHECK_EQ(class_index_set_.values.size(), + IsClassIndexAllowed(0) ? 0 : 1) << "Only ignore class 0 is allowed"; } @@ -1340,11 +1381,12 @@ kernel void scoreKernel( // TODO support better filtering. if (class_index_set_.is_allowlist) { - CHECK_EQ(class_index_set_.values.size(), - IsClassIndexAllowed(0) ? num_classes_ : num_classes_ - 1) + ABSL_CHECK_EQ(class_index_set_.values.size(), + IsClassIndexAllowed(0) ? num_classes_ : num_classes_ - 1) << "Only all classes >= class 0 or >= class 1"; } else { - CHECK_EQ(class_index_set_.values.size(), IsClassIndexAllowed(0) ? 0 : 1) + ABSL_CHECK_EQ(class_index_set_.values.size(), + IsClassIndexAllowed(0) ? 0 : 1) << "Only ignore class 0 is allowed"; } @@ -1370,7 +1412,13 @@ kernel void scoreKernel( Tensor::ElementType::kFloat32, Tensor::Shape{1, num_boxes_ * 2}); // # filter classes supported is hardware dependent. int max_wg_size = score_program_.maxTotalThreadsPerThreadgroup; - CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size; + gpu_has_enough_work_groups_ = num_classes_ < max_wg_size; + if (!gpu_has_enough_work_groups_) { + return absl::FailedPreconditionError(absl::StrFormat( + "Hardware limitation: Processing will be done on CPU, because " + "num_classes %d exceeds the max work_group size %d.", + num_classes_, max_wg_size)); + } } #endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) diff --git a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc index a1cc4e20..5942f234 100644 --- a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc @@ -142,7 +142,7 @@ absl::Status TensorsToLandmarksCalculator::Process(CalculatorContext* cc) { RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32); int num_values = input_tensors[0].shape().num_elements(); const int num_dimensions = num_values / num_landmarks_; - CHECK_GT(num_dimensions, 0); + ABSL_CHECK_GT(num_dimensions, 0); auto view = input_tensors[0].GetCpuReadView(); auto raw_landmarks = view.buffer(); diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index c4b9ab9f..cd4d1ad8 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -13,6 +13,7 @@ # limitations under the License. # +# Placeholder: load py_proto_library load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") licenses(["notice"]) @@ -314,6 +315,7 @@ cc_library( "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ] + select({ "//conditions:default": [ "@org_tensorflow//tensorflow/core:framework", @@ -366,15 +368,14 @@ cc_library( name = "pack_media_sequence_calculator", srcs = ["pack_media_sequence_calculator.cc"], deps = [ + ":pack_media_sequence_calculator_cc_proto", "//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto", - "//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:location", "//mediapipe/framework/formats:location_opencv", "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", "//mediapipe/util/sequence:media_sequence", "//mediapipe/util/sequence:media_sequence_util", "@com_google_absl//absl/container:flat_hash_map", @@ -406,8 +407,13 @@ cc_library( alwayslink = 1, ) -# This dependency removed tensorflow_jellyfish_deps and xprofilez_with_server because they failed -# Boq conformance test. Weigh your use case to see if this will work for you. +# This dependency removed the following 3 targets because they failed Boq conformance test: +# +# tensorflow_jellyfish_deps +# jfprof_lib +# xprofilez_with_server +# +# If you need them plz consider tensorflow_inference_calculator_no_envelope_loader. cc_library( name = "tensorflow_inference_calculator_for_boq", srcs = ["tensorflow_inference_calculator.cc"], @@ -424,7 +430,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/log:check", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", @@ -483,10 +489,10 @@ cc_library( "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/core:protos_all_cc", ] + select({ "//conditions:default": [ @@ -514,10 +520,10 @@ cc_library( ":tensorflow_session_from_frozen_graph_generator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/core:protos_all_cc", ] + select({ "//conditions:default": [ @@ -550,6 +556,7 @@ cc_library( "//mediapipe/framework/deps:file_path", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/cc/saved_model:constants", "@org_tensorflow//tensorflow/cc/saved_model:loader_lite", @@ -627,6 +634,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/cc/saved_model:constants", @@ -648,6 +656,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/core:framework", ], alwayslink = 1, @@ -662,6 +671,7 @@ cc_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", "@org_tensorflow//tensorflow/core:framework", ], alwayslink = 1, @@ -677,6 +687,7 @@ cc_library( "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ] + select({ "//conditions:default": [ "@org_tensorflow//tensorflow/core:framework", @@ -773,6 +784,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/util:audio_decoder_cc_proto", "//mediapipe/util/sequence:media_sequence", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:protos_all_cc", ], @@ -787,6 +799,8 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/core:framework", ], alwayslink = 1, @@ -800,6 +814,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/core:framework", ], alwayslink = 1, @@ -813,6 +828,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/core:framework", ], alwayslink = 1, @@ -826,6 +842,8 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework:packet", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/core:protos_all_cc", ], alwayslink = 1, @@ -920,22 +938,22 @@ cc_test( srcs = ["pack_media_sequence_calculator_test.cc"], deps = [ ":pack_media_sequence_calculator", + ":pack_media_sequence_calculator_cc_proto", "//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto", - "//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", + "//mediapipe/framework:packet", "//mediapipe/framework:timestamp", "//mediapipe/framework/formats:detection_cc_proto", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/formats:location", "//mediapipe/framework/formats:location_opencv", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/util/sequence:media_sequence", - "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -1077,6 +1095,7 @@ cc_test( linkstatic = 1, deps = [ ":tensor_to_image_frame_calculator", + ":tensor_to_image_frame_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/formats:image_frame", @@ -1162,6 +1181,7 @@ cc_test( "//mediapipe/framework/port:rectangle", "//mediapipe/util:audio_decoder_cc_proto", "//mediapipe/util/sequence:media_sequence", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:protos_all_cc", @@ -1243,6 +1263,8 @@ cc_test( "//mediapipe/framework/tool:sink", "//mediapipe/framework/tool:validate_type", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ] + select({ "//conditions:default": [ "@org_tensorflow//tensorflow/core:direct_session", diff --git a/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator.cc index 32a0eb70..bbd5cff3 100644 --- a/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensorflow/matrix_to_tensor_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_check.h" #include "mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_options.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/matrix.h" @@ -28,7 +29,7 @@ namespace mediapipe { namespace { absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, TimeSeriesHeader* header) { - CHECK(header); + ABSL_CHECK(header); if (header_packet.IsEmpty()) { return absl::UnknownError("No header found."); } diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc index 34136440..7a1f2472 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc @@ -12,21 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include #include #include "absl/container/flat_hash_map.h" #include "absl/strings/match.h" +#include "absl/strings/strip.h" #include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" #include "mediapipe/calculators/tensorflow/pack_media_sequence_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/location.h" #include "mediapipe/framework/formats/location_opencv.h" -#include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status.h" #include "mediapipe/util/sequence/media_sequence.h" #include "mediapipe/util/sequence/media_sequence_util.h" #include "tensorflow/core/example/example.pb.h" @@ -36,6 +37,7 @@ namespace mediapipe { const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE"; const char kImageTag[] = "IMAGE"; +const char kImageLabelPrefixTag[] = "IMAGE_LABEL_"; const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_"; const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_"; const char kIntFeaturePrefixTag[] = "INT_FEATURE_"; @@ -44,6 +46,7 @@ const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED"; const char kBBoxTag[] = "BBOX"; const char kKeypointsTag[] = "KEYPOINTS"; const char kSegmentationMaskTag[] = "CLASS_SEGMENTATION"; +const char kClipMediaIdTag[] = "CLIP_MEDIA_ID"; namespace tf = ::tensorflow; namespace mpms = mediapipe::mediasequence; @@ -55,16 +58,21 @@ namespace mpms = mediapipe::mediasequence; // context features can be supplied verbatim in the calculator's options. The // SequenceExample will conform to the description in media_sequence.h. // -// The supported input stream tags are "IMAGE", which stores the encoded -// images from the OpenCVImageEncoderCalculator, "FORWARD_FLOW_ENCODED", which -// stores the encoded optical flow from the same calculator, "BBOX" which stores -// bounding boxes from vector, and streams with the -// "FLOAT_FEATURE_${NAME}" pattern, which stores the values from vector's -// associated with the name ${NAME}. "KEYPOINTS" stores a map of 2D keypoints -// from flat_hash_map>>. "IMAGE_${NAME}", -// "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store prefixed versions of -// each stream, which allows for multiple image streams to be included. However, -// the default names are suppored by more tools. +// The supported input stream tags are: +// * "IMAGE", which stores the encoded images from the +// OpenCVImageEncoderCalculator, +// * "IMAGE_LABEL", which stores whole image labels from Detection, +// * "FORWARD_FLOW_ENCODED", which stores the encoded optical flow from the same +// calculator, +// * "BBOX" which stores bounding boxes from vector, +// * streams with the "FLOAT_FEATURE_${NAME}" pattern, which stores the values +// from vector's associated with the name ${NAME}, +// * "KEYPOINTS" stores a map of 2D keypoints from flat_hash_map>>, +// * "CLIP_MEDIA_ID", which stores the clip's media ID as a string. +// "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store +// prefixed versions of each stream, which allows for multiple image streams to +// be included. However, the default names are suppored by more tools. // // Example config: // node { @@ -100,6 +108,9 @@ class PackMediaSequenceCalculator : public CalculatorBase { static absl::Status GetContract(CalculatorContract* cc) { RET_CHECK(cc->InputSidePackets().HasTag(kSequenceExampleTag)); cc->InputSidePackets().Tag(kSequenceExampleTag).Set(); + if (cc->InputSidePackets().HasTag(kClipMediaIdTag)) { + cc->InputSidePackets().Tag(kClipMediaIdTag).Set(); + } if (cc->Inputs().HasTag(kForwardFlowEncodedTag)) { cc->Inputs() @@ -112,6 +123,10 @@ class PackMediaSequenceCalculator : public CalculatorBase { for (const auto& tag : cc->Inputs().GetTags()) { if (absl::StartsWith(tag, kImageTag)) { + if (absl::StartsWith(tag, kImageLabelPrefixTag)) { + cc->Inputs().Tag(tag).Set(); + continue; + } std::string key = ""; if (tag != kImageTag) { int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; @@ -164,8 +179,8 @@ class PackMediaSequenceCalculator : public CalculatorBase { } } - CHECK(cc->Outputs().HasTag(kSequenceExampleTag) || - cc->OutputSidePackets().HasTag(kSequenceExampleTag)) + RET_CHECK(cc->Outputs().HasTag(kSequenceExampleTag) || + cc->OutputSidePackets().HasTag(kSequenceExampleTag)) << "Neither the output stream nor the output side packet is set to " "output the sequence example."; if (cc->Outputs().HasTag(kSequenceExampleTag)) { @@ -184,6 +199,11 @@ class PackMediaSequenceCalculator : public CalculatorBase { cc->InputSidePackets() .Tag(kSequenceExampleTag) .Get()); + if (cc->InputSidePackets().HasTag(kClipMediaIdTag) && + !cc->InputSidePackets().Tag(kClipMediaIdTag).IsEmpty()) { + clip_media_id_ = + cc->InputSidePackets().Tag(kClipMediaIdTag).Get(); + } const auto& context_features = cc->Options().context_feature_map(); @@ -199,6 +219,16 @@ class PackMediaSequenceCalculator : public CalculatorBase { .replace_data_instead_of_append()) { for (const auto& tag : cc->Inputs().GetTags()) { if (absl::StartsWith(tag, kImageTag)) { + if (absl::StartsWith(tag, kImageLabelPrefixTag)) { + std::string key = + std::string(absl::StripPrefix(tag, kImageLabelPrefixTag)); + mpms::ClearImageLabelString(key, sequence_.get()); + mpms::ClearImageLabelConfidence(key, sequence_.get()); + if (!key.empty() || mpms::HasImageEncoded(*sequence_)) { + mpms::ClearImageTimestamp(key, sequence_.get()); + } + continue; + } std::string key = ""; if (tag != kImageTag) { int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; @@ -227,6 +257,7 @@ class PackMediaSequenceCalculator : public CalculatorBase { mpms::ClearBBoxNumRegions(key, sequence_.get()); mpms::ClearBBoxLabelString(key, sequence_.get()); mpms::ClearBBoxLabelIndex(key, sequence_.get()); + mpms::ClearBBoxLabelConfidence(key, sequence_.get()); mpms::ClearBBoxClassString(key, sequence_.get()); mpms::ClearBBoxClassIndex(key, sequence_.get()); mpms::ClearBBoxTrackString(key, sequence_.get()); @@ -343,6 +374,34 @@ class PackMediaSequenceCalculator : public CalculatorBase { if (absl::StartsWith(tag, kImageTag) && !cc->Inputs().Tag(tag).IsEmpty()) { std::string key = ""; + if (absl::StartsWith(tag, kImageLabelPrefixTag)) { + std::string key = + std::string(absl::StripPrefix(tag, kImageLabelPrefixTag)); + const auto& detection = cc->Inputs().Tag(tag).Get(); + if (detection.label().empty()) continue; + RET_CHECK(detection.label_size() == detection.score_size()) + << "Wrong image label data format: " << detection.label_size() + << " vs " << detection.score_size(); + if (!detection.label_id().empty()) { + RET_CHECK(detection.label_id_size() == detection.label_size()) + << "Wrong image label ID format: " << detection.label_id_size() + << " vs " << detection.label_size(); + } + std::vector labels(detection.label().begin(), + detection.label().end()); + std::vector confidences(detection.score().begin(), + detection.score().end()); + std::vector ids(detection.label_id().begin(), + detection.label_id().end()); + if (!key.empty() || mpms::HasImageEncoded(*sequence_)) { + mpms::AddImageTimestamp(key, cc->InputTimestamp().Value(), + sequence_.get()); + } + mpms::AddImageLabelString(key, labels, sequence_.get()); + mpms::AddImageLabelConfidence(key, confidences, sequence_.get()); + if (!ids.empty()) mpms::AddImageLabelIndex(key, ids, sequence_.get()); + continue; + } if (tag != kImageTag) { int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; if (tag[tag_length] == '_') { @@ -393,6 +452,7 @@ class PackMediaSequenceCalculator : public CalculatorBase { mpms::ClearBBoxNumRegions(prefix, sequence_.get()); mpms::ClearBBoxLabelString(prefix, sequence_.get()); mpms::ClearBBoxLabelIndex(prefix, sequence_.get()); + mpms::ClearBBoxLabelConfidence(prefix, sequence_.get()); mpms::ClearBBoxClassString(prefix, sequence_.get()); mpms::ClearBBoxClassIndex(prefix, sequence_.get()); mpms::ClearBBoxTrackString(prefix, sequence_.get()); @@ -460,6 +520,7 @@ class PackMediaSequenceCalculator : public CalculatorBase { } std::vector predicted_locations; std::vector predicted_class_strings; + std::vector predicted_class_confidences; std::vector predicted_label_ids; for (auto& detection : cc->Inputs().Tag(tag).Get>()) { @@ -488,6 +549,9 @@ class PackMediaSequenceCalculator : public CalculatorBase { if (detection.label_id_size() > 0) { predicted_label_ids.push_back(detection.label_id(0)); } + if (detection.score_size() > 0) { + predicted_class_confidences.push_back(detection.score(0)); + } } } if (!predicted_locations.empty()) { @@ -501,6 +565,10 @@ class PackMediaSequenceCalculator : public CalculatorBase { if (!predicted_label_ids.empty()) { mpms::AddBBoxLabelIndex(key, predicted_label_ids, sequence_.get()); } + if (!predicted_class_confidences.empty()) { + mpms::AddBBoxLabelConfidence(key, predicted_class_confidences, + sequence_.get()); + } } } } @@ -548,10 +616,14 @@ class PackMediaSequenceCalculator : public CalculatorBase { } } } + if (clip_media_id_.has_value()) { + mpms::SetClipMediaId(*clip_media_id_, sequence_.get()); + } return absl::OkStatus(); } std::unique_ptr sequence_; + std::optional clip_media_id_ = std::nullopt; std::map features_present_; bool replace_keypoints_; }; diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc index 752db621..3fb48d1e 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc @@ -12,28 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +#include +#include -#include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" -#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" #include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" #include "mediapipe/calculators/tensorflow/pack_media_sequence_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/formats/detection.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/location.h" #include "mediapipe/framework/formats/location_opencv.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h" #include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/framework/timestamp.h" #include "mediapipe/util/sequence/media_sequence.h" #include "tensorflow/core/example/example.pb.h" #include "tensorflow/core/example/feature.pb.h" +#include "testing/base/public/gmock.h" +#include "testing/base/public/gunit.h" namespace mediapipe { namespace { @@ -59,9 +60,12 @@ constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER"; constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST"; constexpr char kIntFeatureOtherTag[] = "INT_FEATURE_OTHER"; constexpr char kIntFeatureTestTag[] = "INT_FEATURE_TEST"; +constexpr char kImageLabelTestTag[] = "IMAGE_LABEL_TEST"; +constexpr char kImageLabelOtherTag[] = "IMAGE_LABEL_OTHER"; constexpr char kImagePrefixTag[] = "IMAGE_PREFIX"; constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE"; constexpr char kImageTag[] = "IMAGE"; +constexpr char kClipMediaIdTag[] = "CLIP_MEDIA_ID"; class PackMediaSequenceCalculatorTest : public ::testing::Test { protected: @@ -69,10 +73,14 @@ class PackMediaSequenceCalculatorTest : public ::testing::Test { const tf::Features& features, const bool output_only_if_all_present, const bool replace_instead_of_append, - const bool output_as_zero_timestamp = false) { + const bool output_as_zero_timestamp = false, + const std::vector& input_side_packets = { + "SEQUENCE_EXAMPLE:input_sequence"}) { CalculatorGraphConfig::Node config; config.set_calculator("PackMediaSequenceCalculator"); - config.add_input_side_packet("SEQUENCE_EXAMPLE:input_sequence"); + for (const std::string& side_packet : input_side_packets) { + config.add_input_side_packet(side_packet); + } config.add_output_stream("SEQUENCE_EXAMPLE:output_sequence"); for (const std::string& stream : input_streams) { config.add_input_stream(stream); @@ -96,7 +104,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) { mpms::SetClipMediaId(test_video_id, input_sequence.get()); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); OpenCvImageEncoderCalculatorResults encoded_image; encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_width(2); @@ -139,7 +148,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) { mpms::SetClipMediaId(test_video_id, input_sequence.get()); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); OpenCvImageEncoderCalculatorResults encoded_image; encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_width(2); @@ -312,6 +322,76 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBytesLists) { } } +TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImageLabels) { + SetUpCalculator( + {"IMAGE_LABEL_TEST:test_labels", "IMAGE_LABEL_OTHER:test_labels2"}, {}, + false, true); + auto input_sequence = ::absl::make_unique(); + + int num_timesteps = 2; + for (int i = 0; i < num_timesteps; ++i) { + Detection detection1; + detection1.add_label(absl::StrCat("foo", 2 << i)); + detection1.add_label_id(i); + detection1.add_score(0.1 * i); + detection1.add_label(absl::StrCat("foo", 2 << i)); + detection1.add_label_id(i); + detection1.add_score(0.1 * i); + auto label_ptr1 = ::absl::make_unique(detection1); + runner_->MutableInputs() + ->Tag(kImageLabelTestTag) + .packets.push_back(Adopt(label_ptr1.release()).At(Timestamp(i))); + Detection detection2; + detection2.add_label(absl::StrCat("bar", 2 << i)); + detection2.add_score(0.2 * i); + detection2.add_label(absl::StrCat("bar", 2 << i)); + detection2.add_score(0.2 * i); + auto label_ptr2 = ::absl::make_unique(detection2); + runner_->MutableInputs() + ->Tag(kImageLabelOtherTag) + .packets.push_back(Adopt(label_ptr2.release()).At(Timestamp(i))); + } + runner_->MutableSidePackets()->Tag(kSequenceExampleTag) = + Adopt(input_sequence.release()); + + MP_ASSERT_OK(runner_->Run()); + + const std::vector& output_packets = + runner_->Outputs().Tag(kSequenceExampleTag).packets; + ASSERT_EQ(1, output_packets.size()); + const tf::SequenceExample& output_sequence = + output_packets[0].Get(); + + ASSERT_EQ(num_timesteps, + mpms::GetImageTimestampSize("TEST", output_sequence)); + ASSERT_EQ(num_timesteps, + mpms::GetImageLabelStringSize("TEST", output_sequence)); + ASSERT_EQ(num_timesteps, + mpms::GetImageLabelConfidenceSize("TEST", output_sequence)); + ASSERT_EQ(num_timesteps, + mpms::GetImageTimestampSize("OTHER", output_sequence)); + ASSERT_EQ(num_timesteps, + mpms::GetImageLabelStringSize("OTHER", output_sequence)); + ASSERT_EQ(num_timesteps, + mpms::GetImageLabelConfidenceSize("OTHER", output_sequence)); + for (int i = 0; i < num_timesteps; ++i) { + ASSERT_EQ(i, mpms::GetImageTimestampAt("TEST", output_sequence, i)); + ASSERT_THAT(mpms::GetImageLabelStringAt("TEST", output_sequence, i), + ::testing::ElementsAreArray( + std::vector(2, absl::StrCat("foo", 2 << i)))); + ASSERT_THAT(mpms::GetImageLabelIndexAt("TEST", output_sequence, i), + ::testing::ElementsAreArray(std::vector(2, i))); + ASSERT_THAT(mpms::GetImageLabelConfidenceAt("TEST", output_sequence, i), + ::testing::ElementsAreArray(std::vector(2, 0.1 * i))); + ASSERT_EQ(i, mpms::GetImageTimestampAt("OTHER", output_sequence, i)); + ASSERT_THAT(mpms::GetImageLabelStringAt("OTHER", output_sequence, i), + ::testing::ElementsAreArray( + std::vector(2, absl::StrCat("bar", 2 << i)))); + ASSERT_THAT(mpms::GetImageLabelConfidenceAt("OTHER", output_sequence, i), + ::testing::ElementsAreArray(std::vector(2, 0.2 * i))); + } +} + TEST_F(PackMediaSequenceCalculatorTest, OutputAsZeroTimestamp) { SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true, true); auto input_sequence = ::absl::make_unique(); @@ -378,7 +458,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) { Adopt(input_sequence.release()); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); OpenCvImageEncoderCalculatorResults encoded_image; encoded_image.set_encoded_image(bytes.data(), bytes.size()); auto image_ptr = @@ -410,7 +491,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) { cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); std::string test_flow_string(bytes.begin(), bytes.end()); OpenCvImageEncoderCalculatorResults encoded_flow; encoded_flow.set_encoded_image(test_flow_string); @@ -526,6 +608,10 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBBoxDetections) { auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i); ASSERT_EQ(0, class_indices[0]); ASSERT_EQ(1, class_indices[1]); + auto class_scores = + mpms::GetPredictedBBoxLabelConfidenceAt(output_sequence, i); + ASSERT_FLOAT_EQ(0.5, class_scores[0]); + ASSERT_FLOAT_EQ(0.75, class_scores[1]); } } @@ -618,7 +704,8 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) { } cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); OpenCvImageEncoderCalculatorResults encoded_image; encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_width(width); @@ -667,6 +754,10 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) { auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i); ASSERT_EQ(0, class_indices[0]); ASSERT_EQ(1, class_indices[1]); + auto class_scores = + mpms::GetPredictedBBoxLabelConfidenceAt(output_sequence, i); + ASSERT_FLOAT_EQ(0.5, class_scores[0]); + ASSERT_FLOAT_EQ(0.75, class_scores[1]); } } @@ -757,6 +848,88 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoMaskDetections) { testing::ElementsAreArray(::std::vector({"mask"}))); } +TEST_F(PackMediaSequenceCalculatorTest, AddClipMediaId) { + SetUpCalculator( + /*input_streams=*/{"FLOAT_FEATURE_TEST:test", + "FLOAT_FEATURE_OTHER:test2"}, + /*features=*/{}, + /*output_only_if_all_present=*/false, + /*replace_instead_of_append=*/true, + /*output_as_zero_timestamp=*/false, /*input_side_packets=*/ + {"SEQUENCE_EXAMPLE:input_sequence", "CLIP_MEDIA_ID:video_id"}); + auto input_sequence = absl::make_unique(); + const std::string test_video_id = "test_video_id"; + + int num_timesteps = 2; + for (int i = 0; i < num_timesteps; ++i) { + auto vf_ptr = ::absl::make_unique>(2, 2 << i); + runner_->MutableInputs() + ->Tag(kFloatFeatureTestTag) + .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i))); + vf_ptr = ::absl::make_unique>(2, 2 << i); + runner_->MutableInputs() + ->Tag(kFloatFeatureOtherTag) + .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i))); + } + + runner_->MutableSidePackets()->Tag(kClipMediaIdTag) = + MakePacket(test_video_id); + runner_->MutableSidePackets()->Tag(kSequenceExampleTag) = + Adopt(input_sequence.release()); + + MP_ASSERT_OK(runner_->Run()); + + const std::vector& output_packets = + runner_->Outputs().Tag(kSequenceExampleTag).packets; + ASSERT_EQ(1, output_packets.size()); + const tf::SequenceExample& output_sequence = + output_packets[0].Get(); + + ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); +} + +TEST_F(PackMediaSequenceCalculatorTest, ReplaceClipMediaId) { + SetUpCalculator( + /*input_streams=*/{"FLOAT_FEATURE_TEST:test", + "FLOAT_FEATURE_OTHER:test2"}, + /*features=*/{}, + /*output_only_if_all_present=*/false, + /*replace_instead_of_append=*/true, + /*output_as_zero_timestamp=*/false, /*input_side_packets=*/ + {"SEQUENCE_EXAMPLE:input_sequence", "CLIP_MEDIA_ID:video_id"}); + auto input_sequence = absl::make_unique(); + const std::string existing_video_id = "existing_video_id"; + mpms::SetClipMediaId(existing_video_id, input_sequence.get()); + const std::string test_video_id = "test_video_id"; + + int num_timesteps = 2; + for (int i = 0; i < num_timesteps; ++i) { + auto vf_ptr = ::absl::make_unique>(2, 2 << i); + runner_->MutableInputs() + ->Tag(kFloatFeatureTestTag) + .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i))); + vf_ptr = ::absl::make_unique>(2, 2 << i); + runner_->MutableInputs() + ->Tag(kFloatFeatureOtherTag) + .packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i))); + } + + runner_->MutableSidePackets()->Tag(kClipMediaIdTag) = + MakePacket(test_video_id).At(Timestamp(0)); + runner_->MutableSidePackets()->Tag(kSequenceExampleTag) = + Adopt(input_sequence.release()); + + MP_ASSERT_OK(runner_->Run()); + + const std::vector& output_packets = + runner_->Outputs().Tag(kSequenceExampleTag).packets; + ASSERT_EQ(1, output_packets.size()); + const tf::SequenceExample& output_sequence = + output_packets[0].Get(); + + ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence)); +} + TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) { SetUpCalculator( {"FORWARD_FLOW_ENCODED:flow", "FLOAT_FEATURE_I3D_FLOW:feature"}, {}, @@ -767,7 +940,8 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) { cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); std::string test_flow_string(bytes.begin(), bytes.end()); OpenCvImageEncoderCalculatorResults encoded_flow; encoded_flow.set_encoded_image(test_flow_string); @@ -813,7 +987,8 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamNotOK) { mpms::SetClipMediaId(test_video_id, input_sequence.get()); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); std::string test_flow_string(bytes.begin(), bytes.end()); OpenCvImageEncoderCalculatorResults encoded_flow; encoded_flow.set_encoded_image(test_flow_string); @@ -970,7 +1145,8 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) { auto input_sequence = ::absl::make_unique(); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); OpenCvImageEncoderCalculatorResults encoded_image; encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_width(2); @@ -1021,7 +1197,8 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) { auto input_sequence = ::absl::make_unique(); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); std::vector bytes; - ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80})); + ASSERT_TRUE( + cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1})); OpenCvImageEncoderCalculatorResults encoded_image; encoded_image.set_encoded_image(bytes.data(), bytes.size()); int height = 2; @@ -1057,6 +1234,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) { mpms::AddBBoxNumRegions(-1, input_sequence.get()); mpms::AddBBoxLabelString({"anything"}, input_sequence.get()); mpms::AddBBoxLabelIndex({-1}, input_sequence.get()); + mpms::AddBBoxLabelConfidence({-1}, input_sequence.get()); mpms::AddBBoxClassString({"anything"}, input_sequence.get()); mpms::AddBBoxClassIndex({-1}, input_sequence.get()); mpms::AddBBoxTrackString({"anything"}, input_sequence.get()); diff --git a/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.cc b/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.cc index ad87297a..8b938a86 100644 --- a/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/ret_check.h" @@ -99,10 +100,11 @@ class TensorSqueezeDimensionsCalculator : public CalculatorBase { } } if (remove_dims_.empty()) { - LOG(ERROR) << "TensorSqueezeDimensionsCalculator is squeezing input with " - "no single-dimensions. Calculator will be a no-op."; - LOG(ERROR) << "Input to TensorSqueezeDimensionsCalculator has shape " - << tensor_shape.DebugString(); + ABSL_LOG(ERROR) + << "TensorSqueezeDimensionsCalculator is squeezing input with " + "no single-dimensions. Calculator will be a no-op."; + ABSL_LOG(ERROR) << "Input to TensorSqueezeDimensionsCalculator has shape " + << tensor_shape.DebugString(); } } }; diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc index 34e397b3..3b4d5381 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.cc @@ -14,6 +14,7 @@ #include +#include "absl/log/absl_check.h" #include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" @@ -65,6 +66,7 @@ class TensorToImageFrameCalculator : public CalculatorBase { private: float scale_factor_; + bool scale_per_frame_min_max_; }; REGISTER_CALCULATOR(TensorToImageFrameCalculator); @@ -88,6 +90,8 @@ absl::Status TensorToImageFrameCalculator::GetContract(CalculatorContract* cc) { absl::Status TensorToImageFrameCalculator::Open(CalculatorContext* cc) { scale_factor_ = cc->Options().scale_factor(); + scale_per_frame_min_max_ = cc->Options() + .scale_per_frame_min_max(); cc->SetOffset(TimestampDiff(0)); return absl::OkStatus(); } @@ -96,7 +100,7 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) { const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get(); int32_t depth = 1; if (input_tensor.dims() != 2) { // Depth is 1 for 2D tensors. - CHECK(3 == input_tensor.dims()) + ABSL_CHECK(3 == input_tensor.dims()) << "Only 2 or 3-D Tensors can be converted to frames. Instead got: " << input_tensor.dims(); depth = input_tensor.dim_size(2); @@ -109,16 +113,38 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) { auto format = (depth == 3 ? ImageFormat::SRGB : ImageFormat::GRAY8); const int32_t total_size = height * width * depth; + if (scale_per_frame_min_max_) { + RET_CHECK_EQ(input_tensor.dtype(), tensorflow::DT_FLOAT) + << "Setting scale_per_frame_min_max requires FLOAT input tensors."; + } ::std::unique_ptr output; if (input_tensor.dtype() == tensorflow::DT_FLOAT) { // Allocate buffer with alignments. std::unique_ptr buffer( new (std::align_val_t(EIGEN_MAX_ALIGN_BYTES)) uint8_t[total_size]); auto data = input_tensor.flat().data(); + float min = 1e23; + float max = -1e23; + if (scale_per_frame_min_max_) { + for (int i = 0; i < total_size; ++i) { + float d = scale_factor_ * data[i]; + if (d < min) { + min = d; + } + if (d > max) { + max = d; + } + } + } for (int i = 0; i < total_size; ++i) { - float d = scale_factor_ * data[i]; - if (d < 0) d = 0; - if (d > 255) d = 255; + float d = data[i]; + if (scale_per_frame_min_max_) { + d = 255 * (d - min) / (max - min + 1e-9); + } else { + d = scale_factor_ * d; + if (d < 0) d = 0; + if (d > 255) d = 255; + } buffer[i] = d; } output = ::absl::make_unique( diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto index 3410068d..c60448c1 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto +++ b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.proto @@ -26,4 +26,8 @@ message TensorToImageFrameCalculatorOptions { // Multiples floating point tensor outputs by this value before converting to // uint8. This is useful for converting from range [0, 1] to [0, 255] optional float scale_factor = 1 [default = 1.0]; + + // If true, scales any FLOAT tensor input of [min, max] to be between [0, 255] + // per frame. This overrides any explicit scale_factor. + optional bool scale_per_frame_min_max = 2 [default = false]; } diff --git a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc index aee9fee9..13255ac4 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator_test.cc @@ -11,7 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include +#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/formats/image_frame.h" @@ -32,11 +34,14 @@ constexpr char kImage[] = "IMAGE"; template class TensorToImageFrameCalculatorTest : public ::testing::Test { protected: - void SetUpRunner() { + void SetUpRunner(bool scale_per_frame_min_max = false) { CalculatorGraphConfig::Node config; config.set_calculator("TensorToImageFrameCalculator"); config.add_input_stream("TENSOR:input_tensor"); config.add_output_stream("IMAGE:output_image"); + config.mutable_options() + ->MutableExtension(mediapipe::TensorToImageFrameCalculatorOptions::ext) + ->set_scale_per_frame_min_max(scale_per_frame_min_max); runner_ = absl::make_unique(config); } @@ -157,4 +162,47 @@ TYPED_TEST(TensorToImageFrameCalculatorTest, } } +TYPED_TEST(TensorToImageFrameCalculatorTest, + Converts3DTensorToImageFrame2DGrayWithScaling) { + this->SetUpRunner(true); + auto& runner = this->runner_; + constexpr int kWidth = 16; + constexpr int kHeight = 8; + const tf::TensorShape tensor_shape{kHeight, kWidth}; + auto tensor = absl::make_unique( + tf::DataTypeToEnum::v(), tensor_shape); + auto tensor_vec = tensor->template flat().data(); + + // Writing sequence of integers as floats which we want normalized. + tensor_vec[0] = 255; + for (int i = 1; i < kWidth * kHeight; ++i) { + tensor_vec[i] = 200; + } + + const int64_t time = 1234; + runner->MutableInputs()->Tag(kTensor).packets.push_back( + Adopt(tensor.release()).At(Timestamp(time))); + + if (!std::is_same::value) { + EXPECT_FALSE(runner->Run().ok()); + return; // Short circuit because does not apply to other types. + } else { + EXPECT_TRUE(runner->Run().ok()); + const std::vector& output_packets = + runner->Outputs().Tag(kImage).packets; + EXPECT_EQ(1, output_packets.size()); + EXPECT_EQ(time, output_packets[0].Timestamp().Value()); + const ImageFrame& output_image = output_packets[0].Get(); + EXPECT_EQ(ImageFormat::GRAY8, output_image.Format()); + EXPECT_EQ(kWidth, output_image.Width()); + EXPECT_EQ(kHeight, output_image.Height()); + + EXPECT_EQ(255, output_image.PixelData()[0]); + for (int i = 1; i < kWidth * kHeight; ++i) { + const uint8_t pixel_value = output_image.PixelData()[i]; + ASSERT_EQ(0, pixel_value); + } + } +} + } // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc index 081e0c83..dc3d9784 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc @@ -15,6 +15,7 @@ // Calculator converts from one-dimensional Tensor of DT_FLOAT to Matrix // OR from (batched) two-dimensional Tensor of DT_FLOAT to Matrix. +#include "absl/log/absl_check.h" #include "mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/matrix.h" @@ -36,7 +37,7 @@ constexpr char kReference[] = "REFERENCE"; absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, TimeSeriesHeader* header) { - CHECK(header); + ABSL_CHECK(header); if (header_packet.IsEmpty()) { return absl::UnknownError("No header found."); } @@ -191,7 +192,7 @@ absl::Status TensorToMatrixCalculator::Process(CalculatorContext* cc) { << "Tensor stream packet does not contain a Tensor."; const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get(); - CHECK(1 == input_tensor.dims() || 2 == input_tensor.dims()) + ABSL_CHECK(1 == input_tensor.dims() || 2 == input_tensor.dims()) << "Only 1-D or 2-D Tensors can be converted to matrices."; const int32_t length = input_tensor.dim_size(input_tensor.dims() - 1); const int32_t width = diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc index 2608b1c5..84c32fed 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc @@ -20,6 +20,7 @@ #include #include "absl/base/thread_annotations.h" +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/strings/str_split.h" #include "absl/synchronization/mutex.h" @@ -515,7 +516,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase { tf::Tensor concated; const tf::Status concat_status = tf::tensor::Concat(keyed_tensors.second, &concated); - CHECK(concat_status.ok()) << concat_status.ToString(); + ABSL_CHECK(concat_status.ok()) << concat_status.ToString(); input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first], concated); } @@ -597,7 +598,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase { std::vector split_tensors; const tf::Status split_status = tf::tensor::Split(outputs[i], split_vector, &split_tensors); - CHECK(split_status.ok()) << split_status.ToString(); + ABSL_CHECK(split_status.ok()) << split_status.ToString(); // Loop over timestamps so that we don't copy the padding. for (int j = 0; j < inference_state->batch_timestamps_.size(); ++j) { tf::Tensor output_tensor(split_tensors[j]); diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc index c9300837..708f1711 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator_test.cc @@ -17,6 +17,8 @@ #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -118,7 +120,7 @@ class TensorflowInferenceCalculatorTest : public ::testing::Test { // Create tensor from Vector and add as a Packet to the provided tag as input. void AddVectorToInputsAsPacket(const std::vector& packets, const std::string& tag) { - CHECK(!packets.empty()) + ABSL_CHECK(!packets.empty()) << "Please specify at least some data in the packet"; auto packets_ptr = absl::make_unique>(packets); runner_->MutableInputs()->Tag(tag).packets.push_back( @@ -586,12 +588,12 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStates) { runner_->Outputs().Tag(kMultipliedTag).packets; ASSERT_EQ(2, output_packets_mult.size()); const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - LOG(INFO) << "timestamp: " << 0; + ABSL_LOG(INFO) << "timestamp: " << 0; auto expected_tensor = tf::test::AsTensor({3, 8, 15}); tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); auto expected_tensor1 = tf::test::AsTensor({9, 32, 75}); - LOG(INFO) << "timestamp: " << 1; + ABSL_LOG(INFO) << "timestamp: " << 1; tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); EXPECT_EQ(2, runner_ @@ -627,12 +629,12 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStateOverride) { runner_->Outputs().Tag(kMultipliedTag).packets; ASSERT_EQ(2, output_packets_mult.size()); const tf::Tensor& tensor_mult = output_packets_mult[0].Get(); - LOG(INFO) << "timestamp: " << 0; + ABSL_LOG(INFO) << "timestamp: " << 0; auto expected_tensor = tf::test::AsTensor({3, 4, 5}); tf::test::ExpectTensorEqual(tensor_mult, expected_tensor); const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get(); auto expected_tensor1 = tf::test::AsTensor({3, 4, 5}); - LOG(INFO) << "timestamp: " << 1; + ABSL_LOG(INFO) << "timestamp: " << 1; tf::test::ExpectTensorEqual(tensor_mult1, expected_tensor1); EXPECT_EQ(2, runner_ diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc index 1bb2c41f..358b50cd 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc @@ -23,12 +23,12 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/deps/clock.h" #include "mediapipe/framework/deps/monotonic_clock.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/tool/status_util.h" @@ -156,8 +156,8 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase { cc->OutputSidePackets().Tag(kSessionTag).Set(Adopt(session.release())); const uint64_t end_time = absl::ToUnixMicros(clock->TimeNow()); - LOG(INFO) << "Loaded frozen model in: " << end_time - start_time - << " microseconds."; + ABSL_LOG(INFO) << "Loaded frozen model in: " << end_time - start_time + << " microseconds."; return absl::OkStatus(); } diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc index dc39458d..e340a098 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.cc @@ -24,13 +24,13 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/deps/clock.h" #include "mediapipe/framework/deps/monotonic_clock.h" #include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/tool/status_util.h" @@ -155,8 +155,8 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator { output_side_packets->Tag(kSessionTag) = Adopt(session.release()); const uint64_t end_time = absl::ToUnixMicros(clock->TimeNow()); - LOG(INFO) << "Loaded frozen model in: " << end_time - start_time - << " microseconds."; + ABSL_LOG(INFO) << "Loaded frozen model in: " << end_time - start_time + << " microseconds."; return absl::OkStatus(); } }; diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc index 18bddbbe..4ca4cb8d 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc @@ -17,6 +17,7 @@ #if !defined(__ANDROID__) #include "mediapipe/framework/port/file_helpers.h" #endif +#include "absl/log/absl_log.h" #include "absl/strings/str_replace.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h" @@ -69,7 +70,7 @@ const std::string MaybeConvertSignatureToTag( [](unsigned char c) { return std::toupper(c); }); output = absl::StrReplaceAll( output, {{"/", "_"}, {"-", "_"}, {".", "_"}, {":", "_"}}); - LOG(INFO) << "Renamed TAG from: " << name << " to " << output; + ABSL_LOG(INFO) << "Renamed TAG from: " << name << " to " << output; return output; } else { return name; diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc index ee69ec56..95962244 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc @@ -19,6 +19,7 @@ #if !defined(__ANDROID__) #include "mediapipe/framework/port/file_helpers.h" #endif +#include "absl/log/absl_log.h" #include "absl/strings/str_replace.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h" @@ -75,7 +76,7 @@ const std::string MaybeConvertSignatureToTag( [](unsigned char c) { return std::toupper(c); }); output = absl::StrReplaceAll( output, {{"/", "_"}, {"-", "_"}, {".", "_"}, {":", "_"}}); - LOG(INFO) << "Renamed TAG from: " << name << " to " << output; + ABSL_LOG(INFO) << "Renamed TAG from: " << name << " to " << output; return output; } else { return name; diff --git a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc index a14c6bd9..c77c0f3f 100644 --- a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc +++ b/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "absl/container/flat_hash_map.h" +#include "absl/log/absl_log.h" #include "absl/strings/match.h" #include "mediapipe/calculators/core/packet_resampler_calculator.pb.h" #include "mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.pb.h" @@ -201,8 +202,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase { first_timestamp_seen_ = Timestamp::OneOverPostStream().Value(); for (const auto& map_kv : sequence_->feature_lists().feature_list()) { if (absl::StrContains(map_kv.first, "/timestamp")) { - LOG(INFO) << "Found feature timestamps: " << map_kv.first - << " with size: " << map_kv.second.feature_size(); + ABSL_LOG(INFO) << "Found feature timestamps: " << map_kv.first + << " with size: " << map_kv.second.feature_size(); int64_t recent_timestamp = Timestamp::PreStream().Value(); for (int i = 0; i < map_kv.second.feature_size(); ++i) { int64_t next_timestamp = @@ -309,8 +310,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase { audio_decoder_options->set_end_time( end_time + options.extra_padding_from_media_decoder()); } - LOG(INFO) << "Created AudioDecoderOptions:\n" - << audio_decoder_options->DebugString(); + ABSL_LOG(INFO) << "Created AudioDecoderOptions:\n" + << audio_decoder_options->DebugString(); cc->OutputSidePackets() .Tag(kAudioDecoderOptions) .Set(Adopt(audio_decoder_options.release())); @@ -331,8 +332,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase { ->set_end_time(Timestamp::FromSeconds(end_time).Value()); } - LOG(INFO) << "Created PacketResamplerOptions:\n" - << resampler_options->DebugString(); + ABSL_LOG(INFO) << "Created PacketResamplerOptions:\n" + << resampler_options->DebugString(); cc->OutputSidePackets() .Tag(kPacketResamplerOptions) .Set(Adopt(resampler_options.release())); @@ -351,7 +352,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase { absl::Status Process(CalculatorContext* cc) override { if (timestamps_.empty()) { // This occurs when we only have metadata to unpack. - LOG(INFO) << "only unpacking metadata because there are no timestamps."; + ABSL_LOG(INFO) + << "only unpacking metadata because there are no timestamps."; return tool::StatusStop(); } // In Process(), we loop through timestamps on a reference stream and emit diff --git a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc b/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc index addb4a27..2fa70de3 100644 --- a/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/unpack_media_sequence_calculator_test.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/numbers.h" #include "mediapipe/calculators/core/packet_resampler_calculator.pb.h" @@ -81,7 +82,7 @@ class UnpackMediaSequenceCalculatorTest : public ::testing::Test { if (options != nullptr) { *config.mutable_options() = *options; } - LOG(INFO) << config.DebugString(); + ABSL_LOG(INFO) << config.DebugString(); runner_ = absl::make_unique(config); } diff --git a/mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc b/mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc index efb3037f..12f2ade0 100644 --- a/mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc +++ b/mediapipe/calculators/tensorflow/unpack_yt8m_sequence_example_calculator.cc @@ -14,6 +14,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/packet.h" @@ -46,7 +48,7 @@ std::string GetQuantizedFeature( .Get(index) .bytes_list() .value(); - CHECK_EQ(1, bytes_list.size()); + ABSL_CHECK_EQ(1, bytes_list.size()); return bytes_list.Get(0); } } // namespace @@ -149,8 +151,9 @@ class UnpackYt8mSequenceExampleCalculator : public CalculatorBase { .Set(MakePacket(segment_size)); } } - LOG(INFO) << "Reading the sequence example that contains yt8m id: " - << yt8m_id << ". Feature list length: " << feature_list_length_; + ABSL_LOG(INFO) << "Reading the sequence example that contains yt8m id: " + << yt8m_id + << ". Feature list length: " << feature_list_length_; return absl::OkStatus(); } diff --git a/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc index 28184a8c..dd0991cb 100644 --- a/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator.cc @@ -14,6 +14,7 @@ // // Converts vector (or vector>) to 1D (or 2D) tf::Tensor. +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_options.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/ret_check.h" @@ -68,7 +69,7 @@ absl::Status VectorFloatToTensorCalculator::GetContract( // Output vector. ); } else { - LOG(FATAL) << "input size not supported"; + ABSL_LOG(FATAL) << "input size not supported"; } RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) << "Only one output stream is supported."; @@ -125,7 +126,7 @@ absl::Status VectorFloatToTensorCalculator::Process(CalculatorContext* cc) { } cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); } else { - LOG(FATAL) << "input size not supported"; + ABSL_LOG(FATAL) << "input size not supported"; } return absl::OkStatus(); } diff --git a/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc index cb90276a..f4a89202 100644 --- a/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator.cc @@ -15,6 +15,8 @@ // Converts a single int or vector or vector> to 1D (or 2D) // tf::Tensor. +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/ret_check.h" @@ -86,7 +88,7 @@ absl::Status VectorIntToTensorCalculator::GetContract(CalculatorContract* cc) { cc->Inputs().Tag(kVectorInt).Set>(); } } else { - LOG(FATAL) << "input size not supported"; + ABSL_LOG(FATAL) << "input size not supported"; } RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) << "Only one output stream is supported."; @@ -113,11 +115,11 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) { .Get>>(); const int32_t rows = input.size(); - CHECK_GE(rows, 1); + ABSL_CHECK_GE(rows, 1); const int32_t cols = input[0].size(); - CHECK_GE(cols, 1); + ABSL_CHECK_GE(cols, 1); for (int i = 1; i < rows; ++i) { - CHECK_EQ(input[i].size(), cols); + ABSL_CHECK_EQ(input[i].size(), cols); } if (options_.transpose()) { tensor_shape = tf::TensorShape({cols, rows}); @@ -140,7 +142,7 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) { AssignMatrixValue(c, r, input[r][c], output.get()); break; default: - LOG(FATAL) << "tensor data type is not supported."; + ABSL_LOG(FATAL) << "tensor data type is not supported."; } } } @@ -158,7 +160,7 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) { AssignMatrixValue(r, c, input[r][c], output.get()); break; default: - LOG(FATAL) << "tensor data type is not supported."; + ABSL_LOG(FATAL) << "tensor data type is not supported."; } } } @@ -171,7 +173,7 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) { } else { input = cc->Inputs().Tag(kVectorInt).Value().Get>(); } - CHECK_GE(input.size(), 1); + ABSL_CHECK_GE(input.size(), 1); const int32_t length = input.size(); tensor_shape = tf::TensorShape({length}); auto output = ::absl::make_unique(options_.tensor_data_type(), @@ -188,12 +190,12 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) { output->tensor()(i) = input.at(i); break; default: - LOG(FATAL) << "tensor data type is not supported."; + ABSL_LOG(FATAL) << "tensor data type is not supported."; } } cc->Outputs().Tag(kTensorOut).Add(output.release(), cc->InputTimestamp()); } else { - LOG(FATAL) << "input size not supported"; + ABSL_LOG(FATAL) << "input size not supported"; } return absl::OkStatus(); } diff --git a/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator.cc b/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator.cc index 13951127..57ee553c 100644 --- a/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator.cc @@ -15,6 +15,7 @@ // Converts vector (or vector>) to 1D (or 2D) // tf::Tensor. +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_options.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/ret_check.h" @@ -69,7 +70,7 @@ absl::Status VectorStringToTensorCalculator::GetContract( // Input vector. ); } else { - LOG(FATAL) << "input size not supported"; + ABSL_LOG(FATAL) << "input size not supported"; } RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) << "Only one output stream is supported."; @@ -129,7 +130,7 @@ absl::Status VectorStringToTensorCalculator::Process(CalculatorContext* cc) { } cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); } else { - LOG(FATAL) << "input size not supported"; + ABSL_LOG(FATAL) << "input size not supported"; } return absl::OkStatus(); } diff --git a/mediapipe/calculators/tflite/BUILD b/mediapipe/calculators/tflite/BUILD index 333de206..ed9f47a8 100644 --- a/mediapipe/calculators/tflite/BUILD +++ b/mediapipe/calculators/tflite/BUILD @@ -103,6 +103,8 @@ cc_library( "//mediapipe/framework/formats/object_detection:anchor_cc_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) @@ -196,10 +198,13 @@ cc_library( deps = [ ":tflite_inference_calculator_cc_proto", "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/util/tflite:config", "//mediapipe/util/tflite:tflite_model_loader", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate", @@ -275,6 +280,7 @@ cc_library( "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/util:resource_util", "//mediapipe/util/tflite:config", + "@com_google_absl//absl/log:absl_check", "@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", ] + selects.with_or({ @@ -392,6 +398,8 @@ cc_library( "//mediapipe/framework/formats/object_detection:anchor_cc_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/util/tflite:config", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", "@org_tensorflow//tensorflow/lite:framework", @@ -428,6 +436,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/util:resource_util", "@com_google_absl//absl/container:node_hash_map", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", "@org_tensorflow//tensorflow/lite:framework", @@ -456,6 +465,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/log:absl_check", "@org_tensorflow//tensorflow/lite:framework", ], alwayslink = 1, diff --git a/mediapipe/calculators/tflite/ssd_anchors_calculator.cc b/mediapipe/calculators/tflite/ssd_anchors_calculator.cc index 5ed5a95d..d5303d65 100644 --- a/mediapipe/calculators/tflite/ssd_anchors_calculator.cc +++ b/mediapipe/calculators/tflite/ssd_anchors_calculator.cc @@ -16,6 +16,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tflite/ssd_anchors_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/object_detection/anchor.pb.h" @@ -272,13 +274,13 @@ absl::Status SsdAnchorsCalculator::GenerateAnchors( if (options.feature_map_height_size()) { if (options.strides_size()) { - LOG(ERROR) << "Found feature map shapes. Strides will be ignored."; + ABSL_LOG(ERROR) << "Found feature map shapes. Strides will be ignored."; } - CHECK_EQ(options.feature_map_height_size(), kNumLayers); - CHECK_EQ(options.feature_map_height_size(), - options.feature_map_width_size()); + ABSL_CHECK_EQ(options.feature_map_height_size(), kNumLayers); + ABSL_CHECK_EQ(options.feature_map_height_size(), + options.feature_map_width_size()); } else { - CHECK_EQ(options.strides_size(), kNumLayers); + ABSL_CHECK_EQ(options.strides_size(), kNumLayers); } if (options.multiscale_anchor_generation()) { diff --git a/mediapipe/calculators/tflite/tflite_converter_calculator.cc b/mediapipe/calculators/tflite/tflite_converter_calculator.cc index ff6b2ff9..7188cbc5 100644 --- a/mediapipe/calculators/tflite/tflite_converter_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_converter_calculator.cc @@ -15,6 +15,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/calculators/tflite/tflite_converter_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" @@ -643,7 +644,7 @@ absl::Status TfLiteConverterCalculator::LoadOptions(CalculatorContext* cc) { if (options.has_output_tensor_float_range()) { output_range_.emplace(options.output_tensor_float_range().min(), options.output_tensor_float_range().max()); - CHECK_GT(output_range_->second, output_range_->first); + ABSL_CHECK_GT(output_range_->second, output_range_->first); } // Custom div and sub values. @@ -661,9 +662,9 @@ absl::Status TfLiteConverterCalculator::LoadOptions(CalculatorContext* cc) { // Get desired way to handle input channels. max_num_channels_ = options.max_num_channels(); - CHECK_GE(max_num_channels_, 1); - CHECK_LE(max_num_channels_, 4); - CHECK_NE(max_num_channels_, 2); + ABSL_CHECK_GE(max_num_channels_, 1); + ABSL_CHECK_LE(max_num_channels_, 4); + ABSL_CHECK_NE(max_num_channels_, 2); #if defined(MEDIAPIPE_IOS) if (cc->Inputs().HasTag(kGpuBufferTag)) // Currently on iOS, tflite gpu input tensor must be 4 channels, diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.cc b/mediapipe/calculators/tflite/tflite_inference_calculator.cc index add9bb1a..d875b694 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.cc @@ -17,9 +17,12 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/util/tflite/config.h" @@ -109,8 +112,8 @@ std::unique_ptr BuildEdgeTpuInterpreter( edgetpu::EdgeTpuContext* edgetpu_context) { resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); std::unique_ptr interpreter; - CHECK_EQ(tflite::InterpreterBuilder(model, *resolver)(&interpreter), - kTfLiteOk); + ABSL_CHECK_EQ(tflite::InterpreterBuilder(model, *resolver)(&interpreter), + kTfLiteOk); interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context); return interpreter; } @@ -406,11 +409,12 @@ absl::Status TfLiteInferenceCalculator::Open(CalculatorContext* cc) { } if (use_advanced_gpu_api_ && !gpu_input_) { - LOG(WARNING) << "Cannot use advanced GPU APIs, input must be GPU buffers." - "Falling back to the default TFLite API."; + ABSL_LOG(WARNING) + << "Cannot use advanced GPU APIs, input must be GPU buffers." + "Falling back to the default TFLite API."; use_advanced_gpu_api_ = false; } - CHECK(!use_advanced_gpu_api_ || gpu_inference_); + ABSL_CHECK(!use_advanced_gpu_api_ || gpu_inference_); MP_RETURN_IF_ERROR(LoadModel(cc)); @@ -802,9 +806,10 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner( const int tensor_idx = interpreter_->inputs()[i]; interpreter_->SetTensorParametersReadWrite(tensor_idx, kTfLiteFloat32, "", shape, quant); - CHECK(interpreter_->ResizeInputTensor(tensor_idx, shape) == kTfLiteOk); + ABSL_CHECK(interpreter_->ResizeInputTensor(tensor_idx, shape) == + kTfLiteOk); } - CHECK(interpreter_->AllocateTensors() == kTfLiteOk); + ABSL_CHECK(interpreter_->AllocateTensors() == kTfLiteOk); } // Create and bind OpenGL buffers for outputs. @@ -1053,7 +1058,7 @@ absl::Status TfLiteInferenceCalculator::LoadDelegate(CalculatorContext* cc) { gpu_data_in_[i]->shape.w * gpu_data_in_[i]->shape.c; // Input to model can be RGBA only. if (tensor->dims->data[3] != 4) { - LOG(WARNING) << "Please ensure input GPU tensor is 4 channels."; + ABSL_LOG(WARNING) << "Please ensure input GPU tensor is 4 channels."; } const std::string shader_source = absl::Substitute(R"(#include diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc index 4d28b91e..98ab4b1d 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc @@ -17,6 +17,7 @@ #include #include "absl/container/node_hash_map.h" +#include "absl/log/absl_check.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" #include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h" @@ -172,7 +173,7 @@ absl::Status TfLiteTensorsToClassificationCalculator::Process( // Note that partial_sort will raise error when top_k_ > // classification_list->classification_size(). - CHECK_GE(classification_list->classification_size(), top_k_); + ABSL_CHECK_GE(classification_list->classification_size(), top_k_); auto raw_classification_list = classification_list->mutable_classification(); if (top_k_ > 0 && classification_list->classification_size() >= top_k_) { std::partial_sort(raw_classification_list->begin(), diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc index 2ed62c46..269661f7 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc @@ -15,6 +15,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" #include "mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.pb.h" @@ -93,7 +95,7 @@ void ConvertRawValuesToAnchors(const float* raw_anchors, int num_boxes, void ConvertAnchorsToRawValues(const std::vector& anchors, int num_boxes, float* raw_anchors) { - CHECK_EQ(anchors.size(), num_boxes); + ABSL_CHECK_EQ(anchors.size(), num_boxes); int box = 0; for (const auto& anchor : anchors) { raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center(); @@ -288,14 +290,14 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessCPU( const TfLiteTensor* raw_score_tensor = &input_tensors[1]; // TODO: Add flexible input tensor size handling. - CHECK_EQ(raw_box_tensor->dims->size, 3); - CHECK_EQ(raw_box_tensor->dims->data[0], 1); - CHECK_EQ(raw_box_tensor->dims->data[1], num_boxes_); - CHECK_EQ(raw_box_tensor->dims->data[2], num_coords_); - CHECK_EQ(raw_score_tensor->dims->size, 3); - CHECK_EQ(raw_score_tensor->dims->data[0], 1); - CHECK_EQ(raw_score_tensor->dims->data[1], num_boxes_); - CHECK_EQ(raw_score_tensor->dims->data[2], num_classes_); + ABSL_CHECK_EQ(raw_box_tensor->dims->size, 3); + ABSL_CHECK_EQ(raw_box_tensor->dims->data[0], 1); + ABSL_CHECK_EQ(raw_box_tensor->dims->data[1], num_boxes_); + ABSL_CHECK_EQ(raw_box_tensor->dims->data[2], num_coords_); + ABSL_CHECK_EQ(raw_score_tensor->dims->size, 3); + ABSL_CHECK_EQ(raw_score_tensor->dims->data[0], 1); + ABSL_CHECK_EQ(raw_score_tensor->dims->data[1], num_boxes_); + ABSL_CHECK_EQ(raw_score_tensor->dims->data[2], num_classes_); const float* raw_boxes = raw_box_tensor->data.f; const float* raw_scores = raw_score_tensor->data.f; @@ -303,13 +305,13 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessCPU( if (!anchors_init_) { if (input_tensors.size() == kNumInputTensorsWithAnchors) { const TfLiteTensor* anchor_tensor = &input_tensors[2]; - CHECK_EQ(anchor_tensor->dims->size, 2); - CHECK_EQ(anchor_tensor->dims->data[0], num_boxes_); - CHECK_EQ(anchor_tensor->dims->data[1], kNumCoordsPerBox); + ABSL_CHECK_EQ(anchor_tensor->dims->size, 2); + ABSL_CHECK_EQ(anchor_tensor->dims->data[0], num_boxes_); + ABSL_CHECK_EQ(anchor_tensor->dims->data[1], kNumCoordsPerBox); const float* raw_anchors = anchor_tensor->data.f; ConvertRawValuesToAnchors(raw_anchors, num_boxes_, &anchors_); } else if (side_packet_anchors_) { - CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); + ABSL_CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); anchors_ = cc->InputSidePackets().Tag("ANCHORS").Get>(); } else { @@ -409,7 +411,7 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU( CopyBuffer(input_tensors[1], gpu_data_->raw_scores_buffer)); if (!anchors_init_) { if (side_packet_anchors_) { - CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); + ABSL_CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); const auto& anchors = cc->InputSidePackets().Tag("ANCHORS").Get>(); std::vector raw_anchors(num_boxes_ * kNumCoordsPerBox); @@ -417,7 +419,7 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU( MP_RETURN_IF_ERROR(gpu_data_->raw_anchors_buffer.Write( absl::MakeSpan(raw_anchors))); } else { - CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors); + ABSL_CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors); MP_RETURN_IF_ERROR( CopyBuffer(input_tensors[2], gpu_data_->raw_anchors_buffer)); } @@ -477,7 +479,7 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU( commandBuffer:[gpu_helper_ commandBuffer]]; if (!anchors_init_) { if (side_packet_anchors_) { - CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); + ABSL_CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); const auto& anchors = cc->InputSidePackets().Tag("ANCHORS").Get>(); std::vector raw_anchors(num_boxes_ * kNumCoordsPerBox); @@ -541,7 +543,7 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU( output_detections)); #else - LOG(ERROR) << "GPU input on non-Android not supported yet."; + ABSL_LOG(ERROR) << "GPU input on non-Android not supported yet."; #endif // MEDIAPIPE_TFLITE_GL_INFERENCE return absl::OkStatus(); } @@ -567,12 +569,12 @@ absl::Status TfLiteTensorsToDetectionsCalculator::LoadOptions( num_coords_ = options_.num_coords(); // Currently only support 2D when num_values_per_keypoint equals to 2. - CHECK_EQ(options_.num_values_per_keypoint(), 2); + ABSL_CHECK_EQ(options_.num_values_per_keypoint(), 2); // Check if the output size is equal to the requested boxes and keypoints. - CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + - kNumCoordsPerBox, - num_coords_); + ABSL_CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + + kNumCoordsPerBox, + num_coords_); for (int i = 0; i < options_.ignore_classes_size(); ++i) { ignore_classes_.insert(options_.ignore_classes(i)); @@ -897,10 +899,11 @@ void main() { int max_wg_size; // typically <= 1024 glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1, &max_wg_size); // y-dim - CHECK_LT(num_classes_, max_wg_size) + ABSL_CHECK_LT(num_classes_, max_wg_size) << "# classes must be < " << max_wg_size; // TODO support better filtering. - CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed"; + ABSL_CHECK_LE(ignore_classes_.size(), 1) + << "Only ignore class 0 is allowed"; // Shader program GlShader score_shader; @@ -1115,7 +1118,7 @@ kernel void scoreKernel( ignore_classes_.size() ? 1 : 0); // TODO support better filtering. - CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed"; + ABSL_CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed"; { // Shader program @@ -1147,7 +1150,8 @@ kernel void scoreKernel( options:MTLResourceStorageModeShared]; // # filter classes supported is hardware dependent. int max_wg_size = gpu_data_->score_program.maxTotalThreadsPerThreadgroup; - CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size; + ABSL_CHECK_LT(num_classes_, max_wg_size) + << "# classes must be <" << max_wg_size; } #endif // MEDIAPIPE_TFLITE_GL_INFERENCE diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc index 1be83bbe..6740f0af 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_check.h" #include "mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/landmark.pb.h" @@ -199,7 +200,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Process( num_values *= raw_tensor->dims->data[i]; } const int num_dimensions = num_values / num_landmarks_; - CHECK_GT(num_dimensions, 0); + ABSL_CHECK_GT(num_dimensions, 0); const float* raw_landmarks = raw_tensor->data.f; diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index 4c92e949..ad75c65d 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -183,9 +183,9 @@ cc_library( "//mediapipe/framework:calculator_options_cc_proto", "//mediapipe/framework:timestamp", "//mediapipe/framework/deps:clock", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", ], @@ -248,11 +248,12 @@ cc_library( ":annotation_overlay_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:image_opencv", "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:status", @@ -260,6 +261,7 @@ cc_library( "//mediapipe/util:annotation_renderer", "//mediapipe/util:color_cc_proto", "//mediapipe/util:render_data_cc_proto", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ] + select({ "//mediapipe/gpu:disable_gpu": [], @@ -374,9 +376,10 @@ cc_library( "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:location", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:rectangle", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) @@ -675,6 +678,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/util:color_cc_proto", "//mediapipe/util:render_data_cc_proto", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], @@ -731,6 +735,7 @@ cc_library( "//mediapipe/framework/port:statusor", "//mediapipe/util:color_cc_proto", "//mediapipe/util:render_data_cc_proto", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -746,6 +751,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/util:color_cc_proto", "//mediapipe/util:render_data_cc_proto", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) @@ -1149,6 +1155,7 @@ cc_library( "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) @@ -1209,6 +1216,7 @@ cc_library( "//mediapipe/framework/port:rectangle", "//mediapipe/framework/port:status", "//mediapipe/util:rectangle_util", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", ], alwayslink = 1, @@ -1480,6 +1488,7 @@ cc_library( "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", ], alwayslink = 1, diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc index 34093702..f31ce915 100644 --- a/mediapipe/calculators/util/annotation_overlay_calculator.cc +++ b/mediapipe/calculators/util/annotation_overlay_calculator.cc @@ -14,15 +14,17 @@ #include +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "mediapipe/calculators/util/annotation_overlay_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" +#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/image_opencv.h" #include "mediapipe/framework/formats/video_stream_header.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/status.h" @@ -45,6 +47,7 @@ namespace { constexpr char kVectorTag[] = "VECTOR"; constexpr char kGpuBufferTag[] = "IMAGE_GPU"; constexpr char kImageFrameTag[] = "IMAGE"; +constexpr char kImageTag[] = "UIMAGE"; // Universal Image enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; @@ -57,13 +60,16 @@ size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT constexpr uchar kAnnotationBackgroundColor = 2; // Grayscale value. // Future Image type. -inline bool HasImageTag(mediapipe::CalculatorContext* cc) { return false; } +inline bool HasImageTag(mediapipe::CalculatorContext* cc) { + return cc->Inputs().HasTag(kImageTag); +} } // namespace // A calculator for rendering data on images. // // Inputs: // 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer), +// or UIMAGE (an Image). // containing the input image. // If output is CPU, and input isn't provided, the renderer creates a // blank canvas with the width, height and color provided in the options. @@ -76,6 +82,7 @@ inline bool HasImageTag(mediapipe::CalculatorContext* cc) { return false; } // // Output: // 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer), +// or UIMAGE (an Image). // Note: Output types should match their corresponding input stream type. // // For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The @@ -135,6 +142,9 @@ class AnnotationOverlayCalculator : public CalculatorBase { absl::Status CreateRenderTargetCpu(CalculatorContext* cc, std::unique_ptr& image_mat, ImageFormat::Format* target_format); + absl::Status CreateRenderTargetCpuImage(CalculatorContext* cc, + std::unique_ptr& image_mat, + ImageFormat::Format* target_format); template absl::Status CreateRenderTargetGpu(CalculatorContext* cc, std::unique_ptr& image_mat); @@ -172,30 +182,38 @@ class AnnotationOverlayCalculator : public CalculatorBase { REGISTER_CALCULATOR(AnnotationOverlayCalculator); absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) { - CHECK_GE(cc->Inputs().NumEntries(), 1); + RET_CHECK_GE(cc->Inputs().NumEntries(), 1); bool use_gpu = false; - if (cc->Inputs().HasTag(kImageFrameTag) && - cc->Inputs().HasTag(kGpuBufferTag)) { - return absl::InternalError("Cannot have multiple input images."); - } - if (cc->Inputs().HasTag(kGpuBufferTag) != - cc->Outputs().HasTag(kGpuBufferTag)) { - return absl::InternalError("GPU output must have GPU input."); - } + RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) + + cc->Inputs().HasTag(kGpuBufferTag) + + cc->Inputs().HasTag(kImageTag) <= + 1); + RET_CHECK(cc->Outputs().HasTag(kImageFrameTag) + + cc->Outputs().HasTag(kGpuBufferTag) + + cc->Outputs().HasTag(kImageTag) == + 1); // Input image to render onto copy of. Should be same type as output. #if !MEDIAPIPE_DISABLE_GPU if (cc->Inputs().HasTag(kGpuBufferTag)) { cc->Inputs().Tag(kGpuBufferTag).Set(); - CHECK(cc->Outputs().HasTag(kGpuBufferTag)); + RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag)); use_gpu = true; } #endif // !MEDIAPIPE_DISABLE_GPU if (cc->Inputs().HasTag(kImageFrameTag)) { cc->Inputs().Tag(kImageFrameTag).Set(); - CHECK(cc->Outputs().HasTag(kImageFrameTag)); + RET_CHECK(cc->Outputs().HasTag(kImageFrameTag)); + } + + if (cc->Inputs().HasTag(kImageTag)) { + cc->Inputs().Tag(kImageTag).Set(); + RET_CHECK(cc->Outputs().HasTag(kImageTag)); +#if !MEDIAPIPE_DISABLE_GPU + use_gpu = true; // Prepare GPU resources because images can come in on GPU. +#endif } // Data streams to render. @@ -220,6 +238,9 @@ absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) { if (cc->Outputs().HasTag(kImageFrameTag)) { cc->Outputs().Tag(kImageFrameTag).Set(); } + if (cc->Outputs().HasTag(kImageTag)) { + cc->Outputs().Tag(kImageTag).Set(); + } if (use_gpu) { #if !MEDIAPIPE_DISABLE_GPU @@ -252,9 +273,14 @@ absl::Status AnnotationOverlayCalculator::Open(CalculatorContext* cc) { renderer_ = absl::make_unique(); renderer_->SetFlipTextVertically(options_.flip_text_vertically()); if (use_gpu_) renderer_->SetScaleFactor(options_.gpu_scale_factor()); + if (renderer_->GetScaleFactor() < 1.0 && HasImageTag(cc)) + ABSL_LOG(WARNING) + << "Annotation scale factor only supports GPU backed Image."; // Set the output header based on the input header (if present). - const char* tag = use_gpu_ ? kGpuBufferTag : kImageFrameTag; + const char* tag = HasImageTag(cc) ? kImageTag + : use_gpu_ ? kGpuBufferTag + : kImageFrameTag; if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) { const auto& input_header = cc->Inputs().Tag(tag).Header().Get(); @@ -280,6 +306,12 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) { cc->Inputs().Tag(kImageFrameTag).IsEmpty()) { return absl::OkStatus(); } + if (cc->Inputs().HasTag(kImageTag) && cc->Inputs().Tag(kImageTag).IsEmpty()) { + return absl::OkStatus(); + } + if (HasImageTag(cc)) { + use_gpu_ = cc->Inputs().Tag(kImageTag).Get().UsesGpu(); + } // Initialize render target, drawn with OpenCV. std::unique_ptr image_mat; @@ -289,10 +321,17 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) { if (!gpu_initialized_) { MP_RETURN_IF_ERROR( gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { + if (HasImageTag(cc)) { + return GlSetup(cc); + } return GlSetup(cc); })); gpu_initialized_ = true; } + if (HasImageTag(cc)) { + MP_RETURN_IF_ERROR( + (CreateRenderTargetGpu(cc, image_mat))); + } if (cc->Inputs().HasTag(kGpuBufferTag)) { MP_RETURN_IF_ERROR( (CreateRenderTargetGpu( @@ -300,6 +339,10 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) { } #endif // !MEDIAPIPE_DISABLE_GPU } else { + if (cc->Outputs().HasTag(kImageTag)) { + MP_RETURN_IF_ERROR( + CreateRenderTargetCpuImage(cc, image_mat, &target_format)); + } if (cc->Outputs().HasTag(kImageFrameTag)) { MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); } @@ -339,6 +382,9 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) { uchar* image_mat_ptr = image_mat->data; MP_RETURN_IF_ERROR( gpu_helper_.RunInGlContext([this, cc, image_mat_ptr]() -> absl::Status { + if (HasImageTag(cc)) { + return RenderToGpu(cc, image_mat_ptr); + } return RenderToGpu( cc, image_mat_ptr); })); @@ -381,6 +427,10 @@ absl::Status AnnotationOverlayCalculator::RenderToCpu( ImageFrame::kDefaultAlignmentBoundary); #endif // !MEDIAPIPE_DISABLE_GPU + if (HasImageTag(cc)) { + auto out = std::make_unique(std::move(output_frame)); + cc->Outputs().Tag(kImageTag).Add(out.release(), cc->InputTimestamp()); + } if (cc->Outputs().HasTag(kImageFrameTag)) { cc->Outputs() .Tag(kImageFrameTag) @@ -487,6 +537,54 @@ absl::Status AnnotationOverlayCalculator::CreateRenderTargetCpu( return absl::OkStatus(); } +absl::Status AnnotationOverlayCalculator::CreateRenderTargetCpuImage( + CalculatorContext* cc, std::unique_ptr& image_mat, + ImageFormat::Format* target_format) { + if (image_frame_available_) { + const auto& input_frame = + cc->Inputs().Tag(kImageTag).Get(); + + int target_mat_type; + switch (input_frame.image_format()) { + case ImageFormat::SRGBA: + *target_format = ImageFormat::SRGBA; + target_mat_type = CV_8UC4; + break; + case ImageFormat::SRGB: + *target_format = ImageFormat::SRGB; + target_mat_type = CV_8UC3; + break; + case ImageFormat::GRAY8: + *target_format = ImageFormat::SRGB; + target_mat_type = CV_8UC3; + break; + default: + return absl::UnknownError("Unexpected image frame format."); + break; + } + + image_mat = absl::make_unique( + input_frame.height(), input_frame.width(), target_mat_type); + + auto input_mat = formats::MatView(&input_frame); + if (input_frame.image_format() == ImageFormat::GRAY8) { + cv::Mat rgb_mat; + cv::cvtColor(*input_mat, rgb_mat, cv::COLOR_GRAY2RGB); + rgb_mat.copyTo(*image_mat); + } else { + input_mat->copyTo(*image_mat); + } + } else { + image_mat = absl::make_unique( + options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3, + cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(), + options_.canvas_color().b())); + *target_format = ImageFormat::SRGB; + } + + return absl::OkStatus(); +} + template absl::Status AnnotationOverlayCalculator::CreateRenderTargetGpu( CalculatorContext* cc, std::unique_ptr& image_mat) { diff --git a/mediapipe/calculators/util/association_calculator.h b/mediapipe/calculators/util/association_calculator.h index 037ea838..1cec63c8 100644 --- a/mediapipe/calculators/util/association_calculator.h +++ b/mediapipe/calculators/util/association_calculator.h @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "mediapipe/calculators/util/association_calculator.pb.h" #include "mediapipe/framework/calculator_context.h" @@ -72,7 +73,7 @@ class AssociationCalculator : public CalculatorBase { prev_input_stream_id_ = cc->Inputs().GetId("PREV", 0); } options_ = cc->Options<::mediapipe::AssociationCalculatorOptions>(); - CHECK_GE(options_.min_similarity_threshold(), 0); + ABSL_CHECK_GE(options_.min_similarity_threshold(), 0); return absl::OkStatus(); } diff --git a/mediapipe/calculators/util/detection_label_id_to_text_calculator.cc b/mediapipe/calculators/util/detection_label_id_to_text_calculator.cc index 0c1d6892..44b7a210 100644 --- a/mediapipe/calculators/util/detection_label_id_to_text_calculator.cc +++ b/mediapipe/calculators/util/detection_label_id_to_text_calculator.cc @@ -19,6 +19,7 @@ #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/proto_ns.h" #include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/util/label_map.pb.h" #include "mediapipe/util/resource_util.h" @@ -85,7 +86,8 @@ absl::Status DetectionLabelIdToTextCalculator::Open(CalculatorContext* cc) { ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(options.label_map_path())); std::string label_map_string; - MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string)); + MP_RETURN_IF_ERROR( + mediapipe::GetResourceContents(string_path, &label_map_string)); std::istringstream stream(label_map_string); std::string line; diff --git a/mediapipe/calculators/util/detections_to_render_data_calculator.cc b/mediapipe/calculators/util/detections_to_render_data_calculator.cc index 25d74ba6..73c2cb1d 100644 --- a/mediapipe/calculators/util/detections_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/detections_to_render_data_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -233,13 +234,13 @@ void DetectionsToRenderDataCalculator::AddLabels( const Detection& detection, const DetectionsToRenderDataCalculatorOptions& options, float text_line_height, RenderData* render_data) { - CHECK(detection.label().empty() || detection.label_id().empty() || - detection.label_size() == detection.label_id_size()) + ABSL_CHECK(detection.label().empty() || detection.label_id().empty() || + detection.label_size() == detection.label_id_size()) << "String or integer labels should be of same size. Or only one of them " "is present."; const auto num_labels = std::max(detection.label_size(), detection.label_id_size()); - CHECK_EQ(detection.score_size(), num_labels) + ABSL_CHECK_EQ(detection.score_size(), num_labels) << "Number of scores and labels should match for detection."; // Extracts all "label(_id),score" for the detection. @@ -361,9 +362,9 @@ void DetectionsToRenderDataCalculator::AddDetectionToRenderData( const Detection& detection, const DetectionsToRenderDataCalculatorOptions& options, RenderData* render_data) { - CHECK(detection.location_data().format() == LocationData::BOUNDING_BOX || - detection.location_data().format() == - LocationData::RELATIVE_BOUNDING_BOX) + ABSL_CHECK(detection.location_data().format() == LocationData::BOUNDING_BOX || + detection.location_data().format() == + LocationData::RELATIVE_BOUNDING_BOX) << "Only Detection with formats of BOUNDING_BOX or RELATIVE_BOUNDING_BOX " "are supported."; double text_line_height; diff --git a/mediapipe/calculators/util/labels_to_render_data_calculator.cc b/mediapipe/calculators/util/labels_to_render_data_calculator.cc index dcd76d47..314640ed 100644 --- a/mediapipe/calculators/util/labels_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/labels_to_render_data_calculator.cc @@ -19,6 +19,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "mediapipe/calculators/util/labels_to_render_data_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -114,7 +115,8 @@ absl::Status LabelsToRenderDataCalculator::Process(CalculatorContext* cc) { video_height_ = video_header.height; return absl::OkStatus(); } else { - CHECK_EQ(options_.location(), LabelsToRenderDataCalculatorOptions::TOP_LEFT) + ABSL_CHECK_EQ(options_.location(), + LabelsToRenderDataCalculatorOptions::TOP_LEFT) << "Only TOP_LEFT is supported without VIDEO_PRESTREAM."; } @@ -144,7 +146,7 @@ absl::Status LabelsToRenderDataCalculator::Process(CalculatorContext* cc) { if (cc->Inputs().HasTag(kScoresTag)) { std::vector score_vector = cc->Inputs().Tag(kScoresTag).Get>(); - CHECK_EQ(label_vector.size(), score_vector.size()); + ABSL_CHECK_EQ(label_vector.size(), score_vector.size()); scores.resize(label_vector.size()); for (int i = 0; i < label_vector.size(); ++i) { scores[i] = score_vector[i]; diff --git a/mediapipe/calculators/util/landmarks_refinement_calculator.cc b/mediapipe/calculators/util/landmarks_refinement_calculator.cc index 8f734ac8..87394c6c 100644 --- a/mediapipe/calculators/util/landmarks_refinement_calculator.cc +++ b/mediapipe/calculators/util/landmarks_refinement_calculator.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "mediapipe/calculators/util/landmarks_refinement_calculator.pb.h" #include "mediapipe/framework/api2/node.h" @@ -102,7 +103,8 @@ void RefineZ( ->set_z(z_average); } } else { - CHECK(false) << "Z refinement is either not specified or not supported"; + ABSL_CHECK(false) + << "Z refinement is either not specified or not supported"; } } diff --git a/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc b/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc index 263ef85c..b0d4f417 100644 --- a/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc @@ -322,27 +322,30 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) { options_.presence_threshold(), options_.connection_color(), thickness, /*normalized=*/false, render_data.get()); } - for (int i = 0; i < landmarks.landmark_size(); ++i) { - const Landmark& landmark = landmarks.landmark(i); + if (options_.render_landmarks()) { + for (int i = 0; i < landmarks.landmark_size(); ++i) { + const Landmark& landmark = landmarks.landmark(i); - if (!IsLandmarkVisibleAndPresent( - landmark, options_.utilize_visibility(), - options_.visibility_threshold(), options_.utilize_presence(), - options_.presence_threshold())) { - continue; - } + if (!IsLandmarkVisibleAndPresent( + landmark, options_.utilize_visibility(), + options_.visibility_threshold(), options_.utilize_presence(), + options_.presence_threshold())) { + continue; + } - auto* landmark_data_render = AddPointRenderData( - options_.landmark_color(), thickness, render_data.get()); - if (visualize_depth) { - SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render, - options_.min_depth_circle_thickness(), - options_.max_depth_circle_thickness()); + auto* landmark_data_render = AddPointRenderData( + options_.landmark_color(), thickness, render_data.get()); + if (visualize_depth) { + SetColorSizeValueFromZ(landmark.z(), z_min, z_max, + landmark_data_render, + options_.min_depth_circle_thickness(), + options_.max_depth_circle_thickness()); + } + auto* landmark_data = landmark_data_render->mutable_point(); + landmark_data->set_normalized(false); + landmark_data->set_x(landmark.x()); + landmark_data->set_y(landmark.y()); } - auto* landmark_data = landmark_data_render->mutable_point(); - landmark_data->set_normalized(false); - landmark_data->set_x(landmark.x()); - landmark_data->set_y(landmark.y()); } } @@ -368,27 +371,30 @@ absl::Status LandmarksToRenderDataCalculator::Process(CalculatorContext* cc) { options_.presence_threshold(), options_.connection_color(), thickness, /*normalized=*/true, render_data.get()); } - for (int i = 0; i < landmarks.landmark_size(); ++i) { - const NormalizedLandmark& landmark = landmarks.landmark(i); + if (options_.render_landmarks()) { + for (int i = 0; i < landmarks.landmark_size(); ++i) { + const NormalizedLandmark& landmark = landmarks.landmark(i); - if (!IsLandmarkVisibleAndPresent( - landmark, options_.utilize_visibility(), - options_.visibility_threshold(), options_.utilize_presence(), - options_.presence_threshold())) { - continue; - } + if (!IsLandmarkVisibleAndPresent( + landmark, options_.utilize_visibility(), + options_.visibility_threshold(), options_.utilize_presence(), + options_.presence_threshold())) { + continue; + } - auto* landmark_data_render = AddPointRenderData( - options_.landmark_color(), thickness, render_data.get()); - if (visualize_depth) { - SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render, - options_.min_depth_circle_thickness(), - options_.max_depth_circle_thickness()); + auto* landmark_data_render = AddPointRenderData( + options_.landmark_color(), thickness, render_data.get()); + if (visualize_depth) { + SetColorSizeValueFromZ(landmark.z(), z_min, z_max, + landmark_data_render, + options_.min_depth_circle_thickness(), + options_.max_depth_circle_thickness()); + } + auto* landmark_data = landmark_data_render->mutable_point(); + landmark_data->set_normalized(true); + landmark_data->set_x(landmark.x()); + landmark_data->set_y(landmark.y()); } - auto* landmark_data = landmark_data_render->mutable_point(); - landmark_data->set_normalized(true); - landmark_data->set_x(landmark.x()); - landmark_data->set_y(landmark.y()); } } diff --git a/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto b/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto index 99091954..67dca84a 100644 --- a/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto +++ b/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto @@ -32,6 +32,10 @@ message LandmarksToRenderDataCalculatorOptions { // Color of the landmarks. optional Color landmark_color = 2; + + // Whether to render landmarks as points. + optional bool render_landmarks = 14 [default = true]; + // Color of the connections. optional Color connection_color = 3; diff --git a/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc b/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc index a9bc51f6..d83ff67c 100644 --- a/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc +++ b/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc @@ -15,6 +15,7 @@ #include #include +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/status.h" @@ -58,7 +59,7 @@ class LocalFilePatternContentsCalculator : public CalculatorBase { absl::Status Process(CalculatorContext* cc) override { if (current_output_ < filenames_.size()) { auto contents = absl::make_unique(); - LOG(INFO) << filenames_[current_output_]; + ABSL_LOG(INFO) << filenames_[current_output_]; MP_RETURN_IF_ERROR(mediapipe::file::GetContents( filenames_[current_output_], contents.get())); ++current_output_; diff --git a/mediapipe/calculators/util/non_max_suppression_calculator.cc b/mediapipe/calculators/util/non_max_suppression_calculator.cc index 535e2a71..be3a8da7 100644 --- a/mediapipe/calculators/util/non_max_suppression_calculator.cc +++ b/mediapipe/calculators/util/non_max_suppression_calculator.cc @@ -18,12 +18,13 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/calculators/util/non_max_suppression_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/rectangle.h" #include "mediapipe/framework/port/status.h" @@ -47,8 +48,8 @@ bool RetainMaxScoringLabelOnly(Detection* detection) { if (detection->label_id_size() == 0 && detection->label_size() == 0) { return false; } - CHECK(detection->label_id_size() == detection->score_size() || - detection->label_size() == detection->score_size()) + ABSL_CHECK(detection->label_id_size() == detection->score_size() || + detection->label_size() == detection->score_size()) << "Number of scores must be equal to number of detections."; std::vector> indexed_scores; @@ -92,7 +93,7 @@ float OverlapSimilarity( normalization = rect1.Area() + rect2.Area() - intersection_area; break; default: - LOG(FATAL) << "Unrecognized overlap type: " << overlap_type; + ABSL_LOG(FATAL) << "Unrecognized overlap type: " << overlap_type; } return normalization > 0.0f ? intersection_area / normalization : 0.0f; } @@ -171,9 +172,9 @@ class NonMaxSuppressionCalculator : public CalculatorBase { cc->SetOffset(TimestampDiff(0)); options_ = cc->Options(); - CHECK_GT(options_.num_detection_streams(), 0) + ABSL_CHECK_GT(options_.num_detection_streams(), 0) << "At least one detection stream need to be specified."; - CHECK_NE(options_.max_num_detections(), 0) + ABSL_CHECK_NE(options_.max_num_detections(), 0) << "max_num_detections=0 is not a valid value. Please choose a " << "positive number of you want to limit the number of output " << "detections, or set -1 if you do not want any limit."; diff --git a/mediapipe/calculators/util/packet_latency_calculator.cc b/mediapipe/calculators/util/packet_latency_calculator.cc index 6509f016..39c98bdd 100644 --- a/mediapipe/calculators/util/packet_latency_calculator.cc +++ b/mediapipe/calculators/util/packet_latency_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/time/time.h" #include "mediapipe/calculators/util/latency.pb.h" @@ -20,7 +21,6 @@ #include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/deps/clock.h" #include "mediapipe/framework/deps/monotonic_clock.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/timestamp.h" @@ -237,7 +237,7 @@ absl::Status PacketLatencyCalculator::Process(CalculatorContext* cc) { } if (first_process_time_usec_ < 0) { - LOG(WARNING) << "No reference packet received."; + ABSL_LOG(WARNING) << "No reference packet received."; return absl::OkStatus(); } diff --git a/mediapipe/calculators/util/rect_to_render_data_calculator.cc b/mediapipe/calculators/util/rect_to_render_data_calculator.cc index bbc08255..002471ca 100644 --- a/mediapipe/calculators/util/rect_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/rect_to_render_data_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_check.h" #include "mediapipe/calculators/util/rect_to_render_data_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/rect.pb.h" @@ -41,8 +42,8 @@ RenderAnnotation::Rectangle* NewRect( annotation->set_thickness(options.thickness()); if (options.has_top_left_thickness()) { - CHECK(!options.oval()); - CHECK(!options.filled()); + ABSL_CHECK(!options.oval()); + ABSL_CHECK(!options.filled()); annotation->mutable_rectangle()->set_top_left_thickness( options.top_left_thickness()); } diff --git a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc index 59b21d57..30dc11db 100644 --- a/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc +++ b/mediapipe/calculators/util/refine_landmarks_from_heatmap_calculator.cc @@ -124,7 +124,7 @@ absl::StatusOr RefineLandmarksFromHeatMap( int center_row = out_lms.landmark(lm_index).y() * hm_height; // Point is outside of the image let's keep it intact. if (center_col < 0 || center_col >= hm_width || center_row < 0 || - center_col >= hm_height) { + center_row >= hm_height) { continue; } diff --git a/mediapipe/calculators/video/BUILD b/mediapipe/calculators/video/BUILD index 7245b13c..f17747d2 100644 --- a/mediapipe/calculators/video/BUILD +++ b/mediapipe/calculators/video/BUILD @@ -130,9 +130,9 @@ cc_library( "//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_video", - "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) @@ -154,6 +154,7 @@ cc_library( "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -169,6 +170,7 @@ cc_library( "//mediapipe/framework/formats/motion:optical_flow_field", "//mediapipe/framework/port:opencv_video", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/synchronization", ], alwayslink = 1, @@ -194,6 +196,8 @@ cc_library( "//mediapipe/util/tracking:motion_estimation", "//mediapipe/util/tracking:motion_models", "//mediapipe/util/tracking:region_flow_cc_proto", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -206,10 +210,11 @@ cc_library( ":flow_packager_calculator_cc_proto", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/util/tracking:camera_motion_cc_proto", "//mediapipe/util/tracking:flow_packager", "//mediapipe/util/tracking:region_flow_cc_proto", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ], @@ -226,7 +231,6 @@ cc_library( "//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/formats:video_stream_header", # fixdeps: keep -- required for exobazel build. "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", @@ -237,6 +241,8 @@ cc_library( "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/container:node_hash_map", "@com_google_absl//absl/container:node_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -252,7 +258,6 @@ cc_library( "//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/formats:video_stream_header", # fixdeps: keep -- required for exobazel build. "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_features2d", "//mediapipe/framework/port:ret_check", @@ -264,6 +269,8 @@ cc_library( "//mediapipe/util/tracking:box_tracker_cc_proto", "//mediapipe/util/tracking:flow_packager_cc_proto", "//mediapipe/util/tracking:tracking_visualization_utilities", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ] + select({ @@ -341,7 +348,6 @@ cc_test( "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/tool:test_util", - "@com_google_absl//absl/flags:flag", ], ) @@ -361,13 +367,12 @@ cc_test( "//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_video", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/tool:test_util", - "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_log", ], ) @@ -451,7 +456,8 @@ cc_test( "//mediapipe/framework/tool:test_util", "//mediapipe/util/tracking:box_tracker_cc_proto", "//mediapipe/util/tracking:tracking_cc_proto", - "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/calculators/video/box_detector_calculator.cc b/mediapipe/calculators/video/box_detector_calculator.cc index 14ac12e5..51f57b7e 100644 --- a/mediapipe/calculators/video/box_detector_calculator.cc +++ b/mediapipe/calculators/video/box_detector_calculator.cc @@ -17,6 +17,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/numbers.h" #include "mediapipe/calculators/video/box_detector_calculator.pb.h" @@ -25,7 +27,6 @@ #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/video_stream_header.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_features2d_inc.h" #include "mediapipe/framework/port/ret_check.h" @@ -198,7 +199,8 @@ absl::Status BoxDetectorCalculator::Open(CalculatorContext* cc) { if (!predefined_index.ParseFromString(cc->InputSidePackets() .Tag(kIndexProtoStringTag) .Get())) { - LOG(FATAL) << "failed to parse BoxDetectorIndex from INDEX_PROTO_STRING"; + ABSL_LOG(FATAL) + << "failed to parse BoxDetectorIndex from INDEX_PROTO_STRING"; } box_detector_->AddBoxDetectorIndex(predefined_index); } @@ -210,7 +212,7 @@ absl::Status BoxDetectorCalculator::Open(CalculatorContext* cc) { MP_RETURN_IF_ERROR(file::GetContents(string_path, &index_string)); BoxDetectorIndex predefined_index; if (!predefined_index.ParseFromString(index_string)) { - LOG(FATAL) + ABSL_LOG(FATAL) << "failed to parse BoxDetectorIndex from index_proto_filename"; } box_detector_->AddBoxDetectorIndex(predefined_index); @@ -248,7 +250,7 @@ absl::Status BoxDetectorCalculator::Process(CalculatorContext* cc) { BoxDetectorIndex predefined_index; if (!predefined_index.ParseFromString( add_index_stream->Get())) { - LOG(FATAL) << "failed to parse BoxDetectorIndex from ADD_INDEX"; + ABSL_LOG(FATAL) << "failed to parse BoxDetectorIndex from ADD_INDEX"; } box_detector_->AddBoxDetectorIndex(predefined_index); } @@ -276,8 +278,8 @@ absl::Status BoxDetectorCalculator::Process(CalculatorContext* cc) { ? &(cc->Inputs().Tag(kDescriptorsTag)) : nullptr; - CHECK(track_stream != nullptr || video_stream != nullptr || - (feature_stream != nullptr && descriptor_stream != nullptr)) + ABSL_CHECK(track_stream != nullptr || video_stream != nullptr || + (feature_stream != nullptr && descriptor_stream != nullptr)) << "One and only one of {tracking_data, input image frame, " "feature/descriptor} need to be valid."; @@ -295,7 +297,7 @@ absl::Status BoxDetectorCalculator::Process(CalculatorContext* cc) { const TrackingData& tracking_data = track_stream->Get(); - CHECK(tracked_boxes_stream != nullptr) << "tracked_boxes needed."; + ABSL_CHECK(tracked_boxes_stream != nullptr) << "tracked_boxes needed."; const TimedBoxProtoList tracked_boxes = tracked_boxes_stream->Get(); @@ -359,7 +361,7 @@ absl::Status BoxDetectorCalculator::Process(CalculatorContext* cc) { const auto& descriptors = descriptor_stream->Get>(); const int dims = options_.detector_options().descriptor_dims(); - CHECK_GE(descriptors.size(), feature_size * dims); + ABSL_CHECK_GE(descriptors.size(), feature_size * dims); cv::Mat descriptors_mat(feature_size, dims, CV_32F); for (int j = 0; j < feature_size; ++j) { features_vec[j].Set(features[j].pt.x * inv_scale, diff --git a/mediapipe/calculators/video/box_tracker_calculator.cc b/mediapipe/calculators/video/box_tracker_calculator.cc index b5f3b5b0..4a8f4543 100644 --- a/mediapipe/calculators/video/box_tracker_calculator.cc +++ b/mediapipe/calculators/video/box_tracker_calculator.cc @@ -22,6 +22,8 @@ #include "absl/container/flat_hash_set.h" #include "absl/container/node_hash_map.h" #include "absl/container/node_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/numbers.h" #include "mediapipe/calculators/video/box_tracker_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -29,7 +31,6 @@ #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/video_stream_header.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -315,16 +316,16 @@ void ConvertCoordinateForRotation(float in_top, float in_left, float in_bottom, float in_right, int rotation, float* out_top, float* out_left, float* out_bottom, float* out_right) { - CHECK(out_top != nullptr); - CHECK(out_left != nullptr); - CHECK(out_bottom != nullptr); - CHECK(out_right != nullptr); + ABSL_CHECK(out_top != nullptr); + ABSL_CHECK(out_left != nullptr); + ABSL_CHECK(out_bottom != nullptr); + ABSL_CHECK(out_right != nullptr); const float in_center_x = (in_left + in_right) * 0.5f; const float in_center_y = (in_top + in_bottom) * 0.5f; const float in_width = in_right - in_left; const float in_height = in_bottom - in_top; - CHECK_GT(in_width, 0); - CHECK_GT(in_height, 0); + ABSL_CHECK_GT(in_width, 0); + ABSL_CHECK_GT(in_height, 0); float out_center_x; float out_center_y; float out_width; @@ -358,7 +359,7 @@ void ConvertCoordinateForRotation(float in_top, float in_left, float in_bottom, out_height = in_width; break; default: - LOG(ERROR) << "invalid rotation " << rotation; + ABSL_LOG(ERROR) << "invalid rotation " << rotation; out_center_x = in_center_x; out_center_y = in_center_y; out_width = in_width; @@ -373,7 +374,7 @@ void ConvertCoordinateForRotation(float in_top, float in_left, float in_bottom, void AddStateToPath(const MotionBoxState& state, int64_t time_msec, PathSegment* path) { - CHECK(path); + ABSL_CHECK(path); TimedBox result; TimedBoxFromMotionBoxState(state, &result); result.time_msec = time_msec; @@ -384,7 +385,8 @@ void AddStateToPath(const MotionBoxState& state, int64_t time_msec, path->insert(insert_pos, InternalTimedBox(result, new MotionBoxState(state))); } else { - LOG(ERROR) << "Box at time " << time_msec << " already present; ignoring"; + ABSL_LOG(ERROR) << "Box at time " << time_msec + << " already present; ignoring"; } } @@ -486,8 +488,9 @@ absl::Status BoxTrackerCalculator::Open(CalculatorContext* cc) { #if !defined(__ANDROID__) && !defined(__APPLE__) && !defined(__EMSCRIPTEN__) if (cc->InputSidePackets().HasTag(kInitialPosTag)) { - LOG(INFO) << "Parsing: " - << cc->InputSidePackets().Tag(kInitialPosTag).Get(); + ABSL_LOG(INFO) + << "Parsing: " + << cc->InputSidePackets().Tag(kInitialPosTag).Get(); initial_pos_ = ParseTextProtoOrDie( cc->InputSidePackets().Tag(kInitialPosTag).Get()); } @@ -624,7 +627,7 @@ absl::Status BoxTrackerCalculator::Process(CalculatorContext* cc) { if (cancel_object_id_stream && !cancel_object_id_stream->IsEmpty()) { const int cancel_object_id = cancel_object_id_stream->Get(); if (streaming_motion_boxes_.erase(cancel_object_id) == 0) { - LOG(WARNING) << "box id " << cancel_object_id << " does not exist."; + ABSL_LOG(WARNING) << "box id " << cancel_object_id << " does not exist."; } } @@ -649,7 +652,7 @@ absl::Status BoxTrackerCalculator::Process(CalculatorContext* cc) { // present at this frame. TimedBoxProtoList box_track_list; - CHECK(box_tracker_ || track_stream) + ABSL_CHECK(box_tracker_ || track_stream) << "Expected either batch or streaming mode"; // Corresponding list of box states for rendering. For each id present at @@ -944,7 +947,7 @@ void BoxTrackerCalculator::OutputRandomAccessTrack( const bool forward_track = start.time_msec() < end_time_msec; if (track_timestamps_.empty()) { - LOG(WARNING) << "No tracking data cached yet."; + ABSL_LOG(WARNING) << "No tracking data cached yet."; continue; } @@ -954,27 +957,27 @@ void BoxTrackerCalculator::OutputRandomAccessTrack( const int64_t tracking_end_timestamp_msec = track_timestamps_.back().Microseconds() / 1000; if (start.time_msec() < tracking_start_timestamp_msec) { - LOG(WARNING) << "Request start timestamp " << start.time_msec() - << " too old. First frame in the window: " - << tracking_start_timestamp_msec; + ABSL_LOG(WARNING) << "Request start timestamp " << start.time_msec() + << " too old. First frame in the window: " + << tracking_start_timestamp_msec; continue; } if (start.time_msec() > tracking_end_timestamp_msec) { - LOG(WARNING) << "Request start timestamp " << start.time_msec() - << " too new. Last frame in the window: " - << tracking_end_timestamp_msec; + ABSL_LOG(WARNING) << "Request start timestamp " << start.time_msec() + << " too new. Last frame in the window: " + << tracking_end_timestamp_msec; continue; } if (end_time_msec < tracking_start_timestamp_msec) { - LOG(WARNING) << "Request end timestamp " << end_time_msec - << " too old. First frame in the window: " - << tracking_start_timestamp_msec; + ABSL_LOG(WARNING) << "Request end timestamp " << end_time_msec + << " too old. First frame in the window: " + << tracking_start_timestamp_msec; continue; } if (end_time_msec > tracking_end_timestamp_msec) { - LOG(WARNING) << "Request end timestamp " << end_time_msec - << " too new. Last frame in the window: " - << tracking_end_timestamp_msec; + ABSL_LOG(WARNING) << "Request end timestamp " << end_time_msec + << " too new. Last frame in the window: " + << tracking_end_timestamp_msec; continue; } @@ -982,7 +985,7 @@ void BoxTrackerCalculator::OutputRandomAccessTrack( GetRandomAccessTimestampPos(start, forward_track); if (timestamp_pos == track_timestamps_.end()) { - LOG(ERROR) << "Random access outside cached range"; + ABSL_LOG(ERROR) << "Random access outside cached range"; continue; } @@ -993,13 +996,13 @@ void BoxTrackerCalculator::OutputRandomAccessTrack( // TODO: Interpolate random access tracking start_data instead // of dropping the request in the case of missing processed frame. if (start_data == tracking_data_cache_.end()) { - LOG(ERROR) << "Random access starts at unprocessed frame."; + ABSL_LOG(ERROR) << "Random access starts at unprocessed frame."; continue; } const int init_frame = timestamp_pos - track_timestamps_.begin() + track_timestamps_base_index_; - CHECK_GE(init_frame, 0); + ABSL_CHECK_GE(init_frame, 0); MotionBoxMap single_map = PrepareRandomAccessTrack(start, init_frame, forward_track, start_data); @@ -1010,7 +1013,7 @@ void BoxTrackerCalculator::OutputRandomAccessTrack( &single_map, end_time_msec); if (track_error) { - LOG(ERROR) << "Could not track box."; + ABSL_LOG(ERROR) << "Could not track box."; continue; } @@ -1166,8 +1169,8 @@ void BoxTrackerCalculator::StreamTrack(const TrackingData& data, int64_t duration_ms, bool forward, MotionBoxMap* box_map, std::vector* failed_ids) { - CHECK(box_map); - CHECK(failed_ids); + ABSL_CHECK(box_map); + ABSL_CHECK(failed_ids); // Cache the actively discarded tracked ids from the new tracking data. for (const int discarded_id : @@ -1197,7 +1200,7 @@ void BoxTrackerCalculator::StreamTrack(const TrackingData& data, if (!motion_box.second.box.TrackStep(from_frame, // from frame. mvf, forward)) { failed_ids->push_back(motion_box.first); - LOG(INFO) << "lost track. pushed failed id: " << motion_box.first; + ABSL_LOG(INFO) << "lost track. pushed failed id: " << motion_box.first; } else { // Store result. PathSegment& path = motion_box.second.path; @@ -1224,8 +1227,8 @@ void BoxTrackerCalculator::FastForwardStartPos( track_timestamps_.end(), timestamp); if (timestamp_pos == track_timestamps_.end()) { - LOG(WARNING) << "Received start pos beyond current timestamp, " - << "Starting to track once frame arrives."; + ABSL_LOG(WARNING) << "Received start pos beyond current timestamp, " + << "Starting to track once frame arrives."; *initial_pos_.add_box() = start_pos; continue; } @@ -1233,7 +1236,7 @@ void BoxTrackerCalculator::FastForwardStartPos( // Start at previous frame. const int init_frame = timestamp_pos - track_timestamps_.begin() + track_timestamps_base_index_; - CHECK_GE(init_frame, 0); + ABSL_CHECK_GE(init_frame, 0); // Locate corresponding tracking data. auto start_data = std::find_if( @@ -1242,8 +1245,9 @@ void BoxTrackerCalculator::FastForwardStartPos( -> bool { return item.first == timestamp_pos[0]; }); if (start_data == tracking_data_cache_.end()) { - LOG(ERROR) << "Box to fast forward outside tracking data cache. Ignoring." - << " To avoid this error consider increasing the cache size."; + ABSL_LOG(ERROR) + << "Box to fast forward outside tracking data cache. Ignoring." + << " To avoid this error consider increasing the cache size."; continue; } @@ -1281,7 +1285,8 @@ void BoxTrackerCalculator::FastForwardStartPos( true, // forward &single_map, &failed_box); if (!failed_box.empty()) { - LOG(WARNING) << "Unable to fast forward box at frame " << curr_frame; + ABSL_LOG(WARNING) << "Unable to fast forward box at frame " + << curr_frame; track_error = true; break; } diff --git a/mediapipe/calculators/video/flow_packager_calculator.cc b/mediapipe/calculators/video/flow_packager_calculator.cc index 2965cd8e..b0453499 100644 --- a/mediapipe/calculators/video/flow_packager_calculator.cc +++ b/mediapipe/calculators/video/flow_packager_calculator.cc @@ -17,12 +17,13 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "mediapipe/calculators/video/flow_packager_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/util/tracking/camera_motion.pb.h" #include "mediapipe/util/tracking/flow_packager.h" #include "mediapipe/util/tracking/region_flow.pb.h" @@ -160,7 +161,7 @@ absl::Status FlowPackagerCalculator::Process(CalculatorContext* cc) { timestamp.Value() / 1000 / options_.caching_chunk_size_msec(); tracking_chunk_.set_first_chunk(true); } - CHECK_GE(chunk_idx_, 0); + ABSL_CHECK_GE(chunk_idx_, 0); TrackingDataChunk::Item* item = tracking_chunk_.add_item(); item->set_frame_idx(frame_idx_); @@ -227,10 +228,11 @@ absl::Status FlowPackagerCalculator::Close(CalculatorContext* cc) { void FlowPackagerCalculator::WriteChunk(const TrackingDataChunk& chunk) const { if (chunk.item_size() == 0) { - LOG(ERROR) << "Write chunk called with empty tracking data." - << "This can only occur if the spacing between frames " - << "is larger than the requested chunk size. Try increasing " - << "the chunk size"; + ABSL_LOG(ERROR) + << "Write chunk called with empty tracking data." + << "This can only occur if the spacing between frames " + << "is larger than the requested chunk size. Try increasing " + << "the chunk size"; return; } @@ -242,7 +244,7 @@ void FlowPackagerCalculator::WriteChunk(const TrackingDataChunk& chunk) const { chunk_file = cache_dir_ + "/" + absl::StrFormat(*format_runtime, chunk_idx_); } else { - LOG(ERROR) << "chache_file_format wrong. fall back to chunk_%04d."; + ABSL_LOG(ERROR) << "chache_file_format wrong. fall back to chunk_%04d."; chunk_file = cache_dir_ + "/" + absl::StrFormat("chunk_%04d", chunk_idx_); } @@ -252,23 +254,23 @@ void FlowPackagerCalculator::WriteChunk(const TrackingDataChunk& chunk) const { const char* temp_filename = tempnam(cache_dir_.c_str(), nullptr); std::ofstream out_file(temp_filename); if (!out_file) { - LOG(ERROR) << "Could not open " << temp_filename; + ABSL_LOG(ERROR) << "Could not open " << temp_filename; } else { out_file.write(data.data(), data.size()); } if (rename(temp_filename, chunk_file.c_str()) != 0) { - LOG(ERROR) << "Failed to rename to " << chunk_file; + ABSL_LOG(ERROR) << "Failed to rename to " << chunk_file; } - LOG(INFO) << "Wrote chunk : " << chunk_file; + ABSL_LOG(INFO) << "Wrote chunk : " << chunk_file; } void FlowPackagerCalculator::PrepareCurrentForNextChunk( TrackingDataChunk* chunk) { - CHECK(chunk); + ABSL_CHECK(chunk); if (chunk->item_size() == 0) { - LOG(ERROR) << "Called with empty chunk. Unexpected."; + ABSL_LOG(ERROR) << "Called with empty chunk. Unexpected."; return; } diff --git a/mediapipe/calculators/video/motion_analysis_calculator.cc b/mediapipe/calculators/video/motion_analysis_calculator.cc index 544439ae..601b8b04 100644 --- a/mediapipe/calculators/video/motion_analysis_calculator.cc +++ b/mediapipe/calculators/video/motion_analysis_calculator.cc @@ -17,6 +17,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/numbers.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" @@ -348,8 +350,8 @@ absl::Status MotionAnalysisCalculator::Open(CalculatorContext* cc) { video_header = &(cc->Inputs().Tag(kSelectionTag).Header().Get()); } else { - LOG(WARNING) << "No input video header found. Downstream calculators " - "expecting video headers are likely to fail."; + ABSL_LOG(WARNING) << "No input video header found. Downstream calculators " + "expecting video headers are likely to fail."; } with_saliency_ = options_.analysis_options().compute_motion_saliency(); @@ -357,9 +359,9 @@ absl::Status MotionAnalysisCalculator::Open(CalculatorContext* cc) { if (cc->Outputs().HasTag(kSaliencyTag)) { with_saliency_ = true; if (!options_.analysis_options().compute_motion_saliency()) { - LOG(WARNING) << "Enable saliency computation. Set " - << "compute_motion_saliency to true to silence this " - << "warning."; + ABSL_LOG(WARNING) << "Enable saliency computation. Set " + << "compute_motion_saliency to true to silence this " + << "warning."; options_.mutable_analysis_options()->set_compute_motion_saliency(true); } } @@ -428,7 +430,7 @@ absl::Status MotionAnalysisCalculator::Process(CalculatorContext* cc) { selection_input_ ? &(cc->Inputs().Tag(kSelectionTag)) : nullptr; // Checked on Open. - CHECK(video_stream || selection_stream); + ABSL_CHECK(video_stream || selection_stream); // Lazy init. if (frame_width_ < 0 || frame_height_ < 0) { @@ -472,7 +474,7 @@ absl::Status MotionAnalysisCalculator::Process(CalculatorContext* cc) { // Always use frame if selection is not activated. bool use_frame = !selection_input_; if (selection_input_) { - CHECK(selection_stream); + ABSL_CHECK(selection_stream); // Fill in timestamps we process. if (!selection_stream->Value().IsEmpty()) { @@ -603,8 +605,8 @@ absl::Status MotionAnalysisCalculator::Close(CalculatorContext* cc) { } if (csv_file_input_) { if (!meta_motions_.empty()) { - LOG(ERROR) << "More motions than frames. Unexpected! Remainder: " - << meta_motions_.size(); + ABSL_LOG(ERROR) << "More motions than frames. Unexpected! Remainder: " + << meta_motions_.size(); } } return absl::OkStatus(); @@ -620,7 +622,7 @@ void MotionAnalysisCalculator::OutputMotionAnalyzedFrames( const int num_results = motion_analysis_->GetResults( flush, &features, &camera_motions, with_saliency_ ? &saliency : nullptr); - CHECK_LE(num_results, buffer_size); + ABSL_CHECK_LE(num_results, buffer_size); if (num_results == 0) { return; @@ -695,7 +697,7 @@ void MotionAnalysisCalculator::OutputMotionAnalyzedFrames( if (hybrid_meta_analysis_) { hybrid_meta_offset_ -= num_results; - CHECK_GE(hybrid_meta_offset_, 0); + ABSL_CHECK_GE(hybrid_meta_offset_, 0); } timestamp_buffer_.erase(timestamp_buffer_.begin(), @@ -741,8 +743,8 @@ absl::Status MotionAnalysisCalculator::InitOnProcess( } if (region_options->image_format() != image_format && region_options->image_format() != image_format2) { - LOG(WARNING) << "Requested image format in RegionFlowComputation " - << "does not match video stream format. Overriding."; + ABSL_LOG(WARNING) << "Requested image format in RegionFlowComputation " + << "does not match video stream format. Overriding."; region_options->set_image_format(image_format); } @@ -761,12 +763,12 @@ absl::Status MotionAnalysisCalculator::InitOnProcess( frame_width_ = camera_motion.frame_width(); frame_height_ = camera_motion.frame_height(); } else { - LOG(FATAL) << "Either VIDEO or SELECTION stream need to be specified."; + ABSL_LOG(FATAL) << "Either VIDEO or SELECTION stream need to be specified."; } // Filled by CSV file parsing. if (!meta_homographies_.empty()) { - CHECK(csv_file_input_); + ABSL_CHECK(csv_file_input_); AppendCameraMotionsFromHomographies(meta_homographies_, true, // append identity. &meta_motions_, &meta_features_); @@ -800,7 +802,7 @@ bool MotionAnalysisCalculator::ParseModelCSV( for (const auto& value : values) { double value_64f; if (!absl::SimpleAtod(value, &value_64f)) { - LOG(ERROR) << "Not a double, expected!"; + ABSL_LOG(ERROR) << "Not a double, expected!"; return false; } @@ -813,12 +815,12 @@ bool MotionAnalysisCalculator::ParseModelCSV( bool MotionAnalysisCalculator::HomographiesFromValues( const std::vector& homog_values, std::deque* homographies) { - CHECK(homographies); + ABSL_CHECK(homographies); // Obvious constants are obvious :D constexpr int kHomographyValues = 9; if (homog_values.size() % kHomographyValues != 0) { - LOG(ERROR) << "Contents not a multiple of " << kHomographyValues; + ABSL_LOG(ERROR) << "Contents not a multiple of " << kHomographyValues; return false; } @@ -830,7 +832,7 @@ bool MotionAnalysisCalculator::HomographiesFromValues( // Normalize last entry to 1. if (h_vals[kHomographyValues - 1] == 0) { - LOG(ERROR) << "Degenerate homography, last entry is zero"; + ABSL_LOG(ERROR) << "Degenerate homography, last entry is zero"; return false; } @@ -844,8 +846,8 @@ bool MotionAnalysisCalculator::HomographiesFromValues( } if (homographies->size() % options_.meta_models_per_frame() != 0) { - LOG(ERROR) << "Total homographies not a multiple of specified models " - << "per frame."; + ABSL_LOG(ERROR) << "Total homographies not a multiple of specified models " + << "per frame."; return false; } @@ -855,7 +857,7 @@ bool MotionAnalysisCalculator::HomographiesFromValues( void MotionAnalysisCalculator::SubtractMetaMotion( const CameraMotion& meta_motion, RegionFlowFeatureList* features) { if (meta_motion.mixture_homography().model_size() > 0) { - CHECK(row_weights_ != nullptr); + ABSL_CHECK(row_weights_ != nullptr); RegionFlowFeatureListViaTransform(meta_motion.mixture_homography(), features, -1.0f, 1.0f, // subtract transformed. @@ -901,7 +903,7 @@ void MotionAnalysisCalculator::AddMetaMotion( const CameraMotion& meta_motion, const RegionFlowFeatureList& meta_features, RegionFlowFeatureList* features, CameraMotion* motion) { // Restore old feature location. - CHECK_EQ(meta_features.feature_size(), features->feature_size()); + ABSL_CHECK_EQ(meta_features.feature_size(), features->feature_size()); for (int k = 0; k < meta_features.feature_size(); ++k) { auto feature = features->mutable_feature(k); const auto& meta_feature = meta_features.feature(k); @@ -922,8 +924,8 @@ void MotionAnalysisCalculator::AppendCameraMotionsFromHomographies( const std::deque& homographies, bool append_identity, std::deque* camera_motions, std::deque* features) { - CHECK(camera_motions); - CHECK(features); + ABSL_CHECK(camera_motions); + ABSL_CHECK(features); CameraMotion identity; identity.set_frame_width(frame_width_); @@ -947,8 +949,9 @@ void MotionAnalysisCalculator::AppendCameraMotionsFromHomographies( } const int models_per_frame = options_.meta_models_per_frame(); - CHECK_GT(models_per_frame, 0) << "At least one model per frame is needed"; - CHECK_EQ(0, homographies.size() % models_per_frame); + ABSL_CHECK_GT(models_per_frame, 0) + << "At least one model per frame is needed"; + ABSL_CHECK_EQ(0, homographies.size() % models_per_frame); const int num_frames = homographies.size() / models_per_frame; // Heuristic sigma, similar to what we use for rolling shutter removal. diff --git a/mediapipe/calculators/video/opencv_video_decoder_calculator.cc b/mediapipe/calculators/video/opencv_video_decoder_calculator.cc index 9e04f33c..cda7085d 100644 --- a/mediapipe/calculators/video/opencv_video_decoder_calculator.cc +++ b/mediapipe/calculators/video/opencv_video_decoder_calculator.cc @@ -14,6 +14,7 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" @@ -168,9 +169,10 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase { .Tag(kSavedAudioPathTag) .Set(MakePacket(saved_audio_path)); } else { - LOG(WARNING) << "FFmpeg can't extract audio from " << input_file_path - << " by executing the following command: " - << ffmpeg_command; + ABSL_LOG(WARNING) << "FFmpeg can't extract audio from " + << input_file_path + << " by executing the following command: " + << ffmpeg_command; cc->OutputSidePackets() .Tag(kSavedAudioPathTag) .Set(MakePacket(std::string())); @@ -227,9 +229,9 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase { cap_->release(); } if (decoded_frames_ != frame_count_) { - LOG(WARNING) << "Not all the frames are decoded (total frames: " - << frame_count_ << " vs decoded frames: " << decoded_frames_ - << ")."; + ABSL_LOG(WARNING) << "Not all the frames are decoded (total frames: " + << frame_count_ + << " vs decoded frames: " << decoded_frames_ << ")."; } return absl::OkStatus(); } diff --git a/mediapipe/calculators/video/opencv_video_encoder_calculator.cc b/mediapipe/calculators/video/opencv_video_encoder_calculator.cc index 4af8c595..5979d57b 100644 --- a/mediapipe/calculators/video/opencv_video_encoder_calculator.cc +++ b/mediapipe/calculators/video/opencv_video_encoder_calculator.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/strings/str_split.h" #include "mediapipe/calculators/video/opencv_video_encoder_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -187,9 +188,10 @@ absl::Status OpenCvVideoEncoderCalculator::Close(CalculatorContext* cc) { const std::string& audio_file_path = cc->InputSidePackets().Tag(kAudioFilePathTag).Get(); if (audio_file_path.empty()) { - LOG(WARNING) << "OpenCvVideoEncoderCalculator isn't able to attach the " - "audio tracks to the generated video because the audio " - "file path is not specified."; + ABSL_LOG(WARNING) + << "OpenCvVideoEncoderCalculator isn't able to attach the " + "audio tracks to the generated video because the audio " + "file path is not specified."; } else { // A temp output file is needed because FFmpeg can't do in-place editing. const std::string temp_file_path = std::tmpnam(nullptr); diff --git a/mediapipe/calculators/video/tool/BUILD b/mediapipe/calculators/video/tool/BUILD index 408461d2..2a32c680 100644 --- a/mediapipe/calculators/video/tool/BUILD +++ b/mediapipe/calculators/video/tool/BUILD @@ -44,6 +44,7 @@ cc_library( "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/tool:status_util", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) diff --git a/mediapipe/calculators/video/tool/flow_quantizer_model.cc b/mediapipe/calculators/video/tool/flow_quantizer_model.cc index f0b00063..146dc4a7 100644 --- a/mediapipe/calculators/video/tool/flow_quantizer_model.cc +++ b/mediapipe/calculators/video/tool/flow_quantizer_model.cc @@ -14,6 +14,7 @@ #include "mediapipe/calculators/video/tool/flow_quantizer_model.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/type_map.h" @@ -21,7 +22,7 @@ namespace mediapipe { // Uniform normalization to 0-255. uint8_t FlowQuantizerModel::Apply(const float val, const int channel) const { - CHECK_LT(channel, model_.min_value_size()); + ABSL_CHECK_LT(channel, model_.min_value_size()); const auto& min_value = model_.min_value(channel); const auto& max_value = model_.max_value(channel); QCHECK_GT(max_value, min_value); @@ -51,7 +52,7 @@ const QuantizerModelData& FlowQuantizerModel::GetModelData() const { // TODO: Taking the min and max over all training flow fields might be // sensitive to noise. We should use more robust statistics. void FlowQuantizerModel::AddSampleFlowField(const OpticalFlowField& flow) { - CHECK_EQ(model_.min_value_size(), 2); + ABSL_CHECK_EQ(model_.min_value_size(), 2); const cv::Mat_& flow_mat = flow.flow_data(); for (int i = 0; i != flow.width(); ++i) { for (int j = 0; j != flow.height(); ++j) { diff --git a/mediapipe/calculators/video/tracking_graph_test.cc b/mediapipe/calculators/video/tracking_graph_test.cc index 8fd8806b..1ccc6121 100644 --- a/mediapipe/calculators/video/tracking_graph_test.cc +++ b/mediapipe/calculators/video/tracking_graph_test.cc @@ -19,6 +19,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/calculators/video/box_tracker_calculator.pb.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -52,7 +54,7 @@ bool LoadBinaryTestGraph(const std::string& graph_path, bool success = config->ParseFromZeroCopyStream(&in_stream); ifs.close(); if (!success) { - LOG(ERROR) << "could not parse test graph: " << graph_path; + ABSL_LOG(ERROR) << "could not parse test graph: " << graph_path; } return success; } @@ -297,7 +299,7 @@ std::unique_ptr TrackingGraphTest::CreateRandomAccessTrackingBoxList( const std::vector& start_timestamps, const std::vector& end_timestamps) const { - CHECK_EQ(start_timestamps.size(), end_timestamps.size()); + ABSL_CHECK_EQ(start_timestamps.size(), end_timestamps.size()); auto ra_boxes = absl::make_unique(); for (int i = 0; i < start_timestamps.size(); ++i) { auto start_box_list = @@ -620,7 +622,7 @@ TEST_F(TrackingGraphTest, TestTransitionFramesForReacquisition) { // Add TRACK_TIME stream queries in between 2 frames. if (j > 0) { Timestamp track_time = Timestamp((j - 0.5f) * kFrameIntervalUs); - LOG(INFO) << track_time.Value(); + ABSL_LOG(INFO) << track_time.Value(); Packet track_time_packet = Adopt(new Timestamp).At(track_time); MP_EXPECT_OK( graph_.AddPacketToInputStream("track_time", track_time_packet)); diff --git a/mediapipe/calculators/video/tvl1_optical_flow_calculator.cc b/mediapipe/calculators/video/tvl1_optical_flow_calculator.cc index 56f3253e..e60df028 100644 --- a/mediapipe/calculators/video/tvl1_optical_flow_calculator.cc +++ b/mediapipe/calculators/video/tvl1_optical_flow_calculator.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "absl/base/macros.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" @@ -158,7 +159,7 @@ absl::Status Tvl1OpticalFlowCalculator::Process(CalculatorContext* cc) { absl::Status Tvl1OpticalFlowCalculator::CalculateOpticalFlow( const ImageFrame& current_frame, const ImageFrame& next_frame, OpticalFlowField* flow) { - CHECK(flow); + ABSL_CHECK(flow); if (!ImageSizesMatch(current_frame, next_frame)) { return tool::StatusInvalid("Images are different sizes."); } @@ -182,7 +183,7 @@ absl::Status Tvl1OpticalFlowCalculator::CalculateOpticalFlow( flow->Allocate(first.cols, first.rows); cv::Mat cv_flow(flow->mutable_flow_data()); tvl1_computer->calc(first, second, cv_flow); - CHECK_EQ(flow->mutable_flow_data().data, cv_flow.data); + ABSL_CHECK_EQ(flow->mutable_flow_data().data, cv_flow.data); // Inserts the idle DenseOpticalFlow object back to the cache for reuse. { absl::MutexLock lock(&mutex_); diff --git a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar index 943f0cbf..afba1092 100644 Binary files a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar and b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar differ diff --git a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties index 50832291..4e86b927 100644 --- a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties +++ b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/mediapipe/examples/coral/BUILD b/mediapipe/examples/coral/BUILD index 68244d57..0c7c6b11 100644 --- a/mediapipe/examples/coral/BUILD +++ b/mediapipe/examples/coral/BUILD @@ -35,6 +35,7 @@ cc_library( "//mediapipe/framework/port:status", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/examples/coral/demo_run_graph_main.cc b/mediapipe/examples/coral/demo_run_graph_main.cc index 6f1c5626..692f2600 100644 --- a/mediapipe/examples/coral/demo_run_graph_main.cc +++ b/mediapipe/examples/coral/demo_run_graph_main.cc @@ -17,6 +17,7 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -45,17 +46,17 @@ absl::Status RunMPPGraph() { MP_RETURN_IF_ERROR(mediapipe::file::GetContents( absl::GetFlag(FLAGS_calculator_graph_config_file), &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; + ABSL_LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( calculator_graph_config_contents); - LOG(INFO) << "Initialize the calculator graph."; + ABSL_LOG(INFO) << "Initialize the calculator graph."; mediapipe::CalculatorGraph graph; MP_RETURN_IF_ERROR(graph.Initialize(config)); - LOG(INFO) << "Initialize the camera or load the video."; + ABSL_LOG(INFO) << "Initialize the camera or load the video."; cv::VideoCapture capture; const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty(); if (load_video) { @@ -68,7 +69,7 @@ absl::Status RunMPPGraph() { cv::VideoWriter writer; const bool save_video = !absl::GetFlag(FLAGS_output_video_path).empty(); if (save_video) { - LOG(INFO) << "Prepare video writer."; + ABSL_LOG(INFO) << "Prepare video writer."; cv::Mat test_frame; capture.read(test_frame); // Consume first frame. capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. @@ -85,12 +86,12 @@ absl::Status RunMPPGraph() { capture.set(cv::CAP_PROP_FPS, 30); } - LOG(INFO) << "Start running the calculator graph."; + ABSL_LOG(INFO) << "Start running the calculator graph."; ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, graph.AddOutputStreamPoller(kOutputStream)); MP_RETURN_IF_ERROR(graph.StartRun({})); - LOG(INFO) << "Start grabbing and processing frames."; + ABSL_LOG(INFO) << "Start grabbing and processing frames."; bool grab_frames = true; while (grab_frames) { // Capture opencv camera or video frame. @@ -135,7 +136,7 @@ absl::Status RunMPPGraph() { } } - LOG(INFO) << "Shutting down."; + ABSL_LOG(INFO) << "Shutting down."; if (writer.isOpened()) writer.release(); MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); return graph.WaitUntilDone(); @@ -146,10 +147,10 @@ int main(int argc, char** argv) { absl::ParseCommandLine(argc, argv); absl::Status run_status = RunMPPGraph(); if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + ABSL_LOG(ERROR) << "Failed to run the graph: " << run_status.message(); return EXIT_FAILURE; } else { - LOG(INFO) << "Success!"; + ABSL_LOG(INFO) << "Success!"; } return EXIT_SUCCESS; } diff --git a/mediapipe/examples/desktop/BUILD b/mediapipe/examples/desktop/BUILD index eec485ef..3d59c059 100644 --- a/mediapipe/examples/desktop/BUILD +++ b/mediapipe/examples/desktop/BUILD @@ -31,6 +31,7 @@ cc_library( "//mediapipe/framework/port:statusor", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) @@ -51,6 +52,7 @@ cc_library( "//mediapipe/util:resource_util", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", ], ) @@ -77,5 +79,6 @@ cc_library( "//mediapipe/util:resource_util", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/examples/desktop/autoflip/calculators/BUILD b/mediapipe/examples/desktop/autoflip/calculators/BUILD index a3b2ace2..4ae45ac8 100644 --- a/mediapipe/examples/desktop/autoflip/calculators/BUILD +++ b/mediapipe/examples/desktop/autoflip/calculators/BUILD @@ -306,6 +306,7 @@ cc_library( "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) diff --git a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc index 299f60b1..da655cb6 100644 --- a/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc +++ b/mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_log.h" #include "mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" @@ -112,8 +113,8 @@ void ShotBoundaryCalculator::Transmit(mediapipe::CalculatorContext* cc, is_shot_change = false; } if (is_shot_change) { - LOG(INFO) << "Shot change at: " << cc->InputTimestamp().Seconds() - << " seconds."; + ABSL_LOG(INFO) << "Shot change at: " << cc->InputTimestamp().Seconds() + << " seconds."; cc->Outputs() .Tag(kShotChangeTag) .AddPacket(Adopt(std::make_unique(true).release()) diff --git a/mediapipe/examples/desktop/autoflip/quality/BUILD b/mediapipe/examples/desktop/autoflip/quality/BUILD index 20e28610..0aeeffaa 100644 --- a/mediapipe/examples/desktop/autoflip/quality/BUILD +++ b/mediapipe/examples/desktop/autoflip/quality/BUILD @@ -53,6 +53,7 @@ cc_library( "//mediapipe/examples/desktop/autoflip:autoflip_messages_cc_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], ) @@ -67,6 +68,7 @@ cc_library( hdrs = ["piecewise_linear_function.h"], deps = [ "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ], ) @@ -192,6 +194,7 @@ cc_library( "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", ], ) @@ -234,6 +237,7 @@ cc_test( "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:status", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -281,6 +285,7 @@ cc_test( "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ], ) @@ -327,6 +332,7 @@ cc_library( "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc b/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc index 5916d182..947676cd 100644 --- a/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc +++ b/mediapipe/examples/desktop/autoflip/quality/frame_crop_region_computer.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/examples/desktop/autoflip/quality/utils.h" #include "mediapipe/framework/port/ret_check.h" @@ -137,7 +138,7 @@ void FrameCropRegionComputer::UpdateCropRegionScore( const float feature_score, const bool is_required, float* crop_region_score) { if (feature_score < 0.0) { - LOG(WARNING) << "Ignoring negative score"; + ABSL_LOG(WARNING) << "Ignoring negative score"; return; } @@ -161,7 +162,8 @@ void FrameCropRegionComputer::UpdateCropRegionScore( break; } default: { - LOG(WARNING) << "Unknown CropRegionScoreType " << score_aggregation_type; + ABSL_LOG(WARNING) << "Unknown CropRegionScoreType " + << score_aggregation_type; break; } } diff --git a/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc b/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc index fb8f44f1..6e1fc99e 100644 --- a/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc +++ b/mediapipe/examples/desktop/autoflip/quality/piecewise_linear_function.cc @@ -20,6 +20,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/status.h" namespace mediapipe { @@ -27,7 +28,7 @@ namespace autoflip { void PiecewiseLinearFunction::AddPoint(double x, double y) { if (!points_.empty()) { - CHECK_GE(x, points_.back().x) + ABSL_CHECK_GE(x, points_.back().x) << "Points must be provided in non-decreasing x order."; } points_.push_back(PiecewiseLinearFunction::Point(x, y)); @@ -45,8 +46,8 @@ PiecewiseLinearFunction::GetIntervalIterator(double input) const { double PiecewiseLinearFunction::Interpolate( const PiecewiseLinearFunction::Point& p1, const PiecewiseLinearFunction::Point& p2, double input) const { - CHECK_LT(p1.x, input); - CHECK_GE(p2.x, input); + ABSL_CHECK_LT(p1.x, input); + ABSL_CHECK_GE(p2.x, input); return p2.y - (p2.x - input) / (p2.x - p1.x) * (p2.y - p1.y); } diff --git a/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc b/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc index c21245cd..7870fb43 100644 --- a/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc +++ b/mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver_test.cc @@ -14,6 +14,7 @@ #include "mediapipe/examples/desktop/autoflip/quality/polynomial_regression_path_solver.h" +#include "absl/log/absl_check.h" #include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" @@ -145,8 +146,8 @@ void GenerateDataPointsFromRealVideo( const int prior_focus_point_frames_length, std::vector* focus_point_frames, std::vector* prior_focus_point_frames) { - CHECK(focus_point_frames_length + prior_focus_point_frames_length <= - kNumObservations); + ABSL_CHECK(focus_point_frames_length + prior_focus_point_frames_length <= + kNumObservations); for (int i = 0; i < prior_focus_point_frames_length; i++) { FocusPoint sp; sp.set_norm_point_x(data[i]); diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h b/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h index d7f06a02..a1528a7d 100644 --- a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h +++ b/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer.h @@ -43,7 +43,7 @@ namespace autoflip { // SceneCameraMotionAnalyzer analyzer(options); // SceneKeyFrameCropSummary scene_summary; // std::vector focus_point_frames; -// CHECK_OK(analyzer.AnalyzeScenePopulateFocusPointFrames( +// ABSL_CHECK_OK(analyzer.AnalyzeScenePopulateFocusPointFrames( // key_frame_crop_infos, key_frame_crop_options, key_frame_crop_results, // scene_frame_width, scene_frame_height, scene_frame_timestamps, // &scene_summary, &focus_point_frames)); diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc b/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc index aa3ba5c6..3b286e00 100644 --- a/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc +++ b/mediapipe/examples/desktop/autoflip/quality/scene_camera_motion_analyzer_test.cc @@ -20,6 +20,7 @@ #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" #include "absl/strings/str_split.h" #include "mediapipe/examples/desktop/autoflip/autoflip_messages.pb.h" #include "mediapipe/examples/desktop/autoflip/quality/focus_point.pb.h" @@ -744,7 +745,7 @@ TEST(SceneCameraMotionAnalyzerTest, std::vector r = absl::StrSplit(line, ','); records.insert(records.end(), r.begin(), r.end()); } - CHECK_EQ(records.size(), kNumSceneFrames * 3 + 1); + ABSL_CHECK_EQ(records.size(), kNumSceneFrames * 3 + 1); std::vector focus_point_frames; MP_EXPECT_OK(analyzer.PopulateFocusPointFrames( diff --git a/mediapipe/examples/desktop/autoflip/quality/scene_cropper.h b/mediapipe/examples/desktop/autoflip/quality/scene_cropper.h index 0e5c332d..c3c8a35c 100644 --- a/mediapipe/examples/desktop/autoflip/quality/scene_cropper.h +++ b/mediapipe/examples/desktop/autoflip/quality/scene_cropper.h @@ -41,7 +41,7 @@ namespace autoflip { // SceneCropperOptions scene_cropper_options; // SceneCropper scene_cropper(scene_cropper_options); // std::vector cropped_frames; -// CHECK_OK(scene_cropper.CropFrames( +// ABSL_CHECK_OK(scene_cropper.CropFrames( // scene_summary, scene_frames, focus_point_frames, // prior_focus_point_frames, &cropped_frames)); class SceneCropper { diff --git a/mediapipe/examples/desktop/autoflip/quality/utils.cc b/mediapipe/examples/desktop/autoflip/quality/utils.cc index 91945926..0695ff75 100644 --- a/mediapipe/examples/desktop/autoflip/quality/utils.cc +++ b/mediapipe/examples/desktop/autoflip/quality/utils.cc @@ -19,6 +19,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/examples/desktop/autoflip/quality/math_utils.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" @@ -121,12 +122,12 @@ absl::Status PackKeyFrameInfo(const int64_t frame_timestamp_ms, ScaleRect(original_detection.location(), scale_x, scale_y, &location); } else { has_valid_location = false; - LOG(ERROR) << "Detection missing a bounding box, skipped."; + ABSL_LOG(ERROR) << "Detection missing a bounding box, skipped."; } if (has_valid_location) { if (!ClampRect(original_frame_width, original_frame_height, &location) .ok()) { - LOG(ERROR) << "Invalid detection bounding box, skipped."; + ABSL_LOG(ERROR) << "Invalid detection bounding box, skipped."; continue; } auto* detection = processed_detections->add_detections(); diff --git a/mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc b/mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc index 9ae61200..661922fd 100644 --- a/mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc +++ b/mediapipe/examples/desktop/autoflip/quality/visual_scorer.cc @@ -21,6 +21,7 @@ #include #include +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/ret_check.h" @@ -106,7 +107,7 @@ absl::Status VisualScorer::CalculateScore(const cv::Mat& image, *score = (area_score + sharpness_score + colorfulness_score) / weight_sum; if (*score > 1.0f || *score < 0.0f) { - LOG(WARNING) << "Score of region outside expected range: " << *score; + ABSL_LOG(WARNING) << "Score of region outside expected range: " << *score; } return absl::OkStatus(); } diff --git a/mediapipe/examples/desktop/demo_run_graph_main.cc b/mediapipe/examples/desktop/demo_run_graph_main.cc index bb70d3df..ba36ba6c 100644 --- a/mediapipe/examples/desktop/demo_run_graph_main.cc +++ b/mediapipe/examples/desktop/demo_run_graph_main.cc @@ -17,6 +17,7 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -46,17 +47,17 @@ absl::Status RunMPPGraph() { MP_RETURN_IF_ERROR(mediapipe::file::GetContents( absl::GetFlag(FLAGS_calculator_graph_config_file), &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; + ABSL_LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( calculator_graph_config_contents); - LOG(INFO) << "Initialize the calculator graph."; + ABSL_LOG(INFO) << "Initialize the calculator graph."; mediapipe::CalculatorGraph graph; MP_RETURN_IF_ERROR(graph.Initialize(config)); - LOG(INFO) << "Initialize the camera or load the video."; + ABSL_LOG(INFO) << "Initialize the camera or load the video."; cv::VideoCapture capture; const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty(); if (load_video) { @@ -77,12 +78,12 @@ absl::Status RunMPPGraph() { #endif } - LOG(INFO) << "Start running the calculator graph."; + ABSL_LOG(INFO) << "Start running the calculator graph."; ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, graph.AddOutputStreamPoller(kOutputStream)); MP_RETURN_IF_ERROR(graph.StartRun({})); - LOG(INFO) << "Start grabbing and processing frames."; + ABSL_LOG(INFO) << "Start grabbing and processing frames."; bool grab_frames = true; while (grab_frames) { // Capture opencv camera or video frame. @@ -90,10 +91,10 @@ absl::Status RunMPPGraph() { capture >> camera_frame_raw; if (camera_frame_raw.empty()) { if (!load_video) { - LOG(INFO) << "Ignore empty frames from camera."; + ABSL_LOG(INFO) << "Ignore empty frames from camera."; continue; } - LOG(INFO) << "Empty frame, end of video reached."; + ABSL_LOG(INFO) << "Empty frame, end of video reached."; break; } cv::Mat camera_frame; @@ -126,7 +127,7 @@ absl::Status RunMPPGraph() { cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); if (save_video) { if (!writer.isOpened()) { - LOG(INFO) << "Prepare video writer."; + ABSL_LOG(INFO) << "Prepare video writer."; writer.open(absl::GetFlag(FLAGS_output_video_path), mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 capture.get(cv::CAP_PROP_FPS), output_frame_mat.size()); @@ -141,7 +142,7 @@ absl::Status RunMPPGraph() { } } - LOG(INFO) << "Shutting down."; + ABSL_LOG(INFO) << "Shutting down."; if (writer.isOpened()) writer.release(); MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); return graph.WaitUntilDone(); @@ -152,10 +153,10 @@ int main(int argc, char** argv) { absl::ParseCommandLine(argc, argv); absl::Status run_status = RunMPPGraph(); if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + ABSL_LOG(ERROR) << "Failed to run the graph: " << run_status.message(); return EXIT_FAILURE; } else { - LOG(INFO) << "Success!"; + ABSL_LOG(INFO) << "Success!"; } return EXIT_SUCCESS; } diff --git a/mediapipe/examples/desktop/demo_run_graph_main_gpu.cc b/mediapipe/examples/desktop/demo_run_graph_main_gpu.cc index 8336e567..5702bca7 100644 --- a/mediapipe/examples/desktop/demo_run_graph_main_gpu.cc +++ b/mediapipe/examples/desktop/demo_run_graph_main_gpu.cc @@ -18,6 +18,7 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -50,23 +51,23 @@ absl::Status RunMPPGraph() { MP_RETURN_IF_ERROR(mediapipe::file::GetContents( absl::GetFlag(FLAGS_calculator_graph_config_file), &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; + ABSL_LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( calculator_graph_config_contents); - LOG(INFO) << "Initialize the calculator graph."; + ABSL_LOG(INFO) << "Initialize the calculator graph."; mediapipe::CalculatorGraph graph; MP_RETURN_IF_ERROR(graph.Initialize(config)); - LOG(INFO) << "Initialize the GPU."; + ABSL_LOG(INFO) << "Initialize the GPU."; ASSIGN_OR_RETURN(auto gpu_resources, mediapipe::GpuResources::Create()); MP_RETURN_IF_ERROR(graph.SetGpuResources(std::move(gpu_resources))); mediapipe::GlCalculatorHelper gpu_helper; gpu_helper.InitializeForTest(graph.GetGpuResources().get()); - LOG(INFO) << "Initialize the camera or load the video."; + ABSL_LOG(INFO) << "Initialize the camera or load the video."; cv::VideoCapture capture; const bool load_video = !absl::GetFlag(FLAGS_input_video_path).empty(); if (load_video) { @@ -87,12 +88,12 @@ absl::Status RunMPPGraph() { #endif } - LOG(INFO) << "Start running the calculator graph."; + ABSL_LOG(INFO) << "Start running the calculator graph."; ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, graph.AddOutputStreamPoller(kOutputStream)); MP_RETURN_IF_ERROR(graph.StartRun({})); - LOG(INFO) << "Start grabbing and processing frames."; + ABSL_LOG(INFO) << "Start grabbing and processing frames."; bool grab_frames = true; while (grab_frames) { // Capture opencv camera or video frame. @@ -100,10 +101,10 @@ absl::Status RunMPPGraph() { capture >> camera_frame_raw; if (camera_frame_raw.empty()) { if (!load_video) { - LOG(INFO) << "Ignore empty frames from camera."; + ABSL_LOG(INFO) << "Ignore empty frames from camera."; continue; } - LOG(INFO) << "Empty frame, end of video reached."; + ABSL_LOG(INFO) << "Empty frame, end of video reached."; break; } cv::Mat camera_frame; @@ -169,7 +170,7 @@ absl::Status RunMPPGraph() { cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); if (save_video) { if (!writer.isOpened()) { - LOG(INFO) << "Prepare video writer."; + ABSL_LOG(INFO) << "Prepare video writer."; writer.open(absl::GetFlag(FLAGS_output_video_path), mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 capture.get(cv::CAP_PROP_FPS), output_frame_mat.size()); @@ -184,7 +185,7 @@ absl::Status RunMPPGraph() { } } - LOG(INFO) << "Shutting down."; + ABSL_LOG(INFO) << "Shutting down."; if (writer.isOpened()) writer.release(); MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); return graph.WaitUntilDone(); @@ -195,10 +196,10 @@ int main(int argc, char** argv) { absl::ParseCommandLine(argc, argv); absl::Status run_status = RunMPPGraph(); if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + ABSL_LOG(ERROR) << "Failed to run the graph: " << run_status.message(); return EXIT_FAILURE; } else { - LOG(INFO) << "Success!"; + ABSL_LOG(INFO) << "Success!"; } return EXIT_SUCCESS; } diff --git a/mediapipe/examples/desktop/hello_world/BUILD b/mediapipe/examples/desktop/hello_world/BUILD index 27aa088e..14eff2db 100644 --- a/mediapipe/examples/desktop/hello_world/BUILD +++ b/mediapipe/examples/desktop/hello_world/BUILD @@ -22,8 +22,9 @@ cc_binary( deps = [ "//mediapipe/calculators/core:pass_through_calculator", "//mediapipe/framework:calculator_graph", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/examples/desktop/hello_world/hello_world.cc b/mediapipe/examples/desktop/hello_world/hello_world.cc index fde821b5..85cf6c32 100644 --- a/mediapipe/examples/desktop/hello_world/hello_world.cc +++ b/mediapipe/examples/desktop/hello_world/hello_world.cc @@ -14,8 +14,9 @@ // // A simple example to print out "Hello World!" from a MediaPipe graph. +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_graph.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/status.h" @@ -54,7 +55,7 @@ absl::Status PrintHelloWorld() { mediapipe::Packet packet; // Get the output packets string. while (poller.Next(&packet)) { - LOG(INFO) << packet.Get(); + ABSL_LOG(INFO) << packet.Get(); } return graph.WaitUntilDone(); } @@ -62,6 +63,6 @@ absl::Status PrintHelloWorld() { int main(int argc, char** argv) { google::InitGoogleLogging(argv[0]); - CHECK(mediapipe::PrintHelloWorld().ok()); + ABSL_CHECK(mediapipe::PrintHelloWorld().ok()); return 0; } diff --git a/mediapipe/examples/desktop/iris_tracking/BUILD b/mediapipe/examples/desktop/iris_tracking/BUILD index b9f3f6f4..147a0ac2 100644 --- a/mediapipe/examples/desktop/iris_tracking/BUILD +++ b/mediapipe/examples/desktop/iris_tracking/BUILD @@ -33,6 +33,7 @@ cc_binary( "//mediapipe/graphs/iris_tracking:iris_depth_cpu_deps", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc b/mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc index 928ebb20..37476b2b 100644 --- a/mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc +++ b/mediapipe/examples/desktop/iris_tracking/iris_depth_from_image_desktop.cc @@ -19,6 +19,7 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -55,11 +56,11 @@ absl::StatusOr ReadFileToString(const std::string& file_path) { } absl::Status ProcessImage(std::unique_ptr graph) { - LOG(INFO) << "Load the image."; + ABSL_LOG(INFO) << "Load the image."; ASSIGN_OR_RETURN(const std::string raw_image, ReadFileToString(absl::GetFlag(FLAGS_input_image_path))); - LOG(INFO) << "Start running the calculator graph."; + ABSL_LOG(INFO) << "Start running the calculator graph."; ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller output_image_poller, graph->AddOutputStreamPoller(kOutputImageStream)); ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller left_iris_depth_poller, @@ -108,7 +109,7 @@ absl::Status ProcessImage(std::unique_ptr graph) { cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); const bool save_image = !absl::GetFlag(FLAGS_output_image_path).empty(); if (save_image) { - LOG(INFO) << "Saving image to file..."; + ABSL_LOG(INFO) << "Saving image to file..."; cv::imwrite(absl::GetFlag(FLAGS_output_image_path), output_frame_mat); } else { cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); @@ -117,7 +118,7 @@ absl::Status ProcessImage(std::unique_ptr graph) { cv::waitKey(0); } - LOG(INFO) << "Shutting down."; + ABSL_LOG(INFO) << "Shutting down."; MP_RETURN_IF_ERROR(graph->CloseInputStream(kInputStream)); return graph->WaitUntilDone(); } @@ -126,13 +127,13 @@ absl::Status RunMPPGraph() { std::string calculator_graph_config_contents; MP_RETURN_IF_ERROR(mediapipe::file::GetContents( kCalculatorGraphConfigFile, &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; + ABSL_LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( calculator_graph_config_contents); - LOG(INFO) << "Initialize the calculator graph."; + ABSL_LOG(INFO) << "Initialize the calculator graph."; std::unique_ptr graph = absl::make_unique(); MP_RETURN_IF_ERROR(graph->Initialize(config)); @@ -152,10 +153,10 @@ int main(int argc, char** argv) { absl::ParseCommandLine(argc, argv); absl::Status run_status = RunMPPGraph(); if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + ABSL_LOG(ERROR) << "Failed to run the graph: " << run_status.message(); return EXIT_FAILURE; } else { - LOG(INFO) << "Success!"; + ABSL_LOG(INFO) << "Success!"; } return EXIT_SUCCESS; } diff --git a/mediapipe/examples/desktop/media_sequence/BUILD b/mediapipe/examples/desktop/media_sequence/BUILD index 1a88aa10..53f93294 100644 --- a/mediapipe/examples/desktop/media_sequence/BUILD +++ b/mediapipe/examples/desktop/media_sequence/BUILD @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Placeholder: load py_library +# Placeholder: load py_binary + licenses(["notice"]) package(default_visibility = ["//mediapipe/examples:__subpackages__"]) @@ -27,6 +30,7 @@ cc_library( "//mediapipe/framework/port:status", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc b/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc index 06212b01..a14c7734 100644 --- a/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc +++ b/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc @@ -19,6 +19,7 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_split.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/file_helpers.h" @@ -43,8 +44,8 @@ absl::Status RunMPPGraph() { MP_RETURN_IF_ERROR(mediapipe::file::GetContents( absl::GetFlag(FLAGS_calculator_graph_config_file), &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; + ABSL_LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( calculator_graph_config_contents); @@ -61,12 +62,12 @@ absl::Status RunMPPGraph() { input_side_packets[name_and_value[0]] = mediapipe::MakePacket(input_side_packet_contents); } - LOG(INFO) << "Initialize the calculator graph."; + ABSL_LOG(INFO) << "Initialize the calculator graph."; mediapipe::CalculatorGraph graph; MP_RETURN_IF_ERROR(graph.Initialize(config, input_side_packets)); - LOG(INFO) << "Start running the calculator graph."; + ABSL_LOG(INFO) << "Start running the calculator graph."; MP_RETURN_IF_ERROR(graph.Run()); - LOG(INFO) << "Gathering output side packets."; + ABSL_LOG(INFO) << "Gathering output side packets."; kv_pairs = absl::StrSplit(absl::GetFlag(FLAGS_output_side_packets), ','); for (const std::string& kv_pair : kv_pairs) { std::vector name_and_value = absl::StrSplit(kv_pair, '='); @@ -88,10 +89,10 @@ int main(int argc, char** argv) { absl::ParseCommandLine(argc, argv); absl::Status run_status = RunMPPGraph(); if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + ABSL_LOG(ERROR) << "Failed to run the graph: " << run_status.message(); return EXIT_FAILURE; } else { - LOG(INFO) << "Success!"; + ABSL_LOG(INFO) << "Success!"; } return EXIT_SUCCESS; } diff --git a/mediapipe/examples/desktop/simple_run_graph_main.cc b/mediapipe/examples/desktop/simple_run_graph_main.cc index 96d9839a..e794902d 100644 --- a/mediapipe/examples/desktop/simple_run_graph_main.cc +++ b/mediapipe/examples/desktop/simple_run_graph_main.cc @@ -22,6 +22,7 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" @@ -102,8 +103,8 @@ absl::Status RunMPPGraph() { MP_RETURN_IF_ERROR(mediapipe::file::GetContents( absl::GetFlag(FLAGS_calculator_graph_config_file), &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; + ABSL_LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( calculator_graph_config_contents); @@ -119,14 +120,14 @@ absl::Status RunMPPGraph() { mediapipe::MakePacket(name_and_value[1]); } } - LOG(INFO) << "Initialize the calculator graph."; + ABSL_LOG(INFO) << "Initialize the calculator graph."; mediapipe::CalculatorGraph graph; MP_RETURN_IF_ERROR(graph.Initialize(config, input_side_packets)); if (!absl::GetFlag(FLAGS_output_stream).empty() && !absl::GetFlag(FLAGS_output_stream_file).empty()) { ASSIGN_OR_RETURN(auto poller, graph.AddOutputStreamPoller( absl::GetFlag(FLAGS_output_stream))); - LOG(INFO) << "Start running the calculator graph."; + ABSL_LOG(INFO) << "Start running the calculator graph."; MP_RETURN_IF_ERROR(graph.StartRun({})); MP_RETURN_IF_ERROR(OutputStreamToLocalFile(poller)); } else { @@ -134,7 +135,7 @@ absl::Status RunMPPGraph() { absl::GetFlag(FLAGS_output_stream_file).empty()) << "--output_stream and --output_stream_file should be specified in " "pair."; - LOG(INFO) << "Start running the calculator graph."; + ABSL_LOG(INFO) << "Start running the calculator graph."; MP_RETURN_IF_ERROR(graph.StartRun({})); } MP_RETURN_IF_ERROR(graph.WaitUntilDone()); @@ -146,10 +147,10 @@ int main(int argc, char** argv) { absl::ParseCommandLine(argc, argv); absl::Status run_status = RunMPPGraph(); if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + ABSL_LOG(ERROR) << "Failed to run the graph: " << run_status.message(); return EXIT_FAILURE; } else { - LOG(INFO) << "Success!"; + ABSL_LOG(INFO) << "Success!"; } return EXIT_SUCCESS; } diff --git a/mediapipe/examples/desktop/youtube8m/BUILD b/mediapipe/examples/desktop/youtube8m/BUILD index e0e44c4d..783c7a9d 100644 --- a/mediapipe/examples/desktop/youtube8m/BUILD +++ b/mediapipe/examples/desktop/youtube8m/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Placeholder: load py_binary + licenses(["notice"]) cc_binary( @@ -20,6 +22,7 @@ cc_binary( deps = [ "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:matrix", diff --git a/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc b/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc index 9030e925..dbabf84b 100644 --- a/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc +++ b/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc @@ -19,6 +19,7 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_split.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/matrix.h" @@ -44,8 +45,8 @@ absl::Status RunMPPGraph() { MP_RETURN_IF_ERROR(mediapipe::file::GetContents( absl::GetFlag(FLAGS_calculator_graph_config_file), &calculator_graph_config_contents)); - LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; + ABSL_LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( calculator_graph_config_contents); @@ -102,12 +103,12 @@ absl::Status RunMPPGraph() { input_side_packets["vggish_pca_projection_matrix"] = mediapipe::MakePacket(vggish_pca_projection_matrix); - LOG(INFO) << "Initialize the calculator graph."; + ABSL_LOG(INFO) << "Initialize the calculator graph."; mediapipe::CalculatorGraph graph; MP_RETURN_IF_ERROR(graph.Initialize(config, input_side_packets)); - LOG(INFO) << "Start running the calculator graph."; + ABSL_LOG(INFO) << "Start running the calculator graph."; MP_RETURN_IF_ERROR(graph.Run()); - LOG(INFO) << "Gathering output side packets."; + ABSL_LOG(INFO) << "Gathering output side packets."; kv_pairs = absl::StrSplit(absl::GetFlag(FLAGS_output_side_packets), ','); for (const std::string& kv_pair : kv_pairs) { std::vector name_and_value = absl::StrSplit(kv_pair, '='); @@ -129,10 +130,10 @@ int main(int argc, char** argv) { absl::ParseCommandLine(argc, argv); absl::Status run_status = RunMPPGraph(); if (!run_status.ok()) { - LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + ABSL_LOG(ERROR) << "Failed to run the graph: " << run_status.message(); return EXIT_FAILURE; } else { - LOG(INFO) << "Success!"; + ABSL_LOG(INFO) << "Success!"; } return EXIT_SUCCESS; } diff --git a/mediapipe/examples/ios/BUILD b/mediapipe/examples/ios/BUILD index fd611a61..1aed0228 100644 --- a/mediapipe/examples/ios/BUILD +++ b/mediapipe/examples/ios/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Placeholder: load py_test + licenses(["notice"]) package(default_visibility = ["//visibility:public"]) diff --git a/mediapipe/examples/ios/facedetectioncpu/BUILD b/mediapipe/examples/ios/facedetectioncpu/BUILD index 9424fdde..30090190 100644 --- a/mediapipe/examples/ios/facedetectioncpu/BUILD +++ b/mediapipe/examples/ios/facedetectioncpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "facedetectioncpu", diff --git a/mediapipe/examples/ios/facedetectiongpu/BUILD b/mediapipe/examples/ios/facedetectiongpu/BUILD index 8ed689b4..d3725aa3 100644 --- a/mediapipe/examples/ios/facedetectiongpu/BUILD +++ b/mediapipe/examples/ios/facedetectiongpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "facedetectiongpu", diff --git a/mediapipe/examples/ios/faceeffect/BUILD b/mediapipe/examples/ios/faceeffect/BUILD index 1152bed3..c9415068 100644 --- a/mediapipe/examples/ios/faceeffect/BUILD +++ b/mediapipe/examples/ios/faceeffect/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "faceeffect", diff --git a/mediapipe/examples/ios/facemeshgpu/BUILD b/mediapipe/examples/ios/facemeshgpu/BUILD index 6caf8c09..250a8bca 100644 --- a/mediapipe/examples/ios/facemeshgpu/BUILD +++ b/mediapipe/examples/ios/facemeshgpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "facemeshgpu", diff --git a/mediapipe/examples/ios/handdetectiongpu/BUILD b/mediapipe/examples/ios/handdetectiongpu/BUILD index 9b925537..6deb1be1 100644 --- a/mediapipe/examples/ios/handdetectiongpu/BUILD +++ b/mediapipe/examples/ios/handdetectiongpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "handdetectiongpu", diff --git a/mediapipe/examples/ios/handtrackinggpu/BUILD b/mediapipe/examples/ios/handtrackinggpu/BUILD index c5b8e7b5..b8f1442f 100644 --- a/mediapipe/examples/ios/handtrackinggpu/BUILD +++ b/mediapipe/examples/ios/handtrackinggpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "handtrackinggpu", diff --git a/mediapipe/examples/ios/helloworld/BUILD b/mediapipe/examples/ios/helloworld/BUILD index 6bfcfaae..3bed7484 100644 --- a/mediapipe/examples/ios/helloworld/BUILD +++ b/mediapipe/examples/ios/helloworld/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "helloworld", diff --git a/mediapipe/examples/ios/holistictrackinggpu/BUILD b/mediapipe/examples/ios/holistictrackinggpu/BUILD index cd10877d..56c74148 100644 --- a/mediapipe/examples/ios/holistictrackinggpu/BUILD +++ b/mediapipe/examples/ios/holistictrackinggpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "holistictrackinggpu", diff --git a/mediapipe/examples/ios/iristrackinggpu/BUILD b/mediapipe/examples/ios/iristrackinggpu/BUILD index 646d2e5a..78d4bbd1 100644 --- a/mediapipe/examples/ios/iristrackinggpu/BUILD +++ b/mediapipe/examples/ios/iristrackinggpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "iristrackinggpu", diff --git a/mediapipe/examples/ios/objectdetectioncpu/BUILD b/mediapipe/examples/ios/objectdetectioncpu/BUILD index 7638c741..47bde166 100644 --- a/mediapipe/examples/ios/objectdetectioncpu/BUILD +++ b/mediapipe/examples/ios/objectdetectioncpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "objectdetectioncpu", diff --git a/mediapipe/examples/ios/objectdetectiongpu/BUILD b/mediapipe/examples/ios/objectdetectiongpu/BUILD index 3b925c07..174db758 100644 --- a/mediapipe/examples/ios/objectdetectiongpu/BUILD +++ b/mediapipe/examples/ios/objectdetectiongpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "objectdetectiongpu", diff --git a/mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD b/mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD index 2236c525..cb8626cc 100644 --- a/mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD +++ b/mediapipe/examples/ios/objectdetectiontrackinggpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "objectdetectiontrackinggpu", diff --git a/mediapipe/examples/ios/posetrackinggpu/BUILD b/mediapipe/examples/ios/posetrackinggpu/BUILD index 4fbc2280..855d3295 100644 --- a/mediapipe/examples/ios/posetrackinggpu/BUILD +++ b/mediapipe/examples/ios/posetrackinggpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "posetrackinggpu", diff --git a/mediapipe/examples/ios/selfiesegmentationgpu/BUILD b/mediapipe/examples/ios/selfiesegmentationgpu/BUILD index 1ba7997e..2abf0561 100644 --- a/mediapipe/examples/ios/selfiesegmentationgpu/BUILD +++ b/mediapipe/examples/ios/selfiesegmentationgpu/BUILD @@ -24,7 +24,7 @@ load( licenses(["notice"]) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" alias( name = "selfiesegmentationgpu", diff --git a/mediapipe/framework/BUILD b/mediapipe/framework/BUILD index a7d9e0a6..b289fc58 100644 --- a/mediapipe/framework/BUILD +++ b/mediapipe/framework/BUILD @@ -44,6 +44,9 @@ bzl_library( "encode_binary_proto.bzl", ], visibility = ["//visibility:public"], + deps = [ + "@bazel_skylib//lib:paths", + ], ) alias( @@ -179,8 +182,8 @@ cc_library( ":timestamp", "//mediapipe/framework/deps:registration", "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:status", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", ], ) @@ -201,6 +204,7 @@ cc_library( ":timestamp", "//mediapipe/framework/port:any_proto", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ], ) @@ -217,6 +221,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/tool:tag_map", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/synchronization", ], @@ -269,6 +274,8 @@ cc_library( ], deps = [ ":calculator_base", + ":calculator_context", + ":calculator_contract", ":calculator_graph", ":calculator_registry", ":counter_factory", @@ -318,7 +325,6 @@ cc_library( ":input_stream_manager", ":mediapipe_profiling", ":output_side_packet_impl", - ":output_stream", ":output_stream_manager", ":output_stream_poller", ":output_stream_shard", @@ -332,6 +338,7 @@ cc_library( ":scheduler_queue", ":status_handler", ":status_handler_cc_proto", + ":subgraph", ":thread_pool_executor", ":thread_pool_executor_cc_proto", ":timestamp", @@ -339,6 +346,7 @@ cc_library( "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:map_util", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", @@ -352,11 +360,13 @@ cc_library( "//mediapipe/gpu:graph_support", "//mediapipe/util:cpu_util", "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/container:fixed_array", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", @@ -425,6 +435,8 @@ cc_library( "//mediapipe/framework/tool:tag_map", "//mediapipe/framework/tool:validate_name", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", @@ -451,11 +463,12 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":calculator_framework", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:sink", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], @@ -481,6 +494,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/tool:options_map", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -498,11 +512,12 @@ cc_library( deps = [ ":collection_item_id", ":type_map", - "//mediapipe/framework/port:logging", "//mediapipe/framework/tool:tag_map", "//mediapipe/framework/tool:tag_map_helper", "//mediapipe/framework/tool:validate_name", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], @@ -536,6 +551,8 @@ cc_library( "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:map_util", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", @@ -606,10 +623,11 @@ cc_library( ":packet_set", ":packet_type", ":timestamp", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -624,6 +642,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -642,6 +661,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:fill_packet_set", + "@com_google_absl//absl/log:absl_check", ], ) @@ -680,6 +700,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:tag_map", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -700,6 +721,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -719,6 +741,7 @@ cc_library( "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -758,6 +781,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ], ) @@ -796,6 +820,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/tool:tag_map", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/synchronization", ], ) @@ -814,6 +839,7 @@ cc_library( ":timestamp", "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/synchronization", ], ) @@ -824,6 +850,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":graph_output_stream", + "@com_google_absl//absl/log:absl_check", ], ) @@ -840,6 +867,7 @@ cc_library( ":timestamp", "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -864,6 +892,8 @@ cc_library( "//mediapipe/framework/port:statusor", "//mediapipe/framework/tool:type_util", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", @@ -944,6 +974,8 @@ cc_library( "//mediapipe/framework/tool:type_util", "//mediapipe/framework/tool:validate_name", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:span", @@ -1020,6 +1052,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/synchronization", ], ) @@ -1079,6 +1112,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@eigen_archive//:eigen3", ], @@ -1129,6 +1163,8 @@ cc_library( "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) @@ -1149,6 +1185,8 @@ cc_library( "//mediapipe/framework/tool:status_util", "//mediapipe/framework/tool:type_util", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/synchronization", ], alwayslink = 1, @@ -1178,7 +1216,6 @@ cc_library( ":calculator_contract", ":graph_service_manager", ":legacy_calculator_support", - ":packet", ":packet_generator", ":packet_generator_cc_proto", ":packet_set", @@ -1189,7 +1226,6 @@ cc_library( ":stream_handler_cc_proto", ":subgraph", ":thread_pool_executor_cc_proto", - ":timestamp", "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", @@ -1203,10 +1239,12 @@ cc_library( "//mediapipe/framework/tool:subgraph_expansion", "//mediapipe/framework/tool:validate", "//mediapipe/framework/tool:validate_name", - "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_protobuf//:protobuf", ], ) @@ -1288,10 +1326,11 @@ cc_test( ":calculator_node", "//mediapipe/calculators/core:pass_through_calculator", "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:source", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", ], ) @@ -1355,6 +1394,23 @@ cc_test( ], ) +cc_test( + name = "calculator_graph_summary_packet_test", + srcs = ["calculator_graph_summary_packet_test.cc"], + deps = [ + ":calculator_framework", + ":packet", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/api2:packet", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/stream_handler:immediate_input_stream_handler", + "//mediapipe/framework/tool:sink", + "@com_google_absl//absl/status", + ], +) + cc_test( name = "calculator_runner_test", size = "medium", @@ -1368,8 +1424,8 @@ cc_test( ":packet_type", ":timestamp", "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) @@ -1431,7 +1487,6 @@ cc_test( "//mediapipe/calculators/core:mux_calculator", "//mediapipe/calculators/core:pass_through_calculator", "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", @@ -1446,7 +1501,10 @@ cc_test( "//mediapipe/framework/tool:status_util", "//mediapipe/gpu:gpu_service", "@com_google_absl//absl/container:fixed_array", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/time", @@ -1501,11 +1559,11 @@ cc_test( "//mediapipe/calculators/core:mux_calculator", "//mediapipe/calculators/core:pass_through_calculator", "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:sink", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/time", ], ) @@ -1632,6 +1690,7 @@ cc_test( ":packet", ":packet_test_cc_proto", ":type_map", + "//mediapipe/framework/api2:builder", "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:gtest_main", "@com_google_absl//absl/strings", @@ -1676,6 +1735,7 @@ cc_test( "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/tool:template_parser", + "@com_google_absl//absl/log:absl_check", ], ) diff --git a/mediapipe/framework/api2/BUILD b/mediapipe/framework/api2/BUILD index 8a394689..5c5ec04e 100644 --- a/mediapipe/framework/api2/BUILD +++ b/mediapipe/framework/api2/BUILD @@ -1,3 +1,5 @@ +# Placeholder: load py_test + package( default_visibility = ["//visibility:public"], features = ["-use_header_modules"], @@ -20,6 +22,7 @@ cc_library( "//mediapipe/framework/port:any_proto", "//mediapipe/framework/port:ret_check", "@com_google_absl//absl/container:btree", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", ], @@ -112,6 +115,7 @@ cc_test( "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], ) @@ -123,6 +127,7 @@ cc_library( ":tuple", "//mediapipe/framework:packet", "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/meta:type_traits", ], ) @@ -152,6 +157,7 @@ cc_library( "//mediapipe/framework:output_side_packet", "//mediapipe/framework/port:logging", "//mediapipe/framework/tool:type_util", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/framework/api2/builder.h b/mediapipe/framework/api2/builder.h index 51e59973..fde28112 100644 --- a/mediapipe/framework/api2/builder.h +++ b/mediapipe/framework/api2/builder.h @@ -11,6 +11,7 @@ #include #include "absl/container/btree_map.h" +#include "absl/log/absl_check.h" #include "absl/strings/string_view.h" #include "google/protobuf/message_lite.h" #include "mediapipe/framework/api2/port.h" @@ -32,7 +33,7 @@ template struct dependent_false : std::false_type {}; template -T& GetWithAutoGrow(std::vector>* vecp, int index) { +T& GetWithAutoGrow(std::vector>* vecp, size_t index) { auto& vec = *vecp; if (vec.size() <= index) { vec.resize(index + 1); @@ -109,7 +110,7 @@ class MultiPort : public Single { : Single(vec), vec_(*vec) {} Single operator[](int index) { - CHECK_GE(index, 0); + ABSL_CHECK_GE(index, 0); return Single{&GetWithAutoGrow(&vec_, index)}; } @@ -193,7 +194,7 @@ class SourceImpl { template {}, int>::type = 0> Src& ConnectTo(const Dst& dest) { - CHECK(dest.base_.source == nullptr); + ABSL_CHECK(dest.base_.source == nullptr); dest.base_.source = base_; base_->dests_.emplace_back(&dest.base_); return *this; @@ -721,14 +722,14 @@ class Graph { config.set_type(type_); } FixUnnamedConnections(); - CHECK_OK(UpdateBoundaryConfig(&config)); + ABSL_CHECK_OK(UpdateBoundaryConfig(&config)); for (const std::unique_ptr& node : nodes_) { auto* out_node = config.add_node(); - CHECK_OK(UpdateNodeConfig(*node, out_node)); + ABSL_CHECK_OK(UpdateNodeConfig(*node, out_node)); } for (const std::unique_ptr& node : packet_gens_) { auto* out_node = config.add_packet_generator(); - CHECK_OK(UpdateNodeConfig(*node, out_node)); + ABSL_CHECK_OK(UpdateNodeConfig(*node, out_node)); } return config; } @@ -782,7 +783,7 @@ class Graph { config->set_calculator(node.type_); node.in_streams_.Visit( [&](const TagIndexLocation& loc, const DestinationBase& endpoint) { - CHECK(endpoint.source != nullptr); + ABSL_CHECK(endpoint.source != nullptr); config->add_input_stream(TaggedName(loc, endpoint.source->name_)); }); node.out_streams_.Visit( @@ -791,7 +792,7 @@ class Graph { }); node.in_sides_.Visit([&](const TagIndexLocation& loc, const DestinationBase& endpoint) { - CHECK(endpoint.source != nullptr); + ABSL_CHECK(endpoint.source != nullptr); config->add_input_side_packet(TaggedName(loc, endpoint.source->name_)); }); node.out_sides_.Visit( @@ -812,7 +813,7 @@ class Graph { config->set_packet_generator(node.type_); node.in_sides_.Visit([&](const TagIndexLocation& loc, const DestinationBase& endpoint) { - CHECK(endpoint.source != nullptr); + ABSL_CHECK(endpoint.source != nullptr); config->add_input_side_packet(TaggedName(loc, endpoint.source->name_)); }); node.out_sides_.Visit( @@ -829,7 +830,7 @@ class Graph { absl::Status UpdateBoundaryConfig(CalculatorGraphConfig* config) { graph_boundary_.in_streams_.Visit( [&](const TagIndexLocation& loc, const DestinationBase& endpoint) { - CHECK(endpoint.source != nullptr); + ABSL_CHECK(endpoint.source != nullptr); config->add_output_stream(TaggedName(loc, endpoint.source->name_)); }); graph_boundary_.out_streams_.Visit( @@ -838,7 +839,7 @@ class Graph { }); graph_boundary_.in_sides_.Visit([&](const TagIndexLocation& loc, const DestinationBase& endpoint) { - CHECK(endpoint.source != nullptr); + ABSL_CHECK(endpoint.source != nullptr); config->add_output_side_packet(TaggedName(loc, endpoint.source->name_)); }); graph_boundary_.out_sides_.Visit( diff --git a/mediapipe/framework/api2/node.h b/mediapipe/framework/api2/node.h index 14c09824..58cebf1e 100644 --- a/mediapipe/framework/api2/node.h +++ b/mediapipe/framework/api2/node.h @@ -64,58 +64,13 @@ class CalculatorBaseFactoryFor< namespace api2 { namespace internal { -// Defining a member of this type causes P to be ODR-used, which forces its -// instantiation if it's a static member of a template. -// Previously we depended on the pointer's value to determine whether the size -// of a character array is 0 or 1, forcing it to be instantiated so the -// compiler can determine the object's layout. But using it as a template -// argument is more compact. -template -struct ForceStaticInstantiation { -#ifdef _MSC_VER - // Just having it as the template argument does not count as a use for - // MSVC. - static constexpr bool Use() { return P != nullptr; } - char force_static[Use()]; -#endif // _MSC_VER -}; +MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE( + NodeRegistrator, mediapipe::CalculatorBaseRegistry, T::kCalculatorName, + absl::make_unique>) -// Helper template for forcing the definition of a static registration token. -template -struct NodeRegistrationStatic { - static NoDestructor registration; - - static mediapipe::RegistrationToken Make() { - return mediapipe::CalculatorBaseRegistry::Register( - T::kCalculatorName, - absl::make_unique>, - __FILE__, __LINE__); - } - - using RequireStatics = ForceStaticInstantiation<®istration>; -}; - -// Static members of template classes can be defined in the header. -template -NoDestructor - NodeRegistrationStatic::registration(NodeRegistrationStatic::Make()); - -template -struct SubgraphRegistrationImpl { - static NoDestructor registration; - - static mediapipe::RegistrationToken Make() { - return mediapipe::SubgraphRegistry::Register( - T::kCalculatorName, absl::make_unique, __FILE__, __LINE__); - } - - using RequireStatics = ForceStaticInstantiation<®istration>; -}; - -template -NoDestructor - SubgraphRegistrationImpl::registration( - SubgraphRegistrationImpl::Make()); +MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(SubgraphRegistrator, + mediapipe::SubgraphRegistry, + T::kCalculatorName, absl::make_unique) } // namespace internal @@ -128,14 +83,7 @@ template class RegisteredNode; template -class RegisteredNode : public Node { - private: - // The member below triggers instantiation of the registration static. - // Note that the constructor of calculator subclasses is only invoked through - // the registration token, and so we cannot simply use the static in the - // constructor. - typename internal::NodeRegistrationStatic::RequireStatics register_; -}; +class RegisteredNode : public Node, private internal::NodeRegistrator {}; // No-op version for backwards compatibility. template <> @@ -217,31 +165,27 @@ class NodeImpl : public RegisteredNode, public Intf { // TODO: verify that the subgraph config fully implements the // declared interface. template -class SubgraphImpl : public Subgraph, public Intf { - private: - typename internal::SubgraphRegistrationImpl::RequireStatics register_; -}; +class SubgraphImpl : public Subgraph, + public Intf, + private internal::SubgraphRegistrator {}; // This macro is used to register a calculator that does not use automatic // registration. Deprecated. -#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl) \ - static mediapipe::NoDestructor \ - REGISTRY_STATIC_VAR(calculator_registration, \ - __LINE__)(mediapipe::CalculatorBaseRegistry::Register( \ - Impl::kCalculatorName, \ - absl::make_unique>, \ - __FILE__, __LINE__)) +#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl) \ + MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \ + mediapipe::CalculatorBaseRegistry, calculator_registration, \ + Impl::kCalculatorName, \ + absl::make_unique>) // This macro is used to register a non-split-contract calculator. Deprecated. #define MEDIAPIPE_REGISTER_NODE(name) REGISTER_CALCULATOR(name) // This macro is used to define a subgraph that does not use automatic // registration. Deprecated. -#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl) \ - static mediapipe::NoDestructor \ - REGISTRY_STATIC_VAR(subgraph_registration, \ - __LINE__)(mediapipe::SubgraphRegistry::Register( \ - Impl::kCalculatorName, absl::make_unique, __FILE__, __LINE__)) +#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl) \ + MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \ + mediapipe::SubgraphRegistry, subgraph_registration, \ + Impl::kCalculatorName, absl::make_unique) } // namespace api2 } // namespace mediapipe diff --git a/mediapipe/framework/api2/node_test.cc b/mediapipe/framework/api2/node_test.cc index 152cbb0e..ac1ca601 100644 --- a/mediapipe/framework/api2/node_test.cc +++ b/mediapipe/framework/api2/node_test.cc @@ -3,6 +3,7 @@ #include #include +#include "absl/log/absl_log.h" #include "mediapipe/framework/api2/packet.h" #include "mediapipe/framework/api2/port.h" #include "mediapipe/framework/api2/test_contracts.h" @@ -570,7 +571,7 @@ struct LogSinkNode : public Node { MEDIAPIPE_NODE_CONTRACT(kIn); absl::Status Process(CalculatorContext* cc) override { - LOG(INFO) << "LogSinkNode received: " << kIn(cc).Get(); + ABSL_LOG(INFO) << "LogSinkNode received: " << kIn(cc).Get(); return {}; } }; diff --git a/mediapipe/framework/api2/packet.h b/mediapipe/framework/api2/packet.h index c059a988..f231f4c8 100644 --- a/mediapipe/framework/api2/packet.h +++ b/mediapipe/framework/api2/packet.h @@ -13,6 +13,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/meta/type_traits.h" #include "mediapipe/framework/api2/tuple.h" #include "mediapipe/framework/packet.h" @@ -102,9 +103,9 @@ mediapipe::Packet ToOldPacket(PacketBase&& p); template inline const T& PacketBase::Get() const { - CHECK(payload_); + ABSL_CHECK(payload_); packet_internal::Holder* typed_payload = payload_->As(); - CHECK(typed_payload) << absl::StrCat( + ABSL_CHECK(typed_payload) << absl::StrCat( "The Packet stores \"", payload_->DebugTypeName(), "\", but \"", MediaPipeTypeStringOrDemangled(), "\" was requested."); return typed_payload->data(); @@ -134,17 +135,17 @@ namespace internal { template inline void CheckCompatibleType(const HolderBase& holder, internal::Wrap) { const packet_internal::Holder* typed_payload = holder.As(); - CHECK(typed_payload) << absl::StrCat( + ABSL_CHECK(typed_payload) << absl::StrCat( "The Packet stores \"", holder.DebugTypeName(), "\", but \"", MediaPipeTypeStringOrDemangled(), "\" was requested."); - // CHECK(payload_->has_type()); + // ABSL_CHECK(payload_->has_type()); } template inline void CheckCompatibleType(const HolderBase& holder, internal::Wrap>) { bool compatible = (holder.As() || ...); - CHECK(compatible) + ABSL_CHECK(compatible) << "The Packet stores \"" << holder.DebugTypeName() << "\", but one of " << absl::StrJoin( {absl::StrCat("\"", MediaPipeTypeStringOrDemangled(), "\"")...}, @@ -211,9 +212,9 @@ class Packet : public Packet { Packet At(Timestamp timestamp) &&; const T& Get() const { - CHECK(payload_); + ABSL_CHECK(payload_); packet_internal::Holder* typed_payload = payload_->As(); - CHECK(typed_payload); + ABSL_CHECK(typed_payload); return typed_payload->data(); } const T& operator*() const { return Get(); } @@ -330,9 +331,9 @@ class Packet> : public PacketBase { template > const U& Get() const { - CHECK(payload_); + ABSL_CHECK(payload_); packet_internal::Holder* typed_payload = payload_->As(); - CHECK(typed_payload); + ABSL_CHECK(typed_payload); return typed_payload->data(); } @@ -343,7 +344,7 @@ class Packet> : public PacketBase { template auto Visit(const F&... args) const { - CHECK(payload_); + ABSL_CHECK(payload_); auto f = internal::Overload{args...}; using FirstT = typename internal::First::type; using ResultType = absl::result_of_t; @@ -364,7 +365,7 @@ class Packet> : public PacketBase { template auto ConsumeAndVisit(const F&... args) { - CHECK(payload_); + ABSL_CHECK(payload_); auto f = internal::Overload{args...}; using FirstT = typename internal::First::type; using VisitorResultType = diff --git a/mediapipe/framework/api2/port.h b/mediapipe/framework/api2/port.h index 18a78607..075e8843 100644 --- a/mediapipe/framework/api2/port.h +++ b/mediapipe/framework/api2/port.h @@ -20,6 +20,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/api2/const_str.h" @@ -243,8 +244,8 @@ class MultiplePortAccess { // container? int Count() { return count_; } AccessT operator[](int pos) { - CHECK_GE(pos, 0); - CHECK_LT(pos, count_); + ABSL_CHECK_GE(pos, 0); + ABSL_CHECK_LT(pos, count_); return SinglePortAccess(cc_, &first_[pos]); } diff --git a/mediapipe/framework/api2/stream/BUILD b/mediapipe/framework/api2/stream/BUILD new file mode 100644 index 00000000..f57dd46b --- /dev/null +++ b/mediapipe/framework/api2/stream/BUILD @@ -0,0 +1,84 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "loopback", + hdrs = ["loopback.h"], + deps = [ + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2:port", + ], +) + +cc_test( + name = "loopback_test", + srcs = ["loopback_test.cc"], + deps = [ + ":loopback", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/port:gtest", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status_matchers", + ], +) + +cc_library( + name = "image_size", + hdrs = ["image_size.h"], + deps = [ + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/gpu:gpu_buffer", + ], +) + +cc_test( + name = "image_size_test", + srcs = ["image_size_test.cc"], + deps = [ + ":image_size", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/port:gtest", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status_matchers", + "//mediapipe/gpu:gpu_buffer", + ], +) + +cc_library( + name = "rect_transformation", + srcs = ["rect_transformation.cc"], + hdrs = ["rect_transformation.h"], + deps = [ + "//mediapipe/calculators/util:rect_transformation_calculator", + "//mediapipe/calculators/util:rect_transformation_calculator_cc_proto", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/formats:rect_cc_proto", + "@com_google_absl//absl/types:optional", + ], +) + +cc_test( + name = "rect_transformation_test", + srcs = ["rect_transformation_test.cc"], + deps = [ + ":rect_transformation", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:gtest", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + ], +) diff --git a/mediapipe/framework/api2/stream/image_size.h b/mediapipe/framework/api2/stream/image_size.h new file mode 100644 index 00000000..b726f07a --- /dev/null +++ b/mediapipe/framework/api2/stream/image_size.h @@ -0,0 +1,34 @@ +#ifndef MEDIAPIPE_FRAMEWORK_API2_STREAM_IMAGE_SIZE_H_ +#define MEDIAPIPE_FRAMEWORK_API2_STREAM_IMAGE_SIZE_H_ + +#include + +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/gpu/gpu_buffer.h" + +namespace mediapipe::api2::builder { + +// Updates graph to calculate image size and returns corresponding stream. +// +// @image image represented as ImageFrame/Image/GpuBuffer. +// @graph graph to update. +template +Stream> GetImageSize( + Stream image, mediapipe::api2::builder::Graph& graph) { + auto& img_props_node = graph.AddNode("ImagePropertiesCalculator"); + if constexpr (std::is_same_v || + std::is_same_v) { + image.ConnectTo(img_props_node.In("IMAGE")); + } else if constexpr (std::is_same_v) { + image.ConnectTo(img_props_node.In("IMAGE_GPU")); + } else { + static_assert(dependent_false::value, "Type not supported."); + } + return img_props_node.Out("SIZE").Cast>(); +} + +} // namespace mediapipe::api2::builder + +#endif // MEDIAPIPE_FRAMEWORK_API2_STREAM_IMAGE_SIZE_H_ diff --git a/mediapipe/framework/api2/stream/image_size_test.cc b/mediapipe/framework/api2/stream/image_size_test.cc new file mode 100644 index 00000000..3b080ba0 --- /dev/null +++ b/mediapipe/framework/api2/stream/image_size_test.cc @@ -0,0 +1,57 @@ +#include "mediapipe/framework/api2/stream/image_size.h" + +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/gpu/gpu_buffer.h" + +namespace mediapipe::api2::builder { +namespace { + +TEST(GetImageSize, VerifyConfig) { + Graph graph; + + Stream image_frame = graph.In("IMAGE_FRAME").Cast(); + image_frame.SetName("image_frame"); + Stream gpu_buffer = graph.In("GPU_BUFFER").Cast(); + gpu_buffer.SetName("gpu_buffer"); + Stream image = graph.In("IMAGE").Cast(); + image.SetName("image"); + + GetImageSize(image_frame, graph).SetName("image_frame_size"); + GetImageSize(gpu_buffer, graph).SetName("gpu_buffer_size"); + GetImageSize(image, graph).SetName("image_size"); + + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image_frame" + output_stream: "SIZE:image_frame_size" + } + node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE_GPU:gpu_buffer" + output_stream: "SIZE:gpu_buffer_size" + } + node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image" + output_stream: "SIZE:image_size" + } + input_stream: "GPU_BUFFER:gpu_buffer" + input_stream: "IMAGE:image" + input_stream: "IMAGE_FRAME:image_frame" + )pb"))); + + CalculatorGraph calcualtor_graph; + MP_EXPECT_OK(calcualtor_graph.Initialize(graph.GetConfig())); +} +} // namespace +} // namespace mediapipe::api2::builder diff --git a/mediapipe/framework/api2/stream/loopback.h b/mediapipe/framework/api2/stream/loopback.h new file mode 100644 index 00000000..3ad2f0a2 --- /dev/null +++ b/mediapipe/framework/api2/stream/loopback.h @@ -0,0 +1,55 @@ +#ifndef MEDIAPIPE_FRAMEWORK_API2_STREAM_LOOPBACK_H_ +#define MEDIAPIPE_FRAMEWORK_API2_STREAM_LOOPBACK_H_ + +#include +#include + +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/port.h" + +namespace mediapipe::api2::builder { + +// Returns a pair of two values: +// - A stream with loopback data. Such stream, for each new packet in @tick +// stream, provides a packet previously calculated within the graph. +// - A function to define/set loopback data producing stream. +// NOTE: +// * function must be called and only once, otherwise graph validation will +// fail. +// * calling function after graph is destroyed results in undefined behavior +// +// The function wraps `PreviousLoopbackCalculator` into a convenience function +// and allows graph input to be processed together with some previous output. +// +// ------- +// +// Example: +// +// ``` +// +// Graph graph; +// Stream<...> tick = ...; // E.g. main input can surve as a tick. +// auto [prev_data, set_loopback_fn] = GetLoopbackData(tick, graph); +// ... +// Stream data = ...; +// set_loopback_fn(data); +// +// ``` +template +std::pair, std::function)>> GetLoopbackData( + Stream tick, mediapipe::api2::builder::Graph& graph) { + auto& prev = graph.AddNode("PreviousLoopbackCalculator"); + tick.ConnectTo(prev.In("MAIN")); + return {prev.Out("PREV_LOOP").template Cast(), + [prev_ptr = &prev](Stream data) { + // TODO: input stream info must be specified, but + // builder api doesn't support it at the moment. As a workaround, + // input stream info is added by GraphBuilder as a graph building + // post processing step. + data.ConnectTo(prev_ptr->In("LOOP")); + }}; +} + +} // namespace mediapipe::api2::builder + +#endif // MEDIAPIPE_FRAMEWORK_API2_STREAM_LOOPBACK_H_ diff --git a/mediapipe/framework/api2/stream/loopback_test.cc b/mediapipe/framework/api2/stream/loopback_test.cc new file mode 100644 index 00000000..50c3041e --- /dev/null +++ b/mediapipe/framework/api2/stream/loopback_test.cc @@ -0,0 +1,56 @@ +#include "mediapipe/framework/api2/stream/loopback.h" + +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe::api2::builder { +namespace { + +class TestDataProducer : public NodeIntf { + public: + static constexpr Input kLoopbackData{"LOOPBACK_DATA"}; + static constexpr Output kProducedData{"PRODUCED_DATA"}; + MEDIAPIPE_NODE_INTERFACE(TestDataProducer, kLoopbackData, kProducedData); +}; + +TEST(LoopbackTest, GetLoopbackData) { + Graph graph; + + Stream tick = graph.In("TICK").Cast(); + + auto [data, set_loopback_data_fn] = GetLoopbackData(tick, graph); + + auto& producer = graph.AddNode(); + data.ConnectTo(producer[TestDataProducer::kLoopbackData]); + Stream data_to_loopback(producer[TestDataProducer::kProducedData]); + + set_loopback_data_fn(data_to_loopback); + + // PreviousLoopbackCalculator configuration is incorrect here and should be + // updated when corresponding b/175887687 is fixed. + // Use mediapipe::aimatter::GraphBuilder to fix back edges in the graph. + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "PreviousLoopbackCalculator" + input_stream: "LOOP:__stream_2" + input_stream: "MAIN:__stream_0" + output_stream: "PREV_LOOP:__stream_1" + } + node { + calculator: "TestDataProducer" + input_stream: "LOOPBACK_DATA:__stream_1" + output_stream: "PRODUCED_DATA:__stream_2" + } + input_stream: "TICK:__stream_0" + )pb"))); +} + +} // namespace +} // namespace mediapipe::api2::builder diff --git a/mediapipe/framework/api2/stream/rect_transformation.cc b/mediapipe/framework/api2/stream/rect_transformation.cc new file mode 100644 index 00000000..3e63375f --- /dev/null +++ b/mediapipe/framework/api2/stream/rect_transformation.cc @@ -0,0 +1,108 @@ +#include "mediapipe/framework/api2/stream/rect_transformation.h" + +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "mediapipe/calculators/util/rect_transformation_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/rect.pb.h" + +namespace mediapipe::api2::builder { + +namespace { + +using ::mediapipe::NormalizedRect; +using ::mediapipe::api2::builder::GenericNode; +using ::mediapipe::api2::builder::Graph; + +template +Stream InternalScaleAndShift( + Stream transformee, Stream> image_size, + float scale_x_factor, float scale_y_factor, std::optional shift_x, + std::optional shift_y, bool square_long, Graph& graph) { + auto& node = graph.AddNode("RectTransformationCalculator"); + auto& node_opts = + node.GetOptions(); + node_opts.set_scale_x(scale_x_factor); + node_opts.set_scale_y(scale_y_factor); + if (shift_x) { + node_opts.set_shift_x(shift_x.value()); + } + if (shift_y) { + node_opts.set_shift_y(shift_y.value()); + } + if (square_long) { + node_opts.set_square_long(square_long); + } + image_size.ConnectTo(node.In("IMAGE_SIZE")); + if constexpr (std::is_same_v>) { + transformee.ConnectTo(node.In("NORM_RECTS")); + } else if constexpr (std::is_same_v) { + transformee.ConnectTo(node.In("NORM_RECT")); + } else { + static_assert(dependent_false::value, "Unsupported type."); + } + return node.Out("").template Cast(); +} + +} // namespace + +Stream ScaleAndMakeSquare( + Stream rect, Stream> image_size, + float scale_x_factor, float scale_y_factor, Graph& graph) { + return InternalScaleAndShift(rect, image_size, scale_x_factor, scale_y_factor, + /*shift_x=*/std::nullopt, + /*shift_y=*/std::nullopt, + /*square_long=*/true, graph); +} + +Stream Scale(Stream rect, + Stream> image_size, + float scale_x_factor, float scale_y_factor, + Graph& graph) { + return InternalScaleAndShift(rect, image_size, scale_x_factor, scale_y_factor, + /*shift_x=*/std::nullopt, + /*shift_y=*/std::nullopt, + /*square_long=*/false, graph); +} + +Stream> ScaleAndShiftAndMakeSquareLong( + Stream> rects, + Stream> image_size, float scale_x_factor, + float scale_y_factor, float shift_x, float shift_y, Graph& graph) { + return InternalScaleAndShift(rects, image_size, scale_x_factor, + scale_y_factor, shift_x, shift_y, + /*square_long=*/true, graph); +} + +Stream> ScaleAndShift( + Stream> rects, + Stream> image_size, float scale_x_factor, + float scale_y_factor, float shift_x, float shift_y, Graph& graph) { + return InternalScaleAndShift(rects, image_size, scale_x_factor, + scale_y_factor, shift_x, shift_y, + /*square_long=*/false, graph); +} + +Stream ScaleAndShiftAndMakeSquareLong( + Stream rect, Stream> image_size, + float scale_x_factor, float scale_y_factor, float shift_x, float shift_y, + Graph& graph) { + return InternalScaleAndShift(rect, image_size, scale_x_factor, scale_y_factor, + shift_x, shift_y, + /*square_long=*/true, graph); +} + +Stream ScaleAndShift(Stream rect, + Stream> image_size, + float scale_x_factor, float scale_y_factor, + float shift_x, float shift_y, + Graph& graph) { + return InternalScaleAndShift(rect, image_size, scale_x_factor, scale_y_factor, + shift_x, shift_y, /*square_long=*/false, graph); +} + +} // namespace mediapipe::api2::builder diff --git a/mediapipe/framework/api2/stream/rect_transformation.h b/mediapipe/framework/api2/stream/rect_transformation.h new file mode 100644 index 00000000..9f6a9898 --- /dev/null +++ b/mediapipe/framework/api2/stream/rect_transformation.h @@ -0,0 +1,67 @@ +#ifndef MEDIAPIPE_FRAMEWORK_API2_STREAM_RECT_TRANSFORMATION_H_ +#define MEDIAPIPE_FRAMEWORK_API2_STREAM_RECT_TRANSFORMATION_H_ + +#include +#include + +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/rect.pb.h" + +namespace mediapipe::api2::builder { + +// Updates @graph to scale @rect according to passed parameters. +Stream Scale(Stream rect, + Stream> image_size, + float scale_x_factor, + float scale_y_factor, + mediapipe::api2::builder::Graph& graph); + +// Updates @graph to scale @rect according to passed parameters and make it a +// square that has the same center and rotation, and with the side of the square +// equal to the long side of the rect. +// +// TODO: consider removing after migrating to `Scale`. +Stream ScaleAndMakeSquare( + Stream rect, + Stream> image_size, float scale_x_factor, + float scale_y_factor, mediapipe::api2::builder::Graph& graph); + +// Updates @graph to scale and shift vector of @rects according to parameters. +Stream> ScaleAndShift( + Stream> rects, + Stream> image_size, float scale_x_factor, + float scale_y_factor, float shift_x, float shift_y, + mediapipe::api2::builder::Graph& graph); + +// Updates @graph to scale and shift vector of @rects according to passed +// parameters and make each a square that has the same center and rotation, and +// with the side of the square equal to the long side of a particular rect. +// +// TODO: consider removing after migrating to `ScaleAndShift`. +Stream> ScaleAndShiftAndMakeSquareLong( + Stream> rects, + Stream> image_size, float scale_x_factor, + float scale_y_factor, float shift_x, float shift_y, + mediapipe::api2::builder::Graph& graph); + +// Updates @graph to scale, shift @rect according to passed parameters. +Stream ScaleAndShift( + Stream rect, + Stream> image_size, float scale_x_factor, + float scale_y_factor, float shift_x, float shift_y, + mediapipe::api2::builder::Graph& graph); + +// Updates @graph to scale and shift @rect according to passed parameters and +// make it a square that has the same center and rotation, and with the side of +// the square equal to the long side of the rect. +// +// TODO: consider removing after migrating to `ScaleAndShift`. +Stream ScaleAndShiftAndMakeSquareLong( + Stream rect, + Stream> image_size, float scale_x_factor, + float scale_y_factor, float shift_x, float shift_y, + mediapipe::api2::builder::Graph& graph); + +} // namespace mediapipe::api2::builder + +#endif // MEDIAPIPE_FRAMEWORK_API2_STREAM_RECT_TRANSFORMATION_H_ diff --git a/mediapipe/framework/api2/stream/rect_transformation_test.cc b/mediapipe/framework/api2/stream/rect_transformation_test.cc new file mode 100644 index 00000000..79fa6617 --- /dev/null +++ b/mediapipe/framework/api2/stream/rect_transformation_test.cc @@ -0,0 +1,217 @@ +#include "mediapipe/framework/api2/stream/rect_transformation.h" + +#include +#include + +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" + +namespace mediapipe::api2::builder { + +namespace { + +using ::mediapipe::NormalizedRect; + +TEST(RectTransformation, ScaleAndMakeSquare) { + mediapipe::api2::builder::Graph graph; + + Stream rect = graph.In("RECT").Cast(); + Stream> size = + graph.In("SIZE").Cast>(); + Stream transformed_rect = ScaleAndMakeSquare( + rect, size, /*scale_x_factor=*/2, /*scale_y_factor=*/7, graph); + transformed_rect.SetName("transformed_rect"); + + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_1" + input_stream: "NORM_RECT:__stream_0" + output_stream: "transformed_rect" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 2 + scale_y: 7 + square_long: true + } + } + } + input_stream: "RECT:__stream_0" + input_stream: "SIZE:__stream_1" + )pb"))); +} + +TEST(RectTransformation, Scale) { + mediapipe::api2::builder::Graph graph; + + Stream rect = graph.In("RECT").Cast(); + Stream> size = + graph.In("SIZE").Cast>(); + Stream transformed_rect = + Scale(rect, size, /*scale_x_factor=*/2, /*scale_y_factor=*/7, graph); + transformed_rect.SetName("transformed_rect"); + + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_1" + input_stream: "NORM_RECT:__stream_0" + output_stream: "transformed_rect" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 2 + scale_y: 7 + } + } + } + input_stream: "RECT:__stream_0" + input_stream: "SIZE:__stream_1" + )pb"))); +} + +TEST(RectTransformation, ScaleAndShift) { + mediapipe::api2::builder::Graph graph; + + Stream rect = graph.In("RECT").Cast(); + Stream> size = + graph.In("SIZE").Cast>(); + Stream transformed_rect = + ScaleAndShift(rect, size, /*scale_x_factor=*/2, /*scale_y_factor=*/7, + /*shift_x=*/10, /*shift_y=*/0.5f, graph); + transformed_rect.SetName("transformed_rect"); + + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_1" + input_stream: "NORM_RECT:__stream_0" + output_stream: "transformed_rect" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 2 + scale_y: 7 + shift_x: 10 + shift_y: 0.5 + } + } + } + input_stream: "RECT:__stream_0" + input_stream: "SIZE:__stream_1" + )pb"))); +} + +TEST(RectTransformation, ScaleAndShiftAndMakeSquareLong) { + mediapipe::api2::builder::Graph graph; + + Stream rect = graph.In("RECT").Cast(); + Stream> size = + graph.In("SIZE").Cast>(); + Stream transformed_rect = ScaleAndShiftAndMakeSquareLong( + rect, size, /*scale_x_factor=*/2, /*scale_y_factor=*/7, + /*shift_x=*/10, /*shift_y=*/0.5f, graph); + transformed_rect.SetName("transformed_rect"); + + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_1" + input_stream: "NORM_RECT:__stream_0" + output_stream: "transformed_rect" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 2 + scale_y: 7 + shift_x: 10 + shift_y: 0.5 + square_long: true + } + } + } + input_stream: "RECT:__stream_0" + input_stream: "SIZE:__stream_1" + )pb"))); +} + +TEST(RectTransformation, ScaleAndShiftMultipleRects) { + mediapipe::api2::builder::Graph graph; + + Stream> rects = + graph.In("RECTS").Cast>(); + Stream> size = + graph.In("SIZE").Cast>(); + Stream> transformed_rects = + ScaleAndShift(rects, size, /*scale_x_factor=*/2, /*scale_y_factor=*/7, + /*shift_x=*/10, /*shift_y=*/0.5f, graph); + transformed_rects.SetName("transformed_rects"); + + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_1" + input_stream: "NORM_RECTS:__stream_0" + output_stream: "transformed_rects" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 2 + scale_y: 7 + shift_x: 10 + shift_y: 0.5 + } + } + } + input_stream: "RECTS:__stream_0" + input_stream: "SIZE:__stream_1" + )pb"))); +} + +TEST(RectTransformation, ScaleAndShiftAndMakeSquareLongMultipleRects) { + mediapipe::api2::builder::Graph graph; + + Stream> rects = + graph.In("RECTS").Cast>(); + Stream> size = + graph.In("SIZE").Cast>(); + Stream> transformed_rects = + ScaleAndShiftAndMakeSquareLong(rects, size, /*scale_x_factor=*/2, + /*scale_y_factor=*/7, + /*shift_x=*/10, /*shift_y=*/0.5f, graph); + transformed_rects.SetName("transformed_rects"); + + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "RectTransformationCalculator" + input_stream: "IMAGE_SIZE:__stream_1" + input_stream: "NORM_RECTS:__stream_0" + output_stream: "transformed_rects" + options { + [mediapipe.RectTransformationCalculatorOptions.ext] { + scale_x: 2 + scale_y: 7 + shift_x: 10 + shift_y: 0.5 + square_long: true + } + } + } + input_stream: "RECTS:__stream_0" + input_stream: "SIZE:__stream_1" + )pb"))); +} + +} // namespace +} // namespace mediapipe::api2::builder diff --git a/mediapipe/framework/calculator_base.h b/mediapipe/framework/calculator_base.h index 19f37f9d..1f4c8216 100644 --- a/mediapipe/framework/calculator_base.h +++ b/mediapipe/framework/calculator_base.h @@ -17,14 +17,16 @@ #ifndef MEDIAPIPE_FRAMEWORK_CALCULATOR_BASE_H_ #define MEDIAPIPE_FRAMEWORK_CALCULATOR_BASE_H_ +#include +#include #include #include "absl/memory/memory.h" +#include "absl/status/status.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_contract.h" #include "mediapipe/framework/deps/registration.h" #include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/timestamp.h" namespace mediapipe { @@ -150,8 +152,9 @@ class CalculatorBase { // Packets may be output during a call to Close(). However, output packets // are silently discarded if Close() is called after a graph run has ended. // - // NOTE: If Close() needs to perform an action only when processing is - // complete, Close() must check if cc->GraphStatus() is OK. + // NOTE: Do not call cc->GraphStatus() in Close() if you need to check if the + // processing is complete. Please, see CalculatorContext::GraphStatus + // documentation for the suggested solution. virtual absl::Status Close(CalculatorContext* cc) { return absl::OkStatus(); } // Returns a value according to which the framework selects diff --git a/mediapipe/framework/calculator_base_test.cc b/mediapipe/framework/calculator_base_test.cc index c26006e0..42c03696 100644 --- a/mediapipe/framework/calculator_base_test.cc +++ b/mediapipe/framework/calculator_base_test.cc @@ -183,8 +183,7 @@ TEST(CalculatorTest, CreateByNameWhitelisted) { CalculatorBaseRegistry::Register( "::mediapipe::test_ns::whitelisted_ns::DeadCalculator", absl::make_unique>, - __FILE__, __LINE__); + mediapipe::test_ns::whitelisted_ns::DeadCalculator>>); // A whitelisted calculator can be found in its own namespace. MP_EXPECT_OK(CalculatorBaseRegistry::CreateByNameInNamespace( // diff --git a/mediapipe/framework/calculator_context.cc b/mediapipe/framework/calculator_context.cc index 4452f45e..25f29222 100644 --- a/mediapipe/framework/calculator_context.cc +++ b/mediapipe/framework/calculator_context.cc @@ -14,35 +14,37 @@ #include "mediapipe/framework/calculator_context.h" +#include "absl/log/absl_check.h" + namespace mediapipe { const std::string& CalculatorContext::CalculatorType() const { - CHECK(calculator_state_); + ABSL_CHECK(calculator_state_); return calculator_state_->CalculatorType(); } const CalculatorOptions& CalculatorContext::Options() const { - CHECK(calculator_state_); + ABSL_CHECK(calculator_state_); return calculator_state_->Options(); } const std::string& CalculatorContext::NodeName() const { - CHECK(calculator_state_); + ABSL_CHECK(calculator_state_); return calculator_state_->NodeName(); } int CalculatorContext::NodeId() const { - CHECK(calculator_state_); + ABSL_CHECK(calculator_state_); return calculator_state_->NodeId(); } Counter* CalculatorContext::GetCounter(const std::string& name) { - CHECK(calculator_state_); + ABSL_CHECK(calculator_state_); return calculator_state_->GetCounter(name); } CounterFactory* CalculatorContext::GetCounterFactory() { - CHECK(calculator_state_); + ABSL_CHECK(calculator_state_); return calculator_state_->GetCounterFactory(); } diff --git a/mediapipe/framework/calculator_context.h b/mediapipe/framework/calculator_context.h index 284226d9..315d2651 100644 --- a/mediapipe/framework/calculator_context.h +++ b/mediapipe/framework/calculator_context.h @@ -20,6 +20,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/calculator_state.h" #include "mediapipe/framework/counter.h" #include "mediapipe/framework/graph_service.h" @@ -109,9 +110,20 @@ class CalculatorContext { // use OutputStream::SetOffset() directly. void SetOffset(TimestampDiff offset); - // Returns the status of the graph run. + // DEPRECATED: This was intended to get graph run status during + // `CalculatorBase::Close` call. However, `Close` can run simultaneously with + // other calculators `CalculatorBase::Process`, hence the actual graph + // status may change any time and returned graph status here does not + // necessarily reflect the actual graph status. // - // NOTE: This method should only be called during CalculatorBase::Close(). + // As an alternative, instead of checking graph status in `Close` and doing + // work for "done" state, you can enable timestamp bound processing for your + // calculator (`CalculatorContract::SetProcessTimestampBounds`) to trigger + // `Process` on timestamp bound updates and handle "done" state there. + // Check examples in: + // mediapipe/framework/calculator_graph_summary_packet_test.cc. + // + ABSL_DEPRECATED("Does not reflect the actual graph status.") absl::Status GraphStatus() const { return graph_status_; } ProfilingContext* GetProfilingContext() const { @@ -136,7 +148,7 @@ class CalculatorContext { } void PopInputTimestamp() { - CHECK(!input_timestamps_.empty()); + ABSL_CHECK(!input_timestamps_.empty()); input_timestamps_.pop(); } diff --git a/mediapipe/framework/calculator_context_manager.cc b/mediapipe/framework/calculator_context_manager.cc index acd70dd9..7da3d277 100644 --- a/mediapipe/framework/calculator_context_manager.cc +++ b/mediapipe/framework/calculator_context_manager.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/port/logging.h" @@ -27,7 +28,7 @@ void CalculatorContextManager::Initialize( std::shared_ptr input_tag_map, std::shared_ptr output_tag_map, bool calculator_run_in_parallel) { - CHECK(calculator_state); + ABSL_CHECK(calculator_state); calculator_state_ = calculator_state; input_tag_map_ = std::move(input_tag_map); output_tag_map_ = std::move(output_tag_map); @@ -51,15 +52,15 @@ void CalculatorContextManager::CleanupAfterRun() { CalculatorContext* CalculatorContextManager::GetDefaultCalculatorContext() const { - CHECK(default_context_.get()); + ABSL_CHECK(default_context_.get()); return default_context_.get(); } CalculatorContext* CalculatorContextManager::GetFrontCalculatorContext( Timestamp* context_input_timestamp) { - CHECK(calculator_run_in_parallel_); + ABSL_CHECK(calculator_run_in_parallel_); absl::MutexLock lock(&contexts_mutex_); - CHECK(!active_contexts_.empty()); + ABSL_CHECK(!active_contexts_.empty()); *context_input_timestamp = active_contexts_.begin()->first; return active_contexts_.begin()->second.get(); } @@ -70,7 +71,7 @@ CalculatorContext* CalculatorContextManager::PrepareCalculatorContext( return GetDefaultCalculatorContext(); } absl::MutexLock lock(&contexts_mutex_); - CHECK(!mediapipe::ContainsKey(active_contexts_, input_timestamp)) + ABSL_CHECK(!mediapipe::ContainsKey(active_contexts_, input_timestamp)) << "Multiple invocations with the same timestamps are not allowed with " "parallel execution, input_timestamp = " << input_timestamp; diff --git a/mediapipe/framework/calculator_context_manager.h b/mediapipe/framework/calculator_context_manager.h index 6b988b03..ae697e12 100644 --- a/mediapipe/framework/calculator_context_manager.h +++ b/mediapipe/framework/calculator_context_manager.h @@ -21,6 +21,7 @@ #include #include "absl/base/thread_annotations.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_state.h" @@ -97,18 +98,18 @@ class CalculatorContextManager { void PushInputTimestampToContext(CalculatorContext* calculator_context, Timestamp input_timestamp) { - CHECK(calculator_context); + ABSL_CHECK(calculator_context); calculator_context->PushInputTimestamp(input_timestamp); } void PopInputTimestampFromContext(CalculatorContext* calculator_context) { - CHECK(calculator_context); + ABSL_CHECK(calculator_context); calculator_context->PopInputTimestamp(); } void SetGraphStatusInContext(CalculatorContext* calculator_context, const absl::Status& status) { - CHECK(calculator_context); + ABSL_CHECK(calculator_context); calculator_context->SetGraphStatus(status); } diff --git a/mediapipe/framework/calculator_framework.h b/mediapipe/framework/calculator_framework.h index afb73fb3..8f193fde 100644 --- a/mediapipe/framework/calculator_framework.h +++ b/mediapipe/framework/calculator_framework.h @@ -52,6 +52,8 @@ #define MEDIAPIPE_FRAMEWORK_CALCULATOR_FRAMEWORK_H_ #include "mediapipe/framework/calculator_base.h" +#include "mediapipe/framework/calculator_context.h" +#include "mediapipe/framework/calculator_contract.h" #include "mediapipe/framework/calculator_graph.h" #include "mediapipe/framework/calculator_registry.h" #include "mediapipe/framework/counter_factory.h" diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc index 2a2088c6..03c5d229 100644 --- a/mediapipe/framework/calculator_graph.cc +++ b/mediapipe/framework/calculator_graph.cc @@ -17,14 +17,17 @@ #include #include +#include +#include #include #include -#include +#include #include #include -#include "absl/container/fixed_array.h" #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -37,9 +40,15 @@ #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/counter_factory.h" #include "mediapipe/framework/delegating_executor.h" +#include "mediapipe/framework/executor.h" +#include "mediapipe/framework/graph_output_stream.h" #include "mediapipe/framework/graph_service_manager.h" #include "mediapipe/framework/input_stream_manager.h" #include "mediapipe/framework/mediapipe_profiling.h" +#include "mediapipe/framework/output_side_packet_impl.h" +#include "mediapipe/framework/output_stream_manager.h" +#include "mediapipe/framework/output_stream_poller.h" +#include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet_generator.h" #include "mediapipe/framework/packet_generator.pb.h" #include "mediapipe/framework/packet_set.h" @@ -48,14 +57,18 @@ #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/core_proto_inc.h" #include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/map_util.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/source_location.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/framework/scheduler.h" #include "mediapipe/framework/status_handler.h" #include "mediapipe/framework/status_handler.pb.h" #include "mediapipe/framework/thread_pool_executor.h" #include "mediapipe/framework/thread_pool_executor.pb.h" +#include "mediapipe/framework/timestamp.h" #include "mediapipe/framework/tool/fill_packet_set.h" #include "mediapipe/framework/tool/status_util.h" #include "mediapipe/framework/tool/tag_map.h" @@ -75,6 +88,11 @@ namespace { constexpr int kMaxNumAccumulatedErrors = 1000; constexpr char kApplicationThreadExecutorType[] = "ApplicationThreadExecutor"; +// Do not log status payloads, but do include stack traces. +constexpr absl::StatusToStringMode kStatusLogFlags = + absl::StatusToStringMode::kWithEverything & + (~absl::StatusToStringMode::kWithPayload); + } // namespace void CalculatorGraph::ScheduleAllOpenableNodes() { @@ -127,10 +145,10 @@ CalculatorGraph::CalculatorGraph(CalculatorGraphConfig config) // they only need to be fully visible here, where their destructor is // instantiated. CalculatorGraph::~CalculatorGraph() { - // Stop periodic profiler output to ublock Executor destructors. + // Stop periodic profiler output to unblock Executor destructors. absl::Status status = profiler()->Stop(); if (!status.ok()) { - LOG(ERROR) << "During graph destruction: " << status; + ABSL_LOG(ERROR) << "During graph destruction: " << status; } } @@ -155,7 +173,7 @@ absl::Status CalculatorGraph::InitializePacketGeneratorGraph( Executor* default_executor = nullptr; if (!use_application_thread_) { default_executor = executors_[""].get(); - CHECK(default_executor); + ABSL_CHECK(default_executor); } // If default_executor is nullptr, then packet_generator_graph_ will create // its own DelegatingExecutor to use the application thread. @@ -174,6 +192,7 @@ absl::Status CalculatorGraph::InitializeStreams() { const EdgeInfo& edge_info = validated_graph_->InputStreamInfos()[index]; MP_RETURN_IF_ERROR(input_stream_managers_[index].Initialize( edge_info.name, edge_info.packet_type, edge_info.back_edge)); + input_stream_to_index_[&input_stream_managers_[index]] = index; } // Create and initialize the output streams. @@ -382,6 +401,7 @@ absl::Status CalculatorGraph::InitializeDefaultExecutor( "", std::make_shared( std::bind(&internal::Scheduler::AddApplicationThreadTask, &scheduler_, std::placeholders::_1)))); + VLOG(1) << "Using default executor and application thread."; return absl::OkStatus(); } @@ -401,6 +421,8 @@ absl::Status CalculatorGraph::InitializeDefaultExecutor( } MP_RETURN_IF_ERROR( CreateDefaultThreadPool(default_executor_options, num_threads)); + VLOG(1) << absl::StrCat("Using default executor with num_threads: ", + num_threads); return absl::OkStatus(); } @@ -579,7 +601,7 @@ absl::Status CalculatorGraph::MaybeSetUpGpuServiceFromLegacySidePacket( if (legacy_sp.IsEmpty()) return absl::OkStatus(); auto gpu_resources = service_manager_.GetServiceObject(kGpuService); if (gpu_resources) { - LOG(WARNING) + ABSL_LOG(WARNING) << "::mediapipe::GpuSharedData provided as a side packet while the " << "graph already had one; ignoring side packet"; return absl::OkStatus(); @@ -707,7 +729,7 @@ absl::Status CalculatorGraph::PrepareForRun( absl::Status error_status; if (has_error_) { GetCombinedErrors(&error_status); - LOG(ERROR) << error_status; + ABSL_LOG(ERROR) << error_status.ToString(kStatusLogFlags); return error_status; } @@ -786,7 +808,7 @@ absl::Status CalculatorGraph::PrepareForRun( } if (GetCombinedErrors(&error_status)) { - LOG(ERROR) << error_status; + ABSL_LOG(ERROR) << error_status.ToString(kStatusLogFlags); CleanupAfterRun(&error_status); return error_status; } @@ -840,14 +862,17 @@ absl::Status CalculatorGraph::PrepareForRun( absl::Status CalculatorGraph::WaitUntilIdle() { if (has_sources_) { - LOG(WARNING) << "WaitUntilIdle called on a graph with source nodes, which " - "is not fully supported at the moment."; + ABSL_LOG_FIRST_N(WARNING, 1) + << "WaitUntilIdle called on a graph with source nodes, which " + "is not fully supported at the moment. Source nodes: " + << ListSourceNodes(); } + MP_RETURN_IF_ERROR(scheduler_.WaitUntilIdle()); VLOG(2) << "Scheduler idle."; absl::Status status = absl::OkStatus(); if (GetCombinedErrors(&status)) { - LOG(ERROR) << status; + ABSL_LOG(ERROR) << status.ToString(kStatusLogFlags); } return status; } @@ -901,7 +926,7 @@ absl::Status CalculatorGraph::AddPacketToInputStreamInternal( "graph input stream.", stream_name); int node_id = mediapipe::FindOrDie(graph_input_stream_node_ids_, stream_name); - CHECK_GE(node_id, validated_graph_->CalculatorInfos().size()); + ABSL_CHECK_GE(node_id, validated_graph_->CalculatorInfos().size()); { absl::MutexLock lock(&full_input_streams_mutex_); if (full_input_streams_.empty()) { @@ -1040,17 +1065,17 @@ void CalculatorGraph::RecordError(const absl::Status& error) { } if (errors_.size() > kMaxNumAccumulatedErrors) { for (const absl::Status& error : errors_) { - LOG(ERROR) << error; + ABSL_LOG(ERROR) << error; } - LOG(FATAL) << "Forcefully aborting to prevent the framework running out " - "of memory."; + ABSL_LOG(FATAL) + << "Forcefully aborting to prevent the framework running out " + "of memory."; } } } bool CalculatorGraph::GetCombinedErrors(absl::Status* error_status) { - return GetCombinedErrors("CalculatorGraph::Run() failed in Run: ", - error_status); + return GetCombinedErrors("CalculatorGraph::Run() failed: ", error_status); } bool CalculatorGraph::GetCombinedErrors(const std::string& error_prefix, @@ -1089,7 +1114,8 @@ void CalculatorGraph::CallStatusHandlers(GraphRunState graph_run_state, absl::StatusOr> static_access_statusor = internal::StaticAccessToStatusHandlerRegistry:: CreateByNameInNamespace(validated_graph_->Package(), handler_type); - CHECK(static_access_statusor.ok()) << handler_type << " is not registered."; + ABSL_CHECK(static_access_statusor.ok()) + << handler_type << " is not registered."; auto static_access = std::move(static_access_statusor).value(); absl::Status handler_result; if (graph_run_state == GraphRunState::PRE_RUN) { @@ -1130,7 +1156,7 @@ void CalculatorGraph::UpdateThrottledNodes(InputStreamManager* stream, upstream_nodes = &validated_graph_->CalculatorInfos()[node_index].AncestorSources(); } - CHECK(upstream_nodes); + ABSL_CHECK(upstream_nodes); std::vector nodes_to_schedule; { @@ -1152,10 +1178,10 @@ void CalculatorGraph::UpdateThrottledNodes(InputStreamManager* stream, .set_stream_id(&stream->Name())); bool was_throttled = !full_input_streams_[node_id].empty(); if (stream_is_full) { - DCHECK_EQ(full_input_streams_[node_id].count(stream), 0); + ABSL_DCHECK_EQ(full_input_streams_[node_id].count(stream), 0); full_input_streams_[node_id].insert(stream); } else { - DCHECK_EQ(full_input_streams_[node_id].count(stream), 1); + ABSL_DCHECK_EQ(full_input_streams_[node_id].count(stream), 1); full_input_streams_[node_id].erase(stream); } @@ -1212,7 +1238,7 @@ bool CalculatorGraph::UnthrottleSources() { // NOTE: We can be sure that this function will grow input streams enough // to unthrottle at least one source node. The current stream queue sizes // will remain unchanged until at least one source node becomes unthrottled. - // This is a sufficient because succesfully growing at least one full input + // This is a sufficient because successfully growing at least one full input // stream during each call to UnthrottleSources will eventually resolve // each deadlock. absl::flat_hash_set full_streams; @@ -1232,7 +1258,8 @@ bool CalculatorGraph::UnthrottleSources() { for (InputStreamManager* stream : full_streams) { if (Config().report_deadlock()) { RecordError(absl::UnavailableError(absl::StrCat( - "Detected a deadlock due to input throttling for: \"", stream->Name(), + "Detected a deadlock due to input throttling for input stream: \"", + stream->Name(), "\" of a node \"", GetParentNodeDebugName(stream), "\". All calculators are idle while packet sources remain active " "and throttled. Consider adjusting \"max_queue_size\" or " "\"report_deadlock\"."))); @@ -1240,10 +1267,11 @@ bool CalculatorGraph::UnthrottleSources() { } int new_size = stream->QueueSize() + 1; stream->SetMaxQueueSize(new_size); - LOG_EVERY_N(WARNING, 100) - << "Resolved a deadlock by increasing max_queue_size of input stream: " - << stream->Name() << " to: " << new_size - << ". Consider increasing max_queue_size for better performance."; + ABSL_LOG_EVERY_N(WARNING, 100) << absl::StrCat( + "Resolved a deadlock by increasing max_queue_size of input stream: \"", + stream->Name(), "\" of a node \"", GetParentNodeDebugName(stream), + "\" to ", new_size, + ". Consider increasing max_queue_size for better performance."); } return !full_streams.empty(); } @@ -1337,7 +1365,7 @@ void CalculatorGraph::CleanupAfterRun(absl::Status* status) { // Obtain the combined status again, so that it includes the new errors // added by CallStatusHandlers. GetCombinedErrors(status); - CHECK(!status->ok()); + ABSL_CHECK(!status->ok()); } else { MEDIAPIPE_CHECK_OK(*status); } @@ -1372,6 +1400,37 @@ const OutputStreamManager* CalculatorGraph::FindOutputStreamManager( .get()[validated_graph_->OutputStreamIndex(name)]; } +std::string CalculatorGraph::ListSourceNodes() const { + std::vector sources; + for (auto& node : nodes_) { + if (node->IsSource()) { + sources.push_back(node->DebugName()); + } + } + return absl::StrJoin(sources, ", "); +} + +std::string CalculatorGraph::GetParentNodeDebugName( + InputStreamManager* stream) const { + auto iter = input_stream_to_index_.find(stream); + if (iter == input_stream_to_index_.end()) { + return absl::StrCat("Unknown (node with input stream: ", stream->Name(), + ")"); + } + + const int input_stream_index = iter->second; + const EdgeInfo& edge_info = + validated_graph_->InputStreamInfos()[input_stream_index]; + const int node_index = edge_info.parent_node.index; + const CalculatorGraphConfig& config = validated_graph_->Config(); + if (node_index < 0 || node_index >= config.node_size()) { + return absl::StrCat("Unknown (node index: ", node_index, + ", with input stream: ", stream->Name(), ")"); + } + + return DebugName(config.node(node_index)); +} + namespace { void PrintTimingToInfo(const std::string& label, int64_t timer_value) { const int64_t total_seconds = timer_value / 1000000ll; @@ -1380,12 +1439,13 @@ void PrintTimingToInfo(const std::string& label, int64_t timer_value) { const int64_t minutes = (total_seconds / 60ll) % 60ll; const int64_t seconds = total_seconds % 60ll; const int64_t milliseconds = (timer_value / 1000ll) % 1000ll; - LOG(INFO) << label << " took " - << absl::StrFormat( - "%02lld days, %02lld:%02lld:%02lld.%03lld (total seconds: " - "%lld.%06lld)", - days, hours, minutes, seconds, milliseconds, total_seconds, - timer_value % int64_t{1000000}); + ABSL_LOG(INFO) + << label << " took " + << absl::StrFormat( + "%02lld days, %02lld:%02lld:%02lld.%03lld (total seconds: " + "%lld.%06lld)", + days, hours, minutes, seconds, milliseconds, total_seconds, + timer_value % int64_t{1000000}); } bool MetricElementComparator(const std::pair& e1, diff --git a/mediapipe/framework/calculator_graph.h b/mediapipe/framework/calculator_graph.h index 748d2fb3..4284beb7 100644 --- a/mediapipe/framework/calculator_graph.h +++ b/mediapipe/framework/calculator_graph.h @@ -26,10 +26,12 @@ #include #include -#include "absl/base/macros.h" -#include "absl/container/fixed_array.h" +#include "absl/base/attributes.h" +#include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_base.h" @@ -41,18 +43,17 @@ #include "mediapipe/framework/graph_service_manager.h" #include "mediapipe/framework/mediapipe_profiling.h" #include "mediapipe/framework/output_side_packet_impl.h" -#include "mediapipe/framework/output_stream.h" #include "mediapipe/framework/output_stream_manager.h" #include "mediapipe/framework/output_stream_poller.h" #include "mediapipe/framework/output_stream_shard.h" #include "mediapipe/framework/packet.h" -#include "mediapipe/framework/packet_generator.pb.h" #include "mediapipe/framework/packet_generator_graph.h" -#include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/scheduler.h" +#include "mediapipe/framework/scheduler_shared.h" +#include "mediapipe/framework/subgraph.h" #include "mediapipe/framework/thread_pool_executor.pb.h" +#include "mediapipe/framework/timestamp.h" +#include "mediapipe/framework/validated_graph_config.h" namespace mediapipe { @@ -597,6 +598,12 @@ class CalculatorGraph { // status before taking any action. void UpdateThrottledNodes(InputStreamManager* stream, bool* stream_was_full); + // Returns a comma-separated list of source nodes. + std::string ListSourceNodes() const; + + // Returns a parent node name for the given input stream. + std::string GetParentNodeDebugName(InputStreamManager* stream) const; + #if !MEDIAPIPE_DISABLE_GPU // Owns the legacy GpuSharedData if we need to create one for backwards // compatibility. @@ -652,6 +659,9 @@ class CalculatorGraph { std::vector> full_input_streams_ ABSL_GUARDED_BY(full_input_streams_mutex_); + // Input stream to index within `input_stream_managers_` mapping. + absl::flat_hash_map input_stream_to_index_; + // Maps stream names to graph input stream objects. absl::flat_hash_map> graph_input_streams_; diff --git a/mediapipe/framework/calculator_graph_side_packet_test.cc b/mediapipe/framework/calculator_graph_side_packet_test.cc index a9567c80..6f42f585 100644 --- a/mediapipe/framework/calculator_graph_side_packet_test.cc +++ b/mediapipe/framework/calculator_graph_side_packet_test.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/time/clock.h" #include "absl/time/time.h" #include "mediapipe/framework/calculator.pb.h" @@ -24,7 +25,6 @@ #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -128,7 +128,7 @@ class IntegerOutputSidePacketCalculator : public CalculatorBase { } absl::Status Process(CalculatorContext* cc) final { - LOG(FATAL) << "Not reached."; + ABSL_LOG(FATAL) << "Not reached."; return absl::OkStatus(); } }; @@ -153,7 +153,7 @@ class SidePacketAdderCalculator : public CalculatorBase { } absl::Status Process(CalculatorContext* cc) final { - LOG(FATAL) << "Not reached."; + ABSL_LOG(FATAL) << "Not reached."; return absl::OkStatus(); } }; @@ -778,7 +778,7 @@ class OutputSidePacketCachedCalculator : public CalculatorBase { } absl::Status Process(CalculatorContext* cc) final { - LOG(FATAL) << "Not reached."; + ABSL_LOG(FATAL) << "Not reached."; return absl::OkStatus(); } }; diff --git a/mediapipe/framework/calculator_graph_summary_packet_test.cc b/mediapipe/framework/calculator_graph_summary_packet_test.cc new file mode 100644 index 00000000..e6a04e06 --- /dev/null +++ b/mediapipe/framework/calculator_graph_summary_packet_test.cc @@ -0,0 +1,430 @@ +#include "absl/status/status.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/api2/packet.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe { + +using ::mediapipe::api2::Input; +using ::mediapipe::api2::Node; +using ::mediapipe::api2::Output; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::IsEmpty; +using ::testing::Value; + +namespace { + +MATCHER_P2(IntPacket, value, timestamp, "") { + *result_listener << "where object is (value: " << arg.template Get() + << ", timestamp: " << arg.Timestamp() << ")"; + return Value(arg.template Get(), Eq(value)) && + Value(arg.Timestamp(), Eq(timestamp)); +} + +// Calculates and produces sum of all passed inputs when no more packets can be +// expected on the input stream. +class SummaryPacketCalculator : public Node { + public: + static constexpr Input kIn{"IN"}; + static constexpr Output kOut{"SUMMARY"}; + + MEDIAPIPE_NODE_CONTRACT(kIn, kOut); + + static absl::Status UpdateContract(CalculatorContract* cc) { + // Makes sure there are no automatic timestamp bound updates when Process + // is called. + cc->SetTimestampOffset(TimestampDiff::Unset()); + // Currently, only ImmediateInputStreamHandler supports "done" timestamp + // bound update. (ImmediateInputStreamhandler handles multiple input + // streams differently, so, in that case, calculator adjustments may be + // required.) + // TODO: update all input stream handlers to support "done" + // timestamp bound update. + cc->SetInputStreamHandler("ImmediateInputStreamHandler"); + // Enables processing timestamp bound updates. For this use case we are + // specifically interested in "done" timestamp bound update. (E.g. when + // all input packet sources are closed.) + cc->SetProcessTimestampBounds(true); + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext* cc) final { + if (!kIn(cc).IsEmpty()) { + value_ += kIn(cc).Get(); + value_set_ = true; + } + + if (kOut(cc).IsClosed()) { + // This can happen: + // 1. If, during previous invocation, kIn(cc).IsDone() == true (e.g. + // source calculator finished generating packets sent to kIn) and + // HasNextAllowedInStream() == true (which is an often case). + // 2. For Timestamp::PreStream, ImmediateInputStreamHandler will still + // invoke Process() with Timestamp::Max to indicate "Done" timestamp + // bound update. + return absl::OkStatus(); + } + + // TODO: input stream holding a packet with timestamp that has + // no next timestamp allowed in stream should always result in + // InputStream::IsDone() == true. + if (kIn(cc).IsDone() || !cc->InputTimestamp().HasNextAllowedInStream()) { + // `Process` may or may not be invoked for "done" timestamp bound when + // upstream calculator fails in `Close`. Hence, extra care is needed to + // identify whether the calculator needs to send output. + // TODO: remove when "done" timestamp bound flakiness fixed. + if (value_set_) { + // kOut(cc).Send(value_) can be used here as well, however in the case + // of source calculator sending inputs into kIn the resulting timestamp + // is not well defined (e.g. it can be the last packet timestamp or + // Timestamp::Max()) + // TODO: last packet from source should always result in + // InputStream::IsDone() == true. + kOut(cc).Send(value_, Timestamp::Max()); + } + kOut(cc).Close(); + } + return absl::OkStatus(); + } + + private: + int value_ = 0; + bool value_set_ = false; +}; +MEDIAPIPE_REGISTER_NODE(SummaryPacketCalculator); + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnClosingAllPacketSources) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: 'input' + node { + calculator: "SummaryPacketCalculator" + input_stream: 'IN:input' + output_stream: 'SUMMARY:output' + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp(10)); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + send_packet(20, Timestamp(11)); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max()))); +} + +TEST(SummaryPacketCalculatorUseCaseTest, ProducesSummaryPacketOnMaxTimestamp) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: 'input' + node { + calculator: "SummaryPacketCalculator" + input_stream: 'IN:input' + output_stream: 'SUMMARY:output' + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp(10)); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + send_packet(20, Timestamp::Max()); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max()))); + + output_packets.clear(); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, IsEmpty()); +} + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnPreStreamTimestamp) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: 'input' + node { + calculator: "SummaryPacketCalculator" + input_stream: 'IN:input' + output_stream: 'SUMMARY:output' + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp::PreStream()); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max()))); + + output_packets.clear(); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, IsEmpty()); +} + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnPostStreamTimestamp) { + std::vector output_packets; + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie(R"pb( + input_stream: 'input' + node { + calculator: "SummaryPacketCalculator" + input_stream: 'IN:input' + output_stream: 'SUMMARY:output' + } + )pb"); + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp::PostStream()); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(10, Timestamp::Max()))); + + output_packets.clear(); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, IsEmpty()); +} + +class IntGeneratorCalculator : public Node { + public: + static constexpr Output kOut{"INT"}; + + MEDIAPIPE_NODE_CONTRACT(kOut); + + absl::Status Process(CalculatorContext* cc) final { + kOut(cc).Send(20, Timestamp(0)); + kOut(cc).Send(10, Timestamp(1000)); + return tool::StatusStop(); + } +}; +MEDIAPIPE_REGISTER_NODE(IntGeneratorCalculator); + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnSourceCalculatorCompletion) { + std::vector output_packets; + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie(R"pb( + node { + calculator: "IntGeneratorCalculator" + output_stream: "INT:int_value" + } + node { + calculator: "SummaryPacketCalculator" + input_stream: "IN:int_value" + output_stream: "SUMMARY:output" + } + )pb"); + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_EXPECT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max()))); +} + +class EmitOnCloseCalculator : public Node { + public: + static constexpr Input kIn{"IN"}; + static constexpr Output kOut{"INT"}; + + MEDIAPIPE_NODE_CONTRACT(kIn, kOut); + + absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); } + + absl::Status Close(CalculatorContext* cc) final { + kOut(cc).Send(20, Timestamp(0)); + kOut(cc).Send(10, Timestamp(1000)); + return absl::OkStatus(); + } +}; +MEDIAPIPE_REGISTER_NODE(EmitOnCloseCalculator); + +TEST(SummaryPacketCalculatorUseCaseTest, + ProducesSummaryPacketOnAnotherCalculatorClosure) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: "input" + node { + calculator: "EmitOnCloseCalculator" + input_stream: "IN:input" + output_stream: "INT:int_value" + } + node { + calculator: "SummaryPacketCalculator" + input_stream: "IN:int_value" + output_stream: "SUMMARY:output" + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + MP_ASSERT_OK(graph.CloseInputStream("input")); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, ElementsAre(IntPacket(30, Timestamp::Max()))); + + output_packets.clear(); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); + EXPECT_THAT(output_packets, IsEmpty()); +} + +class FailureInCloseCalculator : public Node { + public: + static constexpr Input kIn{"IN"}; + static constexpr Output kOut{"INT"}; + + MEDIAPIPE_NODE_CONTRACT(kIn, kOut); + + absl::Status Process(CalculatorContext* cc) final { return absl::OkStatus(); } + + absl::Status Close(CalculatorContext* cc) final { + return absl::InternalError("error"); + } +}; +MEDIAPIPE_REGISTER_NODE(FailureInCloseCalculator); + +TEST(SummaryPacketCalculatorUseCaseTest, + DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInClose) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: "input" + node { + calculator: "FailureInCloseCalculator" + input_stream: "IN:input" + output_stream: "INT:int_value" + } + node { + calculator: "SummaryPacketCalculator" + input_stream: "IN:int_value" + output_stream: "SUMMARY:output" + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + MP_ASSERT_OK(graph.CloseInputStream("input")); + EXPECT_THAT(graph.WaitUntilIdle(), + StatusIs(absl::StatusCode::kInternal, HasSubstr("error"))); + EXPECT_THAT(output_packets, IsEmpty()); +} + +class FailureInProcessCalculator : public Node { + public: + static constexpr Input kIn{"IN"}; + static constexpr Output kOut{"INT"}; + + MEDIAPIPE_NODE_CONTRACT(kIn, kOut); + + absl::Status Process(CalculatorContext* cc) final { + return absl::InternalError("error"); + } +}; +MEDIAPIPE_REGISTER_NODE(FailureInProcessCalculator); + +TEST(SummaryPacketCalculatorUseCaseTest, + DoesNotProduceSummaryPacketWhenUpstreamCalculatorFailsInProcess) { + auto graph_config = ParseTextProtoOrDie(R"pb( + input_stream: "input" + node { + calculator: "FailureInProcessCalculator" + input_stream: "IN:input" + output_stream: "INT:int_value" + } + node { + calculator: "SummaryPacketCalculator" + input_stream: "IN:int_value" + output_stream: "SUMMARY:output" + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("output", &graph_config, &output_packets); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config, {})); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.WaitUntilIdle()); + EXPECT_THAT(output_packets, IsEmpty()); + + auto send_packet = [&graph](int value, Timestamp timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input", MakePacket(value).At(timestamp))); + }; + + send_packet(10, Timestamp::PostStream()); + EXPECT_THAT(graph.WaitUntilIdle(), + StatusIs(absl::StatusCode::kInternal, HasSubstr("error"))); + EXPECT_THAT(output_packets, IsEmpty()); +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/framework/calculator_graph_test.cc b/mediapipe/framework/calculator_graph_test.cc index 2e7d99ef..91bf72e3 100644 --- a/mediapipe/framework/calculator_graph_test.cc +++ b/mediapipe/framework/calculator_graph_test.cc @@ -17,16 +17,22 @@ #include #include +#include #include #include +#include #include #include +#include #include #include #include #include "absl/container/fixed_array.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" +#include "absl/status/status.h" #include "absl/strings/escaping.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -47,7 +53,6 @@ #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -725,13 +730,13 @@ class SlowCountingSinkCalculator : public CalculatorBase { absl::Status Process(CalculatorContext* cc) override { absl::SleepFor(absl::Milliseconds(10)); int value = cc->Inputs().Index(0).Get(); - CHECK_EQ(value, counter_); + ABSL_CHECK_EQ(value, counter_); ++counter_; return absl::OkStatus(); } absl::Status Close(CalculatorContext* cc) override { - CHECK_EQ(10, counter_); + ABSL_CHECK_EQ(10, counter_); return absl::OkStatus(); } @@ -1014,7 +1019,7 @@ class CheckInputTimestampSourceCalculator : public CalculatorBase { absl::Status Close(CalculatorContext* cc) final { // Must use CHECK instead of RET_CHECK in Close(), because the framework // may call the Close() method of a source node with .IgnoreError(). - CHECK_EQ(cc->InputTimestamp(), Timestamp::Done()); + ABSL_CHECK_EQ(cc->InputTimestamp(), Timestamp::Done()); return absl::OkStatus(); } @@ -1092,7 +1097,7 @@ class CheckInputTimestamp2SourceCalculator : public CalculatorBase { absl::Status Close(CalculatorContext* cc) final { // Must use CHECK instead of RET_CHECK in Close(), because the framework // may call the Close() method of a source node with .IgnoreError(). - CHECK_EQ(cc->InputTimestamp(), Timestamp::Done()); + ABSL_CHECK_EQ(cc->InputTimestamp(), Timestamp::Done()); return absl::OkStatus(); } @@ -1242,8 +1247,8 @@ REGISTER_STATUS_HANDLER(IncrementingStatusHandler); class CurrentThreadExecutor : public Executor { public: ~CurrentThreadExecutor() override { - CHECK(!executing_); - CHECK(tasks_.empty()); + ABSL_CHECK(!executing_); + ABSL_CHECK(tasks_.empty()); } void Schedule(std::function task) override { @@ -1254,7 +1259,7 @@ class CurrentThreadExecutor : public Executor { // running) to avoid an indefinitely-deep call stack. tasks_.emplace_back(std::move(task)); } else { - CHECK(tasks_.empty()); + ABSL_CHECK(tasks_.empty()); executing_ = true; task(); while (!tasks_.empty()) { @@ -1406,7 +1411,7 @@ void RunComprehensiveTest(CalculatorGraph* graph, // Call graph->Run() several times, to make sure that the appropriate // cleanup happens between iterations. for (int iteration = 0; iteration < 2; ++iteration) { - LOG(INFO) << "Loop iteration " << iteration; + ABSL_LOG(INFO) << "Loop iteration " << iteration; dumped_final_sum_packet = Packet(); dumped_final_stddev_packet = Packet(); dumped_final_packet = Packet(); @@ -1448,7 +1453,7 @@ void RunComprehensiveTest(CalculatorGraph* graph, ->GetCounter("copy_range5-PassThrough") ->Get()); } - LOG(INFO) << "After Loop Runs."; + ABSL_LOG(INFO) << "After Loop Runs."; // Verify that the graph can still run (but not successfully) when // one of the nodes is caused to fail. extra_side_packets.clear(); @@ -1459,9 +1464,9 @@ void RunComprehensiveTest(CalculatorGraph* graph, dumped_final_sum_packet = Packet(); dumped_final_stddev_packet = Packet(); dumped_final_packet = Packet(); - LOG(INFO) << "Expect an error to be logged here."; + ABSL_LOG(INFO) << "Expect an error to be logged here."; ASSERT_FALSE(graph->Run(extra_side_packets).ok()); - LOG(INFO) << "Error should have been logged."; + ABSL_LOG(INFO) << "Error should have been logged."; } TEST(CalculatorGraph, BadInitialization) { @@ -2549,6 +2554,129 @@ TEST(CalculatorGraph, OutputPacketInOpen2) { EXPECT_EQ(Timestamp(i), packet_dump[i].Timestamp()); } +TEST(CalculatorGraph, DeadlockIsReportedAndSufficientInfoProvided) { + CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie(R"pb( + report_deadlock: true + max_queue_size: 1 + input_stream: 'input1' + input_stream: 'input2' + node { + calculator: 'PassThroughCalculator' + input_stream: 'input1' + input_stream: 'input2' + output_stream: 'output1' + output_stream: 'output2' + } + )pb"); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(config)); + MP_ASSERT_OK(graph.StartRun({})); + + Packet packet = MakePacket(1); + MP_EXPECT_OK(graph.AddPacketToInputStream("input1", packet.At(Timestamp(0)))); + absl::Status status = + graph.AddPacketToInputStream("input1", packet.At(Timestamp(1))); + + EXPECT_EQ(status.code(), absl::StatusCode::kUnavailable); + EXPECT_THAT(status.message(), + testing::AllOf(testing::HasSubstr("deadlock"), + testing::HasSubstr("input1"), + testing::HasSubstr("PassThroughCalculator"))); + graph.Cancel(); +} + +TEST(CalculatorGraph, + DeadlockIsReportedAndSufficientInfoProvidedMultipleCalculators) { + CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie(R"pb( + report_deadlock: true + max_queue_size: 1 + input_stream: 'input1' + input_stream: 'input2' + node { + calculator: 'PassThroughCalculator' + input_stream: 'input1' + input_stream: 'input2' + output_stream: 'output1' + output_stream: 'output2' + } + node { + calculator: 'MergeCalculator' + input_stream: 'output1' + input_stream: 'output2' + output_stream: 'output3' + } + )pb"); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(config)); + MP_ASSERT_OK(graph.StartRun({})); + + Packet packet = MakePacket(1); + MP_EXPECT_OK(graph.AddPacketToInputStream("input1", packet.At(Timestamp(0)))); + absl::Status status = + graph.AddPacketToInputStream("input1", packet.At(Timestamp(1))); + + EXPECT_EQ(status.code(), absl::StatusCode::kUnavailable); + EXPECT_THAT(status.message(), + testing::AllOf(testing::HasSubstr("deadlock"), + testing::HasSubstr("input1"), + testing::HasSubstr("PassThroughCalculator"))); + graph.Cancel(); +} + +TEST(CalculatorGraph, TwoDeadlocksAreReportedAndSufficientInfoProvided) { + CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie(R"pb( + report_deadlock: true + max_queue_size: 1 + input_stream: 'input1' + input_stream: 'input2' + node { + calculator: 'PassThroughCalculator' + input_stream: 'input1' + input_stream: 'input2' + output_stream: 'output1' + output_stream: 'output2' + } + node { + calculator: 'PassThroughCalculator' + input_stream: 'output1' + input_stream: 'output2' + output_stream: 'output3' + output_stream: 'output4' + } + node { + calculator: 'MergeCalculator' + input_stream: 'input1' + input_stream: 'output1' + input_stream: 'output2' + input_stream: 'output3' + input_stream: 'output4' + output_stream: 'output5' + } + )pb"); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(config)); + MP_ASSERT_OK(graph.StartRun({})); + + Packet packet = MakePacket(1); + MP_EXPECT_OK(graph.AddPacketToInputStream("input1", packet.At(Timestamp(0)))); + absl::Status status = + graph.AddPacketToInputStream("input1", packet.At(Timestamp(1))); + + EXPECT_EQ(status.code(), absl::StatusCode::kUnavailable); + EXPECT_THAT(status.message(), + testing::AllOf(testing::HasSubstr("deadlock"), + testing::HasSubstr("input1"), + testing::HasSubstr("PassThroughCalculator"), + testing::HasSubstr("MergeCalculator"))); + graph.Cancel(); +} + // Tests that no packets are available on input streams in Open(), even if the // upstream calculator outputs a packet in Open(). TEST(CalculatorGraph, EmptyInputInOpen) { @@ -2619,7 +2747,7 @@ TEST(CalculatorGraph, UnthrottleRespectsLayers) { std::map input_side_packets; input_side_packets["global_counter"] = Adopt(new auto(&global_counter)); // TODO: Set this value to true. When the calculator outputs a - // packet in Open, it will trigget b/33568859, and the test will fail. Use + // packet in Open, it will trigger b/33568859, and the test will fail. Use // this test to verify that b/33568859 is fixed. constexpr bool kOutputInOpen = true; input_side_packets["output_in_open"] = MakePacket(kOutputInOpen); @@ -3339,7 +3467,7 @@ TEST(CalculatorGraph, SetInputStreamMaxQueueSizeWorksSlowCalculator) { // Verify the scheduler unthrottles the graph input stream to avoid a deadlock, // and won't enter a busy loop. TEST(CalculatorGraph, AddPacketNoBusyLoop) { - // The DecimatorCalculator ouputs 1 out of every 101 input packets and drops + // The DecimatorCalculator outputs 1 out of every 101 input packets and drops // the rest, without setting the next timestamp bound on its output. As a // result, the MergeCalculator is not runnable in between and packets on its // "in" input stream will be queued and exceed the max queue size. @@ -3467,7 +3595,7 @@ REGISTER_CALCULATOR(::mediapipe::nested_ns::ProcessCallbackCalculator); TEST(CalculatorGraph, CalculatorInNamepsace) { CalculatorGraphConfig config; - CHECK(proto_ns::TextFormat::ParseFromString(R"( + ABSL_CHECK(proto_ns::TextFormat::ParseFromString(R"( input_stream: 'in_a' node { calculator: 'mediapipe.nested_ns.ProcessCallbackCalculator' @@ -3476,7 +3604,7 @@ TEST(CalculatorGraph, CalculatorInNamepsace) { input_side_packet: 'callback_1' } )", - &config)); + &config)); CalculatorGraph graph; MP_ASSERT_OK(graph.Initialize(config)); nested_ns::ProcessFunction callback_1; diff --git a/mediapipe/framework/calculator_node.cc b/mediapipe/framework/calculator_node.cc index f6a1c7db..c0aff3b1 100644 --- a/mediapipe/framework/calculator_node.cc +++ b/mediapipe/framework/calculator_node.cc @@ -19,6 +19,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -59,7 +61,7 @@ const PacketType* GetPacketType(const PacketTypeSet& packet_type_set, } else { id = packet_type_set.GetId(tag, 0); } - CHECK(id.IsValid()) << "Internal mediapipe error."; + ABSL_CHECK(id.IsValid()) << "Internal mediapipe error."; return &packet_type_set.Get(id); } @@ -341,7 +343,7 @@ absl::Status CalculatorNode::ConnectShardsToStreams( void CalculatorNode::SetExecutor(const std::string& executor) { absl::MutexLock status_lock(&status_mutex_); - CHECK_LT(status_, kStateOpened); + ABSL_CHECK_LT(status_, kStateOpened); executor_ = executor; } @@ -366,7 +368,7 @@ bool CalculatorNode::Closed() const { } void CalculatorNode::SetMaxInputStreamQueueSize(int max_queue_size) { - CHECK(input_stream_handler_); + ABSL_CHECK(input_stream_handler_); input_stream_handler_->SetMaxQueueSize(max_queue_size); } @@ -506,7 +508,7 @@ absl::Status CalculatorNode::OpenNode() { Timestamp(0)); } - LOG_IF(FATAL, result == tool::StatusStop()) << absl::Substitute( + ABSL_LOG_IF(FATAL, result == tool::StatusStop()) << absl::Substitute( "Open() on node \"$0\" returned tool::StatusStop() which should only be " "used to signal that a source node is done producing data.", DebugName()); @@ -519,7 +521,7 @@ absl::Status CalculatorNode::OpenNode() { offset_enabled = offset_enabled || stream->Spec()->offset_enabled; } if (offset_enabled && input_stream_handler_->SyncSetCount() > 1) { - LOG(WARNING) << absl::Substitute( + ABSL_LOG(WARNING) << absl::Substitute( "Calculator node \"$0\" is configured with multiple input sync-sets " "and an output timestamp-offset, which will often conflict due to " "the order of packet arrival. With multiple input sync-sets, use " @@ -539,7 +541,7 @@ absl::Status CalculatorNode::OpenNode() { void CalculatorNode::ActivateNode() { absl::MutexLock status_lock(&status_mutex_); - CHECK_EQ(status_, kStateOpened) << DebugName(); + ABSL_CHECK_EQ(status_, kStateOpened) << DebugName(); status_ = kStateActive; } @@ -601,7 +603,7 @@ absl::Status CalculatorNode::CloseNode(const absl::Status& graph_status, } needs_to_close_ = false; - LOG_IF(FATAL, result == tool::StatusStop()) << absl::Substitute( + ABSL_LOG_IF(FATAL, result == tool::StatusStop()) << absl::Substitute( "Close() on node \"$0\" returned tool::StatusStop() which should only be " "used to signal that a source node is done producing data.", DebugName()); @@ -694,8 +696,8 @@ void CalculatorNode::InputStreamHeadersReady() { bool ready_for_open = false; { absl::MutexLock lock(&status_mutex_); - CHECK_EQ(status_, kStatePrepared) << DebugName(); - CHECK(!input_stream_headers_ready_called_); + ABSL_CHECK_EQ(status_, kStatePrepared) << DebugName(); + ABSL_CHECK(!input_stream_headers_ready_called_); input_stream_headers_ready_called_ = true; input_stream_headers_ready_ = true; ready_for_open = input_side_packets_ready_; @@ -709,8 +711,8 @@ void CalculatorNode::InputSidePacketsReady() { bool ready_for_open = false; { absl::MutexLock lock(&status_mutex_); - CHECK_EQ(status_, kStatePrepared) << DebugName(); - CHECK(!input_side_packets_ready_called_); + ABSL_CHECK_EQ(status_, kStatePrepared) << DebugName(); + ABSL_CHECK(!input_side_packets_ready_called_); input_side_packets_ready_called_ = true; input_side_packets_ready_ = true; ready_for_open = input_stream_headers_ready_; @@ -760,7 +762,7 @@ void CalculatorNode::EndScheduling() { return; } --current_in_flight_; - CHECK_GE(current_in_flight_, 0); + ABSL_CHECK_GE(current_in_flight_, 0); if (scheduling_state_ == kScheduling) { // Changes the state to scheduling pending if another thread is doing the @@ -790,7 +792,7 @@ std::string CalculatorNode::DebugInputStreamNames() const { } std::string CalculatorNode::DebugName() const { - DCHECK(calculator_state_); + ABSL_DCHECK(calculator_state_); return calculator_state_->NodeName(); } @@ -893,9 +895,9 @@ absl::Status CalculatorNode::ProcessNode( // open input streams for Process(). So this node needs to be closed // too. // If the streams are closed, there shouldn't be more input. - CHECK_EQ(calculator_context_manager_.NumberOfContextTimestamps( - *calculator_context), - 1); + ABSL_CHECK_EQ(calculator_context_manager_.NumberOfContextTimestamps( + *calculator_context), + 1); return CloseNode(absl::OkStatus(), /*graph_run_ended=*/false); } else { RET_CHECK_FAIL() @@ -910,7 +912,7 @@ absl::Status CalculatorNode::ProcessNode( void CalculatorNode::SetQueueSizeCallbacks( InputStreamManager::QueueSizeCallback becomes_full_callback, InputStreamManager::QueueSizeCallback becomes_not_full_callback) { - CHECK(input_stream_handler_); + ABSL_CHECK(input_stream_handler_); input_stream_handler_->SetQueueSizeCallbacks( std::move(becomes_full_callback), std::move(becomes_not_full_callback)); } diff --git a/mediapipe/framework/calculator_node_test.cc b/mediapipe/framework/calculator_node_test.cc index 1c62a714..deac61f1 100644 --- a/mediapipe/framework/calculator_node_test.cc +++ b/mediapipe/framework/calculator_node_test.cc @@ -18,11 +18,12 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_macros.h" @@ -95,7 +96,8 @@ int CountCalculator::num_destroyed_ = 0; void SourceNodeOpenedNoOp() {} void CheckFail(const absl::Status& status) { - LOG(FATAL) << "The test triggered the error callback with status: " << status; + ABSL_LOG(FATAL) << "The test triggered the error callback with status: " + << status; } class CalculatorNodeTest : public ::testing::Test { @@ -103,7 +105,7 @@ class CalculatorNodeTest : public ::testing::Test { void ReadyForOpen(int* count) { ++(*count); } void Notification(CalculatorContext* cc, int* count) { - CHECK(cc); + ABSL_CHECK(cc); cc_ = cc; ++(*count); } diff --git a/mediapipe/framework/calculator_runner.cc b/mediapipe/framework/calculator_runner.cc index 1bd3211e..800f041c 100644 --- a/mediapipe/framework/calculator_runner.cc +++ b/mediapipe/framework/calculator_runner.cc @@ -16,10 +16,11 @@ #include "mediapipe/framework/calculator_runner.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -139,7 +140,7 @@ CalculatorRunner::CalculatorRunner(const std::string& calculator_type, #if !defined(MEDIAPIPE_PROTO_LITE) CalculatorRunner::CalculatorRunner(const std::string& node_config_string) { CalculatorGraphConfig::Node node_config; - CHECK( + ABSL_CHECK( proto_ns::TextFormat::ParseFromString(node_config_string, &node_config)); MEDIAPIPE_CHECK_OK(InitializeFromNodeConfig(node_config)); } @@ -149,8 +150,8 @@ CalculatorRunner::CalculatorRunner(const std::string& calculator_type, int num_inputs, int num_outputs, int num_side_packets) { node_config_.set_calculator(calculator_type); - CHECK(proto_ns::TextFormat::ParseFromString(options_string, - node_config_.mutable_options())); + ABSL_CHECK(proto_ns::TextFormat::ParseFromString( + options_string, node_config_.mutable_options())); SetNumInputs(num_inputs); SetNumOutputs(num_outputs); SetNumInputSidePackets(num_side_packets); @@ -188,7 +189,7 @@ void CalculatorRunner::SetNumInputSidePackets(int n) { } void CalculatorRunner::InitializeInputs(const tool::TagAndNameInfo& info) { - CHECK(graph_ == nullptr); + ABSL_CHECK(graph_ == nullptr); MEDIAPIPE_CHECK_OK( tool::SetFromTagAndNameInfo(info, node_config_.mutable_input_stream())); inputs_.reset(new StreamContentsSet(info)); @@ -196,7 +197,7 @@ void CalculatorRunner::InitializeInputs(const tool::TagAndNameInfo& info) { } void CalculatorRunner::InitializeOutputs(const tool::TagAndNameInfo& info) { - CHECK(graph_ == nullptr); + ABSL_CHECK(graph_ == nullptr); MEDIAPIPE_CHECK_OK( tool::SetFromTagAndNameInfo(info, node_config_.mutable_output_stream())); outputs_.reset(new StreamContentsSet(info)); @@ -205,7 +206,7 @@ void CalculatorRunner::InitializeOutputs(const tool::TagAndNameInfo& info) { void CalculatorRunner::InitializeInputSidePackets( const tool::TagAndNameInfo& info) { - CHECK(graph_ == nullptr); + ABSL_CHECK(graph_ == nullptr); MEDIAPIPE_CHECK_OK(tool::SetFromTagAndNameInfo( info, node_config_.mutable_input_side_packet())); input_side_packets_.reset(new PacketSet(info)); @@ -262,16 +263,18 @@ absl::Status CalculatorRunner::BuildGraph() { if (log_calculator_proto_) { #if defined(MEDIAPIPE_PROTO_LITE) - LOG(INFO) << "Please initialize CalculatorRunner using the recommended " - "constructor:\n CalculatorRunner runner(node_config);"; + ABSL_LOG(INFO) + << "Please initialize CalculatorRunner using the recommended " + "constructor:\n CalculatorRunner runner(node_config);"; #else std::string config_string; proto_ns::TextFormat::Printer printer; printer.SetInitialIndentLevel(4); printer.PrintToString(node_config_, &config_string); - LOG(INFO) << "Please initialize CalculatorRunner using the recommended " - "constructor:\n CalculatorRunner runner(R\"(\n" - << config_string << "\n )\");"; + ABSL_LOG(INFO) + << "Please initialize CalculatorRunner using the recommended " + "constructor:\n CalculatorRunner runner(R\"(\n" + << config_string << "\n )\");"; #endif } diff --git a/mediapipe/framework/calculator_runner_test.cc b/mediapipe/framework/calculator_runner_test.cc index a7890bad..7fd118cc 100644 --- a/mediapipe/framework/calculator_runner_test.cc +++ b/mediapipe/framework/calculator_runner_test.cc @@ -16,6 +16,7 @@ #include "mediapipe/framework/calculator_runner.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/calculator_registry.h" @@ -24,7 +25,6 @@ #include "mediapipe/framework/packet_type.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/framework/timestamp.h" @@ -136,7 +136,7 @@ TEST(CalculatorRunner, RunsCalculator) { // Run CalculatorRunner::Run() several times, with different inputs. This // tests that a CalculatorRunner instance can be reused. for (int iter = 0; iter < 3; ++iter) { - LOG(INFO) << "iter: " << iter; + ABSL_LOG(INFO) << "iter: " << iter; const int length = iter; // Generate the inputs at timestamps 0 ... length-1, at timestamp t having // values t and t*2 for the two streams, respectively. diff --git a/mediapipe/framework/calculator_state.cc b/mediapipe/framework/calculator_state.cc index 3b0264e9..9ff47868 100644 --- a/mediapipe/framework/calculator_state.cc +++ b/mediapipe/framework/calculator_state.cc @@ -18,6 +18,7 @@ #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/port/logging.h" @@ -46,23 +47,23 @@ void CalculatorState::ResetBetweenRuns() { } void CalculatorState::SetInputSidePackets(const PacketSet* input_side_packets) { - CHECK(input_side_packets); + ABSL_CHECK(input_side_packets); input_side_packets_ = input_side_packets; } void CalculatorState::SetOutputSidePackets( OutputSidePacketSet* output_side_packets) { - CHECK(output_side_packets); + ABSL_CHECK(output_side_packets); output_side_packets_ = output_side_packets; } Counter* CalculatorState::GetCounter(const std::string& name) { - CHECK(counter_factory_); + ABSL_CHECK(counter_factory_); return counter_factory_->GetCounter(absl::StrCat(NodeName(), "-", name)); } CounterFactory* CalculatorState::GetCounterFactory() { - CHECK(counter_factory_); + ABSL_CHECK(counter_factory_); return counter_factory_; } diff --git a/mediapipe/framework/collection.h b/mediapipe/framework/collection.h index c7b6fb0d..d955c9cb 100644 --- a/mediapipe/framework/collection.h +++ b/mediapipe/framework/collection.h @@ -24,11 +24,12 @@ #include #include "absl/base/macros.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/collection_item_id.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/tool/tag_map.h" #include "mediapipe/framework/tool/tag_map_helper.h" #include "mediapipe/framework/tool/validate_name.h" @@ -52,7 +53,7 @@ struct CollectionErrorHandlerFatal { // get away with only one version of this function (which is const // but returns a non-const reference). T& GetFallback(const absl::string_view tag, int index) const { - LOG(FATAL) << "Failed to get tag \"" << tag << "\" index " << index; + ABSL_LOG(FATAL) << "Failed to get tag \"" << tag << "\" index " << index; std::abort(); } }; @@ -365,7 +366,7 @@ class Collection { std::unique_ptr data_; // A class which allows errors to be reported flexibly. The default - // instantiation performs a LOG(FATAL) and does not have any member + // instantiation performs a ABSL_LOG(FATAL) and does not have any member // variables (zero size). ErrorHandler error_handler_; }; @@ -413,16 +414,16 @@ bool Collection::UsesTags() const { template typename Collection::value_type& Collection::Get(CollectionItemId id) { - CHECK_LE(BeginId(), id); - CHECK_LT(id, EndId()); + ABSL_CHECK_LE(BeginId(), id); + ABSL_CHECK_LT(id, EndId()); return begin()[id.value()]; } template const typename Collection::value_type& Collection::Get(CollectionItemId id) const { - CHECK_LE(BeginId(), id); - CHECK_LT(id, EndId()); + ABSL_CHECK_LE(BeginId(), id); + ABSL_CHECK_LT(id, EndId()); return begin()[id.value()]; } @@ -433,8 +434,8 @@ Collection::GetPtr(CollectionItemId id) { "mediapipe::internal::Collection::GetPtr() is only " "available for collections that were defined with template " "argument storage == CollectionStorage::kStorePointer."); - CHECK_LE(BeginId(), id); - CHECK_LT(id, EndId()); + ABSL_CHECK_LE(BeginId(), id); + ABSL_CHECK_LT(id, EndId()); return data_[id.value()]; } @@ -445,8 +446,8 @@ Collection::GetPtr(CollectionItemId id) const { "mediapipe::internal::Collection::GetPtr() is only " "available for collections that were defined with template " "argument storage == CollectionStorage::kStorePointer."); - CHECK_LE(BeginId(), id); - CHECK_LT(id, EndId()); + ABSL_CHECK_LE(BeginId(), id); + ABSL_CHECK_LT(id, EndId()); return data_[id.value()]; } diff --git a/mediapipe/framework/counter_factory.cc b/mediapipe/framework/counter_factory.cc index 895b44ea..b4da1043 100644 --- a/mediapipe/framework/counter_factory.cc +++ b/mediapipe/framework/counter_factory.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_log.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" @@ -59,9 +60,9 @@ void CounterSet::PublishCounters() ABSL_LOCKS_EXCLUDED(mu_) {} void CounterSet::PrintCounters() ABSL_LOCKS_EXCLUDED(mu_) { absl::ReaderMutexLock lock(&mu_); - LOG_IF(INFO, !counters_.empty()) << "MediaPipe Counters:"; + ABSL_LOG_IF(INFO, !counters_.empty()) << "MediaPipe Counters:"; for (const auto& counter : counters_) { - LOG(INFO) << counter.first << ": " << counter.second->Get(); + ABSL_LOG(INFO) << counter.first << ": " << counter.second->Get(); } } diff --git a/mediapipe/framework/deps/BUILD b/mediapipe/framework/deps/BUILD index 7fe37bae..6b670952 100644 --- a/mediapipe/framework/deps/BUILD +++ b/mediapipe/framework/deps/BUILD @@ -77,8 +77,9 @@ cc_library( ], visibility = ["//visibility:public"], deps = [ - "//mediapipe/framework/port:logging", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", ], @@ -130,8 +131,9 @@ cc_library( deps = [ "//mediapipe/framework/port", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -149,7 +151,10 @@ cc_library( # Use this library through "mediapipe/framework/port:map_util". visibility = ["//mediapipe/framework/port:__pkg__"], - deps = ["//mediapipe/framework/port:logging"], + deps = [ + "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", + ], ) cc_library( @@ -160,7 +165,7 @@ cc_library( ], deps = [ "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", ], ) @@ -228,12 +233,13 @@ cc_library( ], deps = [ ":registration_token", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/meta:type_traits", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", @@ -276,8 +282,8 @@ cc_library( visibility = ["//mediapipe/framework/port:__pkg__"], deps = [ ":source_location", - "//mediapipe/framework/port:logging", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", @@ -344,6 +350,8 @@ cc_library( deps = [ ":thread_options", "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -358,6 +366,8 @@ cc_library( visibility = ["//mediapipe/framework/port:__pkg__"], deps = [ "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -369,7 +379,7 @@ cc_library( visibility = ["//mediapipe/framework/port:__pkg__"], deps = [ "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/utility", ], ) @@ -415,10 +425,10 @@ cc_test( ":clock", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:threadpool", "//mediapipe/framework/tool:simulation_clock", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", diff --git a/mediapipe/framework/deps/cleanup.h b/mediapipe/framework/deps/cleanup.h index 125cc740..0541e314 100644 --- a/mediapipe/framework/deps/cleanup.h +++ b/mediapipe/framework/deps/cleanup.h @@ -26,7 +26,7 @@ // DataObject d; // while (ReadDataObject(fp, &d)) { // if (d.IsBad()) { -// LOG(ERROR) << "Bad Data"; +// ABSL_LOG(ERROR) << "Bad Data"; // return; // } // PushGoodData(d); diff --git a/mediapipe/framework/deps/clock.cc b/mediapipe/framework/deps/clock.cc index f6814386..418d8281 100644 --- a/mediapipe/framework/deps/clock.cc +++ b/mediapipe/framework/deps/clock.cc @@ -14,8 +14,8 @@ #include "mediapipe/framework/deps/clock.h" +#include "absl/log/absl_log.h" #include "absl/time/clock.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -28,7 +28,7 @@ namespace { class RealTimeClock : public Clock { public: virtual ~RealTimeClock() { - LOG(FATAL) << "RealTimeClock should never be destroyed"; + ABSL_LOG(FATAL) << "RealTimeClock should never be destroyed"; } absl::Time TimeNow() override { return absl::Now(); } diff --git a/mediapipe/framework/deps/map_util.h b/mediapipe/framework/deps/map_util.h index 05d47b7e..940ff03f 100644 --- a/mediapipe/framework/deps/map_util.h +++ b/mediapipe/framework/deps/map_util.h @@ -27,6 +27,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -53,7 +54,7 @@ template const typename M::value_type::second_type& FindOrDie( const M& m, const typename M::value_type::first_type& key) { auto it = m.find(key); - CHECK(it != m.end()) << "Map key not found: " << key; + ABSL_CHECK(it != m.end()) << "Map key not found: " << key; return it->second; } @@ -63,7 +64,7 @@ typename M::value_type::second_type& FindOrDie( M& m, // NOLINT const typename M::value_type::first_type& key) { auto it = m.find(key); - CHECK(it != m.end()) << "Map key not found: " << key; + ABSL_CHECK(it != m.end()) << "Map key not found: " << key; return it->second; } @@ -138,7 +139,7 @@ bool InsertIfNotPresent(M* m, const typename M::value_type::first_type& key, // inserted. template bool ReverseMap(const M& m, ReverseM* reverse) { - CHECK(reverse != nullptr); + ABSL_CHECK(reverse != nullptr); for (const auto& kv : m) { if (!InsertIfNotPresent(reverse, kv.second, kv.first)) { return false; diff --git a/mediapipe/framework/deps/mathutil.h b/mediapipe/framework/deps/mathutil.h index 315b78c4..a3d8b6e8 100644 --- a/mediapipe/framework/deps/mathutil.h +++ b/mediapipe/framework/deps/mathutil.h @@ -23,8 +23,8 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -354,7 +354,7 @@ class MathUtil { template // T models LessThanComparable. static const T& Clamp(const T& low, const T& high, const T& value) { // Prevents errors in ordering the arguments. - DCHECK(!(high < low)); + ABSL_DCHECK(!(high < low)); if (high < value) return high; if (value < low) return low; return value; @@ -364,7 +364,7 @@ class MathUtil { // absolute margin of error. template static bool WithinMargin(const T x, const T y, const T margin) { - DCHECK_GE(margin, 0); + ABSL_DCHECK_GE(margin, 0); return (std::abs(x) <= std::abs(y) + margin) && (std::abs(x) >= std::abs(y) - margin); } diff --git a/mediapipe/framework/deps/monotonic_clock.cc b/mediapipe/framework/deps/monotonic_clock.cc index 503ef5cf..17542b6f 100644 --- a/mediapipe/framework/deps/monotonic_clock.cc +++ b/mediapipe/framework/deps/monotonic_clock.cc @@ -16,9 +16,10 @@ #include "absl/base/macros.h" #include "absl/base/thread_annotations.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" #include "absl/time/time.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -60,7 +61,7 @@ class MonotonicClockImpl : public MonotonicClock { // Absolve this object of responsibility for state_. void ReleaseState() { - CHECK(state_owned_); + ABSL_CHECK(state_owned_); state_owned_ = false; } @@ -80,7 +81,7 @@ class MonotonicClockImpl : public MonotonicClock { absl::MutexLock m(&state_->lock); // Check consistency of internal data with state_. - CHECK_LE(last_raw_time_, state_->max_time) + ABSL_CHECK_LE(last_raw_time_, state_->max_time) << "non-monotonic behavior: last_raw_time_=" << last_raw_time_ << ", max_time=" << state_->max_time; @@ -107,7 +108,7 @@ class MonotonicClockImpl : public MonotonicClock { // First, update correction metrics. ++correction_count_; absl::Duration delta = state_->max_time - raw_time; - CHECK_LT(absl::ZeroDuration(), delta); + ABSL_CHECK_LT(absl::ZeroDuration(), delta); if (delta > max_correction_) { max_correction_ = delta; } @@ -205,7 +206,7 @@ MonotonicClock* MonotonicClock::CreateSynchronizedMonotonicClock() { // Test access methods. void MonotonicClockAccess::SynchronizedMonotonicClockReset() { - LOG(INFO) << "Resetting SynchronizedMonotonicClock"; + ABSL_LOG(INFO) << "Resetting SynchronizedMonotonicClock"; State* sync_state = GlobalSyncState(); absl::MutexLock m(&sync_state->lock); sync_state->max_time = absl::UnixEpoch(); diff --git a/mediapipe/framework/deps/monotonic_clock_test.cc b/mediapipe/framework/deps/monotonic_clock_test.cc index 0a049392..9b57ffe5 100644 --- a/mediapipe/framework/deps/monotonic_clock_test.cc +++ b/mediapipe/framework/deps/monotonic_clock_test.cc @@ -21,13 +21,13 @@ #include #include "absl/base/thread_annotations.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/synchronization/mutex.h" #include "absl/time/clock.h" #include "absl/time/time.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/threadpool.h" #include "mediapipe/framework/tool/simulation_clock.h" @@ -254,8 +254,8 @@ TEST_F(MonotonicClockTest, RealTime) { // Just out of curiousity -- did real clock go backwards? int clock_num_corrections; mono_clock->GetCorrectionMetrics(&clock_num_corrections, NULL); - LOG(INFO) << clock_num_corrections << " corrections in " << num_calls - << " calls to mono_clock->Now()"; + ABSL_LOG(INFO) << clock_num_corrections << " corrections in " << num_calls + << " calls to mono_clock->Now()"; delete mono_clock; } @@ -523,13 +523,13 @@ TEST_F(MonotonicClockTest, RealFrenzy) { // Just out of curiousity -- did real clock go backwards? int clock_num_corrections; m1->GetCorrectionMetrics(&clock_num_corrections, NULL); - LOG_IF(INFO, clock_num_corrections > 0) + ABSL_LOG_IF(INFO, clock_num_corrections > 0) << clock_num_corrections << " corrections"; m2->GetCorrectionMetrics(&clock_num_corrections, NULL); - LOG_IF(INFO, clock_num_corrections > 0) + ABSL_LOG_IF(INFO, clock_num_corrections > 0) << clock_num_corrections << " corrections"; m3->GetCorrectionMetrics(&clock_num_corrections, NULL); - LOG_IF(INFO, clock_num_corrections > 0) + ABSL_LOG_IF(INFO, clock_num_corrections > 0) << clock_num_corrections << " corrections"; delete m1; delete m2; diff --git a/mediapipe/framework/deps/re2.h b/mediapipe/framework/deps/re2.h index 61f7985e..89dc8fcd 100644 --- a/mediapipe/framework/deps/re2.h +++ b/mediapipe/framework/deps/re2.h @@ -19,7 +19,7 @@ namespace mediapipe { -// Implementats a subset of RE2 using std::regex_match. +// Implements a subset of RE2 using std::regex_match. class RE2 { public: RE2(const std::string& pattern) : std_regex_(pattern) {} diff --git a/mediapipe/framework/deps/registration.h b/mediapipe/framework/deps/registration.h index 7965539b..f974d689 100644 --- a/mediapipe/framework/deps/registration.h +++ b/mediapipe/framework/deps/registration.h @@ -16,7 +16,6 @@ #define MEDIAPIPE_DEPS_REGISTRATION_H_ #include -#include #include #include #include @@ -29,6 +28,8 @@ #include "absl/base/thread_annotations.h" #include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/meta/type_traits.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" @@ -36,7 +37,6 @@ #include "absl/synchronization/mutex.h" #include "mediapipe/framework/deps/registration_token.h" #include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/statusor.h" namespace mediapipe { @@ -145,6 +145,23 @@ template struct WrapStatusOr> { using type = absl::StatusOr; }; + +// Defining a member of this type causes P to be ODR-used, which forces its +// instantiation if it's a static member of a template. +// Previously we depended on the pointer's value to determine whether the size +// of a character array is 0 or 1, forcing it to be instantiated so the +// compiler can determine the object's layout. But using it as a template +// argument is more compact. +template +struct ForceStaticInstantiation { +#ifdef _MSC_VER + // Just having it as the template argument does not count as a use for + // MSVC. + static constexpr bool Use() { return P != nullptr; } + char force_static[Use()]; +#endif // _MSC_VER +}; + } // namespace registration_internal class NamespaceAllowlist { @@ -162,8 +179,7 @@ class FunctionRegistry { FunctionRegistry(const FunctionRegistry&) = delete; FunctionRegistry& operator=(const FunctionRegistry&) = delete; - RegistrationToken Register(absl::string_view name, Function func, - std::string filename, uint64_t line) + RegistrationToken Register(absl::string_view name, Function func) ABSL_LOCKS_EXCLUDED(lock_) { std::string normalized_name = GetNormalizedName(name); absl::WriterMutexLock lock(&lock_); @@ -173,21 +189,10 @@ class FunctionRegistry { } if (functions_.insert(std::make_pair(normalized_name, std::move(func))) .second) { -#ifndef NDEBUG - locations_.emplace(normalized_name, - std::make_pair(std::move(filename), line)); -#endif return RegistrationToken( [this, normalized_name]() { Unregister(normalized_name); }); } -#ifndef NDEBUG - LOG(FATAL) << "Function with name " << name << " already registered." - << " First registration at " - << locations_.at(normalized_name).first << ":" - << locations_.at(normalized_name).second; -#else - LOG(FATAL) << "Function with name " << name << " already registered."; -#endif + ABSL_LOG(FATAL) << "Function with name " << name << " already registered."; return RegistrationToken([]() {}); } @@ -266,7 +271,7 @@ class FunctionRegistry { if (names[0].empty()) { names.erase(names.begin()); } else { - CHECK_EQ(1u, names.size()) + ABSL_CHECK_EQ(1u, names.size()) << "A registered class name must be either fully qualified " << "with a leading :: or unqualified, got: " << name << "."; } @@ -316,11 +321,6 @@ class FunctionRegistry { private: mutable absl::Mutex lock_; absl::flat_hash_map functions_ ABSL_GUARDED_BY(lock_); -#ifndef NDEBUG - // Stores filename and line number for useful debug log. - absl::flat_hash_map> locations_ - ABSL_GUARDED_BY(lock_); -#endif // For names included in NamespaceAllowlist, strips the namespace. std::string GetAdjustedName(absl::string_view name) { @@ -351,10 +351,8 @@ class GlobalFactoryRegistry { public: static RegistrationToken Register(absl::string_view name, - typename Functions::Function func, - std::string filename, uint64_t line) { - return functions()->Register(name, std::move(func), std::move(filename), - line); + typename Functions::Function func) { + return functions()->Register(name, std::move(func)); } // Invokes the specified factory function and returns the result. @@ -411,15 +409,181 @@ class GlobalFactoryRegistry { #define REGISTRY_STATIC_VAR(var_name, line) \ REGISTRY_STATIC_VAR_INNER(var_name, line) -#define MEDIAPIPE_REGISTER_FACTORY_FUNCTION(RegistryType, name, ...) \ - static auto* REGISTRY_STATIC_VAR(registration_##name, __LINE__) = \ - new mediapipe::RegistrationToken( \ - RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__)) +// Disables all static registration in MediaPipe accomplished using: +// - REGISTER_FACTORY_FUNCTION_QUALIFIED +// - MEDIAPIPE_REGISTER_FACTORY_FUNCTION +// - MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE +// +// Which includes: +// - calculators +// - input stream handlers +// - output stream handlers +// - generators +// - anything else registered using above macros +#if !defined(MEDIAPIPE_DISABLE_STATIC_REGISTRATION) +#define MEDIAPIPE_DISABLE_STATIC_REGISTRATION 0 +#endif // !defined(MEDIAPIPE_DISABLE_STATIC_REGISTRATION) +// Enables "Dry Run" for MediaPipe static registration: MediaPipe logs the +// registration code, instead of actual registration. +// +// The intended use: if you plan to disable static registration using +// MEDIAPIPE_DISABLE_STATIC_REGISTRATION, you may find it useful to build your +// MediaPipe dependency first with only: +// MEDIAPIPE_ENABLE_STATIC_REGISTRATION_DRY_RUN +// and load it to see what manual registration will be required when you build +// with: +// MEDIAPIPE_DISABLE_STATIC_REGISTRATION +#if !defined(MEDIAPIPE_ENABLE_STATIC_REGISTRATION_DRY_RUN) +#define MEDIAPIPE_ENABLE_STATIC_REGISTRATION_DRY_RUN 0 +#endif // !defined(MEDIAPIPE_ENABLE_STATIC_REGISTRATION_DRY_RUN) + +#if MEDIAPIPE_DISABLE_STATIC_REGISTRATION && \ + MEDIAPIPE_ENABLE_STATIC_REGISTRATION_DRY_RUN +static_assert(false, + "Cannot do static registration Dry Run as static registration is " + "disabled."); +#endif // MEDIAPIPE_DISABLE_STATIC_REGISTRATION && + // MEDIAPIPE_ENABLE_STATIC_REGISTRATION_DRY_RUN + +#if MEDIAPIPE_DISABLE_STATIC_REGISTRATION +// When static registration is disabled, make sure corresponding macros don't do +// any registration. + +#define MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, \ + name, ...) +#define MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(RegistratorName, RegistryType, \ + name, ...) \ + template \ + class RegistratorName {}; + +#elif MEDIAPIPE_ENABLE_STATIC_REGISTRATION_DRY_RUN +// When static registration is enabled and running in Dry-Run mode, make sure +// corresponding macros print registration details instead of doing actual +// registration. + +#define INTERNAL_MEDIAPIPE_REGISTER_FACTORY_STRINGIFY_HELPER(x) #x +#define INTERNAL_MEDIAPIPE_REGISTER_FACTORY_STRINGIFY(x) \ + INTERNAL_MEDIAPIPE_REGISTER_FACTORY_STRINGIFY_HELPER(x) + +#define MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, \ + name, ...) \ + static mediapipe::RegistrationToken* REGISTRY_STATIC_VAR(var_name, \ + __LINE__) = []() { \ + ABSL_RAW_LOG(WARNING, "Registration Dry Run: %s", \ + INTERNAL_MEDIAPIPE_REGISTER_FACTORY_STRINGIFY( \ + RegistryType::Register(name, __VA_ARGS__))); \ + return nullptr; \ + }(); + +#define MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(RegistratorName, RegistryType, \ + names, ...) \ + template \ + struct Internal##RegistratorName { \ + static NoDestructor registration; \ + \ + static mediapipe::RegistrationToken Make() { \ + ABSL_RAW_LOG(WARNING, "Registration Dry Run: %s", \ + INTERNAL_MEDIAPIPE_REGISTER_FACTORY_STRINGIFY( \ + RegistryType::Register(names, __VA_ARGS__))); \ + ABSL_RAW_LOG(WARNING, "Where typeid(T).name() is: %s", \ + typeid(T).name()); \ + return {}; \ + } \ + \ + using RequireStatics = \ + registration_internal::ForceStaticInstantiation<®istration>; \ + }; \ + /* Static members of template classes can be defined in the header. */ \ + template \ + NoDestructor \ + Internal##RegistratorName::registration( \ + Internal##RegistratorName::Make()); \ + \ + template \ + class RegistratorName { \ + private: \ + /* The member below triggers instantiation of the registration static. */ \ + typename Internal##RegistratorName::RequireStatics register_; \ + }; + +#else +// When static registration is enabled and NOT running in Dry-Run mode, make +// sure corresponding macros do proper static registration. + +#define MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, \ + name, ...) \ + static mediapipe::RegistrationToken* REGISTRY_STATIC_VAR(var_name, \ + __LINE__) = \ + new mediapipe::RegistrationToken( \ + RegistryType::Register(name, __VA_ARGS__)); + +// Defines a utility registrator class which can be used to automatically +// register factory functions. +// +// Example: +// === Defining a registry ================================================ +// +// class Component {}; +// +// using ComponentRegistry = GlobalFactoryRegistry>; +// +// === Defining a registrator ============================================= +// +// MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(ComponentRegistrator, +// ComponentRegistry, T::kName, +// absl::make_unique); +// +// === Defining and registering a new component. ========================== +// +// class MyComponent : public Component, +// private ComponentRegistrator { +// public: +// static constexpr char kName[] = "MyComponent"; +// ... +// }; +// +// NOTE: +// - MyComponent is automatically registered in ComponentRegistry by +// "MyComponent" name. +// - Every component is require to provide its name (T::kName here.) +#define MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(RegistratorName, RegistryType, \ + name, ...) \ + template \ + struct Internal##RegistratorName { \ + static NoDestructor registration; \ + \ + static mediapipe::RegistrationToken Make() { \ + return RegistryType::Register(name, __VA_ARGS__); \ + } \ + \ + using RequireStatics = \ + registration_internal::ForceStaticInstantiation<®istration>; \ + }; \ + /* Static members of template classes can be defined in the header. */ \ + template \ + NoDestructor \ + Internal##RegistratorName::registration( \ + Internal##RegistratorName::Make()); \ + \ + template \ + class RegistratorName { \ + private: \ + /* The member below triggers instantiation of the registration static. */ \ + typename Internal##RegistratorName::RequireStatics register_; \ + }; + +#endif // MEDIAPIPE_DISABLE_STATIC_REGISTRATION + +#define MEDIAPIPE_REGISTER_FACTORY_FUNCTION(RegistryType, name, ...) \ + MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \ + RegistryType, registration_##name, #name, __VA_ARGS__) + +// TODO: migrate usages to use +// MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED. #define REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, name, ...) \ - static auto* REGISTRY_STATIC_VAR(var_name, __LINE__) = \ - new mediapipe::RegistrationToken( \ - RegistryType::Register(#name, __VA_ARGS__, __FILE__, __LINE__)) + MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED(RegistryType, var_name, #name, \ + __VA_ARGS__) } // namespace mediapipe diff --git a/mediapipe/framework/deps/safe_int.h b/mediapipe/framework/deps/safe_int.h index f6dbb931..37d8663c 100644 --- a/mediapipe/framework/deps/safe_int.h +++ b/mediapipe/framework/deps/safe_int.h @@ -34,7 +34,7 @@ // define any custom policy they desire. // // PolicyTypes: -// LogFatalOnError: LOG(FATAL) when a error occurs. +// LogFatalOnError: ABSL_LOG(FATAL) when a error occurs. #ifndef MEDIAPIPE_DEPS_SAFE_INT_H_ #define MEDIAPIPE_DEPS_SAFE_INT_H_ @@ -44,8 +44,9 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/deps/strong_int.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { namespace intops { @@ -67,17 +68,17 @@ class SafeIntStrongIntValidator { // Check that the underlying integral type provides a range that is // compatible with two's complement. if (std::numeric_limits::is_signed) { - CHECK_EQ(-1, - std::numeric_limits::min() + std::numeric_limits::max()) + ABSL_CHECK_EQ( + -1, std::numeric_limits::min() + std::numeric_limits::max()) << "unexpected integral bounds"; } // Check that division truncates towards 0 (implementation defined in // C++'03, but standard in C++'11). - CHECK_EQ(12, 127 / 10) << "division does not truncate towards 0"; - CHECK_EQ(-12, -127 / 10) << "division does not truncate towards 0"; - CHECK_EQ(-12, 127 / -10) << "division does not truncate towards 0"; - CHECK_EQ(12, -127 / -10) << "division does not truncate towards 0"; + ABSL_CHECK_EQ(12, 127 / 10) << "division does not truncate towards 0"; + ABSL_CHECK_EQ(-12, -127 / 10) << "division does not truncate towards 0"; + ABSL_CHECK_EQ(-12, 127 / -10) << "division does not truncate towards 0"; + ABSL_CHECK_EQ(12, -127 / -10) << "division does not truncate towards 0"; } public: @@ -284,15 +285,15 @@ class SafeIntStrongIntValidator { } }; -// A SafeIntStrongIntValidator policy class to LOG(FATAL) on errors. +// A SafeIntStrongIntValidator policy class to ABSL_LOG(FATAL) on errors. struct LogFatalOnError { template static void Error(const char* error, Tlhs lhs, Trhs rhs, const char* op) { - LOG(FATAL) << error << ": (" << lhs << " " << op << " " << rhs << ")"; + ABSL_LOG(FATAL) << error << ": (" << lhs << " " << op << " " << rhs << ")"; } template static void Error(const char* error, Tval val, const char* op) { - LOG(FATAL) << error << ": (" << op << val << ")"; + ABSL_LOG(FATAL) << error << ": (" << op << val << ")"; } }; diff --git a/mediapipe/framework/deps/status.h b/mediapipe/framework/deps/status.h index 492e4d43..8ee38f32 100644 --- a/mediapipe/framework/deps/status.h +++ b/mediapipe/framework/deps/status.h @@ -21,9 +21,9 @@ #include #include "absl/base/attributes.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -44,7 +44,7 @@ inline std::string* MediaPipeCheckOpHelper(absl::Status v, const char* msg) { #define MEDIAPIPE_DO_CHECK_OK(val, level) \ while (auto _result = mediapipe::MediaPipeCheckOpHelper(val, #val)) \ - LOG(level) << *(_result) + ABSL_LOG(level) << *(_result) #define MEDIAPIPE_CHECK_OK(val) MEDIAPIPE_DO_CHECK_OK(val, FATAL) #define MEDIAPIPE_QCHECK_OK(val) MEDIAPIPE_DO_CHECK_OK(val, QFATAL) @@ -53,7 +53,7 @@ inline std::string* MediaPipeCheckOpHelper(absl::Status v, const char* msg) { #define MEDIAPIPE_DCHECK_OK(val) MEDIAPIPE_CHECK_OK(val) #else #define MEDIAPIPE_DCHECK_OK(val) \ - while (false && (absl::OkStatus() == (val))) LOG(FATAL) + while (false && (absl::OkStatus() == (val))) ABSL_LOG(FATAL) #endif #define CHECK_OK MEDIAPIPE_CHECK_OK diff --git a/mediapipe/framework/deps/strong_int.h b/mediapipe/framework/deps/strong_int.h index 3ddb6d0b..b4bfef77 100644 --- a/mediapipe/framework/deps/strong_int.h +++ b/mediapipe/framework/deps/strong_int.h @@ -103,6 +103,7 @@ #include #include "absl/base/macros.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/port.h" @@ -134,7 +135,7 @@ struct NullStrongIntValidator { // // template // static void ValidateInit(U arg) { - // if (arg < 0) LOG(FATAL) << "arg < 0"; + // if (arg < 0) ABSL_LOG(FATAL) << "arg < 0"; // } // // template diff --git a/mediapipe/framework/deps/threadpool_pthread_impl.cc b/mediapipe/framework/deps/threadpool_pthread_impl.cc index d9c32d35..5033b752 100644 --- a/mediapipe/framework/deps/threadpool_pthread_impl.cc +++ b/mediapipe/framework/deps/threadpool_pthread_impl.cc @@ -18,6 +18,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "mediapipe/framework/deps/threadpool.h" @@ -48,7 +50,7 @@ ThreadPool::WorkerThread::WorkerThread(ThreadPool* pool, const std::string& name_prefix) : pool_(pool), name_prefix_(name_prefix) { int res = pthread_create(&thread_, nullptr, ThreadBody, this); - CHECK_EQ(res, 0) << "pthread_create failed"; + ABSL_CHECK_EQ(res, 0) << "pthread_create failed"; } ThreadPool::WorkerThread::~WorkerThread() {} @@ -67,9 +69,9 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) { if (nice(nice_priority_level) != -1 || errno == 0) { VLOG(1) << "Changed the nice priority level by " << nice_priority_level; } else { - LOG(ERROR) << "Error : " << strerror(errno) << std::endl - << "Could not change the nice priority level by " - << nice_priority_level; + ABSL_LOG(ERROR) << "Error : " << strerror(errno) << std::endl + << "Could not change the nice priority level by " + << nice_priority_level; } } if (!selected_cpus.empty()) { @@ -84,27 +86,27 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) { VLOG(1) << "Pinned the thread pool executor to processor " << absl::StrJoin(selected_cpus, ", processor ") << "."; } else { - LOG(ERROR) << "Error : " << strerror(errno) << std::endl - << "Failed to set processor affinity. Ignore processor " - "affinity setting for now."; + ABSL_LOG(ERROR) << "Error : " << strerror(errno) << std::endl + << "Failed to set processor affinity. Ignore processor " + "affinity setting for now."; } } int error = pthread_setname_np(pthread_self(), name.c_str()); if (error != 0) { - LOG(ERROR) << "Error : " << strerror(error) << std::endl - << "Failed to set name for thread: " << name; + ABSL_LOG(ERROR) << "Error : " << strerror(error) << std::endl + << "Failed to set name for thread: " << name; } #else const std::string name = internal::CreateThreadName(thread->name_prefix_, 0); if (nice_priority_level != 0 || !selected_cpus.empty()) { - LOG(ERROR) << "Thread priority and processor affinity feature aren't " - "supported on the current platform."; + ABSL_LOG(ERROR) << "Thread priority and processor affinity feature aren't " + "supported on the current platform."; } #if __APPLE__ int error = pthread_setname_np(name.c_str()); if (error != 0) { - LOG(ERROR) << "Error : " << strerror(error) << std::endl - << "Failed to set name for thread: " << name; + ABSL_LOG(ERROR) << "Error : " << strerror(error) << std::endl + << "Failed to set name for thread: " << name; } #endif // __APPLE__ #endif // __linux__ diff --git a/mediapipe/framework/deps/threadpool_std_thread_impl.cc b/mediapipe/framework/deps/threadpool_std_thread_impl.cc index 4a902495..a5f86eeb 100644 --- a/mediapipe/framework/deps/threadpool_std_thread_impl.cc +++ b/mediapipe/framework/deps/threadpool_std_thread_impl.cc @@ -17,18 +17,10 @@ #include // NOLINT(build/c++11) -#include "mediapipe/framework/deps/threadpool.h" - -#ifdef _WIN32 -#include -#else -#include -#include -#endif - +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" -#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/deps/threadpool.h" namespace mediapipe { @@ -67,8 +59,9 @@ void* ThreadPool::WorkerThread::ThreadBody(void* arg) { thread->pool_->thread_options().nice_priority_level(); const std::set selected_cpus = thread->pool_->thread_options().cpu_set(); if (nice_priority_level != 0 || !selected_cpus.empty()) { - LOG(ERROR) << "Thread priority and processor affinity feature aren't " - "supported by the std::thread threadpool implementation."; + ABSL_LOG(ERROR) + << "Thread priority and processor affinity feature aren't " + "supported by the std::thread threadpool implementation."; } thread->pool_->RunWorker(); return nullptr; diff --git a/mediapipe/framework/deps/topologicalsorter.cc b/mediapipe/framework/deps/topologicalsorter.cc index 67fc6adc..ba906ea6 100644 --- a/mediapipe/framework/deps/topologicalsorter.cc +++ b/mediapipe/framework/deps/topologicalsorter.cc @@ -16,18 +16,19 @@ #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { TopologicalSorter::TopologicalSorter(int num_nodes) : num_nodes_(num_nodes) { - CHECK_GE(num_nodes_, 0); + ABSL_CHECK_GE(num_nodes_, 0); adjacency_lists_.resize(num_nodes_); } void TopologicalSorter::AddEdge(int from, int to) { - CHECK(!traversal_started_ && from < num_nodes_ && to < num_nodes_ && - from >= 0 && to >= 0); + ABSL_CHECK(!traversal_started_ && from < num_nodes_ && to < num_nodes_ && + from >= 0 && to >= 0); adjacency_lists_[from].push_back(to); } diff --git a/mediapipe/framework/deps/topologicalsorter.h b/mediapipe/framework/deps/topologicalsorter.h index d5027477..2270f294 100644 --- a/mediapipe/framework/deps/topologicalsorter.h +++ b/mediapipe/framework/deps/topologicalsorter.h @@ -40,7 +40,7 @@ namespace mediapipe { // if (cyclic) { // PrintCycleNodes(cycle_nodes); // } else { -// LOG(INFO) << idx; +// ABSL_LOG(INFO) << idx; // } // } class TopologicalSorter { diff --git a/mediapipe/framework/deps/vector.h b/mediapipe/framework/deps/vector.h index 2d4de82f..5d1400ef 100644 --- a/mediapipe/framework/deps/vector.h +++ b/mediapipe/framework/deps/vector.h @@ -24,9 +24,9 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/utility/utility.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" template class Vector2; @@ -78,13 +78,13 @@ class BasicVector { void Clear() { AsD() = D(); } T& operator[](int b) { - DCHECK_GE(b, 0); - DCHECK_LT(b, SIZE); + ABSL_DCHECK_GE(b, 0); + ABSL_DCHECK_LT(b, SIZE); return static_cast(*this).Data()[b]; } T operator[](int b) const { - DCHECK_GE(b, 0); - DCHECK_LT(b, SIZE); + ABSL_DCHECK_GE(b, 0); + ABSL_DCHECK_LT(b, SIZE); return static_cast(*this).Data()[b]; } diff --git a/mediapipe/framework/encode_binary_proto.bzl b/mediapipe/framework/encode_binary_proto.bzl index e849d971..bf7f0583 100644 --- a/mediapipe/framework/encode_binary_proto.bzl +++ b/mediapipe/framework/encode_binary_proto.bzl @@ -37,29 +37,33 @@ Args: output: The desired name of the output file. Optional. """ +load("@bazel_skylib//lib:paths.bzl", "paths") + PROTOC = "@com_google_protobuf//:protoc" -def _canonicalize_proto_path_oss(all_protos, genfile_path): - """For the protos from external repository, canonicalize the proto path and the file name. +def _canonicalize_proto_path_oss(f): + if not f.root.path: + return struct( + proto_path = ".", + file_name = f.short_path, + ) - Returns: - Proto path list and proto source file list. - """ - proto_paths = [] - proto_file_names = [] - for s in all_protos.to_list(): - if s.path.startswith(genfile_path): - repo_name, _, file_name = s.path[len(genfile_path + "/external/"):].partition("/") + # `f.path` looks like "/external//(_virtual_imports//)?" + repo_name, _, file_name = f.path[len(paths.join(f.root.path, "external") + "/"):].partition("/") + if file_name.startswith("_virtual_imports/"): + # This is a virtual import; move "_virtual_imports/" from `repo_name` to `file_name`. + repo_name = paths.join(repo_name, *file_name.split("/", 2)[:2]) + file_name = file_name.split("/", 2)[-1] + return struct( + proto_path = paths.join(f.root.path, "external", repo_name), + file_name = file_name, + ) - # handle virtual imports - if file_name.startswith("_virtual_imports"): - repo_name = repo_name + "/" + "/".join(file_name.split("/", 2)[:2]) - file_name = file_name.split("/", 2)[-1] - proto_paths.append(genfile_path + "/external/" + repo_name) - proto_file_names.append(file_name) - else: - proto_file_names.append(s.path) - return ([" --proto_path=" + path for path in proto_paths], proto_file_names) +def _map_root_path(f): + return _canonicalize_proto_path_oss(f).proto_path + +def _map_short_path(f): + return _canonicalize_proto_path_oss(f).file_name def _get_proto_provider(dep): """Get the provider for protocol buffers from a dependnecy. @@ -90,25 +94,37 @@ def _encode_binary_proto_impl(ctx): sibling = textpb, ) - path_list, file_list = _canonicalize_proto_path_oss(all_protos, ctx.genfiles_dir.path) + args = ctx.actions.args() + args.add(textpb) + args.add(binarypb) + args.add(ctx.executable._proto_compiler) + args.add(ctx.attr.message_type, format = "--encode=%s") + args.add("--proto_path=.") + args.add_all( + all_protos, + map_each = _map_root_path, + format_each = "--proto_path=%s", + uniquify = True, + ) + args.add_all( + all_protos, + map_each = _map_short_path, + uniquify = True, + ) # Note: the combination of absolute_paths and proto_path, as well as the exact # order of gendir before ., is needed for the proto compiler to resolve # import statements that reference proto files produced by a genrule. ctx.actions.run_shell( - tools = all_protos.to_list() + [textpb, ctx.executable._proto_compiler], - outputs = [binarypb], - command = " ".join( - [ - ctx.executable._proto_compiler.path, - "--encode=" + ctx.attr.message_type, - "--proto_path=" + ctx.genfiles_dir.path, - "--proto_path=" + ctx.bin_dir.path, - "--proto_path=.", - ] + path_list + file_list + - ["<", textpb.path, ">", binarypb.path], + tools = depset( + direct = [textpb, ctx.executable._proto_compiler], + transitive = [all_protos], ), + outputs = [binarypb], + command = "${@:3} < $1 > $2", + arguments = [args], mnemonic = "EncodeProto", + toolchain = None, ) output_depset = depset([binarypb]) diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index b23209f7..9a570d52 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -104,7 +104,7 @@ cc_library( srcs = ["deleting_file.cc"], hdrs = ["deleting_file.h"], deps = [ - "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_log", ], ) @@ -119,6 +119,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", "@eigen_archive//:eigen3", ], ) @@ -155,11 +156,12 @@ cc_library( "//mediapipe/framework/port:aligned_malloc_and_free", "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:source_location", "//mediapipe/framework/tool:type_util", "@com_google_absl//absl/base", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ] + select({ @@ -206,7 +208,6 @@ cc_library( "//mediapipe/framework/formats/annotation:locus_cc_proto", "//mediapipe/framework/formats/annotation:rasterization_cc_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:point", "//mediapipe/framework/port:rectangle", "//mediapipe/framework/port:ret_check", @@ -214,6 +215,8 @@ cc_library( "//mediapipe/framework/port:statusor", "//mediapipe/framework/tool:status_util", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_protobuf//:protobuf", @@ -234,6 +237,7 @@ cc_library( ":location", "//mediapipe/framework/formats/annotation:rasterization_cc_proto", "//mediapipe/framework/port:opencv_imgproc", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) @@ -339,6 +343,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer_format", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/synchronization", ] + select({ "//conditions:default": [ @@ -363,6 +368,7 @@ cc_library( ":image_frame_pool", "//mediapipe/framework:port", "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/synchronization", ] + select({ @@ -400,6 +406,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:statusor", + "@com_google_absl//absl/log:absl_check", ], ) @@ -427,6 +434,17 @@ cc_test( ], ) +# Used by vendor processes that don't have access to libandroid.so, but want to use AHardwareBuffer. +config_setting( + name = "android_link_native_window", + define_values = { + "MEDIAPIPE_ANDROID_LINK_NATIVE_WINDOW": "1", + "MEDIAPIPE_NO_JNI": "1", + }, + values = {"crosstool_top": "//external:android/crosstool"}, + visibility = ["//visibility:private"], +) + cc_library( name = "tensor", srcs = @@ -449,7 +467,13 @@ cc_library( "//conditions:default": [], }), defines = select({ + # Excludes AHardwareBuffer features from vendor processes "//mediapipe/framework:android_no_jni": ["MEDIAPIPE_NO_JNI"], + # unless they're linked against nativewindow. + ":android_link_native_window": [ + "MEDIAPIPE_ANDROID_LINK_NATIVE_WINDOW", + "MEDIAPIPE_NO_JNI", + ], "//conditions:default": [], }), linkopts = select({ @@ -462,11 +486,15 @@ cc_library( "//mediapipe:android": [ "-landroid", ], + ":android_link_native_window": [ + "-lnativewindow", # Provides to vendor processes on Android API >= 26. + ], }), deps = [ "//mediapipe/framework:port", - "//mediapipe/framework/port:logging", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/synchronization", ] + select({ @@ -500,7 +528,7 @@ cc_library( hdrs = ["frame_buffer.h"], deps = [ "//mediapipe/framework/port:integral_types", - "@com_google_absl//absl/log:check", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", ], diff --git a/mediapipe/framework/formats/body_rig.proto b/mediapipe/framework/formats/body_rig.proto index 5420ccc1..88964d99 100644 --- a/mediapipe/framework/formats/body_rig.proto +++ b/mediapipe/framework/formats/body_rig.proto @@ -19,7 +19,7 @@ package mediapipe; // Joint of a 3D human model (e.g. elbow, knee, wrist). Contains 3D rotation of // the joint and its visibility. message Joint { - // Joint rotation in 6D contineous representation ordered as + // Joint rotation in 6D continuous representation ordered as // [a1, b1, a2, b2, a3, b3]. // // Such representation is more sutable for NN model training and can be diff --git a/mediapipe/framework/formats/deleting_file.cc b/mediapipe/framework/formats/deleting_file.cc index 977a7894..b759a5f6 100644 --- a/mediapipe/framework/formats/deleting_file.cc +++ b/mediapipe/framework/formats/deleting_file.cc @@ -17,7 +17,7 @@ #include -#include "mediapipe/framework/port/logging.h" +#include "absl/log/absl_log.h" namespace mediapipe { @@ -27,7 +27,7 @@ DeletingFile::DeletingFile(const std::string& path, bool delete_on_destruction) DeletingFile::~DeletingFile() { if (delete_on_destruction_) { if (remove(path_.c_str()) != 0) { - LOG(ERROR) << "Unable to delete file: " << path_; + ABSL_LOG(ERROR) << "Unable to delete file: " << path_; } } } diff --git a/mediapipe/framework/formats/frame_buffer.h b/mediapipe/framework/formats/frame_buffer.h index 21a5f537..71e15457 100644 --- a/mediapipe/framework/formats/frame_buffer.h +++ b/mediapipe/framework/formats/frame_buffer.h @@ -18,7 +18,7 @@ limitations under the License. #include -#include "absl/log/check.h" +#include "absl/log/absl_check.h" #include "absl/status/statusor.h" #include "mediapipe/framework/port/integral_types.h" @@ -147,15 +147,15 @@ class FrameBuffer { // Returns plane indexed by the input `index`. const Plane& plane(int index) const { - CHECK_GE(index, 0); - CHECK_LT(static_cast(index), planes_.size()); + ABSL_CHECK_GE(index, 0); + ABSL_CHECK_LT(static_cast(index), planes_.size()); return planes_[index]; } // Returns mutable plane indexed by the input `index`. Plane mutable_plane(int index) { - CHECK_GE(index, 0); - CHECK_LT(static_cast(index), planes_.size()); + ABSL_CHECK_GE(index, 0); + ABSL_CHECK_LT(static_cast(index), planes_.size()); return planes_[index]; } diff --git a/mediapipe/framework/formats/image.cc b/mediapipe/framework/formats/image.cc index 1ef7e3cb..b37d95aa 100644 --- a/mediapipe/framework/formats/image.cc +++ b/mediapipe/framework/formats/image.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/formats/image.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/type_map.h" #if !MEDIAPIPE_DISABLE_GPU diff --git a/mediapipe/framework/formats/image_frame.cc b/mediapipe/framework/formats/image_frame.cc index 2de819a3..472da76a 100644 --- a/mediapipe/framework/formats/image_frame.cc +++ b/mediapipe/framework/formats/image_frame.cc @@ -23,10 +23,11 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/port/aligned_malloc_and_free.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/proto_ns.h" namespace mediapipe { @@ -98,8 +99,8 @@ void ImageFrame::Reset(ImageFormat::Format format, int width, int height, format_ = format; width_ = width; height_ = height; - CHECK_NE(ImageFormat::UNKNOWN, format_); - CHECK(IsValidAlignmentNumber(alignment_boundary)); + ABSL_CHECK_NE(ImageFormat::UNKNOWN, format_); + ABSL_CHECK(IsValidAlignmentNumber(alignment_boundary)); width_step_ = width * NumberOfChannels() * ByteDepth(); if (alignment_boundary == 1) { pixel_data_ = {new uint8_t[height * width_step_], @@ -124,8 +125,8 @@ void ImageFrame::AdoptPixelData(ImageFormat::Format format, int width, height_ = height; width_step_ = width_step; - CHECK_NE(ImageFormat::UNKNOWN, format_); - CHECK_GE(width_step_, width * NumberOfChannels() * ByteDepth()); + ABSL_CHECK_NE(ImageFormat::UNKNOWN, format_); + ABSL_CHECK_GE(width_step_, width * NumberOfChannels() * ByteDepth()); pixel_data_ = {pixel_data, deleter}; } @@ -136,8 +137,8 @@ std::unique_ptr ImageFrame::Release() { void ImageFrame::InternalCopyFrom(int width, int height, int width_step, int channel_size, const uint8_t* pixel_data) { - CHECK_EQ(width_, width); - CHECK_EQ(height_, height); + ABSL_CHECK_EQ(width_, width); + ABSL_CHECK_EQ(height_, height); // row_bytes = channel_size * num_channels * width const int row_bytes = channel_size * NumberOfChannels() * width; if (width_step == 0) { @@ -187,8 +188,8 @@ void ImageFrame::SetAlignmentPaddingAreas() { if (!pixel_data_) { return; } - CHECK_GE(width_, 1); - CHECK_GE(height_, 1); + ABSL_CHECK_GE(width_, 1); + ABSL_CHECK_GE(height_, 1); const int pixel_size = ByteDepth() * NumberOfChannels(); const int padding_size = width_step_ - width_ * pixel_size; @@ -222,7 +223,7 @@ bool ImageFrame::IsContiguous() const { } bool ImageFrame::IsAligned(uint32_t alignment_boundary) const { - CHECK(IsValidAlignmentNumber(alignment_boundary)); + ABSL_CHECK(IsValidAlignmentNumber(alignment_boundary)); if (!pixel_data_) { return false; } @@ -287,7 +288,7 @@ int ImageFrame::NumberOfChannelsForFormat(ImageFormat::Format format) { case ImageFormat::SBGRA: return 4; default: - LOG(FATAL) << InvalidFormatString(format); + ABSL_LOG(FATAL) << InvalidFormatString(format); } } @@ -318,7 +319,7 @@ int ImageFrame::ChannelSizeForFormat(ImageFormat::Format format) { case ImageFormat::SBGRA: return sizeof(uint8_t); default: - LOG(FATAL) << InvalidFormatString(format); + ABSL_LOG(FATAL) << InvalidFormatString(format); } } @@ -349,7 +350,7 @@ int ImageFrame::ByteDepthForFormat(ImageFormat::Format format) { case ImageFormat::SBGRA: return 1; default: - LOG(FATAL) << InvalidFormatString(format); + ABSL_LOG(FATAL) << InvalidFormatString(format); } } @@ -359,7 +360,7 @@ void ImageFrame::CopyFrom(const ImageFrame& image_frame, Reset(image_frame.Format(), image_frame.Width(), image_frame.Height(), alignment_boundary); - CHECK_EQ(format_, image_frame.Format()); + ABSL_CHECK_EQ(format_, image_frame.Format()); InternalCopyFrom(image_frame.Width(), image_frame.Height(), image_frame.WidthStep(), image_frame.ChannelSize(), image_frame.PixelData()); @@ -382,10 +383,10 @@ void ImageFrame::CopyPixelData(ImageFormat::Format format, int width, } void ImageFrame::CopyToBuffer(uint8_t* buffer, int buffer_size) const { - CHECK(buffer); - CHECK_EQ(1, ByteDepth()); + ABSL_CHECK(buffer); + ABSL_CHECK_EQ(1, ByteDepth()); const int data_size = width_ * height_ * NumberOfChannels(); - CHECK_LE(data_size, buffer_size); + ABSL_CHECK_LE(data_size, buffer_size); if (IsContiguous()) { // The data is stored contiguously, we can just copy. const uint8_t* src = reinterpret_cast(pixel_data_.get()); @@ -397,10 +398,10 @@ void ImageFrame::CopyToBuffer(uint8_t* buffer, int buffer_size) const { } void ImageFrame::CopyToBuffer(uint16_t* buffer, int buffer_size) const { - CHECK(buffer); - CHECK_EQ(2, ByteDepth()); + ABSL_CHECK(buffer); + ABSL_CHECK_EQ(2, ByteDepth()); const int data_size = width_ * height_ * NumberOfChannels(); - CHECK_LE(data_size, buffer_size); + ABSL_CHECK_LE(data_size, buffer_size); if (IsContiguous()) { // The data is stored contiguously, we can just copy. const uint16_t* src = reinterpret_cast(pixel_data_.get()); @@ -412,10 +413,10 @@ void ImageFrame::CopyToBuffer(uint16_t* buffer, int buffer_size) const { } void ImageFrame::CopyToBuffer(float* buffer, int buffer_size) const { - CHECK(buffer); - CHECK_EQ(4, ByteDepth()); + ABSL_CHECK(buffer); + ABSL_CHECK_EQ(4, ByteDepth()); const int data_size = width_ * height_ * NumberOfChannels(); - CHECK_LE(data_size, buffer_size); + ABSL_CHECK_LE(data_size, buffer_size); if (IsContiguous()) { // The data is stored contiguously, we can just copy. const float* src = reinterpret_cast(pixel_data_.get()); diff --git a/mediapipe/framework/formats/image_multi_pool.cc b/mediapipe/framework/formats/image_multi_pool.cc index 655064d3..a38e30a6 100644 --- a/mediapipe/framework/formats/image_multi_pool.cc +++ b/mediapipe/framework/formats/image_multi_pool.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/port/logging.h" @@ -43,7 +44,7 @@ ImageMultiPool::SimplePoolGpu ImageMultiPool::MakeSimplePoolGpu( IBufferSpec spec) { OSType cv_format = mediapipe::CVPixelFormatForGpuBufferFormat( GpuBufferFormatForImageFormat(spec.format)); - CHECK_NE(cv_format, -1) << "unsupported pixel format"; + ABSL_CHECK_NE(cv_format, -1) << "unsupported pixel format"; return MakeCFHolderAdopting(mediapipe::CreateCVPixelBufferPool( spec.width, spec.height, cv_format, kKeepCount, 0.1 /* max age in seconds */)); @@ -61,11 +62,11 @@ Image ImageMultiPool::GetBufferFromSimplePool( // pool to give us contiguous data. OSType cv_format = mediapipe::CVPixelFormatForGpuBufferFormat( mediapipe::GpuBufferFormatForImageFormat(spec.format)); - CHECK_NE(cv_format, -1) << "unsupported pixel format"; + ABSL_CHECK_NE(cv_format, -1) << "unsupported pixel format"; CVPixelBufferRef buffer; CVReturn err = mediapipe::CreateCVPixelBufferWithoutPool( spec.width, spec.height, cv_format, &buffer); - CHECK(!err) << "Error creating pixel buffer: " << err; + ABSL_CHECK(!err) << "Error creating pixel buffer: " << err; return Image(MakeCFHolderAdopting(buffer)); #else CVPixelBufferRef buffer; @@ -87,7 +88,7 @@ Image ImageMultiPool::GetBufferFromSimplePool( } }, &buffer); - CHECK(!err) << "Error creating pixel buffer: " << err; + ABSL_CHECK(!err) << "Error creating pixel buffer: " << err; return Image(MakeCFHolderAdopting(buffer)); #endif // TARGET_IPHONE_SIMULATOR } @@ -188,7 +189,7 @@ Image ImageMultiPool::GetBuffer(int width, int height, bool use_gpu, ImageMultiPool::~ImageMultiPool() { #if !MEDIAPIPE_DISABLE_GPU #ifdef __APPLE__ - CHECK_EQ(texture_caches_.size(), 0) + ABSL_CHECK_EQ(texture_caches_.size(), 0) << "Failed to unregister texture caches before deleting pool"; #endif // defined(__APPLE__) #endif // !MEDIAPIPE_DISABLE_GPU @@ -199,8 +200,8 @@ ImageMultiPool::~ImageMultiPool() { void ImageMultiPool::RegisterTextureCache(mediapipe::CVTextureCacheType cache) { absl::MutexLock lock(&mutex_gpu_); - CHECK(std::find(texture_caches_.begin(), texture_caches_.end(), cache) == - texture_caches_.end()) + ABSL_CHECK(std::find(texture_caches_.begin(), texture_caches_.end(), cache) == + texture_caches_.end()) << "Attempting to register a texture cache twice"; texture_caches_.emplace_back(cache); } @@ -210,7 +211,7 @@ void ImageMultiPool::UnregisterTextureCache( absl::MutexLock lock(&mutex_gpu_); auto it = std::find(texture_caches_.begin(), texture_caches_.end(), cache); - CHECK(it != texture_caches_.end()) + ABSL_CHECK(it != texture_caches_.end()) << "Attempting to unregister an unknown texture cache"; texture_caches_.erase(it); } diff --git a/mediapipe/framework/formats/image_opencv.cc b/mediapipe/framework/formats/image_opencv.cc index 498c7831..387afb5e 100644 --- a/mediapipe/framework/formats/image_opencv.cc +++ b/mediapipe/framework/formats/image_opencv.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/formats/image_opencv.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/port/logging.h" @@ -100,7 +101,7 @@ std::shared_ptr MatView(const mediapipe::Image* image) { auto owner = std::make_shared(const_cast(image)); uint8_t* data_ptr = owner->lock.Pixels(); - CHECK(data_ptr != nullptr); + ABSL_CHECK(data_ptr != nullptr); // Use Image to initialize in-place. Image still owns memory. if (steps[0] == sizes[1] * image->channels() * ImageFrame::ByteDepthForFormat(image->image_format())) { diff --git a/mediapipe/framework/formats/location.cc b/mediapipe/framework/formats/location.cc index 205edf19..b9dd97e7 100644 --- a/mediapipe/framework/formats/location.cc +++ b/mediapipe/framework/formats/location.cc @@ -18,13 +18,14 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/substitute.h" #include "mediapipe/framework/formats/annotation/locus.pb.h" #include "mediapipe/framework/formats/annotation/rasterization.pb.h" #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/point2.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -39,7 +40,7 @@ namespace { // the location_data, the tightest bounding box, that contains all pixels // encoded in the rasterizations. Rectangle_i MaskToRectangle(const LocationData& location_data) { - CHECK(location_data.mask().has_rasterization()); + ABSL_CHECK(location_data.mask().has_rasterization()); const auto& rasterization = location_data.mask().rasterization(); if (rasterization.interval_size() == 0) { return Rectangle_i(0, 0, 0, 0); @@ -63,7 +64,7 @@ Location::Location() {} Location::Location(const LocationData& location_data) : location_data_(location_data) { - CHECK(IsValidLocationData(location_data_)); + ABSL_CHECK(IsValidLocationData(location_data_)); } Location Location::CreateGlobalLocation() { @@ -152,15 +153,15 @@ bool Location::IsValidLocationData(const LocationData& location_data) { template <> Rectangle_i Location::GetBBox() const { - CHECK_EQ(LocationData::BOUNDING_BOX, location_data_.format()); + ABSL_CHECK_EQ(LocationData::BOUNDING_BOX, location_data_.format()); const auto& box = location_data_.bounding_box(); return Rectangle_i(box.xmin(), box.ymin(), box.width(), box.height()); } Location& Location::Scale(const float scale) { - CHECK(!location_data_.has_mask()) + ABSL_CHECK(!location_data_.has_mask()) << "Location mask scaling is not implemented."; - CHECK_GT(scale, 0.0f); + ABSL_CHECK_GT(scale, 0.0f); switch (location_data_.format()) { case LocationData::GLOBAL: { // Do nothing. @@ -187,7 +188,8 @@ Location& Location::Scale(const float scale) { break; } case LocationData::MASK: { - LOG(FATAL) << "Scaling for location data of type MASK is not supported."; + ABSL_LOG(FATAL) + << "Scaling for location data of type MASK is not supported."; break; } } @@ -232,7 +234,8 @@ Location& Location::Square(int image_width, int image_height) { break; } case LocationData::MASK: { - LOG(FATAL) << "Squaring for location data of type MASK is not supported."; + ABSL_LOG(FATAL) + << "Squaring for location data of type MASK is not supported."; break; } } @@ -247,7 +250,7 @@ namespace { // This function is inteded to shift boundaries of intervals such that they // best fit within an image. float BestShift(float min_value, float max_value, float range) { - CHECK_LE(min_value, max_value); + ABSL_CHECK_LE(min_value, max_value); const float value_range = max_value - min_value; if (value_range > range) { return 0.5f * (range - min_value - max_value); @@ -294,8 +297,8 @@ Location& Location::ShiftToFitBestIntoImage(int image_width, int image_height) { const float y_shift = BestShift(mask_bounding_box.xmin(), mask_bounding_box.xmax(), image_height); auto* mask = location_data_.mutable_mask(); - CHECK_EQ(image_width, mask->width()); - CHECK_EQ(image_height, mask->height()); + ABSL_CHECK_EQ(image_width, mask->width()); + ABSL_CHECK_EQ(image_height, mask->height()); for (auto& interval : *mask->mutable_rasterization()->mutable_interval()) { interval.set_y(interval.y() + y_shift); @@ -327,7 +330,7 @@ Location& Location::Crop(const Rectangle_i& crop_box) { break; } case LocationData::RELATIVE_BOUNDING_BOX: - LOG(FATAL) + ABSL_LOG(FATAL) << "Can't crop a relative bounding box using absolute coordinates. " "Use the 'Rectangle_f version of Crop() instead"; case LocationData::MASK: { @@ -361,7 +364,7 @@ Location& Location::Crop(const Rectangle_f& crop_box) { // Do nothing. break; case LocationData::BOUNDING_BOX: - LOG(FATAL) + ABSL_LOG(FATAL) << "Can't crop an absolute bounding box using relative coordinates. " "Use the 'Rectangle_i version of Crop() instead"; case LocationData::RELATIVE_BOUNDING_BOX: { @@ -377,8 +380,9 @@ Location& Location::Crop(const Rectangle_f& crop_box) { break; } case LocationData::MASK: - LOG(FATAL) << "Can't crop a mask using relative coordinates. Use the " - "'Rectangle_i' version of Crop() instead"; + ABSL_LOG(FATAL) + << "Can't crop a mask using relative coordinates. Use the " + "'Rectangle_i' version of Crop() instead"; } return *this; } @@ -418,7 +422,7 @@ Rectangle_i Location::ConvertToBBox(int image_width, } Rectangle_f Location::GetRelativeBBox() const { - CHECK_EQ(LocationData::RELATIVE_BOUNDING_BOX, location_data_.format()); + ABSL_CHECK_EQ(LocationData::RELATIVE_BOUNDING_BOX, location_data_.format()); const auto& box = location_data_.relative_bounding_box(); return Rectangle_f(box.xmin(), box.ymin(), box.width(), box.height()); } @@ -457,7 +461,7 @@ Rectangle_f Location::ConvertToRelativeBBox(int image_width, template <> ::mediapipe::BoundingBox Location::GetBBox<::mediapipe::BoundingBox>() const { - CHECK_EQ(LocationData::BOUNDING_BOX, location_data_.format()); + ABSL_CHECK_EQ(LocationData::BOUNDING_BOX, location_data_.format()); const auto& box = location_data_.bounding_box(); ::mediapipe::BoundingBox bounding_box; bounding_box.set_left_x(box.xmin()); @@ -480,7 +484,7 @@ template <> } std::vector Location::GetRelativeKeypoints() const { - CHECK_EQ(LocationData::RELATIVE_BOUNDING_BOX, location_data_.format()); + ABSL_CHECK_EQ(LocationData::RELATIVE_BOUNDING_BOX, location_data_.format()); std::vector keypoints; for (const auto& keypoint : location_data_.relative_keypoints()) { keypoints.emplace_back(Point2_f(keypoint.x(), keypoint.y())); diff --git a/mediapipe/framework/formats/location_opencv.cc b/mediapipe/framework/formats/location_opencv.cc index 6e15b299..4b69cc6d 100644 --- a/mediapipe/framework/formats/location_opencv.cc +++ b/mediapipe/framework/formats/location_opencv.cc @@ -14,11 +14,12 @@ #include "mediapipe/framework/formats/location_opencv.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/substitute.h" #include "mediapipe/framework/formats/annotation/rasterization.pb.h" #include "mediapipe/framework/formats/location.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/statusor.h" @@ -26,7 +27,7 @@ namespace mediapipe { namespace { Rectangle_i MaskToRectangle(const LocationData& location_data) { - CHECK(location_data.mask().has_rasterization()); + ABSL_CHECK(location_data.mask().has_rasterization()); const auto& rasterization = location_data.mask().rasterization(); if (rasterization.interval_size() == 0) { return Rectangle_i(0, 0, 0, 0); @@ -85,7 +86,7 @@ Location CreateBBoxLocation(const cv::Rect& rect) { std::unique_ptr GetCvMask(const Location& location) { const auto location_data = location.ConvertToProto(); - CHECK_EQ(LocationData::MASK, location_data.format()); + ABSL_CHECK_EQ(LocationData::MASK, location_data.format()); const auto& mask = location_data.mask(); std::unique_ptr mat( new cv::Mat(mask.height(), mask.width(), CV_8UC1, cv::Scalar(0))); @@ -108,7 +109,7 @@ std::unique_ptr ConvertToCvMask(const Location& location, image_width, image_height, location.ConvertToBBox(image_width, image_height)); if (!status_or_mat.ok()) { - LOG(ERROR) << status_or_mat.status().message(); + ABSL_LOG(ERROR) << status_or_mat.status().message(); return nullptr; } return std::move(status_or_mat).value(); @@ -120,15 +121,15 @@ std::unique_ptr ConvertToCvMask(const Location& location, // This should never happen; a new LocationData::Format enum was introduced // without updating this function's switch(...) to support it. #if !defined(MEDIAPIPE_MOBILE) && !defined(MEDIAPIPE_LITE) - LOG(ERROR) << "Location's LocationData has format not supported by " - "Location::ConvertToMask: " - << location_data.DebugString(); + ABSL_LOG(ERROR) << "Location's LocationData has format not supported by " + "Location::ConvertToMask: " + << location_data.DebugString(); #endif return nullptr; } void EnlargeLocation(Location& location, const float factor) { - CHECK_GT(factor, 0.0f); + ABSL_CHECK_GT(factor, 0.0f); if (factor == 1.0f) return; auto location_data = location.ConvertToProto(); switch (location_data.format()) { @@ -183,7 +184,7 @@ void EnlargeLocation(Location& location, const float factor) { template Location CreateCvMaskLocation(const cv::Mat_& mask) { - CHECK_EQ(1, mask.channels()) + ABSL_CHECK_EQ(1, mask.channels()) << "The specified cv::Mat mask should be single-channel."; LocationData location_data; diff --git a/mediapipe/framework/formats/matrix.cc b/mediapipe/framework/formats/matrix.cc index 42f2df5f..34ffc6e7 100644 --- a/mediapipe/framework/formats/matrix.cc +++ b/mediapipe/framework/formats/matrix.cc @@ -15,6 +15,7 @@ #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/core_proto_inc.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/proto_ns.h" @@ -33,8 +34,8 @@ void MatrixDataProtoFromMatrix(const Matrix& matrix, MatrixData* matrix_data) { } void MatrixFromMatrixDataProto(const MatrixData& matrix_data, Matrix* matrix) { - CHECK_EQ(matrix_data.rows() * matrix_data.cols(), - matrix_data.packed_data_size()); + ABSL_CHECK_EQ(matrix_data.rows() * matrix_data.cols(), + matrix_data.packed_data_size()); if (matrix_data.layout() == MatrixData::ROW_MAJOR) { matrix->resize(matrix_data.cols(), matrix_data.rows()); } else { @@ -56,9 +57,9 @@ std::string MatrixAsTextProto(const Matrix& matrix) { } void MatrixFromTextProto(const std::string& text_proto, Matrix* matrix) { - CHECK(matrix); + ABSL_CHECK(matrix); MatrixData matrix_data; - CHECK(proto_ns::TextFormat::ParseFromString(text_proto, &matrix_data)); + ABSL_CHECK(proto_ns::TextFormat::ParseFromString(text_proto, &matrix_data)); MatrixFromMatrixDataProto(matrix_data, matrix); } #endif // !defined(MEDIAPIPE_MOBILE) && !defined(MEDIAPIPE_LITE) diff --git a/mediapipe/framework/formats/motion/BUILD b/mediapipe/framework/formats/motion/BUILD index 919b8240..8f40202c 100644 --- a/mediapipe/framework/formats/motion/BUILD +++ b/mediapipe/framework/formats/motion/BUILD @@ -39,11 +39,12 @@ cc_library( "//mediapipe/framework/formats:location_opencv", "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:point", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:framework", ], @@ -61,8 +62,9 @@ cc_test( "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/core:framework", ], ) diff --git a/mediapipe/framework/formats/motion/optical_flow_field.cc b/mediapipe/framework/formats/motion/optical_flow_field.cc index a9650419..fd9b8e30 100644 --- a/mediapipe/framework/formats/motion/optical_flow_field.cc +++ b/mediapipe/framework/formats/motion/optical_flow_field.cc @@ -18,6 +18,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/deps/mathutil.h" @@ -25,7 +27,6 @@ #include "mediapipe/framework/formats/location_opencv.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/point2.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/type_map.h" @@ -40,8 +41,8 @@ const float kFloFileHeaderOnRead = 202021.25; void CartesianToPolarCoordinates(const cv::Mat& cartesian, cv::Mat* magnitudes, cv::Mat* angles) { - CHECK(magnitudes != nullptr); - CHECK(angles != nullptr); + ABSL_CHECK(magnitudes != nullptr); + ABSL_CHECK(angles != nullptr); cv::Mat cartesian_components[2]; cv::split(cartesian, cartesian_components); cv::cartToPolar(cartesian_components[0], cartesian_components[1], *magnitudes, @@ -105,7 +106,7 @@ cv::Mat OpticalFlowField::GetVisualizationInternal( std::max(std::numeric_limits::epsilon(), MaxAbsoluteValueIgnoringHuge(magnitudes, kHugeToIgnore)); } - CHECK_LT(0, max_magnitude); + ABSL_CHECK_LT(0, max_magnitude); cv::Mat hsv = MakeVisualizationHsv(angles, magnitudes, max_magnitude); cv::Mat viz; cv::cvtColor(hsv, viz, 71 /*cv::COLOR_HSV2RGB_FULL*/); @@ -119,7 +120,7 @@ cv::Mat OpticalFlowField::GetVisualization() const { cv::Mat OpticalFlowField::GetVisualizationSaturatedAt( float max_magnitude) const { - CHECK_LT(0, max_magnitude) + ABSL_CHECK_LT(0, max_magnitude) << "Specified saturation magnitude must be positive."; return GetVisualizationInternal(max_magnitude, true); } @@ -147,9 +148,9 @@ void OpticalFlowField::Resize(int new_width, int new_height) { } void OpticalFlowField::CopyFromTensor(const tensorflow::Tensor& tensor) { - CHECK_EQ(tensorflow::DT_FLOAT, tensor.dtype()); - CHECK_EQ(3, tensor.dims()) << "Tensor must be height x width x 2."; - CHECK_EQ(2, tensor.dim_size(2)) << "Tensor must be height x width x 2."; + ABSL_CHECK_EQ(tensorflow::DT_FLOAT, tensor.dtype()); + ABSL_CHECK_EQ(3, tensor.dims()) << "Tensor must be height x width x 2."; + ABSL_CHECK_EQ(2, tensor.dim_size(2)) << "Tensor must be height x width x 2."; const int height = tensor.dim_size(0); const int width = tensor.dim_size(1); Allocate(width, height); @@ -163,8 +164,8 @@ void OpticalFlowField::CopyFromTensor(const tensorflow::Tensor& tensor) { } void OpticalFlowField::SetFromProto(const OpticalFlowFieldData& proto) { - CHECK_EQ(proto.width() * proto.height(), proto.dx_size()); - CHECK_EQ(proto.width() * proto.height(), proto.dy_size()); + ABSL_CHECK_EQ(proto.width() * proto.height(), proto.dx_size()); + ABSL_CHECK_EQ(proto.width() * proto.height(), proto.dy_size()); flow_data_.create(proto.height(), proto.width()); int i = 0; for (int r = 0; r < flow_data_.rows; ++r) { @@ -191,8 +192,8 @@ void OpticalFlowField::ConvertToProto(OpticalFlowFieldData* proto) const { bool OpticalFlowField::FollowFlow(float x, float y, float* new_x, float* new_y) const { - CHECK(new_x); - CHECK(new_y); + ABSL_CHECK(new_x); + ABSL_CHECK(new_y); if (x < 0 || x > flow_data_.cols - 1 || // horizontal bounds y < 0 || y > flow_data_.rows - 1) { // vertical bounds return false; @@ -205,10 +206,10 @@ bool OpticalFlowField::FollowFlow(float x, float y, float* new_x, cv::Point2f OpticalFlowField::InterpolatedFlowAt(float x, float y) const { // Sanity bounds checks. - CHECK_GE(x, 0); - CHECK_GE(y, 0); - CHECK_LE(x, flow_data_.cols - 1); - CHECK_LE(y, flow_data_.rows - 1); + ABSL_CHECK_GE(x, 0); + ABSL_CHECK_GE(y, 0); + ABSL_CHECK_LE(x, flow_data_.cols - 1); + ABSL_CHECK_LE(y, flow_data_.rows - 1); const int x0 = static_cast(std::floor(x)); const int y0 = static_cast(std::floor(y)); @@ -253,7 +254,7 @@ bool OpticalFlowField::AllWithinMargin(const OpticalFlowField& other, const cv::Point2f& other_motion = other.flow_data().at(r, c); if (!MathUtil::WithinMargin(this_motion.x, other_motion.x, margin) || !MathUtil::WithinMargin(this_motion.y, other_motion.y, margin)) { - LOG(INFO) << "First failure at" << r << " " << c; + ABSL_LOG(INFO) << "First failure at" << r << " " << c; return false; } } @@ -265,9 +266,9 @@ void OpticalFlowField::EstimateMotionConsistencyOcclusions( const OpticalFlowField& forward, const OpticalFlowField& backward, double spatial_distance_threshold, Location* occluded_mask, Location* disoccluded_mask) { - CHECK_EQ(forward.width(), backward.width()) + ABSL_CHECK_EQ(forward.width(), backward.width()) << "Flow fields have different widths."; - CHECK_EQ(forward.height(), backward.height()) + ABSL_CHECK_EQ(forward.height(), backward.height()) << "Flow fields have different heights."; if (occluded_mask != nullptr) { *occluded_mask = FindMotionInconsistentPixels(forward, backward, diff --git a/mediapipe/framework/formats/motion/optical_flow_field_test.cc b/mediapipe/framework/formats/motion/optical_flow_field_test.cc index fdce418f..2647c261 100644 --- a/mediapipe/framework/formats/motion/optical_flow_field_test.cc +++ b/mediapipe/framework/formats/motion/optical_flow_field_test.cc @@ -19,12 +19,13 @@ #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/formats/location_opencv.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "tensorflow/core/framework/tensor.h" namespace mediapipe { diff --git a/mediapipe/framework/formats/tensor.cc b/mediapipe/framework/formats/tensor.cc index 0445712c..2f2bfaae 100644 --- a/mediapipe/framework/formats/tensor.cc +++ b/mediapipe/framework/formats/tensor.cc @@ -17,9 +17,10 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/logging.h" #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 #include "mediapipe/gpu/gl_base.h" #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 @@ -81,7 +82,7 @@ void* AllocateVirtualMemory(size_t size) { vm_address_t data; auto error = vm_allocate(mach_task_self(), &data, AlignToPageSize(size), VM_FLAGS_ANYWHERE); - LOG_IF(FATAL, error != KERN_SUCCESS) + ABSL_LOG_IF(FATAL, error != KERN_SUCCESS) << "Can't allocate virtual memory for Tensor."; return reinterpret_cast(data); } @@ -113,10 +114,10 @@ void MtlBufferView::AllocateMtlBuffer(const Tensor& tensor, MtlBufferView MtlBufferView::GetReadView(const Tensor& tensor, id command_buffer) { - LOG_IF(FATAL, tensor.valid_ == Tensor::kValidNone) + ABSL_LOG_IF(FATAL, tensor.valid_ == Tensor::kValidNone) << "Tensor must be written prior to read from."; - LOG_IF(FATAL, - !(tensor.valid_ & (Tensor::kValidCpu | Tensor::kValidMetalBuffer))) + ABSL_LOG_IF( + FATAL, !(tensor.valid_ & (Tensor::kValidCpu | Tensor::kValidMetalBuffer))) << "Tensor conversion between different GPU backing formats is not " "supported yet."; auto lock(absl::make_unique(&tensor.view_mutex_)); @@ -152,7 +153,7 @@ bool Tensor::NeedsHalfFloatRenderTarget() const { if (!has_color_buffer_float) { static bool has_color_buffer_half_float = gl_context_->HasGlExtension("EXT_color_buffer_half_float"); - LOG_IF(FATAL, !has_color_buffer_half_float) + ABSL_LOG_IF(FATAL, !has_color_buffer_half_float) << "EXT_color_buffer_half_float or WEBGL_color_buffer_float " << "required on web to use MP tensor"; return true; @@ -161,9 +162,9 @@ bool Tensor::NeedsHalfFloatRenderTarget() const { } Tensor::OpenGlTexture2dView Tensor::GetOpenGlTexture2dReadView() const { - LOG_IF(FATAL, valid_ == kValidNone) + ABSL_LOG_IF(FATAL, valid_ == kValidNone) << "Tensor must be written prior to read from."; - LOG_IF(FATAL, !(valid_ & (kValidCpu | kValidOpenGlTexture2d))) + ABSL_LOG_IF(FATAL, !(valid_ & (kValidCpu | kValidOpenGlTexture2d))) << "Tensor conversion between different GPU backing formats is not " "supported yet."; auto lock = absl::make_unique(&view_mutex_); @@ -266,7 +267,7 @@ Tensor::OpenGlTexture2dView::GetLayoutDimensions(const Tensor::Shape& shape, float power = std::log2(std::sqrt(static_cast(num_pixels))); w = 1 << static_cast(power); int h = (num_pixels + w - 1) / w; - LOG_IF(FATAL, w > max_size || h > max_size) + ABSL_LOG_IF(FATAL, w > max_size || h > max_size) << "The tensor can't fit into OpenGL Texture2D View."; *width = w; *height = h; @@ -276,7 +277,7 @@ Tensor::OpenGlTexture2dView::GetLayoutDimensions(const Tensor::Shape& shape, void Tensor::AllocateOpenGlTexture2d() const { if (opengl_texture2d_ == GL_INVALID_INDEX) { gl_context_ = mediapipe::GlContext::GetCurrent(); - LOG_IF(FATAL, !gl_context_) << "GlContext is not bound to the thread."; + ABSL_LOG_IF(FATAL, !gl_context_) << "GlContext is not bound to the thread."; glGenTextures(1, &opengl_texture2d_); glBindTexture(GL_TEXTURE_2D, opengl_texture2d_); // Texture2D represents a buffer with computable data so should be fetched @@ -302,7 +303,7 @@ void Tensor::AllocateOpenGlTexture2d() const { // once for OES_texture_float extension, to save time. static bool has_oes_extension = gl_context_->HasGlExtension("OES_texture_float"); - LOG_IF(FATAL, !has_oes_extension) + ABSL_LOG_IF(FATAL, !has_oes_extension) << "OES_texture_float extension required in order to use MP tensor " << "with GLES 2.0"; // Allocate the image data; note that it's no longer RGBA32F, so will be @@ -328,13 +329,13 @@ void Tensor::AllocateOpenGlTexture2d() const { #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const { - LOG_IF(FATAL, valid_ == kValidNone) + ABSL_LOG_IF(FATAL, valid_ == kValidNone) << "Tensor must be written prior to read from."; - LOG_IF(FATAL, !(valid_ & (kValidCpu | + ABSL_LOG_IF(FATAL, !(valid_ & (kValidCpu | #ifdef MEDIAPIPE_TENSOR_USE_AHWB - kValidAHardwareBuffer | + kValidAHardwareBuffer | #endif // MEDIAPIPE_TENSOR_USE_AHWB - kValidOpenGlBuffer))) + kValidOpenGlBuffer))) << "Tensor conversion between different GPU backing formats is not " "supported yet."; auto lock(absl::make_unique(&view_mutex_)); @@ -347,7 +348,7 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const { void* ptr = glMapBufferRange(GL_SHADER_STORAGE_BUFFER, 0, bytes(), GL_MAP_INVALIDATE_BUFFER_BIT | GL_MAP_WRITE_BIT); - CHECK(ptr) << "glMapBufferRange failed: " << glGetError(); + ABSL_CHECK(ptr) << "glMapBufferRange failed: " << glGetError(); std::memcpy(ptr, cpu_buffer_, bytes()); glUnmapBuffer(GL_SHADER_STORAGE_BUFFER); } @@ -374,7 +375,7 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferWriteView( void Tensor::AllocateOpenGlBuffer() const { if (opengl_buffer_ == GL_INVALID_INDEX) { gl_context_ = mediapipe::GlContext::GetCurrent(); - LOG_IF(FATAL, !gl_context_) << "GlContext is not bound to the thread."; + ABSL_LOG_IF(FATAL, !gl_context_) << "GlContext is not bound to the thread."; glGenBuffers(1, &opengl_buffer_); glBindBuffer(GL_SHADER_STORAGE_BUFFER, opengl_buffer_); if (!use_ahwb_ || !AllocateAhwbMapToSsbo()) { @@ -528,7 +529,7 @@ void Tensor::Invalidate() { Tensor::CpuReadView Tensor::GetCpuReadView() const { auto lock = absl::make_unique(&view_mutex_); - LOG_IF(FATAL, valid_ == kValidNone) + ABSL_LOG_IF(FATAL, valid_ == kValidNone) << "Tensor must be written prior to read from."; #ifdef MEDIAPIPE_TENSOR_USE_AHWB if (__builtin_available(android 26, *)) { @@ -537,7 +538,7 @@ Tensor::CpuReadView Tensor::GetCpuReadView() const { valid_ |= kValidCpu; return {ptr, std::move(lock), [ahwb = ahwb_] { auto error = AHardwareBuffer_unlock(ahwb, nullptr); - CHECK(error == 0) << "AHardwareBuffer_unlock " << error; + ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error; }}; } } @@ -548,7 +549,7 @@ Tensor::CpuReadView Tensor::GetCpuReadView() const { // GPU-to-CPU synchronization and read-back. #if MEDIAPIPE_METAL_ENABLED if (valid_ & kValidMetalBuffer) { - LOG_IF(FATAL, !mtl_resources_->command_buffer) + ABSL_LOG_IF(FATAL, !mtl_resources_->command_buffer) << "Metal -> CPU synchronization " "requires MTLCommandBuffer to be set."; if (mtl_resources_->command_buffer) { @@ -621,7 +622,7 @@ Tensor::CpuWriteView Tensor::GetCpuWriteView( if (ptr) { return {ptr, std::move(lock), [ahwb = ahwb_, fence_fd = &fence_fd_] { auto error = AHardwareBuffer_unlock(ahwb, fence_fd); - CHECK(error == 0) << "AHardwareBuffer_unlock " << error; + ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error; }}; } } diff --git a/mediapipe/framework/formats/tensor.h b/mediapipe/framework/formats/tensor.h index 1d670d80..701707de 100644 --- a/mediapipe/framework/formats/tensor.h +++ b/mediapipe/framework/formats/tensor.h @@ -25,16 +25,21 @@ #include #include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/formats/tensor/internal.h" #include "mediapipe/framework/port.h" -#ifndef MEDIAPIPE_NO_JNI +// Supported use cases for tensor_ahwb: +// 1. Native code running in Android apps. +// 2. Android vendor processes linked against nativewindow. +#if !defined(MEDIAPIPE_NO_JNI) || defined(MEDIAPIPE_ANDROID_LINK_NATIVE_WINDOW) #if __ANDROID_API__ >= 26 || defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__) #define MEDIAPIPE_TENSOR_USE_AHWB 1 #endif // __ANDROID_API__ >= 26 || // defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__) -#endif // MEDIAPIPE_NO_JNI +#endif // !defined(MEDIAPIPE_NO_JNI) || + // defined(MEDIAPIPE_ANDROID_LINK_NATIVE_WINDOW) #ifdef MEDIAPIPE_TENSOR_USE_AHWB #include @@ -117,11 +122,18 @@ class Tensor { Shape() = default; Shape(std::initializer_list dimensions) : dims(dimensions) {} Shape(const std::vector& dimensions) : dims(dimensions) {} + Shape(std::initializer_list dimensions, bool is_dynamic) + : dims(dimensions), is_dynamic(is_dynamic) {} + Shape(const std::vector& dimensions, bool is_dynamic) + : dims(dimensions), is_dynamic(is_dynamic) {} int num_elements() const { return std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); } std::vector dims; + // The Tensor has dynamic rather than static shape so the TFLite interpreter + // needs to be reallocated. Only relevant for CPU. + bool is_dynamic = false; }; // Quantization parameters corresponding to the zero_point and scale value // made available by TfLite quantized (uint8/int8) tensors. @@ -193,12 +205,12 @@ class Tensor { } int file_descriptor() const { return file_descriptor_; } void SetReadingFinishedFunc(FinishingFunc&& func) { - CHECK(ahwb_written_) + ABSL_CHECK(ahwb_written_) << "AHWB write view can't accept 'reading finished callback'"; *ahwb_written_ = std::move(func); } void SetWritingFinishedFD(int fd, FinishingFunc func = nullptr) { - CHECK(fence_fd_) + ABSL_CHECK(fence_fd_) << "AHWB read view can't accept 'writing finished file descriptor'"; *fence_fd_ = fd; *ahwb_written_ = std::move(func); diff --git a/mediapipe/framework/formats/tensor_ahwb.cc b/mediapipe/framework/formats/tensor_ahwb.cc index 525f05f3..339148e9 100644 --- a/mediapipe/framework/formats/tensor_ahwb.cc +++ b/mediapipe/framework/formats/tensor_ahwb.cc @@ -7,9 +7,10 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/gpu/gl_base.h" #endif // MEDIAPIPE_TENSOR_USE_AHWB @@ -208,12 +209,13 @@ class DelayedReleaser { Tensor::AHardwareBufferView Tensor::GetAHardwareBufferReadView() const { auto lock(absl::make_unique(&view_mutex_)); - CHECK(valid_ != kValidNone) << "Tensor must be written prior to read from."; - CHECK(!(valid_ & kValidOpenGlTexture2d)) + ABSL_CHECK(valid_ != kValidNone) + << "Tensor must be written prior to read from."; + ABSL_CHECK(!(valid_ & kValidOpenGlTexture2d)) << "Tensor conversion between OpenGL texture and AHardwareBuffer is not " "supported."; bool transfer = !ahwb_; - CHECK(AllocateAHardwareBuffer()) + ABSL_CHECK(AllocateAHardwareBuffer()) << "AHardwareBuffer is not supported on the target system."; valid_ |= kValidAHardwareBuffer; if (transfer) { @@ -253,7 +255,7 @@ void Tensor::CreateEglSyncAndFd() const { Tensor::AHardwareBufferView Tensor::GetAHardwareBufferWriteView( int size_alignment) const { auto lock(absl::make_unique(&view_mutex_)); - CHECK(AllocateAHardwareBuffer(size_alignment)) + ABSL_CHECK(AllocateAHardwareBuffer(size_alignment)) << "AHardwareBuffer is not supported on the target system."; valid_ = kValidAHardwareBuffer; return {ahwb_, @@ -319,7 +321,7 @@ void Tensor::MoveCpuOrSsboToAhwb() const { if (__builtin_available(android 26, *)) { auto error = AHardwareBuffer_lock( ahwb_, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1, nullptr, &dest); - CHECK(error == 0) << "AHardwareBuffer_lock " << error; + ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error; } if (valid_ & kValidCpu) { std::memcpy(dest, cpu_buffer_, bytes()); @@ -342,11 +344,12 @@ void Tensor::MoveCpuOrSsboToAhwb() const { // of the Ahwb at the next request to the OpenGlBufferView. valid_ &= ~kValidOpenGlBuffer; } else { - LOG(FATAL) << "Can't convert tensor with mask " << valid_ << " into AHWB."; + ABSL_LOG(FATAL) << "Can't convert tensor with mask " << valid_ + << " into AHWB."; } if (__builtin_available(android 26, *)) { auto error = AHardwareBuffer_unlock(ahwb_, nullptr); - CHECK(error == 0) << "AHardwareBuffer_unlock " << error; + ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error; } } @@ -421,9 +424,10 @@ void* Tensor::MapAhwbToCpuRead() const { // TODO: Use tflite::gpu::GlBufferSync and GlActiveSync. gl_context_->Run([]() { glFinish(); }); } else if (valid_ & kValidAHardwareBuffer) { - CHECK(ahwb_written_) << "Ahwb-to-Cpu synchronization requires the " - "completion function to be set"; - CHECK(ahwb_written_(true)) + ABSL_CHECK(ahwb_written_) + << "Ahwb-to-Cpu synchronization requires the " + "completion function to be set"; + ABSL_CHECK(ahwb_written_(true)) << "An error oqcured while waiting for the buffer to be written"; } } @@ -431,7 +435,7 @@ void* Tensor::MapAhwbToCpuRead() const { auto error = AHardwareBuffer_lock(ahwb_, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN, ssbo_written_, nullptr, &ptr); - CHECK(error == 0) << "AHardwareBuffer_lock " << error; + ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error; close(ssbo_written_); ssbo_written_ = -1; return ptr; @@ -449,7 +453,7 @@ void* Tensor::MapAhwbToCpuWrite() const { void* ptr; auto error = AHardwareBuffer_lock( ahwb_, AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN, -1, nullptr, &ptr); - CHECK(error == 0) << "AHardwareBuffer_lock " << error; + ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error; return ptr; } } diff --git a/mediapipe/framework/formats/tensor_test.cc b/mediapipe/framework/formats/tensor_test.cc index 4ad4e18e..468af4ab 100644 --- a/mediapipe/framework/formats/tensor_test.cc +++ b/mediapipe/framework/formats/tensor_test.cc @@ -2,6 +2,7 @@ #include #include +#include #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" @@ -34,6 +35,17 @@ TEST(General, TestDataTypes) { EXPECT_EQ(t_bool.bytes(), t_bool.shape().num_elements() * sizeof(bool)); } +TEST(General, TestDynamic) { + Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape({1, 2, 3, 4}, true)); + EXPECT_EQ(t1.shape().num_elements(), 1 * 2 * 3 * 4); + EXPECT_TRUE(t1.shape().is_dynamic); + + std::vector t2_dims = {4, 3, 2, 3}; + Tensor t2(Tensor::ElementType::kFloat16, Tensor::Shape(t2_dims, true)); + EXPECT_EQ(t2.shape().num_elements(), 4 * 3 * 2 * 3); + EXPECT_TRUE(t2.shape().is_dynamic); +} + TEST(Cpu, TestMemoryAllocation) { Tensor t1(Tensor::ElementType::kFloat32, Tensor::Shape{4, 3, 2, 3}); auto v1 = t1.GetCpuWriteView(); diff --git a/mediapipe/framework/graph_output_stream.cc b/mediapipe/framework/graph_output_stream.cc index de024dfe..e456c653 100644 --- a/mediapipe/framework/graph_output_stream.cc +++ b/mediapipe/framework/graph_output_stream.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/graph_output_stream.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/port/status.h" @@ -153,7 +154,7 @@ void OutputStreamPollerImpl::Reset() { } void OutputStreamPollerImpl::SetMaxQueueSize(int queue_size) { - CHECK(queue_size >= -1) + ABSL_CHECK(queue_size >= -1) << "Max queue size must be either -1 or non-negative."; input_stream_handler_->SetMaxQueueSize(queue_size); } @@ -175,7 +176,7 @@ void OutputStreamPollerImpl::NotifyError() { } bool OutputStreamPollerImpl::Next(Packet* packet) { - CHECK(packet); + ABSL_CHECK(packet); bool empty_queue = true; bool timestamp_bound_changed = false; Timestamp min_timestamp = Timestamp::Unset(); @@ -212,7 +213,7 @@ bool OutputStreamPollerImpl::Next(Packet* packet) { bool stream_is_done = false; *packet = input_stream_->PopPacketAtTimestamp( min_timestamp, &num_packets_dropped, &stream_is_done); - CHECK_EQ(num_packets_dropped, 0) + ABSL_CHECK_EQ(num_packets_dropped, 0) << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", num_packets_dropped, input_stream_->Name()); } else if (timestamp_bound_changed) { diff --git a/mediapipe/framework/graph_output_stream.h b/mediapipe/framework/graph_output_stream.h index b541aec1..7308be11 100644 --- a/mediapipe/framework/graph_output_stream.h +++ b/mediapipe/framework/graph_output_stream.h @@ -22,6 +22,7 @@ #include "absl/base/attributes.h" #include "absl/base/thread_annotations.h" +#include "absl/log/absl_log.h" #include "absl/strings/substitute.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/input_stream_handler.h" @@ -30,7 +31,6 @@ #include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet_set.h" #include "mediapipe/framework/packet_type.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/timestamp.h" @@ -76,7 +76,7 @@ class GraphOutputStream { // TODO: Simplify this. We are forced to use an ISH just to // receive a packet, even though we do not need to do any of the things an ISH // normally does. The fact that we have to disable required overrides with - // LOG(FATAL) shows that this is the wrong interface. + // ABSL_LOG(FATAL) shows that this is the wrong interface. class GraphOutputStreamHandler : public InputStreamHandler { public: GraphOutputStreamHandler(std::shared_ptr tag_map, @@ -88,15 +88,15 @@ class GraphOutputStream { protected: NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override { - LOG(FATAL) << "GraphOutputStreamHandler::GetNodeReadiness should " - "never be invoked."; + ABSL_LOG(FATAL) << "GraphOutputStreamHandler::GetNodeReadiness should " + "never be invoked."; return NodeReadiness::kNotReady; } void FillInputSet(Timestamp input_timestamp, InputStreamShardSet* input_set) override { - LOG(FATAL) << "GraphOutputStreamHandler::FillInputSet should " - "never be invoked."; + ABSL_LOG(FATAL) << "GraphOutputStreamHandler::FillInputSet should " + "never be invoked."; } }; diff --git a/mediapipe/framework/graph_service.h b/mediapipe/framework/graph_service.h index 12b2ccb3..95f55bbd 100644 --- a/mediapipe/framework/graph_service.h +++ b/mediapipe/framework/graph_service.h @@ -19,6 +19,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/status.h" @@ -125,7 +126,7 @@ class ServiceBinding { public: bool IsAvailable() { return service_ != nullptr; } T& GetObject() { - CHECK(service_) << "Service is unavailable."; + ABSL_CHECK(service_) << "Service is unavailable."; return *service_; } diff --git a/mediapipe/framework/graph_validation_test.cc b/mediapipe/framework/graph_validation_test.cc index c9898383..3982adbe 100644 --- a/mediapipe/framework/graph_validation_test.cc +++ b/mediapipe/framework/graph_validation_test.cc @@ -19,6 +19,7 @@ #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" @@ -121,7 +122,7 @@ TEST(GraphValidationTest, InitializeGraphFromLinker) { TEST(GraphValidationTest, InitializeTemplateFromProtos) { mediapipe::tool::TemplateParser::Parser parser; CalculatorGraphTemplate config_1; - CHECK(parser.ParseFromString(R"( + ABSL_CHECK(parser.ParseFromString(R"( type: "PassThroughGraph" input_stream: % "INPUT:" + in_name % output_stream: "OUTPUT:stream_2" @@ -132,7 +133,7 @@ TEST(GraphValidationTest, InitializeTemplateFromProtos) { output_stream: "stream_2" # Same as input. } )", - &config_1)); + &config_1)); auto config_2 = ParseTextProtoOrDie(R"pb( input_stream: "INPUT:stream_1" output_stream: "OUTPUT:stream_2" diff --git a/mediapipe/framework/input_side_packet_handler.cc b/mediapipe/framework/input_side_packet_handler.cc index 9b01cc31..b2eccf0d 100644 --- a/mediapipe/framework/input_side_packet_handler.cc +++ b/mediapipe/framework/input_side_packet_handler.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/input_side_packet_handler.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_builder.h" @@ -82,7 +83,7 @@ absl::Status InputSidePacketHandler::SetInternal(CollectionItemId id, void InputSidePacketHandler::TriggerErrorCallback( const absl::Status& status) const { - CHECK(error_callback_); + ABSL_CHECK(error_callback_); error_callback_(status); } diff --git a/mediapipe/framework/input_stream_handler.cc b/mediapipe/framework/input_stream_handler.cc index a7bd9ef4..e222c2e6 100644 --- a/mediapipe/framework/input_stream_handler.cc +++ b/mediapipe/framework/input_stream_handler.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/input_stream_handler.h" +#include "absl/log/absl_check.h" #include "absl/strings/str_join.h" #include "absl/strings/substitute.h" #include "mediapipe/framework/collection_item_id.h" @@ -102,7 +103,7 @@ void InputStreamHandler::SetHeader(CollectionItemId id, const Packet& header) { return; } if (!input_stream_managers_.Get(id)->BackEdge()) { - CHECK_GT(unset_header_count_, 0); + ABSL_CHECK_GT(unset_header_count_, 0); if (unset_header_count_.fetch_sub(1, std::memory_order_acq_rel) == 1) { headers_ready_callback_(); } @@ -111,7 +112,7 @@ void InputStreamHandler::SetHeader(CollectionItemId id, const Packet& header) { void InputStreamHandler::UpdateInputShardHeaders( InputStreamShardSet* input_shards) { - CHECK(input_shards); + ABSL_CHECK(input_shards); for (CollectionItemId id = input_stream_managers_.BeginId(); id < input_stream_managers_.EndId(); ++id) { input_shards->Get(id).SetHeader(input_stream_managers_.Get(id)->Header()); @@ -198,7 +199,7 @@ bool InputStreamHandler::ScheduleInvocations(int max_allowance, TraceEvent(TraceEvent::READY_FOR_PROCESS) .set_node_id(calculator_context->NodeId())); } else { - CHECK(node_readiness == NodeReadiness::kReadyForClose); + ABSL_CHECK(node_readiness == NodeReadiness::kReadyForClose); // If any parallel invocations are in progress or a calculator context has // been prepared for Close(), we shouldn't prepare another calculator // context for Close(). @@ -302,7 +303,7 @@ void InputStreamHandler::SetNextTimestampBound(CollectionItemId id, void InputStreamHandler::ClearCurrentInputs( CalculatorContext* calculator_context) { - CHECK(calculator_context); + ABSL_CHECK(calculator_context); calculator_context_manager_->PopInputTimestampFromContext(calculator_context); for (auto& input : calculator_context->Inputs()) { // Invokes InputStreamShard's private method to clear packet. @@ -317,18 +318,20 @@ void InputStreamHandler::Close() { } void InputStreamHandler::SetBatchSize(int batch_size) { - CHECK(!calculator_run_in_parallel_ || batch_size == 1) + ABSL_CHECK(!calculator_run_in_parallel_ || batch_size == 1) << "Batching cannot be combined with parallel execution."; - CHECK(!late_preparation_ || batch_size == 1) + ABSL_CHECK(!late_preparation_ || batch_size == 1) << "Batching cannot be combined with late preparation."; - CHECK_GE(batch_size, 1) << "Batch size has to be greater than or equal to 1."; + ABSL_CHECK_GE(batch_size, 1) + << "Batch size has to be greater than or equal to 1."; // Source nodes shouldn't specify batch_size even if it's set to 1. - CHECK_GE(NumInputStreams(), 0) << "Source nodes cannot batch input packets."; + ABSL_CHECK_GE(NumInputStreams(), 0) + << "Source nodes cannot batch input packets."; batch_size_ = batch_size; } void InputStreamHandler::SetLatePreparation(bool late_preparation) { - CHECK(batch_size_ == 1 || !late_preparation_) + ABSL_CHECK(batch_size_ == 1 || !late_preparation_) << "Batching cannot be combined with late preparation."; late_preparation_ = late_preparation; } @@ -404,15 +407,15 @@ Timestamp SyncSet::MinPacketTimestamp() const { void SyncSet::FillInputSet(Timestamp input_timestamp, InputStreamShardSet* input_set) { - CHECK(input_timestamp.IsAllowedInStream()); - CHECK(input_set); + ABSL_CHECK(input_timestamp.IsAllowedInStream()); + ABSL_CHECK(input_set); for (CollectionItemId id : stream_ids_) { const auto& stream = input_stream_handler_->input_stream_managers_.Get(id); int num_packets_dropped = 0; bool stream_is_done = false; Packet current_packet = stream->PopPacketAtTimestamp( input_timestamp, &num_packets_dropped, &stream_is_done); - CHECK_EQ(num_packets_dropped, 0) + ABSL_CHECK_EQ(num_packets_dropped, 0) << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", num_packets_dropped, stream->Name()); input_stream_handler_->AddPacketToShard( diff --git a/mediapipe/framework/input_stream_manager.cc b/mediapipe/framework/input_stream_manager.cc index 1af2e2cc..fe63b62e 100644 --- a/mediapipe/framework/input_stream_manager.cc +++ b/mediapipe/framework/input_stream_manager.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/packet.h" @@ -244,7 +245,7 @@ Timestamp InputStreamManager::MinTimestampOrBoundHelper() const Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp, int* num_packets_dropped, bool* stream_is_done) { - CHECK(enable_timestamps_); + ABSL_CHECK(enable_timestamps_); *num_packets_dropped = -1; *stream_is_done = false; bool queue_became_non_full = false; @@ -252,7 +253,7 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp, { absl::MutexLock stream_lock(&stream_mutex_); // Make sure timestamp didn't decrease from last time. - CHECK_LE(last_select_timestamp_, timestamp); + ABSL_CHECK_LE(last_select_timestamp_, timestamp); last_select_timestamp_ = timestamp; // Make sure AddPacket and SetNextTimestampBound are not called with @@ -299,7 +300,7 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp, } Packet InputStreamManager::PopQueueHead(bool* stream_is_done) { - CHECK(!enable_timestamps_); + ABSL_CHECK(!enable_timestamps_); *stream_is_done = false; bool queue_became_non_full = false; Packet packet; diff --git a/mediapipe/framework/input_stream_shard.cc b/mediapipe/framework/input_stream_shard.cc index 8e3348dd..c7d1df8a 100644 --- a/mediapipe/framework/input_stream_shard.cc +++ b/mediapipe/framework/input_stream_shard.cc @@ -14,12 +14,14 @@ #include "mediapipe/framework/input_stream_shard.h" +#include "absl/log/absl_check.h" + namespace mediapipe { void InputStreamShard::AddPacket(Packet&& value, bool is_done) { // A packet can be added if the shard is still active or the packet being // added is empty. An empty packet corresponds to absence of a packet. - CHECK(!is_done_ || value.IsEmpty()); + ABSL_CHECK(!is_done_ || value.IsEmpty()); packet_queue_.emplace(std::move(value)); is_done_ = is_done; } diff --git a/mediapipe/framework/legacy_calculator_support.h b/mediapipe/framework/legacy_calculator_support.h index 9378d14f..6ec0d953 100644 --- a/mediapipe/framework/legacy_calculator_support.h +++ b/mediapipe/framework/legacy_calculator_support.h @@ -66,7 +66,7 @@ class LegacyCalculatorSupport { }; }; -#if !defined(_MSC_VER) +#if !defined(_MSC_VER) || defined(__clang__) // We only declare this variable for two specializations of the template because // it is only meant to be used for these two types. // Note that, since these variables are members of specific template diff --git a/mediapipe/framework/mediapipe_cc_test.bzl b/mediapipe/framework/mediapipe_cc_test.bzl index 0fc0a462..5e1daca7 100644 --- a/mediapipe/framework/mediapipe_cc_test.bzl +++ b/mediapipe/framework/mediapipe_cc_test.bzl @@ -15,7 +15,7 @@ def mediapipe_cc_test( platforms = ["linux", "android", "ios", "wasm"], exclude_platforms = None, # ios_unit_test arguments - ios_minimum_os_version = "11.0", + ios_minimum_os_version = "12.0", # android_cc_test arguments open_gl_driver = None, emulator_mini_boot = True, diff --git a/mediapipe/framework/output_side_packet_impl.cc b/mediapipe/framework/output_side_packet_impl.cc index 94bc518f..dcb54140 100644 --- a/mediapipe/framework/output_side_packet_impl.cc +++ b/mediapipe/framework/output_side_packet_impl.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/output_side_packet_impl.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/source_location.h" #include "mediapipe/framework/port/status_builder.h" @@ -42,7 +43,7 @@ void OutputSidePacketImpl::Set(const Packet& packet) { void OutputSidePacketImpl::AddMirror( InputSidePacketHandler* input_side_packet_handler, CollectionItemId id) { - CHECK(input_side_packet_handler); + ABSL_CHECK(input_side_packet_handler); mirrors_.emplace_back(input_side_packet_handler, id); } @@ -81,7 +82,7 @@ absl::Status OutputSidePacketImpl::SetInternal(const Packet& packet) { void OutputSidePacketImpl::TriggerErrorCallback( const absl::Status& status) const { - CHECK(error_callback_); + ABSL_CHECK(error_callback_); error_callback_(status); } diff --git a/mediapipe/framework/output_stream_handler.cc b/mediapipe/framework/output_stream_handler.cc index ba8f4671..377de6c8 100644 --- a/mediapipe/framework/output_stream_handler.cc +++ b/mediapipe/framework/output_stream_handler.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/output_stream_handler.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/output_stream_shard.h" @@ -31,7 +32,7 @@ absl::Status OutputStreamHandler::InitializeOutputStreamManagers( absl::Status OutputStreamHandler::SetupOutputShards( OutputStreamShardSet* output_shards) { - CHECK(output_shards); + ABSL_CHECK(output_shards); for (CollectionItemId id = output_stream_managers_.BeginId(); id < output_stream_managers_.EndId(); ++id) { OutputStreamManager* manager = output_stream_managers_.Get(id); @@ -52,7 +53,7 @@ void OutputStreamHandler::PrepareForRun( } void OutputStreamHandler::Open(OutputStreamShardSet* output_shards) { - CHECK(output_shards); + ABSL_CHECK(output_shards); PropagateOutputPackets(Timestamp::Unstarted(), output_shards); for (auto& manager : output_stream_managers_) { manager->PropagateHeader(); @@ -62,7 +63,7 @@ void OutputStreamHandler::Open(OutputStreamShardSet* output_shards) { void OutputStreamHandler::PrepareOutputs(Timestamp input_timestamp, OutputStreamShardSet* output_shards) { - CHECK(output_shards); + ABSL_CHECK(output_shards); for (CollectionItemId id = output_stream_managers_.BeginId(); id < output_stream_managers_.EndId(); ++id) { output_stream_managers_.Get(id)->ResetShard(&output_shards->Get(id)); @@ -79,7 +80,7 @@ void OutputStreamHandler::UpdateTaskTimestampBound(Timestamp timestamp) { if (task_timestamp_bound_ == timestamp) { return; } - CHECK_GT(timestamp, task_timestamp_bound_); + ABSL_CHECK_GT(timestamp, task_timestamp_bound_); task_timestamp_bound_ = timestamp; if (propagation_state_ == kPropagatingBound) { propagation_state_ = kPropagationPending; @@ -149,7 +150,7 @@ void OutputStreamHandler::Close(OutputStreamShardSet* output_shards) { void OutputStreamHandler::PropagateOutputPackets( Timestamp input_timestamp, OutputStreamShardSet* output_shards) { - CHECK(output_shards); + ABSL_CHECK(output_shards); for (CollectionItemId id = output_stream_managers_.BeginId(); id < output_stream_managers_.EndId(); ++id) { OutputStreamManager* manager = output_stream_managers_.Get(id); diff --git a/mediapipe/framework/output_stream_handler.h b/mediapipe/framework/output_stream_handler.h index 0b8dbed2..cb6b2d6e 100644 --- a/mediapipe/framework/output_stream_handler.h +++ b/mediapipe/framework/output_stream_handler.h @@ -25,6 +25,7 @@ // TODO: Move protos in another CL after the C++ code migration. #include "absl/base/thread_annotations.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_context_manager.h" #include "mediapipe/framework/collection.h" @@ -63,7 +64,7 @@ class OutputStreamHandler { calculator_context_manager_(calculator_context_manager), options_(options), calculator_run_in_parallel_(calculator_run_in_parallel) { - CHECK(calculator_context_manager_); + ABSL_CHECK(calculator_context_manager_); } virtual ~OutputStreamHandler() = default; diff --git a/mediapipe/framework/output_stream_manager.cc b/mediapipe/framework/output_stream_manager.cc index b092313e..0cb59294 100644 --- a/mediapipe/framework/output_stream_manager.cc +++ b/mediapipe/framework/output_stream_manager.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/output_stream_manager.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/input_stream_handler.h" #include "mediapipe/framework/port/status_builder.h" @@ -80,7 +81,7 @@ void OutputStreamManager::PropagateHeader() { void OutputStreamManager::AddMirror(InputStreamHandler* input_stream_handler, CollectionItemId id) { - CHECK(input_stream_handler); + ABSL_CHECK(input_stream_handler); mirrors_.emplace_back(input_stream_handler, id); } @@ -163,7 +164,7 @@ Timestamp OutputStreamManager::ComputeOutputTimestampBound( // TODO Consider moving the propagation logic to OutputStreamHandler. void OutputStreamManager::PropagateUpdatesToMirrors( Timestamp next_timestamp_bound, OutputStreamShard* output_stream_shard) { - CHECK(output_stream_shard); + ABSL_CHECK(output_stream_shard); { if (next_timestamp_bound != Timestamp::Unset()) { absl::MutexLock lock(&stream_mutex_); diff --git a/mediapipe/framework/output_stream_poller.h b/mediapipe/framework/output_stream_poller.h index 26c0e72b..98ebda31 100644 --- a/mediapipe/framework/output_stream_poller.h +++ b/mediapipe/framework/output_stream_poller.h @@ -17,6 +17,7 @@ #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/graph_output_stream.h" namespace mediapipe { @@ -34,7 +35,7 @@ class OutputStreamPoller { // Resets OutputStramPollerImpl and cleans the internal packet queue. void Reset() { auto poller = internal_poller_impl_.lock(); - CHECK(poller) << "OutputStreamPollerImpl is already destroyed."; + ABSL_CHECK(poller) << "OutputStreamPollerImpl is already destroyed."; poller->Reset(); } @@ -50,14 +51,14 @@ class OutputStreamPoller { void SetMaxQueueSize(int queue_size) { auto poller = internal_poller_impl_.lock(); - CHECK(poller) << "OutputStreamPollerImpl is already destroyed."; + ABSL_CHECK(poller) << "OutputStreamPollerImpl is already destroyed."; return poller->SetMaxQueueSize(queue_size); } // Returns the number of packets in the queue. int QueueSize() { auto poller = internal_poller_impl_.lock(); - CHECK(poller) << "OutputStreamPollerImpl is already destroyed."; + ABSL_CHECK(poller) << "OutputStreamPollerImpl is already destroyed."; return poller->QueueSize(); } diff --git a/mediapipe/framework/output_stream_shard.cc b/mediapipe/framework/output_stream_shard.cc index 682c704c..3b24321f 100644 --- a/mediapipe/framework/output_stream_shard.cc +++ b/mediapipe/framework/output_stream_shard.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/output_stream_shard.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/source_location.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" @@ -23,7 +24,7 @@ namespace mediapipe { OutputStreamShard::OutputStreamShard() : closed_(false) {} void OutputStreamShard::SetSpec(OutputStreamSpec* output_stream_spec) { - CHECK(output_stream_spec); + ABSL_CHECK(output_stream_spec); output_stream_spec_ = output_stream_spec; } diff --git a/mediapipe/framework/output_stream_shard.h b/mediapipe/framework/output_stream_shard.h index 718174c4..81a89759 100644 --- a/mediapipe/framework/output_stream_shard.h +++ b/mediapipe/framework/output_stream_shard.h @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/output_stream.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet_type.h" @@ -34,7 +35,7 @@ struct OutputStreamSpec { // Triggers the error callback with absl::Status info when an error // occurs. void TriggerErrorCallback(const absl::Status& status) const { - CHECK(error_callback); + ABSL_CHECK(error_callback); error_callback(status); } diff --git a/mediapipe/framework/packet.cc b/mediapipe/framework/packet.cc index 05d3c6c5..edcdaf19 100644 --- a/mediapipe/framework/packet.cc +++ b/mediapipe/framework/packet.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/packet.h" +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/canonical_errors.h" @@ -135,10 +136,11 @@ absl::Status Packet::ValidateAsProtoMessageLite() const { } const proto_ns::MessageLite& Packet::GetProtoMessageLite() const { - CHECK(holder_ != nullptr) << "The packet is empty."; + ABSL_CHECK(holder_ != nullptr) << "The packet is empty."; const proto_ns::MessageLite* proto = holder_->GetProtoMessageLite(); - CHECK(proto != nullptr) << "The Packet stores '" << holder_->DebugTypeName() - << "', it cannot be converted to MessageLite type."; + ABSL_CHECK(proto != nullptr) + << "The Packet stores '" << holder_->DebugTypeName() + << "', it cannot be converted to MessageLite type."; return *proto; } diff --git a/mediapipe/framework/packet.h b/mediapipe/framework/packet.h index af2ec5a9..770dd9d4 100644 --- a/mediapipe/framework/packet.h +++ b/mediapipe/framework/packet.h @@ -18,11 +18,14 @@ #define MEDIAPIPE_FRAMEWORK_PACKET_H_ #include +#include #include #include #include #include "absl/base/macros.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" @@ -368,11 +371,14 @@ class HolderBase { } // Returns a printable string identifying the type stored in the holder. virtual const std::string DebugTypeName() const = 0; + // Returns debug data id. + virtual int64_t DebugDataId() const = 0; // Returns the registered type name if it's available, otherwise the // empty string. virtual const std::string RegisteredTypeName() const = 0; // Get the type id of the underlying data type. virtual TypeId GetTypeId() const = 0; + // Downcasts this to Holder. Returns nullptr if deserialization // failed or if the requested type is not what is stored. template @@ -451,61 +457,37 @@ struct is_concrete_proto_t !std::is_same{} && !std::is_same{}> {}; -// Registers a message type. T must be a non-cv-qualified concrete proto type. template -struct MessageRegistrationImpl { - static NoDestructor registration; - // This could have been a lambda inside registration's initializer below, but - // MSVC has a bug with lambdas, so we put it here as a workaround. - static std::unique_ptr> CreateMessageHolder() { - return absl::make_unique>(new T); - } -}; +std::unique_ptr CreateMessageHolder() { + return absl::make_unique>(new T); +} -// Static members of template classes can be defined in the header. -template -NoDestructor - MessageRegistrationImpl::registration(MessageHolderRegistry::Register( - T{}.GetTypeName(), MessageRegistrationImpl::CreateMessageHolder, - __FILE__, __LINE__)); +// Registers a message type. T must be a non-cv-qualified concrete proto type. +MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(MessageRegistrator, MessageHolderRegistry, + T{}.GetTypeName(), CreateMessageHolder) // For non-Message payloads, this does nothing. template -struct HolderSupport { - static void EnsureStaticInit() {} -}; +struct HolderPayloadRegistrator {}; // This template ensures that, for each concrete MessageLite subclass that is // stored in a Packet, we register a function that allows us to create a // Holder with the correct payload type from the proto's type name. +// +// We must use std::remove_cv to ensure we don't try to register Foo twice if +// there are Holder and Holder. TODO: lift this +// up to Holder? template -struct HolderSupport{}>::type> { - // We must use std::remove_cv to ensure we don't try to register Foo twice if - // there are Holder and Holder. TODO: lift this - // up to Holder? - using R = MessageRegistrationImpl::type>; - // For the registration static member to be instantiated, it needs to be - // referenced in a context that requires the definition to exist (see ISO/IEC - // C++ 2003 standard, 14.7.1). Calling this ensures that's the case. - // We need two different call-sites to cover proto types for which packets - // are only ever created (i.e. the protos are only produced by calculators) - // and proto types for which packets are only ever consumed (i.e. the protos - // are only consumed by calculators). - static void EnsureStaticInit() { CHECK(R::registration.get() != nullptr); } -}; +struct HolderPayloadRegistrator< + T, typename std::enable_if{}>::type> + : private MessageRegistrator::type> {}; template -class Holder : public HolderBase { +class Holder : public HolderBase, private HolderPayloadRegistrator { public: - explicit Holder(const T* ptr) : ptr_(ptr) { - HolderSupport::EnsureStaticInit(); - } + explicit Holder(const T* ptr) : ptr_(ptr) {} ~Holder() override { delete_helper(); } - const T& data() const { - HolderSupport::EnsureStaticInit(); - return *ptr_; - } + const T& data() const { return *ptr_; } TypeId GetTypeId() const final { return kTypeId; } // Releases the underlying data pointer and transfers the ownership to a // unique pointer. @@ -535,6 +517,7 @@ class Holder : public HolderBase { const std::string DebugTypeName() const final { return MediaPipeTypeStringOrDemangled(); } + int64_t DebugDataId() const final { return reinterpret_cast(ptr_); } const std::string RegisteredTypeName() const final { const std::string* type_string = MediaPipeTypeString(); if (type_string) { @@ -743,7 +726,7 @@ inline Packet& Packet::operator=(Packet&& packet) { inline bool Packet::IsEmpty() const { return holder_ == nullptr; } inline TypeId Packet::GetTypeId() const { - CHECK(holder_); + ABSL_CHECK(holder_); return holder_->GetTypeId(); } @@ -753,7 +736,7 @@ inline const T& Packet::Get() const { if (holder == nullptr) { // Produce a good error message. absl::Status status = ValidateAsType(); - LOG(FATAL) << "Packet::Get() failed: " << status.message(); + ABSL_LOG(FATAL) << "Packet::Get() failed: " << status.message(); } return holder->data(); } @@ -762,13 +745,13 @@ inline Timestamp Packet::Timestamp() const { return timestamp_; } template Packet Adopt(const T* ptr) { - CHECK(ptr != nullptr); + ABSL_CHECK(ptr != nullptr); return packet_internal::Create(new packet_internal::Holder(ptr)); } template Packet PointToForeign(const T* ptr) { - CHECK(ptr != nullptr); + ABSL_CHECK(ptr != nullptr); return packet_internal::Create(new packet_internal::ForeignHolder(ptr)); } diff --git a/mediapipe/framework/packet_registration_test.cc b/mediapipe/framework/packet_registration_test.cc index 30c7c789..7b2ea1f7 100644 --- a/mediapipe/framework/packet_registration_test.cc +++ b/mediapipe/framework/packet_registration_test.cc @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include + #include "absl/strings/str_cat.h" +#include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet_test.pb.h" @@ -24,6 +28,9 @@ namespace mediapipe { namespace { +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Stream; + namespace test_ns { constexpr char kOutTag[] = "OUT"; @@ -48,7 +55,7 @@ REGISTER_CALCULATOR(TestSinkCalculator); } // namespace test_ns -TEST(PacketTest, InputTypeRegistration) { +TEST(PacketRegistrationTest, InputTypeRegistration) { using testing::Contains; ASSERT_EQ(mediapipe::InputOnlyProto{}.GetTypeName(), "mediapipe.InputOnlyProto"); @@ -56,5 +63,33 @@ TEST(PacketTest, InputTypeRegistration) { Contains("mediapipe.InputOnlyProto")); } +TEST(PacketRegistrationTest, AdoptingRegisteredProtoWorks) { + CalculatorGraphConfig config; + { + Graph graph; + Stream input = + graph.In(0).SetName("in").Cast(); + + auto& sink_node = graph.AddNode("TestSinkCalculator"); + input.ConnectTo(sink_node.In(test_ns::kInTag)); + Stream output = sink_node.Out(test_ns::kOutTag).Cast(); + + output.ConnectTo(graph.Out(0)).SetName("out"); + + config = graph.GetConfig(); + } + + CalculatorGraph calculator_graph; + MP_ASSERT_OK(calculator_graph.Initialize(std::move(config))); + MP_ASSERT_OK(calculator_graph.StartRun({})); + + int value = 10; + auto proto = std::make_unique(); + proto->set_x(value); + MP_ASSERT_OK(calculator_graph.AddPacketToInputStream( + "in", Adopt(proto.release()).At(Timestamp(0)))); + MP_ASSERT_OK(calculator_graph.WaitUntilIdle()); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/framework/packet_type.h b/mediapipe/framework/packet_type.h index 9b4bbd36..10496f05 100644 --- a/mediapipe/framework/packet_type.h +++ b/mediapipe/framework/packet_type.h @@ -23,6 +23,8 @@ #include #include "absl/base/macros.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" @@ -162,15 +164,15 @@ class PacketTypeSetErrorHandler { if (!missing_) { missing_ = absl::make_unique(); } - CHECK(!missing_->initialized_errors); + ABSL_CHECK(!missing_->initialized_errors); std::string key = absl::StrCat(tag, ":", index); return missing_->entries[key]; } // In the const setting produce a FATAL error. const PacketType& GetFallback(const absl::string_view tag, int index) const { - LOG(FATAL) << "Failed to get tag \"" << tag << "\" index " << index - << ". Unable to defer error due to const specifier."; + ABSL_LOG(FATAL) << "Failed to get tag \"" << tag << "\" index " << index + << ". Unable to defer error due to const specifier."; std::abort(); } @@ -181,9 +183,9 @@ class PacketTypeSetErrorHandler { // Get the error messages that have been deferred. // This function can only be called if HasError() is true. const std::vector& ErrorMessages() const { - CHECK(missing_) << "ErrorMessages() can only be called if errors have " - "occurred. Call HasError() before calling this " - "function."; + ABSL_CHECK(missing_) << "ErrorMessages() can only be called if errors have " + "occurred. Call HasError() before calling this " + "function."; if (!missing_->initialized_errors) { for (const auto& entry : missing_->entries) { // Optional entries that were missing are not considered errors. diff --git a/mediapipe/framework/port/BUILD b/mediapipe/framework/port/BUILD index cae439bc..f8c95d68 100644 --- a/mediapipe/framework/port/BUILD +++ b/mediapipe/framework/port/BUILD @@ -261,8 +261,8 @@ cc_library( ) cc_library( - name = "opencv_highgui", - hdrs = ["opencv_highgui_inc.h"], + name = "opencv_photo", + hdrs = ["opencv_photo_inc.h"], deps = [ ":opencv_core", "//third_party:opencv", @@ -297,6 +297,15 @@ cc_library( ], ) +cc_library( + name = "opencv_highgui", + hdrs = ["opencv_highgui_inc.h"], + deps = [ + ":opencv_core", + "//third_party:opencv", + ], +) + cc_library( name = "opencv_videoio", hdrs = ["opencv_videoio_inc.h"], @@ -317,6 +326,7 @@ cc_library( ":core_proto", ":logging", "//mediapipe/framework:port", + "@com_google_absl//absl/log:absl_check", ], ) diff --git a/mediapipe/framework/port/opencv_highgui_inc.h b/mediapipe/framework/port/opencv_highgui_inc.h index c3ca4b7f..c79804e1 100644 --- a/mediapipe/framework/port/opencv_highgui_inc.h +++ b/mediapipe/framework/port/opencv_highgui_inc.h @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef MEDIAPIPE_PORT_OPENCV_HIGHGUI_INC_H_ -#define MEDIAPIPE_PORT_OPENCV_HIGHGUI_INC_H_ +#ifndef MEDIAPIPE_FRAMEWORK_PORT_OPENCV_HIGHGUI_INC_H_ +#define MEDIAPIPE_FRAMEWORK_PORT_OPENCV_HIGHGUI_INC_H_ #include @@ -25,4 +25,4 @@ #include #endif -#endif // MEDIAPIPE_PORT_OPENCV_HIGHGUI_INC_H_ +#endif // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_HIGHGUI_INC_H_ diff --git a/mediapipe/framework/port/opencv_imgcodecs_inc.h b/mediapipe/framework/port/opencv_imgcodecs_inc.h index 60bcd49e..4c867ed5 100644 --- a/mediapipe/framework/port/opencv_imgcodecs_inc.h +++ b/mediapipe/framework/port/opencv_imgcodecs_inc.h @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2022 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/mediapipe/calculators/core/clip_detection_vector_size_calculator.cc b/mediapipe/framework/port/opencv_photo_inc.h similarity index 59% rename from mediapipe/calculators/core/clip_detection_vector_size_calculator.cc rename to mediapipe/framework/port/opencv_photo_inc.h index 55bcf2fe..1416fda7 100644 --- a/mediapipe/calculators/core/clip_detection_vector_size_calculator.cc +++ b/mediapipe/framework/port/opencv_photo_inc.h @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#ifndef MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_ +#define MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_ -#include "mediapipe/calculators/core/clip_vector_size_calculator.h" -#include "mediapipe/framework/formats/detection.pb.h" +#include "third_party/OpenCV/photo.hpp" -namespace mediapipe { - -typedef ClipVectorSizeCalculator<::mediapipe::Detection> - ClipDetectionVectorSizeCalculator; -REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator); - -} // namespace mediapipe +#endif // MEDIAPIPE_PORT_OPENCV_PHOTO_INC_H_ diff --git a/mediapipe/framework/port/opencv_video_inc.h b/mediapipe/framework/port/opencv_video_inc.h index dc84bf59..5f06d923 100644 --- a/mediapipe/framework/port/opencv_video_inc.h +++ b/mediapipe/framework/port/opencv_video_inc.h @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2022 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/mediapipe/framework/port/parse_text_proto.h b/mediapipe/framework/port/parse_text_proto.h index c352d4f0..722ded6e 100644 --- a/mediapipe/framework/port/parse_text_proto.h +++ b/mediapipe/framework/port/parse_text_proto.h @@ -15,6 +15,7 @@ #ifndef MEDIAPIPE_PORT_PARSE_TEXT_PROTO_H_ #define MEDIAPIPE_PORT_PARSE_TEXT_PROTO_H_ +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/core_proto_inc.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/proto_ns.h" @@ -29,7 +30,7 @@ bool ParseTextProto(const std::string& input, T* proto) { template T ParseTextProtoOrDie(const std::string& input) { T result; - CHECK(ParseTextProto(input, &result)); + ABSL_CHECK(ParseTextProto(input, &result)); return result; } diff --git a/mediapipe/framework/profiler/BUILD b/mediapipe/framework/profiler/BUILD index 53aeb1ea..99699f2c 100644 --- a/mediapipe/framework/profiler/BUILD +++ b/mediapipe/framework/profiler/BUILD @@ -116,13 +116,14 @@ cc_library( "//mediapipe/framework/port:advanced_proto_lite", "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:re2", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:name_util", "//mediapipe/framework/tool:tag_map", "//mediapipe/framework/tool:validate_name", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", @@ -218,11 +219,11 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_options_cc_proto", "//mediapipe/framework:mediapipe_options_cc_proto", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/framework/tool:tag_map", "//mediapipe/framework/tool:tag_map_helper", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", ], ) @@ -257,6 +258,7 @@ cc_test( "//mediapipe/framework/tool:simulation_clock_executor", "//mediapipe/framework/tool:status_util", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/time", ], ) @@ -268,9 +270,9 @@ cc_test( ":sharded_map", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:threadpool", "@com_google_absl//absl/container:node_hash_map", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", ], @@ -374,6 +376,7 @@ cc_test( "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/profiler/reporter:reporter_lib", "//mediapipe/framework/tool:test_util", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", ], diff --git a/mediapipe/framework/profiler/gl_context_profiler.cc b/mediapipe/framework/profiler/gl_context_profiler.cc index 59c9f01f..667d153d 100644 --- a/mediapipe/framework/profiler/gl_context_profiler.cc +++ b/mediapipe/framework/profiler/gl_context_profiler.cc @@ -14,6 +14,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/time/clock.h" #include "absl/time/time.h" diff --git a/mediapipe/framework/profiler/graph_profiler.cc b/mediapipe/framework/profiler/graph_profiler.cc index 6aead525..94995511 100644 --- a/mediapipe/framework/profiler/graph_profiler.cc +++ b/mediapipe/framework/profiler/graph_profiler.cc @@ -17,13 +17,14 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/substitute.h" #include "absl/synchronization/mutex.h" #include "absl/time/time.h" #include "mediapipe/framework/port/advanced_proto_lite_inc.h" #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/proto_ns.h" #include "mediapipe/framework/port/re2.h" #include "mediapipe/framework/port/ret_check.h" @@ -158,7 +159,7 @@ void GraphProfiler::Initialize( const ValidatedGraphConfig& validated_graph_config) { absl::WriterMutexLock lock(&profiler_mutex_); validated_graph_ = &validated_graph_config; - CHECK(!is_initialized_) + ABSL_CHECK(!is_initialized_) << "Cannot initialize the profiler for the same graph multiple times."; profiler_config_ = validated_graph_config.Config().profiler_config(); int64 interval_size_usec = profiler_config_.histogram_interval_size_usec(); @@ -190,7 +191,7 @@ void GraphProfiler::Initialize( } auto iter = calculator_profiles_.insert({node_name, profile}); - CHECK(iter.second) << absl::Substitute( + ABSL_CHECK(iter.second) << absl::Substitute( "Calculator \"$0\" has already been added.", node_name); } profile_builder_ = std::make_unique(this); @@ -201,7 +202,7 @@ void GraphProfiler::Initialize( void GraphProfiler::SetClock(const std::shared_ptr& clock) { absl::WriterMutexLock lock(&profiler_mutex_); - CHECK(clock) << "GraphProfiler::SetClock() is called with a nullptr."; + ABSL_CHECK(clock) << "GraphProfiler::SetClock() is called with a nullptr."; clock_ = clock; } @@ -251,10 +252,10 @@ absl::Status GraphProfiler::Start(mediapipe::Executor* executor) { file::SetContents(absl::StrCat(trace_log_path, "trace_writing_check"), "can write trace logs to this location"); if (status.ok()) { - LOG(INFO) << "trace_log_path: " << trace_log_path; + ABSL_LOG(INFO) << "trace_log_path: " << trace_log_path; } else { - LOG(ERROR) << "cannot write to trace_log_path: " << trace_log_path << ": " - << status; + ABSL_LOG(ERROR) << "cannot write to trace_log_path: " << trace_log_path + << ": " << status; } is_running_ = true; @@ -315,7 +316,7 @@ void GraphProfiler::AddPacketInfo(const TraceEvent& packet_info) { return; } if (!packet_timestamp.IsRangeValue()) { - LOG(WARNING) << absl::Substitute( + ABSL_LOG(WARNING) << absl::Substitute( "Skipped adding packet info because the timestamp $0 for stream " "\"$1\" is not valid.", packet_timestamp.Value(), stream_name); @@ -386,7 +387,7 @@ std::set GraphProfiler::GetBackEdgeIds( tool::ParseTagIndex(input_stream_info.tag_index(), &tag, &index)) << absl::Substitute("Cannot parse TAG or index for the backedge \"$0\"", input_stream_info.tag_index()); - CHECK(0 <= index && index < input_tag_map.NumEntries(tag)) + ABSL_CHECK(0 <= index && index < input_tag_map.NumEntries(tag)) << absl::Substitute( "The input_stream_info for tag \"$0\" (index " "$1) does not match any input_stream.", @@ -445,7 +446,7 @@ void GraphProfiler::SetOpenRuntime(const CalculatorContext& calculator_context, const std::string& node_name = calculator_context.NodeName(); int64 time_usec = end_time_usec - start_time_usec; auto profile_iter = calculator_profiles_.find(node_name); - CHECK(profile_iter != calculator_profiles_.end()) << absl::Substitute( + ABSL_CHECK(profile_iter != calculator_profiles_.end()) << absl::Substitute( "Calculator \"$0\" has not been added during initialization.", calculator_context.NodeName()); CalculatorProfile* calculator_profile = &profile_iter->second; @@ -467,7 +468,7 @@ void GraphProfiler::SetCloseRuntime(const CalculatorContext& calculator_context, const std::string& node_name = calculator_context.NodeName(); int64 time_usec = end_time_usec - start_time_usec; auto profile_iter = calculator_profiles_.find(node_name); - CHECK(profile_iter != calculator_profiles_.end()) << absl::Substitute( + ABSL_CHECK(profile_iter != calculator_profiles_.end()) << absl::Substitute( "Calculator \"$0\" has not been added during initialization.", calculator_context.NodeName()); CalculatorProfile* calculator_profile = &profile_iter->second; @@ -482,7 +483,7 @@ void GraphProfiler::SetCloseRuntime(const CalculatorContext& calculator_context, void GraphProfiler::AddTimeSample(int64 start_time_usec, int64 end_time_usec, TimeHistogram* histogram) { if (end_time_usec < start_time_usec) { - LOG(ERROR) << absl::Substitute( + ABSL_LOG(ERROR) << absl::Substitute( "end_time_usec ($0) is < start_time_usec ($1)", end_time_usec, start_time_usec); return; @@ -519,8 +520,8 @@ int64 GraphProfiler::AddInputStreamTimeSamples( // This is a condition rather than a failure CHECK because // under certain conditions the consumer calculator's Process() // can start before the producer calculator's Process() is finished. - LOG_FIRST_N(WARNING, 10) << "Expected packet info is missing for: " - << PacketIdToString(packet_id); + ABSL_LOG_FIRST_N(WARNING, 10) << "Expected packet info is missing for: " + << PacketIdToString(packet_id); continue; } AddTimeSample( @@ -545,7 +546,7 @@ void GraphProfiler::AddProcessSample( const std::string& node_name = calculator_context.NodeName(); auto profile_iter = calculator_profiles_.find(node_name); - CHECK(profile_iter != calculator_profiles_.end()) << absl::Substitute( + ABSL_CHECK(profile_iter != calculator_profiles_.end()) << absl::Substitute( "Calculator \"$0\" has not been added during initialization.", calculator_context.NodeName()); CalculatorProfile* calculator_profile = &profile_iter->second; diff --git a/mediapipe/framework/profiler/graph_profiler_test.cc b/mediapipe/framework/profiler/graph_profiler_test.cc index e9badaa2..8a9bc141 100644 --- a/mediapipe/framework/profiler/graph_profiler_test.cc +++ b/mediapipe/framework/profiler/graph_profiler_test.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/profiler/graph_profiler.h" +#include "absl/log/absl_log.h" #include "absl/status/statusor.h" #include "absl/synchronization/mutex.h" #include "absl/time/time.h" @@ -59,7 +60,8 @@ CalculatorProfile GetProfileWithName( return p; } } - LOG(FATAL) << "Cannot find calulator profile with name " << calculator_name; + ABSL_LOG(FATAL) << "Cannot find calulator profile with name " + << calculator_name; return CalculatorProfile::default_instance(); } @@ -1227,7 +1229,7 @@ TEST(GraphProfilerTest, ParallelReads) { EXPECT_EQ(1003, profiles[0].process_runtime().count(0)); EXPECT_EQ(1000, profiles[1].process_runtime().count(0)); } else { - LOG(FATAL) << "Unexpected profile name " << profiles[0].name(); + ABSL_LOG(FATAL) << "Unexpected profile name " << profiles[0].name(); } EXPECT_EQ(1001, out_1_packets.size()); } diff --git a/mediapipe/framework/profiler/graph_tracer_test.cc b/mediapipe/framework/profiler/graph_tracer_test.cc index c1cc819c..4fe9826c 100644 --- a/mediapipe/framework/profiler/graph_tracer_test.cc +++ b/mediapipe/framework/profiler/graph_tracer_test.cc @@ -22,6 +22,7 @@ #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" #include "absl/time/time.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -332,7 +333,7 @@ TEST_F(GraphTracerTest, GraphTrace) { class GraphTracerE2ETest : public ::testing::Test { protected: void SetUpPassThroughGraph() { - CHECK(proto_ns::TextFormat::ParseFromString(R"( + ABSL_CHECK(proto_ns::TextFormat::ParseFromString(R"( input_stream: "input_0" node { calculator: "LambdaCalculator" @@ -346,11 +347,11 @@ class GraphTracerE2ETest : public ::testing::Test { trace_enabled: true } )", - &graph_config_)); + &graph_config_)); } void SetUpDemuxInFlightGraph() { - CHECK(proto_ns::TextFormat::ParseFromString(R"( + ABSL_CHECK(proto_ns::TextFormat::ParseFromString(R"( node { calculator: "LambdaCalculator" input_side_packet: 'callback_2' @@ -404,7 +405,7 @@ class GraphTracerE2ETest : public ::testing::Test { trace_enabled: true } )", - &graph_config_)); + &graph_config_)); } absl::Time ParseTime(const std::string& date_time_str) { @@ -1372,7 +1373,7 @@ TEST_F(GraphTracerE2ETest, GpuTaskTrace) { // Show that trace_enabled activates the GlContextProfiler. TEST_F(GraphTracerE2ETest, GpuTracing) { - CHECK(proto_ns::TextFormat::ParseFromString(R"( + ABSL_CHECK(proto_ns::TextFormat::ParseFromString(R"( input_stream: "input_buffer" input_stream: "render_data" output_stream: "annotated_buffer" @@ -1386,7 +1387,7 @@ TEST_F(GraphTracerE2ETest, GpuTracing) { trace_enabled: true } )", - &graph_config_)); + &graph_config_)); // Create the CalculatorGraph with only trace_enabled set. MP_ASSERT_OK(graph_.Initialize(graph_config_, {})); @@ -1423,5 +1424,13 @@ TEST_F(GraphTracerE2ETest, DestructGraph) { } } +TEST(TraceBuilderTest, EventDataIsExtracted) { + int value = 10; + Packet p = PointToForeign(&value); + TraceEvent event; + event.set_packet_data_id(&p); + EXPECT_EQ(event.event_data, reinterpret_cast(&value)); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/framework/profiler/reporter_test.cc b/mediapipe/framework/profiler/reporter_test.cc index e5bc541a..6ca6c642 100644 --- a/mediapipe/framework/profiler/reporter_test.cc +++ b/mediapipe/framework/profiler/reporter_test.cc @@ -21,6 +21,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator.pb.h" @@ -43,15 +44,15 @@ using ::testing::IsSupersetOf; void LoadGraphProfile(const std::string& path, GraphProfile* proto) { int fd = open(path.c_str(), O_RDONLY); if (fd == -1) { - LOG(ERROR) << "could not open test graph: " << path - << ", error: " << strerror(errno); + ABSL_LOG(ERROR) << "could not open test graph: " << path + << ", error: " << strerror(errno); return; } proto_ns::io::FileInputStream input(fd); bool success = proto->ParseFromZeroCopyStream(&input); close(fd); if (!success) { - LOG(ERROR) << "could not parse test graph: " << path; + ABSL_LOG(ERROR) << "could not parse test graph: " << path; } } diff --git a/mediapipe/framework/profiler/sharded_map_test.cc b/mediapipe/framework/profiler/sharded_map_test.cc index e551b25c..5a47b390 100644 --- a/mediapipe/framework/profiler/sharded_map_test.cc +++ b/mediapipe/framework/profiler/sharded_map_test.cc @@ -17,13 +17,13 @@ #include #include "absl/container/node_hash_map.h" +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" #include "absl/time/clock.h" #include "absl/time/time.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/threadpool.h" namespace { @@ -134,9 +134,9 @@ TEST(ShardedMapTest, TestParallelAccess) { ShardedMap sharded_map(4999); TestParallelAccess(sharded_map, 13); }); - LOG(INFO) << "Ellapsed time: simple_map: " << simple_time; - LOG(INFO) << "Ellapsed time: safe_map: " << safe_time; - LOG(INFO) << "Ellapsed time: sharded_map: " << sharded_time; + ABSL_LOG(INFO) << "Ellapsed time: simple_map: " << simple_time; + ABSL_LOG(INFO) << "Ellapsed time: safe_map: " << safe_time; + ABSL_LOG(INFO) << "Ellapsed time: sharded_map: " << sharded_time; } } // namespace diff --git a/mediapipe/framework/profiler/test_context_builder.h b/mediapipe/framework/profiler/test_context_builder.h index abf9ee74..4018a034 100644 --- a/mediapipe/framework/profiler/test_context_builder.h +++ b/mediapipe/framework/profiler/test_context_builder.h @@ -21,11 +21,11 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/mediapipe_options.pb.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/statusor.h" #include "mediapipe/framework/tool/tag_map.h" @@ -92,7 +92,7 @@ class TestContextBuilder { spec.name = output_map_->Names()[id.value()]; spec.packet_type = packet_type; spec.error_callback = [](const absl::Status& status) { - LOG(ERROR) << status; + ABSL_LOG(ERROR) << status; }; output_specs_[spec.name] = spec; } diff --git a/mediapipe/framework/profiler/testing/BUILD b/mediapipe/framework/profiler/testing/BUILD index 67668ef7..55b3613f 100644 --- a/mediapipe/framework/profiler/testing/BUILD +++ b/mediapipe/framework/profiler/testing/BUILD @@ -23,6 +23,7 @@ cc_library( deps = [ "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) diff --git a/mediapipe/framework/profiler/testing/simple_calculator.cc b/mediapipe/framework/profiler/testing/simple_calculator.cc index 18ba67b9..fa1123ee 100644 --- a/mediapipe/framework/profiler/testing/simple_calculator.cc +++ b/mediapipe/framework/profiler/testing/simple_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/status.h" @@ -28,7 +29,7 @@ class SimpleCalculator : public CalculatorBase { } absl::Status Process(CalculatorContext* cc) final { - LOG(WARNING) << "Simple Calculator Process called, count_: " << count_; + ABSL_LOG(WARNING) << "Simple Calculator Process called, count_: " << count_; int max_count = 1; if (cc->InputSidePackets().HasTag("MAX_COUNT")) { max_count = cc->InputSidePackets().Tag("MAX_COUNT").Get(); diff --git a/mediapipe/framework/profiler/trace_buffer.h b/mediapipe/framework/profiler/trace_buffer.h index b5e2d999..8dc09aef 100644 --- a/mediapipe/framework/profiler/trace_buffer.h +++ b/mediapipe/framework/profiler/trace_buffer.h @@ -15,6 +15,9 @@ #ifndef MEDIAPIPE_FRAMEWORK_PROFILER_TRACE_BUFFER_H_ #define MEDIAPIPE_FRAMEWORK_PROFILER_TRACE_BUFFER_H_ +#include +#include + #include "absl/time/time.h" #include "mediapipe/framework/calculator_profile.pb.h" #include "mediapipe/framework/packet.h" @@ -23,17 +26,6 @@ namespace mediapipe { -namespace packet_internal { -// Returns a hash of the packet data address from a packet data holder. -inline const int64 GetPacketDataId(const HolderBase* holder) { - if (holder == nullptr) { - return 0; - } - const void* address = &(static_cast*>(holder)->data()); - return reinterpret_cast(address); -} -} // namespace packet_internal - // Packet trace log event. struct TraceEvent { using EventType = GraphTrace::EventType; @@ -75,8 +67,12 @@ struct TraceEvent { return *this; } inline TraceEvent& set_packet_data_id(const Packet* packet) { - this->event_data = - packet_internal::GetPacketDataId(packet_internal::GetHolder(*packet)); + const auto* holder = packet_internal::GetHolder(*packet); + int64_t data_id = 0; + if (holder != nullptr) { + data_id = holder->DebugDataId(); + } + this->event_data = data_id; return *this; } inline TraceEvent& set_thread_id(int thread_id) { diff --git a/mediapipe/framework/scheduler.cc b/mediapipe/framework/scheduler.cc index ceadce78..36effe01 100644 --- a/mediapipe/framework/scheduler.cc +++ b/mediapipe/framework/scheduler.cc @@ -19,6 +19,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_graph.h" @@ -77,7 +78,7 @@ void Scheduler::Reset() { void Scheduler::CloseAllSourceNodes() { shared_.stopping = true; } void Scheduler::SetExecutor(Executor* executor) { - CHECK_EQ(state_, STATE_NOT_STARTED) + ABSL_CHECK_EQ(state_, STATE_NOT_STARTED) << "SetExecutor must not be called after the scheduler has started"; default_queue_.SetExecutor(executor); } @@ -147,7 +148,7 @@ void Scheduler::HandleIdle() { // Note: TryToScheduleNextSourceLayer unlocks and locks state_mutex_ // internally. bool did_activate = TryToScheduleNextSourceLayer(); - CHECK(did_activate || active_sources_.empty()); + ABSL_CHECK(did_activate || active_sources_.empty()); continue; } @@ -183,7 +184,7 @@ void Scheduler::HandleIdle() { void Scheduler::Quit() { // All calls to Calculator::Process() have returned (even if we had an // error). - CHECK(state_ == STATE_RUNNING || state_ == STATE_CANCELLING); + ABSL_CHECK(state_ == STATE_RUNNING || state_ == STATE_CANCELLING); SetQueuesRunning(false); shared_.timer.EndRun(); @@ -198,7 +199,7 @@ void Scheduler::Start() { shared_.timer.StartRun(); { absl::MutexLock lock(&state_mutex_); - CHECK_EQ(state_, STATE_NOT_STARTED); + ABSL_CHECK_EQ(state_, STATE_NOT_STARTED); state_ = STATE_RUNNING; SetQueuesRunning(true); @@ -270,13 +271,6 @@ absl::Status Scheduler::WaitForObservedOutput() { return observed ? absl::OkStatus() : absl::OutOfRangeError("Graph is done."); } -// Idleness requires: -// 1. either the graph has no source nodes or all source nodes are closed, and -// 2. no packets are added to graph input streams. -// For simplicity, we only fully support WaitUntilIdle() to be called on a graph -// with no source nodes. -// The application must ensure no other threads are adding packets to graph -// input streams while a WaitUntilIdle() call is in progress. absl::Status Scheduler::WaitUntilIdle() { RET_CHECK_NE(state_, STATE_NOT_STARTED); ApplicationThreadAwait(std::bind(&Scheduler::IsIdle, this)); @@ -333,15 +327,15 @@ void Scheduler::ClosedAllGraphInputStreams() { // container. void Scheduler::ScheduleNodeIfNotThrottled( CalculatorNode* node, CalculatorContext* calculator_context) { - DCHECK(node); - DCHECK(calculator_context); + ABSL_DCHECK(node); + ABSL_DCHECK(calculator_context); if (!graph_->IsNodeThrottled(node->Id())) { node->GetSchedulerQueue()->AddNode(node, calculator_context); } } void Scheduler::ScheduleNodeForOpen(CalculatorNode* node) { - DCHECK(node); + ABSL_DCHECK(node); VLOG(1) << "Scheduling OpenNode of calculator " << node->DebugName(); node->GetSchedulerQueue()->AddNodeForOpen(node); } @@ -351,7 +345,7 @@ void Scheduler::ScheduleUnthrottledReadyNodes( for (CalculatorNode* node : nodes_to_schedule) { // Source nodes always reuse the default calculator context because they // can't be executed in parallel. - CHECK(node->IsSource()); + ABSL_CHECK(node->IsSource()); CalculatorContext* default_context = node->GetDefaultCalculatorContext(); node->GetSchedulerQueue()->AddNode(node, default_context); } @@ -374,8 +368,8 @@ void Scheduler::CleanupActiveSources() { bool Scheduler::TryToScheduleNextSourceLayer() { VLOG(3) << "TryToScheduleNextSourceLayer"; - CHECK(active_sources_.empty()); - CHECK(!sources_queue_.empty()); + ABSL_CHECK(active_sources_.empty()); + ABSL_CHECK(!sources_queue_.empty()); if (!unopened_sources_.empty() && (*unopened_sources_.begin())->source_layer() < @@ -427,8 +421,9 @@ bool Scheduler::TryToScheduleNextSourceLayer() { } void Scheduler::AddUnopenedSourceNode(CalculatorNode* node) { - CHECK_EQ(state_, STATE_NOT_STARTED) << "AddUnopenedSourceNode can only be " - "called before starting the scheduler"; + ABSL_CHECK_EQ(state_, STATE_NOT_STARTED) + << "AddUnopenedSourceNode can only be " + "called before starting the scheduler"; unopened_sources_.insert(node); } @@ -445,7 +440,7 @@ void Scheduler::AssignNodeToSchedulerQueue(CalculatorNode* node) { SchedulerQueue* queue; if (!node->Executor().empty()) { auto iter = non_default_queues_.find(node->Executor()); - CHECK(iter != non_default_queues_.end()); + ABSL_CHECK(iter != non_default_queues_.end()); queue = iter->second.get(); } else { queue = &default_queue_; @@ -528,7 +523,7 @@ void Scheduler::CleanupAfterRun() { while (!sources_queue_.empty()) { sources_queue_.pop(); } - CHECK(app_thread_tasks_.empty()); + ABSL_CHECK(app_thread_tasks_.empty()); } for (auto queue : scheduler_queues_) { queue->CleanupAfterRun(); @@ -539,7 +534,7 @@ void Scheduler::CleanupAfterRun() { } internal::SchedulerTimes Scheduler::GetSchedulerTimes() { - CHECK_EQ(state_, STATE_TERMINATED); + ABSL_CHECK_EQ(state_, STATE_TERMINATED); return shared_.timer.GetSchedulerTimes(); } diff --git a/mediapipe/framework/scheduler.h b/mediapipe/framework/scheduler.h index 8a6d079e..22d552c7 100644 --- a/mediapipe/framework/scheduler.h +++ b/mediapipe/framework/scheduler.h @@ -76,6 +76,16 @@ class Scheduler { // be scheduled and nothing is running in the worker threads. This function // can be called only after Start(). // Runs application thread tasks while waiting. + // + // Idleness requires: + // 1. either the graph has no source nodes or all source nodes are closed, and + // 2. no packets are added to graph input streams. + // + // For simplicity, we only fully support WaitUntilIdle() to be called on a + // graph with no source nodes. + // + // The application must ensure no other threads are adding packets to graph + // input streams while a WaitUntilIdle() call is in progress. absl::Status WaitUntilIdle() ABSL_LOCKS_EXCLUDED(state_mutex_); // Wait until any graph input stream has been unthrottled. diff --git a/mediapipe/framework/scheduler_queue.cc b/mediapipe/framework/scheduler_queue.cc index 33214cf6..557d7e40 100644 --- a/mediapipe/framework/scheduler_queue.cc +++ b/mediapipe/framework/scheduler_queue.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_node.h" #include "mediapipe/framework/executor.h" @@ -36,8 +37,8 @@ namespace internal { SchedulerQueue::Item::Item(CalculatorNode* node, CalculatorContext* cc) : node_(node), cc_(cc) { - CHECK(node); - CHECK(cc); + ABSL_CHECK(node); + ABSL_CHECK(cc); is_source_ = node->IsSource(); id_ = node->Id(); if (is_source_) { @@ -48,7 +49,7 @@ SchedulerQueue::Item::Item(CalculatorNode* node, CalculatorContext* cc) SchedulerQueue::Item::Item(CalculatorNode* node) : node_(node), cc_(nullptr), is_open_node_(true) { - CHECK(node); + ABSL_CHECK(node); is_source_ = node->IsSource(); id_ = node->Id(); if (is_source_) { @@ -104,7 +105,7 @@ bool SchedulerQueue::IsIdle() { void SchedulerQueue::SetRunning(bool running) { absl::MutexLock lock(&mutex_); running_count_ += running ? 1 : -1; - DCHECK_LE(running_count_, 1); + ABSL_DCHECK_LE(running_count_, 1); } void SchedulerQueue::AddNode(CalculatorNode* node, CalculatorContext* cc) { @@ -117,7 +118,7 @@ void SchedulerQueue::AddNode(CalculatorNode* node, CalculatorContext* cc) { // Only happens when the framework tries to schedule an unthrottled source // node while it's running. For non-source nodes, if a calculator context is // prepared, it is committed to be scheduled. - CHECK(node->IsSource()) << node->DebugName(); + ABSL_CHECK(node->IsSource()) << node->DebugName(); return; } AddItemToQueue(Item(node, cc)); @@ -192,15 +193,16 @@ void SchedulerQueue::RunNextTask() { { absl::MutexLock lock(&mutex_); - CHECK(!queue_.empty()) << "Called RunNextTask when the queue is empty. " - "This should not happen."; + ABSL_CHECK(!queue_.empty()) + << "Called RunNextTask when the queue is empty. " + "This should not happen."; node = queue_.top().Node(); calculator_context = queue_.top().Context(); is_open_node = queue_.top().IsOpenNode(); queue_.pop(); - CHECK(!node->Closed()) + ABSL_CHECK(!node->Closed()) << "Scheduled a node that was closed. This should not happen."; } @@ -211,7 +213,7 @@ void SchedulerQueue::RunNextTask() { // do it here to ensure all executors are covered. AUTORELEASEPOOL { if (is_open_node) { - DCHECK(!calculator_context); + ABSL_DCHECK(!calculator_context); OpenCalculatorNode(node); } else { RunCalculatorNode(node, calculator_context); @@ -221,7 +223,7 @@ void SchedulerQueue::RunNextTask() { bool is_idle; { absl::MutexLock lock(&mutex_); - DCHECK_GT(num_pending_tasks_, 0); + ABSL_DCHECK_GT(num_pending_tasks_, 0); --num_pending_tasks_; is_idle = IsIdle(); } @@ -266,8 +268,8 @@ void SchedulerQueue::RunCalculatorNode(CalculatorNode* node, // that all sources will be closed and no further sources should be // scheduled. The graph will be terminated as soon as its scheduler // queue becomes empty. - CHECK(!node->IsSource()); // ProcessNode takes care of StatusStop() - // from sources. + ABSL_CHECK(!node->IsSource()); // ProcessNode takes care of + // StatusStop() from sources. shared_->stopping = true; } else { // If we have an error in this calculator. @@ -299,8 +301,8 @@ void SchedulerQueue::CleanupAfterRun() { { absl::MutexLock lock(&mutex_); was_idle = IsIdle(); - CHECK_EQ(num_pending_tasks_, 0); - CHECK_EQ(num_tasks_to_add_, queue_.size()); + ABSL_CHECK_EQ(num_pending_tasks_, 0); + ABSL_CHECK_EQ(num_tasks_to_add_, queue_.size()); num_tasks_to_add_ = 0; while (!queue_.empty()) { queue_.pop(); diff --git a/mediapipe/framework/stream_handler/BUILD b/mediapipe/framework/stream_handler/BUILD index 8b54ade8..c3eb334f 100644 --- a/mediapipe/framework/stream_handler/BUILD +++ b/mediapipe/framework/stream_handler/BUILD @@ -53,8 +53,16 @@ mediapipe_proto_library( cc_library( name = "barrier_input_stream_handler", srcs = ["barrier_input_stream_handler.cc"], + hdrs = ["barrier_input_stream_handler.h"], deps = [ + "//mediapipe/framework:calculator_context_manager", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:collection_item_id", "//mediapipe/framework:input_stream_handler", + "//mediapipe/framework:mediapipe_options_cc_proto", + "//mediapipe/framework/tool:tag_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", ], alwayslink = 1, ) @@ -74,8 +82,15 @@ cc_library( cc_library( name = "early_close_input_stream_handler", srcs = ["early_close_input_stream_handler.cc"], + hdrs = ["early_close_input_stream_handler.h"], deps = [ + "//mediapipe/framework:calculator_context_manager", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:collection_item_id", "//mediapipe/framework:input_stream_handler", + "//mediapipe/framework:mediapipe_options_cc_proto", + "//mediapipe/framework/tool:tag_map", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -84,10 +99,21 @@ cc_library( cc_library( name = "fixed_size_input_stream_handler", srcs = ["fixed_size_input_stream_handler.cc"], + hdrs = ["fixed_size_input_stream_handler.h"], deps = [ ":default_input_stream_handler", ":fixed_size_input_stream_handler_cc_proto", + "//mediapipe/framework:calculator_context_manager", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:collection_item_id", "//mediapipe/framework:input_stream_handler", + "//mediapipe/framework:mediapipe_options_cc_proto", + "//mediapipe/framework:packet", + "//mediapipe/framework/tool:tag_map", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/synchronization", ], alwayslink = 1, ) @@ -95,8 +121,18 @@ cc_library( cc_library( name = "immediate_input_stream_handler", srcs = ["immediate_input_stream_handler.cc"], + hdrs = ["immediate_input_stream_handler.h"], deps = [ + "//mediapipe/framework:calculator_context_manager", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:collection_item_id", "//mediapipe/framework:input_stream_handler", + "//mediapipe/framework:mediapipe_options_cc_proto", + "//mediapipe/framework/tool:tag_map", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/synchronization", ], alwayslink = 1, ) @@ -115,6 +151,7 @@ cc_library( "//mediapipe/framework:packet_set", "//mediapipe/framework:timestamp", "//mediapipe/framework/tool:tag_map", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) @@ -122,9 +159,13 @@ cc_library( cc_library( name = "mux_input_stream_handler", srcs = ["mux_input_stream_handler.cc"], + hdrs = ["mux_input_stream_handler.h"], deps = [ + "//mediapipe/framework:calculator_context_manager", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:collection_item_id", "//mediapipe/framework:input_stream_handler", - "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -134,16 +175,23 @@ cc_library( cc_library( name = "sync_set_input_stream_handler", srcs = ["sync_set_input_stream_handler.cc"], + hdrs = ["sync_set_input_stream_handler.h"], deps = [ ":sync_set_input_stream_handler_cc_proto", - "//mediapipe/framework:collection", + "//mediapipe/framework:calculator_context_manager", + "//mediapipe/framework:calculator_framework", "//mediapipe/framework:collection_item_id", "//mediapipe/framework:input_stream_handler", "//mediapipe/framework:mediapipe_options_cc_proto", "//mediapipe/framework:packet_set", "//mediapipe/framework:timestamp", + "//mediapipe/framework/port:map_util", + "//mediapipe/framework/port:status", "//mediapipe/framework/tool:tag_map", - "@com_google_absl//absl/strings", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", "@com_google_absl//absl/synchronization", ], alwayslink = 1, @@ -152,12 +200,19 @@ cc_library( cc_library( name = "timestamp_align_input_stream_handler", srcs = ["timestamp_align_input_stream_handler.cc"], + hdrs = ["timestamp_align_input_stream_handler.h"], deps = [ ":timestamp_align_input_stream_handler_cc_proto", + "//mediapipe/framework:calculator_context_manager", + "//mediapipe/framework:calculator_framework", "//mediapipe/framework:collection_item_id", "//mediapipe/framework:input_stream_handler", + "//mediapipe/framework:mediapipe_options_cc_proto", "//mediapipe/framework:timestamp", "//mediapipe/framework/tool:validate_name", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -176,6 +231,7 @@ cc_test( "//mediapipe/framework/tool:tag_map", "//mediapipe/framework/tool:tag_map_helper", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", ], ) @@ -194,6 +250,7 @@ cc_test( "//mediapipe/framework/tool:tag_map", "//mediapipe/framework/tool:tag_map_helper", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", ], ) diff --git a/mediapipe/framework/stream_handler/barrier_input_stream_handler.cc b/mediapipe/framework/stream_handler/barrier_input_stream_handler.cc index ece873b1..4150fafa 100644 --- a/mediapipe/framework/stream_handler/barrier_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/barrier_input_stream_handler.cc @@ -11,84 +11,70 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/framework/stream_handler/barrier_input_stream_handler.h" -#include -#include -#include +#include +#include +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/input_stream_handler.h" namespace mediapipe { -// Implementation of an input stream handler that considers a node as ready for -// Process() if all input streams have a packet available. This implies it must -// consider a node as ready for Close() if any input stream is done. -class BarrierInputStreamHandler : public InputStreamHandler { - public: - BarrierInputStreamHandler() = delete; - BarrierInputStreamHandler( - std::shared_ptr tag_map, - CalculatorContextManager* calculator_context_manager, - const MediaPipeOptions& options, bool calculator_run_in_parallel) - : InputStreamHandler(std::move(tag_map), calculator_context_manager, - options, calculator_run_in_parallel) {} - - void PrepareForRun( - std::function headers_ready_callback, - std::function notification_callback, - std::function schedule_callback, - std::function error_callback) override { - InputStreamHandler::PrepareForRun( - std::move(headers_ready_callback), std::move(notification_callback), - std::move(schedule_callback), std::move(error_callback)); - for (auto& stream : input_stream_managers_) { - stream->DisableTimestamps(); - } +void BarrierInputStreamHandler::PrepareForRun( + std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) { + InputStreamHandler::PrepareForRun( + std::move(headers_ready_callback), std::move(notification_callback), + std::move(schedule_callback), std::move(error_callback)); + for (auto& stream : input_stream_managers_) { + stream->DisableTimestamps(); } +} - protected: - // In BarrierInputStreamHandler, a node is "ready" if: - // - any stream is done (need to call Close() in this case), or - // - all streams have a packet available. - NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override { - DCHECK(min_stream_timestamp); - *min_stream_timestamp = Timestamp::Done(); - bool all_available = true; - for (const auto& stream : input_stream_managers_) { - bool empty; - Timestamp stream_timestamp = stream->MinTimestampOrBound(&empty); - if (empty) { - if (stream_timestamp == Timestamp::Done()) { - *min_stream_timestamp = Timestamp::Done(); - return NodeReadiness::kReadyForClose; - } - all_available = false; +NodeReadiness BarrierInputStreamHandler::GetNodeReadiness( + Timestamp* min_stream_timestamp) { + ABSL_DCHECK(min_stream_timestamp); + *min_stream_timestamp = Timestamp::Done(); + bool all_available = true; + for (const auto& stream : input_stream_managers_) { + bool empty; + Timestamp stream_timestamp = stream->MinTimestampOrBound(&empty); + if (empty) { + if (stream_timestamp == Timestamp::Done()) { + *min_stream_timestamp = Timestamp::Done(); + return NodeReadiness::kReadyForClose; } - *min_stream_timestamp = std::min(*min_stream_timestamp, stream_timestamp); + all_available = false; } - - CHECK_NE(*min_stream_timestamp, Timestamp::Done()); - if (all_available) { - return NodeReadiness::kReadyForProcess; - } - return NodeReadiness::kNotReady; + *min_stream_timestamp = std::min(*min_stream_timestamp, stream_timestamp); } - // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. - void FillInputSet(Timestamp input_timestamp, - InputStreamShardSet* input_set) override { - CHECK(input_timestamp.IsAllowedInStream()); - CHECK(input_set); - for (CollectionItemId id = input_stream_managers_.BeginId(); - id < input_stream_managers_.EndId(); ++id) { - auto& stream = input_stream_managers_.Get(id); - bool stream_is_done = false; - Packet current_packet = stream->PopQueueHead(&stream_is_done); - AddPacketToShard(&input_set->Get(id), std::move(current_packet), - stream_is_done); - } + ABSL_CHECK_NE(*min_stream_timestamp, Timestamp::Done()); + if (all_available) { + return NodeReadiness::kReadyForProcess; } -}; + return NodeReadiness::kNotReady; +} + +void BarrierInputStreamHandler::FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) { + ABSL_CHECK(input_timestamp.IsAllowedInStream()); + ABSL_CHECK(input_set); + for (CollectionItemId id = input_stream_managers_.BeginId(); + id < input_stream_managers_.EndId(); ++id) { + auto& stream = input_stream_managers_.Get(id); + bool stream_is_done = false; + Packet current_packet = stream->PopQueueHead(&stream_is_done); + AddPacketToShard(&input_set->Get(id), std::move(current_packet), + stream_is_done); + } +} REGISTER_INPUT_STREAM_HANDLER(BarrierInputStreamHandler); diff --git a/mediapipe/framework/stream_handler/barrier_input_stream_handler.h b/mediapipe/framework/stream_handler/barrier_input_stream_handler.h new file mode 100644 index 00000000..55a21d33 --- /dev/null +++ b/mediapipe/framework/stream_handler/barrier_input_stream_handler.h @@ -0,0 +1,64 @@ + +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_BARRIER_INPUT_STREAM_HANDLER_H_ +#define MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_BARRIER_INPUT_STREAM_HANDLER_H_ + +#include +#include +#include + +#include "absl/status/status.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/mediapipe_options.pb.h" +#include "mediapipe/framework/tool/tag_map.h" + +namespace mediapipe { + +// Implementation of an input stream handler that considers a node as ready for +// Process() if all input streams have a packet available. This implies it must +// consider a node as ready for Close() if any input stream is done. +class BarrierInputStreamHandler : public InputStreamHandler { + public: + BarrierInputStreamHandler() = delete; + BarrierInputStreamHandler( + std::shared_ptr tag_map, + CalculatorContextManager* calculator_context_manager, + const mediapipe::MediaPipeOptions& options, + bool calculator_run_in_parallel) + : InputStreamHandler(std::move(tag_map), calculator_context_manager, + options, calculator_run_in_parallel) {} + + void PrepareForRun(std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) override; + + protected: + // In BarrierInputStreamHandler, a node is "ready" if: + // - any stream is done (need to call Close() in this case), or + // - all streams have a packet available. + NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; + + // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. + void FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) override; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_BARRIER_INPUT_STREAM_HANDLER_H_ diff --git a/mediapipe/framework/stream_handler/barrier_input_stream_handler_test.cc b/mediapipe/framework/stream_handler/barrier_input_stream_handler_test.cc index 9f341ba5..deb04fc3 100644 --- a/mediapipe/framework/stream_handler/barrier_input_stream_handler_test.cc +++ b/mediapipe/framework/stream_handler/barrier_input_stream_handler_test.cc @@ -18,6 +18,7 @@ #include #include "absl/base/macros.h" +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context_manager.h" @@ -105,7 +106,7 @@ class BarrierInputStreamHandlerTest : public ::testing::Test { void NotifyNoOp() {} void Schedule(CalculatorContext* calculator_context) { - CHECK(calculator_context); + ABSL_CHECK(calculator_context); calculator_context_ = calculator_context; } diff --git a/mediapipe/framework/stream_handler/early_close_input_stream_handler.cc b/mediapipe/framework/stream_handler/early_close_input_stream_handler.cc index 983b986c..3a7dd867 100644 --- a/mediapipe/framework/stream_handler/early_close_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/early_close_input_stream_handler.cc @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,81 +11,70 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/framework/stream_handler/early_close_input_stream_handler.h" #include -#include -#include +#include "absl/log/absl_check.h" #include "absl/strings/substitute.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/input_stream_handler.h" namespace mediapipe { -// Implementation of an input stream handler that considers a node as ready for -// Close() if any input stream is done. -class EarlyCloseInputStreamHandler : public InputStreamHandler { - public: - EarlyCloseInputStreamHandler() = delete; - EarlyCloseInputStreamHandler(std::shared_ptr tag_map, - CalculatorContextManager* cc_manager, - const MediaPipeOptions& options, - bool calculator_run_in_parallel) - : InputStreamHandler(std::move(tag_map), cc_manager, options, - calculator_run_in_parallel) {} - - protected: - // In EarlyCloseInputStreamHandler, a node is "ready" if: - // - any stream is done (need to call Close() in this case), or - // - the minimum bound (over all empty streams) is greater than the smallest - // timestamp of any stream, which means we have received all the packets - // that will be available at the next timestamp. - NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override { - DCHECK(min_stream_timestamp); - *min_stream_timestamp = Timestamp::Done(); - Timestamp min_bound = Timestamp::Done(); - for (const auto& stream : input_stream_managers_) { - bool empty; - Timestamp stream_timestamp = stream->MinTimestampOrBound(&empty); - if (empty) { - if (stream_timestamp == Timestamp::Done()) { - *min_stream_timestamp = Timestamp::Done(); - return NodeReadiness::kReadyForClose; - } - min_bound = std::min(min_bound, stream_timestamp); +// In EarlyCloseInputStreamHandler, a node is "ready" if: +// - any stream is done (need to call Close() in this case), or +// - the minimum bound (over all empty streams) is greater than the smallest +// timestamp of any stream, which means we have received all the packets +// that will be available at the next timestamp. +NodeReadiness EarlyCloseInputStreamHandler::GetNodeReadiness( + Timestamp* min_stream_timestamp) { + ABSL_DCHECK(min_stream_timestamp); + *min_stream_timestamp = Timestamp::Done(); + Timestamp min_bound = Timestamp::Done(); + for (const auto& stream : input_stream_managers_) { + bool empty; + Timestamp stream_timestamp = stream->MinTimestampOrBound(&empty); + if (empty) { + if (stream_timestamp == Timestamp::Done()) { + *min_stream_timestamp = Timestamp::Done(); + return NodeReadiness::kReadyForClose; } - *min_stream_timestamp = std::min(*min_stream_timestamp, stream_timestamp); + min_bound = std::min(min_bound, stream_timestamp); } - - CHECK_NE(*min_stream_timestamp, Timestamp::Done()); - - if (min_bound > *min_stream_timestamp) { - return NodeReadiness::kReadyForProcess; - } - - CHECK_EQ(min_bound, *min_stream_timestamp); - return NodeReadiness::kNotReady; + *min_stream_timestamp = std::min(*min_stream_timestamp, stream_timestamp); } - // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. - void FillInputSet(Timestamp input_timestamp, - InputStreamShardSet* input_set) override { - CHECK(input_timestamp.IsAllowedInStream()); - CHECK(input_set); - for (CollectionItemId id = input_stream_managers_.BeginId(); - id < input_stream_managers_.EndId(); ++id) { - auto& stream = input_stream_managers_.Get(id); - int num_packets_dropped = 0; - bool stream_is_done = false; - Packet current_packet = stream->PopPacketAtTimestamp( - input_timestamp, &num_packets_dropped, &stream_is_done); - CHECK_EQ(num_packets_dropped, 0) - << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", - num_packets_dropped, stream->Name()); - AddPacketToShard(&input_set->Get(id), std::move(current_packet), - stream_is_done); - } + ABSL_CHECK_NE(*min_stream_timestamp, Timestamp::Done()); + + if (min_bound > *min_stream_timestamp) { + return NodeReadiness::kReadyForProcess; } -}; + + ABSL_CHECK_EQ(min_bound, *min_stream_timestamp); + return NodeReadiness::kNotReady; +} + +// Only invoked when associated GetNodeReadiness() returned kReadyForProcess. +void EarlyCloseInputStreamHandler::FillInputSet( + Timestamp input_timestamp, InputStreamShardSet* input_set) { + ABSL_CHECK(input_timestamp.IsAllowedInStream()); + ABSL_CHECK(input_set); + for (CollectionItemId id = input_stream_managers_.BeginId(); + id < input_stream_managers_.EndId(); ++id) { + auto& stream = input_stream_managers_.Get(id); + int num_packets_dropped = 0; + bool stream_is_done = false; + Packet current_packet = stream->PopPacketAtTimestamp( + input_timestamp, &num_packets_dropped, &stream_is_done); + ABSL_CHECK_EQ(num_packets_dropped, 0) + << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", + num_packets_dropped, stream->Name()); + AddPacketToShard(&input_set->Get(id), std::move(current_packet), + stream_is_done); + } +} REGISTER_INPUT_STREAM_HANDLER(EarlyCloseInputStreamHandler); diff --git a/mediapipe/framework/stream_handler/early_close_input_stream_handler.h b/mediapipe/framework/stream_handler/early_close_input_stream_handler.h new file mode 100644 index 00000000..081954ef --- /dev/null +++ b/mediapipe/framework/stream_handler/early_close_input_stream_handler.h @@ -0,0 +1,56 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_EARLY_CLOSE_INPUT_STREAM_HANDLER_H_ +#define MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_EARLY_CLOSE_INPUT_STREAM_HANDLER_H_ + +#include +#include + +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/mediapipe_options.pb.h" +#include "mediapipe/framework/tool/tag_map.h" + +namespace mediapipe { + +// Implementation of an input stream handler that considers a node as ready for +// Close() if any input stream is done. +class EarlyCloseInputStreamHandler : public InputStreamHandler { + public: + EarlyCloseInputStreamHandler() = delete; + EarlyCloseInputStreamHandler(std::shared_ptr tag_map, + CalculatorContextManager* cc_manager, + const mediapipe::MediaPipeOptions& options, + bool calculator_run_in_parallel) + : InputStreamHandler(std::move(tag_map), cc_manager, options, + calculator_run_in_parallel) {} + + protected: + // In EarlyCloseInputStreamHandler, a node is "ready" if: + // - any stream is done (need to call Close() in this case), or + // - the minimum bound (over all empty streams) is greater than the smallest + // timestamp of any stream, which means we have received all the packets + // that will be available at the next timestamp. + NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; + + // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. + void FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) override; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_EARLY_CLOSE_INPUT_STREAM_HANDLER_H_ diff --git a/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc b/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc index fd51a738..cb4e0faf 100644 --- a/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -11,219 +11,185 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/framework/stream_handler/fixed_size_input_stream_handler.h" +#include +#include #include +#include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/mediapipe_options.pb.h" +#include "mediapipe/framework/packet.h" #include "mediapipe/framework/stream_handler/default_input_stream_handler.h" -// TODO: Move protos in another CL after the C++ code migration. #include "mediapipe/framework/stream_handler/fixed_size_input_stream_handler.pb.h" +#include "mediapipe/framework/tool/tag_map.h" namespace mediapipe { -// Input stream handler that limits each input queue to a maximum of -// target_queue_size packets, discarding older packets as needed. When a -// timestamp is dropped from a stream, it is dropped from all others as well. -// -// For example, a calculator node with one input stream and the following input -// stream handler specs: -// -// node { -// calculator: "CalculatorRunningAtOneFps" -// input_stream: "packets_streaming_in_at_ten_fps" -// input_stream_handler { -// input_stream_handler: "FixedSizeInputStreamHandler" -// } -// } -// -// will always try to keep the newest packet in the input stream. -// -// A few details: FixedSizeInputStreamHandler takes action when any stream grows -// to trigger_queue_size or larger. It then keeps at most target_queue_size -// packets in every InputStreamImpl. Every stream is truncated at the same -// timestamp, so that each included timestamp delivers the same packets as -// DefaultInputStreamHandler includes. -// -class FixedSizeInputStreamHandler : public DefaultInputStreamHandler { - public: - FixedSizeInputStreamHandler() = delete; - FixedSizeInputStreamHandler(std::shared_ptr tag_map, - CalculatorContextManager* cc_manager, - const MediaPipeOptions& options, - bool calculator_run_in_parallel) - : DefaultInputStreamHandler(std::move(tag_map), cc_manager, options, - calculator_run_in_parallel) { - const auto& ext = - options.GetExtension(FixedSizeInputStreamHandlerOptions::ext); - trigger_queue_size_ = ext.trigger_queue_size(); - target_queue_size_ = ext.target_queue_size(); - fixed_min_size_ = ext.fixed_min_size(); - pending_ = false; - kept_timestamp_ = Timestamp::Unset(); - // TODO: Either re-enable SetLatePreparation(true) with - // CalculatorContext::InputTimestamp set correctly, or remove the - // implementation of SetLatePreparation. - } +FixedSizeInputStreamHandler::FixedSizeInputStreamHandler( + std::shared_ptr tag_map, CalculatorContextManager* cc_manager, + const mediapipe::MediaPipeOptions& options, bool calculator_run_in_parallel) + : DefaultInputStreamHandler(std::move(tag_map), cc_manager, options, + calculator_run_in_parallel) { + const auto& ext = + options.GetExtension(mediapipe::FixedSizeInputStreamHandlerOptions::ext); + trigger_queue_size_ = ext.trigger_queue_size(); + target_queue_size_ = ext.target_queue_size(); + fixed_min_size_ = ext.fixed_min_size(); + pending_ = false; + kept_timestamp_ = Timestamp::Unset(); + // TODO: Either re-enable SetLatePreparation(true) with + // CalculatorContext::InputTimestamp set correctly, or remove the + // implementation of SetLatePreparation. +} - private: - // Drops packets if all input streams exceed trigger_queue_size. - void EraseAllSurplus() ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) { - Timestamp min_timestamp_all_streams = Timestamp::Max(); - for (const auto& stream : input_stream_managers_) { - // Check whether every InputStreamImpl grew beyond trigger_queue_size. - if (stream->QueueSize() < trigger_queue_size_) { - return; - } - Timestamp min_timestamp = - stream->GetMinTimestampAmongNLatest(target_queue_size_); - - // Record the min timestamp among the newest target_queue_size_ packets - // across all InputStreamImpls. - min_timestamp_all_streams = - std::min(min_timestamp_all_streams, min_timestamp); +void FixedSizeInputStreamHandler::EraseAllSurplus() { + Timestamp min_timestamp_all_streams = Timestamp::Max(); + for (const auto& stream : input_stream_managers_) { + // Check whether every InputStreamImpl grew beyond trigger_queue_size. + if (stream->QueueSize() < trigger_queue_size_) { + return; } - for (auto& stream : input_stream_managers_) { - stream->ErasePacketsEarlierThan(min_timestamp_all_streams); + Timestamp min_timestamp = + stream->GetMinTimestampAmongNLatest(target_queue_size_); + + // Record the min timestamp among the newest target_queue_size_ packets + // across all InputStreamImpls. + min_timestamp_all_streams = + std::min(min_timestamp_all_streams, min_timestamp); + } + for (auto& stream : input_stream_managers_) { + stream->ErasePacketsEarlierThan(min_timestamp_all_streams); + } +} + +Timestamp FixedSizeInputStreamHandler::PreviousAllowedInStream( + Timestamp bound) { + return bound.IsRangeValue() ? bound - 1 : bound; +} + +Timestamp FixedSizeInputStreamHandler::MinStreamBound() { + Timestamp min_bound = Timestamp::Done(); + for (const auto& stream : input_stream_managers_) { + Timestamp stream_bound = stream->GetMinTimestampAmongNLatest(1); + if (stream_bound > Timestamp::Unset()) { + stream_bound = stream_bound.NextAllowedInStream(); + } else { + stream_bound = stream->MinTimestampOrBound(nullptr); + } + min_bound = std::min(min_bound, stream_bound); + } + return min_bound; +} + +Timestamp FixedSizeInputStreamHandler::MinTimestampToProcess() { + Timestamp min_bound = Timestamp::Done(); + for (const auto& stream : input_stream_managers_) { + bool empty; + Timestamp stream_timestamp = stream->MinTimestampOrBound(&empty); + // If we're using the stream's *bound*, we only want to process up to the + // packet *before* the bound, because a packet may still arrive at that + // time. + if (empty) { + stream_timestamp = PreviousAllowedInStream(stream_timestamp); + } + min_bound = std::min(min_bound, stream_timestamp); + } + return min_bound; +} + +void FixedSizeInputStreamHandler::EraseAnySurplus(bool keep_one) { + // Record the most recent first kept timestamp on any stream. + for (const auto& stream : input_stream_managers_) { + int32_t queue_size = (stream->QueueSize() >= trigger_queue_size_) + ? target_queue_size_ + : trigger_queue_size_ - 1; + if (stream->QueueSize() > queue_size) { + kept_timestamp_ = std::max( + kept_timestamp_, stream->GetMinTimestampAmongNLatest(queue_size + 1) + .NextAllowedInStream()); } } - - // Returns the latest timestamp allowed before a bound. - Timestamp PreviousAllowedInStream(Timestamp bound) { - return bound.IsRangeValue() ? bound - 1 : bound; + if (keep_one) { + // In order to preserve one viable timestamp, do not truncate past + // the timestamp bound of the least current stream. + kept_timestamp_ = + std::min(kept_timestamp_, PreviousAllowedInStream(MinStreamBound())); } - - // Returns the lowest timestamp at which a packet may arrive at any stream. - Timestamp MinStreamBound() { - Timestamp min_bound = Timestamp::Done(); - for (const auto& stream : input_stream_managers_) { - Timestamp stream_bound = stream->GetMinTimestampAmongNLatest(1); - if (stream_bound > Timestamp::Unset()) { - stream_bound = stream_bound.NextAllowedInStream(); - } else { - stream_bound = stream->MinTimestampOrBound(nullptr); - } - min_bound = std::min(min_bound, stream_bound); - } - return min_bound; + for (auto& stream : input_stream_managers_) { + stream->ErasePacketsEarlierThan(kept_timestamp_); } +} - // Returns the lowest timestamp of a packet ready to process. - Timestamp MinTimestampToProcess() { - Timestamp min_bound = Timestamp::Done(); - for (const auto& stream : input_stream_managers_) { - bool empty; - Timestamp stream_timestamp = stream->MinTimestampOrBound(&empty); - // If we're using the stream's *bound*, we only want to process up to the - // packet *before* the bound, because a packet may still arrive at that - // time. - if (empty) { - stream_timestamp = PreviousAllowedInStream(stream_timestamp); - } - min_bound = std::min(min_bound, stream_timestamp); - } - return min_bound; +void FixedSizeInputStreamHandler::EraseSurplusPackets(bool keep_one) { + return (fixed_min_size_) ? EraseAllSurplus() : EraseAnySurplus(keep_one); +} + +NodeReadiness FixedSizeInputStreamHandler::GetNodeReadiness( + Timestamp* min_stream_timestamp) { + ABSL_DCHECK(min_stream_timestamp); + absl::MutexLock lock(&erase_mutex_); + // kReadyForProcess is returned only once until FillInputSet completes. + // In late_preparation mode, GetNodeReadiness must return kReadyForProcess + // exactly once for each input-set produced. Here, GetNodeReadiness + // releases just one input-set at a time and then disables input queue + // truncation until that promised input-set is consumed. + if (pending_) { + return NodeReadiness::kNotReady; } + EraseSurplusPackets(false); + NodeReadiness result = + DefaultInputStreamHandler::GetNodeReadiness(min_stream_timestamp); - // Keeps only the most recent target_queue_size packets in each stream - // exceeding trigger_queue_size. Also, discards all packets older than the - // first kept timestamp on any stream. - void EraseAnySurplus(bool keep_one) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) { - // Record the most recent first kept timestamp on any stream. - for (const auto& stream : input_stream_managers_) { - int32_t queue_size = (stream->QueueSize() >= trigger_queue_size_) - ? target_queue_size_ - : trigger_queue_size_ - 1; - if (stream->QueueSize() > queue_size) { - kept_timestamp_ = std::max( - kept_timestamp_, stream->GetMinTimestampAmongNLatest(queue_size + 1) - .NextAllowedInStream()); - } - } - if (keep_one) { - // In order to preserve one viable timestamp, do not truncate past - // the timestamp bound of the least current stream. - kept_timestamp_ = - std::min(kept_timestamp_, PreviousAllowedInStream(MinStreamBound())); - } - for (auto& stream : input_stream_managers_) { - stream->ErasePacketsEarlierThan(kept_timestamp_); - } - } - - void EraseSurplusPackets(bool keep_one) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_) { - return (fixed_min_size_) ? EraseAllSurplus() : EraseAnySurplus(keep_one); - } - - NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override { - DCHECK(min_stream_timestamp); - absl::MutexLock lock(&erase_mutex_); - // kReadyForProcess is returned only once until FillInputSet completes. - // In late_preparation mode, GetNodeReadiness must return kReadyForProcess - // exactly once for each input-set produced. Here, GetNodeReadiness - // releases just one input-set at a time and then disables input queue - // truncation until that promised input-set is consumed. - if (pending_) { - return NodeReadiness::kNotReady; - } + // If a packet has arrived below kept_timestamp_, recalculate. + while (*min_stream_timestamp < kept_timestamp_ && + result == NodeReadiness::kReadyForProcess) { EraseSurplusPackets(false); - NodeReadiness result = - DefaultInputStreamHandler::GetNodeReadiness(min_stream_timestamp); - - // If a packet has arrived below kept_timestamp_, recalculate. - while (*min_stream_timestamp < kept_timestamp_ && - result == NodeReadiness::kReadyForProcess) { - EraseSurplusPackets(false); - result = - DefaultInputStreamHandler::GetNodeReadiness(min_stream_timestamp); - } - pending_ = (result == NodeReadiness::kReadyForProcess); - return result; + result = DefaultInputStreamHandler::GetNodeReadiness(min_stream_timestamp); } + pending_ = (result == NodeReadiness::kReadyForProcess); + return result; +} - void AddPackets(CollectionItemId id, - const std::list& packets) override { - InputStreamHandler::AddPackets(id, packets); - absl::MutexLock lock(&erase_mutex_); - if (!pending_) { - EraseSurplusPackets(false); - } +void FixedSizeInputStreamHandler::AddPackets(CollectionItemId id, + const std::list& packets) { + InputStreamHandler::AddPackets(id, packets); + absl::MutexLock lock(&erase_mutex_); + if (!pending_) { + EraseSurplusPackets(false); } +} - void MovePackets(CollectionItemId id, std::list* packets) override { - InputStreamHandler::MovePackets(id, packets); - absl::MutexLock lock(&erase_mutex_); - if (!pending_) { - EraseSurplusPackets(false); - } +void FixedSizeInputStreamHandler::MovePackets(CollectionItemId id, + std::list* packets) { + InputStreamHandler::MovePackets(id, packets); + absl::MutexLock lock(&erase_mutex_); + if (!pending_) { + EraseSurplusPackets(false); } +} - void FillInputSet(Timestamp input_timestamp, - InputStreamShardSet* input_set) override { - CHECK(input_set); - absl::MutexLock lock(&erase_mutex_); - if (!pending_) { - LOG(ERROR) << "FillInputSet called without GetNodeReadiness."; - } - // input_timestamp is recalculated here to process the most recent packets. - EraseSurplusPackets(true); - input_timestamp = MinTimestampToProcess(); - DefaultInputStreamHandler::FillInputSet(input_timestamp, input_set); - pending_ = false; +void FixedSizeInputStreamHandler::FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) { + ABSL_CHECK(input_set); + absl::MutexLock lock(&erase_mutex_); + if (!pending_) { + ABSL_LOG(ERROR) << "FillInputSet called without GetNodeReadiness."; } - - private: - int32_t trigger_queue_size_; - int32_t target_queue_size_; - bool fixed_min_size_; - // Indicates that GetNodeReadiness has returned kReadyForProcess once, and - // the corresponding call to FillInputSet has not yet completed. - bool pending_ ABSL_GUARDED_BY(erase_mutex_); - // The timestamp used to truncate all input streams. - Timestamp kept_timestamp_ ABSL_GUARDED_BY(erase_mutex_); - absl::Mutex erase_mutex_; -}; + // input_timestamp is recalculated here to process the most recent packets. + EraseSurplusPackets(true); + input_timestamp = MinTimestampToProcess(); + DefaultInputStreamHandler::FillInputSet(input_timestamp, input_set); + pending_ = false; +} REGISTER_INPUT_STREAM_HANDLER(FixedSizeInputStreamHandler); diff --git a/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.h b/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.h new file mode 100644 index 00000000..a00bdda5 --- /dev/null +++ b/mediapipe/framework/stream_handler/fixed_size_input_stream_handler.h @@ -0,0 +1,108 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_FIXED_SIZE_INPUT_STREAM_HANDLER_H_ +#define MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_FIXED_SIZE_INPUT_STREAM_HANDLER_H_ + +#include +#include +#include + +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/stream_handler/default_input_stream_handler.h" + +namespace mediapipe { + +// Input stream handler that limits each input queue to a maximum of +// target_queue_size packets, discarding older packets as needed. When a +// timestamp is dropped from a stream, it is dropped from all others as well. +// +// For example, a calculator node with one input stream and the following input +// stream handler specs: +// +// node { +// calculator: "CalculatorRunningAtOneFps" +// input_stream: "packets_streaming_in_at_ten_fps" +// input_stream_handler { +// input_stream_handler: "FixedSizeInputStreamHandler" +// } +// } +// +// will always try to keep the newest packet in the input stream. +// +// A few details: FixedSizeInputStreamHandler takes action when any stream grows +// to trigger_queue_size or larger. It then keeps at most target_queue_size +// packets in every InputStreamImpl. Every stream is truncated at the same +// timestamp, so that each included timestamp delivers the same packets as +// DefaultInputStreamHandler includes. +class FixedSizeInputStreamHandler : public DefaultInputStreamHandler { + public: + FixedSizeInputStreamHandler() = delete; + FixedSizeInputStreamHandler(std::shared_ptr tag_map, + CalculatorContextManager* cc_manager, + const MediaPipeOptions& options, + bool calculator_run_in_parallel); + + private: + // Drops packets if all input streams exceed trigger_queue_size. + void EraseAllSurplus() ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_); + + // Returns the latest timestamp allowed before a bound. + Timestamp PreviousAllowedInStream(Timestamp bound); + + // Returns the lowest timestamp at which a packet may arrive at any stream. + Timestamp MinStreamBound(); + + // Returns the lowest timestamp of a packet ready to process. + Timestamp MinTimestampToProcess(); + + // Keeps only the most recent target_queue_size packets in each stream + // exceeding trigger_queue_size. Also, discards all packets older than the + // first kept timestamp on any stream. + void EraseAnySurplus(bool keep_one) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_); + + void EraseSurplusPackets(bool keep_one) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(erase_mutex_); + + NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; + + void AddPackets(CollectionItemId id, + const std::list& packets) override; + + void MovePackets(CollectionItemId id, std::list* packets) override; + + void FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) override; + + private: + int32_t trigger_queue_size_; + int32_t target_queue_size_; + bool fixed_min_size_; + // Indicates that GetNodeReadiness has returned kReadyForProcess once, and + // the corresponding call to FillInputSet has not yet completed. + bool pending_ ABSL_GUARDED_BY(erase_mutex_); + // The timestamp used to truncate all input streams. + Timestamp kept_timestamp_ ABSL_GUARDED_BY(erase_mutex_); + absl::Mutex erase_mutex_; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_FIXED_SIZE_INPUT_STREAM_HANDLER_H_ diff --git a/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc b/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc index c34fc96b..b2fc1aa8 100644 --- a/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/immediate_input_stream_handler.cc @@ -11,65 +11,33 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/framework/stream_handler/immediate_input_stream_handler.h" +#include +#include #include #include +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/mediapipe_options.pb.h" +#include "mediapipe/framework/tool/tag_map.h" namespace mediapipe { using SyncSet = InputStreamHandler::SyncSet; -// An input stream handler that delivers input packets to the Calculator -// immediately, with no dependency between input streams. It also invokes -// Calculator::Process when any input stream becomes done. -// -// NOTE: If packets arrive successively on different input streams with -// identical or decreasing timestamps, this input stream handler will -// invoke its Calculator with a sequence of InputTimestamps that is -// non-increasing. Its Calculator is responsible for accumulating packets -// with the required timetamps before processing and delivering output. -// -class ImmediateInputStreamHandler : public InputStreamHandler { - public: - ImmediateInputStreamHandler() = delete; - ImmediateInputStreamHandler( - std::shared_ptr tag_map, - CalculatorContextManager* calculator_context_manager, - const MediaPipeOptions& options, bool calculator_run_in_parallel); - - protected: - // Reinitializes this InputStreamHandler before each CalculatorGraph run. - void PrepareForRun(std::function headers_ready_callback, - std::function notification_callback, - std::function schedule_callback, - std::function error_callback) override; - - // Returns kReadyForProcess whenever a Packet is available at any of - // the input streams, or any input stream becomes done. - NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; - - // Selects a packet on each stream with an available packet with the - // specified timestamp, leaving other input streams unaffected. - void FillInputSet(Timestamp input_timestamp, - InputStreamShardSet* input_set) override; - - // Returns the number of sync-sets maintained by this input-handler. - int SyncSetCount() override; - - absl::Mutex mutex_; - // The packet-set builder for each input stream. - std::vector sync_sets_ ABSL_GUARDED_BY(mutex_); - // The input timestamp for each kReadyForProcess input stream. - std::vector ready_timestamps_ ABSL_GUARDED_BY(mutex_); -}; REGISTER_INPUT_STREAM_HANDLER(ImmediateInputStreamHandler); ImmediateInputStreamHandler::ImmediateInputStreamHandler( std::shared_ptr tag_map, CalculatorContextManager* calculator_context_manager, - const MediaPipeOptions& options, bool calculator_run_in_parallel) + const mediapipe::MediaPipeOptions& options, bool calculator_run_in_parallel) : InputStreamHandler(tag_map, calculator_context_manager, options, calculator_run_in_parallel) { for (auto id = tag_map->BeginId(); id < tag_map->EndId(); ++id) { @@ -115,7 +83,7 @@ NodeReadiness ImmediateInputStreamHandler::GetNodeReadiness( ready_timestamps_[i] = stream_ts; input_timestamp = std::min(input_timestamp, stream_ts); } else if (readiness == NodeReadiness::kReadyForClose) { - CHECK_EQ(stream_ts, Timestamp::Done()); + ABSL_CHECK_EQ(stream_ts, Timestamp::Done()); if (ProcessTimestampBounds()) { // With kReadyForClose, the timestamp-bound Done is returned. // TODO: Make all InputStreamHandlers process Done() like this. diff --git a/mediapipe/framework/stream_handler/immediate_input_stream_handler.h b/mediapipe/framework/stream_handler/immediate_input_stream_handler.h new file mode 100644 index 00000000..dd15ad99 --- /dev/null +++ b/mediapipe/framework/stream_handler/immediate_input_stream_handler.h @@ -0,0 +1,77 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_IMMEDIATE_INPUT_STREAM_HANDLER_H_ +#define MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_IMMEDIATE_INPUT_STREAM_HANDLER_H_ + +#include +#include +#include + +#include "absl/base/thread_annotations.h" +#include "absl/status/status.h" +#include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/tool/tag_map.h" + +namespace mediapipe { + +// An input stream handler that delivers input packets to the Calculator +// immediately, with no dependency between input streams. It also invokes +// Calculator::Process when any input stream becomes done. +// +// NOTE: If packets arrive successively on different input streams with +// identical or decreasing timestamps, this input stream handler will +// invoke its Calculator with a sequence of InputTimestamps that is +// non-increasing. Its Calculator is responsible for accumulating packets +// with the required timestamps before processing and delivering output. +class ImmediateInputStreamHandler : public InputStreamHandler { + public: + ImmediateInputStreamHandler() = delete; + ImmediateInputStreamHandler( + std::shared_ptr tag_map, + CalculatorContextManager* calculator_context_manager, + const MediaPipeOptions& options, bool calculator_run_in_parallel); + + protected: + // Reinitializes this InputStreamHandler before each CalculatorGraph run. + void PrepareForRun(std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) override; + + // Returns kReadyForProcess whenever a Packet is available at any of + // the input streams, or any input stream becomes done. + NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; + + // Selects a packet on each stream with an available packet with the + // specified timestamp, leaving other input streams unaffected. + void FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) override; + + // Returns the number of sync-sets maintained by this input-handler. + int SyncSetCount() override; + + absl::Mutex mutex_; + // The packet-set builder for each input stream. + std::vector sync_sets_ ABSL_GUARDED_BY(mutex_); + // The input timestamp for each kReadyForProcess input stream. + std::vector ready_timestamps_ ABSL_GUARDED_BY(mutex_); +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_IMMEDIATE_INPUT_STREAM_HANDLER_H_ diff --git a/mediapipe/framework/stream_handler/immediate_input_stream_handler_test.cc b/mediapipe/framework/stream_handler/immediate_input_stream_handler_test.cc index e5de7f0c..04b1c490 100644 --- a/mediapipe/framework/stream_handler/immediate_input_stream_handler_test.cc +++ b/mediapipe/framework/stream_handler/immediate_input_stream_handler_test.cc @@ -18,6 +18,7 @@ #include #include "absl/base/macros.h" +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context_manager.h" @@ -104,7 +105,7 @@ class ImmediateInputStreamHandlerTest : public ::testing::Test { void NotifyNoOp() {} void Schedule(CalculatorContext* cc) { - CHECK(cc); + ABSL_CHECK(cc); cc_ = cc; } @@ -132,7 +133,7 @@ class ImmediateInputStreamHandlerTest : public ::testing::Test { } const InputStream& Input(const CollectionItemId& id) { - CHECK(cc_); + ABSL_CHECK(cc_); return cc_->Inputs().Get(id); } diff --git a/mediapipe/framework/stream_handler/in_order_output_stream_handler.cc b/mediapipe/framework/stream_handler/in_order_output_stream_handler.cc index 9af38ecd..8faaaceb 100644 --- a/mediapipe/framework/stream_handler/in_order_output_stream_handler.cc +++ b/mediapipe/framework/stream_handler/in_order_output_stream_handler.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/stream_handler/in_order_output_stream_handler.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/collection.h" #include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/output_stream_shard.h" @@ -23,7 +24,7 @@ namespace mediapipe { REGISTER_OUTPUT_STREAM_HANDLER(InOrderOutputStreamHandler); void InOrderOutputStreamHandler::PropagationLoop() { - CHECK_EQ(propagation_state_, kIdle); + ABSL_CHECK_EQ(propagation_state_, kIdle); Timestamp context_timestamp; CalculatorContext* calculator_context; if (!calculator_context_manager_->HasActiveContexts()) { @@ -34,7 +35,7 @@ void InOrderOutputStreamHandler::PropagationLoop() { if (!completed_input_timestamps_.empty()) { Timestamp completed_timestamp = *completed_input_timestamps_.begin(); if (context_timestamp != completed_timestamp) { - CHECK_LT(context_timestamp, completed_timestamp); + ABSL_CHECK_LT(context_timestamp, completed_timestamp); return; } propagation_state_ = kPropagatingPackets; @@ -45,7 +46,7 @@ void InOrderOutputStreamHandler::PropagationLoop() { if (propagation_state_ == kPropagatingPackets) { PropagatePackets(&calculator_context, &context_timestamp); } else { - CHECK_EQ(kPropagatingBound, propagation_state_); + ABSL_CHECK_EQ(kPropagatingBound, propagation_state_); PropagationBound(&calculator_context, &context_timestamp); } } @@ -105,12 +106,12 @@ void InOrderOutputStreamHandler::PropagationBound( } // Some recent changes require the propagation thread to recheck if any // new packets can be propagated. - CHECK_EQ(propagation_state_, kPropagationPending); + ABSL_CHECK_EQ(propagation_state_, kPropagationPending); // task_timestamp_bound_ was updated while the propagation thread was // doing timestamp propagation. This thread will redo timestamp // propagation for the new task_timestamp_bound_. if (!calculator_context_manager_->HasActiveContexts()) { - CHECK_LT(bound_to_propagate, task_timestamp_bound_); + ABSL_CHECK_LT(bound_to_propagate, task_timestamp_bound_); propagation_state_ = kPropagatingBound; return; } diff --git a/mediapipe/framework/stream_handler/mux_input_stream_handler.cc b/mediapipe/framework/stream_handler/mux_input_stream_handler.cc index 0303a577..a0253b9c 100644 --- a/mediapipe/framework/stream_handler/mux_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/mux_input_stream_handler.cc @@ -11,151 +11,124 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/framework/stream_handler/mux_input_stream_handler.h" +#include + +#include "absl/log/absl_check.h" #include "absl/strings/substitute.h" #include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/input_stream_handler.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { -// Implementation of the input stream handler for the MuxCalculator. -// -// One of the input streams is the control stream; all the other input streams -// are data streams. To make MuxInputStreamHandler work properly, the tag of the -// input streams must obey the following rules: -// Let N be the number of input streams. Data streams must use tag "INPUT" with -// index 0, ..., N - 2; the control stream must use tag "SELECT". -// -// The control stream carries packets of type 'int'. The 'int' value in a -// control stream packet must be a valid index in the range 0, ..., N - 2 and -// select the data stream at that index. The selected data stream must have a -// packet with the same timestamp as the control stream packet. -// -// When the control stream is done, GetNodeReadiness() returns -// NodeReadiness::kReadyForClose. -// -// TODO: pass the input stream tags to the MuxInputStreamHandler -// constructor so that it can refer to input streams by tag. See b/30125118. -class MuxInputStreamHandler : public InputStreamHandler { - public: - MuxInputStreamHandler() = delete; - MuxInputStreamHandler(std::shared_ptr tag_map, - CalculatorContextManager* cc_manager, - const MediaPipeOptions& options, - bool calculator_run_in_parallel) - : InputStreamHandler(std::move(tag_map), cc_manager, options, - calculator_run_in_parallel) {} +CollectionItemId MuxInputStreamHandler::GetControlStreamId() const { + return input_stream_managers_.EndId() - 1; +} +void MuxInputStreamHandler::RemoveOutdatedDataPackets(Timestamp timestamp) { + const CollectionItemId control_stream_id = GetControlStreamId(); + for (CollectionItemId id = input_stream_managers_.BeginId(); + id < control_stream_id; ++id) { + input_stream_managers_.Get(id)->ErasePacketsEarlierThan(timestamp); + } +} - protected: - // In MuxInputStreamHandler, a node is "ready" if: - // - the control stream is done (need to call Close() in this case), or - // - we have received the packets on the control stream and the selected data - // stream at the next timestamp. - NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override { - DCHECK(min_stream_timestamp); - absl::MutexLock lock(&input_streams_mutex_); +// In MuxInputStreamHandler, a node is "ready" if: +// - the control stream is done (need to call Close() in this case), or +// - we have received the packets on the control stream and the selected data +// stream at the next timestamp. +NodeReadiness MuxInputStreamHandler::GetNodeReadiness( + Timestamp* min_stream_timestamp) { + ABSL_DCHECK(min_stream_timestamp); + absl::MutexLock lock(&input_streams_mutex_); - const auto& control_stream = - input_stream_managers_.Get(input_stream_managers_.EndId() - 1); - bool empty; - *min_stream_timestamp = control_stream->MinTimestampOrBound(&empty); - if (empty) { - if (*min_stream_timestamp == Timestamp::Done()) { - // Calculator is done if the control input stream is done. - return NodeReadiness::kReadyForClose; - } - // Calculator is not ready to run if the control input stream is empty. + const auto& control_stream = input_stream_managers_.Get(GetControlStreamId()); + bool empty; + *min_stream_timestamp = control_stream->MinTimestampOrBound(&empty); + + // Data streams may contain some outdated packets which failed to be popped + // out during "FillInputSet". (This handler doesn't sync input streams, + // hence "FillInputSet" can be triggered before every input stream is + // filled with packets corresponding to the same timestamp.) + RemoveOutdatedDataPackets(*min_stream_timestamp); + if (empty) { + if (*min_stream_timestamp == Timestamp::Done()) { + // Calculator is done if the control input stream is done. + return NodeReadiness::kReadyForClose; + } + // Calculator is not ready to run if the control input stream is empty. + return NodeReadiness::kNotReady; + } + + Packet control_packet = control_stream->QueueHead(); + ABSL_CHECK(!control_packet.IsEmpty()); + int control_value = control_packet.Get(); + ABSL_CHECK_LE(0, control_value); + ABSL_CHECK_LT(control_value, input_stream_managers_.NumEntries() - 1); + const auto& data_stream = input_stream_managers_.Get( + input_stream_managers_.BeginId() + control_value); + + Timestamp stream_timestamp = data_stream->MinTimestampOrBound(&empty); + if (empty) { + if (stream_timestamp <= *min_stream_timestamp) { + // "data_stream" didn't receive a packet corresponding to the current + // "control_stream" packet yet. return NodeReadiness::kNotReady; } - - Packet control_packet = control_stream->QueueHead(); - CHECK(!control_packet.IsEmpty()); - int control_value = control_packet.Get(); - CHECK_LE(0, control_value); - CHECK_LT(control_value, input_stream_managers_.NumEntries() - 1); - const auto& data_stream = input_stream_managers_.Get( - input_stream_managers_.BeginId() + control_value); - - // Data stream may contain some outdated packets which failed to be popped - // out during "FillInputSet". (This handler doesn't sync input streams, - // hence "FillInputSet" can be triggerred before every input stream is - // filled with packets corresponding to the same timestamp.) - data_stream->ErasePacketsEarlierThan(*min_stream_timestamp); - Timestamp stream_timestamp = data_stream->MinTimestampOrBound(&empty); - if (empty) { - if (stream_timestamp <= *min_stream_timestamp) { - // "data_stream" didn't receive a packet corresponding to the current - // "control_stream" packet yet. - return NodeReadiness::kNotReady; - } - // "data_stream" timestamp bound update detected. - return NodeReadiness::kReadyForProcess; - } - if (stream_timestamp > *min_stream_timestamp) { - // The earliest packet "data_stream" holds corresponds to a control packet - // yet to arrive, which means there won't be a "data_stream" packet - // corresponding to the current "control_stream" packet, which should be - // indicated as timestamp boun update. - return NodeReadiness::kReadyForProcess; - } - CHECK_EQ(stream_timestamp, *min_stream_timestamp); + // "data_stream" timestamp bound update detected. return NodeReadiness::kReadyForProcess; } - - // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. - void FillInputSet(Timestamp input_timestamp, - InputStreamShardSet* input_set) override { - CHECK(input_timestamp.IsAllowedInStream()); - CHECK(input_set); - absl::MutexLock lock(&input_streams_mutex_); - - const CollectionItemId control_stream_id = - input_stream_managers_.EndId() - 1; - auto& control_stream = input_stream_managers_.Get(control_stream_id); - int num_packets_dropped = 0; - bool stream_is_done = false; - Packet control_packet = control_stream->PopPacketAtTimestamp( - input_timestamp, &num_packets_dropped, &stream_is_done); - CHECK_EQ(num_packets_dropped, 0) - << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", - num_packets_dropped, control_stream->Name()); - CHECK(!control_packet.IsEmpty()); - int control_value = control_packet.Get(); - AddPacketToShard(&input_set->Get(control_stream_id), - std::move(control_packet), stream_is_done); - - const CollectionItemId data_stream_id = - input_stream_managers_.BeginId() + control_value; - CHECK_LE(input_stream_managers_.BeginId(), data_stream_id); - CHECK_LT(data_stream_id, control_stream_id); - auto& data_stream = input_stream_managers_.Get(data_stream_id); - stream_is_done = false; - Packet data_packet = data_stream->PopPacketAtTimestamp( - input_timestamp, &num_packets_dropped, &stream_is_done); - CHECK_EQ(num_packets_dropped, 0) - << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", - num_packets_dropped, data_stream->Name()); - AddPacketToShard(&input_set->Get(data_stream_id), std::move(data_packet), - stream_is_done); - - // Discard old packets on other streams. - // Note that control_stream_id is the last valid id. - auto next_timestamp = input_timestamp.NextAllowedInStream(); - for (CollectionItemId id = input_stream_managers_.BeginId(); - id < control_stream_id; ++id) { - if (id == data_stream_id) continue; - auto& other_stream = input_stream_managers_.Get(id); - other_stream->ErasePacketsEarlierThan(next_timestamp); - } + if (stream_timestamp > *min_stream_timestamp) { + // The earliest packet "data_stream" holds corresponds to a control packet + // yet to arrive, which means there won't be a "data_stream" packet + // corresponding to the current "control_stream" packet, which should be + // indicated as timestamp boun update. + return NodeReadiness::kReadyForProcess; } + ABSL_CHECK_EQ(stream_timestamp, *min_stream_timestamp); + return NodeReadiness::kReadyForProcess; +} - private: - // Must be acquired when manipulating the control and data streams to ensure - // we have a consistent view of the two streams. - absl::Mutex input_streams_mutex_; -}; +// Only invoked when associated GetNodeReadiness() returned kReadyForProcess. +void MuxInputStreamHandler::FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) { + ABSL_CHECK(input_timestamp.IsAllowedInStream()); + ABSL_CHECK(input_set); + absl::MutexLock lock(&input_streams_mutex_); + + const CollectionItemId control_stream_id = GetControlStreamId(); + auto& control_stream = input_stream_managers_.Get(control_stream_id); + int num_packets_dropped = 0; + bool stream_is_done = false; + Packet control_packet = control_stream->PopPacketAtTimestamp( + input_timestamp, &num_packets_dropped, &stream_is_done); + ABSL_CHECK_EQ(num_packets_dropped, 0) + << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", + num_packets_dropped, control_stream->Name()); + ABSL_CHECK(!control_packet.IsEmpty()); + int control_value = control_packet.Get(); + AddPacketToShard(&input_set->Get(control_stream_id), + std::move(control_packet), stream_is_done); + + const CollectionItemId data_stream_id = + input_stream_managers_.BeginId() + control_value; + ABSL_CHECK_LE(input_stream_managers_.BeginId(), data_stream_id); + ABSL_CHECK_LT(data_stream_id, control_stream_id); + auto& data_stream = input_stream_managers_.Get(data_stream_id); + stream_is_done = false; + Packet data_packet = data_stream->PopPacketAtTimestamp( + input_timestamp, &num_packets_dropped, &stream_is_done); + ABSL_CHECK_EQ(num_packets_dropped, 0) + << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", + num_packets_dropped, data_stream->Name()); + AddPacketToShard(&input_set->Get(data_stream_id), std::move(data_packet), + stream_is_done); + + // Discard old packets on data streams. + RemoveOutdatedDataPackets(input_timestamp.NextAllowedInStream()); +} REGISTER_INPUT_STREAM_HANDLER(MuxInputStreamHandler); diff --git a/mediapipe/framework/stream_handler/mux_input_stream_handler.h b/mediapipe/framework/stream_handler/mux_input_stream_handler.h new file mode 100644 index 00000000..63fdde0e --- /dev/null +++ b/mediapipe/framework/stream_handler/mux_input_stream_handler.h @@ -0,0 +1,80 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_MUX_INPUT_STREAM_HANDLER_H_ +#define MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_MUX_INPUT_STREAM_HANDLER_H_ + +#include +#include + +#include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/input_stream_handler.h" + +namespace mediapipe { + +// Implementation of the input stream handler for the MuxCalculator. +// +// One of the input streams is the control stream; all the other input streams +// are data streams. To make MuxInputStreamHandler work properly, the tag of the +// input streams must obey the following rules: +// Let N be the number of input streams. Data streams must use tag "INPUT" with +// index 0, ..., N - 2; the control stream must use tag "SELECT". +// +// The control stream carries packets of type 'int'. The 'int' value in a +// control stream packet must be a valid index in the range 0, ..., N - 2 and +// select the data stream at that index. The selected data stream must have a +// packet with the same timestamp as the control stream packet. +// +// When the control stream is done, GetNodeReadiness() returns +// NodeReadiness::kReadyForClose. +// +// TODO: pass the input stream tags to the MuxInputStreamHandler +// constructor so that it can refer to input streams by tag. See b/30125118. +class MuxInputStreamHandler : public InputStreamHandler { + public: + MuxInputStreamHandler() = delete; + MuxInputStreamHandler(std::shared_ptr tag_map, + CalculatorContextManager* cc_manager, + const MediaPipeOptions& options, + bool calculator_run_in_parallel) + : InputStreamHandler(std::move(tag_map), cc_manager, options, + calculator_run_in_parallel) {} + + private: + CollectionItemId GetControlStreamId() const; + void RemoveOutdatedDataPackets(Timestamp timestamp); + + protected: + // In MuxInputStreamHandler, a node is "ready" if: + // - the control stream is done (need to call Close() in this case), or + // - we have received the packets on the control stream and the selected data + // stream at the next timestamp. + NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; + + // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. + void FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) override; + + private: + // Must be acquired when manipulating the control and data streams to ensure + // we have a consistent view of the two streams. + absl::Mutex input_streams_mutex_; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_MUX_INPUT_STREAM_HANDLER_H_ diff --git a/mediapipe/framework/stream_handler/mux_input_stream_handler_test.cc b/mediapipe/framework/stream_handler/mux_input_stream_handler_test.cc index f19a3dde..78b2bb3f 100644 --- a/mediapipe/framework/stream_handler/mux_input_stream_handler_test.cc +++ b/mediapipe/framework/stream_handler/mux_input_stream_handler_test.cc @@ -645,5 +645,41 @@ TEST(MuxInputStreamHandlerTest, MP_ASSERT_OK(graph.WaitUntilDone()); } +TEST(MuxInputStreamHandlerTest, RemovesUnusedDataStreamPackets) { + CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie(R"pb( + input_stream: "input0" + input_stream: "input1" + input_stream: "select" + node { + calculator: "MuxCalculator" + input_stream: "INPUT:0:input0" + input_stream: "INPUT:1:input1" + input_stream: "SELECT:select" + output_stream: "OUTPUT:output" + input_stream_handler { input_stream_handler: "MuxInputStreamHandler" } + } + )pb"); + config.set_max_queue_size(1); + config.set_report_deadlock(true); + + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(config)); + MP_ASSERT_OK(graph.StartRun({})); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "select", MakePacket(0).At(Timestamp(2)))); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input0", MakePacket(1000).At(Timestamp(2)))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + + // Add two delayed packets to the deselected input. They should be discarded + // instead of triggering the deadlock detection (max_queue_size = 1). + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input1", MakePacket(900).At(Timestamp(1)))); + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input1", MakePacket(900).At(Timestamp(2)))); + MP_ASSERT_OK(graph.WaitUntilIdle()); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc b/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc index 1001d64f..f6356c17 100644 --- a/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc @@ -11,105 +11,51 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/framework/stream_handler/sync_set_input_stream_handler.h" -#include +#include +#include +#include +#include +#include -// TODO: Move protos in another CL after the C++ code migration. -#include "absl/strings/substitute.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/input_stream_handler.h" -#include "mediapipe/framework/mediapipe_options.pb.h" #include "mediapipe/framework/packet_set.h" +#include "mediapipe/framework/port/map_util.h" +#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/stream_handler/sync_set_input_stream_handler.pb.h" #include "mediapipe/framework/timestamp.h" -#include "mediapipe/framework/tool/tag_map.h" namespace mediapipe { -// An input stream handler which separates the inputs into sets which -// are each independently synchronized. For example, if 5 inputs are -// present, then the first three can be grouped (and will be synchronized -// as if they were in a calculator with only those three streams) and the -// remaining 2 streams can be independently grouped. The calculator will -// always be called with all the available packets from a single sync set -// (never more than one). The input timestamps seen by the calculator -// will be ordered sequentially for each sync set but may jump around -// between sync sets. -class SyncSetInputStreamHandler : public InputStreamHandler { - public: - SyncSetInputStreamHandler() = delete; - SyncSetInputStreamHandler(std::shared_ptr tag_map, - CalculatorContextManager* cc_manager, - const MediaPipeOptions& extendable_options, - bool calculator_run_in_parallel); - - void PrepareForRun(std::function headers_ready_callback, - std::function notification_callback, - std::function schedule_callback, - std::function error_callback) override; - - protected: - // In SyncSetInputStreamHandler, a node is "ready" if any - // of its sync sets are ready in the traditional sense (See - // DefaultInputStreamHandler). - NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; - - // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. - // Populates packets for the ready sync-set, and populates timestamp bounds - // for all sync-sets. - void FillInputSet(Timestamp input_timestamp, - InputStreamShardSet* input_set) override; - - // Populates timestamp bounds for streams outside the ready sync-set. - void FillInputBounds(Timestamp input_timestamp, - InputStreamShardSet* input_set) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - - // Returns the number of sync-sets maintained by this input-handler. - int SyncSetCount() override; - - private: - absl::Mutex mutex_; - // The ids of each set of inputs. - std::vector sync_sets_ ABSL_GUARDED_BY(mutex_); - // The index of the ready sync set. A value of -1 indicates that no - // sync sets are ready. - int ready_sync_set_index_ ABSL_GUARDED_BY(mutex_) = -1; - // The timestamp at which the sync set is ready. If no sync set is - // ready then this variable should be Timestamp::Done() . - Timestamp ready_timestamp_ ABSL_GUARDED_BY(mutex_); -}; - REGISTER_INPUT_STREAM_HANDLER(SyncSetInputStreamHandler); -SyncSetInputStreamHandler::SyncSetInputStreamHandler( - std::shared_ptr tag_map, CalculatorContextManager* cc_manager, - const MediaPipeOptions& extendable_options, bool calculator_run_in_parallel) - : InputStreamHandler(std::move(tag_map), cc_manager, extendable_options, - calculator_run_in_parallel) {} - void SyncSetInputStreamHandler::PrepareForRun( std::function headers_ready_callback, std::function notification_callback, std::function schedule_callback, std::function error_callback) { const auto& handler_options = - options_.GetExtension(SyncSetInputStreamHandlerOptions::ext); + options_.GetExtension(mediapipe::SyncSetInputStreamHandlerOptions::ext); { absl::MutexLock lock(&mutex_); sync_sets_.clear(); std::set used_ids; for (const auto& sync_set : handler_options.sync_set()) { std::vector stream_ids; - CHECK_LT(0, sync_set.tag_index_size()); + ABSL_CHECK_LT(0, sync_set.tag_index_size()); for (const auto& tag_index : sync_set.tag_index()) { std::string tag; int index; MEDIAPIPE_CHECK_OK(tool::ParseTagIndex(tag_index, &tag, &index)); CollectionItemId id = input_stream_managers_.GetId(tag, index); - CHECK(id.IsValid()) << "stream \"" << tag_index << "\" is not found."; - CHECK(!mediapipe::ContainsKey(used_ids, id)) + ABSL_CHECK(id.IsValid()) + << "stream \"" << tag_index << "\" is not found."; + ABSL_CHECK(!mediapipe::ContainsKey(used_ids, id)) << "stream \"" << tag_index << "\" is in more than one sync set."; used_ids.insert(id); stream_ids.push_back(id); @@ -137,7 +83,7 @@ void SyncSetInputStreamHandler::PrepareForRun( NodeReadiness SyncSetInputStreamHandler::GetNodeReadiness( Timestamp* min_stream_timestamp) { - DCHECK(min_stream_timestamp); + ABSL_DCHECK(min_stream_timestamp); absl::MutexLock lock(&mutex_); if (ready_sync_set_index_ >= 0) { *min_stream_timestamp = ready_timestamp_; @@ -185,7 +131,7 @@ void SyncSetInputStreamHandler::FillInputSet(Timestamp input_timestamp, InputStreamShardSet* input_set) { // Assume that all current packets are already cleared. absl::MutexLock lock(&mutex_); - CHECK_LE(0, ready_sync_set_index_); + ABSL_CHECK_LE(0, ready_sync_set_index_); sync_sets_[ready_sync_set_index_].FillInputSet(input_timestamp, input_set); for (int i = 0; i < sync_sets_.size(); ++i) { if (i != ready_sync_set_index_) { diff --git a/mediapipe/framework/stream_handler/sync_set_input_stream_handler.h b/mediapipe/framework/stream_handler/sync_set_input_stream_handler.h new file mode 100644 index 00000000..67f1e49a --- /dev/null +++ b/mediapipe/framework/stream_handler/sync_set_input_stream_handler.h @@ -0,0 +1,97 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_SYNC_SET_INPUT_STREAM_HANDLER_H_ +#define MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_SYNC_SET_INPUT_STREAM_HANDLER_H_ + +#include +#include +#include +#include + +#include "absl/base/thread_annotations.h" +#include "absl/status/status.h" +#include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/mediapipe_options.pb.h" +#include "mediapipe/framework/packet_set.h" +#include "mediapipe/framework/stream_handler/sync_set_input_stream_handler.pb.h" +#include "mediapipe/framework/timestamp.h" +#include "mediapipe/framework/tool/tag_map.h" + +namespace mediapipe { + +// An input stream handler which separates the inputs into sets which +// are each independently synchronized. For example, if 5 inputs are +// present, then the first three can be grouped (and will be synchronized +// as if they were in a calculator with only those three streams) and the +// remaining 2 streams can be independently grouped. The calculator will +// always be called with all the available packets from a single sync set +// (never more than one). The input timestamps seen by the calculator +// will be ordered sequentially for each sync set but may jump around +// between sync sets. +class SyncSetInputStreamHandler : public InputStreamHandler { + public: + SyncSetInputStreamHandler() = delete; + SyncSetInputStreamHandler( + std::shared_ptr tag_map, + CalculatorContextManager* cc_manager, + const mediapipe::MediaPipeOptions& extendable_options, + bool calculator_run_in_parallel) + : InputStreamHandler(std::move(tag_map), cc_manager, extendable_options, + calculator_run_in_parallel) {} + + void PrepareForRun(std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) override; + + protected: + // In SyncSetInputStreamHandler, a node is "ready" if any + // of its sync sets are ready in the traditional sense (See + // DefaultInputStreamHandler). + NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; + + // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. + // Populates packets for the ready sync-set, and populates timestamp bounds + // for all sync-sets. + void FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) override; + + // Populates timestamp bounds for streams outside the ready sync-set. + void FillInputBounds(Timestamp input_timestamp, + InputStreamShardSet* input_set) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Returns the number of sync-sets maintained by this input-handler. + int SyncSetCount() override; + + private: + absl::Mutex mutex_; + // The ids of each set of inputs. + std::vector sync_sets_ ABSL_GUARDED_BY(mutex_); + // The index of the ready sync set. A value of -1 indicates that no + // sync sets are ready. + int ready_sync_set_index_ ABSL_GUARDED_BY(mutex_) = -1; + // The timestamp at which the sync set is ready. If no sync set is + // ready then this variable should be Timestamp::Done() . + Timestamp ready_timestamp_ ABSL_GUARDED_BY(mutex_); +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_SYNC_SET_INPUT_STREAM_HANDLER_H_ diff --git a/mediapipe/framework/stream_handler/sync_set_input_stream_handler_test.cc b/mediapipe/framework/stream_handler/sync_set_input_stream_handler_test.cc index e93f806b..c8cc6a17 100644 --- a/mediapipe/framework/stream_handler/sync_set_input_stream_handler_test.cc +++ b/mediapipe/framework/stream_handler/sync_set_input_stream_handler_test.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/calculator_framework.h" // TODO: Move protos in another CL after the C++ code migration. @@ -215,7 +216,7 @@ TEST(SyncSetInputStreamHandlerTest, OrdinaryOperation) { RandomEngine rng(testing::UnitTest::GetInstance()->random_seed()); for (int iter = 0; iter < 1000; ++iter) { - LOG(INFO) << "Starting command shuffling iteration " << iter; + ABSL_LOG(INFO) << "Starting command shuffling iteration " << iter; // Merge the commands for each sync set together into a serial list. // This is done by randomly choosing which list to grab from next. diff --git a/mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.cc b/mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.cc index ae075d78..1ab5e4e7 100644 --- a/mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.cc +++ b/mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.cc @@ -12,91 +12,45 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.h" + #include +#include +#include #include #include #include +#include "absl/log/absl_check.h" #include "absl/strings/substitute.h" #include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/collection_item_id.h" #include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/mediapipe_options.pb.h" #include "mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.pb.h" #include "mediapipe/framework/timestamp.h" #include "mediapipe/framework/tool/validate_name.h" namespace mediapipe { -// The input streams must have the same time unit but may have different time -// origins (also called epochs). The timestamp_base_tag_index option -// designates an input stream as the timestamp base. -// -// TimestampAlignInputStreamHandler operates in two phases: -// -// 1. Pre-initialization: In this phase, the input stream handler passes -// through input packets in the timestamp base input stream, but buffers the -// input packets in all other input streams. This phase ends when the input -// stream handler has an input packet in every input stream. It uses the -// the timestamps of these input packets to calculate the timestamp offset of -// each input stream with respect to the timestamp base input stream. The -// timestamp offsets are saved for use in the next phase. -// -// 2. Post-initialization: In this phase, the input stream handler behaves -// like the DefaultInputStreamHandler, except that timestamp offsets are -// applied to the packet timestamps. -class TimestampAlignInputStreamHandler : public InputStreamHandler { - public: - TimestampAlignInputStreamHandler() = delete; - TimestampAlignInputStreamHandler(std::shared_ptr tag_map, - CalculatorContextManager* cc_manager, - const MediaPipeOptions& options, - bool calculator_run_in_parallel); - - void PrepareForRun(std::function headers_ready_callback, - std::function notification_callback, - std::function schedule_callback, - std::function error_callback) override; - - protected: - // In TimestampAlignInputStreamHandler, a node is "ready" if: - // - before the timestamp offsets are initialized: we have received a packet - // in the timestamp base input stream, or - // - after the timestamp offsets are initialized: the minimum bound (over - // all empty streams) is greater than the smallest timestamp of any - // stream, which means we have received all the packets that will be - // available at the next timestamp, or - // - all streams are done (need to call Close() in this case). - // Note that all packet timestamps and timestamp bounds are aligned with the - // timestamp base. - NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; - - // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. - void FillInputSet(Timestamp input_timestamp, - InputStreamShardSet* input_set) override; - - private: - CollectionItemId timestamp_base_stream_id_; - - absl::Mutex mutex_; - bool offsets_initialized_ ABSL_GUARDED_BY(mutex_) = false; - std::vector timestamp_offsets_; -}; REGISTER_INPUT_STREAM_HANDLER(TimestampAlignInputStreamHandler); TimestampAlignInputStreamHandler::TimestampAlignInputStreamHandler( std::shared_ptr tag_map, CalculatorContextManager* cc_manager, - const MediaPipeOptions& options, bool calculator_run_in_parallel) + const mediapipe::MediaPipeOptions& options, bool calculator_run_in_parallel) : InputStreamHandler(std::move(tag_map), cc_manager, options, calculator_run_in_parallel), timestamp_offsets_(input_stream_managers_.NumEntries()) { - const auto& handler_options = - options.GetExtension(TimestampAlignInputStreamHandlerOptions::ext); + const auto& handler_options = options.GetExtension( + mediapipe::TimestampAlignInputStreamHandlerOptions::ext); std::string tag; int index; MEDIAPIPE_CHECK_OK(tool::ParseTagIndex( handler_options.timestamp_base_tag_index(), &tag, &index)); timestamp_base_stream_id_ = input_stream_managers_.GetId(tag, index); - CHECK(timestamp_base_stream_id_.IsValid()) + ABSL_CHECK(timestamp_base_stream_id_.IsValid()) << "stream \"" << handler_options.timestamp_base_tag_index() << "\" is not found."; timestamp_offsets_[timestamp_base_stream_id_.value()] = 0; @@ -119,7 +73,7 @@ void TimestampAlignInputStreamHandler::PrepareForRun( NodeReadiness TimestampAlignInputStreamHandler::GetNodeReadiness( Timestamp* min_stream_timestamp) { - DCHECK(min_stream_timestamp); + ABSL_DCHECK(min_stream_timestamp); *min_stream_timestamp = Timestamp::Done(); Timestamp min_bound = Timestamp::Done(); @@ -178,14 +132,14 @@ NodeReadiness TimestampAlignInputStreamHandler::GetNodeReadiness( return NodeReadiness::kReadyForProcess; } - CHECK_EQ(min_bound, *min_stream_timestamp); + ABSL_CHECK_EQ(min_bound, *min_stream_timestamp); return NodeReadiness::kNotReady; } void TimestampAlignInputStreamHandler::FillInputSet( Timestamp input_timestamp, InputStreamShardSet* input_set) { - CHECK(input_timestamp.IsAllowedInStream()); - CHECK(input_set); + ABSL_CHECK(input_timestamp.IsAllowedInStream()); + ABSL_CHECK(input_set); { absl::MutexLock lock(&mutex_); if (!offsets_initialized_) { @@ -198,7 +152,7 @@ void TimestampAlignInputStreamHandler::FillInputSet( if (id == timestamp_base_stream_id_) { current_packet = stream->PopPacketAtTimestamp( input_timestamp, &num_packets_dropped, &stream_is_done); - CHECK_EQ(num_packets_dropped, 0) << absl::Substitute( + ABSL_CHECK_EQ(num_packets_dropped, 0) << absl::Substitute( "Dropped $0 packet(s) on input stream \"$1\".", num_packets_dropped, stream->Name()); } @@ -218,10 +172,10 @@ void TimestampAlignInputStreamHandler::FillInputSet( Packet current_packet = stream->PopPacketAtTimestamp( stream_timestamp, &num_packets_dropped, &stream_is_done); if (!current_packet.IsEmpty()) { - CHECK_EQ(current_packet.Timestamp(), stream_timestamp); + ABSL_CHECK_EQ(current_packet.Timestamp(), stream_timestamp); current_packet = current_packet.At(input_timestamp); } - CHECK_EQ(num_packets_dropped, 0) + ABSL_CHECK_EQ(num_packets_dropped, 0) << absl::Substitute("Dropped $0 packet(s) on input stream \"$1\".", num_packets_dropped, stream->Name()); AddPacketToShard(&input_set->Get(id), std::move(current_packet), diff --git a/mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.h b/mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.h new file mode 100644 index 00000000..dce8fad9 --- /dev/null +++ b/mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.h @@ -0,0 +1,91 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_TIMESTAMP_ALIGN_INPUT_STREAM_HANDLER_H_ +#define MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_TIMESTAMP_ALIGN_INPUT_STREAM_HANDLER_H_ + +#include +#include +#include + +#include "absl/base/thread_annotations.h" +#include "absl/status/status.h" +#include "absl/synchronization/mutex.h" +#include "mediapipe/framework/calculator_context_manager.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/collection_item_id.h" +#include "mediapipe/framework/input_stream_handler.h" +#include "mediapipe/framework/stream_handler/timestamp_align_input_stream_handler.pb.h" +#include "mediapipe/framework/timestamp.h" + +namespace mediapipe { + +// The input streams must have the same time unit but may have different time +// origins (also called epochs). The timestamp_base_tag_index option +// designates an input stream as the timestamp base. +// +// TimestampAlignInputStreamHandler operates in two phases: +// +// 1. Pre-initialization: In this phase, the input stream handler passes +// through input packets in the timestamp base input stream, but buffers the +// input packets in all other input streams. This phase ends when the input +// stream handler has an input packet in every input stream. It uses the +// the timestamps of these input packets to calculate the timestamp offset of +// each input stream with respect to the timestamp base input stream. The +// timestamp offsets are saved for use in the next phase. +// +// 2. Post-initialization: In this phase, the input stream handler behaves +// like the DefaultInputStreamHandler, except that timestamp offsets are +// applied to the packet timestamps. +class TimestampAlignInputStreamHandler : public InputStreamHandler { + public: + TimestampAlignInputStreamHandler() = delete; + TimestampAlignInputStreamHandler(std::shared_ptr tag_map, + CalculatorContextManager* cc_manager, + const mediapipe::MediaPipeOptions& options, + bool calculator_run_in_parallel); + + void PrepareForRun(std::function headers_ready_callback, + std::function notification_callback, + std::function schedule_callback, + std::function error_callback) override; + + protected: + // In TimestampAlignInputStreamHandler, a node is "ready" if: + // - before the timestamp offsets are initialized: we have received a packet + // in the timestamp base input stream, or + // - after the timestamp offsets are initialized: the minimum bound (over + // all empty streams) is greater than the smallest timestamp of any + // stream, which means we have received all the packets that will be + // available at the next timestamp, or + // - all streams are done (need to call Close() in this case). + // Note that all packet timestamps and timestamp bounds are aligned with the + // timestamp base. + NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override; + + // Only invoked when associated GetNodeReadiness() returned kReadyForProcess. + void FillInputSet(Timestamp input_timestamp, + InputStreamShardSet* input_set) override; + + private: + CollectionItemId timestamp_base_stream_id_; + + absl::Mutex mutex_; + bool offsets_initialized_ ABSL_GUARDED_BY(mutex_) = false; + std::vector timestamp_offsets_; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_STREAM_HANDLER_TIMESTAMP_ALIGN_INPUT_STREAM_HANDLER_H_ diff --git a/mediapipe/framework/subgraph.cc b/mediapipe/framework/subgraph.cc index 6c18c9ca..7cbde28b 100644 --- a/mediapipe/framework/subgraph.cc +++ b/mediapipe/framework/subgraph.cc @@ -64,13 +64,13 @@ GraphRegistry::GraphRegistry( void GraphRegistry::Register( const std::string& type_name, std::function()> factory) { - local_factories_.Register(type_name, factory, __FILE__, __LINE__); + local_factories_.Register(type_name, factory); } // TODO: Remove this convenience function. void GraphRegistry::Register(const std::string& type_name, const CalculatorGraphConfig& config) { - Register(type_name, [config] { + local_factories_.Register(type_name, [config] { auto result = absl::make_unique(config); return std::unique_ptr(result.release()); }); @@ -79,7 +79,7 @@ void GraphRegistry::Register(const std::string& type_name, // TODO: Remove this convenience function. void GraphRegistry::Register(const std::string& type_name, const CalculatorGraphTemplate& templ) { - Register(type_name, [templ] { + local_factories_.Register(type_name, [templ] { auto result = absl::make_unique(templ); return std::unique_ptr(result.release()); }); diff --git a/mediapipe/framework/test_calculators.cc b/mediapipe/framework/test_calculators.cc index 6cb30085..1ed1e61b 100644 --- a/mediapipe/framework/test_calculators.cc +++ b/mediapipe/framework/test_calculators.cc @@ -20,6 +20,7 @@ #include #include "Eigen/Core" +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/calculator_framework.h" @@ -203,7 +204,7 @@ class RangeCalculator : public CalculatorBase { // Initializes this object. void Initialize(CalculatorContext* cc) { - CHECK(!initialized_); + ABSL_CHECK(!initialized_); cc->Options(); // Ensure Options() can be called here. std::tie(n_, k_) = @@ -380,10 +381,10 @@ class RandomMatrixCalculator : public CalculatorBase { absl::Status Open(CalculatorContext* cc) override { auto& options = cc->Options(); - CHECK_LT(0, options.timestamp_step()); - CHECK_LT(0, options.rows()); - CHECK_LT(0, options.cols()); - CHECK_LT(options.start_timestamp(), options.limit_timestamp()); + ABSL_CHECK_LT(0, options.timestamp_step()); + ABSL_CHECK_LT(0, options.rows()); + ABSL_CHECK_LT(0, options.cols()); + ABSL_CHECK_LT(options.start_timestamp(), options.limit_timestamp()); current_timestamp_ = Timestamp(options.start_timestamp()); cc->Outputs().Index(0).SetNextTimestampBound(current_timestamp_); @@ -447,13 +448,13 @@ class MeanAndCovarianceCalculator : public CalculatorBase { absl::Status Process(CalculatorContext* cc) override { const Eigen::MatrixXd sample = cc->Inputs().Index(0).Get().cast(); - CHECK_EQ(1, sample.cols()); + ABSL_CHECK_EQ(1, sample.cols()); if (num_samples_ == 0) { rows_ = sample.rows(); sum_vector_ = Eigen::VectorXd::Zero(rows_); outer_product_sum_ = Eigen::MatrixXd::Zero(rows_, rows_); } else { - CHECK_EQ(sample.rows(), rows_); + ABSL_CHECK_EQ(sample.rows(), rows_); } sum_vector_ += sample; outer_product_sum_ += sample * sample.transpose(); diff --git a/mediapipe/framework/timestamp.cc b/mediapipe/framework/timestamp.cc index 05b69747..9183b3c8 100644 --- a/mediapipe/framework/timestamp.cc +++ b/mediapipe/framework/timestamp.cc @@ -16,6 +16,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" namespace mediapipe { @@ -26,7 +28,7 @@ constexpr double Timestamp::kTimestampUnitsPerSecond; // - The safe int type will check for overflow/underflow and other errors. // - The CHECK in the constructor will disallow special values. TimestampDiff Timestamp::operator-(const Timestamp other) const { - CHECK(IsRangeValue() && other.IsRangeValue()) + ABSL_CHECK(IsRangeValue() && other.IsRangeValue()) << "This timestamp is " << DebugString() << " and other was " << other.DebugString(); TimestampBaseType tmp_base = timestamp_ - other.timestamp_; @@ -43,7 +45,7 @@ TimestampDiff TimestampDiff::operator-(const TimestampDiff other) const { // Clamp the addition to the range [Timestamp::Min(), Timestamp::Max()]. Timestamp Timestamp::operator+(const TimestampDiff offset) const { - CHECK(IsRangeValue()) << "Timestamp is: " << DebugString(); + ABSL_CHECK(IsRangeValue()) << "Timestamp is: " << DebugString(); TimestampBaseType offset_base(offset.Value()); if (offset_base >= TimestampBaseType(0)) { if (timestamp_.value() >= Timestamp::Max().Value() - offset_base.value()) { @@ -112,7 +114,7 @@ std::string Timestamp::DebugString() const { } else if (*this == Timestamp::Done()) { return "Timestamp::Done()"; } else { - LOG(FATAL) << "Unknown special type."; + ABSL_LOG(FATAL) << "Unknown special type."; } } return absl::StrCat(timestamp_.value()); @@ -131,6 +133,13 @@ Timestamp Timestamp::NextAllowedInStream() const { return *this + 1; } +bool Timestamp::HasNextAllowedInStream() const { + if (*this >= Max() || *this == PreStream()) { + return false; + } + return true; +} + Timestamp Timestamp::PreviousAllowedInStream() const { if (*this <= Min() || *this == PostStream()) { // Indicates that no previous timestamps may occur. diff --git a/mediapipe/framework/timestamp.h b/mediapipe/framework/timestamp.h index 966ec183..8949dcc8 100644 --- a/mediapipe/framework/timestamp.h +++ b/mediapipe/framework/timestamp.h @@ -47,6 +47,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/deps/safe_int.h" #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/logging.h" @@ -186,6 +187,10 @@ class Timestamp { // CHECKs that this->IsAllowedInStream(). Timestamp NextAllowedInStream() const; + // Returns true if there's a next timestamp in the range [Min .. Max] after + // this one. + bool HasNextAllowedInStream() const; + // Returns the previous timestamp in the range [Min .. Max], or // Unstarted() if no Packets may preceed one with this timestamp. Timestamp PreviousAllowedInStream() const; @@ -266,14 +271,14 @@ std::ostream& operator<<(std::ostream& os, TimestampDiff arg); inline Timestamp::Timestamp() : timestamp_(kint64min) {} inline Timestamp::Timestamp(int64 timestamp) : timestamp_(timestamp) { - CHECK(!IsSpecialValue()) + ABSL_CHECK(!IsSpecialValue()) << "Cannot directly create a Timestamp with a special value: " << CreateNoErrorChecking(timestamp); } inline Timestamp::Timestamp(TimestampBaseType timestamp) : timestamp_(timestamp) { - CHECK(!IsSpecialValue()) + ABSL_CHECK(!IsSpecialValue()) << "Cannot directly create a Timestamp with a special value: " << CreateNoErrorChecking(timestamp.value()); } diff --git a/mediapipe/framework/timestamp_test.cc b/mediapipe/framework/timestamp_test.cc index 5f5cc342..3ba0b5c3 100644 --- a/mediapipe/framework/timestamp_test.cc +++ b/mediapipe/framework/timestamp_test.cc @@ -125,6 +125,22 @@ TEST(TimestampTest, NextAllowedInStream) { Timestamp::PostStream().NextAllowedInStream()); } +TEST(TimestampTest, HasNextAllowedInStream) { + EXPECT_TRUE(Timestamp::Min().HasNextAllowedInStream()); + EXPECT_TRUE((Timestamp::Min() + 1).HasNextAllowedInStream()); + EXPECT_TRUE(Timestamp(-1000).HasNextAllowedInStream()); + EXPECT_TRUE(Timestamp(0).HasNextAllowedInStream()); + EXPECT_TRUE(Timestamp(1000).HasNextAllowedInStream()); + EXPECT_TRUE((Timestamp::Max() - 2).HasNextAllowedInStream()); + EXPECT_TRUE((Timestamp::Max() - 1).HasNextAllowedInStream()); + + EXPECT_FALSE(Timestamp::PreStream().HasNextAllowedInStream()); + EXPECT_FALSE(Timestamp::Max().HasNextAllowedInStream()); + EXPECT_FALSE(Timestamp::PostStream().HasNextAllowedInStream()); + EXPECT_FALSE(Timestamp::OneOverPostStream().HasNextAllowedInStream()); + EXPECT_FALSE(Timestamp::Done().HasNextAllowedInStream()); +} + TEST(TimestampTest, SpecialValueDifferences) { { // Lower range const std::vector timestamps = { diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index 4ae0bb60..8899c89f 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -66,10 +66,12 @@ cc_library( deps = [ "//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework/port:advanced_proto", + "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_log", ], ) @@ -140,6 +142,7 @@ cc_library( "//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework/port:map_util", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ], @@ -165,6 +168,7 @@ cc_test( ":executor_util", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", + "@com_google_absl//absl/log:absl_check", ], ) @@ -281,6 +285,7 @@ cc_binary( "//mediapipe/framework/port:logging", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:parse", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -335,6 +340,7 @@ mediapipe_cc_test( cc_library( name = "packet_generator_wrapper_calculator", srcs = ["packet_generator_wrapper_calculator.cc"], + hdrs = ["packet_generator_wrapper_calculator.h"], visibility = ["//mediapipe/framework:__subpackages__"], deps = [ ":packet_generator_wrapper_calculator_cc_proto", @@ -342,6 +348,9 @@ cc_library( "//mediapipe/framework:calculator_registry", "//mediapipe/framework:output_side_packet", "//mediapipe/framework:packet_generator", + "//mediapipe/framework:packet_set", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/status", ], alwayslink = 1, ) @@ -360,6 +369,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -386,21 +396,23 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":name_util", + ":status_util", "//mediapipe/calculators/internal:callback_packet_calculator", "//mediapipe/calculators/internal:callback_packet_calculator_cc_proto", "//mediapipe/framework:calculator_base", "//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework:calculator_graph", "//mediapipe/framework:calculator_registry", - "//mediapipe/framework:input_stream", "//mediapipe/framework:packet", "//mediapipe/framework:packet_type", - "//mediapipe/framework/port:logging", + "//mediapipe/framework:timestamp", "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", ], alwayslink = 1, ) @@ -415,7 +427,6 @@ cc_library( ":tag_map", "//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework:graph_service_manager", - "//mediapipe/framework:packet_generator", "//mediapipe/framework:packet_generator_cc_proto", "//mediapipe/framework:port", "//mediapipe/framework:status_handler_cc_proto", @@ -425,8 +436,12 @@ cc_library( "//mediapipe/framework/port:map_util", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", ], ) @@ -452,6 +467,7 @@ cc_library( deps = [ "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -501,10 +517,11 @@ cc_library( ":calculator_graph_template_cc_proto", ":proto_util_lite", "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:numbers", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) @@ -525,11 +542,13 @@ cc_library( "//mediapipe/framework/deps:proto_descriptor_cc_proto", "//mediapipe/framework/port:advanced_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:map_util", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], @@ -625,6 +644,7 @@ cc_test( ":tag_map_helper", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:map_util", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) @@ -663,6 +683,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/port:threadpool", "//mediapipe/util:cpu_util", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ], @@ -764,6 +785,7 @@ cc_test( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/tool/testdata:dub_quad_test_subgraph", "//mediapipe/framework/tool/testdata:nested_test_subgraph", + "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], ) @@ -782,11 +804,12 @@ cc_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:advanced_proto", "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -920,6 +943,7 @@ cc_library( "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -940,11 +964,11 @@ cc_test( "//mediapipe/framework:subgraph", "//mediapipe/framework:test_calculators", "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/stream_handler:immediate_input_stream_handler", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/framework/tool/executor_util.h b/mediapipe/framework/tool/executor_util.h index 3167cdd0..5fb25da7 100644 --- a/mediapipe/framework/tool/executor_util.h +++ b/mediapipe/framework/tool/executor_util.h @@ -22,6 +22,10 @@ namespace mediapipe { namespace tool { // Ensures the default executor's stack size is at least min_stack_size. +// +// Note that this will also initialize the default executor; any configuration +// changes, such as num_threads, should be done to the config before calling +// this. void EnsureMinimumDefaultExecutorStackSize(int32 min_stack_size, CalculatorGraphConfig* config); } // namespace tool diff --git a/mediapipe/framework/tool/ios.bzl b/mediapipe/framework/tool/ios.bzl index c97b092e..a0fe0be5 100644 --- a/mediapipe/framework/tool/ios.bzl +++ b/mediapipe/framework/tool/ios.bzl @@ -14,7 +14,7 @@ """MediaPipe Task Library Helper Rules for iOS""" -MPP_TASK_MINIMUM_OS_VERSION = "11.0" +MPP_TASK_MINIMUM_OS_VERSION = "12.0" # When the static framework is built with bazel, the all header files are moved # to the "Headers" directory with no header path prefixes. This auxiliary rule diff --git a/mediapipe/framework/tool/mediapipe_proto.bzl b/mediapipe/framework/tool/mediapipe_proto.bzl index 6e41d054..142560ce 100644 --- a/mediapipe/framework/tool/mediapipe_proto.bzl +++ b/mediapipe/framework/tool/mediapipe_proto.bzl @@ -50,6 +50,7 @@ def mediapipe_proto_library_impl( def_cc_proto = True, def_py_proto = True, def_java_lite_proto = True, + def_kt_lite_proto = True, def_objc_proto = True, def_java_proto = True, def_jspb_proto = True, @@ -72,6 +73,7 @@ def mediapipe_proto_library_impl( def_cc_proto: define the cc_proto_library target def_py_proto: define the py_proto_library target def_java_lite_proto: define the java_lite_proto_library target + def_kt_lite_proto: define the kt_lite_proto_library target def_objc_proto: define the objc_proto_library target def_java_proto: define the java_proto_library target def_jspb_proto: define the jspb_proto_library target @@ -255,6 +257,7 @@ def mediapipe_proto_library( def_cc_proto = True, def_py_proto = True, def_java_lite_proto = True, + def_kt_lite_proto = True, def_portable_proto = True, # @unused def_objc_proto = True, def_java_proto = True, @@ -281,6 +284,7 @@ def mediapipe_proto_library( def_cc_proto: define the cc_proto_library target def_py_proto: define the py_proto_library target def_java_lite_proto: define the java_lite_proto_library target + def_kt_lite_proto: define the kt_lite_proto_library target def_portable_proto: ignored since portable protos are gone def_objc_proto: define the objc_proto_library target def_java_proto: define the java_proto_library target @@ -304,6 +308,7 @@ def mediapipe_proto_library( def_cc_proto = def_cc_proto, def_py_proto = def_py_proto, def_java_lite_proto = def_java_lite_proto, + def_kt_lite_proto = def_kt_lite_proto, def_objc_proto = def_objc_proto, def_java_proto = def_java_proto, def_jspb_proto = def_jspb_proto, @@ -334,6 +339,7 @@ def mediapipe_proto_library( def_cc_proto = def_cc_proto, def_py_proto = def_py_proto, def_java_lite_proto = def_java_lite_proto, + def_kt_lite_proto = def_kt_lite_proto, def_objc_proto = def_objc_proto, def_java_proto = def_java_proto, def_jspb_proto = def_jspb_proto, diff --git a/mediapipe/framework/tool/message_type_util.cc b/mediapipe/framework/tool/message_type_util.cc index fe505ee0..3bc5ea8d 100644 --- a/mediapipe/framework/tool/message_type_util.cc +++ b/mediapipe/framework/tool/message_type_util.cc @@ -4,6 +4,7 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_check.h" #include "absl/strings/ascii.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" @@ -118,14 +119,14 @@ class DescriptorReader { static FileDescriptorSet ReadFileDescriptorSet(const std::string& path) { std::string contents; - CHECK_OK(file::GetContents(path, &contents)); + ABSL_CHECK_OK(file::GetContents(path, &contents)); proto_ns::FileDescriptorSet result; result.ParseFromString(contents); return result; } static void WriteFile(const std::string& path, const std::string& contents) { - CHECK_OK(file::SetContents(path, contents)); + ABSL_CHECK_OK(file::SetContents(path, contents)); } static void WriteMessageTypeName(const std::string& path, diff --git a/mediapipe/framework/tool/packet_generator_wrapper_calculator.cc b/mediapipe/framework/tool/packet_generator_wrapper_calculator.cc index 831918df..07eae6f2 100644 --- a/mediapipe/framework/tool/packet_generator_wrapper_calculator.cc +++ b/mediapipe/framework/tool/packet_generator_wrapper_calculator.cc @@ -1,52 +1,55 @@ +#include "mediapipe/framework/tool/packet_generator_wrapper_calculator.h" + +#include "absl/status/status.h" #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/calculator_registry.h" #include "mediapipe/framework/output_side_packet.h" #include "mediapipe/framework/packet_generator.h" +#include "mediapipe/framework/packet_set.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/framework/tool/packet_generator_wrapper_calculator.pb.h" namespace mediapipe { -class PacketGeneratorWrapperCalculator : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - const auto& options = - cc->Options<::mediapipe::PacketGeneratorWrapperCalculatorOptions>(); - ASSIGN_OR_RETURN(auto static_access, - mediapipe::internal::StaticAccessToGeneratorRegistry:: - CreateByNameInNamespace(options.package(), - options.packet_generator())); - MP_RETURN_IF_ERROR(static_access->FillExpectations( - options.options(), &cc->InputSidePackets(), - &cc->OutputSidePackets())) - .SetPrepend() - << options.packet_generator() << "::FillExpectations() failed: "; - return absl::OkStatus(); - } +absl::Status PacketGeneratorWrapperCalculator::GetContract( + CalculatorContract* cc) { + const auto& options = + cc->Options<::mediapipe::PacketGeneratorWrapperCalculatorOptions>(); + ASSIGN_OR_RETURN(auto static_access, + mediapipe::internal::StaticAccessToGeneratorRegistry:: + CreateByNameInNamespace(options.package(), + options.packet_generator())); + MP_RETURN_IF_ERROR(static_access->FillExpectations(options.options(), + &cc->InputSidePackets(), + &cc->OutputSidePackets())) + .SetPrepend() + << options.packet_generator() << "::FillExpectations() failed: "; + return absl::OkStatus(); +} - absl::Status Open(CalculatorContext* cc) override { - const auto& options = - cc->Options<::mediapipe::PacketGeneratorWrapperCalculatorOptions>(); - ASSIGN_OR_RETURN(auto static_access, - mediapipe::internal::StaticAccessToGeneratorRegistry:: - CreateByNameInNamespace(options.package(), - options.packet_generator())); - mediapipe::PacketSet output_packets(cc->OutputSidePackets().TagMap()); - MP_RETURN_IF_ERROR(static_access->Generate(options.options(), - cc->InputSidePackets(), - &output_packets)) - .SetPrepend() - << options.packet_generator() << "::Generate() failed: "; - for (auto id = output_packets.BeginId(); id < output_packets.EndId(); - ++id) { - cc->OutputSidePackets().Get(id).Set(output_packets.Get(id)); - } - return absl::OkStatus(); +absl::Status PacketGeneratorWrapperCalculator::Open(CalculatorContext* cc) { + const auto& options = + cc->Options<::mediapipe::PacketGeneratorWrapperCalculatorOptions>(); + ASSIGN_OR_RETURN(auto static_access, + mediapipe::internal::StaticAccessToGeneratorRegistry:: + CreateByNameInNamespace(options.package(), + options.packet_generator())); + mediapipe::PacketSet output_packets(cc->OutputSidePackets().TagMap()); + MP_RETURN_IF_ERROR(static_access->Generate(options.options(), + cc->InputSidePackets(), + &output_packets)) + .SetPrepend() + << options.packet_generator() << "::Generate() failed: "; + for (auto id = output_packets.BeginId(); id < output_packets.EndId(); ++id) { + cc->OutputSidePackets().Get(id).Set(output_packets.Get(id)); } + return absl::OkStatus(); +} + +absl::Status PacketGeneratorWrapperCalculator::Process(CalculatorContext* cc) { + return absl::OkStatus(); +} - absl::Status Process(CalculatorContext* cc) override { - return absl::OkStatus(); - } -}; REGISTER_CALCULATOR(PacketGeneratorWrapperCalculator); } // namespace mediapipe diff --git a/mediapipe/framework/tool/packet_generator_wrapper_calculator.h b/mediapipe/framework/tool/packet_generator_wrapper_calculator.h new file mode 100644 index 00000000..012281ca --- /dev/null +++ b/mediapipe/framework/tool/packet_generator_wrapper_calculator.h @@ -0,0 +1,32 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_FRAMEWORK_TOOL_PACKET_GENERATOR_WRAPPER_CALCULATOR_H_ +#define MEDIAPIPE_FRAMEWORK_TOOL_PACKET_GENERATOR_WRAPPER_CALCULATOR_H_ + +#include "absl/status/status.h" +#include "mediapipe/framework/calculator_base.h" + +namespace mediapipe { + +class PacketGeneratorWrapperCalculator : public CalculatorBase { + public: + static absl::Status GetContract(CalculatorContract* cc); + absl::Status Open(CalculatorContext* cc) override; + absl::Status Process(CalculatorContext* cc) override; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_FRAMEWORK_TOOL_PACKET_GENERATOR_WRAPPER_CALCULATOR_H_ diff --git a/mediapipe/framework/tool/proto_util_lite.cc b/mediapipe/framework/tool/proto_util_lite.cc index 745f4a13..285aa220 100644 --- a/mediapipe/framework/tool/proto_util_lite.cc +++ b/mediapipe/framework/tool/proto_util_lite.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_check.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" @@ -411,7 +412,7 @@ static absl::Status DeserializeValue(const FieldValue& bytes, } case W::TYPE_GROUP: case W::TYPE_MESSAGE: - CHECK(false) << "DeserializeValue cannot deserialize a Message."; + ABSL_CHECK(false) << "DeserializeValue cannot deserialize a Message."; case W::TYPE_UINT32: return ReadPrimitive(&input, result); case W::TYPE_ENUM: diff --git a/mediapipe/framework/tool/sink.cc b/mediapipe/framework/tool/sink.cc index f8abf492..b97d27ea 100644 --- a/mediapipe/framework/tool/sink.cc +++ b/mediapipe/framework/tool/sink.cc @@ -18,60 +18,65 @@ #include "mediapipe/framework/tool/sink.h" +#include + +#include +#include #include +#include #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" #include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/calculator_graph.h" #include "mediapipe/framework/calculator_registry.h" -#include "mediapipe/framework/input_stream.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet_type.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/source_location.h" #include "mediapipe/framework/port/status_builder.h" +#include "mediapipe/framework/timestamp.h" #include "mediapipe/framework/tool/name_util.h" +#include "mediapipe/framework/tool/status_util.h" namespace mediapipe { namespace tool { -namespace { -// Produces an output packet with the PostStream timestamp containing the -// input side packet. -class MediaPipeInternalSidePacketToPacketStreamCalculator - : public CalculatorBase { - public: - static absl::Status GetContract(CalculatorContract* cc) { - cc->InputSidePackets().Index(0).SetAny(); - cc->Outputs().Index(0).SetSameAs(&cc->InputSidePackets().Index(0)); - return absl::OkStatus(); - } - absl::Status Open(CalculatorContext* cc) final { - cc->Outputs().Index(0).AddPacket( - cc->InputSidePackets().Index(0).At(Timestamp::PostStream())); - cc->Outputs().Index(0).Close(); - return absl::OkStatus(); - } +absl::Status MediaPipeInternalSidePacketToPacketStreamCalculator::GetContract( + CalculatorContract* cc) { + cc->InputSidePackets().Index(0).SetAny(); + cc->Outputs().Index(0).SetSameAs(&cc->InputSidePackets().Index(0)); + return absl::OkStatus(); +} + +absl::Status MediaPipeInternalSidePacketToPacketStreamCalculator::Open( + CalculatorContext* cc) { + cc->Outputs().Index(0).AddPacket( + cc->InputSidePackets().Index(0).At(Timestamp::PostStream())); + cc->Outputs().Index(0).Close(); + return absl::OkStatus(); +} + +absl::Status MediaPipeInternalSidePacketToPacketStreamCalculator::Process( + CalculatorContext* cc) { + // The framework treats this calculator as a source calculator. + return mediapipe::tool::StatusStop(); +} - absl::Status Process(CalculatorContext* cc) final { - // The framework treats this calculator as a source calculator. - return mediapipe::tool::StatusStop(); - } -}; REGISTER_CALCULATOR(MediaPipeInternalSidePacketToPacketStreamCalculator); -} // namespace void AddVectorSink(const std::string& stream_name, // CalculatorGraphConfig* config, // std::vector* dumped_data) { - CHECK(config); - CHECK(dumped_data); + ABSL_CHECK(config); + ABSL_CHECK(dumped_data); std::string input_side_packet_name; tool::AddCallbackCalculator(stream_name, config, &input_side_packet_name, @@ -90,15 +95,15 @@ void AddVectorSink(const std::string& stream_name, // // Up to 64-bit pointer in hex (16 characters) and an optional "0x" prepended. char address[19]; int written = snprintf(address, sizeof(address), "%p", dumped_data); - CHECK(written > 0 && written < sizeof(address)); + ABSL_CHECK(written > 0 && written < sizeof(address)); options->set_pointer(address); } void AddPostStreamPacketSink(const std::string& stream_name, CalculatorGraphConfig* config, Packet* post_stream_packet) { - CHECK(config); - CHECK(post_stream_packet); + ABSL_CHECK(config); + ABSL_CHECK(post_stream_packet); std::string input_side_packet_name; tool::AddCallbackCalculator(stream_name, config, &input_side_packet_name, @@ -116,14 +121,14 @@ void AddPostStreamPacketSink(const std::string& stream_name, // Up to 64-bit pointer in hex (16 characters) and an optional "0x" prepended. char address[19]; int written = snprintf(address, sizeof(address), "%p", post_stream_packet); - CHECK(written > 0 && written < sizeof(address)); + ABSL_CHECK(written > 0 && written < sizeof(address)); options->set_pointer(address); } void AddSidePacketSink(const std::string& side_packet_name, CalculatorGraphConfig* config, Packet* dumped_packet) { - CHECK(config); - CHECK(dumped_packet); + ABSL_CHECK(config); + ABSL_CHECK(dumped_packet); CalculatorGraphConfig::Node* conversion_node = config->add_node(); const std::string node_name = GetUnusedNodeName( @@ -145,8 +150,8 @@ void AddCallbackCalculator(const std::string& stream_name, CalculatorGraphConfig* config, std::string* callback_side_packet_name, bool use_std_function) { - CHECK(config); - CHECK(callback_side_packet_name); + ABSL_CHECK(config); + ABSL_CHECK(callback_side_packet_name); CalculatorGraphConfig::Node* sink_node = config->add_node(); sink_node->set_name(GetUnusedNodeName( *config, @@ -162,7 +167,7 @@ void AddCallbackCalculator(const std::string& stream_name, sink_node->add_input_side_packet( absl::StrCat("CALLBACK:", input_side_packet_name)); } else { - LOG(FATAL) << "AddCallbackCalculator must use std::function"; + ABSL_LOG(FATAL) << "AddCallbackCalculator must use std::function"; } } @@ -182,8 +187,8 @@ void AddMultiStreamCallback( std::function&)> callback, CalculatorGraphConfig* config, std::map* side_packets, bool observe_timestamp_bounds) { - CHECK(config); - CHECK(side_packets); + ABSL_CHECK(config); + ABSL_CHECK(side_packets); CalculatorGraphConfig::Node* sink_node = config->add_node(); const std::string name = GetUnusedNodeName( *config, absl::StrCat("multi_callback_", absl::StrJoin(streams, "_"))); @@ -217,8 +222,8 @@ void AddCallbackWithHeaderCalculator(const std::string& stream_name, CalculatorGraphConfig* config, std::string* callback_side_packet_name, bool use_std_function) { - CHECK(config); - CHECK(callback_side_packet_name); + ABSL_CHECK(config); + ABSL_CHECK(callback_side_packet_name); CalculatorGraphConfig::Node* sink_node = config->add_node(); sink_node->set_name(GetUnusedNodeName( *config, @@ -237,7 +242,7 @@ void AddCallbackWithHeaderCalculator(const std::string& stream_name, sink_node->add_input_side_packet( absl::StrCat("CALLBACK:", input_side_packet_name)); } else { - LOG(FATAL) << "AddCallbackWithHeaderCalculator must use std::function"; + ABSL_LOG(FATAL) << "AddCallbackWithHeaderCalculator must use std::function"; } } @@ -286,7 +291,7 @@ absl::Status CallbackCalculator::Open(CalculatorContext* cc) { .Tag("VECTOR_CALLBACK") .Get&)>>(); } else { - LOG(FATAL) << "InputSidePackets must use tags."; + ABSL_LOG(FATAL) << "InputSidePackets must use tags."; } if (callback_ == nullptr && vector_callback_ == nullptr) { return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) @@ -326,7 +331,7 @@ absl::Status CallbackWithHeaderCalculator::GetContract(CalculatorContract* cc) { cc->Inputs().Tag("HEADER").SetAny(); if (cc->InputSidePackets().UsesTags()) { - CHECK(cc->InputSidePackets().HasTag("CALLBACK")); + ABSL_CHECK(cc->InputSidePackets().HasTag("CALLBACK")); cc->InputSidePackets() .Tag("CALLBACK") .Set>(); @@ -343,7 +348,7 @@ absl::Status CallbackWithHeaderCalculator::Open(CalculatorContext* cc) { .Tag("CALLBACK") .Get>(); } else { - LOG(FATAL) << "InputSidePackets must use tags."; + ABSL_LOG(FATAL) << "InputSidePackets must use tags."; } if (callback_ == nullptr) { return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) diff --git a/mediapipe/framework/tool/sink.h b/mediapipe/framework/tool/sink.h index f786e60a..4d00b6e6 100644 --- a/mediapipe/framework/tool/sink.h +++ b/mediapipe/framework/tool/sink.h @@ -28,10 +28,12 @@ #ifndef MEDIAPIPE_FRAMEWORK_TOOL_SINK_H_ #define MEDIAPIPE_FRAMEWORK_TOOL_SINK_H_ +#include #include #include #include "absl/base/macros.h" +#include "absl/status/status.h" #include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/packet_type.h" #include "mediapipe/framework/port/status.h" @@ -66,9 +68,9 @@ namespace tool { // // Call tool::AddVectorSink() more times if you wish. Note that each stream // // needs to get its own packet vector. // CalculatorGraph graph; -// CHECK_OK(graph.Initialize(config)); +// ABSL_CHECK_OK(graph.Initialize(config)); // // Set other input side packets. -// CHECK_OK(graph.Run()); +// ABSL_CHECK_OK(graph.Run()); // for (const Packet& packet : packet_dump) { // // Do something. // } @@ -158,7 +160,7 @@ void AddCallbackWithHeaderCalculator(const std::string& stream_name, // tool::AddCallbackCalculator("the_output_stream", &config, // &input_side_packet_name, true); // CalculatorGraph graph(config); -// CHECK_OK(graph.Run( +// ABSL_CHECK_OK(graph.Run( // {{input_side_packet_name, // MakePacket>( // std::bind(&MyClass::MyFunction, this, std::placeholders::_1))}} @@ -205,6 +207,16 @@ class CallbackWithHeaderCalculator : public CalculatorBase { Packet header_packet_; }; +// Produces an output packet with the PostStream timestamp containing the +// input side packet. +class MediaPipeInternalSidePacketToPacketStreamCalculator + : public CalculatorBase { + public: + static absl::Status GetContract(CalculatorContract* cc); + absl::Status Open(CalculatorContext* cc) final; + absl::Status Process(CalculatorContext* cc) final; +}; + } // namespace tool } // namespace mediapipe diff --git a/mediapipe/framework/tool/status_util.cc b/mediapipe/framework/tool/status_util.cc index 0c277a00..19f3fc6b 100644 --- a/mediapipe/framework/tool/status_util.cc +++ b/mediapipe/framework/tool/status_util.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" diff --git a/mediapipe/framework/tool/subgraph_expansion.cc b/mediapipe/framework/tool/subgraph_expansion.cc index dcd055f5..a05aef89 100644 --- a/mediapipe/framework/tool/subgraph_expansion.cc +++ b/mediapipe/framework/tool/subgraph_expansion.cc @@ -23,8 +23,13 @@ #include #include +#include "absl/container/flat_hash_set.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" +#include "absl/status/status.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/graph_service_manager.h" #include "mediapipe/framework/packet_generator.pb.h" #include "mediapipe/framework/port.h" @@ -123,6 +128,19 @@ absl::Status TransformNames( MP_RETURN_IF_ERROR(TransformStreamNames( status_handler.mutable_input_side_packet(), transform)); } + // Prefix executor names, but only those defined in the current graph. + absl::flat_hash_set local_executor_names; + for (auto& executor : *config->mutable_executor()) { + if (!executor.name().empty()) { + local_executor_names.insert(executor.name()); + *executor.mutable_name() = transform(executor.name()); + } + } + for (auto& node : *config->mutable_node()) { + if (local_executor_names.contains(node.executor())) { + *node.mutable_executor() = transform(node.executor()); + } + } return absl::OkStatus(); } @@ -273,6 +291,41 @@ absl::Status ConnectSubgraphStreams( return absl::OkStatus(); } +absl::Status RemoveDuplicateExecutors( + const absl::flat_hash_set& seen_executors, + CalculatorGraphConfig* config) { + auto* mutable_executors = config->mutable_executor(); + auto unique_executors_it = std::remove_if( + mutable_executors->begin(), mutable_executors->end(), + [&seen_executors](const mediapipe::ExecutorConfig& executor_config) { + bool is_duplicate = seen_executors.contains(executor_config.name()); + // This can happen in the following situation: you define an + // executor at the top-level-graph and one or more of your + // subgraphs declare executors with the same name as well. + // + // Historically, executors defined in subgraphs were ignored + // (unless you use your subgraph as a top-level-graph). + // + // Now executors can be defined in subgraphs (their names are + // automatically updated to be prefixed with subgraph name). To be + // backward compatible, MediaPipe will ignore (remove) executors + // defined in subgraphs if they have the same names as one of + // top-level-graph defined executors. + // + // NOTE: If you see this warning, you may want to verify if you + // actually use the same executors and consider removing one or + // another. + if (is_duplicate) { + ABSL_LOG(WARNING) << absl::StrFormat( + "Removing a duplicate of top-level-graph executor: %s", + executor_config.name()); + } + return is_duplicate; + }); + mutable_executors->erase(unique_executors_it, mutable_executors->end()); + return absl::OkStatus(); +} + absl::Status ExpandSubgraphs(CalculatorGraphConfig* config, const GraphRegistry* graph_registry, const Subgraph::SubgraphOptions* graph_options, @@ -283,6 +336,12 @@ absl::Status ExpandSubgraphs(CalculatorGraphConfig* config, MP_RETURN_IF_ERROR(mediapipe::tool::DefineGraphOptions( graph_options ? *graph_options : CalculatorGraphConfig::Node(), config)); + + absl::flat_hash_set seen_executors; + for (int i = 0; i < config->executor_size(); ++i) { + seen_executors.insert(config->executor(i).name()); + } + auto* nodes = config->mutable_node(); while (1) { auto subgraph_nodes_start = std::stable_partition( @@ -303,6 +362,7 @@ absl::Status ExpandSubgraphs(CalculatorGraphConfig* config, config->package(), node.calculator(), &subgraph_context)); MP_RETURN_IF_ERROR(mediapipe::tool::DefineGraphOptions(node, &subgraph)); + MP_RETURN_IF_ERROR(RemoveDuplicateExecutors(seen_executors, &subgraph)); MP_RETURN_IF_ERROR(PrefixNames(node_name, &subgraph)); MP_RETURN_IF_ERROR(ConnectSubgraphStreams(node, &subgraph)); subgraphs.push_back(subgraph); @@ -319,6 +379,9 @@ absl::Status ExpandSubgraphs(CalculatorGraphConfig* config, subgraph.status_handler().end(), proto_ns::RepeatedPtrFieldBackInserter( config->mutable_status_handler())); + std::copy( + subgraph.executor().begin(), subgraph.executor().end(), + proto_ns::RepeatedPtrFieldBackInserter(config->mutable_executor())); } } return absl::OkStatus(); diff --git a/mediapipe/framework/tool/subgraph_expansion_test.cc b/mediapipe/framework/tool/subgraph_expansion_test.cc index b6d9950a..f6988c56 100644 --- a/mediapipe/framework/tool/subgraph_expansion_test.cc +++ b/mediapipe/framework/tool/subgraph_expansion_test.cc @@ -15,6 +15,7 @@ #include +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -771,9 +772,7 @@ class InternalExecutorSubgraph : public Subgraph { }; REGISTER_MEDIAPIPE_GRAPH(InternalExecutorSubgraph); -// This test confirms that none of existing subgraphs can actually create an -// executor when used as subgraphs and not like a final graph. -TEST(SubgraphExpansionTest, SubgraphExecutorIsIgnored) { +TEST(SubgraphExpansionTest, SubgraphExecutorWorks) { CalculatorGraphConfig supergraph = mediapipe::ParseTextProtoOrDie(R"pb( input_stream: "input" @@ -785,23 +784,27 @@ TEST(SubgraphExpansionTest, SubgraphExecutorIsIgnored) { )pb"); CalculatorGraphConfig expected_graph = mediapipe::ParseTextProtoOrDie(R"pb( - input_stream: "input" node { name: "internalexecutorsubgraph__PassThroughCalculator" calculator: "PassThroughCalculator" input_stream: "input" output_stream: "output" - executor: "xyz" + executor: "internalexecutorsubgraph__xyz" + } + input_stream: "input" + executor { + name: "internalexecutorsubgraph__xyz" + type: "ThreadPoolExecutor" + options { + [mediapipe.ThreadPoolExecutorOptions.ext] { num_threads: 1 } + } } )pb"); MP_EXPECT_OK(tool::ExpandSubgraphs(&supergraph)); EXPECT_THAT(supergraph, mediapipe::EqualsProto(expected_graph)); CalculatorGraph calculator_graph; - EXPECT_THAT(calculator_graph.Initialize(supergraph), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("The executor \"xyz\" is " - "not declared in an ExecutorConfig."))); + MP_EXPECT_OK(calculator_graph.Initialize(supergraph)); } class NestedInternalExecutorsSubgraph : public Subgraph { @@ -847,7 +850,7 @@ class NestedInternalExecutorsSubgraph : public Subgraph { }; REGISTER_MEDIAPIPE_GRAPH(NestedInternalExecutorsSubgraph); -TEST(SubgraphExpansionTest, NestedSubgraphExecutorsAreIgnored) { +TEST(SubgraphExpansionTest, NestedSubgraphExecutorsWork) { CalculatorGraphConfig supergraph = mediapipe::ParseTextProtoOrDie(R"pb( input_stream: "input" @@ -864,35 +867,55 @@ TEST(SubgraphExpansionTest, NestedSubgraphExecutorsAreIgnored) { calculator: "PassThroughCalculator" input_stream: "nestedinternalexecutorssubgraph__bar_0" output_stream: "nestedinternalexecutorssubgraph__bar_1" - executor: "xyz" + executor: "nestedinternalexecutorssubgraph__xyz" } node { name: "nestedinternalexecutorssubgraph__PassThroughCalculator_2" calculator: "PassThroughCalculator" input_stream: "nestedinternalexecutorssubgraph__bar_1" output_stream: "output" - executor: "abc" + executor: "nestedinternalexecutorssubgraph__abc" } node { name: "nestedinternalexecutorssubgraph__internalexecutorsubgraph__PassThroughCalculator" calculator: "PassThroughCalculator" input_stream: "input" output_stream: "nestedinternalexecutorssubgraph__bar_0" - executor: "xyz" + executor: "nestedinternalexecutorssubgraph__internalexecutorsubgraph__xyz" } input_stream: "input" + executor { + name: "nestedinternalexecutorssubgraph__xyz" + type: "ThreadPoolExecutor" + options { + [mediapipe.ThreadPoolExecutorOptions.ext] { num_threads: 1 } + } + } + executor { + name: "nestedinternalexecutorssubgraph__abc" + type: "ThreadPoolExecutor" + options { + [mediapipe.ThreadPoolExecutorOptions.ext] { num_threads: 1 } + } + } + executor { + name: "nestedinternalexecutorssubgraph__internalexecutorsubgraph__xyz" + type: "ThreadPoolExecutor" + options { + [mediapipe.ThreadPoolExecutorOptions.ext] { num_threads: 1 } + } + } )pb"); MP_EXPECT_OK(tool::ExpandSubgraphs(&supergraph)); EXPECT_THAT(supergraph, mediapipe::EqualsProto(expected_graph)); CalculatorGraph calculator_graph; - EXPECT_THAT(calculator_graph.Initialize(supergraph), - StatusIs(absl::StatusCode::kInvalidArgument, - HasSubstr("The executor \"xyz\" is " - "not declared in an ExecutorConfig."))); + MP_EXPECT_OK(calculator_graph.Initialize(supergraph)); } -TEST(SubgraphExpansionTest, GraphExecutorsSubstituteSubgraphExecutors) { +// For backward compatibility. +TEST(SubgraphExpansionTest, + TopLevelGraphExecutorsCauseSameNamedSubgraphExecutorsToBeRemoved) { CalculatorGraphConfig supergraph = mediapipe::ParseTextProtoOrDie(R"pb( input_stream: "input" diff --git a/mediapipe/framework/tool/switch_container.cc b/mediapipe/framework/tool/switch_container.cc index daa12992..29307c4f 100644 --- a/mediapipe/framework/tool/switch_container.cc +++ b/mediapipe/framework/tool/switch_container.cc @@ -20,6 +20,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -148,7 +149,7 @@ void ClearContainerOptions(CalculatorGraphConfig::Node* dest) { // Returns an unused name similar to a specified name. std::string UniqueName(std::string name, std::set* names) { - CHECK(names != nullptr); + ABSL_CHECK(names != nullptr); std::string result = name; int suffix = 2; while (names->count(result) > 0) { @@ -161,7 +162,7 @@ std::string UniqueName(std::string name, std::set* names) { // Parses tag, index, and name from a list of stream identifiers. void ParseTags(const proto_ns::RepeatedPtrField& streams, std::map* result) { - CHECK(result != nullptr); + ABSL_CHECK(result != nullptr); std::set used_names; int used_index = -1; for (const std::string& stream : streams) { @@ -177,14 +178,14 @@ void ParseTags(const proto_ns::RepeatedPtrField& streams, // Removes the entry for a tag and index from a map. void EraseTag(const std::string& stream, std::map* streams) { - CHECK(streams != nullptr); + ABSL_CHECK(streams != nullptr); streams->erase(ParseTagIndexFromStream(absl::StrCat(stream, ":u"))); } // Removes the entry for a tag and index from a list. void EraseTag(const std::string& stream, proto_ns::RepeatedPtrField* streams) { - CHECK(streams != nullptr); + ABSL_CHECK(streams != nullptr); TagIndex stream_tag = ParseTagIndexFromStream(absl::StrCat(stream, ":u")); for (int i = streams->size() - 1; i >= 0; --i) { TagIndex tag = ParseTagIndexFromStream(streams->at(i)); @@ -197,7 +198,7 @@ void EraseTag(const std::string& stream, // Returns the stream names for the container node. void GetContainerNodeStreams(const CalculatorGraphConfig::Node& node, CalculatorGraphConfig::Node* result) { - CHECK(result != nullptr); + ABSL_CHECK(result != nullptr); *result->mutable_input_stream() = node.input_stream(); *result->mutable_output_stream() = node.output_stream(); *result->mutable_input_side_packet() = node.input_side_packet(); diff --git a/mediapipe/framework/tool/switch_container_test.cc b/mediapipe/framework/tool/switch_container_test.cc index 08cc4ab5..5ffd26e0 100644 --- a/mediapipe/framework/tool/switch_container_test.cc +++ b/mediapipe/framework/tool/switch_container_test.cc @@ -17,13 +17,13 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/proto_ns.h" #include "mediapipe/framework/port/ret_check.h" @@ -385,7 +385,7 @@ TEST(SwitchContainerTest, RunsWithInputStreamHandler) { CalculatorGraphConfig supergraph = SubnodeContainerExample(R"pb(synchronize_io: true)pb"); MP_EXPECT_OK(tool::ExpandSubgraphs(&supergraph)); - LOG(INFO) << supergraph.DebugString(); + ABSL_LOG(INFO) << supergraph.DebugString(); RunTestContainer(supergraph, true); } diff --git a/mediapipe/framework/tool/tag_map_test.cc b/mediapipe/framework/tool/tag_map_test.cc index a93b9444..68ee94ae 100644 --- a/mediapipe/framework/tool/tag_map_test.cc +++ b/mediapipe/framework/tool/tag_map_test.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/tool/tag_map.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_join.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" @@ -329,8 +330,8 @@ void TestDebugString( tool::TagMap& tag_map = *statusor_tag_map.value(); std::string debug_string = tag_map.DebugString(); std::string short_string = tag_map.ShortDebugString(); - LOG(INFO) << "ShortDebugString:\n" << short_string << "\n"; - LOG(INFO) << "DebugString:\n" << debug_string << "\n\n"; + ABSL_LOG(INFO) << "ShortDebugString:\n" << short_string << "\n"; + ABSL_LOG(INFO) << "DebugString:\n" << debug_string << "\n\n"; std::vector actual_entries; for (const auto& field : tag_map.CanonicalEntries()) { diff --git a/mediapipe/framework/tool/template_expander.cc b/mediapipe/framework/tool/template_expander.cc index a91ea5ad..8f9ef686 100644 --- a/mediapipe/framework/tool/template_expander.cc +++ b/mediapipe/framework/tool/template_expander.cc @@ -15,20 +15,16 @@ #include "mediapipe/framework/tool/template_expander.h" #include -#include #include #include -#include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/ascii.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" -#include "absl/strings/str_join.h" -#include "absl/strings/str_split.h" #include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/numbers.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -183,8 +179,8 @@ FieldType GetFieldType(const TemplateExpression& rule) { int FieldCount(const FieldValue& base, ProtoPath field_path, FieldType field_type) { int result = 0; - CHECK( - ProtoUtilLite::GetFieldCount(base, field_path, field_type, &result).ok()); + ABSL_CHECK_OK( + ProtoUtilLite::GetFieldCount(base, field_path, field_type, &result)); return result; } @@ -647,7 +643,7 @@ class TemplateExpanderImpl { for (int i = 0; i < args.size(); ++i) { if (args[i].has_dict()) { FieldValue dict_bytes; - CHECK(args[i].dict().SerializePartialToString(&dict_bytes)); + ABSL_CHECK(args[i].dict().SerializePartialToString(&dict_bytes)); result->push_back(dict_bytes); } else if (args[i].has_num() || args[i].has_str()) { std::string text_value = args[i].has_num() @@ -694,7 +690,7 @@ absl::Status TemplateExpander::ExpandTemplates( } absl::Status status; for (const absl::Status& error : errors_) { - LOG(ERROR) << error; + ABSL_LOG(ERROR) << error; status.Update(error); } return status; diff --git a/mediapipe/framework/tool/template_parser.cc b/mediapipe/framework/tool/template_parser.cc index ad799c34..d97ec0c2 100644 --- a/mediapipe/framework/tool/template_parser.cc +++ b/mediapipe/framework/tool/template_parser.cc @@ -20,6 +20,9 @@ #include #include +#include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/ascii.h" #include "absl/strings/numbers.h" @@ -30,7 +33,6 @@ #include "mediapipe/framework/deps/proto_descriptor.pb.h" #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/map_util.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -180,11 +182,11 @@ void CheckFieldIndex(const FieldDescriptor* field, int index) { } if (field->is_repeated() && index == -1) { - LOG(DFATAL) << "Index must be in range of repeated field values. " - << "Field: " << field->name(); + ABSL_LOG(ERROR) << "Index must be in range of repeated field values. " + << "Field: " << field->name(); } else if (!field->is_repeated() && index != -1) { - LOG(DFATAL) << "Index must be -1 for singular fields." - << "Field: " << field->name(); + ABSL_LOG(ERROR) << "Index must be -1 for singular fields." + << "Field: " << field->name(); } } @@ -304,7 +306,7 @@ class TemplateParser::Parser::ParserImpl { // Parses the ASCII representation specified in input and saves the // information into the output pointer (a Message). Returns // false if an error occurs (an error will also be logged to - // LOG(ERROR)). + // ABSL_LOG(ERROR)). virtual bool Parse(Message* output) { // Consume fields until we cannot do so anymore. while (true) { @@ -334,12 +336,12 @@ class TemplateParser::Parser::ParserImpl { had_errors_ = true; if (error_collector_ == NULL) { if (line >= 0) { - LOG(ERROR) << "Error parsing text-format " - << root_message_type_->full_name() << ": " << (line + 1) - << ":" << (col + 1) << ": " << message; + ABSL_LOG(ERROR) << "Error parsing text-format " + << root_message_type_->full_name() << ": " << (line + 1) + << ":" << (col + 1) << ": " << message; } else { - LOG(ERROR) << "Error parsing text-format " - << root_message_type_->full_name() << ": " << message; + ABSL_LOG(ERROR) << "Error parsing text-format " + << root_message_type_->full_name() << ": " << message; } } else { error_collector_->AddError(line, col, std::string(message)); @@ -349,12 +351,12 @@ class TemplateParser::Parser::ParserImpl { void ReportWarning(int line, int col, absl::string_view message) { if (error_collector_ == NULL) { if (line >= 0) { - LOG(WARNING) << "Warning parsing text-format " - << root_message_type_->full_name() << ": " << (line + 1) - << ":" << (col + 1) << ": " << message; + ABSL_LOG(WARNING) << "Warning parsing text-format " + << root_message_type_->full_name() << ": " + << (line + 1) << ":" << (col + 1) << ": " << message; } else { - LOG(WARNING) << "Warning parsing text-format " - << root_message_type_->full_name() << ": " << message; + ABSL_LOG(WARNING) << "Warning parsing text-format " + << root_message_type_->full_name() << ": " << message; } } else { error_collector_->AddWarning(line, col, std::string(message)); @@ -470,7 +472,7 @@ class TemplateParser::Parser::ParserImpl { "\" stored in google.protobuf.Any."); return false; } - DO(ConsumeAnyValue(value_descriptor, &serialized_value)); + DO(ConsumeAnyValue(any_value_field, value_descriptor, &serialized_value)); if (singular_overwrite_policy_ == FORBID_SINGULAR_OVERWRITES) { // Fail if any_type_url_field has already been specified. if ((!any_type_url_field->is_repeated() && @@ -564,7 +566,8 @@ class TemplateParser::Parser::ParserImpl { // Skips unknown or reserved fields. if (field == NULL) { - CHECK(allow_unknown_field_ || allow_unknown_extension_ || reserved_field); + ABSL_CHECK(allow_unknown_field_ || allow_unknown_extension_ || + reserved_field); // Try to guess the type of this field. // If this field is not a message, there should be a ":" between the @@ -708,7 +711,7 @@ class TemplateParser::Parser::ParserImpl { // If the parse information tree is not NULL, create a nested one // for the nested message. ParseInfoTree* parent = parse_info_tree_; - if (parent != NULL) { + if (parent) { parse_info_tree_ = parent->CreateNested(field); } @@ -883,7 +886,7 @@ class TemplateParser::Parser::ParserImpl { case FieldDescriptor::CPPTYPE_MESSAGE: { // We should never get here. Put here instead of a default // so that if new types are added, we get a nice compiler warning. - LOG(FATAL) << "Reached an unintended state: CPPTYPE_MESSAGE"; + ABSL_LOG(FATAL) << "Reached an unintended state: CPPTYPE_MESSAGE"; break; } } @@ -1190,8 +1193,20 @@ class TemplateParser::Parser::ParserImpl { // A helper function for reconstructing Any::value. Consumes a text of // full_type_name, then serializes it into serialized_value. - bool ConsumeAnyValue(const Descriptor* value_descriptor, + bool ConsumeAnyValue(const FieldDescriptor* field, + const Descriptor* value_descriptor, std::string* serialized_value) { + if (--recursion_limit_ < 0) { + ReportError("Message is too deep"); + return false; + } + // If the parse information tree is not NULL, create a nested one + // for the nested message. + ParseInfoTree* parent = parse_info_tree_; + if (parent) { + parse_info_tree_ = parent->CreateNested(field); + } + DynamicMessageFactory factory; const Message* value_prototype = factory.GetPrototype(value_descriptor); if (value_prototype == NULL) { @@ -1213,6 +1228,11 @@ class TemplateParser::Parser::ParserImpl { } value->AppendToString(serialized_value); } + + ++recursion_limit_; + + // Reset the parse information tree. + parse_info_tree_ = parent; return true; } @@ -1379,7 +1399,7 @@ bool DeterministicallySerialize(const Message& proto, std::string* result) { void SerializeField(const Message* message, const FieldDescriptor* field, std::vector* result) { ProtoUtilLite::FieldValue message_bytes; - CHECK(DeterministicallySerialize(*message, &message_bytes)); + ABSL_CHECK(DeterministicallySerialize(*message, &message_bytes)); ProtoUtilLite::FieldAccess access( field->number(), static_cast(field->type())); MEDIAPIPE_CHECK_OK(access.SetMessage(message_bytes)); @@ -1430,10 +1450,10 @@ std::vector GetFields(const Message* src) { // Orders map entries in dst to match src. void OrderMapEntries(const Message* src, Message* dst, - std::set* seen = nullptr) { - std::unique_ptr> seen_owner; + absl::flat_hash_set* seen = nullptr) { + std::unique_ptr> seen_owner; if (!seen) { - seen_owner = std::make_unique>(); + seen_owner = std::make_unique>(); seen = seen_owner.get(); } if (seen->count(src) > 0) { @@ -1684,13 +1704,13 @@ class TemplateParser::Parser::MediaPipeParserImpl const std::vector& args) { auto field_type = static_cast(field->type()); ProtoUtilLite::FieldValue message_bytes; - CHECK(message->SerializePartialToString(&message_bytes)); + ABSL_CHECK(message->SerializePartialToString(&message_bytes)); int count; MEDIAPIPE_CHECK_OK(ProtoUtilLite::GetFieldCount( message_bytes, {{field->number(), 0}}, field_type, &count)); MEDIAPIPE_CHECK_OK(ProtoUtilLite::ReplaceFieldRange( &message_bytes, {{field->number(), count}}, 0, field_type, args)); - CHECK(message->ParsePartialFromString(message_bytes)); + ABSL_CHECK(message->ParsePartialFromString(message_bytes)); } // Parse and record a template definition for the current field path. diff --git a/mediapipe/framework/tool/test_util.cc b/mediapipe/framework/tool/test_util.cc index 64b5072c..e5fac11a 100644 --- a/mediapipe/framework/tool/test_util.cc +++ b/mediapipe/framework/tool/test_util.cc @@ -22,6 +22,8 @@ #include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/strings/match.h" @@ -35,7 +37,6 @@ #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/port/advanced_proto_inc.h" #include "mediapipe/framework/port/file_helpers.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/proto_ns.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status_macros.h" @@ -59,8 +60,8 @@ absl::Status CompareDiff(const ImageFrame& image1, const ImageFrame& image2, const float max_avg_diff, std::unique_ptr& diff_image) { // Verify image byte depth matches expected byte depth. - CHECK_EQ(sizeof(T), image1.ByteDepth()); - CHECK_EQ(sizeof(T), image2.ByteDepth()); + ABSL_CHECK_EQ(sizeof(T), image1.ByteDepth()); + ABSL_CHECK_EQ(sizeof(T), image2.ByteDepth()); const int width = image1.Width(); const int height = image1.Height(); @@ -71,8 +72,8 @@ absl::Status CompareDiff(const ImageFrame& image1, const ImageFrame& image2, const int num_channels = std::min(channels1, channels2); // Verify the width steps are multiples of byte depth. - CHECK_EQ(image1.WidthStep() % image1.ByteDepth(), 0); - CHECK_EQ(image2.WidthStep() % image2.ByteDepth(), 0); + ABSL_CHECK_EQ(image1.WidthStep() % image1.ByteDepth(), 0); + ABSL_CHECK_EQ(image2.WidthStep() % image2.ByteDepth(), 0); const int width_padding1 = image1.WidthStep() / image1.ByteDepth() - width * channels1; const int width_padding2 = @@ -143,7 +144,7 @@ absl::Status CompareDiff(const ImageFrame& image1, const ImageFrame& image2, std::string GetBinaryDirectory() { char full_path[PATH_MAX + 1]; int length = readlink("/proc/self/exe", full_path, PATH_MAX + 1); - CHECK_GT(length, 0); + ABSL_CHECK_GT(length, 0); return std::string( ::mediapipe::file::Dirname(absl::string_view(full_path, length))); } @@ -196,7 +197,7 @@ absl::Status CompareImageFrames(const ImageFrame& image1, return CompareDiff(image1, image2, max_color_diff, max_alpha_diff, max_avg_diff, diff_image); default: - LOG(FATAL) << ImageFrame::InvalidFormatString(image1.Format()); + ABSL_LOG(FATAL) << ImageFrame::InvalidFormatString(image1.Format()); } } @@ -228,7 +229,9 @@ absl::Status CompareAndSaveImageOutput( auto status = CompareImageFrames(**expected, actual, options.max_color_diff, options.max_alpha_diff, options.max_avg_diff, diff_img); - ASSIGN_OR_RETURN(auto diff_img_path, SavePngTestOutput(*diff_img, "diff")); + if (diff_img) { + ASSIGN_OR_RETURN(auto diff_img_path, SavePngTestOutput(*diff_img, "diff")); + } return status; } @@ -334,15 +337,15 @@ absl::StatusOr SavePngTestOutput( bool LoadTestGraph(CalculatorGraphConfig* proto, const std::string& path) { int fd = open(path.c_str(), O_RDONLY); if (fd == -1) { - LOG(ERROR) << "could not open test graph: " << path - << ", error: " << strerror(errno); + ABSL_LOG(ERROR) << "could not open test graph: " << path + << ", error: " << strerror(errno); return false; } proto_ns::io::FileInputStream input(fd); bool success = proto->ParseFromZeroCopyStream(&input); close(fd); if (!success) { - LOG(ERROR) << "could not parse test graph: " << path; + ABSL_LOG(ERROR) << "could not parse test graph: " << path; } return success; } @@ -353,7 +356,7 @@ std::unique_ptr GenerateLuminanceImage( const int height = original_image.Height(); const int channels = original_image.NumberOfChannels(); if (channels != 3 && channels != 4) { - LOG(ERROR) << "Invalid number of image channels: " << channels; + ABSL_LOG(ERROR) << "Invalid number of image channels: " << channels; return nullptr; } auto luminance_image = diff --git a/mediapipe/framework/tool/text_to_binary_graph.cc b/mediapipe/framework/tool/text_to_binary_graph.cc index b6b38dea..046f0751 100644 --- a/mediapipe/framework/tool/text_to_binary_graph.cc +++ b/mediapipe/framework/tool/text_to_binary_graph.cc @@ -21,9 +21,11 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/port/advanced_proto_inc.h" #include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -33,10 +35,10 @@ ABSL_FLAG(std::string, proto_source, "", ABSL_FLAG(std::string, proto_output, "", "An output template file in binary CalculatorGraphTemplate form."); -#define EXIT_IF_ERROR(status) \ - if (!status.ok()) { \ - LOG(ERROR) << status; \ - return EXIT_FAILURE; \ +#define EXIT_IF_ERROR(status) \ + if (!status.ok()) { \ + ABSL_LOG(ERROR) << status; \ + return EXIT_FAILURE; \ } namespace mediapipe { diff --git a/mediapipe/framework/tool/validate_type.cc b/mediapipe/framework/tool/validate_type.cc index 4c97a310..38c04fa8 100644 --- a/mediapipe/framework/tool/validate_type.cc +++ b/mediapipe/framework/tool/validate_type.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_contract.h" #include "mediapipe/framework/calculator_framework.h" @@ -78,7 +79,7 @@ absl::Status RunGenerateAndValidateTypes( const PacketGeneratorOptions& extendable_options, const PacketSet& input_side_packets, PacketSet* output_side_packets, const std::string& package) { - CHECK(output_side_packets); + ABSL_CHECK(output_side_packets); // Get static access to functions. ASSIGN_OR_RETURN( auto static_access, diff --git a/mediapipe/framework/type_map.h b/mediapipe/framework/type_map.h index 42f6fe6b..f03f48ce 100644 --- a/mediapipe/framework/type_map.h +++ b/mediapipe/framework/type_map.h @@ -64,6 +64,8 @@ #include #include "absl/base/macros.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/demangle.h" #include "mediapipe/framework/port/status.h" @@ -127,7 +129,7 @@ class StaticMap { } static void GetKeys(std::vector* keys) { - CHECK(keys); + ABSL_CHECK(keys); keys->clear(); const MapType& internal_map = GetMap()->internal_map_; for (typename MapType::const_iterator i = internal_map.begin(); @@ -158,12 +160,12 @@ class StaticMap { // Type has been already registered. const MediaPipeTypeData& existing_data = it->second.second; - CHECK_EQ(existing_data.type_id, value.type_id) + ABSL_CHECK_EQ(existing_data.type_id, value.type_id) << "Found inconsistent type ids (" << existing_data.type_id << " vs " << value.type_id << ") during mediapipe type registration. Previous definition at " << it->second.first << " and current definition at " << file_and_line; - CHECK_EQ(existing_data.type_string, value.type_string) + ABSL_CHECK_EQ(existing_data.type_string, value.type_string) << "Found inconsistent type strings (" << existing_data.type_string << " vs " << value.type_string << ") during mediapipe type registration. Previous registration at " @@ -171,29 +173,31 @@ class StaticMap { << file_and_line; if (value.serialize_fn && value.deserialize_fn) { // Doesn't allow to redefine the existing type serialization functions. - CHECK(!existing_data.serialize_fn && !existing_data.deserialize_fn) + ABSL_CHECK(!existing_data.serialize_fn && !existing_data.deserialize_fn) << "Attempting to redefine serialization functions of type " << value.type_string << ", that have been defined at " << it->second.first << ", at " << file_and_line; const std::string previous_file_and_line = it->second.first; it->second.first = file_and_line; it->second.second = value; - LOG(WARNING) << "Redo mediapipe type registration of type " - << value.type_string << " with serialization function at " - << file_and_line << ". It was registered at " - << previous_file_and_line; + ABSL_LOG(WARNING) << "Redo mediapipe type registration of type " + << value.type_string + << " with serialization function at " << file_and_line + << ". It was registered at " + << previous_file_and_line; } else if (!value.serialize_fn && !value.deserialize_fn) { // Prefers type registration with serialization functions. If type has // been registered with some serialization functions, the // non-serialization version will be ignored. - LOG(WARNING) << "Ignore mediapipe type registration of type " - << value.type_string << " at " << file_and_line - << ", since type has been registered with serialization " - "functions at " - << it->second.first; + ABSL_LOG(WARNING) + << "Ignore mediapipe type registration of type " + << value.type_string << " at " << file_and_line + << ", since type has been registered with serialization " + "functions at " + << it->second.first; } else { // Doesn't allow to only have one of serialize_fn and deserialize_fn. - LOG(FATAL) + ABSL_LOG(FATAL) << "Invalid mediapipe type registration at " << file_and_line << ". Serialization functions should be provided at the same time."; } diff --git a/mediapipe/framework/validated_graph_config.cc b/mediapipe/framework/validated_graph_config.cc index 15eac320..4f918247 100644 --- a/mediapipe/framework/validated_graph_config.cc +++ b/mediapipe/framework/validated_graph_config.cc @@ -15,8 +15,11 @@ #include "mediapipe/framework/validated_graph_config.h" #include +#include #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -33,6 +36,7 @@ #include "mediapipe/framework/port/core_proto_inc.h" #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/proto_ns.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/source_location.h" #include "mediapipe/framework/port/status.h" @@ -49,8 +53,6 @@ namespace mediapipe { -namespace { - // Create a debug string name for a set of edge. An edge can be either // a stream or a side packet. std::string DebugEdgeNames( @@ -78,6 +80,8 @@ std::string DebugName(const CalculatorGraphConfig::Node& node_config) { return name; } +namespace { + std::string DebugName(const PacketGeneratorConfig& node_config) { return absl::StrCat( "[", node_config.packet_generator(), ", ", @@ -98,7 +102,7 @@ std::string DebugName(const CalculatorGraphConfig& config, NodeTypeInfo::NodeType node_type, int node_index) { switch (node_type) { case NodeTypeInfo::NodeType::CALCULATOR: - return DebugName(config.node(node_index)); + return mediapipe::DebugName(config.node(node_index)); case NodeTypeInfo::NodeType::PACKET_GENERATOR: return DebugName(config.packet_generator(node_index)); case NodeTypeInfo::NodeType::GRAPH_INPUT_STREAM: @@ -108,8 +112,8 @@ std::string DebugName(const CalculatorGraphConfig& config, case NodeTypeInfo::NodeType::UNKNOWN: /* Fall through. */ {} } - LOG(FATAL) << "Unknown NodeTypeInfo::NodeType: " - << NodeTypeInfo::NodeTypeToString(node_type); + ABSL_LOG(FATAL) << "Unknown NodeTypeInfo::NodeType: " + << NodeTypeInfo::NodeTypeToString(node_type); } // Adds the ExecutorConfigs for predefined executors, if they are not in @@ -158,8 +162,8 @@ std::string NodeTypeInfo::NodeTypeToString(NodeType node_type) { case NodeTypeInfo::NodeType::UNKNOWN: return "Unknown Node"; } - LOG(FATAL) << "Unknown NodeTypeInfo::NodeType: " - << static_cast(node_type); + ABSL_LOG(FATAL) << "Unknown NodeTypeInfo::NodeType: " + << static_cast(node_type); } absl::Status NodeTypeInfo::Initialize( @@ -692,12 +696,13 @@ absl::Status ValidatedGraphConfig::AddInputStreamsForNode( if (edge_info.back_edge) { // A back edge was specified, but its output side was already seen. if (!need_sorting_ptr) { - LOG(WARNING) << "Input Stream \"" << name - << "\" for node with sorted index " << node_index - << " name " << node_type_info->Contract().GetNodeName() - << " is marked as a back edge, but its output stream is " - "already available. This means it was not necessary " - "to mark it as a back edge."; + ABSL_LOG(WARNING) + << "Input Stream \"" << name << "\" for node with sorted index " + << node_index << " name " + << node_type_info->Contract().GetNodeName() + << " is marked as a back edge, but its output stream is " + "already available. This means it was not necessary " + "to mark it as a back edge."; } } else { edge_info.upstream = iter->second; @@ -744,7 +749,7 @@ int ValidatedGraphConfig::SorterIndexForNode(NodeTypeInfo::NodeRef node) const { case NodeTypeInfo::NodeType::CALCULATOR: return generators_.size() + node.index; default: - CHECK(false); + ABSL_CHECK(false); } } @@ -900,8 +905,8 @@ absl::Status ValidatedGraphConfig::ValidateSidePacketTypes() { "\"$3\" but the connected output side packet will be of type \"$4\"", side_packet.name, NodeTypeInfo::NodeTypeToString(side_packet.parent_node.type), - mediapipe::DebugName(config_, side_packet.parent_node.type, - side_packet.parent_node.index), + DebugName(config_, side_packet.parent_node.type, + side_packet.parent_node.index), side_packet.packet_type->DebugTypeName(), output_side_packets_[side_packet.upstream] .packet_type->DebugTypeName())); diff --git a/mediapipe/framework/validated_graph_config.h b/mediapipe/framework/validated_graph_config.h index 95ecccbb..ec46b62b 100644 --- a/mediapipe/framework/validated_graph_config.h +++ b/mediapipe/framework/validated_graph_config.h @@ -16,15 +16,18 @@ #define MEDIAPIPE_FRAMEWORK_VALIDATED_GRAPH_CONFIG_H_ #include +#include #include #include "absl/container/flat_hash_set.h" +#include "google/protobuf/repeated_ptr_field.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator_contract.h" #include "mediapipe/framework/graph_service_manager.h" #include "mediapipe/framework/packet_generator.pb.h" #include "mediapipe/framework/packet_type.h" #include "mediapipe/framework/port/map_util.h" +#include "mediapipe/framework/port/proto_ns.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" #include "mediapipe/framework/status_handler.pb.h" @@ -34,6 +37,12 @@ namespace mediapipe { class ValidatedGraphConfig; +std::string DebugEdgeNames( + const std::string& edge_type, + const proto_ns::RepeatedPtrField& edges); + +std::string DebugName(const CalculatorGraphConfig::Node& node_config); + // Type information for a graph node (Calculator, Generator, etc). class NodeTypeInfo { public: diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index ee32b91e..74c7e2d0 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -204,6 +204,8 @@ cc_library( "//mediapipe/framework/port:threadpool", "@com_google_absl//absl/base:dynamic_annotations", "@com_google_absl//absl/debugging:leak_check", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", @@ -235,6 +237,8 @@ cc_library( ":gpu_buffer_format", ":gpu_buffer_storage", ":gpu_buffer_storage_image_frame", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", # TODO: remove this dependency. Some other teams' tests # depend on having an indirect image_frame dependency, need to be @@ -295,7 +299,7 @@ cc_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:logging", "@com_google_absl//absl/functional:bind_front", - "@com_google_absl//absl/log:check", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", ] + select({ @@ -332,6 +336,7 @@ cc_library( "//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/port:logging", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", ] + select({ "//conditions:default": [ ":gl_base", @@ -368,6 +373,8 @@ cc_library( ":image_frame_view", "//mediapipe/objc:CFHolder", "//mediapipe/objc:util", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -398,6 +405,7 @@ cc_library( ":pixel_buffer_pool_util", "//mediapipe/framework/port:logging", "//mediapipe/objc:CFHolder", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/synchronization", ], ) @@ -421,6 +429,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/objc:CFHolder", "//mediapipe/objc:util", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/synchronization", ], ) @@ -437,6 +446,7 @@ cc_library( ":image_frame_view", "//mediapipe/framework/formats:frame_buffer", "//mediapipe/framework/formats:image_frame", + "@com_google_absl//absl/log:absl_check", ], ) @@ -476,8 +486,8 @@ cc_library( "//mediapipe/framework/formats:yuv_image", "//mediapipe/util/frame_buffer:frame_buffer_util", "//third_party/libyuv", - "@com_google_absl//absl/log", - "@com_google_absl//absl/log:check", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -635,6 +645,7 @@ cc_library( "//mediapipe/framework/deps:no_destructor", "//mediapipe/framework/port:ret_check", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", ] + select({ "//conditions:default": [], "//mediapipe:apple": [ @@ -756,6 +767,7 @@ cc_library( deps = [ ":gl_base", "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_log", ], ) @@ -818,11 +830,12 @@ cc_library( "//mediapipe/framework/deps:registration", "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:map_util", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/synchronization", ] + select({ @@ -848,6 +861,8 @@ objc_library( "//mediapipe/objc:mediapipe_framework_ios", "//third_party/apple_frameworks:CoreVideo", "//third_party/apple_frameworks:Metal", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@google_toolbox_for_mac//:GTM_Defines", ], ) @@ -990,6 +1005,7 @@ cc_library( "//mediapipe/framework/api2:node", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/synchronization", ], alwayslink = 1, @@ -1121,7 +1137,7 @@ objc_library( alwayslink = 1, ) -MIN_IOS_VERSION = "11.0" +MIN_IOS_VERSION = "12.0" test_suite( name = "ios", @@ -1208,5 +1224,6 @@ mediapipe_cc_test( "//mediapipe/framework/formats:yuv_image", "//mediapipe/framework/port:gtest_main", "//third_party/libyuv", + "@com_google_absl//absl/log:absl_check", ], ) diff --git a/mediapipe/gpu/MPPMetalHelper.mm b/mediapipe/gpu/MPPMetalHelper.mm index c0703e6e..c6648369 100644 --- a/mediapipe/gpu/MPPMetalHelper.mm +++ b/mediapipe/gpu/MPPMetalHelper.mm @@ -14,9 +14,11 @@ #import "mediapipe/gpu/MPPMetalHelper.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #import "mediapipe/gpu/gpu_buffer.h" -#import "mediapipe/gpu/graph_support.h" #import "mediapipe/gpu/gpu_service.h" +#import "mediapipe/gpu/graph_support.h" #import "mediapipe/gpu/metal_shared_resources.h" #import "GTMDefines.h" @@ -78,14 +80,13 @@ class MetalHelperLegacySupport { - (instancetype)initWithSidePackets:(const mediapipe::PacketSet&)inputSidePackets { auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContext(); if (cc) { - CHECK_EQ(&inputSidePackets, &cc->InputSidePackets()); + ABSL_CHECK_EQ(&inputSidePackets, &cc->InputSidePackets()); return [self initWithCalculatorContext:cc]; } // TODO: remove when we can. - LOG(WARNING) - << "CalculatorContext not available. If this calculator uses " - "CalculatorBase, call initWithCalculatorContext instead."; + ABSL_LOG(WARNING) << "CalculatorContext not available. If this calculator uses " + "CalculatorBase, call initWithCalculatorContext instead."; mediapipe::GpuSharedData* gpu_shared = inputSidePackets.Tag(mediapipe::kGpuSharedTagName).Get(); @@ -96,14 +97,13 @@ class MetalHelperLegacySupport { + (absl::Status)setupInputSidePackets:(mediapipe::PacketTypeSet*)inputSidePackets { auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContract(); if (cc) { - CHECK_EQ(inputSidePackets, &cc->InputSidePackets()); + ABSL_CHECK_EQ(inputSidePackets, &cc->InputSidePackets()); return [self updateContract:cc]; } // TODO: remove when we can. - LOG(WARNING) - << "CalculatorContract not available. If you're calling this " - "from a GetContract method, call updateContract instead."; + ABSL_LOG(WARNING) << "CalculatorContract not available. If you're calling this " + "from a GetContract method, call updateContract instead."; auto id = inputSidePackets->GetId(mediapipe::kGpuSharedTagName, 0); RET_CHECK(id.IsValid()) << "A " << mediapipe::kGpuSharedTagName @@ -180,7 +180,7 @@ class MetalHelperLegacySupport { NULL, _gpuResources->metal_shared().resources().mtlTextureCache, mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width, height, plane, &texture); - CHECK_EQ(err, kCVReturnSuccess); + ABSL_CHECK_EQ(err, kCVReturnSuccess); return texture; } diff --git a/mediapipe/gpu/cv_pixel_buffer_pool_wrapper.cc b/mediapipe/gpu/cv_pixel_buffer_pool_wrapper.cc index 6e077ae6..07ac7373 100644 --- a/mediapipe/gpu/cv_pixel_buffer_pool_wrapper.cc +++ b/mediapipe/gpu/cv_pixel_buffer_pool_wrapper.cc @@ -17,6 +17,7 @@ #include #include "CoreFoundation/CFBase.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/objc/CFHolder.h" #include "mediapipe/objc/util.h" @@ -27,7 +28,7 @@ CvPixelBufferPoolWrapper::CvPixelBufferPoolWrapper( int width, int height, GpuBufferFormat format, CFTimeInterval maxAge, CvTextureCacheManager* texture_caches) { OSType cv_format = CVPixelFormatForGpuBufferFormat(format); - CHECK_NE(cv_format, -1) << "unsupported pixel format"; + ABSL_CHECK_NE(cv_format, -1) << "unsupported pixel format"; pool_ = MakeCFHolderAdopting( /* keep count is 0 because the age param keeps buffers around anyway */ CreateCVPixelBufferPool(width, height, cv_format, 0, maxAge)); @@ -58,7 +59,7 @@ CFHolder CvPixelBufferPoolWrapper::GetBuffer() { ++threshold; } } - CHECK(!err) << "Error creating pixel buffer: " << err; + ABSL_CHECK(!err) << "Error creating pixel buffer: " << err; count_ = threshold; return MakeCFHolderAdopting(buffer); } @@ -73,11 +74,11 @@ void CvPixelBufferPoolWrapper::Flush() { CVPixelBufferPoolFlush(*pool_, 0); } CFHolder CvPixelBufferPoolWrapper::CreateBufferWithoutPool( const internal::GpuBufferSpec& spec) { OSType cv_format = CVPixelFormatForGpuBufferFormat(spec.format); - CHECK_NE(cv_format, -1) << "unsupported pixel format"; + ABSL_CHECK_NE(cv_format, -1) << "unsupported pixel format"; CVPixelBufferRef buffer; CVReturn err = CreateCVPixelBufferWithoutPool(spec.width, spec.height, cv_format, &buffer); - CHECK(!err) << "Error creating pixel buffer: " << err; + ABSL_CHECK(!err) << "Error creating pixel buffer: " << err; return MakeCFHolderAdopting(buffer); } diff --git a/mediapipe/gpu/cv_texture_cache_manager.cc b/mediapipe/gpu/cv_texture_cache_manager.cc index b977a899..0c4d2306 100644 --- a/mediapipe/gpu/cv_texture_cache_manager.cc +++ b/mediapipe/gpu/cv_texture_cache_manager.cc @@ -14,6 +14,7 @@ #include "mediapipe/gpu/cv_texture_cache_manager.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -32,8 +33,8 @@ void CvTextureCacheManager::FlushTextureCaches() { void CvTextureCacheManager::RegisterTextureCache(CVTextureCacheType cache) { absl::MutexLock lock(&mutex_); - CHECK(std::find(texture_caches_.begin(), texture_caches_.end(), cache) == - texture_caches_.end()) + ABSL_CHECK(std::find(texture_caches_.begin(), texture_caches_.end(), cache) == + texture_caches_.end()) << "Attempting to register a texture cache twice"; texture_caches_.emplace_back(cache); } @@ -42,13 +43,13 @@ void CvTextureCacheManager::UnregisterTextureCache(CVTextureCacheType cache) { absl::MutexLock lock(&mutex_); auto it = std::find(texture_caches_.begin(), texture_caches_.end(), cache); - CHECK(it != texture_caches_.end()) + ABSL_CHECK(it != texture_caches_.end()) << "Attempting to unregister an unknown texture cache"; texture_caches_.erase(it); } CvTextureCacheManager::~CvTextureCacheManager() { - CHECK_EQ(texture_caches_.size(), 0) + ABSL_CHECK_EQ(texture_caches_.size(), 0) << "Failed to unregister texture caches before deleting manager"; } diff --git a/mediapipe/gpu/gl_calculator_helper.cc b/mediapipe/gpu/gl_calculator_helper.cc index 974525a9..763ac387 100644 --- a/mediapipe/gpu/gl_calculator_helper.cc +++ b/mediapipe/gpu/gl_calculator_helper.cc @@ -14,6 +14,8 @@ #include "mediapipe/gpu/gl_calculator_helper.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/legacy_calculator_support.h" @@ -36,7 +38,7 @@ void GlCalculatorHelper::InitializeInternal(CalculatorContext* cc, } absl::Status GlCalculatorHelper::Open(CalculatorContext* cc) { - CHECK(cc); + ABSL_CHECK(cc); auto gpu_service = cc->Service(kGpuService); RET_CHECK(gpu_service.IsAvailable()) << "GPU service not available. Did you forget to call " @@ -71,12 +73,12 @@ absl::Status GlCalculatorHelper::SetupInputSidePackets( PacketTypeSet* input_side_packets) { auto cc = LegacyCalculatorSupport::Scoped::current(); if (cc) { - CHECK_EQ(input_side_packets, &cc->InputSidePackets()); + ABSL_CHECK_EQ(input_side_packets, &cc->InputSidePackets()); return UpdateContract(cc); } // TODO: remove when we can. - LOG(WARNING) + ABSL_LOG(WARNING) << "CalculatorContract not available. If you're calling this " "from a GetContract method, call GlCalculatorHelper::UpdateContract " "instead."; @@ -183,9 +185,9 @@ GpuBuffer GlCalculatorHelper::GpuBufferCopyingImageFrame( const ImageFrame& image_frame) { #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame); - // Converts absl::StatusOr to absl::Status since CHECK_OK() currently only - // deals with absl::Status in MediaPipe OSS. - CHECK_OK(maybe_buffer.status()); + // Converts absl::StatusOr to absl::Status since ABSL_CHECK_OK() currently + // only deals with absl::Status in MediaPipe OSS. + ABSL_CHECK_OK(maybe_buffer.status()); return GpuBuffer(std::move(maybe_buffer).value()); #else return GpuBuffer(GlTextureBuffer::Create(image_frame)); @@ -194,8 +196,8 @@ GpuBuffer GlCalculatorHelper::GpuBufferCopyingImageFrame( void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, int* width, int* height) { - CHECK(width); - CHECK(height); + ABSL_CHECK(width); + ABSL_CHECK(height); *width = pixel_buffer.width(); *height = pixel_buffer.height(); } @@ -219,6 +221,10 @@ GlTexture GlCalculatorHelper::CreateDestinationTexture( return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView(0)); } +GlTexture GlCalculatorHelper::CreateDestinationTexture(GpuBuffer& gpu_buffer) { + return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView(0)); +} + GlTexture GlCalculatorHelper::CreateSourceTexture( const mediapipe::Image& image) { return CreateSourceTexture(image.GetGpuBuffer()); diff --git a/mediapipe/gpu/gl_calculator_helper.h b/mediapipe/gpu/gl_calculator_helper.h index c1b94fa8..b6430860 100644 --- a/mediapipe/gpu/gl_calculator_helper.h +++ b/mediapipe/gpu/gl_calculator_helper.h @@ -162,6 +162,9 @@ class GlCalculatorHelper { int output_width, int output_height, GpuBufferFormat format = GpuBufferFormat::kBGRA32); + // Allows user provided buffers to be used as rendering destinations. + GlTexture CreateDestinationTexture(GpuBuffer& buffer); + // Creates a destination texture copying and uploading passed image frame. // // WARNING: mind that this functions creates a new texture every time and diff --git a/mediapipe/gpu/gl_context.cc b/mediapipe/gpu/gl_context.cc index d7381bab..5eff88b9 100644 --- a/mediapipe/gpu/gl_context.cc +++ b/mediapipe/gpu/gl_context.cc @@ -22,10 +22,11 @@ #include #include "absl/base/dynamic_annotations.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/synchronization/mutex.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" @@ -59,8 +60,8 @@ static void SetThreadName(const char* name) { thread_name[sizeof(thread_name) - 1] = '\0'; int res = pthread_setname_np(pthread_self(), thread_name); if (res != 0) { - LOG_FIRST_N(INFO, 1) << "Can't set pthread names: name: \"" << name - << "\"; error: " << res; + ABSL_LOG_FIRST_N(INFO, 1) + << "Can't set pthread names: name: \"" << name << "\"; error: " << res; } #elif __APPLE__ pthread_setname_np(name); @@ -69,17 +70,17 @@ static void SetThreadName(const char* name) { } GlContext::DedicatedThread::DedicatedThread() { - CHECK_EQ(pthread_create(&gl_thread_id_, nullptr, ThreadBody, this), 0); + ABSL_CHECK_EQ(pthread_create(&gl_thread_id_, nullptr, ThreadBody, this), 0); } GlContext::DedicatedThread::~DedicatedThread() { if (IsCurrentThread()) { - CHECK(self_destruct_); - CHECK_EQ(pthread_detach(gl_thread_id_), 0); + ABSL_CHECK(self_destruct_); + ABSL_CHECK_EQ(pthread_detach(gl_thread_id_), 0); } else { // Give an invalid job to signal termination. PutJob({}); - CHECK_EQ(pthread_join(gl_thread_id_, nullptr), 0); + ABSL_CHECK_EQ(pthread_join(gl_thread_id_, nullptr), 0); } } @@ -168,7 +169,7 @@ void GlContext::DedicatedThread::RunWithoutWaiting(GlVoidFunction gl_func) { // non-calculator tasks in the presence of GL source calculators, calculator // tasks must always be scheduled as new tasks, or another solution needs to // be set up to avoid starvation. See b/78522434. - CHECK(gl_func); + ABSL_CHECK(gl_func); PutJob(std::move(gl_func)); } @@ -236,9 +237,10 @@ absl::Status GlContext::GetGlExtensions() { // platforms to avoid possible undefined symbol or runtime errors. #if (GL_VERSION_3_0 || GL_ES_VERSION_3_0) && !defined(__EMSCRIPTEN__) if (!SymbolAvailable(&glGetStringi)) { - LOG(ERROR) << "GL major version > 3.0 indicated, but glGetStringi not " - << "defined. Falling back to deprecated GL extensions querying " - << "method."; + ABSL_LOG(ERROR) + << "GL major version > 3.0 indicated, but glGetStringi not " + << "defined. Falling back to deprecated GL extensions querying " + << "method."; return absl::InternalError("glGetStringi not defined, but queried"); } int num_extensions = 0; @@ -269,7 +271,7 @@ absl::Status GlContext::GetGlExtensionsCompat() { const GLubyte* res = glGetString(GL_EXTENSIONS); if (glGetError() != 0 || res == nullptr) { - LOG(ERROR) << "Error querying for GL extensions"; + ABSL_LOG(ERROR) << "Error querying for GL extensions"; return absl::InternalError("Error querying for GL extensions"); } const char* signed_res = reinterpret_cast(res); @@ -297,7 +299,7 @@ absl::Status GlContext::FinishInitialization(bool create_thread) { } else { // This may happen when using SwiftShader, but the numeric versions are // available and will be used instead. - LOG(WARNING) << "failed to get GL_VERSION string"; + ABSL_LOG(WARNING) << "failed to get GL_VERSION string"; } // We will decide later whether we want to use the version numbers we query @@ -315,8 +317,8 @@ absl::Status GlContext::FinishInitialization(bool create_thread) { // parse the version string. if (!ParseGlVersion(version_string, &gl_major_version_, &gl_minor_version_)) { - LOG(WARNING) << "invalid GL_VERSION format: '" << version_string - << "'; assuming 2.0"; + ABSL_LOG(WARNING) << "invalid GL_VERSION format: '" << version_string + << "'; assuming 2.0"; gl_major_version_ = 2; gl_minor_version_ = 0; } @@ -330,18 +332,18 @@ absl::Status GlContext::FinishInitialization(bool create_thread) { // for more details. if (gl_major_version_from_context_creation > 0 && gl_major_version_ != gl_major_version_from_context_creation) { - LOG(WARNING) << "Requested a context with major GL version " - << gl_major_version_from_context_creation - << " but context reports major version " << gl_major_version_ - << ". Setting to " << gl_major_version_from_context_creation - << ".0"; + ABSL_LOG(WARNING) << "Requested a context with major GL version " + << gl_major_version_from_context_creation + << " but context reports major version " + << gl_major_version_ << ". Setting to " + << gl_major_version_from_context_creation << ".0"; gl_major_version_ = gl_major_version_from_context_creation; gl_minor_version_ = 0; } - LOG(INFO) << "GL version: " << gl_major_version_ << "." << gl_minor_version_ - << " (" << version_string - << "), renderer: " << glGetString(GL_RENDERER); + ABSL_LOG(INFO) << "GL version: " << gl_major_version_ << "." + << gl_minor_version_ << " (" << version_string + << "), renderer: " << glGetString(GL_RENDERER); { auto status = GetGlExtensions(); @@ -389,7 +391,7 @@ GlContext::~GlContext() { clear_attachments(); return ExitContext(nullptr); }); - LOG_IF(ERROR, !status.ok()) + ABSL_LOG_IF(ERROR, !status.ok()) << "Failed to deactivate context on thread: " << status; if (thread_->IsCurrentThread()) { thread_.release()->SelfDestruct(); @@ -403,7 +405,7 @@ GlContext::~GlContext() { clear_attachments(); return absl::OkStatus(); }); - LOG_IF(ERROR, !status.ok()) << status; + ABSL_LOG_IF(ERROR, !status.ok()) << status; } } DestroyContext(); @@ -468,7 +470,7 @@ void GlContext::RunWithoutWaiting(GlVoidFunction gl_func) { return absl::OkStatus(); }); if (!status.ok()) { - LOG(ERROR) << "Error in RunWithoutWaiting: " << status; + ABSL_LOG(ERROR) << "Error in RunWithoutWaiting: " << status; } } } @@ -494,10 +496,10 @@ absl::Status GlContext::SwitchContext(ContextBinding* saved_context, } // Check that the context object is consistent with the native context. if (old_context_obj && saved_context) { - DCHECK(old_context_obj->context_ == saved_context->context); + ABSL_DCHECK(old_context_obj->context_ == saved_context->context); } if (new_context_obj) { - DCHECK(new_context_obj->context_ == new_context.context); + ABSL_DCHECK(new_context_obj->context_ == new_context.context); } if (new_context_obj && (old_context_obj == new_context_obj)) { @@ -537,7 +539,7 @@ GlContext::ContextBinding GlContext::ThisContextBinding() { } absl::Status GlContext::EnterContext(ContextBinding* saved_context) { - DCHECK(HasContext()); + ABSL_DCHECK(HasContext()); return SwitchContext(saved_context, ThisContextBinding()); } @@ -848,7 +850,7 @@ bool GlContext::IsAnyContextCurrent() { std::shared_ptr GlContext::CreateSyncTokenForCurrentExternalContext( const std::shared_ptr& delegate_graph_context) { - CHECK(delegate_graph_context); + ABSL_CHECK(delegate_graph_context); if (!IsAnyContextCurrent()) return nullptr; if (delegate_graph_context->ShouldUseFenceSync()) { return std::shared_ptr( @@ -899,7 +901,7 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) { // from the GlContext, and we must wait for gl_finish_count_ to pass it. // Therefore, we need to do at most one more glFinish call. This DCHECK // is used for documentation and sanity-checking purposes. - DCHECK(gl_finish_count_ >= count_to_pass); + ABSL_DCHECK(gl_finish_count_ >= count_to_pass); if (gl_finish_count_ == count_to_pass) { glFinish(); GlFinishCalled(); @@ -920,7 +922,7 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) { // it can signal the right condition variable if it is asked to do a // glFinish. absl::MutexLock other_lock(&other->mutex_); - DCHECK(!other->context_waiting_on_); + ABSL_DCHECK(!other->context_waiting_on_); other->context_waiting_on_ = this; } // We do not schedule this action using Run because we don't necessarily @@ -964,12 +966,12 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) { } void GlContext::WaitSyncToken(const std::shared_ptr& token) { - CHECK(token); + ABSL_CHECK(token); token->Wait(); } bool GlContext::SyncTokenIsReady(const std::shared_ptr& token) { - CHECK(token); + ABSL_CHECK(token); return token->IsReady(); } @@ -982,7 +984,7 @@ bool GlContext::CheckForGlErrors() { return CheckForGlErrors(false); } bool GlContext::CheckForGlErrors(bool force) { #if UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING if (!force) { - LOG_FIRST_N(WARNING, 1) << "OpenGL error checking is disabled"; + ABSL_LOG_FIRST_N(WARNING, 1) << "OpenGL error checking is disabled"; return false; } #endif @@ -994,23 +996,23 @@ bool GlContext::CheckForGlErrors(bool force) { had_error = true; switch (error) { case GL_INVALID_ENUM: - LOG(INFO) << "Found unchecked GL error: GL_INVALID_ENUM"; + ABSL_LOG(INFO) << "Found unchecked GL error: GL_INVALID_ENUM"; break; case GL_INVALID_VALUE: - LOG(INFO) << "Found unchecked GL error: GL_INVALID_VALUE"; + ABSL_LOG(INFO) << "Found unchecked GL error: GL_INVALID_VALUE"; break; case GL_INVALID_OPERATION: - LOG(INFO) << "Found unchecked GL error: GL_INVALID_OPERATION"; + ABSL_LOG(INFO) << "Found unchecked GL error: GL_INVALID_OPERATION"; break; case GL_INVALID_FRAMEBUFFER_OPERATION: - LOG(INFO) + ABSL_LOG(INFO) << "Found unchecked GL error: GL_INVALID_FRAMEBUFFER_OPERATION"; break; case GL_OUT_OF_MEMORY: - LOG(INFO) << "Found unchecked GL error: GL_OUT_OF_MEMORY"; + ABSL_LOG(INFO) << "Found unchecked GL error: GL_OUT_OF_MEMORY"; break; default: - LOG(INFO) << "Found unchecked GL error: UNKNOWN ERROR"; + ABSL_LOG(INFO) << "Found unchecked GL error: UNKNOWN ERROR"; break; } } @@ -1022,16 +1024,16 @@ void GlContext::LogUncheckedGlErrors(bool had_gl_errors) { // TODO: ideally we would print a backtrace here, or at least // the name of the current calculator, to make it easier to find the // culprit. In practice, getting a backtrace from Android without crashing - // is nearly impossible, so screw it. Just change this to LOG(FATAL) when - // you want to debug. - LOG(WARNING) << "Ignoring unchecked GL error."; + // is nearly impossible, so screw it. Just change this to ABSL_LOG(FATAL) + // when you want to debug. + ABSL_LOG(WARNING) << "Ignoring unchecked GL error."; } } const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, int plane) { std::shared_ptr ctx = GlContext::GetCurrent(); - CHECK(ctx != nullptr); + ABSL_CHECK(ctx != nullptr); return GlTextureInfoForGpuBufferFormat(format, plane, ctx->GetGlVersion()); } diff --git a/mediapipe/gpu/gl_context.h b/mediapipe/gpu/gl_context.h index fba0267a..bb3e6a59 100644 --- a/mediapipe/gpu/gl_context.h +++ b/mediapipe/gpu/gl_context.h @@ -22,6 +22,7 @@ #include #include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/executor.h" #include "mediapipe/framework/mediapipe_profiling.h" @@ -295,7 +296,7 @@ class GlContext : public std::enable_shared_from_this { // TOOD: const result? template T& GetCachedAttachment(const Attachment& attachment) { - DCHECK(IsCurrent()); + ABSL_DCHECK(IsCurrent()); internal::AttachmentPtr& entry = attachments_[&attachment]; if (entry == nullptr) { entry = attachment.factory()(*this); diff --git a/mediapipe/gpu/gl_context_eagl.cc b/mediapipe/gpu/gl_context_eagl.cc index 865813c2..5beb9d49 100644 --- a/mediapipe/gpu/gl_context_eagl.cc +++ b/mediapipe/gpu/gl_context_eagl.cc @@ -15,7 +15,6 @@ #include #include "absl/memory/memory.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" diff --git a/mediapipe/gpu/gl_context_egl.cc b/mediapipe/gpu/gl_context_egl.cc index f8784bbb..d573b697 100644 --- a/mediapipe/gpu/gl_context_egl.cc +++ b/mediapipe/gpu/gl_context_egl.cc @@ -14,10 +14,11 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/status/statusor.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" @@ -58,7 +59,7 @@ static void EglThreadExitCallback(void* key_value) { static void MakeEglReleaseThreadKey() { int err = pthread_key_create(&egl_release_thread_key, EglThreadExitCallback); if (err) { - LOG(ERROR) << "cannot create pthread key: " << err; + ABSL_LOG(ERROR) << "cannot create pthread key: " << err; } } @@ -81,8 +82,8 @@ static absl::StatusOr GetInitializedDefaultEglDisplay() { EGLint minor = 0; EGLBoolean egl_initialized = eglInitialize(display, &major, &minor); RET_CHECK(egl_initialized) << "Unable to initialize EGL"; - LOG(INFO) << "Successfully initialized EGL. Major : " << major - << " Minor: " << minor; + ABSL_LOG(INFO) << "Successfully initialized EGL. Major : " << major + << " Minor: " << minor; return display; } @@ -114,7 +115,7 @@ GlContext::StatusOrGlContext GlContext::Create(EGLContext share_context, absl::Status GlContext::CreateContextInternal(EGLContext share_context, int gl_version) { - CHECK(gl_version == 2 || gl_version == 3); + ABSL_CHECK(gl_version == 2 || gl_version == 3); const EGLint config_attr[] = { // clang-format off @@ -180,8 +181,9 @@ absl::Status GlContext::CreateContext(EGLContext share_context) { auto status = CreateContextInternal(share_context, 3); if (!status.ok()) { - LOG(WARNING) << "Creating a context with OpenGL ES 3 failed: " << status; - LOG(WARNING) << "Fall back on OpenGL ES 2."; + ABSL_LOG(WARNING) << "Creating a context with OpenGL ES 3 failed: " + << status; + ABSL_LOG(WARNING) << "Fall back on OpenGL ES 2."; status = CreateContextInternal(share_context, 2); } MP_RETURN_IF_ERROR(status); @@ -208,13 +210,13 @@ void GlContext::DestroyContext() { if (eglMakeCurrent(display_, surface_, surface_, context_)) { glUseProgram(0); } else { - LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase - << std::hex << eglGetError(); + ABSL_LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase + << std::hex << eglGetError(); } return SetCurrentContextBinding(saved_context); }; auto status = thread_ ? thread_->Run(detach_program) : detach_program(); - LOG_IF(ERROR, !status.ok()) << status; + ABSL_LOG_IF(ERROR, !status.ok()) << status; } #endif // __ANDROID__ @@ -236,21 +238,21 @@ void GlContext::DestroyContext() { if (IsCurrent()) { if (!eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE, EGL_NO_CONTEXT)) { - LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase - << std::hex << eglGetError(); + ABSL_LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase + << std::hex << eglGetError(); } } if (surface_ != EGL_NO_SURFACE) { if (!eglDestroySurface(display_, surface_)) { - LOG(ERROR) << "eglDestroySurface() returned error " << std::showbase - << std::hex << eglGetError(); + ABSL_LOG(ERROR) << "eglDestroySurface() returned error " << std::showbase + << std::hex << eglGetError(); } surface_ = EGL_NO_SURFACE; } if (context_ != EGL_NO_CONTEXT) { if (!eglDestroyContext(display_, context_)) { - LOG(ERROR) << "eglDestroyContext() returned error " << std::showbase - << std::hex << eglGetError(); + ABSL_LOG(ERROR) << "eglDestroyContext() returned error " << std::showbase + << std::hex << eglGetError(); } context_ = EGL_NO_CONTEXT; } diff --git a/mediapipe/gpu/gl_context_nsgl.cc b/mediapipe/gpu/gl_context_nsgl.cc index 561474ad..82d92a00 100644 --- a/mediapipe/gpu/gl_context_nsgl.cc +++ b/mediapipe/gpu/gl_context_nsgl.cc @@ -14,8 +14,8 @@ #include +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_builder.h" @@ -83,7 +83,7 @@ absl::Status GlContext::CreateContext(NSOpenGLContext* share_context) { if (!pixel_format_) { // On several Forge machines, the default config fails. For now let's do // this. - LOG(WARNING) + ABSL_LOG(WARNING) << "failed to create pixel format; trying without acceleration"; NSOpenGLPixelFormatAttribute attrs_no_accel[] = {NSOpenGLPFAColorSize, 24, @@ -102,7 +102,8 @@ absl::Status GlContext::CreateContext(NSOpenGLContext* share_context) { // Try to query pixel format from shared context. if (!context_) { - LOG(WARNING) << "Requested context not created, using queried context."; + ABSL_LOG(WARNING) + << "Requested context not created, using queried context."; CGLContextObj cgl_ctx = static_cast([share_context CGLContextObj]); CGLPixelFormatObj cgl_fmt = diff --git a/mediapipe/gpu/gl_context_webgl.cc b/mediapipe/gpu/gl_context_webgl.cc index 25cbed83..0f14581b 100644 --- a/mediapipe/gpu/gl_context_webgl.cc +++ b/mediapipe/gpu/gl_context_webgl.cc @@ -14,6 +14,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" @@ -48,7 +50,7 @@ GlContext::StatusOrGlContext GlContext::Create( absl::Status GlContext::CreateContextInternal( EMSCRIPTEN_WEBGL_CONTEXT_HANDLE external_context, int webgl_version) { - CHECK(webgl_version == 1 || webgl_version == 2); + ABSL_CHECK(webgl_version == 1 || webgl_version == 2); EmscriptenWebGLContextAttributes attrs; emscripten_webgl_init_context_attributes(&attrs); @@ -78,7 +80,7 @@ absl::Status GlContext::CreateContextInternal( // Check for failure if (context_handle <= 0) { - LOG(INFO) << "Couldn't create webGL " << webgl_version << " context."; + ABSL_LOG(INFO) << "Couldn't create webGL " << webgl_version << " context."; return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC) << "emscripten_webgl_create_context() returned error " << context_handle; @@ -103,32 +105,32 @@ absl::Status GlContext::CreateContext( auto status = CreateContextInternal(external_context, 2); if (!status.ok()) { - LOG(WARNING) << "Creating a context with WebGL 2 failed: " << status; - LOG(WARNING) << "Fall back on WebGL 1."; + ABSL_LOG(WARNING) << "Creating a context with WebGL 2 failed: " << status; + ABSL_LOG(WARNING) << "Fall back on WebGL 1."; status = CreateContextInternal(external_context, 1); } MP_RETURN_IF_ERROR(status); - LOG(INFO) << "Successfully created a WebGL context with major version " - << gl_major_version_ << " and handle " << context_; - + VLOG(1) << "Successfully created a WebGL context with major version " + << gl_major_version_ << " and handle " << context_; return absl::OkStatus(); } void GlContext::DestroyContext() { if (thread_) { // For now, we force web MediaPipe to be single-threaded, so error here. - LOG(ERROR) << "thread_ should not exist in DestroyContext() on web."; + ABSL_LOG(ERROR) << "thread_ should not exist in DestroyContext() on web."; } // Destroy the context and surface. if (context_ != 0) { EMSCRIPTEN_RESULT res = emscripten_webgl_destroy_context(context_); if (res != EMSCRIPTEN_RESULT_SUCCESS) { - LOG(ERROR) << "emscripten_webgl_destroy_context() returned error " << res; + ABSL_LOG(ERROR) << "emscripten_webgl_destroy_context() returned error " + << res; } else { - LOG(INFO) << "Successfully destroyed WebGL context with handle " - << context_; + ABSL_LOG(INFO) << "Successfully destroyed WebGL context with handle " + << context_; } context_ = 0; } diff --git a/mediapipe/gpu/gl_scaler_calculator.cc b/mediapipe/gpu/gl_scaler_calculator.cc index fa06c885..14540b52 100644 --- a/mediapipe/gpu/gl_scaler_calculator.cc +++ b/mediapipe/gpu/gl_scaler_calculator.cc @@ -104,6 +104,7 @@ class GlScalerCalculator : public CalculatorBase { bool vertical_flip_output_; bool horizontal_flip_output_; FrameScaleMode scale_mode_ = FrameScaleMode::kStretch; + bool use_nearest_neighbor_interpolation_ = false; }; REGISTER_CALCULATOR(GlScalerCalculator); @@ -186,7 +187,8 @@ absl::Status GlScalerCalculator::Open(CalculatorContext* cc) { scale_mode_ = FrameScaleModeFromProto(options.scale_mode(), FrameScaleMode::kStretch); } - + use_nearest_neighbor_interpolation_ = + options.use_nearest_neighbor_interpolation(); if (HasTagOrIndex(cc->InputSidePackets(), "OUTPUT_DIMENSIONS", 1)) { const auto& dimensions = TagOrIndex(cc->InputSidePackets(), "OUTPUT_DIMENSIONS", 1) @@ -297,6 +299,11 @@ absl::Status GlScalerCalculator::Process(CalculatorContext* cc) { glBindTexture(src2.target(), src2.name()); } + if (use_nearest_neighbor_interpolation_) { + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + } + MP_RETURN_IF_ERROR(renderer->GlRender( src1.width(), src1.height(), dst.width(), dst.height(), scale_mode_, rotation_, horizontal_flip_output_, vertical_flip_output_, diff --git a/mediapipe/gpu/gl_scaler_calculator.proto b/mediapipe/gpu/gl_scaler_calculator.proto index 99c0d439..f746a30f 100644 --- a/mediapipe/gpu/gl_scaler_calculator.proto +++ b/mediapipe/gpu/gl_scaler_calculator.proto @@ -19,7 +19,7 @@ package mediapipe; import "mediapipe/framework/calculator.proto"; import "mediapipe/gpu/scale_mode.proto"; -// Next id: 8. +// Next id: 9. message GlScalerCalculatorOptions { extend CalculatorOptions { optional GlScalerCalculatorOptions ext = 166373014; @@ -39,4 +39,7 @@ message GlScalerCalculatorOptions { // Flip the output texture horizontally. This is applied after rotation. optional bool flip_horizontal = 5; optional ScaleMode.Mode scale_mode = 6; + // Whether to use nearest neighbor interpolation. Default to use linear + // interpolation. + optional bool use_nearest_neighbor_interpolation = 8 [default = false]; } diff --git a/mediapipe/gpu/gl_surface_sink_calculator.cc b/mediapipe/gpu/gl_surface_sink_calculator.cc index ad867c2b..dbbf2526 100644 --- a/mediapipe/gpu/gl_surface_sink_calculator.cc +++ b/mediapipe/gpu/gl_surface_sink_calculator.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/calculator_framework.h" @@ -95,7 +96,7 @@ absl::Status GlSurfaceSinkCalculator::Process(CalculatorContext* cc) { absl::MutexLock lock(&surface_holder_->mutex); EGLSurface surface = surface_holder_->surface; if (surface == EGL_NO_SURFACE) { - LOG_EVERY_N(INFO, 300) << "GlSurfaceSinkCalculator: no surface"; + ABSL_LOG_EVERY_N(INFO, 300) << "GlSurfaceSinkCalculator: no surface"; return absl::OkStatus(); } diff --git a/mediapipe/gpu/gl_texture_buffer.cc b/mediapipe/gpu/gl_texture_buffer.cc index 4e5ce4ee..0ea511c5 100644 --- a/mediapipe/gpu/gl_texture_buffer.cc +++ b/mediapipe/gpu/gl_texture_buffer.cc @@ -14,6 +14,8 @@ #include "mediapipe/gpu/gl_texture_buffer.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/gpu/gl_context.h" #include "mediapipe/gpu/gl_texture_view.h" @@ -47,7 +49,7 @@ std::unique_ptr GlTextureBuffer::Create(int width, int height, auto buf = absl::make_unique(GL_TEXTURE_2D, 0, width, height, format, nullptr); if (!buf->CreateInternal(data, alignment)) { - LOG(WARNING) << "Failed to create a GL texture"; + ABSL_LOG(WARNING) << "Failed to create a GL texture"; return nullptr; } return buf; @@ -108,7 +110,7 @@ GlTextureBuffer::GlTextureBuffer(GLenum target, GLuint name, int width, bool GlTextureBuffer::CreateInternal(const void* data, int alignment) { auto context = GlContext::GetCurrent(); if (!context) { - LOG(WARNING) << "Cannot create a GL texture without a valid context"; + ABSL_LOG(WARNING) << "Cannot create a GL texture without a valid context"; return false; } @@ -127,7 +129,7 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) { if (info.gl_internal_format == GL_RGBA16F && context->GetGlVersion() != GlVersion::kGLES2 && SymbolAvailable(&glTexStorage2D)) { - CHECK(data == nullptr) << "unimplemented"; + ABSL_CHECK(data == nullptr) << "unimplemented"; glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); } else { glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_, @@ -149,10 +151,10 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) { // Use the deletion callback to delete the texture on the context // that created it. - CHECK(!deletion_callback_); + ABSL_CHECK(!deletion_callback_); deletion_callback_ = [this, context](std::shared_ptr sync_token) { - CHECK_NE(name_, 0); + ABSL_CHECK_NE(name_, 0); GLuint name_to_delete = name_; context->RunWithoutWaiting([name_to_delete]() { // Note that we do not wait for consumers to be done before deleting the @@ -200,9 +202,9 @@ void GlTextureBuffer::Reuse() { } void GlTextureBuffer::Updated(std::shared_ptr prod_token) { - CHECK(!producer_sync_) + ABSL_CHECK(!producer_sync_) << "Updated existing texture which had not been marked for reuse!"; - CHECK(prod_token); + ABSL_CHECK(prod_token); producer_sync_ = std::move(prod_token); const auto& synced_context = producer_sync_->GetContext(); if (synced_context) { @@ -216,7 +218,7 @@ void GlTextureBuffer::DidRead(std::shared_ptr cons_token) const { consumer_multi_sync_->Add(std::move(cons_token)); } else { // TODO: change to a CHECK. - LOG_FIRST_N(WARNING, 5) << "unexpected null sync in DidRead"; + ABSL_LOG_FIRST_N(WARNING, 5) << "unexpected null sync in DidRead"; } } @@ -263,11 +265,11 @@ void GlTextureBuffer::WaitForConsumersOnGpu() { GlTextureView GlTextureBuffer::GetReadView(internal::types, int plane) const { auto gl_context = GlContext::GetCurrent(); - CHECK(gl_context); - CHECK_EQ(plane, 0); + ABSL_CHECK(gl_context); + ABSL_CHECK_EQ(plane, 0); // Note that this method is only supposed to be called by GpuBuffer, which // ensures this condition is satisfied. - DCHECK(!weak_from_this().expired()) + ABSL_DCHECK(!weak_from_this().expired()) << "GlTextureBuffer must be held in shared_ptr to get a GlTextureView"; // Insert wait call to sync with the producer. WaitOnGpu(); @@ -284,11 +286,11 @@ GlTextureView GlTextureBuffer::GetReadView(internal::types, GlTextureView GlTextureBuffer::GetWriteView(internal::types, int plane) { auto gl_context = GlContext::GetCurrent(); - CHECK(gl_context); - CHECK_EQ(plane, 0); + ABSL_CHECK(gl_context); + ABSL_CHECK_EQ(plane, 0); // Note that this method is only supposed to be called by GpuBuffer, which // ensures this condition is satisfied. - DCHECK(!weak_from_this().expired()) + ABSL_DCHECK(!weak_from_this().expired()) << "GlTextureBuffer must be held in shared_ptr to get a GlTextureView"; // Insert wait call to sync with the producer. WaitOnGpu(); @@ -345,7 +347,7 @@ static void ReadTexture(GlContext& ctx, const GlTextureView& view, // won't overflow the buffer with glReadPixels, we'd also need to check or // reset several glPixelStore parameters (e.g. what if someone had the // ill-advised idea of setting GL_PACK_SKIP_PIXELS?). - CHECK(view.gl_context()); + ABSL_CHECK(view.gl_context()); GlTextureInfo info = GlTextureInfoForGpuBufferFormat( format, view.plane(), view.gl_context()->GetGlVersion()); diff --git a/mediapipe/gpu/gpu_buffer.cc b/mediapipe/gpu/gpu_buffer.cc index 628e8609..0eb7a1c5 100644 --- a/mediapipe/gpu/gpu_buffer.cc +++ b/mediapipe/gpu/gpu_buffer.cc @@ -4,6 +4,7 @@ #include #include "absl/functional/bind_front.h" +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "mediapipe/framework/port/logging.h" @@ -127,10 +128,11 @@ internal::GpuBufferStorage& GpuBuffer::GetStorageForViewOrDie( TypeId view_provider_type, bool for_writing) const { auto* chosen_storage = GpuBuffer::GetStorageForView(view_provider_type, for_writing); - CHECK(chosen_storage) << "no view provider found for requested view " - << view_provider_type.name() << "; storages available: " - << (holder_ ? holder_->DebugString() : "invalid"); - DCHECK(chosen_storage->can_down_cast_to(view_provider_type)); + ABSL_CHECK(chosen_storage) + << "no view provider found for requested view " + << view_provider_type.name() << "; storages available: " + << (holder_ ? holder_->DebugString() : "invalid"); + ABSL_DCHECK(chosen_storage->can_down_cast_to(view_provider_type)); return *chosen_storage; } diff --git a/mediapipe/gpu/gpu_buffer.h b/mediapipe/gpu/gpu_buffer.h index 93eb1460..20cc05ea 100644 --- a/mediapipe/gpu/gpu_buffer.h +++ b/mediapipe/gpu/gpu_buffer.h @@ -20,7 +20,7 @@ #include #include -#include "absl/log/check.h" +#include "absl/log/absl_check.h" #include "absl/synchronization/mutex.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/gpu/gpu_buffer_format.h" @@ -74,7 +74,7 @@ class GpuBuffer { // GpuBuffers in a portable way from the framework, e.g. using // GpuBufferMultiPool. explicit GpuBuffer(std::shared_ptr storage) { - CHECK(storage) << "Cannot construct GpuBuffer with null storage"; + ABSL_CHECK(storage) << "Cannot construct GpuBuffer with null storage"; holder_ = std::make_shared(std::move(storage)); } diff --git a/mediapipe/gpu/gpu_buffer_format.cc b/mediapipe/gpu/gpu_buffer_format.cc index 00ee9e24..646fb383 100644 --- a/mediapipe/gpu/gpu_buffer_format.cc +++ b/mediapipe/gpu/gpu_buffer_format.cc @@ -15,6 +15,7 @@ #include "mediapipe/gpu/gpu_buffer_format.h" #include "absl/container/flat_hash_map.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/deps/no_destructor.h" #include "mediapipe/framework/port/logging.h" @@ -100,6 +101,10 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, {GL_R8, GL_RED, GL_UNSIGNED_BYTE, 1}, #endif // TARGET_OS_OSX }}, + {GpuBufferFormat::kOneComponent8Alpha, + { + {GL_ALPHA, GL_ALPHA, GL_UNSIGNED_BYTE, 1}, + }}, {GpuBufferFormat::kOneComponent8Red, { {GL_R8, GL_RED, GL_UNSIGNED_BYTE, 1}, @@ -185,16 +190,16 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, } auto iter = format_info->find(format); - CHECK(iter != format_info->end()) + ABSL_CHECK(iter != format_info->end()) << "unsupported format: " << static_cast>(format); const auto& planes = iter->second; #ifndef __APPLE__ - CHECK_EQ(planes.size(), 1) + ABSL_CHECK_EQ(planes.size(), 1) << "multiplanar formats are not supported on this platform"; #endif - CHECK_GE(plane, 0) << "invalid plane number"; - CHECK_LT(plane, planes.size()) << "invalid plane number"; + ABSL_CHECK_GE(plane, 0) << "invalid plane number"; + ABSL_CHECK_LT(plane, planes.size()) << "invalid plane number"; return planes[plane]; } #endif // MEDIAPIPE_DISABLE_GPU @@ -221,6 +226,7 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { case GpuBufferFormat::kRGBA32: // TODO: this likely maps to ImageFormat::SRGBA case GpuBufferFormat::kGrayHalf16: + case GpuBufferFormat::kOneComponent8Alpha: case GpuBufferFormat::kOneComponent8Red: case GpuBufferFormat::kTwoComponent8: case GpuBufferFormat::kTwoComponentHalf16: diff --git a/mediapipe/gpu/gpu_buffer_format.h b/mediapipe/gpu/gpu_buffer_format.h index 5d77afeb..06eabda7 100644 --- a/mediapipe/gpu/gpu_buffer_format.h +++ b/mediapipe/gpu/gpu_buffer_format.h @@ -43,6 +43,7 @@ enum class GpuBufferFormat : uint32_t { kGrayFloat32 = MEDIAPIPE_FOURCC('L', '0', '0', 'f'), kGrayHalf16 = MEDIAPIPE_FOURCC('L', '0', '0', 'h'), kOneComponent8 = MEDIAPIPE_FOURCC('L', '0', '0', '8'), + kOneComponent8Alpha = MEDIAPIPE_FOURCC('A', '0', '0', '8'), kOneComponent8Red = MEDIAPIPE_FOURCC('R', '0', '0', '8'), kTwoComponent8 = MEDIAPIPE_FOURCC('2', 'C', '0', '8'), kTwoComponentHalf16 = MEDIAPIPE_FOURCC('2', 'C', '0', 'h'), @@ -101,6 +102,7 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) { return kCVPixelFormatType_OneComponent32Float; case GpuBufferFormat::kOneComponent8: return kCVPixelFormatType_OneComponent8; + case GpuBufferFormat::kOneComponent8Alpha: case GpuBufferFormat::kOneComponent8Red: return -1; case GpuBufferFormat::kTwoComponent8: diff --git a/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc b/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc index 7cac32b7..ba048351 100644 --- a/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc +++ b/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc @@ -2,6 +2,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/gpu/gl_context.h" #include "mediapipe/gpu/gpu_buffer_storage_image_frame.h" #include "mediapipe/objc/util.h" @@ -17,11 +19,11 @@ typedef CVOpenGLESTextureRef CVTextureType; GpuBufferStorageCvPixelBuffer::GpuBufferStorageCvPixelBuffer( int width, int height, GpuBufferFormat format) { OSType cv_format = CVPixelFormatForGpuBufferFormat(format); - CHECK_NE(cv_format, -1) << "unsupported pixel format"; + ABSL_CHECK_NE(cv_format, -1) << "unsupported pixel format"; CVPixelBufferRef buffer; CVReturn err = CreateCVPixelBufferWithoutPool(width, height, cv_format, &buffer); - CHECK(!err) << "Error creating pixel buffer: " << err; + ABSL_CHECK(!err) << "Error creating pixel buffer: " << err; adopt(buffer); } @@ -29,13 +31,13 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetTexture( int plane, GlTextureView::DoneWritingFn done_writing) const { CVReturn err; auto gl_context = GlContext::GetCurrent(); - CHECK(gl_context); + ABSL_CHECK(gl_context); #if TARGET_OS_OSX CVTextureType cv_texture_temp; err = CVOpenGLTextureCacheCreateTextureFromImage( kCFAllocatorDefault, gl_context->cv_texture_cache(), **this, NULL, &cv_texture_temp); - CHECK(cv_texture_temp && !err) + ABSL_CHECK(cv_texture_temp && !err) << "CVOpenGLTextureCacheCreateTextureFromImage failed: " << err; CFHolder cv_texture; cv_texture.adopt(cv_texture_temp); @@ -53,7 +55,7 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetTexture( GL_TEXTURE_2D, info.gl_internal_format, width() / info.downscale, height() / info.downscale, info.gl_format, info.gl_type, plane, &cv_texture_temp); - CHECK(cv_texture_temp && !err) + ABSL_CHECK(cv_texture_temp && !err) << "CVOpenGLESTextureCacheCreateTextureFromImage failed: " << err; CFHolder cv_texture; cv_texture.adopt(cv_texture_temp); @@ -73,12 +75,12 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetReadView( #if TARGET_IPHONE_SIMULATOR static void ViewDoneWritingSimulatorWorkaround(CVPixelBufferRef pixel_buffer, const GlTextureView& view) { - CHECK(pixel_buffer); + ABSL_CHECK(pixel_buffer); auto ctx = GlContext::GetCurrent().get(); if (!ctx) ctx = view.gl_context(); ctx->Run([pixel_buffer, &view, ctx] { CVReturn err = CVPixelBufferLockBaseAddress(pixel_buffer, 0); - CHECK(err == kCVReturnSuccess) + ABSL_CHECK(err == kCVReturnSuccess) << "CVPixelBufferLockBaseAddress failed: " << err; OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); size_t bytes_per_row = CVPixelBufferGetBytesPerRow(pixel_buffer); @@ -113,10 +115,10 @@ static void ViewDoneWritingSimulatorWorkaround(CVPixelBufferRef pixel_buffer, view.target(), 0, 0); glBindFramebuffer(GL_FRAMEBUFFER, 0); } else { - LOG(ERROR) << "unsupported pixel format: " << pixel_format; + ABSL_LOG(ERROR) << "unsupported pixel format: " << pixel_format; } err = CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); - CHECK(err == kCVReturnSuccess) + ABSL_CHECK(err == kCVReturnSuccess) << "CVPixelBufferUnlockBaseAddress failed: " << err; }); } @@ -149,7 +151,7 @@ static std::shared_ptr ConvertFromImageFrame( std::shared_ptr frame) { auto status_or_buffer = CreateCVPixelBufferForImageFrame(frame->image_frame()); - CHECK(status_or_buffer.ok()); + ABSL_CHECK(status_or_buffer.ok()); return std::make_shared( std::move(status_or_buffer).value()); } diff --git a/mediapipe/gpu/gpu_buffer_storage_image_frame.cc b/mediapipe/gpu/gpu_buffer_storage_image_frame.cc index 316c6cc4..7f46e297 100644 --- a/mediapipe/gpu/gpu_buffer_storage_image_frame.cc +++ b/mediapipe/gpu/gpu_buffer_storage_image_frame.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/formats/frame_buffer.h" #include "mediapipe/framework/formats/image_frame.h" @@ -43,7 +44,7 @@ std::shared_ptr ImageFrameToFrameBuffer( std::shared_ptr image_frame) { FrameBuffer::Format format = FrameBufferFormatForImageFrameFormat(image_frame->Format()); - CHECK(format != FrameBuffer::Format::kUNKNOWN) + ABSL_CHECK(format != FrameBuffer::Format::kUNKNOWN) << "Invalid format. Only SRGB, SRGBA and GRAY8 are supported."; const FrameBuffer::Dimension dimension{/*width=*/image_frame->Width(), /*height=*/image_frame->Height()}; diff --git a/mediapipe/gpu/gpu_buffer_storage_yuv_image.cc b/mediapipe/gpu/gpu_buffer_storage_yuv_image.cc index 41905de7..87fb8957 100644 --- a/mediapipe/gpu/gpu_buffer_storage_yuv_image.cc +++ b/mediapipe/gpu/gpu_buffer_storage_yuv_image.cc @@ -19,8 +19,8 @@ limitations under the License. #include #include -#include "absl/log/check.h" -#include "absl/log/log.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "libyuv/video_common.h" #include "mediapipe/framework/formats/frame_buffer.h" #include "mediapipe/framework/formats/image_frame.h" @@ -87,7 +87,7 @@ std::shared_ptr YuvImageToFrameBuffer( FrameBuffer::Dimension dimension{/*width=*/yuv_image->width(), /*height=*/yuv_image->height()}; std::vector planes; - CHECK(yuv_image->mutable_data(0) != nullptr && yuv_image->stride(0) > 0) + ABSL_CHECK(yuv_image->mutable_data(0) != nullptr && yuv_image->stride(0) > 0) << "Invalid YuvImage. Expected plane at index 0 to be non-null and have " "stride > 0."; planes.emplace_back( @@ -97,7 +97,8 @@ std::shared_ptr YuvImageToFrameBuffer( switch (format) { case FrameBuffer::Format::kNV12: case FrameBuffer::Format::kNV21: { - CHECK(yuv_image->mutable_data(1) != nullptr && yuv_image->stride(1) > 0) + ABSL_CHECK(yuv_image->mutable_data(1) != nullptr && + yuv_image->stride(1) > 0) << "Invalid YuvImage. Expected plane at index 1 to be non-null and " "have stride > 0."; planes.emplace_back( @@ -108,8 +109,9 @@ std::shared_ptr YuvImageToFrameBuffer( } case FrameBuffer::Format::kYV12: case FrameBuffer::Format::kYV21: { - CHECK(yuv_image->mutable_data(1) != nullptr && yuv_image->stride(1) > 0 && - yuv_image->mutable_data(2) != nullptr && yuv_image->stride(2) > 0) + ABSL_CHECK( + yuv_image->mutable_data(1) != nullptr && yuv_image->stride(1) > 0 && + yuv_image->mutable_data(2) != nullptr && yuv_image->stride(2) > 0) << "Invalid YuvImage. Expected planes at indices 1 and 2 to be " "non-null and have stride > 0."; planes.emplace_back( @@ -123,7 +125,7 @@ std::shared_ptr YuvImageToFrameBuffer( break; } default: - LOG(FATAL) + ABSL_LOG(FATAL) << "Invalid format. Only FOURCC_NV12, FOURCC_NV21, FOURCC_YV12 and " "FOURCC_I420 are supported."; } @@ -148,7 +150,7 @@ std::shared_ptr YuvImageToImageFrame( auto rgb_buffer = FrameBuffer(planes, yuv_buffer->dimension(), FrameBuffer::Format::kRGB); // Convert. - CHECK_OK(frame_buffer::Convert(*yuv_buffer, &rgb_buffer)); + ABSL_CHECK_OK(frame_buffer::Convert(*yuv_buffer, &rgb_buffer)); return image_frame; } @@ -156,8 +158,8 @@ std::shared_ptr YuvImageToImageFrame( GpuBufferStorageYuvImage::GpuBufferStorageYuvImage( std::shared_ptr yuv_image) { - CHECK(GpuBufferFormatForFourCC(yuv_image->fourcc()) != - GpuBufferFormat::kUnknown) + ABSL_CHECK(GpuBufferFormatForFourCC(yuv_image->fourcc()) != + GpuBufferFormat::kUnknown) << "Invalid format. Only FOURCC_NV12, FOURCC_NV21, FOURCC_YV12 and " "FOURCC_I420 are supported."; yuv_image_ = yuv_image; @@ -195,7 +197,7 @@ GpuBufferStorageYuvImage::GpuBufferStorageYuvImage(int width, int height, break; } default: - LOG(FATAL) + ABSL_LOG(FATAL) << "Invalid format. Only kNV12, kNV21, kYV12 and kYV21 are supported"; } } @@ -223,6 +225,6 @@ std::shared_ptr GpuBufferStorageYuvImage::GetWriteView( internal::types) { // Not supported on purpose: writes into the resulting ImageFrame cannot // easily be ported back to the original YUV image. - LOG(FATAL) << "GetWriteView is not supported."; + ABSL_LOG(FATAL) << "GetWriteView is not supported."; } } // namespace mediapipe diff --git a/mediapipe/gpu/gpu_shared_data_internal.cc b/mediapipe/gpu/gpu_shared_data_internal.cc index 1098c82e..b9b9c26f 100644 --- a/mediapipe/gpu/gpu_shared_data_internal.cc +++ b/mediapipe/gpu/gpu_shared_data_internal.cc @@ -15,6 +15,7 @@ #include "mediapipe/gpu/gpu_shared_data_internal.h" #include "absl/base/attributes.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/deps/no_destructor.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/gpu/gl_context.h" @@ -120,7 +121,7 @@ GpuResources::~GpuResources() { ABSL_CONST_INIT extern const GraphService kGpuService; absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) { - CHECK(node->Contract().ServiceRequests().contains(kGpuService.key)); + ABSL_CHECK(node->Contract().ServiceRequests().contains(kGpuService.key)); std::string node_id = node->GetCalculatorState().NodeName(); std::string node_type = node->GetCalculatorState().CalculatorType(); std::string context_key; diff --git a/mediapipe/gpu/shader_util.cc b/mediapipe/gpu/shader_util.cc index 5de7e24f..3e3eb462 100644 --- a/mediapipe/gpu/shader_util.cc +++ b/mediapipe/gpu/shader_util.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/logging.h" #if DEBUG @@ -26,7 +27,7 @@ if (log_length > 0) { \ GLchar* log = static_cast(malloc(log_length)); \ glGet##type##InfoLog(object, log_length, &log_length, log); \ - LOG(INFO) << #type " " action " log:\n" << log; \ + ABSL_LOG(INFO) << #type " " action " log:\n" << log; \ free(log); \ } \ } while (0) @@ -41,7 +42,7 @@ if (log_length > 0) { \ GLchar* log = static_cast(malloc(log_length)); \ glGet##type##InfoLog(object, log_length, &log_length, log); \ - LOG(ERROR) << #type " " action " log:\n" << log; \ + ABSL_LOG(ERROR) << #type " " action " log:\n" << log; \ free(log); \ } \ } while (0) @@ -70,13 +71,14 @@ GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader, GLint status; glGetShaderiv(*shader, GL_COMPILE_STATUS, &status); - LOG_IF(ERROR, status == GL_FALSE) << "Failed to compile shader:\n" << source; + ABSL_LOG_IF(ERROR, status == GL_FALSE) << "Failed to compile shader:\n" + << source; if (status == GL_FALSE) { int length = 0; GLchar cmessage[kMaxShaderInfoLength]; glGetShaderInfoLog(*shader, kMaxShaderInfoLength, &length, cmessage); - LOG(ERROR) << "Error message: " << std::string(cmessage, length); + ABSL_LOG(ERROR) << "Error message: " << std::string(cmessage, length); } return status; } @@ -95,7 +97,8 @@ GLint GlhLinkProgram(GLuint program, bool force_log_errors) { GL_DEBUG_LOG(Program, program, "link"); glGetProgramiv(program, GL_LINK_STATUS, &status); - LOG_IF(ERROR, status == GL_FALSE) << "Failed to link program " << program; + ABSL_LOG_IF(ERROR, status == GL_FALSE) + << "Failed to link program " << program; return status; } @@ -108,7 +111,8 @@ GLint GlhValidateProgram(GLuint program) { GL_DEBUG_LOG(Program, program, "validate"); glGetProgramiv(program, GL_VALIDATE_STATUS, &status); - LOG_IF(ERROR, status == GL_FALSE) << "Failed to validate program " << program; + ABSL_LOG_IF(ERROR, status == GL_FALSE) + << "Failed to validate program " << program; return status; } diff --git a/mediapipe/graphs/instant_motion_tracking/calculators/BUILD b/mediapipe/graphs/instant_motion_tracking/calculators/BUILD index 93af68c2..cdfd911d 100644 --- a/mediapipe/graphs/instant_motion_tracking/calculators/BUILD +++ b/mediapipe/graphs/instant_motion_tracking/calculators/BUILD @@ -63,6 +63,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/graphs/object_detection_3d/calculators:model_matrix_cc_proto", "//mediapipe/modules/objectron/calculators:box", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@eigen_archive//:eigen3", diff --git a/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc b/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc index c003135b..a73589a8 100644 --- a/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc +++ b/mediapipe/graphs/instant_motion_tracking/calculators/matrices_manager_calculator.cc @@ -18,6 +18,7 @@ #include "Eigen/Core" #include "Eigen/Dense" #include "Eigen/Geometry" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -116,8 +117,8 @@ class MatricesManagerCalculator : public CalculatorBase { return user_scaling.scale_factor; } } - LOG(WARNING) << "Cannot find sticker_id: " << sticker_id - << ", returning 1.0f scaling"; + ABSL_LOG(WARNING) << "Cannot find sticker_id: " << sticker_id + << ", returning 1.0f scaling"; return 1.0f; } @@ -129,8 +130,8 @@ class MatricesManagerCalculator : public CalculatorBase { return rotation.rotation_radians; } } - LOG(WARNING) << "Cannot find sticker_id: " << sticker_id - << ", returning 0.0f rotation"; + ABSL_LOG(WARNING) << "Cannot find sticker_id: " << sticker_id + << ", returning 0.0f rotation"; return 0.0f; } }; @@ -221,8 +222,9 @@ absl::Status MatricesManagerCalculator::Process(CalculatorContext* cc) { model_matrix = asset_matrices_gif->add_model_matrix(); } else { // Asset 3D if (render_data[render_idx] != 1) { - LOG(ERROR) << "render id: " << render_data[render_idx] - << " is not supported. Fall back to using render_id = 1."; + ABSL_LOG(ERROR) + << "render id: " << render_data[render_idx] + << " is not supported. Fall back to using render_id = 1."; } model_matrix = asset_matrices_1->add_model_matrix(); } @@ -379,8 +381,8 @@ DiagonalMatrix3f MatricesManagerCalculator::GetDefaultRenderScaleDiagonal( break; } default: { - LOG(INFO) << "Unsupported render_id: " << render_id - << ", returning default render_scale"; + ABSL_LOG(INFO) << "Unsupported render_id: " << render_id + << ", returning default render_scale"; break; } } diff --git a/mediapipe/graphs/object_detection_3d/calculators/BUILD b/mediapipe/graphs/object_detection_3d/calculators/BUILD index d4c5c496..c491baf2 100644 --- a/mediapipe/graphs/object_detection_3d/calculators/BUILD +++ b/mediapipe/graphs/object_detection_3d/calculators/BUILD @@ -74,6 +74,8 @@ cc_library( "//mediapipe/gpu:shader_util", "//mediapipe/modules/objectron/calculators:camera_parameters_cc_proto", "//mediapipe/util/android:asset_manager_util", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], alwayslink = 1, ) diff --git a/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc b/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc index a92020ff..5dee74a2 100644 --- a/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc +++ b/mediapipe/graphs/object_detection_3d/calculators/gl_animation_overlay_calculator.cc @@ -19,6 +19,8 @@ #include #endif +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" @@ -35,7 +37,7 @@ namespace { #if defined(GL_DEBUG) #define GLCHECK(command) \ command; \ - if (int err = glGetError()) LOG(ERROR) << "GL error detected: " << err; + if (int err = glGetError()) ABSL_LOG(ERROR) << "GL error detected: " << err; #else #define GLCHECK(command) command #endif @@ -355,12 +357,13 @@ bool GlAnimationOverlayCalculator::ReadBytesFromAsset(AAsset *asset, } // At least log any I/O errors encountered. if (bytes_read < 0) { - LOG(ERROR) << "Error reading from AAsset: " << bytes_read; + ABSL_LOG(ERROR) << "Error reading from AAsset: " << bytes_read; return false; } if (bytes_left > 0) { // Reached EOF before reading in specified number of bytes. - LOG(WARNING) << "Reached EOF before reading in specified number of bytes."; + ABSL_LOG(WARNING) + << "Reached EOF before reading in specified number of bytes."; return false; } return true; @@ -374,7 +377,7 @@ bool GlAnimationOverlayCalculator::LoadAnimationAndroid( Singleton::get(); AAssetManager *asset_manager = mediapipe_asset_manager->GetAssetManager(); if (!asset_manager) { - LOG(ERROR) << "Failed to access Android asset manager."; + ABSL_LOG(ERROR) << "Failed to access Android asset manager."; return false; } @@ -382,7 +385,7 @@ bool GlAnimationOverlayCalculator::LoadAnimationAndroid( AAsset *asset = AAssetManager_open(asset_manager, filename.c_str(), AASSET_MODE_STREAMING); if (!asset) { - LOG(ERROR) << "Failed to open animation asset: " << filename; + ABSL_LOG(ERROR) << "Failed to open animation asset: " << filename; return false; } @@ -400,14 +403,14 @@ bool GlAnimationOverlayCalculator::LoadAnimationAndroid( triangle_mesh.vertices.reset(new float[lengths[0]]); if (!ReadBytesFromAsset(asset, (void *)triangle_mesh.vertices.get(), sizeof(float) * lengths[0])) { - LOG(ERROR) << "Failed to read vertices for frame " << frame_count_; + ABSL_LOG(ERROR) << "Failed to read vertices for frame " << frame_count_; return false; } // Try to read in texture coordinates (4-byte floats) triangle_mesh.texture_coords.reset(new float[lengths[1]]); if (!ReadBytesFromAsset(asset, (void *)triangle_mesh.texture_coords.get(), sizeof(float) * lengths[1])) { - LOG(ERROR) << "Failed to read tex-coords for frame " << frame_count_; + ABSL_LOG(ERROR) << "Failed to read tex-coords for frame " << frame_count_; return false; } // Try to read in indices (2-byte shorts) @@ -415,7 +418,7 @@ bool GlAnimationOverlayCalculator::LoadAnimationAndroid( triangle_mesh.triangle_indices.reset(new int16[lengths[2]]); if (!ReadBytesFromAsset(asset, (void *)triangle_mesh.triangle_indices.get(), sizeof(int16) * lengths[2])) { - LOG(ERROR) << "Failed to read indices for frame " << frame_count_; + ABSL_LOG(ERROR) << "Failed to read indices for frame " << frame_count_; return false; } @@ -426,9 +429,10 @@ bool GlAnimationOverlayCalculator::LoadAnimationAndroid( } AAsset_close(asset); - LOG(INFO) << "Finished parsing " << frame_count_ << " animation frames."; + ABSL_LOG(INFO) << "Finished parsing " << frame_count_ << " animation frames."; if (meshes->empty()) { - LOG(ERROR) << "No animation frames were parsed! Erroring out calculator."; + ABSL_LOG(ERROR) + << "No animation frames were parsed! Erroring out calculator."; return false; } return true; @@ -439,7 +443,7 @@ bool GlAnimationOverlayCalculator::LoadAnimationAndroid( bool GlAnimationOverlayCalculator::LoadAnimation(const std::string &filename) { std::ifstream infile(filename.c_str(), std::ifstream::binary); if (!infile) { - LOG(ERROR) << "Error opening asset with filename: " << filename; + ABSL_LOG(ERROR) << "Error opening asset with filename: " << filename; return false; } @@ -462,7 +466,7 @@ bool GlAnimationOverlayCalculator::LoadAnimation(const std::string &filename) { infile.read((char *)(triangle_mesh.vertices.get()), sizeof(float) * lengths[0]); if (!infile) { - LOG(ERROR) << "Failed to read vertices for frame " << frame_count_; + ABSL_LOG(ERROR) << "Failed to read vertices for frame " << frame_count_; return false; } @@ -471,8 +475,8 @@ bool GlAnimationOverlayCalculator::LoadAnimation(const std::string &filename) { infile.read((char *)(triangle_mesh.texture_coords.get()), sizeof(float) * lengths[1]); if (!infile) { - LOG(ERROR) << "Failed to read texture coordinates for frame " - << frame_count_; + ABSL_LOG(ERROR) << "Failed to read texture coordinates for frame " + << frame_count_; return false; } @@ -482,8 +486,8 @@ bool GlAnimationOverlayCalculator::LoadAnimation(const std::string &filename) { infile.read((char *)(triangle_mesh.triangle_indices.get()), sizeof(int16_t) * lengths[2]); if (!infile) { - LOG(ERROR) << "Failed to read triangle indices for frame " - << frame_count_; + ABSL_LOG(ERROR) << "Failed to read triangle indices for frame " + << frame_count_; return false; } @@ -493,9 +497,10 @@ bool GlAnimationOverlayCalculator::LoadAnimation(const std::string &filename) { frame_count_++; } - LOG(INFO) << "Finished parsing " << frame_count_ << " animation frames."; + ABSL_LOG(INFO) << "Finished parsing " << frame_count_ << " animation frames."; if (triangle_meshes_.empty()) { - LOG(ERROR) << "No animation frames were parsed! Erroring out calculator."; + ABSL_LOG(ERROR) + << "No animation frames were parsed! Erroring out calculator."; return false; } return true; @@ -506,8 +511,8 @@ bool GlAnimationOverlayCalculator::LoadAnimation(const std::string &filename) { void GlAnimationOverlayCalculator::ComputeAspectRatioAndFovFromCameraParameters( const CameraParametersProto &camera_parameters, float *aspect_ratio, float *vertical_fov_degrees) { - CHECK(aspect_ratio != nullptr); - CHECK(vertical_fov_degrees != nullptr); + ABSL_CHECK(aspect_ratio != nullptr); + ABSL_CHECK(vertical_fov_degrees != nullptr); *aspect_ratio = camera_parameters.portrait_width() / camera_parameters.portrait_height(); *vertical_fov_degrees = @@ -560,7 +565,7 @@ absl::Status GlAnimationOverlayCalculator::Open(CalculatorContext *cc) { cc->InputSidePackets().Tag("MASK_ASSET").Get(); loaded_animation = LoadAnimationAndroid(mask_asset_name, &mask_meshes_); if (!loaded_animation) { - LOG(ERROR) << "Failed to load mask asset."; + ABSL_LOG(ERROR) << "Failed to load mask asset."; return absl::UnknownError("Failed to load mask asset."); } } @@ -569,7 +574,7 @@ absl::Status GlAnimationOverlayCalculator::Open(CalculatorContext *cc) { loaded_animation = LoadAnimation(asset_name); #endif if (!loaded_animation) { - LOG(ERROR) << "Failed to load animation asset."; + ABSL_LOG(ERROR) << "Failed to load animation asset."; return absl::UnknownError("Failed to load animation asset."); } @@ -608,7 +613,7 @@ void GlAnimationOverlayCalculator::LoadModelMatrices( current_model_matrices->clear(); for (int i = 0; i < model_matrices.model_matrix_size(); ++i) { const auto &model_matrix = model_matrices.model_matrix(i); - CHECK(model_matrix.matrix_entries_size() == kNumMatrixEntries) + ABSL_CHECK(model_matrix.matrix_entries_size() == kNumMatrixEntries) << "Invalid Model Matrix"; current_model_matrices->emplace_back(); ModelMatrix &new_matrix = current_model_matrices->back(); @@ -669,8 +674,8 @@ absl::Status GlAnimationOverlayCalculator::Process(CalculatorContext *cc) { height = input_frame->height(); dst = helper_.CreateSourceTexture(*input_frame); } else { - LOG(ERROR) << "Unable to consume input video frame for overlay!"; - LOG(ERROR) << "Status returned was: " << result.status(); + ABSL_LOG(ERROR) << "Unable to consume input video frame for overlay!"; + ABSL_LOG(ERROR) << "Status returned was: " << result.status(); dst = helper_.CreateDestinationTexture(width, height); } } else if (!has_video_stream_) { @@ -699,7 +704,7 @@ absl::Status GlAnimationOverlayCalculator::Process(CalculatorContext *cc) { GL_RENDERBUFFER, renderbuffer_)); GLenum status = GLCHECK(glCheckFramebufferStatus(GL_FRAMEBUFFER)); if (status != GL_FRAMEBUFFER_COMPLETE) { - LOG(ERROR) << "Incomplete framebuffer with status: " << status; + ABSL_LOG(ERROR) << "Incomplete framebuffer with status: " << status; } GLCHECK(glClear(GL_DEPTH_BUFFER_BIT)); diff --git a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java index 9321e82b..591b6c98 100644 --- a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java +++ b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java @@ -34,6 +34,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.Nullable; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.opengles.GL10; @@ -303,7 +304,7 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { } // Use this when the texture is not a SurfaceTexture. - public void setNextFrame(TextureFrame frame) { + public void setNextFrame(@Nullable TextureFrame frame) { if (surfaceTexture != null) { Matrix.setIdentityM(textureTransformMatrix, 0 /* offset */); } diff --git a/mediapipe/java/com/google/mediapipe/framework/AppTextureFrame.java b/mediapipe/java/com/google/mediapipe/framework/AppTextureFrame.java index 20c63c06..242cd616 100644 --- a/mediapipe/java/com/google/mediapipe/framework/AppTextureFrame.java +++ b/mediapipe/java/com/google/mediapipe/framework/AppTextureFrame.java @@ -78,17 +78,21 @@ public class AppTextureFrame implements TextureFrame { * Use {@link waitUntilReleasedWithGpuSync} whenever possible. */ public void waitUntilReleased() throws InterruptedException { + GlSyncToken tokenToRelease = null; synchronized (this) { while (inUse && releaseSyncToken == null) { wait(); } if (releaseSyncToken != null) { - releaseSyncToken.waitOnCpu(); - releaseSyncToken.release(); + tokenToRelease = releaseSyncToken; inUse = false; releaseSyncToken = null; } } + if (tokenToRelease != null) { + tokenToRelease.waitOnCpu(); + tokenToRelease.release(); + } } /** @@ -98,17 +102,21 @@ public class AppTextureFrame implements TextureFrame { * TextureFrame. */ public void waitUntilReleasedWithGpuSync() throws InterruptedException { + GlSyncToken tokenToRelease = null; synchronized (this) { while (inUse && releaseSyncToken == null) { wait(); } if (releaseSyncToken != null) { - releaseSyncToken.waitOnGpu(); - releaseSyncToken.release(); + tokenToRelease = releaseSyncToken; inUse = false; releaseSyncToken = null; } } + if (tokenToRelease != null) { + tokenToRelease.waitOnGpu(); + tokenToRelease.release(); + } } /** diff --git a/mediapipe/java/com/google/mediapipe/framework/BUILD b/mediapipe/java/com/google/mediapipe/framework/BUILD index dd5f8f1d..78ae61d0 100644 --- a/mediapipe/java/com/google/mediapipe/framework/BUILD +++ b/mediapipe/java/com/google/mediapipe/framework/BUILD @@ -50,7 +50,6 @@ android_library( "MediaPipeRunner.java", ], visibility = [ - "//java/com/google/android/libraries/camera/effects:__subpackages__", "//mediapipe/java/com/google/mediapipe:__subpackages__", ], exports = [ diff --git a/mediapipe/java/com/google/mediapipe/framework/PacketCreator.java b/mediapipe/java/com/google/mediapipe/framework/PacketCreator.java index 04265cab..e71749d0 100644 --- a/mediapipe/java/com/google/mediapipe/framework/PacketCreator.java +++ b/mediapipe/java/com/google/mediapipe/framework/PacketCreator.java @@ -237,6 +237,10 @@ public class PacketCreator { return Packet.create(nativeCreateInt32Array(mediapipeGraph.getNativeHandle(), data)); } + public Packet createInt32Pair(int first, int second) { + return Packet.create(nativeCreateInt32Pair(mediapipeGraph.getNativeHandle(), first, second)); + } + public Packet createFloat32Array(float[] data) { return Packet.create(nativeCreateFloat32Array(mediapipeGraph.getNativeHandle(), data)); } @@ -449,6 +453,8 @@ public class PacketCreator { private native long nativeCreateInt32Array(long context, int[] data); + private native long nativeCreateInt32Pair(long context, int first, int second); + private native long nativeCreateFloat32Array(long context, float[] data); private native long nativeCreateFloat32Vector(long context, float[] data); diff --git a/mediapipe/java/com/google/mediapipe/framework/PacketGetter.java b/mediapipe/java/com/google/mediapipe/framework/PacketGetter.java index 1c1daadc..5ea12872 100644 --- a/mediapipe/java/com/google/mediapipe/framework/PacketGetter.java +++ b/mediapipe/java/com/google/mediapipe/framework/PacketGetter.java @@ -239,7 +239,7 @@ public final class PacketGetter { /** * Assign the native image buffer array in given ByteBuffer array. It assumes given ByteBuffer - * array has the the same size of image list packet, and assumes the output buffer stores pixels + * array has the same size of image list packet, and assumes the output buffer stores pixels * contiguously. It returns false if this assumption does not hold. * *

If deepCopy is true, it assumes the given buffersArray has allocated the required size of diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/BUILD b/mediapipe/java/com/google/mediapipe/framework/jni/BUILD index 778790b1..0a985f87 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/BUILD +++ b/mediapipe/java/com/google/mediapipe/framework/jni/BUILD @@ -95,13 +95,14 @@ cc_library( "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/port:core_proto", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:singleton", "//mediapipe/framework/port:status", "//mediapipe/framework/port:threadpool", "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/framework/tool:executor_util", "//mediapipe/framework/tool:name_util", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", @@ -138,8 +139,8 @@ cc_library( hdrs = ["jni_util.h"], deps = [ ":class_registry", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/synchronization", ] + select({ "//conditions:default": [ @@ -173,8 +174,8 @@ cc_library( ":class_registry", ":loose_headers", ":mediapipe_framework_jni", - "//mediapipe/framework/port:logging", "@com_google_absl//absl/container:node_hash_map", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ] + select({ diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/android_packet_creator_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/android_packet_creator_jni.cc index cda84ac1..a40112b2 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/android_packet_creator_jni.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/android_packet_creator_jni.cc @@ -19,11 +19,11 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/colorspace.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/graph.h" @@ -49,26 +49,26 @@ std::unique_ptr CreateImageFrameFromBitmap( void* pixel_addr = nullptr; int result = AndroidBitmap_lockPixels(env, bitmap, &pixel_addr); if (result != ANDROID_BITMAP_RESULT_SUCCESS) { - LOG(ERROR) << "AndroidBitmap_lockPixels() failed with result code " - << result; + ABSL_LOG(ERROR) << "AndroidBitmap_lockPixels() failed with result code " + << result; return nullptr; } if (format == mediapipe::ImageFormat::SRGBA) { const int64_t buffer_size = stride * height; if (buffer_size != image_frame->PixelDataSize()) { - LOG(ERROR) << "Bitmap stride: " << stride - << " times bitmap height: " << height - << " is not equal to the expected size: " - << image_frame->PixelDataSize(); + ABSL_LOG(ERROR) << "Bitmap stride: " << stride + << " times bitmap height: " << height + << " is not equal to the expected size: " + << image_frame->PixelDataSize(); return nullptr; } std::memcpy(image_frame->MutablePixelData(), pixel_addr, image_frame->PixelDataSize()); } else if (format == mediapipe::ImageFormat::SRGB) { if (stride != width * 4) { - LOG(ERROR) << "Bitmap stride: " << stride - << "is not equal to 4 times bitmap width: " << width; + ABSL_LOG(ERROR) << "Bitmap stride: " << stride + << "is not equal to 4 times bitmap width: " << width; return nullptr; } const uint8_t* rgba_data = static_cast(pixel_addr); @@ -76,14 +76,14 @@ std::unique_ptr CreateImageFrameFromBitmap( image_frame->MutablePixelData(), image_frame->WidthStep()); } else { - LOG(ERROR) << "unsupported image format: " << format; + ABSL_LOG(ERROR) << "unsupported image format: " << format; return nullptr; } result = AndroidBitmap_unlockPixels(env, bitmap); if (result != ANDROID_BITMAP_RESULT_SUCCESS) { - LOG(ERROR) << "AndroidBitmap_unlockPixels() failed with result code " - << result; + ABSL_LOG(ERROR) << "AndroidBitmap_unlockPixels() failed with result code " + << result; return nullptr; } @@ -98,7 +98,8 @@ JNIEXPORT jlong JNICALL ANDROID_PACKET_CREATOR_METHOD( AndroidBitmapInfo info; int result = AndroidBitmap_getInfo(env, bitmap, &info); if (result != ANDROID_BITMAP_RESULT_SUCCESS) { - LOG(ERROR) << "AndroidBitmap_getInfo() failed with result code " << result; + ABSL_LOG(ERROR) << "AndroidBitmap_getInfo() failed with result code " + << result; return 0L; } @@ -117,7 +118,8 @@ JNIEXPORT jlong JNICALL ANDROID_PACKET_CREATOR_METHOD( AndroidBitmapInfo info; int result = AndroidBitmap_getInfo(env, bitmap, &info); if (result != ANDROID_BITMAP_RESULT_SUCCESS) { - LOG(ERROR) << "AndroidBitmap_getInfo() failed with result code " << result; + ABSL_LOG(ERROR) << "AndroidBitmap_getInfo() failed with result code " + << result; return 0L; } @@ -135,7 +137,8 @@ JNIEXPORT jlong JNICALL ANDROID_PACKET_CREATOR_METHOD(nativeCreateRgbaImage)( AndroidBitmapInfo info; int result = AndroidBitmap_getInfo(env, bitmap, &info); if (result != ANDROID_BITMAP_RESULT_SUCCESS) { - LOG(ERROR) << "AndroidBitmap_getInfo() failed with result code " << result; + ABSL_LOG(ERROR) << "AndroidBitmap_getInfo() failed with result code " + << result; return 0L; } diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc b/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc index d565187d..f129b1a7 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc @@ -18,6 +18,7 @@ #include +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/synchronization/mutex.h" @@ -75,7 +76,7 @@ class CallbackHandler { // The jobject global reference is managed by the Graph directly. // So no-op here. if (java_callback_) { - LOG(ERROR) << "Java callback global reference is not released."; + ABSL_LOG(ERROR) << "Java callback global reference is not released."; } } @@ -135,7 +136,8 @@ Graph::~Graph() { // Cleans up the jni objects. JNIEnv* env = mediapipe::java::GetJNIEnv(); if (env == nullptr) { - LOG(ERROR) << "Can't attach to java thread, no jni clean up performed."; + ABSL_LOG(ERROR) + << "Can't attach to java thread, no jni clean up performed."; return; } for (const auto& handler : callback_handlers_) { @@ -219,12 +221,12 @@ absl::Status Graph::AddMultiStreamCallbackHandler( int64_t Graph::AddSurfaceOutput(const std::string& output_stream_name) { if (!graph_config()) { - LOG(ERROR) << "Graph is not loaded!"; + ABSL_LOG(ERROR) << "Graph is not loaded!"; return 0; } #if MEDIAPIPE_DISABLE_GPU - LOG(FATAL) << "GPU support has been disabled in this build!"; + ABSL_LOG(FATAL) << "GPU support has been disabled in this build!"; #else CalculatorGraphConfig::Node* sink_node = graph_config()->add_node(); sink_node->set_name(mediapipe::tool::GetUnusedNodeName( @@ -291,7 +293,7 @@ CalculatorGraphConfig Graph::GetCalculatorGraphConfig() { CalculatorGraph temp_graph; absl::Status status = InitializeGraph(&temp_graph); if (!status.ok()) { - LOG(ERROR) << "GetCalculatorGraphConfig failed:\n" << status.message(); + ABSL_LOG(ERROR) << "GetCalculatorGraphConfig failed:\n" << status.message(); } return temp_graph.Config(); } @@ -416,13 +418,13 @@ absl::Status Graph::RunGraphUntilClose(JNIEnv* env) { CalculatorGraph calculator_graph; absl::Status status = InitializeGraph(&calculator_graph); if (!status.ok()) { - LOG(ERROR) << status.message(); + ABSL_LOG(ERROR) << status.message(); running_graph_.reset(nullptr); return status; } // TODO: gpu & services set up! status = calculator_graph.Run(CreateCombinedSidePackets()); - LOG(INFO) << "Graph run finished."; + ABSL_LOG(INFO) << "Graph run finished."; return status; } @@ -440,9 +442,9 @@ absl::Status Graph::StartRunningGraph(JNIEnv* env) { // Set the mode for adding packets to graph input streams. running_graph_->SetGraphInputStreamAddMode(graph_input_stream_add_mode_); if (VLOG_IS_ON(2)) { - LOG(INFO) << "input packet streams:"; + ABSL_LOG(INFO) << "input packet streams:"; for (auto& name : graph_config()->input_stream()) { - LOG(INFO) << name; + ABSL_LOG(INFO) << name; } } absl::Status status; @@ -450,7 +452,7 @@ absl::Status Graph::StartRunningGraph(JNIEnv* env) { if (gpu_resources_) { status = running_graph_->SetGpuResources(gpu_resources_); if (!status.ok()) { - LOG(ERROR) << status.message(); + ABSL_LOG(ERROR) << status.message(); running_graph_.reset(nullptr); return status; } @@ -461,7 +463,7 @@ absl::Status Graph::StartRunningGraph(JNIEnv* env) { status = running_graph_->SetServicePacket(*service_packet.first, service_packet.second); if (!status.ok()) { - LOG(ERROR) << status.message(); + ABSL_LOG(ERROR) << status.message(); running_graph_.reset(nullptr); return status; } @@ -469,15 +471,15 @@ absl::Status Graph::StartRunningGraph(JNIEnv* env) { status = InitializeGraph(running_graph_.get()); if (!status.ok()) { - LOG(ERROR) << status.message(); + ABSL_LOG(ERROR) << status.message(); running_graph_.reset(nullptr); return status; } - LOG(INFO) << "Start running the graph, waiting for inputs."; + ABSL_LOG(INFO) << "Start running the graph, waiting for inputs."; status = running_graph_->StartRun(CreateCombinedSidePackets(), stream_headers_); if (!status.ok()) { - LOG(ERROR) << status; + ABSL_LOG(ERROR) << status; running_graph_.reset(nullptr); return status; } @@ -520,12 +522,12 @@ absl::Status Graph::CloseInputStream(std::string stream_name) { if (!running_graph_) { return absl::FailedPreconditionError("Graph must be running."); } - LOG(INFO) << "Close input stream: " << stream_name; + ABSL_LOG(INFO) << "Close input stream: " << stream_name; return running_graph_->CloseInputStream(stream_name); } absl::Status Graph::CloseAllInputStreams() { - LOG(INFO) << "Close all input streams."; + ABSL_LOG(INFO) << "Close all input streams."; if (!running_graph_) { return absl::FailedPreconditionError("Graph must be running."); } @@ -533,7 +535,7 @@ absl::Status Graph::CloseAllInputStreams() { } absl::Status Graph::CloseAllPacketSources() { - LOG(INFO) << "Close all input streams."; + ABSL_LOG(INFO) << "Close all input streams."; if (!running_graph_) { return absl::FailedPreconditionError("Graph must be running."); } @@ -564,7 +566,7 @@ void Graph::SetInputSidePacket(const std::string& stream_name, void Graph::SetStreamHeader(const std::string& stream_name, const Packet& packet) { stream_headers_[stream_name] = packet; - LOG(INFO) << stream_name << " stream header being set."; + ABSL_LOG(INFO) << stream_name << " stream header being set."; } void Graph::SetGraphInputStreamAddMode( @@ -580,7 +582,7 @@ mediapipe::GpuResources* Graph::GetGpuResources() const { absl::Status Graph::SetParentGlContext(int64_t java_gl_context) { #if MEDIAPIPE_DISABLE_GPU - LOG(FATAL) << "GPU support has been disabled in this build!"; + ABSL_LOG(FATAL) << "GPU support has been disabled in this build!"; #else if (gpu_resources_) { return absl::AlreadyExistsError( diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/graph_texture_frame_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/graph_texture_frame_jni.cc index b3bcd14d..a658d01c 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/graph_texture_frame_jni.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/graph_texture_frame_jni.cc @@ -14,6 +14,7 @@ #include "mediapipe/java/com/google/mediapipe/framework/jni/graph_texture_frame_jni.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_context.h" @@ -101,8 +102,8 @@ JNIEXPORT void JNICALL GRAPH_TEXTURE_FRAME_METHOD(nativeDidRead)( // However, `DidRead` may succeed resulting in a later crash and masking the // actual problem.) if (token.use_count() == 0) { - LOG_FIRST_N(ERROR, 5) << absl::StrFormat("invalid sync token ref: %d", - consumerSyncToken); + ABSL_LOG_FIRST_N(ERROR, 5) + << absl::StrFormat("invalid sync token ref: %d", consumerSyncToken); return; } (*buffer)->DidRead(token); diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc b/mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc index 88a1366b..6ccf8d7e 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc @@ -16,8 +16,8 @@ #include +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" namespace { @@ -38,7 +38,7 @@ class JvmThread { case JNI_OK: break; case JNI_EDETACHED: - LOG(INFO) << "GetEnv: not attached"; + ABSL_LOG(INFO) << "GetEnv: not attached"; if (jvm_->AttachCurrentThread( #ifdef __ANDROID__ &jni_env_, @@ -46,16 +46,16 @@ class JvmThread { reinterpret_cast(&jni_env_), #endif // __ANDROID__ nullptr) != 0) { - LOG(ERROR) << "Failed to attach to java thread."; + ABSL_LOG(ERROR) << "Failed to attach to java thread."; break; } attached_ = true; break; case JNI_EVERSION: - LOG(ERROR) << "GetEnv: jni version not supported."; + ABSL_LOG(ERROR) << "GetEnv: jni version not supported."; break; default: - LOG(ERROR) << "GetEnv: unknown status."; + ABSL_LOG(ERROR) << "GetEnv: unknown status."; break; } } @@ -83,7 +83,7 @@ static pthread_once_t key_once = PTHREAD_ONCE_INIT; static void ThreadExitCallback(void* key_value) { JvmThread* jvm_thread = reinterpret_cast(key_value); // Detach the thread when thread exits. - LOG(INFO) << "Exiting thread. Detach thread."; + ABSL_LOG(INFO) << "Exiting thread. Detach thread."; delete jvm_thread; } @@ -187,7 +187,7 @@ bool SetJavaVM(JNIEnv* env) { absl::MutexLock lock(&g_jvm_mutex); if (!g_jvm) { if (env->GetJavaVM(&g_jvm) != JNI_OK) { - LOG(ERROR) << "Can not get the Java VM instance!"; + ABSL_LOG(ERROR) << "Can not get the Java VM instance!"; g_jvm = nullptr; return false; } diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc index f7430e6e..56ddd5e0 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc @@ -16,6 +16,7 @@ #include #include +#include #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -27,6 +28,7 @@ #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/time_series_header.pb.h" #include "mediapipe/framework/formats/video_stream_header.h" +#include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/core_proto_inc.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/colorspace.h" @@ -481,6 +483,15 @@ JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateInt32Array)( return CreatePacketWithContext(context, packet); } +JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateInt32Pair)( + JNIEnv* env, jobject thiz, jlong context, jint first, jint second) { + static_assert(std::is_same::value, "jint must be int32_t"); + + mediapipe::Packet packet = mediapipe::MakePacket>( + std::make_pair(first, second)); + return CreatePacketWithContext(context, packet); +} + JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateStringFromByteArray)( JNIEnv* env, jobject thiz, jlong context, jbyteArray data) { jsize count = env->GetArrayLength(data); diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h b/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h index b3b1043f..92f48261 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h +++ b/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h @@ -118,6 +118,9 @@ JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateFloat32Vector)( JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateInt32Array)( JNIEnv* env, jobject thiz, jlong context, jintArray data); +JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateInt32Pair)( + JNIEnv* env, jobject thiz, jlong context, jint first, jint second); + JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateStringFromByteArray)( JNIEnv* env, jobject thiz, jlong context, jbyteArray data); diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.cc b/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.cc index bef275b4..3f96a404 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.cc @@ -14,8 +14,8 @@ #include "mediapipe/java/com/google/mediapipe/framework/jni/register_natives.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" #if defined(__ANDROID__) @@ -65,9 +65,10 @@ void RegisterNativesVector(JNIEnv *env, jclass cls, // in exchange for flexibility to list out all registrations without worrying // about usage subset by client Java projects. if (!cls || methods.empty()) { - LOG(INFO) << "Skipping registration and clearing exception. Class or " - "native methods not found, may be unused and/or trimmed by " - "Proguard."; + ABSL_LOG(INFO) + << "Skipping registration and clearing exception. Class or " + "native methods not found, may be unused and/or trimmed by " + "Proguard."; env->ExceptionClear(); return; } @@ -81,7 +82,7 @@ void RegisterNativesVector(JNIEnv *env, jclass cls, } // Fatal crash if registration fails. if (env->RegisterNatives(cls, methods_array, methods.size()) < 0) { - LOG(FATAL) + ABSL_LOG(FATAL) << "Failed during native method registration, so likely the " "signature of a method is incorrect. Make sure there are no typos " "and " diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/surface_output_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/surface_output_jni.cc index 51d693b2..2ac43e57 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/surface_output_jni.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/surface_output_jni.cc @@ -17,6 +17,8 @@ #include #endif // __ANDROID__ +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/gpu/egl_surface_holder.h" @@ -51,7 +53,7 @@ JNIEXPORT void JNICALL MEDIAPIPE_SURFACE_OUTPUT_METHOD(nativeSetSurface)( JNIEnv* env, jobject thiz, jlong context, jlong packet, jobject surface) { #ifdef __ANDROID__ mediapipe::GlContext* gl_context = GetGlContext(context); - CHECK(gl_context) << "GPU shared data not created"; + ABSL_CHECK(gl_context) << "GPU shared data not created"; mediapipe::EglSurfaceHolder* surface_holder = GetSurfaceHolder(packet); // ANativeWindow_fromSurface must not be called on the GL thread, it is a @@ -99,14 +101,14 @@ JNIEXPORT void JNICALL MEDIAPIPE_SURFACE_OUTPUT_METHOD(nativeSetSurface)( ANativeWindow_release(window); } #else - LOG(FATAL) << "setSurface is only supported on Android"; + ABSL_LOG(FATAL) << "setSurface is only supported on Android"; #endif // __ANDROID__ } JNIEXPORT void JNICALL MEDIAPIPE_SURFACE_OUTPUT_METHOD(nativeSetEglSurface)( JNIEnv* env, jobject thiz, jlong context, jlong packet, jlong surface) { mediapipe::GlContext* gl_context = GetGlContext(context); - CHECK(gl_context) << "GPU shared data not created"; + ABSL_CHECK(gl_context) << "GPU shared data not created"; auto egl_surface = reinterpret_cast(surface); mediapipe::EglSurfaceHolder* surface_holder = GetSurfaceHolder(packet); EGLSurface old_surface = EGL_NO_SURFACE; diff --git a/mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl b/mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl index 879527ed..8817f283 100644 --- a/mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl +++ b/mediapipe/java/com/google/mediapipe/mediapipe_aar.bzl @@ -197,7 +197,6 @@ def _mediapipe_jni(name, gen_libmediapipe, calculators = []): name = name + "_opencv_cc_lib", srcs = select({ "//mediapipe:android_arm64": ["@android_opencv//:libopencv_java3_so_arm64-v8a"], - "//mediapipe:android_armeabi": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"], "//mediapipe:android_arm": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"], "//mediapipe:android_x86": ["@android_opencv//:libopencv_java3_so_x86"], "//mediapipe:android_x86_64": ["@android_opencv//:libopencv_java3_so_x86_64"], diff --git a/mediapipe/model_maker/__init__.py b/mediapipe/model_maker/__init__.py index 6779524b..8c87c12d 100644 --- a/mediapipe/model_maker/__init__.py +++ b/mediapipe/model_maker/__init__.py @@ -13,11 +13,15 @@ # limitations under the License. +from mediapipe.model_maker.python.vision.core import image_utils from mediapipe.model_maker.python.core.utils import quantization +from mediapipe.model_maker.python.core.utils import model_util + from mediapipe.model_maker.python.vision import image_classifier from mediapipe.model_maker.python.vision import gesture_recognizer from mediapipe.model_maker.python.text import text_classifier from mediapipe.model_maker.python.vision import object_detector +from mediapipe.model_maker.python.vision import face_stylizer # Remove duplicated and non-public API del python diff --git a/mediapipe/model_maker/python/BUILD b/mediapipe/model_maker/python/BUILD index 775ac82d..42681fad 100644 --- a/mediapipe/model_maker/python/BUILD +++ b/mediapipe/model_maker/python/BUILD @@ -24,6 +24,7 @@ package_group( package_group( name = "1p_client", packages = [ + "//cloud/ml/applications/vision/model_garden/model_oss/mediapipe/...", "//research/privacy/learning/fl_eval/pcvr/...", ], ) diff --git a/mediapipe/model_maker/python/core/BUILD b/mediapipe/model_maker/python/core/BUILD index 0ed20a2f..a73e545d 100644 --- a/mediapipe/model_maker/python/core/BUILD +++ b/mediapipe/model_maker/python/core/BUILD @@ -14,7 +14,10 @@ # Placeholder for internal Python strict library and test compatibility macro. -package(default_visibility = ["//mediapipe:__subpackages__"]) +package(default_visibility = [ + "//cloud/ml/applications/vision/model_garden/model_oss/mediapipe:__subpackages__", + "//mediapipe:__subpackages__", +]) licenses(["notice"]) diff --git a/mediapipe/model_maker/python/core/data/BUILD b/mediapipe/model_maker/python/core/data/BUILD index 1c2fb7a4..4364b774 100644 --- a/mediapipe/model_maker/python/core/data/BUILD +++ b/mediapipe/model_maker/python/core/data/BUILD @@ -57,3 +57,14 @@ py_test( srcs = ["classification_dataset_test.py"], deps = [":classification_dataset"], ) + +py_library( + name = "cache_files", + srcs = ["cache_files.py"], +) + +py_test( + name = "cache_files_test", + srcs = ["cache_files_test.py"], + deps = [":cache_files"], +) diff --git a/mediapipe/model_maker/python/core/data/cache_files.py b/mediapipe/model_maker/python/core/data/cache_files.py new file mode 100644 index 00000000..13d3d5b6 --- /dev/null +++ b/mediapipe/model_maker/python/core/data/cache_files.py @@ -0,0 +1,112 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Common TFRecord cache files library.""" + +import dataclasses +import os +import tempfile +from typing import Any, Mapping, Sequence + +import tensorflow as tf +import yaml + + +# Suffix of the meta data file name. +METADATA_FILE_SUFFIX = '_metadata.yaml' + + +@dataclasses.dataclass(frozen=True) +class TFRecordCacheFiles: + """TFRecordCacheFiles dataclass to store and load cached TFRecord files. + + Attributes: + cache_prefix_filename: The cache prefix filename. This is usually provided + as a hash of the original data source to avoid different data sources + resulting in the same cache file. + cache_dir: The cache directory to save TFRecord and metadata file. When + cache_dir is None, a temporary folder will be created and will not be + removed automatically after training which makes it can be used later. + num_shards: Number of shards for output tfrecord files. + """ + + cache_prefix_filename: str = 'cache_prefix' + cache_dir: str = dataclasses.field(default_factory=tempfile.mkdtemp) + num_shards: int = 1 + + def __post_init__(self): + if not tf.io.gfile.exists(self.cache_dir): + tf.io.gfile.makedirs(self.cache_dir) + if not self.cache_prefix_filename: + raise ValueError('cache_prefix_filename cannot be empty.') + if self.num_shards <= 0: + raise ValueError( + f'num_shards must be greater than 0, got {self.num_shards}' + ) + + @property + def cache_prefix(self) -> str: + """The cache prefix including the cache directory and the cache prefix filename.""" + return os.path.join(self.cache_dir, self.cache_prefix_filename) + + @property + def tfrecord_files(self) -> Sequence[str]: + """The TFRecord files.""" + tfrecord_files = [ + self.cache_prefix + '-%05d-of-%05d.tfrecord' % (i, self.num_shards) + for i in range(self.num_shards) + ] + return tfrecord_files + + @property + def metadata_file(self) -> str: + """The metadata file.""" + return self.cache_prefix + METADATA_FILE_SUFFIX + + def get_writers(self) -> Sequence[tf.io.TFRecordWriter]: + """Gets an array of TFRecordWriter objects. + + Note that these writers should each be closed using .close() when done. + + Returns: + Array of TFRecordWriter objects + """ + return [tf.io.TFRecordWriter(path) for path in self.tfrecord_files] + + def save_metadata(self, metadata): + """Writes metadata to file. + + Args: + metadata: A dictionary of metadata content to write. Exact format is + dependent on the specific dataset, but typically includes a 'size' and + 'label_names' entry. + """ + with tf.io.gfile.GFile(self.metadata_file, 'w') as f: + yaml.dump(metadata, f) + + def load_metadata(self) -> Mapping[Any, Any]: + """Reads metadata from file. + + Returns: + Dictionary object containing metadata + """ + if not tf.io.gfile.exists(self.metadata_file): + return {} + with tf.io.gfile.GFile(self.metadata_file, 'r') as f: + metadata = yaml.load(f, Loader=yaml.FullLoader) + return metadata + + def is_cached(self) -> bool: + """Checks whether this CacheFiles is already cached.""" + all_cached_files = list(self.tfrecord_files) + [self.metadata_file] + return all(tf.io.gfile.exists(f) for f in all_cached_files) diff --git a/mediapipe/model_maker/python/core/data/cache_files_test.py b/mediapipe/model_maker/python/core/data/cache_files_test.py new file mode 100644 index 00000000..ac727b3f --- /dev/null +++ b/mediapipe/model_maker/python/core/data/cache_files_test.py @@ -0,0 +1,77 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf + +from mediapipe.model_maker.python.core.data import cache_files + + +class CacheFilesTest(tf.test.TestCase): + + def test_tfrecord_cache_files(self): + cf = cache_files.TFRecordCacheFiles( + cache_prefix_filename='tfrecord', + cache_dir='/tmp/cache_dir', + num_shards=2, + ) + self.assertEqual(cf.cache_prefix, '/tmp/cache_dir/tfrecord') + self.assertEqual( + cf.metadata_file, + '/tmp/cache_dir/tfrecord' + cache_files.METADATA_FILE_SUFFIX, + ) + expected_tfrecord_files = [ + '/tmp/cache_dir/tfrecord-%05d-of-%05d.tfrecord' % (i, 2) + for i in range(2) + ] + self.assertEqual(cf.tfrecord_files, expected_tfrecord_files) + + # Writing TFRecord Files + self.assertFalse(cf.is_cached()) + for tfrecord_file in cf.tfrecord_files: + self.assertFalse(tf.io.gfile.exists(tfrecord_file)) + writers = cf.get_writers() + for writer in writers: + writer.close() + for tfrecord_file in cf.tfrecord_files: + self.assertTrue(tf.io.gfile.exists(tfrecord_file)) + self.assertFalse(cf.is_cached()) + + # Writing Metadata Files + original_metadata = {'size': 10, 'label_names': ['label1', 'label2']} + cf.save_metadata(original_metadata) + self.assertTrue(cf.is_cached()) + metadata = cf.load_metadata() + self.assertEqual(metadata, original_metadata) + + def test_recordio_cache_files_error(self): + with self.assertRaisesRegex( + ValueError, 'cache_prefix_filename cannot be empty' + ): + cache_files.TFRecordCacheFiles( + cache_prefix_filename='', + cache_dir='/tmp/cache_dir', + num_shards=2, + ) + with self.assertRaisesRegex( + ValueError, 'num_shards must be greater than 0, got 0' + ): + cache_files.TFRecordCacheFiles( + cache_prefix_filename='tfrecord', + cache_dir='/tmp/cache_dir', + num_shards=0, + ) + + +if __name__ == '__main__': + tf.test.main() diff --git a/mediapipe/model_maker/python/core/data/classification_dataset.py b/mediapipe/model_maker/python/core/data/classification_dataset.py index b1df3b6d..352caca6 100644 --- a/mediapipe/model_maker/python/core/data/classification_dataset.py +++ b/mediapipe/model_maker/python/core/data/classification_dataset.py @@ -13,7 +13,7 @@ # limitations under the License. """Common classification dataset library.""" -from typing import List, Tuple +from typing import List, Optional, Tuple import tensorflow as tf @@ -23,8 +23,12 @@ from mediapipe.model_maker.python.core.data import dataset as ds class ClassificationDataset(ds.Dataset): """Dataset Loader for classification models.""" - def __init__(self, dataset: tf.data.Dataset, size: int, - label_names: List[str]): + def __init__( + self, + dataset: tf.data.Dataset, + label_names: List[str], + size: Optional[int] = None, + ): super().__init__(dataset, size) self._label_names = label_names diff --git a/mediapipe/model_maker/python/core/data/classification_dataset_test.py b/mediapipe/model_maker/python/core/data/classification_dataset_test.py index d21803f4..dfcea7da 100644 --- a/mediapipe/model_maker/python/core/data/classification_dataset_test.py +++ b/mediapipe/model_maker/python/core/data/classification_dataset_test.py @@ -36,9 +36,14 @@ class ClassificationDatasetTest(tf.test.TestCase): value: A value variable stored by the mock dataset class for testing. """ - def __init__(self, dataset: tf.data.Dataset, size: int, - label_names: List[str], value: Any): - super().__init__(dataset=dataset, size=size, label_names=label_names) + def __init__( + self, + dataset: tf.data.Dataset, + label_names: List[str], + value: Any, + size: int, + ): + super().__init__(dataset=dataset, label_names=label_names, size=size) self.value = value def split(self, fraction: float) -> Tuple[_DatasetT, _DatasetT]: @@ -52,7 +57,8 @@ class ClassificationDatasetTest(tf.test.TestCase): # Create data loader from sample data. ds = tf.data.Dataset.from_tensor_slices([[0, 1], [1, 1], [0, 0], [1, 0]]) data = MagicClassificationDataset( - dataset=ds, size=len(ds), label_names=label_names, value=magic_value) + dataset=ds, label_names=label_names, value=magic_value, size=len(ds) + ) # Train/Test data split. fraction = .25 diff --git a/mediapipe/model_maker/python/core/data/dataset.py b/mediapipe/model_maker/python/core/data/dataset.py index bfdc5b0f..0cfccb14 100644 --- a/mediapipe/model_maker/python/core/data/dataset.py +++ b/mediapipe/model_maker/python/core/data/dataset.py @@ -56,15 +56,14 @@ class Dataset(object): def size(self) -> Optional[int]: """Returns the size of the dataset. - Note that this function may return None becuase the exact size of the - dataset isn't a necessary parameter to create an instance of this class, - and tf.data.Dataset donesn't support a function to get the length directly - since it's lazy-loaded and may be infinite. - In most cases, however, when an instance of this class is created by helper - functions like 'from_folder', the size of the dataset will be preprocessed, - and this function can return an int representing the size of the dataset. + Same functionality as calling __len__. See the __len__ method definition for + more information. + + Raises: + TypeError if self._size is not set and the cardinality of self._dataset + is INFINITE_CARDINALITY or UNKNOWN_CARDINALITY. """ - return self._size + return self.__len__() def gen_tf_dataset( self, @@ -116,8 +115,22 @@ class Dataset(object): # here. return dataset - def __len__(self): - """Returns the number of element of the dataset.""" + def __len__(self) -> int: + """Returns the number of element of the dataset. + + If size is not set, this method will fallback to using the __len__ method + of the tf.data.Dataset in self._dataset. Calling __len__ on a + tf.data.Dataset instance may throw a TypeError because the dataset may + be lazy-loaded with an unknown size or have infinite size. + + In most cases, however, when an instance of this class is created by helper + functions like 'from_folder', the size of the dataset will be preprocessed, + and the _size instance variable will be already set. + + Raises: + TypeError if self._size is not set and the cardinality of self._dataset + is INFINITE_CARDINALITY or UNKNOWN_CARDINALITY. + """ if self._size is not None: return self._size else: @@ -152,15 +165,25 @@ class Dataset(object): Returns: The splitted two sub datasets. + + Raises: + ValueError: if the provided fraction is not between 0 and 1. + ValueError: if this dataset does not have a set size. """ - assert (fraction > 0 and fraction < 1) + if not (fraction > 0 and fraction < 1): + raise ValueError(f'Fraction must be between 0 and 1. Got:{fraction}') + if not self._size: + raise ValueError( + 'Dataset size unknown. Cannot split the dataset when ' + 'the size is unknown.' + ) dataset = self._dataset train_size = int(self._size * fraction) - trainset = self.__class__(dataset.take(train_size), train_size, *args) + trainset = self.__class__(dataset.take(train_size), *args, size=train_size) test_size = self._size - train_size - testset = self.__class__(dataset.skip(train_size), test_size, *args) + testset = self.__class__(dataset.skip(train_size), *args, size=test_size) return trainset, testset diff --git a/mediapipe/model_maker/python/core/hyperparameters.py b/mediapipe/model_maker/python/core/hyperparameters.py index 22471655..92e1856c 100644 --- a/mediapipe/model_maker/python/core/hyperparameters.py +++ b/mediapipe/model_maker/python/core/hyperparameters.py @@ -15,7 +15,7 @@ import dataclasses import tempfile -from typing import Optional +from typing import Mapping, Optional import tensorflow as tf @@ -36,6 +36,8 @@ class BaseHParams: steps_per_epoch: An optional integer indicate the number of training steps per epoch. If not set, the training pipeline calculates the default steps per epoch as the training dataset size divided by batch size. + class_weights: An optional mapping of indices to weights for weighting the + loss function during training. shuffle: True if the dataset is shuffled before training. export_dir: The location of the model checkpoint files. distribution_strategy: A string specifying which Distribution Strategy to @@ -57,6 +59,7 @@ class BaseHParams: batch_size: int epochs: int steps_per_epoch: Optional[int] = None + class_weights: Optional[Mapping[int, float]] = None # Dataset-related parameters shuffle: bool = False diff --git a/mediapipe/model_maker/python/core/tasks/classifier.py b/mediapipe/model_maker/python/core/tasks/classifier.py index 60c00f0d..d504defb 100644 --- a/mediapipe/model_maker/python/core/tasks/classifier.py +++ b/mediapipe/model_maker/python/core/tasks/classifier.py @@ -43,7 +43,7 @@ class Classifier(custom_model.CustomModel): self._model: tf.keras.Model = None self._optimizer: Union[str, tf.keras.optimizers.Optimizer] = None self._loss_function: Union[str, tf.keras.losses.Loss] = None - self._metric_function: Union[str, tf.keras.metrics.Metric] = None + self._metric_functions: Sequence[Union[str, tf.keras.metrics.Metric]] = None self._callbacks: Sequence[tf.keras.callbacks.Callback] = None self._hparams: hp.BaseHParams = None self._history: tf.keras.callbacks.History = None @@ -92,7 +92,8 @@ class Classifier(custom_model.CustomModel): self._model.compile( optimizer=self._optimizer, loss=self._loss_function, - metrics=[self._metric_function]) + metrics=self._metric_functions, + ) latest_checkpoint = ( tf.train.latest_checkpoint(checkpoint_path) @@ -109,7 +110,9 @@ class Classifier(custom_model.CustomModel): # dataset is exhausted even if there are epochs remaining. steps_per_epoch=None, validation_data=validation_dataset, - callbacks=self._callbacks) + callbacks=self._callbacks, + class_weight=self._hparams.class_weights, + ) def evaluate(self, data: dataset.Dataset, batch_size: int = 32) -> Any: """Evaluates the classifier with the provided evaluation dataset. diff --git a/mediapipe/model_maker/python/core/utils/BUILD b/mediapipe/model_maker/python/core/utils/BUILD index ef9cab29..c5e03124 100644 --- a/mediapipe/model_maker/python/core/utils/BUILD +++ b/mediapipe/model_maker/python/core/utils/BUILD @@ -19,6 +19,13 @@ licenses(["notice"]) package(default_visibility = ["//mediapipe:__subpackages__"]) +filegroup( + name = "testdata", + srcs = glob([ + "testdata/**", + ]), +) + py_library( name = "test_util", testonly = 1, @@ -56,11 +63,26 @@ py_library( py_test( name = "file_util_test", srcs = ["file_util_test.py"], - data = ["//mediapipe/model_maker/python/core/utils/testdata"], + data = [":testdata"], tags = ["requires-net:external"], deps = [":file_util"], ) +py_library( + name = "hub_loader", + srcs = ["hub_loader.py"], +) + +py_test( + name = "hub_loader_test", + srcs = ["hub_loader_test.py"], + data = [":testdata"], + deps = [ + ":hub_loader", + "//mediapipe/tasks/python/test:test_utils", + ], +) + py_library( name = "loss_functions", srcs = ["loss_functions.py"], @@ -80,10 +102,30 @@ py_test( deps = [":loss_functions"], ) +###################################################################### +# Public target of the MediaPipe Model Maker Quantization Config. + +# Quantization Config is used to export a quantized model. Please refer +# to the specific task documentations such as: +# https://developers.google.com/mediapipe/solutions/vision/image_classifier/customize +# for usage information. +###################################################################### +py_library( + name = "metrics", + srcs = ["metrics.py"], +) + +py_test( + name = "metrics_test", + srcs = ["metrics_test.py"], + deps = [":metrics"], +) + py_library( name = "quantization", srcs = ["quantization.py"], srcs_version = "PY3", + visibility = ["//visibility:public"], deps = ["//mediapipe/model_maker/python/core/data:dataset"], ) diff --git a/mediapipe/model_maker/python/core/utils/hub_loader.py b/mediapipe/model_maker/python/core/utils/hub_loader.py new file mode 100644 index 00000000..a5209988 --- /dev/null +++ b/mediapipe/model_maker/python/core/utils/hub_loader.py @@ -0,0 +1,97 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Handles both V1 and V2 modules.""" + +import tensorflow_hub as hub + + +class HubKerasLayerV1V2(hub.KerasLayer): + """Class to loads TF v1 and TF v2 hub modules that could be fine-tuned. + + Since TF v1 modules couldn't be retrained in hub.KerasLayer. This class + provides a workaround for retraining the whole tf1 model in tf2. In + particular, it extract self._func._self_unconditional_checkpoint_dependencies + into trainable variable in tf1. + + Doesn't update moving-mean/moving-variance for BatchNormalization during + fine-tuning. + """ + + def _setup_layer(self, trainable=False, **kwargs): + if self._is_hub_module_v1: + self._setup_layer_v1(trainable, **kwargs) + else: + # call _setup_layer from the base class for v2. + super(HubKerasLayerV1V2, self)._setup_layer(trainable, **kwargs) + + def _check_trainability(self): + if self._is_hub_module_v1: + self._check_trainability_v1() + else: + # call _check_trainability from the base class for v2. + super(HubKerasLayerV1V2, self)._check_trainability() + + def _setup_layer_v1(self, trainable=False, **kwargs): + """Constructs keras layer with relevant weights and losses.""" + # Initialize an empty layer, then add_weight() etc. as needed. + super(hub.KerasLayer, self).__init__(trainable=trainable, **kwargs) + + if not self._is_hub_module_v1: + raise ValueError( + 'Only supports to set up v1 hub module in this function.' + ) + + # v2 trainable_variable: + if hasattr(self._func, 'trainable_variables'): + for v in self._func.trainable_variables: + self._add_existing_weight(v, trainable=True) + trainable_variables = {id(v) for v in self._func.trainable_variables} + else: + trainable_variables = set() + + if not hasattr(self._func, '_self_unconditional_checkpoint_dependencies'): + raise ValueError( + "_func doesn't contains attribute " + '_self_unconditional_checkpoint_dependencies.' + ) + dependencies = self._func._self_unconditional_checkpoint_dependencies # pylint: disable=protected-access + + # Adds trainable variables. + for dep in dependencies: + if dep.name == 'variables': + for v in dep.ref: + if id(v) not in trainable_variables: + self._add_existing_weight(v, trainable=True) + trainable_variables.add(id(v)) + + # Adds non-trainable variables. + if hasattr(self._func, 'variables'): + for v in self._func.variables: + if id(v) not in trainable_variables: + self._add_existing_weight(v, trainable=False) + + # Forward the callable's regularization losses (if any). + if hasattr(self._func, 'regularization_losses'): + for l in self._func.regularization_losses: + if not callable(l): + raise ValueError( + 'hub.KerasLayer(obj) expects obj.regularization_losses to be an ' + 'iterable of callables, each returning a scalar loss term.' + ) + self.add_loss(self._call_loss_if_trainable(l)) # Supports callables. + + def _check_trainability_v1(self): + """Ignores trainability checks for V1.""" + if self._is_hub_module_v1: + return # Nothing to do. diff --git a/mediapipe/model_maker/python/core/utils/hub_loader_test.py b/mediapipe/model_maker/python/core/utils/hub_loader_test.py new file mode 100644 index 00000000..8ea15b5d --- /dev/null +++ b/mediapipe/model_maker/python/core/utils/hub_loader_test.py @@ -0,0 +1,59 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from absl.testing import parameterized +import tensorflow as tf + +from mediapipe.model_maker.python.core.utils import hub_loader +from mediapipe.tasks.python.test import test_utils + + +class HubKerasLayerV1V2Test(tf.test.TestCase, parameterized.TestCase): + + @parameterized.parameters( + ("hub_module_v1_mini", True), + ("saved_model_v2_mini", True), + ("hub_module_v1_mini", False), + ("saved_model_v2_mini", False), + ) + def test_load_with_defaults(self, module_name, trainable): + inputs, expected_outputs = 10.0, 11.0 # Test modules perform increment op. + path = test_utils.get_test_data_path(module_name) + layer = hub_loader.HubKerasLayerV1V2(path, trainable=trainable) + output = layer(inputs) + self.assertEqual(output, expected_outputs) + + def test_trainable_variable(self): + path = test_utils.get_test_data_path("hub_module_v1_mini_train") + layer = hub_loader.HubKerasLayerV1V2(path, trainable=True) + # Checks trainable variables. + self.assertLen(layer.trainable_variables, 2) + self.assertEqual(layer.trainable_variables[0].name, "a:0") + self.assertEqual(layer.trainable_variables[1].name, "b:0") + self.assertEqual(layer.variables, layer.trainable_variables) + # Checks non-trainable variables. + self.assertEmpty(layer.non_trainable_variables) + + layer = hub_loader.HubKerasLayerV1V2(path, trainable=False) + # Checks trainable variables. + self.assertEmpty(layer.trainable_variables) + # Checks non-trainable variables. + self.assertLen(layer.non_trainable_variables, 2) + self.assertEqual(layer.non_trainable_variables[0].name, "a:0") + self.assertEqual(layer.non_trainable_variables[1].name, "b:0") + self.assertEqual(layer.variables, layer.non_trainable_variables) + + +if __name__ == "__main__": + tf.test.main() diff --git a/mediapipe/model_maker/python/core/utils/loss_functions.py b/mediapipe/model_maker/python/core/utils/loss_functions.py index 504ba91e..c741e428 100644 --- a/mediapipe/model_maker/python/core/utils/loss_functions.py +++ b/mediapipe/model_maker/python/core/utils/loss_functions.py @@ -59,7 +59,7 @@ class FocalLoss(tf.keras.losses.Loss): """ def __init__(self, gamma, class_weight: Optional[Sequence[float]] = None): - """Constructor. + """Initializes FocalLoss. Args: gamma: Focal loss gamma, as described in class docs. @@ -115,6 +115,51 @@ class FocalLoss(tf.keras.losses.Loss): return tf.reduce_sum(losses) / batch_size +class SparseFocalLoss(FocalLoss): + """Sparse implementation of Focal Loss. + + This is the same as FocalLoss, except the labels are expected to be class ids + instead of 1-hot encoded vectors. See FocalLoss class documentation defined + in this same file for more details. + + Example usage: + >>> y_true = [1, 2] + >>> y_pred = [[0.05, 0.95, 0], [0.1, 0.8, 0.1]] + >>> gamma = 2 + >>> focal_loss = SparseFocalLoss(gamma, 3) + >>> focal_loss(y_true, y_pred).numpy() + 0.9326 + + >>> # Calling with 'sample_weight'. + >>> focal_loss(y_true, y_pred, sample_weight=tf.constant([0.3, 0.7])).numpy() + 0.6528 + """ + + def __init__( + self, gamma, num_classes, class_weight: Optional[Sequence[float]] = None + ): + """Initializes SparseFocalLoss. + + Args: + gamma: Focal loss gamma, as described in class docs. + num_classes: Number of classes. + class_weight: A weight to apply to the loss, one for each class. The + weight is applied for each input where the ground truth label matches. + """ + super().__init__(gamma, class_weight=class_weight) + self._num_classes = num_classes + + def __call__( + self, + y_true: tf.Tensor, + y_pred: tf.Tensor, + sample_weight: Optional[tf.Tensor] = None, + ) -> tf.Tensor: + y_true = tf.cast(tf.reshape(y_true, [-1]), tf.int32) + y_true_one_hot = tf.one_hot(y_true, self._num_classes) + return super().__call__(y_true_one_hot, y_pred, sample_weight=sample_weight) + + @dataclasses.dataclass class PerceptualLossWeight: """The weight for each perceptual loss. diff --git a/mediapipe/model_maker/python/core/utils/loss_functions_test.py b/mediapipe/model_maker/python/core/utils/loss_functions_test.py index 01f9a667..3a14567e 100644 --- a/mediapipe/model_maker/python/core/utils/loss_functions_test.py +++ b/mediapipe/model_maker/python/core/utils/loss_functions_test.py @@ -101,6 +101,23 @@ class FocalLossTest(tf.test.TestCase, parameterized.TestCase): self.assertNear(loss, expected_loss, 1e-4) +class SparseFocalLossTest(tf.test.TestCase): + + def test_sparse_focal_loss_matches_focal_loss(self): + num_classes = 2 + y_pred = tf.constant([[0.8, 0.2], [0.3, 0.7]]) + y_true = tf.constant([1, 0]) + y_true_one_hot = tf.one_hot(y_true, num_classes) + for gamma in [0.0, 0.5, 1.0]: + expected_loss_fn = loss_functions.FocalLoss(gamma=gamma) + loss_fn = loss_functions.SparseFocalLoss( + gamma=gamma, num_classes=num_classes + ) + expected_loss = expected_loss_fn(y_true_one_hot, y_pred) + loss = loss_fn(y_true, y_pred) + self.assertNear(loss, expected_loss, 1e-4) + + class MockPerceptualLoss(loss_functions.PerceptualLoss): """A mock class with implementation of abstract methods for testing.""" diff --git a/mediapipe/model_maker/python/core/utils/metrics.py b/mediapipe/model_maker/python/core/utils/metrics.py new file mode 100644 index 00000000..31014616 --- /dev/null +++ b/mediapipe/model_maker/python/core/utils/metrics.py @@ -0,0 +1,104 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Metrics utility library.""" + +import tensorflow as tf + + +def _get_binary_sparse_metric(metric: tf.metrics.Metric): + """Helper method to create a BinarySparse version of a tf.keras.Metric. + + BinarySparse is an implementation where the update_state(y_true, y_pred) takes + in shapes y_true=(batch_size, 1) y_pred=(batch_size, 2). Note that this only + supports the binary classification case, and that class_id=0 is the negative + class and class_id=1 is the positive class. + + Currently supported tf.metric.Metric classes + 1. BinarySparseRecallAtPrecision + 2. BinarySparsePrecisionAtRecall + + Args: + metric: A tf.metric.Metric class for which we want to generate a + BinarySparse version of this metric. + + Returns: + A class for the BinarySparse version of the specified tf.metrics.Metric + """ + + class BinarySparseMetric(metric): + """A BinarySparse wrapper class for a tf.keras.Metric. + + This class has the same parameters and functions as the underlying + metric class. For example, the parameters for BinarySparseRecallAtPrecision + is the same as tf.keras.metrics.RecallAtPrecision. The only new constraint + is that class_id must be set to 1 (or not specified) for the Binary metric. + """ + + def __init__(self, *args, **kwargs): + if 'class_id' in kwargs and kwargs['class_id'] != 1: + raise ValueError( + f'Custom BinarySparseMetric for class:{metric.__name__} is ' + 'only supported for class_id=1, got class_id=' + f'{kwargs["class_id"]} instead' + ) + else: + kwargs['class_id'] = 1 + super().__init__(*args, **kwargs) + + def update_state(self, y_true, y_pred, sample_weight=None): + y_true = tf.cast(tf.reshape(y_true, [-1]), tf.int32) + y_true_one_hot = tf.one_hot(y_true, 2) + return super().update_state( + y_true_one_hot, y_pred, sample_weight=sample_weight + ) + + return BinarySparseMetric + + +def _get_sparse_metric(metric: tf.metrics.Metric): + """Helper method to create a Sparse version of a tf.keras.Metric. + + Sparse is an implementation where the update_state(y_true, y_pred) takes in + shapes y_true=(batch_size, 1) and y_pred=(batch_size, num_classes). + + Currently supported tf.metrics.Metric classes: + 1. tf.metrics.Recall + 2. tf.metrics.Precision + + Args: + metric: A tf.metric.Metric class for which we want to generate a Sparse + version of this metric. + + Returns: + A class for the Sparse version of the specified tf.keras.Metric. + """ + + class SparseMetric(metric): + """A Sparse wrapper class for a tf.keras.Metric.""" + + def update_state(self, y_true, y_pred, sample_weight=None): + y_pred = tf.math.argmax(y_pred, axis=-1) + return super().update_state(y_true, y_pred, sample_weight=sample_weight) + + return SparseMetric + + +SparseRecall = _get_sparse_metric(tf.metrics.Recall) +SparsePrecision = _get_sparse_metric(tf.metrics.Precision) +BinarySparseRecallAtPrecision = _get_binary_sparse_metric( + tf.metrics.RecallAtPrecision +) +BinarySparsePrecisionAtRecall = _get_binary_sparse_metric( + tf.metrics.PrecisionAtRecall +) diff --git a/mediapipe/model_maker/python/core/utils/metrics_test.py b/mediapipe/model_maker/python/core/utils/metrics_test.py new file mode 100644 index 00000000..84233527 --- /dev/null +++ b/mediapipe/model_maker/python/core/utils/metrics_test.py @@ -0,0 +1,74 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from absl.testing import parameterized +import tensorflow as tf + +from mediapipe.model_maker.python.core.utils import metrics + + +class SparseMetricTest(tf.test.TestCase, parameterized.TestCase): + + def setUp(self): + super().setUp() + self.y_true = [0, 0, 1, 1, 0, 1] + self.y_pred = [ + [0.9, 0.1], # 0, 0 y + [0.8, 0.2], # 0, 0 y + [0.7, 0.3], # 0, 1 n + [0.6, 0.4], # 0, 1 n + [0.3, 0.7], # 1, 0 y + [0.3, 0.7], # 1, 1 y + ] + self.num_classes = 3 + + def _assert_metric_equals(self, metric, value): + metric.update_state(self.y_true, self.y_pred) + self.assertEqual(metric.result(), value) + + def test_sparse_recall(self): + metric = metrics.SparseRecall() + self._assert_metric_equals(metric, 1 / 3) + + def test_sparse_precision(self): + metric = metrics.SparsePrecision() + self._assert_metric_equals(metric, 1 / 2) + + def test_binary_sparse_recall_at_precision(self): + metric = metrics.BinarySparseRecallAtPrecision(1.0) + self._assert_metric_equals(metric, 0.0) # impossible to achieve precision=1 + metric = metrics.BinarySparseRecallAtPrecision(0.4) + self._assert_metric_equals(metric, 1.0) + + def test_binary_sparse_precision_at_recall(self): + metric = metrics.BinarySparsePrecisionAtRecall(1.0) + self._assert_metric_equals(metric, 3 / 4) + metric = metrics.BinarySparsePrecisionAtRecall(0.7) + self._assert_metric_equals(metric, 3 / 4) + + def test_binary_sparse_precision_at_recall_class_id_error(self): + # class_id=1 case should not error + _ = metrics.BinarySparsePrecisionAtRecall(1.0, class_id=1) + # class_id=2 case should error + with self.assertRaisesRegex( + ValueError, + 'Custom BinarySparseMetric for class:PrecisionAtRecall is only' + ' supported for class_id=1, got class_id=2 instead', + ): + _ = metrics.BinarySparsePrecisionAtRecall(1.0, class_id=2) + + +if __name__ == '__main__': + tf.test.main() diff --git a/mediapipe/model_maker/python/core/utils/model_util.py b/mediapipe/model_maker/python/core/utils/model_util.py index 5ca2c2b7..fd11c60b 100644 --- a/mediapipe/model_maker/python/core/utils/model_util.py +++ b/mediapipe/model_maker/python/core/utils/model_util.py @@ -112,6 +112,39 @@ def get_steps_per_epoch(steps_per_epoch: Optional[int] = None, return len(train_data) // batch_size +def convert_to_tflite_from_file( + saved_model_file: str, + quantization_config: Optional[quantization.QuantizationConfig] = None, + supported_ops: Tuple[tf.lite.OpsSet, ...] = ( + tf.lite.OpsSet.TFLITE_BUILTINS, + ), + preprocess: Optional[Callable[..., Any]] = None, +) -> bytearray: + """Converts the input Keras model to TFLite format. + + Args: + saved_model_file: Keras model to be converted to TFLite. + quantization_config: Configuration for post-training quantization. + supported_ops: A list of supported ops in the converted TFLite file. + preprocess: A callable to preprocess the representative dataset for + quantization. The callable takes three arguments in order: feature, label, + and is_training. + + Returns: + bytearray of TFLite model + """ + converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_file) + + if quantization_config: + converter = quantization_config.set_converter_with_quantization( + converter, preprocess=preprocess + ) + + converter.target_spec.supported_ops = supported_ops + tflite_model = converter.convert() + return tflite_model + + def convert_to_tflite( model: tf.keras.Model, quantization_config: Optional[quantization.QuantizationConfig] = None, @@ -135,16 +168,14 @@ def convert_to_tflite( """ with tempfile.TemporaryDirectory() as temp_dir: save_path = os.path.join(temp_dir, 'saved_model') - model.save(save_path, include_optimizer=False, save_format='tf') - converter = tf.lite.TFLiteConverter.from_saved_model(save_path) - - if quantization_config: - converter = quantization_config.set_converter_with_quantization( - converter, preprocess=preprocess) - - converter.target_spec.supported_ops = supported_ops - tflite_model = converter.convert() - return tflite_model + model.save( + save_path, + include_optimizer=False, + save_format='tf', + ) + return convert_to_tflite_from_file( + save_path, quantization_config, supported_ops, preprocess + ) def save_tflite(tflite_model: bytearray, tflite_file: str) -> None: diff --git a/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini/saved_model.pb b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini/saved_model.pb new file mode 100644 index 00000000..e60e04a2 Binary files /dev/null and b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini/saved_model.pb differ diff --git a/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini/tfhub_module.pb b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini/tfhub_module.pb new file mode 100644 index 00000000..d65dd8f1 --- /dev/null +++ b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini/tfhub_module.pb @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/saved_model.pb b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/saved_model.pb new file mode 100644 index 00000000..69519fef Binary files /dev/null and b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/saved_model.pb differ diff --git a/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/tfhub_module.pb b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/tfhub_module.pb new file mode 100644 index 00000000..d65dd8f1 --- /dev/null +++ b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/tfhub_module.pb @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/variables/variables.data-00000-of-00001 b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/variables/variables.data-00000-of-00001 new file mode 100644 index 00000000..3474955e --- /dev/null +++ b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/variables/variables.data-00000-of-00001 @@ -0,0 +1,2 @@ +øÌû¾âì¿ + diff --git a/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/variables/variables.index b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/variables/variables.index new file mode 100644 index 00000000..d0e35ab8 Binary files /dev/null and b/mediapipe/model_maker/python/core/utils/testdata/hub_module_v1_mini_train/variables/variables.index differ diff --git a/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/saved_model.pb b/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/saved_model.pb new file mode 100644 index 00000000..314ea74f Binary files /dev/null and b/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/saved_model.pb differ diff --git a/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/variables/variables.data-00000-of-00001 b/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/variables/variables.data-00000-of-00001 new file mode 100644 index 00000000..09dbb330 Binary files /dev/null and b/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/variables/variables.data-00000-of-00001 differ diff --git a/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/variables/variables.index b/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/variables/variables.index new file mode 100644 index 00000000..7cfb9ffd Binary files /dev/null and b/mediapipe/model_maker/python/core/utils/testdata/saved_model_v2_mini/variables/variables.index differ diff --git a/mediapipe/model_maker/python/text/core/bert_model_spec.py b/mediapipe/model_maker/python/text/core/bert_model_spec.py index 792c2c9a..4a847ac3 100644 --- a/mediapipe/model_maker/python/text/core/bert_model_spec.py +++ b/mediapipe/model_maker/python/text/core/bert_model_spec.py @@ -14,7 +14,7 @@ """Specification for a BERT model.""" import dataclasses -from typing import Dict +from typing import Dict, Union from mediapipe.model_maker.python.core import hyperparameters as hp from mediapipe.model_maker.python.core.utils import file_util @@ -35,7 +35,9 @@ class BertModelSpec: Transformers for Language Understanding) for more details. Attributes: - downloaded_files: A DownloadedFiles object of the model files + files: Either a TFHub url string which can be passed directly to + hub.KerasLayer or a DownloadedFiles object of the model files. + is_tf2: If True, the checkpoint is TF2 format. Else use TF1 format. hparams: Hyperparameters used for training. model_options: Configurable options for a BERT model. do_lower_case: boolean, whether to lower case the input text. Should be @@ -45,15 +47,28 @@ class BertModelSpec: name: The name of the object. """ - downloaded_files: file_util.DownloadedFiles - hparams: hp.BaseHParams = hp.BaseHParams( - epochs=3, - batch_size=32, - learning_rate=3e-5, - distribution_strategy='mirrored') - model_options: bert_model_options.BertModelOptions = ( - bert_model_options.BertModelOptions()) + files: Union[str, file_util.DownloadedFiles] + is_tf2: bool = True + hparams: hp.BaseHParams = dataclasses.field( + default_factory=lambda: hp.BaseHParams( + epochs=3, + batch_size=32, + learning_rate=3e-5, + distribution_strategy='mirrored', + ) + ) + model_options: bert_model_options.BertModelOptions = dataclasses.field( + default_factory=bert_model_options.BertModelOptions + ) do_lower_case: bool = True tflite_input_name: Dict[str, str] = dataclasses.field( default_factory=lambda: _DEFAULT_TFLITE_INPUT_NAME) name: str = 'Bert' + + def get_path(self) -> str: + if isinstance(self.files, file_util.DownloadedFiles): + return self.files.get_path() + elif isinstance(self.files, str): + return self.files + else: + raise ValueError(f'files has unsupported type: {type(self.files)}') diff --git a/mediapipe/model_maker/python/text/text_classifier/BUILD b/mediapipe/model_maker/python/text/text_classifier/BUILD index 9fe96849..e32733e3 100644 --- a/mediapipe/model_maker/python/text/text_classifier/BUILD +++ b/mediapipe/model_maker/python/text/text_classifier/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary and library compatibility macro. # Placeholder for internal Python strict test compatibility macro. package(default_visibility = ["//mediapipe:__subpackages__"]) @@ -31,11 +31,11 @@ py_library( visibility = ["//visibility:public"], deps = [ ":dataset", + ":hyperparameters", ":model_options", ":model_spec", ":text_classifier", ":text_classifier_options", - "//mediapipe/model_maker/python/core:hyperparameters", ], ) @@ -45,12 +45,18 @@ py_library( deps = ["//mediapipe/model_maker/python/text/core:bert_model_options"], ) +py_library( + name = "hyperparameters", + srcs = ["hyperparameters.py"], + deps = ["//mediapipe/model_maker/python/core:hyperparameters"], +) + py_library( name = "model_spec", srcs = ["model_spec.py"], deps = [ + ":hyperparameters", ":model_options", - "//mediapipe/model_maker/python/core:hyperparameters", "//mediapipe/model_maker/python/core/utils:file_util", "//mediapipe/model_maker/python/text/core:bert_model_spec", ], @@ -61,16 +67,19 @@ py_test( srcs = ["model_spec_test.py"], tags = ["requires-net:external"], deps = [ + ":hyperparameters", ":model_options", ":model_spec", - "//mediapipe/model_maker/python/core:hyperparameters", ], ) py_library( name = "dataset", srcs = ["dataset.py"], - deps = ["//mediapipe/model_maker/python/core/data:classification_dataset"], + deps = [ + "//mediapipe/model_maker/python/core/data:cache_files", + "//mediapipe/model_maker/python/core/data:classification_dataset", + ], ) py_test( @@ -82,7 +91,10 @@ py_test( py_library( name = "preprocessor", srcs = ["preprocessor.py"], - deps = [":dataset"], + deps = [ + ":dataset", + "//mediapipe/model_maker/python/core/data:cache_files", + ], ) py_test( @@ -93,6 +105,7 @@ py_test( ":dataset", ":model_spec", ":preprocessor", + "//mediapipe/model_maker/python/core/data:cache_files", ], ) @@ -100,9 +113,9 @@ py_library( name = "text_classifier_options", srcs = ["text_classifier_options.py"], deps = [ + ":hyperparameters", ":model_options", ":model_spec", - "//mediapipe/model_maker/python/core:hyperparameters", ], ) @@ -111,13 +124,16 @@ py_library( srcs = ["text_classifier.py"], deps = [ ":dataset", + ":hyperparameters", ":model_options", ":model_spec", ":preprocessor", ":text_classifier_options", - "//mediapipe/model_maker/python/core:hyperparameters", "//mediapipe/model_maker/python/core/data:dataset", "//mediapipe/model_maker/python/core/tasks:classifier", + "//mediapipe/model_maker/python/core/utils:hub_loader", + "//mediapipe/model_maker/python/core/utils:loss_functions", + "//mediapipe/model_maker/python/core/utils:metrics", "//mediapipe/model_maker/python/core/utils:model_util", "//mediapipe/model_maker/python/core/utils:quantization", "//mediapipe/tasks/python/metadata/metadata_writers:metadata_writer", @@ -140,6 +156,7 @@ py_test( ], deps = [ ":text_classifier_import", + "//mediapipe/model_maker/python/core/utils:loss_functions", "//mediapipe/tasks/python/test:test_utils", ], ) diff --git a/mediapipe/model_maker/python/text/text_classifier/__init__.py b/mediapipe/model_maker/python/text/text_classifier/__init__.py index 4df3a771..7eb0f925 100644 --- a/mediapipe/model_maker/python/text/text_classifier/__init__.py +++ b/mediapipe/model_maker/python/text/text_classifier/__init__.py @@ -13,19 +13,23 @@ # limitations under the License. """MediaPipe Public Python API for Text Classifier.""" -from mediapipe.model_maker.python.core import hyperparameters from mediapipe.model_maker.python.text.text_classifier import dataset +from mediapipe.model_maker.python.text.text_classifier import hyperparameters from mediapipe.model_maker.python.text.text_classifier import model_options from mediapipe.model_maker.python.text.text_classifier import model_spec from mediapipe.model_maker.python.text.text_classifier import text_classifier from mediapipe.model_maker.python.text.text_classifier import text_classifier_options -HParams = hyperparameters.BaseHParams + +AverageWordEmbeddingHParams = hyperparameters.AverageWordEmbeddingHParams +AverageWordEmbeddingModelOptions = ( + model_options.AverageWordEmbeddingModelOptions +) +BertOptimizer = hyperparameters.BertOptimizer +BertHParams = hyperparameters.BertHParams +BertModelOptions = model_options.BertModelOptions CSVParams = dataset.CSVParameters Dataset = dataset.Dataset -AverageWordEmbeddingModelOptions = ( - model_options.AverageWordEmbeddingModelOptions) -BertModelOptions = model_options.BertModelOptions SupportedModels = model_spec.SupportedModels TextClassifier = text_classifier.TextClassifier TextClassifierOptions = text_classifier_options.TextClassifierOptions diff --git a/mediapipe/model_maker/python/text/text_classifier/dataset.py b/mediapipe/model_maker/python/text/text_classifier/dataset.py index 63605b47..1f8798df 100644 --- a/mediapipe/model_maker/python/text/text_classifier/dataset.py +++ b/mediapipe/model_maker/python/text/text_classifier/dataset.py @@ -15,11 +15,15 @@ import csv import dataclasses +import hashlib +import os import random +import tempfile +from typing import List, Optional, Sequence -from typing import Optional, Sequence import tensorflow as tf +from mediapipe.model_maker.python.core.data import cache_files as cache_files_lib from mediapipe.model_maker.python.core.data import classification_dataset @@ -46,21 +50,49 @@ class CSVParameters: class Dataset(classification_dataset.ClassificationDataset): """Dataset library for text classifier.""" + def __init__( + self, + dataset: tf.data.Dataset, + label_names: List[str], + tfrecord_cache_files: Optional[cache_files_lib.TFRecordCacheFiles] = None, + size: Optional[int] = None, + ): + super().__init__(dataset, label_names, size) + if not tfrecord_cache_files: + tfrecord_cache_files = cache_files_lib.TFRecordCacheFiles( + cache_prefix_filename="tfrecord", num_shards=1 + ) + self.tfrecord_cache_files = tfrecord_cache_files + @classmethod - def from_csv(cls, - filename: str, - csv_params: CSVParameters, - shuffle: bool = True) -> "Dataset": + def from_csv( + cls, + filename: str, + csv_params: CSVParameters, + shuffle: bool = True, + cache_dir: Optional[str] = None, + num_shards: int = 1, + ) -> "Dataset": """Loads text with labels from a CSV file. Args: filename: Name of the CSV file. csv_params: Parameters used for reading the CSV file. shuffle: If True, randomly shuffle the data. + cache_dir: Optional parameter to specify where to store the preprocessed + dataset. Only used for BERT models. + num_shards: Optional parameter for num shards of the preprocessed dataset. + Note that using more than 1 shard will reorder the dataset. Only used + for BERT models. Returns: Dataset containing (text, label) pairs and other related info. """ + if cache_dir is None: + cache_dir = tempfile.mkdtemp() + # calculate hash for cache based off of files + hasher = hashlib.md5() + hasher.update(os.path.basename(filename).encode("utf-8")) with tf.io.gfile.GFile(filename, "r") as f: reader = csv.DictReader( f, @@ -69,6 +101,9 @@ class Dataset(classification_dataset.ClassificationDataset): quotechar=csv_params.quotechar) lines = list(reader) + for line in lines: + hasher.update(str(line).encode("utf-8")) + if shuffle: random.shuffle(lines) @@ -81,8 +116,18 @@ class Dataset(classification_dataset.ClassificationDataset): index_by_label[line[csv_params.label_column]] for line in lines ] label_index_ds = tf.data.Dataset.from_tensor_slices( - tf.cast(label_indices, tf.int64)) + tf.cast(label_indices, tf.int64) + ) text_label_ds = tf.data.Dataset.zip((text_ds, label_index_ds)) + hasher.update(str(num_shards).encode("utf-8")) + cache_prefix_filename = hasher.hexdigest() + tfrecord_cache_files = cache_files_lib.TFRecordCacheFiles( + cache_prefix_filename, cache_dir, num_shards + ) return Dataset( - dataset=text_label_ds, size=len(texts), label_names=label_names) + dataset=text_label_ds, + label_names=label_names, + tfrecord_cache_files=tfrecord_cache_files, + size=len(texts), + ) diff --git a/mediapipe/model_maker/python/text/text_classifier/dataset_test.py b/mediapipe/model_maker/python/text/text_classifier/dataset_test.py index 012476e0..2fa90b86 100644 --- a/mediapipe/model_maker/python/text/text_classifier/dataset_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/dataset_test.py @@ -53,7 +53,7 @@ class DatasetTest(tf.test.TestCase): def test_split(self): ds = tf.data.Dataset.from_tensor_slices(['good', 'bad', 'neutral', 'odd']) - data = dataset.Dataset(ds, 4, ['pos', 'neg']) + data = dataset.Dataset(ds, ['pos', 'neg'], size=4) train_data, test_data = data.split(0.5) expected_train_data = [b'good', b'bad'] expected_test_data = [b'neutral', b'odd'] diff --git a/mediapipe/model_maker/python/text/text_classifier/hyperparameters.py b/mediapipe/model_maker/python/text/text_classifier/hyperparameters.py new file mode 100644 index 00000000..71470edb --- /dev/null +++ b/mediapipe/model_maker/python/text/text_classifier/hyperparameters.py @@ -0,0 +1,72 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Hyperparameters for training object detection models.""" + +import dataclasses +import enum +from typing import Sequence, Union + +from mediapipe.model_maker.python.core import hyperparameters as hp + + +@dataclasses.dataclass +class AverageWordEmbeddingHParams(hp.BaseHParams): + """The hyperparameters for an AverageWordEmbeddingClassifier.""" + + +@enum.unique +class BertOptimizer(enum.Enum): + """Supported Optimizers for Bert Text Classifier.""" + + ADAMW = "adamw" + LAMB = "lamb" + + +@dataclasses.dataclass +class BertHParams(hp.BaseHParams): + """The hyperparameters for a Bert Classifier. + + Attributes: + learning_rate: Learning rate to use for gradient descent training. + end_learning_rate: End learning rate for linear decay. Defaults to 0. + batch_size: Batch size for training. Defaults to 48. + epochs: Number of training iterations over the dataset. Defaults to 2. + optimizer: Optimizer to use for training. Supported values are defined in + BertOptimizer enum: ADAMW and LAMB. + weight_decay: Weight decay of the optimizer. Defaults to 0.01. + desired_precisions: If specified, adds a RecallAtPrecision metric per + desired_precisions[i] entry which tracks the recall given the constraint + on precision. Only supported for binary classification. + desired_recalls: If specified, adds a PrecisionAtRecall metric per + desired_recalls[i] entry which tracks the precision given the constraint + on recall. Only supported for binary classification. + gamma: Gamma parameter for focal loss. To use cross entropy loss, set this + value to 0. Defaults to 2.0. + """ + + learning_rate: float = 3e-5 + end_learning_rate: float = 0.0 + + batch_size: int = 48 + epochs: int = 2 + optimizer: BertOptimizer = BertOptimizer.ADAMW + weight_decay: float = 0.01 + + desired_precisions: Sequence[float] = dataclasses.field(default_factory=list) + desired_recalls: Sequence[float] = dataclasses.field(default_factory=list) + + gamma: float = 2.0 + + +HParams = Union[BertHParams, AverageWordEmbeddingHParams] diff --git a/mediapipe/model_maker/python/text/text_classifier/model_spec.py b/mediapipe/model_maker/python/text/text_classifier/model_spec.py index e947f8c1..01d1432c 100644 --- a/mediapipe/model_maker/python/text/text_classifier/model_spec.py +++ b/mediapipe/model_maker/python/text/text_classifier/model_spec.py @@ -17,18 +17,14 @@ import dataclasses import enum import functools -from mediapipe.model_maker.python.core import hyperparameters as hp from mediapipe.model_maker.python.core.utils import file_util from mediapipe.model_maker.python.text.core import bert_model_spec +from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp from mediapipe.model_maker.python.text.text_classifier import model_options as mo -# BERT-based text classifier spec inherited from BertModelSpec -BertClassifierSpec = bert_model_spec.BertModelSpec -MOBILEBERT_TINY_FILES = file_util.DownloadedFiles( - 'text_classifier/mobilebert_tiny', - 'https://storage.googleapis.com/mediapipe-assets/mobilebert_tiny.tar.gz', - is_folder=True, +MOBILEBERT_FILES = ( + 'https://tfhub.dev/google/mobilebert/uncased_L-24_H-128_B-512_A-4_F-4_OPT/1' ) @@ -43,28 +39,38 @@ class AverageWordEmbeddingClassifierSpec: """ # `learning_rate` is unused for the average word embedding model - hparams: hp.BaseHParams = hp.BaseHParams( - epochs=10, batch_size=32, learning_rate=0) - model_options: mo.AverageWordEmbeddingModelOptions = ( - mo.AverageWordEmbeddingModelOptions()) + hparams: hp.AverageWordEmbeddingHParams = dataclasses.field( + default_factory=lambda: hp.AverageWordEmbeddingHParams( + epochs=10, batch_size=32, learning_rate=0 + ) + ) + model_options: mo.AverageWordEmbeddingModelOptions = dataclasses.field( + default_factory=mo.AverageWordEmbeddingModelOptions + ) name: str = 'AverageWordEmbedding' - average_word_embedding_classifier_spec = functools.partial( AverageWordEmbeddingClassifierSpec) + +@dataclasses.dataclass +class BertClassifierSpec(bert_model_spec.BertModelSpec): + """Specification for a Bert classifier model. + + Only overrides the hparams attribute since the rest of the attributes are + inherited from the BertModelSpec. + """ + + hparams: hp.BertHParams = dataclasses.field(default_factory=hp.BertHParams) + mobilebert_classifier_spec = functools.partial( BertClassifierSpec, - downloaded_files=MOBILEBERT_TINY_FILES, - hparams=hp.BaseHParams( + files=MOBILEBERT_FILES, + hparams=hp.BertHParams( epochs=3, batch_size=48, learning_rate=3e-5, distribution_strategy='off' ), - name='MobileBert', - tflite_input_name={ - 'ids': 'serving_default_input_1:0', - 'mask': 'serving_default_input_3:0', - 'segment_ids': 'serving_default_input_2:0', - }, + name='MobileBERT', + is_tf2=False, ) diff --git a/mediapipe/model_maker/python/text/text_classifier/model_spec_test.py b/mediapipe/model_maker/python/text/text_classifier/model_spec_test.py index a8d40558..d1e578b8 100644 --- a/mediapipe/model_maker/python/text/text_classifier/model_spec_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/model_spec_test.py @@ -19,7 +19,7 @@ from unittest import mock as unittest_mock import tensorflow as tf -from mediapipe.model_maker.python.core import hyperparameters as hp +from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp from mediapipe.model_maker.python.text.text_classifier import model_options as classifier_model_options from mediapipe.model_maker.python.text.text_classifier import model_spec as ms @@ -42,26 +42,30 @@ class ModelSpecTest(tf.test.TestCase): def test_predefined_bert_spec(self): model_spec_obj = ms.SupportedModels.MOBILEBERT_CLASSIFIER.value() self.assertIsInstance(model_spec_obj, ms.BertClassifierSpec) - self.assertEqual(model_spec_obj.name, 'MobileBert') - self.assertTrue(os.path.exists(model_spec_obj.downloaded_files.get_path())) + self.assertEqual(model_spec_obj.name, 'MobileBERT') + self.assertTrue(model_spec_obj.files) self.assertTrue(model_spec_obj.do_lower_case) self.assertEqual( - model_spec_obj.tflite_input_name, { - 'ids': 'serving_default_input_1:0', - 'mask': 'serving_default_input_3:0', - 'segment_ids': 'serving_default_input_2:0' - }) + model_spec_obj.tflite_input_name, + { + 'ids': 'serving_default_input_word_ids:0', + 'mask': 'serving_default_input_mask:0', + 'segment_ids': 'serving_default_input_type_ids:0', + }, + ) self.assertEqual( model_spec_obj.model_options, classifier_model_options.BertModelOptions( seq_len=128, do_fine_tuning=True, dropout_rate=0.1)) self.assertEqual( model_spec_obj.hparams, - hp.BaseHParams( + hp.BertHParams( epochs=3, batch_size=48, learning_rate=3e-5, - distribution_strategy='off')) + distribution_strategy='off', + ), + ) def test_predefined_average_word_embedding_spec(self): model_spec_obj = ( @@ -78,7 +82,7 @@ class ModelSpecTest(tf.test.TestCase): dropout_rate=0.2)) self.assertEqual( model_spec_obj.hparams, - hp.BaseHParams( + hp.AverageWordEmbeddingHParams( epochs=10, batch_size=32, learning_rate=0, @@ -101,7 +105,7 @@ class ModelSpecTest(tf.test.TestCase): custom_bert_classifier_options) def test_custom_average_word_embedding_spec(self): - custom_hparams = hp.BaseHParams( + custom_hparams = hp.AverageWordEmbeddingHParams( learning_rate=0.4, batch_size=64, epochs=10, @@ -110,7 +114,8 @@ class ModelSpecTest(tf.test.TestCase): export_dir='foo/bar', distribution_strategy='mirrored', num_gpus=3, - tpu='tpu/address') + tpu='tpu/address', + ) custom_average_word_embedding_model_options = ( classifier_model_options.AverageWordEmbeddingModelOptions( seq_len=512, diff --git a/mediapipe/model_maker/python/text/text_classifier/preprocessor.py b/mediapipe/model_maker/python/text/text_classifier/preprocessor.py index 15b9d90d..68a5df2f 100644 --- a/mediapipe/model_maker/python/text/text_classifier/preprocessor.py +++ b/mediapipe/model_maker/python/text/text_classifier/preprocessor.py @@ -15,16 +15,16 @@ """Preprocessors for text classification.""" import collections +import hashlib import os import re -import tempfile from typing import Mapping, Sequence, Tuple, Union import tensorflow as tf import tensorflow_hub +from mediapipe.model_maker.python.core.data import cache_files as cache_files_lib from mediapipe.model_maker.python.text.text_classifier import dataset as text_classifier_ds -from official.nlp.data import classifier_data_lib from official.nlp.tools import tokenization @@ -75,19 +75,20 @@ def _decode_record( return bert_features, example["label_ids"] -def _single_file_dataset( - input_file: str, name_to_features: Mapping[str, tf.io.FixedLenFeature] +def _tfrecord_dataset( + tfrecord_files: Sequence[str], + name_to_features: Mapping[str, tf.io.FixedLenFeature], ) -> tf.data.TFRecordDataset: """Creates a single-file dataset to be passed for BERT custom training. Args: - input_file: Filepath for the dataset. + tfrecord_files: Filepaths for the dataset. name_to_features: Maps record keys to feature types. Returns: Dataset containing BERT model input features and labels. """ - d = tf.data.TFRecordDataset(input_file) + d = tf.data.TFRecordDataset(tfrecord_files) d = d.map( lambda record: _decode_record(record, name_to_features), num_parallel_calls=tf.data.AUTOTUNE) @@ -221,15 +222,23 @@ class BertClassifierPreprocessor: seq_len: Length of the input sequence to the model. vocab_file: File containing the BERT vocab. tokenizer: BERT tokenizer. + model_name: Name of the model provided by the model_spec. Used to associate + cached files with specific Bert model vocab. """ - def __init__(self, seq_len: int, do_lower_case: bool, uri: str): + def __init__( + self, seq_len: int, do_lower_case: bool, uri: str, model_name: str + ): self._seq_len = seq_len # Vocab filepath is tied to the BERT module's URI. self._vocab_file = os.path.join( - tensorflow_hub.resolve(uri), "assets", "vocab.txt") - self._tokenizer = tokenization.FullTokenizer(self._vocab_file, - do_lower_case) + tensorflow_hub.resolve(uri), "assets", "vocab.txt" + ) + self._do_lower_case = do_lower_case + self._tokenizer = tokenization.FullTokenizer( + self._vocab_file, self._do_lower_case + ) + self._model_name = model_name def _get_name_to_features(self): """Gets the dictionary mapping record keys to feature types.""" @@ -244,8 +253,62 @@ class BertClassifierPreprocessor: """Returns the vocab file of the BertClassifierPreprocessor.""" return self._vocab_file + def _get_tfrecord_cache_files( + self, ds_cache_files + ) -> cache_files_lib.TFRecordCacheFiles: + """Helper to regenerate cache prefix filename using preprocessor info. + + We need to update the dataset cache_prefix cache because the actual cached + dataset depends on the preprocessor parameters such as model_name, seq_len, + and do_lower_case in addition to the raw dataset parameters which is already + included in the ds_cache_files.cache_prefix_filename + + Specifically, the new cache_prefix_filename used by the preprocessor will + be a hash generated from the following: + 1. cache_prefix_filename of the initial raw dataset + 2. model_name + 3. seq_len + 4. do_lower_case + + Args: + ds_cache_files: TFRecordCacheFiles from the original raw dataset object + + Returns: + A new TFRecordCacheFiles object which incorporates the preprocessor + parameters. + """ + hasher = hashlib.md5() + hasher.update(ds_cache_files.cache_prefix_filename.encode("utf-8")) + hasher.update(self._model_name.encode("utf-8")) + hasher.update(str(self._seq_len).encode("utf-8")) + hasher.update(str(self._do_lower_case).encode("utf-8")) + cache_prefix_filename = hasher.hexdigest() + return cache_files_lib.TFRecordCacheFiles( + cache_prefix_filename, + ds_cache_files.cache_dir, + ds_cache_files.num_shards, + ) + + def _process_bert_features(self, text: str) -> Mapping[str, Sequence[int]]: + tokens = self._tokenizer.tokenize(text) + tokens = tokens[0 : (self._seq_len - 2)] # account for [CLS] and [SEP] + tokens.insert(0, "[CLS]") + tokens.append("[SEP]") + input_ids = self._tokenizer.convert_tokens_to_ids(tokens) + input_mask = [1] * len(input_ids) + while len(input_ids) < self._seq_len: + input_ids.append(0) + input_mask.append(0) + segment_ids = [0] * self._seq_len + return { + "input_ids": input_ids, + "input_mask": input_mask, + "segment_ids": segment_ids, + } + def preprocess( - self, dataset: text_classifier_ds.Dataset) -> text_classifier_ds.Dataset: + self, dataset: text_classifier_ds.Dataset + ) -> text_classifier_ds.Dataset: """Preprocesses data into input for a BERT-based classifier. Args: @@ -254,32 +317,54 @@ class BertClassifierPreprocessor: Returns: Dataset containing (bert_features, label) data. """ - examples = [] - for index, (text, label) in enumerate(dataset.gen_tf_dataset()): - _validate_text_and_label(text, label) - examples.append( - classifier_data_lib.InputExample( - guid=str(index), - text_a=text.numpy()[0].decode("utf-8"), - text_b=None, - # InputExample expects the label name rather than the int ID - label=dataset.label_names[label.numpy()[0]])) + ds_cache_files = dataset.tfrecord_cache_files + # Get new tfrecord_cache_files by including preprocessor information. + tfrecord_cache_files = self._get_tfrecord_cache_files(ds_cache_files) + if not tfrecord_cache_files.is_cached(): + print(f"Writing new cache files to {tfrecord_cache_files.cache_prefix}") + writers = tfrecord_cache_files.get_writers() + size = 0 + for index, (text, label) in enumerate(dataset.gen_tf_dataset()): + _validate_text_and_label(text, label) + feature = self._process_bert_features(text.numpy()[0].decode("utf-8")) + def create_int_feature(values): + f = tf.train.Feature( + int64_list=tf.train.Int64List(value=list(values)) + ) + return f - tfrecord_file = os.path.join(tempfile.mkdtemp(), "bert_features.tfrecord") - classifier_data_lib.file_based_convert_examples_to_features( - examples=examples, - label_list=dataset.label_names, - max_seq_length=self._seq_len, - tokenizer=self._tokenizer, - output_file=tfrecord_file) - preprocessed_ds = _single_file_dataset(tfrecord_file, - self._get_name_to_features()) + features = collections.OrderedDict() + features["input_ids"] = create_int_feature(feature["input_ids"]) + features["input_mask"] = create_int_feature(feature["input_mask"]) + features["segment_ids"] = create_int_feature(feature["segment_ids"]) + features["label_ids"] = create_int_feature([label.numpy()[0]]) + tf_example = tf.train.Example( + features=tf.train.Features(feature=features) + ) + writers[index % len(writers)].write(tf_example.SerializeToString()) + size = index + 1 + for writer in writers: + writer.close() + metadata = {"size": size, "label_names": dataset.label_names} + tfrecord_cache_files.save_metadata(metadata) + else: + print( + f"Using existing cache files at {tfrecord_cache_files.cache_prefix}" + ) + metadata = tfrecord_cache_files.load_metadata() + size = metadata["size"] + label_names = metadata["label_names"] + preprocessed_ds = _tfrecord_dataset( + tfrecord_cache_files.tfrecord_files, self._get_name_to_features() + ) return text_classifier_ds.Dataset( dataset=preprocessed_ds, - size=dataset.size, - label_names=dataset.label_names) + size=size, + label_names=label_names, + tfrecord_cache_files=tfrecord_cache_files, + ) -TextClassifierPreprocessor = ( - Union[BertClassifierPreprocessor, - AverageWordEmbeddingClassifierPreprocessor]) +TextClassifierPreprocessor = Union[ + BertClassifierPreprocessor, AverageWordEmbeddingClassifierPreprocessor +] diff --git a/mediapipe/model_maker/python/text/text_classifier/preprocessor_test.py b/mediapipe/model_maker/python/text/text_classifier/preprocessor_test.py index 27e98e26..ff901549 100644 --- a/mediapipe/model_maker/python/text/text_classifier/preprocessor_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/preprocessor_test.py @@ -13,14 +13,17 @@ # limitations under the License. import csv +import io import os import tempfile from unittest import mock as unittest_mock +import mock import numpy as np import numpy.testing as npt import tensorflow as tf +from mediapipe.model_maker.python.core.data import cache_files from mediapipe.model_maker.python.text.text_classifier import dataset as text_classifier_ds from mediapipe.model_maker.python.text.text_classifier import model_spec from mediapipe.model_maker.python.text.text_classifier import preprocessor @@ -88,7 +91,8 @@ class PreprocessorTest(tf.test.TestCase): bert_preprocessor = preprocessor.BertClassifierPreprocessor( seq_len=5, do_lower_case=bert_spec.do_lower_case, - uri=bert_spec.downloaded_files.get_path(), + uri=bert_spec.get_path(), + model_name=bert_spec.name, ) preprocessed_dataset = bert_preprocessor.preprocess(dataset) labels = [] @@ -97,18 +101,87 @@ class PreprocessorTest(tf.test.TestCase): self.assertEqual(label.shape, [1]) labels.append(label.numpy()[0]) self.assertSameElements( - features.keys(), ['input_word_ids', 'input_mask', 'input_type_ids']) + features.keys(), ['input_word_ids', 'input_mask', 'input_type_ids'] + ) for feature in features.values(): self.assertEqual(feature.shape, [1, 5]) input_masks.append(features['input_mask'].numpy()[0]) - npt.assert_array_equal(features['input_type_ids'].numpy()[0], - [0, 0, 0, 0, 0]) + npt.assert_array_equal( + features['input_type_ids'].numpy()[0], [0, 0, 0, 0, 0] + ) npt.assert_array_equal( - np.stack(input_masks), np.array([[1, 1, 1, 1, 1], [1, 1, 1, 1, 0]])) + np.stack(input_masks), np.array([[1, 1, 1, 1, 1], [1, 1, 1, 1, 0]]) + ) self.assertEqual(labels, [1, 0]) + def test_bert_preprocessor_cache(self): + csv_file = self._get_csv_file() + dataset = text_classifier_ds.Dataset.from_csv( + filename=csv_file, + csv_params=self.CSV_PARAMS_, + cache_dir=self.get_temp_dir(), + ) + bert_spec = model_spec.SupportedModels.MOBILEBERT_CLASSIFIER.value() + bert_preprocessor = preprocessor.BertClassifierPreprocessor( + seq_len=5, + do_lower_case=bert_spec.do_lower_case, + uri=bert_spec.get_path(), + model_name=bert_spec.name, + ) + ds_cache_files = dataset.tfrecord_cache_files + preprocessed_cache_files = bert_preprocessor._get_tfrecord_cache_files( + ds_cache_files + ) + self.assertFalse(preprocessed_cache_files.is_cached()) + preprocessed_dataset = bert_preprocessor.preprocess(dataset) + self.assertTrue(preprocessed_cache_files.is_cached()) + self.assertEqual( + preprocessed_dataset.tfrecord_cache_files, preprocessed_cache_files + ) + + # The second time running preprocessor, it should load from cache directly + mock_stdout = io.StringIO() + with mock.patch('sys.stdout', mock_stdout): + _ = bert_preprocessor.preprocess(dataset) + self.assertEqual( + mock_stdout.getvalue(), + 'Using existing cache files at' + f' {preprocessed_cache_files.cache_prefix}\n', + ) + + def _get_new_prefix(self, cf, bert_spec, seq_len, do_lower_case): + bert_preprocessor = preprocessor.BertClassifierPreprocessor( + seq_len=seq_len, + do_lower_case=do_lower_case, + uri=bert_spec.get_path(), + model_name=bert_spec.name, + ) + new_cf = bert_preprocessor._get_tfrecord_cache_files(cf) + return new_cf.cache_prefix_filename + + def test_bert_get_tfrecord_cache_files(self): + # Test to ensure regenerated cache_files have different prefixes + all_cf_prefixes = set() + cf = cache_files.TFRecordCacheFiles( + cache_prefix_filename='cache_prefix', + cache_dir=self.get_temp_dir(), + num_shards=1, + ) + mobilebert_spec = model_spec.SupportedModels.MOBILEBERT_CLASSIFIER.value() + all_cf_prefixes.add(self._get_new_prefix(cf, mobilebert_spec, 5, True)) + all_cf_prefixes.add(self._get_new_prefix(cf, mobilebert_spec, 10, True)) + all_cf_prefixes.add(self._get_new_prefix(cf, mobilebert_spec, 5, False)) + new_cf = cache_files.TFRecordCacheFiles( + cache_prefix_filename='new_cache_prefix', + cache_dir=self.get_temp_dir(), + num_shards=1, + ) + all_cf_prefixes.add(self._get_new_prefix(new_cf, mobilebert_spec, 5, True)) + + # Each item of all_cf_prefixes should be unique. + self.assertLen(all_cf_prefixes, 4) + if __name__ == '__main__': # Load compressed models from tensorflow_hub - os.environ['TFHUB_MODEL_LOAD_FORMAT'] = 'COMPRESSED' tf.test.main() diff --git a/mediapipe/model_maker/python/text/text_classifier/testdata/bert_metadata.json b/mediapipe/model_maker/python/text/text_classifier/testdata/bert_metadata.json index 24214a80..22fb220f 100644 --- a/mediapipe/model_maker/python/text/text_classifier/testdata/bert_metadata.json +++ b/mediapipe/model_maker/python/text/text_classifier/testdata/bert_metadata.json @@ -16,8 +16,8 @@ } }, { - "name": "mask", - "description": "Mask with 1 for real tokens and 0 for padding tokens.", + "name": "segment_ids", + "description": "0 for the first sequence, 1 for the second sequence if exists.", "content": { "content_properties_type": "FeatureProperties", "content_properties": { @@ -27,8 +27,8 @@ } }, { - "name": "segment_ids", - "description": "0 for the first sequence, 1 for the second sequence if exists.", + "name": "mask", + "description": "Mask with 1 for real tokens and 0 for padding tokens.", "content": { "content_properties_type": "FeatureProperties", "content_properties": { diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py index c3dd48be..75275223 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py @@ -19,14 +19,18 @@ import tempfile from typing import Any, Optional, Sequence, Tuple import tensorflow as tf +from tensorflow_addons import optimizers as tfa_optimizers import tensorflow_hub as hub -from mediapipe.model_maker.python.core import hyperparameters as hp from mediapipe.model_maker.python.core.data import dataset as ds from mediapipe.model_maker.python.core.tasks import classifier +from mediapipe.model_maker.python.core.utils import hub_loader +from mediapipe.model_maker.python.core.utils import loss_functions +from mediapipe.model_maker.python.core.utils import metrics from mediapipe.model_maker.python.core.utils import model_util from mediapipe.model_maker.python.core.utils import quantization from mediapipe.model_maker.python.text.text_classifier import dataset as text_ds +from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp from mediapipe.model_maker.python.text.text_classifier import model_options as mo from mediapipe.model_maker.python.text.text_classifier import model_spec as ms from mediapipe.model_maker.python.text.text_classifier import preprocessor @@ -49,27 +53,34 @@ def _validate(options: text_classifier_options.TextClassifierOptions): if options.model_options is None: return - if (isinstance(options.model_options, mo.AverageWordEmbeddingModelOptions) and - (options.supported_model != - ms.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER)): - raise ValueError("Expected AVERAGE_WORD_EMBEDDING_CLASSIFIER," - f" got {options.supported_model}") - if (isinstance(options.model_options, mo.BertModelOptions) and - (options.supported_model != ms.SupportedModels.MOBILEBERT_CLASSIFIER)): + if isinstance( + options.model_options, mo.AverageWordEmbeddingModelOptions + ) and ( + options.supported_model + != ms.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER + ): raise ValueError( - f"Expected MOBILEBERT_CLASSIFIER, got {options.supported_model}") + "Expected AVERAGE_WORD_EMBEDDING_CLASSIFIER," + f" got {options.supported_model}" + ) + if isinstance(options.model_options, mo.BertModelOptions) and ( + not isinstance(options.supported_model.value(), ms.BertClassifierSpec) + ): + raise ValueError( + f"Expected a Bert Classifier, got {options.supported_model}" + ) class TextClassifier(classifier.Classifier): """API for creating and training a text classification model.""" - def __init__(self, model_spec: Any, hparams: hp.BaseHParams, - label_names: Sequence[str]): + def __init__( + self, model_spec: Any, label_names: Sequence[str], shuffle: bool + ): super().__init__( - model_spec=model_spec, label_names=label_names, shuffle=hparams.shuffle) + model_spec=model_spec, label_names=label_names, shuffle=shuffle + ) self._model_spec = model_spec - self._hparams = hparams - self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir) self._text_preprocessor: preprocessor.TextClassifierPreprocessor = None @classmethod @@ -106,29 +117,39 @@ class TextClassifier(classifier.Classifier): if options.hparams is None: options.hparams = options.supported_model.value().hparams - if options.supported_model == ms.SupportedModels.MOBILEBERT_CLASSIFIER: - text_classifier = ( - _BertClassifier.create_bert_classifier(train_data, validation_data, - options, - train_data.label_names)) - elif (options.supported_model == - ms.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER): - text_classifier = ( - _AverageWordEmbeddingClassifier - .create_average_word_embedding_classifier(train_data, validation_data, - options, - train_data.label_names)) + if isinstance(options.supported_model.value(), ms.BertClassifierSpec): + text_classifier = _BertClassifier.create_bert_classifier( + train_data, validation_data, options + ) + elif isinstance( + options.supported_model.value(), ms.AverageWordEmbeddingClassifierSpec + ): + text_classifier = _AverageWordEmbeddingClassifier.create_average_word_embedding_classifier( + train_data, validation_data, options + ) else: raise ValueError(f"Unknown model {options.supported_model}") return text_classifier - def evaluate(self, data: ds.Dataset, batch_size: int = 32) -> Any: + def evaluate( + self, + data: ds.Dataset, + batch_size: int = 32, + desired_precisions: Optional[Sequence[float]] = None, + desired_recalls: Optional[Sequence[float]] = None, + ) -> Any: """Overrides Classifier.evaluate(). Args: data: Evaluation dataset. Must be a TextClassifier Dataset. batch_size: Number of samples per evaluation step. + desired_precisions: If specified, adds a RecallAtPrecision metric per + desired_precisions[i] entry which tracks the recall given the constraint + on precision. Only supported for binary classification. + desired_recalls: If specified, adds a PrecisionAtRecall metric per + desired_recalls[i] entry which tracks the precision given the constraint + on recall. Only supported for binary classification. Returns: The loss value and accuracy. @@ -144,7 +165,28 @@ class TextClassifier(classifier.Classifier): processed_data = self._text_preprocessor.preprocess(data) dataset = processed_data.gen_tf_dataset(batch_size, is_training=False) - return self._model.evaluate(dataset) + + with self._hparams.get_strategy().scope(): + return self._model.evaluate(dataset) + + def save_model( + self, + model_name: str = "saved_model", + ): + """Saves the model in SavedModel format. + + For more information, see https://www.tensorflow.org/guide/saved_model. + + Args: + model_name: Name of the saved model. + """ + tf.io.gfile.makedirs(self._hparams.export_dir) + saved_model_file = os.path.join(self._hparams.export_dir, model_name) + self._model.save( + saved_model_file, + include_optimizer=False, + save_format="tf", + ) def export_model( self, @@ -161,20 +203,23 @@ class TextClassifier(classifier.Classifier): path is {self._hparams.export_dir}/{model_name}. quantization_config: The configuration for model quantization. """ - if not tf.io.gfile.exists(self._hparams.export_dir): - tf.io.gfile.makedirs(self._hparams.export_dir) + tf.io.gfile.makedirs(self._hparams.export_dir) tflite_file = os.path.join(self._hparams.export_dir, model_name) metadata_file = os.path.join(self._hparams.export_dir, "metadata.json") - tflite_model = model_util.convert_to_tflite( - model=self._model, quantization_config=quantization_config) + self.save_model(model_name="saved_model") + saved_model_file = os.path.join(self._hparams.export_dir, "saved_model") + + tflite_model = model_util.convert_to_tflite_from_file( + saved_model_file, quantization_config=quantization_config + ) vocab_filepath = os.path.join(tempfile.mkdtemp(), "vocab.txt") self._save_vocab(vocab_filepath) writer = self._get_metadata_writer(tflite_model, vocab_filepath) tflite_model_with_metadata, metadata_json = writer.populate() model_util.save_tflite(tflite_model_with_metadata, tflite_file) - with open(metadata_file, "w") as f: + with tf.io.gfile.GFile(metadata_file, "w") as f: f.write(metadata_json) @abc.abstractmethod @@ -191,28 +236,39 @@ class _AverageWordEmbeddingClassifier(TextClassifier): _DELIM_REGEX_PATTERN = r"[^\w\']+" - def __init__(self, model_spec: ms.AverageWordEmbeddingClassifierSpec, - model_options: mo.AverageWordEmbeddingModelOptions, - hparams: hp.BaseHParams, label_names: Sequence[str]): - super().__init__(model_spec, hparams, label_names) + def __init__( + self, + model_spec: ms.AverageWordEmbeddingClassifierSpec, + model_options: mo.AverageWordEmbeddingModelOptions, + hparams: hp.AverageWordEmbeddingHParams, + label_names: Sequence[str], + ): + super().__init__(model_spec, label_names, hparams.shuffle) self._model_options = model_options + self._hparams = hparams + self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir) self._loss_function = "sparse_categorical_crossentropy" - self._metric_function = "accuracy" + self._metric_functions = [ + "accuracy", + metrics.SparsePrecision(name="precision", dtype=tf.float32), + metrics.SparseRecall(name="recall", dtype=tf.float32), + ] self._text_preprocessor: ( preprocessor.AverageWordEmbeddingClassifierPreprocessor) = None @classmethod def create_average_word_embedding_classifier( - cls, train_data: text_ds.Dataset, validation_data: text_ds.Dataset, + cls, + train_data: text_ds.Dataset, + validation_data: text_ds.Dataset, options: text_classifier_options.TextClassifierOptions, - label_names: Sequence[str]) -> "_AverageWordEmbeddingClassifier": + ) -> "_AverageWordEmbeddingClassifier": """Creates, trains, and returns an Average Word Embedding classifier. Args: train_data: Training data. validation_data: Validation data. options: Options for creating and training the text classifier. - label_names: Label names used in the data. Returns: An Average Word Embedding classifier. @@ -306,30 +362,37 @@ class _BertClassifier(TextClassifier): _INITIALIZER_RANGE = 0.02 - def __init__(self, model_spec: ms.BertClassifierSpec, - model_options: mo.BertModelOptions, hparams: hp.BaseHParams, - label_names: Sequence[str]): - super().__init__(model_spec, hparams, label_names) + def __init__( + self, + model_spec: ms.BertClassifierSpec, + model_options: mo.BertModelOptions, + hparams: hp.BertHParams, + label_names: Sequence[str], + ): + super().__init__(model_spec, label_names, hparams.shuffle) + self._hparams = hparams + self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir) self._model_options = model_options - with self._hparams.get_strategy().scope(): - self._loss_function = tf.keras.losses.SparseCategoricalCrossentropy() - self._metric_function = tf.keras.metrics.SparseCategoricalAccuracy( - "test_accuracy", dtype=tf.float32 - ) self._text_preprocessor: preprocessor.BertClassifierPreprocessor = None + with self._hparams.get_strategy().scope(): + self._loss_function = loss_functions.SparseFocalLoss( + self._hparams.gamma, self._num_classes + ) + self._metric_functions = self._create_metrics() @classmethod def create_bert_classifier( - cls, train_data: text_ds.Dataset, validation_data: text_ds.Dataset, + cls, + train_data: text_ds.Dataset, + validation_data: text_ds.Dataset, options: text_classifier_options.TextClassifierOptions, - label_names: Sequence[str]) -> "_BertClassifier": + ) -> "_BertClassifier": """Creates, trains, and returns a BERT-based classifier. Args: train_data: Training data. validation_data: Validation data. options: Options for creating and training the text classifier. - label_names: Label names used in the data. Returns: A BERT-based classifier. @@ -372,10 +435,60 @@ class _BertClassifier(TextClassifier): self._text_preprocessor = preprocessor.BertClassifierPreprocessor( seq_len=self._model_options.seq_len, do_lower_case=self._model_spec.do_lower_case, - uri=self._model_spec.downloaded_files.get_path(), + uri=self._model_spec.get_path(), + model_name=self._model_spec.name, ) - return (self._text_preprocessor.preprocess(train_data), - self._text_preprocessor.preprocess(validation_data)) + return ( + self._text_preprocessor.preprocess(train_data), + self._text_preprocessor.preprocess(validation_data), + ) + + def _create_metrics(self): + """Creates metrics for training and evaluation. + + The default metrics are accuracy, precision, and recall. + + For binary classification tasks only (num_classes=2): + Users can configure PrecisionAtRecall and RecallAtPrecision metrics using + the desired_presisions and desired_recalls fields in BertHParams. + + Returns: + A list of tf.keras.Metric subclasses which can be used with model.compile + """ + metric_functions = [ + tf.keras.metrics.SparseCategoricalAccuracy( + "accuracy", dtype=tf.float32 + ), + metrics.SparsePrecision(name="precision", dtype=tf.float32), + metrics.SparseRecall(name="recall", dtype=tf.float32), + ] + if self._num_classes == 2: + if self._hparams.desired_precisions: + for desired_precision in self._hparams.desired_precisions: + metric_functions.append( + metrics.BinarySparseRecallAtPrecision( + desired_precision, + name=f"recall_at_precision_{desired_precision}", + num_thresholds=1000, + ) + ) + if self._hparams.desired_recalls: + for desired_recall in self._hparams.desired_recalls: + metric_functions.append( + metrics.BinarySparseRecallAtPrecision( + desired_recall, + name=f"precision_at_recall_{desired_recall}", + num_thresholds=1000, + ) + ) + else: + if self._hparams.desired_precisions or self._hparams.desired_recalls: + raise ValueError( + "desired_recalls and desired_precisions parameters are binary" + " metrics and not supported for num_classes > 2. Found" + f" num_classes: {self._num_classes}" + ) + return metric_functions def _create_model(self): """Creates a BERT-based classifier model. @@ -385,30 +498,58 @@ class _BertClassifier(TextClassifier): """ encoder_inputs = dict( input_word_ids=tf.keras.layers.Input( - shape=(self._model_options.seq_len,), dtype=tf.int32), + shape=(self._model_options.seq_len,), + dtype=tf.int32, + name="input_word_ids", + ), input_mask=tf.keras.layers.Input( - shape=(self._model_options.seq_len,), dtype=tf.int32), + shape=(self._model_options.seq_len,), + dtype=tf.int32, + name="input_mask", + ), input_type_ids=tf.keras.layers.Input( - shape=(self._model_options.seq_len,), dtype=tf.int32), + shape=(self._model_options.seq_len,), + dtype=tf.int32, + name="input_type_ids", + ), ) - encoder = hub.KerasLayer( - self._model_spec.downloaded_files.get_path(), - trainable=self._model_options.do_fine_tuning, - ) - encoder_outputs = encoder(encoder_inputs) - pooled_output = encoder_outputs["pooled_output"] + if self._model_spec.is_tf2: + encoder = hub.KerasLayer( + self._model_spec.get_path(), + trainable=self._model_options.do_fine_tuning, + load_options=tf.saved_model.LoadOptions( + experimental_io_device="/job:localhost" + ), + ) + encoder_outputs = encoder(encoder_inputs) + pooled_output = encoder_outputs["pooled_output"] + else: + renamed_inputs = dict( + input_ids=encoder_inputs["input_word_ids"], + input_mask=encoder_inputs["input_mask"], + segment_ids=encoder_inputs["input_type_ids"], + ) + encoder = hub_loader.HubKerasLayerV1V2( + self._model_spec.get_path(), + signature="tokens", + output_key="pooled_output", + trainable=self._model_options.do_fine_tuning, + ) + pooled_output = encoder(renamed_inputs) output = tf.keras.layers.Dropout(rate=self._model_options.dropout_rate)( - pooled_output) + pooled_output + ) initializer = tf.keras.initializers.TruncatedNormal( - stddev=self._INITIALIZER_RANGE) + stddev=self._INITIALIZER_RANGE + ) output = tf.keras.layers.Dense( self._num_classes, kernel_initializer=initializer, name="output", activation="softmax", - dtype=tf.float32)( - output) + dtype=tf.float32, + )(output) self._model = tf.keras.Model(inputs=encoder_inputs, outputs=output) def _create_optimizer(self, train_data: text_ds.Dataset): @@ -431,18 +572,38 @@ class _BertClassifier(TextClassifier): lr_schedule = tf.keras.optimizers.schedules.PolynomialDecay( initial_learning_rate=initial_lr, decay_steps=total_steps, - end_learning_rate=0.0, - power=1.0) + end_learning_rate=self._hparams.end_learning_rate, + power=1.0, + ) if warmup_steps: lr_schedule = model_util.WarmUp( initial_learning_rate=initial_lr, decay_schedule_fn=lr_schedule, - warmup_steps=warmup_steps) - - self._optimizer = tf.keras.optimizers.experimental.AdamW( - lr_schedule, weight_decay=0.01, epsilon=1e-6, global_clipnorm=1.0) - self._optimizer.exclude_from_weight_decay( - var_names=["LayerNorm", "layer_norm", "bias"]) + warmup_steps=warmup_steps, + ) + if self._hparams.optimizer == hp.BertOptimizer.ADAMW: + self._optimizer = tf.keras.optimizers.experimental.AdamW( + lr_schedule, + weight_decay=self._hparams.weight_decay, + epsilon=1e-6, + global_clipnorm=1.0, + ) + self._optimizer.exclude_from_weight_decay( + var_names=["LayerNorm", "layer_norm", "bias"] + ) + elif self._hparams.optimizer == hp.BertOptimizer.LAMB: + self._optimizer = tfa_optimizers.LAMB( + lr_schedule, + weight_decay_rate=self._hparams.weight_decay, + epsilon=1e-6, + exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"], + global_clipnorm=1.0, + ) + else: + raise ValueError( + "BertHParams.optimizer must be set to ADAM or " + f"LAMB. Got {self._hparams.optimizer}." + ) def _save_vocab(self, vocab_filepath: str): tf.io.gfile.copy( diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py index c3d1711d..b646a15a 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier_demo.py @@ -66,14 +66,16 @@ def run(data_dir, quantization_config = None if (supported_model == text_classifier.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER): - hparams = text_classifier.HParams( - epochs=10, batch_size=32, learning_rate=0, export_dir=export_dir) + hparams = text_classifier.AverageWordEmbeddingHParams( + epochs=10, batch_size=32, learning_rate=0, export_dir=export_dir + ) # Warning: This takes extremely long to run on CPU elif ( supported_model == text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER): quantization_config = quantization.QuantizationConfig.for_dynamic() - hparams = text_classifier.HParams( - epochs=3, batch_size=48, learning_rate=3e-5, export_dir=export_dir) + hparams = text_classifier.BertHParams( + epochs=3, batch_size=48, learning_rate=3e-5, export_dir=export_dir + ) # Fine-tunes the model. options = text_classifier.TextClassifierOptions( @@ -82,8 +84,8 @@ def run(data_dir, options) # Gets evaluation results. - _, acc = model.evaluate(validation_data) - print('Eval accuracy: %f' % acc) + metrics = model.evaluate(validation_data) + print('Eval accuracy: %f' % metrics[1]) model.export_model(quantization_config=quantization_config) model.export_labels(export_dir=options.hparams.export_dir) diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier_options.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier_options.py index c62fb27b..b61731f1 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier_options.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier_options.py @@ -16,7 +16,7 @@ import dataclasses from typing import Optional -from mediapipe.model_maker.python.core import hyperparameters as hp +from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp from mediapipe.model_maker.python.text.text_classifier import model_options as mo from mediapipe.model_maker.python.text.text_classifier import model_spec as ms @@ -34,5 +34,5 @@ class TextClassifierOptions: architecture of the `supported_model`. """ supported_model: ms.SupportedModels - hparams: Optional[hp.BaseHParams] = None + hparams: Optional[hp.HParams] = None model_options: Optional[mo.TextClassifierModelOptions] = None diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py index 34830c9f..122182dd 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py @@ -16,17 +16,17 @@ import csv import filecmp import os import tempfile -import unittest from unittest import mock as unittest_mock +from absl.testing import parameterized import tensorflow as tf +from mediapipe.model_maker.python.core.utils import loss_functions from mediapipe.model_maker.python.text import text_classifier from mediapipe.tasks.python.test import test_utils -@unittest.skip('b/275624089') -class TextClassifierTest(tf.test.TestCase): +class TextClassifierTest(tf.test.TestCase, parameterized.TestCase): _AVERAGE_WORD_EMBEDDING_JSON_FILE = ( test_utils.get_test_data_path('average_word_embedding_metadata.json')) @@ -66,18 +66,20 @@ class TextClassifierTest(tf.test.TestCase): def test_create_and_train_average_word_embedding_model(self): train_data, validation_data = self._get_data() - options = ( - text_classifier.TextClassifierOptions( - supported_model=(text_classifier.SupportedModels - .AVERAGE_WORD_EMBEDDING_CLASSIFIER), - hparams=text_classifier.HParams( - epochs=1, batch_size=1, learning_rate=0))) + options = text_classifier.TextClassifierOptions( + supported_model=( + text_classifier.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER + ), + hparams=text_classifier.AverageWordEmbeddingHParams( + epochs=1, batch_size=1, learning_rate=0 + ), + ) average_word_embedding_classifier = ( text_classifier.TextClassifier.create(train_data, validation_data, options)) - _, accuracy = average_word_embedding_classifier.evaluate(validation_data) - self.assertGreaterEqual(accuracy, 0.0) + metrics = average_word_embedding_classifier.evaluate(validation_data) + self.assertGreaterEqual(metrics[1], 0.0) # metrics[1] is accuracy # Test export_model average_word_embedding_classifier.export_model() @@ -96,24 +98,36 @@ class TextClassifierTest(tf.test.TestCase): filecmp.cmp( output_metadata_file, self._AVERAGE_WORD_EMBEDDING_JSON_FILE, - shallow=False)) + shallow=False, + ) + ) - def test_create_and_train_bert(self): + @parameterized.named_parameters( + # Skipping mobilebert b/c OSS test timeout/flakiness: b/275624089 + dict( + testcase_name='mobilebert', + supported_model=text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER, + ), + ) + def test_create_and_train_bert(self, supported_model): train_data, validation_data = self._get_data() options = text_classifier.TextClassifierOptions( - supported_model=text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER, + supported_model=supported_model, model_options=text_classifier.BertModelOptions( - do_fine_tuning=False, seq_len=2), - hparams=text_classifier.HParams( + do_fine_tuning=False, seq_len=2 + ), + hparams=text_classifier.BertHParams( epochs=1, batch_size=1, learning_rate=3e-5, - distribution_strategy='off')) + distribution_strategy='off', + ), + ) bert_classifier = text_classifier.TextClassifier.create( train_data, validation_data, options) - _, accuracy = bert_classifier.evaluate(validation_data) - self.assertGreaterEqual(accuracy, 0.0) + metrics = bert_classifier.evaluate(validation_data) + self.assertGreaterEqual(metrics[1], 0.0) # metrics[1] is accuracy # Test export_model bert_classifier.export_model() @@ -137,45 +151,93 @@ class TextClassifierTest(tf.test.TestCase): ) def test_label_mismatch(self): - options = ( - text_classifier.TextClassifierOptions( - supported_model=( - text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER))) + options = text_classifier.TextClassifierOptions( + supported_model=(text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER) + ) train_tf_dataset = tf.data.Dataset.from_tensor_slices([[0]]) - train_data = text_classifier.Dataset(train_tf_dataset, 1, ['foo']) + train_data = text_classifier.Dataset(train_tf_dataset, ['foo'], 1) validation_tf_dataset = tf.data.Dataset.from_tensor_slices([[0]]) - validation_data = text_classifier.Dataset(validation_tf_dataset, 1, ['bar']) + validation_data = text_classifier.Dataset(validation_tf_dataset, ['bar'], 1) with self.assertRaisesRegex( ValueError, - 'Training data label names .* not equal to validation data label names' + 'Training data label names .* not equal to validation data label names', ): - text_classifier.TextClassifier.create(train_data, validation_data, - options) + text_classifier.TextClassifier.create( + train_data, validation_data, options + ) def test_options_mismatch(self): train_data, validation_data = self._get_data() - avg_options = ( - text_classifier.TextClassifierOptions( - supported_model=( - text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER), - model_options=text_classifier.AverageWordEmbeddingModelOptions())) - with self.assertRaisesRegex( - ValueError, 'Expected AVERAGE_WORD_EMBEDDING_CLASSIFIER, got' - ' SupportedModels.MOBILEBERT_CLASSIFIER'): - text_classifier.TextClassifier.create(train_data, validation_data, - avg_options) + avg_options = text_classifier.TextClassifierOptions( + supported_model=(text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER), + model_options=text_classifier.AverageWordEmbeddingModelOptions(), + ) + with self.assertRaisesWithLiteralMatch( + ValueError, + 'Expected AVERAGE_WORD_EMBEDDING_CLASSIFIER, got' + ' SupportedModels.MOBILEBERT_CLASSIFIER', + ): + text_classifier.TextClassifier.create( + train_data, validation_data, avg_options + ) - bert_options = ( - text_classifier.TextClassifierOptions( - supported_model=(text_classifier.SupportedModels - .AVERAGE_WORD_EMBEDDING_CLASSIFIER), - model_options=text_classifier.BertModelOptions())) - with self.assertRaisesRegex( - ValueError, 'Expected MOBILEBERT_CLASSIFIER, got' - ' SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER'): - text_classifier.TextClassifier.create(train_data, validation_data, - bert_options) + bert_options = text_classifier.TextClassifierOptions( + supported_model=( + text_classifier.SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER + ), + model_options=text_classifier.BertModelOptions(), + ) + with self.assertRaisesWithLiteralMatch( + ValueError, + 'Expected a Bert Classifier, got' + ' SupportedModels.AVERAGE_WORD_EMBEDDING_CLASSIFIER', + ): + text_classifier.TextClassifier.create( + train_data, validation_data, bert_options + ) + + def test_bert_loss_and_metrics_creation(self): + train_data, validation_data = self._get_data() + supported_model = text_classifier.SupportedModels.MOBILEBERT_CLASSIFIER + hparams = text_classifier.BertHParams( + desired_recalls=[0.2], + desired_precisions=[0.9], + epochs=1, + batch_size=1, + learning_rate=3e-5, + distribution_strategy='off', + gamma=3.5, + ) + options = text_classifier.TextClassifierOptions( + supported_model=supported_model, hparams=hparams + ) + bert_classifier = text_classifier.TextClassifier.create( + train_data, validation_data, options + ) + loss_fn = bert_classifier._loss_function + self.assertIsInstance(loss_fn, loss_functions.SparseFocalLoss) + self.assertEqual(loss_fn._gamma, 3.5) + self.assertEqual(loss_fn._num_classes, 2) + metric_names = [m.name for m in bert_classifier._metric_functions] + expected_metric_names = [ + 'accuracy', + 'recall', + 'precision', + 'precision_at_recall_0.2', + 'recall_at_precision_0.9', + ] + self.assertCountEqual(metric_names, expected_metric_names) + + # Non-binary data + tf_dataset = tf.data.Dataset.from_tensor_slices([[0]]) + data = text_classifier.Dataset(tf_dataset, ['foo', 'bar', 'baz'], 1) + with self.assertRaisesWithLiteralMatch( + ValueError, + 'desired_recalls and desired_precisions parameters are binary metrics' + ' and not supported for num_classes > 2. Found num_classes: 3', + ): + text_classifier.TextClassifier.create(data, data, options) if __name__ == '__main__': diff --git a/mediapipe/model_maker/python/vision/face_stylizer/BUILD b/mediapipe/model_maker/python/vision/face_stylizer/BUILD index a2e30a11..5e4c2245 100644 --- a/mediapipe/model_maker/python/vision/face_stylizer/BUILD +++ b/mediapipe/model_maker/python/vision/face_stylizer/BUILD @@ -20,13 +20,6 @@ licenses(["notice"]) package(default_visibility = ["//mediapipe:__subpackages__"]) -filegroup( - name = "testdata", - srcs = glob([ - "testdata/**", - ]), -) - py_library( name = "constants", srcs = ["constants.py"], @@ -72,18 +65,11 @@ py_library( name = "dataset", srcs = ["dataset.py"], deps = [ + ":constants", "//mediapipe/model_maker/python/core/data:classification_dataset", - "//mediapipe/model_maker/python/vision/core:image_utils", - ], -) - -py_test( - name = "dataset_test", - srcs = ["dataset_test.py"], - data = [":testdata"], - deps = [ - ":dataset", - "//mediapipe/tasks/python/test:test_utils", + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:face_aligner", ], ) @@ -100,6 +86,7 @@ py_library( "//mediapipe/model_maker/python/core/utils:loss_functions", "//mediapipe/model_maker/python/core/utils:model_util", "//mediapipe/model_maker/python/vision/core:image_preprocessing", + "//mediapipe/tasks/python/metadata/metadata_writers:face_stylizer", ], ) diff --git a/mediapipe/model_maker/python/vision/face_stylizer/constants.py b/mediapipe/model_maker/python/vision/face_stylizer/constants.py index e7a03aeb..ac767523 100644 --- a/mediapipe/model_maker/python/vision/face_stylizer/constants.py +++ b/mediapipe/model_maker/python/vision/face_stylizer/constants.py @@ -41,5 +41,11 @@ FACE_STYLIZER_W_FILES = file_util.DownloadedFiles( 'https://storage.googleapis.com/mediapipe-assets/face_stylizer_w_avg.npy', ) +FACE_ALIGNER_TASK_FILES = file_util.DownloadedFiles( + 'face_stylizer/face_landmarker_v2.task', + 'https://storage.googleapis.com/mediapipe-assets/face_landmarker_v2.task', + is_folder=False, +) + # Dimension of the input style vector to the decoder STYLE_DIM = 512 diff --git a/mediapipe/model_maker/python/vision/face_stylizer/dataset.py b/mediapipe/model_maker/python/vision/face_stylizer/dataset.py index d517fd9c..eb324028 100644 --- a/mediapipe/model_maker/python/vision/face_stylizer/dataset.py +++ b/mediapipe/model_maker/python/vision/face_stylizer/dataset.py @@ -13,13 +13,43 @@ # limitations under the License. """Face stylizer dataset library.""" +from typing import Sequence import logging import os import tensorflow as tf from mediapipe.model_maker.python.core.data import classification_dataset -from mediapipe.model_maker.python.vision.core import image_utils +from mediapipe.model_maker.python.vision.face_stylizer import constants +from mediapipe.python._framework_bindings import image as image_module +from mediapipe.tasks.python.core import base_options as base_options_module +from mediapipe.tasks.python.vision import face_aligner + + +def _preprocess_face_dataset( + all_image_paths: Sequence[str], +) -> Sequence[tf.Tensor]: + """Preprocess face image dataset by aligning the face.""" + path = constants.FACE_ALIGNER_TASK_FILES.get_path() + base_options = base_options_module.BaseOptions(model_asset_path=path) + options = face_aligner.FaceAlignerOptions(base_options=base_options) + aligner = face_aligner.FaceAligner.create_from_options(options) + + preprocessed_images = [] + for path in all_image_paths: + tf.compat.v1.logging.info('Preprocess image %s', path) + image = image_module.Image.create_from_file(path) + aligned_image = aligner.align(image) + if aligned_image is None: + raise ValueError( + 'ERROR: Invalid image. No face is detected and aligned. Please make' + ' sure the image has a single face that is facing straightforward and' + ' not significantly rotated.' + ) + aligned_image_tensor = tf.convert_to_tensor(aligned_image.numpy_view()) + preprocessed_images.append(aligned_image_tensor) + + return preprocessed_images # TODO: Change to a unlabeled dataset if it makes sense. @@ -27,72 +57,41 @@ class Dataset(classification_dataset.ClassificationDataset): """Dataset library for face stylizer fine tuning.""" @classmethod - def from_folder( - cls, dirname: str + def from_image( + cls, filename: str ) -> classification_dataset.ClassificationDataset: - """Loads images from the given directory. + """Creates a dataset from single image. - The style image dataset directory is expected to contain one subdirectory - whose name represents the label of the style. There can be one or multiple - images of the same style in that subdirectory. Supported input image formats - include 'jpg', 'jpeg', 'png'. + Supported input image formats include 'jpg', 'jpeg', 'png'. Args: - dirname: Name of the directory containing the image files. + filename: Name of the image file. Returns: - Dataset containing images and labels and other related info. - Raises: - ValueError: if the input data directory is empty. + Dataset containing image and label and other related info. """ - data_root = os.path.abspath(dirname) + file_path = os.path.abspath(filename) + image_filename = os.path.basename(filename) + image_name, ext_name = os.path.splitext(image_filename) - # Assumes the image data of the same label are in the same subdirectory, - # gets image path and label names. - all_image_paths = list(tf.io.gfile.glob(data_root + r'/*/*')) - all_image_size = len(all_image_paths) - if all_image_size == 0: - raise ValueError('Invalid input data directory') - if not any( - fname.endswith(('.jpg', '.jpeg', '.png')) for fname in all_image_paths - ): - raise ValueError('No images found under given directory') + if not ext_name.endswith(('.jpg', '.jpeg', '.png')): + raise ValueError('Unsupported image formats: %s' % ext_name) - label_names = sorted( - name - for name in os.listdir(data_root) - if os.path.isdir(os.path.join(data_root, name)) - ) - all_label_size = len(label_names) - index_by_label = dict( - (name, index) for index, name in enumerate(label_names) - ) - # Get the style label from the subdirectory name. - all_image_labels = [ - index_by_label[os.path.basename(os.path.dirname(path))] - for path in all_image_paths - ] + image_data = _preprocess_face_dataset([file_path]) + label_names = [image_name] - path_ds = tf.data.Dataset.from_tensor_slices(all_image_paths) - - image_ds = path_ds.map( - image_utils.load_image, num_parallel_calls=tf.data.AUTOTUNE - ) + image_ds = tf.data.Dataset.from_tensor_slices(image_data) # Load label - label_ds = tf.data.Dataset.from_tensor_slices( - tf.cast(all_image_labels, tf.int64) - ) + label_ds = tf.data.Dataset.from_tensor_slices(tf.cast([0], tf.int64)) # Create a dataset of (image, label) pairs image_label_ds = tf.data.Dataset.zip((image_ds, label_ds)) - logging.info( - 'Load images dataset with size: %d, num_label: %d, labels: %s.', - all_image_size, - all_label_size, - ', '.join(label_names), - ) + logging.info('Create dataset for style: %s.', image_name) + return Dataset( - dataset=image_label_ds, size=all_image_size, label_names=label_names + dataset=image_label_ds, + label_names=label_names, + size=1, ) diff --git a/mediapipe/model_maker/python/vision/face_stylizer/dataset_test.py b/mediapipe/model_maker/python/vision/face_stylizer/dataset_test.py index 73140f30..24214081 100644 --- a/mediapipe/model_maker/python/vision/face_stylizer/dataset_test.py +++ b/mediapipe/model_maker/python/vision/face_stylizer/dataset_test.py @@ -12,8 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import numpy as np import tensorflow as tf +from mediapipe.model_maker.python.vision.core import image_utils from mediapipe.model_maker.python.vision.face_stylizer import dataset from mediapipe.tasks.python.test import test_utils @@ -22,25 +24,24 @@ class DatasetTest(tf.test.TestCase): def setUp(self): super().setUp() - self._test_data_dirname = 'input/style' - def test_from_folder(self): - input_data_dir = test_utils.get_test_data_path(self._test_data_dirname) - data = dataset.Dataset.from_folder(dirname=input_data_dir) - self.assertEqual(data.num_classes, 2) - self.assertEqual(data.label_names, ['cartoon', 'sketch']) - self.assertLen(data, 2) + def test_from_image(self): + test_image_file = 'input/style/cartoon/cartoon.jpg' + input_image_path = test_utils.get_test_data_path(test_image_file) + data = dataset.Dataset.from_image(filename=input_image_path) + self.assertEqual(data.num_classes, 1) + self.assertEqual(data.label_names, ['cartoon']) + self.assertLen(data, 1) - def test_from_folder_raise_value_error_for_invalid_path(self): - with self.assertRaisesRegex(ValueError, 'Invalid input data directory'): - dataset.Dataset.from_folder(dirname='invalid') + def test_from_image_raise_value_error_for_invalid_path(self): + with self.assertRaisesRegex(ValueError, 'Unsupported image formats: .zip'): + dataset.Dataset.from_image(filename='input/style/cartoon/cartoon.zip') - def test_from_folder_raise_value_error_for_valid_no_data_path(self): - input_data_dir = test_utils.get_test_data_path('face_stylizer') - with self.assertRaisesRegex( - ValueError, 'No images found under given directory' - ): - dataset.Dataset.from_folder(dirname=input_data_dir) + def test_from_image_raise_value_error_for_invalid_image(self): + with self.assertRaisesRegex(ValueError, 'Invalid image'): + test_image_file = 'input/style/sketch/boy-6030802_1280.jpg' + input_image_path = test_utils.get_test_data_path(test_image_file) + dataset.Dataset.from_image(filename=input_image_path) if __name__ == '__main__': diff --git a/mediapipe/model_maker/python/vision/face_stylizer/face_stylizer.py b/mediapipe/model_maker/python/vision/face_stylizer/face_stylizer.py index 5758ac7b..e635d2b3 100644 --- a/mediapipe/model_maker/python/vision/face_stylizer/face_stylizer.py +++ b/mediapipe/model_maker/python/vision/face_stylizer/face_stylizer.py @@ -13,8 +13,10 @@ # limitations under the License. """APIs to train face stylization model.""" +import logging import os -from typing import Callable, Optional +from typing import Any, Callable, Optional +import zipfile import numpy as np import tensorflow as tf @@ -28,6 +30,16 @@ from mediapipe.model_maker.python.vision.face_stylizer import face_stylizer_opti from mediapipe.model_maker.python.vision.face_stylizer import hyperparameters as hp from mediapipe.model_maker.python.vision.face_stylizer import model_options as model_opt from mediapipe.model_maker.python.vision.face_stylizer import model_spec as ms +from mediapipe.tasks.python.metadata.metadata_writers import face_stylizer as metadata_writer + +# Face detector model and face landmarks detector file names. +_FACE_DETECTOR_MODEL = 'face_detector.tflite' +_FACE_LANDMARKS_DETECTOR_MODEL = 'face_landmarks_detector.tflite' + +# The mean value used in the input tensor normalization for the face stylizer +# model. +_NORM_MEAN = 0.0 +_NORM_STD = 255.0 class FaceStylizer(object): @@ -54,7 +66,6 @@ class FaceStylizer(object): self._model_spec = model_spec self._model_options = model_options self._hparams = hparams - # TODO: Support face alignment in image preprocessor. self._preprocessor = image_preprocessing.Preprocessor( input_shape=self._model_spec.input_image_shape, num_classes=1, @@ -93,6 +104,37 @@ class FaceStylizer(object): face_stylizer._create_and_train_model(train_data) return face_stylizer + def stylize( + self, data: classification_ds.ClassificationDataset + ) -> classification_ds.ClassificationDataset: + """Stylizes the images represented by the input dataset. + + Args: + data: Dataset of input images, can contain multiple images. + + Returns: + A dataset contains the stylized images + """ + input_dataset = data.gen_tf_dataset(preprocess=self._preprocessor) + output_img_list = [] + for sample in input_dataset: + image = sample[0] + w = self._encoder(image, training=True) + x = self._decoder({'inputs': w + self.w_avg}, training=True) + output_batch = x['image'][-1] + output_img_tensor = (tf.squeeze(output_batch).numpy() + 1.0) * 127.5 + output_img_list.append(output_img_tensor) + + image_ds = tf.data.Dataset.from_tensor_slices(output_img_list) + + logging.info('Stylized %s images.', len(output_img_list)) + + return classification_ds.ClassificationDataset( + dataset=image_ds, + label_names=['stylized'], + size=len(output_img_list), + ) + def _create_and_train_model( self, train_data: classification_ds.ClassificationDataset ): @@ -128,7 +170,7 @@ class FaceStylizer(object): def _train_model( self, train_data: classification_ds.ClassificationDataset, - preprocessor: Optional[Callable[..., bool]] = None, + preprocessor: Optional[Callable[..., Any]] = None, ): """Trains the face stylizer model. @@ -147,7 +189,7 @@ class FaceStylizer(object): batch_size = self._hparams.batch_size label_in = tf.zeros(shape=[batch_size, 0]) - style_encoding = self._encoder(style_img) + style_encoding = self._encoder(style_img, training=True) + self.w_avg optimizer = tf.keras.optimizers.Adam( learning_rate=self._hparams.learning_rate, @@ -177,10 +219,7 @@ class FaceStylizer(object): ) with tf.GradientTape() as tape: - outputs = self._decoder( - {'inputs': in_latent + self.w_avg}, - training=True, - ) + outputs = self._decoder({'inputs': in_latent.numpy()}, training=True) gen_img = outputs['image'][-1] real_feature = self._discriminator( @@ -195,40 +234,86 @@ class FaceStylizer(object): tf.keras.losses.MeanAbsoluteError()(real_feature, gen_feature) * self._model_options.adv_loss_weight ) - tf.compat.v1.logging.info(f'Iteration {i} loss: {style_loss.numpy()}') + print(f'Iteration {i} loss: {style_loss.numpy()}') tvars = self._decoder.trainable_variables grads = tape.gradient(style_loss, tvars) optimizer.apply_gradients(list(zip(grads, tvars))) - # TODO: Add a metadata writer for face sytlizer model. - def export_model(self, model_name: str = 'model.tflite'): - """Converts and saves the model to a TFLite file with metadata included. + def export_model(self, model_name: str = 'face_stylizer.task'): + """Converts the model to TFLite and exports as a model bundle file. - Note that only the TFLite file is needed for deployment. This function - also saves a metadata.json file to the same directory as the TFLite file - which can be used to interpret the metadata content in the TFLite file. + Saves a model bundle file and metadata json file to hparams.export_dir. The + resulting model bundle file will contain necessary models for face + detection, face landmarks detection, and customized face stylization. Only + the model bundle file is needed for the downstream face stylization task. + The metadata.json file is saved only to interpret the contents of the model + bundle file. The face detection model and face landmarks detection model are + from https://storage.googleapis.com/mediapipe-assets/face_landmarker_v2.task + and the customized face stylization model is trained in this library. Args: - model_name: File name to save TFLite model with metadata. The full export - path is {self._hparams.export_dir}/{model_name}. + model_name: Face stylizer model bundle file name. The full export path is + {self._hparams.export_dir}/{model_name}. """ if not tf.io.gfile.exists(self._hparams.export_dir): tf.io.gfile.makedirs(self._hparams.export_dir) - tflite_file = os.path.join(self._hparams.export_dir, model_name) + model_bundle_file = os.path.join(self._hparams.export_dir, model_name) + metadata_file = os.path.join(self._hparams.export_dir, 'metadata.json') # Create an end-to-end model by concatenating encoder and decoder inputs = tf.keras.Input(shape=(256, 256, 3)) - x = self._encoder(inputs) - x = self._decoder({'inputs': x + self.w_avg}) + x = self._encoder(inputs, training=True) + x = self._decoder({'inputs': x + self.w_avg}, training=True) x = x['image'][-1] # Scale the data range from [-1, 1] to [0, 1] to support running inference # on both CPU and GPU. outputs = (x + 1.0) / 2.0 model = tf.keras.Model(inputs=inputs, outputs=outputs) - tflite_model = model_util.convert_to_tflite( + face_stylizer_model_buffer = model_util.convert_to_tflite( model=model, + quantization_config=None, + supported_ops=( + tf.lite.OpsSet.TFLITE_BUILTINS, + tf.lite.OpsSet.SELECT_TF_OPS, + ), preprocess=self._preprocessor, ) - model_util.save_tflite(tflite_model, tflite_file) + + face_aligner_task_file_path = constants.FACE_ALIGNER_TASK_FILES.get_path() + + with zipfile.ZipFile(face_aligner_task_file_path, 'r') as zf: + file_list = zf.namelist() + if _FACE_DETECTOR_MODEL not in file_list: + raise ValueError( + '{0} is not packed in face aligner task file'.format( + _FACE_DETECTOR_MODEL + ) + ) + if _FACE_LANDMARKS_DETECTOR_MODEL not in file_list: + raise ValueError( + '{0} is not packed in face aligner task file'.format( + _FACE_LANDMARKS_DETECTOR_MODEL + ) + ) + + with zf.open(_FACE_DETECTOR_MODEL) as f: + face_detector_model_buffer = f.read() + + with zf.open(_FACE_LANDMARKS_DETECTOR_MODEL) as f: + face_landmarks_detector_model_buffer = f.read() + + writer = metadata_writer.MetadataWriter.create( + bytearray(face_stylizer_model_buffer), + bytearray(face_detector_model_buffer), + bytearray(face_landmarks_detector_model_buffer), + input_norm_mean=[_NORM_MEAN], + input_norm_std=[_NORM_STD], + ) + + model_bundle_content, metadata_json = writer.populate() + with open(model_bundle_file, 'wb') as f: + f.write(model_bundle_content) + with open(metadata_file, 'w') as f: + f.write(metadata_json) diff --git a/mediapipe/model_maker/python/vision/face_stylizer/face_stylizer_test.py b/mediapipe/model_maker/python/vision/face_stylizer/face_stylizer_test.py index 354ce799..bd44fe7f 100644 --- a/mediapipe/model_maker/python/vision/face_stylizer/face_stylizer_test.py +++ b/mediapipe/model_maker/python/vision/face_stylizer/face_stylizer_test.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import zipfile import tensorflow as tf @@ -23,11 +24,22 @@ from mediapipe.tasks.python.test import test_utils class FaceStylizerTest(tf.test.TestCase): - def _load_data(self): - """Loads training dataset.""" - input_data_dir = test_utils.get_test_data_path('input/style') + def _create_training_dataset(self): + """Creates training dataset.""" + input_style_image_file = test_utils.get_test_data_path( + 'input/style/cartoon/cartoon.jpg' + ) - data = face_stylizer.Dataset.from_folder(dirname=input_data_dir) + data = face_stylizer.Dataset.from_image(filename=input_style_image_file) + return data + + def _create_eval_dataset(self): + """Create evaluation dataset.""" + input_test_image_file = test_utils.get_test_data_path( + 'input/raw/face/portrait.jpg' + ) + + data = face_stylizer.Dataset.from_image(filename=input_test_image_file) return data def _evaluate_saved_model(self, model: face_stylizer.FaceStylizer): @@ -40,7 +52,8 @@ class FaceStylizerTest(tf.test.TestCase): def setUp(self): super().setUp() - self._train_data = self._load_data() + self._train_data = self._create_training_dataset() + self._eval_data = self._create_eval_dataset() def test_finetuning_face_stylizer_with_single_input_style_image(self): with self.test_session(use_gpu=True): @@ -53,6 +66,21 @@ class FaceStylizerTest(tf.test.TestCase): ) self._evaluate_saved_model(model) + def test_evaluate_face_stylizer(self): + with self.test_session(use_gpu=True): + face_stylizer_options = face_stylizer.FaceStylizerOptions( + model=face_stylizer.SupportedModels.BLAZE_FACE_STYLIZER_256, + hparams=face_stylizer.HParams(epochs=1), + ) + model = face_stylizer.FaceStylizer.create( + train_data=self._train_data, options=face_stylizer_options + ) + eval_output = model.stylize(self._eval_data) + self.assertLen(eval_output, 1) + eval_output_data = eval_output.gen_tf_dataset() + iterator = iter(eval_output_data) + self.assertEqual(iterator.get_next().shape, (1, 256, 256, 3)) + def test_export_face_stylizer_tflite_model(self): with self.test_session(use_gpu=True): model_enum = face_stylizer.SupportedModels.BLAZE_FACE_STYLIZER_256 @@ -65,10 +93,23 @@ class FaceStylizerTest(tf.test.TestCase): model = face_stylizer.FaceStylizer.create( train_data=self._train_data, options=face_stylizer_options ) - tflite_model_name = 'custom_face_stylizer.tflite' - model.export_model(model_name=tflite_model_name) + model.export_model() + model_bundle_file = os.path.join( + self.get_temp_dir(), 'face_stylizer.task' + ) + with zipfile.ZipFile(model_bundle_file) as zf: + self.assertEqual( + set(zf.namelist()), + set([ + 'face_detector.tflite', + 'face_landmarks_detector.tflite', + 'face_stylizer.tflite', + ]), + ) + zf.extractall(self.get_temp_dir()) + face_stylizer_tflite_file = os.path.join( - self.get_temp_dir(), tflite_model_name + self.get_temp_dir(), 'face_stylizer.tflite' ) spec = face_stylizer.SupportedModels.get(model_enum) input_image_shape = spec.input_image_shape diff --git a/mediapipe/model_maker/python/vision/face_stylizer/testdata/input/style/sketch/boy-6030802_1280.jpg b/mediapipe/model_maker/python/vision/face_stylizer/testdata/input/style/sketch/boy-6030802_1280.jpg new file mode 100644 index 00000000..042f2c9d Binary files /dev/null and b/mediapipe/model_maker/python/vision/face_stylizer/testdata/input/style/sketch/boy-6030802_1280.jpg differ diff --git a/mediapipe/model_maker/python/vision/gesture_recognizer/BUILD b/mediapipe/model_maker/python/vision/gesture_recognizer/BUILD index ecd2a712..969887e6 100644 --- a/mediapipe/model_maker/python/vision/gesture_recognizer/BUILD +++ b/mediapipe/model_maker/python/vision/gesture_recognizer/BUILD @@ -13,7 +13,7 @@ # limitations under the License. # Placeholder for internal Python strict test compatibility macro. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary and library compatibility macro. licenses(["notice"]) diff --git a/mediapipe/model_maker/python/vision/gesture_recognizer/dataset.py b/mediapipe/model_maker/python/vision/gesture_recognizer/dataset.py index 1ba626be..8e2095a3 100644 --- a/mediapipe/model_maker/python/vision/gesture_recognizer/dataset.py +++ b/mediapipe/model_maker/python/vision/gesture_recognizer/dataset.py @@ -249,5 +249,6 @@ class Dataset(classification_dataset.ClassificationDataset): len(valid_hand_data), len(label_names), ','.join(label_names))) return Dataset( dataset=hand_embedding_label_ds, + label_names=label_names, size=len(valid_hand_data), - label_names=label_names) + ) diff --git a/mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer.py b/mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer.py index 66934304..8335968b 100644 --- a/mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer.py +++ b/mediapipe/model_maker/python/vision/gesture_recognizer/gesture_recognizer.py @@ -54,7 +54,7 @@ class GestureRecognizer(classifier.Classifier): self._model_options = model_options self._hparams = hparams self._loss_function = loss_functions.FocalLoss(gamma=self._hparams.gamma) - self._metric_function = 'categorical_accuracy' + self._metric_functions = ['categorical_accuracy'] self._optimizer = 'adam' self._callbacks = self._get_callbacks() self._history = None diff --git a/mediapipe/model_maker/python/vision/image_classifier/BUILD b/mediapipe/model_maker/python/vision/image_classifier/BUILD index 73d1d2f7..a9d91e84 100644 --- a/mediapipe/model_maker/python/vision/image_classifier/BUILD +++ b/mediapipe/model_maker/python/vision/image_classifier/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary and library compatibility macro. # Placeholder for internal Python library rule. licenses(["notice"]) diff --git a/mediapipe/model_maker/python/vision/image_classifier/dataset.py b/mediapipe/model_maker/python/vision/image_classifier/dataset.py index 6bc180be..f627dfec 100644 --- a/mediapipe/model_maker/python/vision/image_classifier/dataset.py +++ b/mediapipe/model_maker/python/vision/image_classifier/dataset.py @@ -15,28 +15,12 @@ import os import random - -from typing import List, Optional import tensorflow as tf -import tensorflow_datasets as tfds from mediapipe.model_maker.python.core.data import classification_dataset from mediapipe.model_maker.python.vision.core import image_utils -def _create_data( - name: str, data: tf.data.Dataset, info: tfds.core.DatasetInfo, - label_names: List[str] -) -> Optional[classification_dataset.ClassificationDataset]: - """Creates a Dataset object from tfds data.""" - if name not in data: - return None - data = data[name] - data = data.map(lambda a: (a['image'], a['label'])) - size = info.splits[name].num_examples - return Dataset(data, size, label_names) - - class Dataset(classification_dataset.ClassificationDataset): """Dataset library for image classifier.""" @@ -99,4 +83,5 @@ class Dataset(classification_dataset.ClassificationDataset): 'Load image with size: %d, num_label: %d, labels: %s.', all_image_size, all_label_size, ', '.join(label_names)) return Dataset( - dataset=image_label_ds, size=all_image_size, label_names=label_names) + dataset=image_label_ds, label_names=label_names, size=all_image_size + ) diff --git a/mediapipe/model_maker/python/vision/image_classifier/dataset_test.py b/mediapipe/model_maker/python/vision/image_classifier/dataset_test.py index 63fa666b..33101382 100644 --- a/mediapipe/model_maker/python/vision/image_classifier/dataset_test.py +++ b/mediapipe/model_maker/python/vision/image_classifier/dataset_test.py @@ -41,7 +41,7 @@ class DatasetTest(tf.test.TestCase): def test_split(self): ds = tf.data.Dataset.from_tensor_slices([[0, 1], [1, 1], [0, 0], [1, 0]]) - data = dataset.Dataset(dataset=ds, size=4, label_names=['pos', 'neg']) + data = dataset.Dataset(dataset=ds, label_names=['pos', 'neg'], size=4) train_data, test_data = data.split(fraction=0.5) self.assertLen(train_data, 2) diff --git a/mediapipe/model_maker/python/vision/image_classifier/image_classifier.py b/mediapipe/model_maker/python/vision/image_classifier/image_classifier.py index 3838a5a1..8acf59f6 100644 --- a/mediapipe/model_maker/python/vision/image_classifier/image_classifier.py +++ b/mediapipe/model_maker/python/vision/image_classifier/image_classifier.py @@ -59,7 +59,7 @@ class ImageClassifier(classifier.Classifier): self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir) self._loss_function = tf.keras.losses.CategoricalCrossentropy( label_smoothing=self._hparams.label_smoothing) - self._metric_function = 'accuracy' + self._metric_functions = ['accuracy'] self._history = None # Training history returned from `keras_model.fit`. @classmethod diff --git a/mediapipe/model_maker/python/vision/image_classifier/image_classifier_test.py b/mediapipe/model_maker/python/vision/image_classifier/image_classifier_test.py index 4b1ea607..71a47d9e 100644 --- a/mediapipe/model_maker/python/vision/image_classifier/image_classifier_test.py +++ b/mediapipe/model_maker/python/vision/image_classifier/image_classifier_test.py @@ -52,8 +52,9 @@ class ImageClassifierTest(tf.test.TestCase, parameterized.TestCase): ds = tf.data.Dataset.from_generator( self._gen, (tf.uint8, tf.int64), (tf.TensorShape( [self.IMAGE_SIZE, self.IMAGE_SIZE, 3]), tf.TensorShape([]))) - data = image_classifier.Dataset(ds, self.IMAGES_PER_CLASS * 3, - ['cyan', 'magenta', 'yellow']) + data = image_classifier.Dataset( + ds, ['cyan', 'magenta', 'yellow'], self.IMAGES_PER_CLASS * 3 + ) return data def setUp(self): diff --git a/mediapipe/model_maker/python/vision/object_detector/BUILD b/mediapipe/model_maker/python/vision/object_detector/BUILD index 75c08dbc..14d378a1 100644 --- a/mediapipe/model_maker/python/vision/object_detector/BUILD +++ b/mediapipe/model_maker/python/vision/object_detector/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary and library compatibility macro. # Placeholder for internal Python strict test compatibility macro. licenses(["notice"]) @@ -54,6 +54,7 @@ py_library( srcs = ["dataset.py"], deps = [ ":dataset_util", + "//mediapipe/model_maker/python/core/data:cache_files", "//mediapipe/model_maker/python/core/data:classification_dataset", ], ) @@ -73,6 +74,7 @@ py_test( py_library( name = "dataset_util", srcs = ["dataset_util.py"], + deps = ["//mediapipe/model_maker/python/core/data:cache_files"], ) py_test( diff --git a/mediapipe/model_maker/python/vision/object_detector/dataset.py b/mediapipe/model_maker/python/vision/object_detector/dataset.py index c18a071b..f7751915 100644 --- a/mediapipe/model_maker/python/vision/object_detector/dataset.py +++ b/mediapipe/model_maker/python/vision/object_detector/dataset.py @@ -16,8 +16,8 @@ from typing import Optional import tensorflow as tf -import yaml +from mediapipe.model_maker.python.core.data import cache_files from mediapipe.model_maker.python.core.data import classification_dataset from mediapipe.model_maker.python.vision.object_detector import dataset_util from official.vision.dataloaders import tf_example_decoder @@ -76,14 +76,16 @@ class Dataset(classification_dataset.ClassificationDataset): ValueError: If the label_name for id 0 is set to something other than the 'background' class. """ - cache_files = dataset_util.get_cache_files_coco(data_dir, cache_dir) - if not dataset_util.is_cached(cache_files): + tfrecord_cache_files = dataset_util.get_cache_files_coco( + data_dir, cache_dir + ) + if not tfrecord_cache_files.is_cached(): label_map = dataset_util.get_label_map_coco(data_dir) cache_writer = dataset_util.COCOCacheFilesWriter( label_map=label_map, max_num_images=max_num_images ) - cache_writer.write_files(cache_files, data_dir) - return cls.from_cache(cache_files.cache_prefix) + cache_writer.write_files(tfrecord_cache_files, data_dir) + return cls.from_cache(tfrecord_cache_files) @classmethod def from_pascal_voc_folder( @@ -134,47 +136,48 @@ class Dataset(classification_dataset.ClassificationDataset): Raises: ValueError: if the input data directory is empty. """ - cache_files = dataset_util.get_cache_files_pascal_voc(data_dir, cache_dir) - if not dataset_util.is_cached(cache_files): + tfrecord_cache_files = dataset_util.get_cache_files_pascal_voc( + data_dir, cache_dir + ) + if not tfrecord_cache_files.is_cached(): label_map = dataset_util.get_label_map_pascal_voc(data_dir) cache_writer = dataset_util.PascalVocCacheFilesWriter( label_map=label_map, max_num_images=max_num_images ) - cache_writer.write_files(cache_files, data_dir) + cache_writer.write_files(tfrecord_cache_files, data_dir) - return cls.from_cache(cache_files.cache_prefix) + return cls.from_cache(tfrecord_cache_files) @classmethod - def from_cache(cls, cache_prefix: str) -> 'Dataset': + def from_cache( + cls, tfrecord_cache_files: cache_files.TFRecordCacheFiles + ) -> 'Dataset': """Loads the TFRecord data from cache. Args: - cache_prefix: The cache prefix including the cache directory and the cache - prefix filename, e.g: '/tmp/cache/train'. + tfrecord_cache_files: The TFRecordCacheFiles object containing the already + cached TFRecord and metadata files. Returns: ObjectDetectorDataset object. + + Raises: + ValueError if tfrecord_cache_files are not already cached. """ - # Get TFRecord Files - tfrecord_file_pattern = cache_prefix + '*.tfrecord' - matched_files = tf.io.gfile.glob(tfrecord_file_pattern) - if not matched_files: - raise ValueError('TFRecord files are empty.') + if not tfrecord_cache_files.is_cached(): + raise ValueError( + 'Cache files must be already cached to use the from_cache method.' + ) - # Load meta_data. - meta_data_file = cache_prefix + dataset_util.META_DATA_FILE_SUFFIX - if not tf.io.gfile.exists(meta_data_file): - raise ValueError("Metadata file %s doesn't exist." % meta_data_file) - with tf.io.gfile.GFile(meta_data_file, 'r') as f: - meta_data = yaml.load(f, Loader=yaml.FullLoader) + metadata = tfrecord_cache_files.load_metadata() - dataset = tf.data.TFRecordDataset(matched_files) + dataset = tf.data.TFRecordDataset(tfrecord_cache_files.tfrecord_files) decoder = tf_example_decoder.TfExampleDecoder(regenerate_source_id=False) dataset = dataset.map(decoder.decode, num_parallel_calls=tf.data.AUTOTUNE) - label_map = meta_data['label_map'] + label_map = metadata['label_map'] label_names = [label_map[k] for k in sorted(label_map.keys())] return Dataset( - dataset=dataset, size=meta_data['size'], label_names=label_names + dataset=dataset, label_names=label_names, size=metadata['size'] ) diff --git a/mediapipe/model_maker/python/vision/object_detector/dataset_util.py b/mediapipe/model_maker/python/vision/object_detector/dataset_util.py index 74d082f9..fbb821b3 100644 --- a/mediapipe/model_maker/python/vision/object_detector/dataset_util.py +++ b/mediapipe/model_maker/python/vision/object_detector/dataset_util.py @@ -15,25 +15,20 @@ import abc import collections -import dataclasses import hashlib import json import math import os import tempfile -from typing import Any, Dict, List, Mapping, Optional, Sequence +from typing import Any, Dict, List, Mapping, Optional import xml.etree.ElementTree as ET import tensorflow as tf -import yaml +from mediapipe.model_maker.python.core.data import cache_files from official.vision.data import tfrecord_lib -# Suffix of the meta data file name. -META_DATA_FILE_SUFFIX = '_meta_data.yaml' - - def _xml_get(node: ET.Element, name: str) -> ET.Element: """Gets a named child from an XML Element node. @@ -71,18 +66,9 @@ def _get_dir_basename(data_dir: str) -> str: return os.path.basename(os.path.abspath(data_dir)) -@dataclasses.dataclass(frozen=True) -class CacheFiles: - """Cache files for object detection.""" - - cache_prefix: str - tfrecord_files: Sequence[str] - meta_data_file: str - - def _get_cache_files( cache_dir: Optional[str], cache_prefix_filename: str, num_shards: int = 10 -) -> CacheFiles: +) -> cache_files.TFRecordCacheFiles: """Creates an object of CacheFiles class. Args: @@ -96,28 +82,16 @@ def _get_cache_files( An object of CacheFiles class. """ cache_dir = _get_cache_dir_or_create(cache_dir) - # The cache prefix including the cache directory and the cache prefix - # filename, e.g: '/tmp/cache/train'. - cache_prefix = os.path.join(cache_dir, cache_prefix_filename) - tf.compat.v1.logging.info( - 'Cache will be stored in %s with prefix filename %s. Cache_prefix is %s' - % (cache_dir, cache_prefix_filename, cache_prefix) - ) - - # Cached files including the TFRecord files and the meta data file. - tfrecord_files = [ - cache_prefix + '-%05d-of-%05d.tfrecord' % (i, num_shards) - for i in range(num_shards) - ] - meta_data_file = cache_prefix + META_DATA_FILE_SUFFIX - return CacheFiles( - cache_prefix=cache_prefix, - tfrecord_files=tuple(tfrecord_files), - meta_data_file=meta_data_file, + return cache_files.TFRecordCacheFiles( + cache_prefix_filename=cache_prefix_filename, + cache_dir=cache_dir, + num_shards=num_shards, ) -def get_cache_files_coco(data_dir: str, cache_dir: str) -> CacheFiles: +def get_cache_files_coco( + data_dir: str, cache_dir: str +) -> cache_files.TFRecordCacheFiles: """Creates an object of CacheFiles class using a COCO formatted dataset. Args: @@ -152,7 +126,9 @@ def get_cache_files_coco(data_dir: str, cache_dir: str) -> CacheFiles: return _get_cache_files(cache_dir, cache_prefix_filename, num_shards) -def get_cache_files_pascal_voc(data_dir: str, cache_dir: str) -> CacheFiles: +def get_cache_files_pascal_voc( + data_dir: str, cache_dir: str +) -> cache_files.TFRecordCacheFiles: """Gets an object of CacheFiles using a PASCAL VOC formatted dataset. Args: @@ -181,14 +157,6 @@ def get_cache_files_pascal_voc(data_dir: str, cache_dir: str) -> CacheFiles: return _get_cache_files(cache_dir, cache_prefix_filename, num_shards) -def is_cached(cache_files: CacheFiles) -> bool: - """Checks whether cache files are already cached.""" - all_cached_files = list(cache_files.tfrecord_files) + [ - cache_files.meta_data_file - ] - return all(tf.io.gfile.exists(path) for path in all_cached_files) - - class CacheFilesWriter(abc.ABC): """CacheFilesWriter class to write the cached files.""" @@ -208,19 +176,22 @@ class CacheFilesWriter(abc.ABC): self.label_map = label_map self.max_num_images = max_num_images - def write_files(self, cache_files: CacheFiles, *args, **kwargs) -> None: - """Writes TFRecord and meta_data files. + def write_files( + self, + tfrecord_cache_files: cache_files.TFRecordCacheFiles, + *args, + **kwargs, + ) -> None: + """Writes TFRecord and metadata files. Args: - cache_files: CacheFiles object including a list of TFRecord files and the - meta data yaml file to save the meta_data including data size and - label_map. + tfrecord_cache_files: TFRecordCacheFiles object including a list of + TFRecord files and the meta data yaml file to save the metadata + including data size and label_map. *args: Non-keyword of parameters used in the `_get_example` method. **kwargs: Keyword parameters used in the `_get_example` method. """ - writers = [ - tf.io.TFRecordWriter(path) for path in cache_files.tfrecord_files - ] + writers = tfrecord_cache_files.get_writers() # Writes tf.Example into TFRecord files. size = 0 @@ -235,10 +206,9 @@ class CacheFilesWriter(abc.ABC): for writer in writers: writer.close() - # Writes meta_data into meta_data_file. - meta_data = {'size': size, 'label_map': self.label_map} - with tf.io.gfile.GFile(cache_files.meta_data_file, 'w') as f: - yaml.dump(meta_data, f) + # Writes metadata into metadata_file. + metadata = {'size': size, 'label_map': self.label_map} + tfrecord_cache_files.save_metadata(metadata) @abc.abstractmethod def _get_example(self, *args, **kwargs): diff --git a/mediapipe/model_maker/python/vision/object_detector/dataset_util_test.py b/mediapipe/model_maker/python/vision/object_detector/dataset_util_test.py index 6daea1f4..250c5d45 100644 --- a/mediapipe/model_maker/python/vision/object_detector/dataset_util_test.py +++ b/mediapipe/model_maker/python/vision/object_detector/dataset_util_test.py @@ -19,7 +19,6 @@ import shutil from unittest import mock as unittest_mock import tensorflow as tf -import yaml from mediapipe.model_maker.python.vision.core import test_utils from mediapipe.model_maker.python.vision.object_detector import dataset_util @@ -30,13 +29,10 @@ class DatasetUtilTest(tf.test.TestCase): def _assert_cache_files_equal(self, cf1, cf2): self.assertEqual(cf1.cache_prefix, cf2.cache_prefix) - self.assertCountEqual(cf1.tfrecord_files, cf2.tfrecord_files) - self.assertEqual(cf1.meta_data_file, cf2.meta_data_file) + self.assertEqual(cf1.num_shards, cf2.num_shards) def _assert_cache_files_not_equal(self, cf1, cf2): self.assertNotEqual(cf1.cache_prefix, cf2.cache_prefix) - self.assertNotEqual(cf1.tfrecord_files, cf2.tfrecord_files) - self.assertNotEqual(cf1.meta_data_file, cf2.meta_data_file) def _get_cache_files_and_assert_neq_fn(self, cache_files_fn): def get_cache_files_and_assert_neq(cf, data_dir, cache_dir): @@ -57,7 +53,7 @@ class DatasetUtilTest(tf.test.TestCase): self.assertEqual( cache_files.tfrecord_files[0], '/tmp/train-00000-of-00001.tfrecord' ) - self.assertEqual(cache_files.meta_data_file, '/tmp/train_meta_data.yaml') + self.assertEqual(cache_files.metadata_file, '/tmp/train_metadata.yaml') def test_matching_get_cache_files_coco(self): cache_dir = self.create_tempdir() @@ -118,7 +114,7 @@ class DatasetUtilTest(tf.test.TestCase): self.assertEqual( cache_files.tfrecord_files[0], '/tmp/train-00000-of-00001.tfrecord' ) - self.assertEqual(cache_files.meta_data_file, '/tmp/train_meta_data.yaml') + self.assertEqual(cache_files.metadata_file, '/tmp/train_metadata.yaml') def test_matching_get_cache_files_pascal_voc(self): cache_dir = self.create_tempdir() @@ -173,13 +169,13 @@ class DatasetUtilTest(tf.test.TestCase): cache_files = dataset_util.get_cache_files_coco( tasks_test_utils.get_test_data_path('coco_data'), cache_dir=tempdir ) - self.assertFalse(dataset_util.is_cached(cache_files)) + self.assertFalse(cache_files.is_cached()) with open(cache_files.tfrecord_files[0], 'w') as f: f.write('test') - self.assertFalse(dataset_util.is_cached(cache_files)) - with open(cache_files.meta_data_file, 'w') as f: + self.assertFalse(cache_files.is_cached()) + with open(cache_files.metadata_file, 'w') as f: f.write('test') - self.assertTrue(dataset_util.is_cached(cache_files)) + self.assertTrue(cache_files.is_cached()) def test_get_label_map_coco(self): coco_dir = tasks_test_utils.get_test_data_path('coco_data') @@ -203,13 +199,11 @@ class DatasetUtilTest(tf.test.TestCase): self.assertTrue(os.path.isfile(cache_files.tfrecord_files[0])) self.assertGreater(os.path.getsize(cache_files.tfrecord_files[0]), 0) - # Checks the meta_data file - self.assertTrue(os.path.isfile(cache_files.meta_data_file)) - self.assertGreater(os.path.getsize(cache_files.meta_data_file), 0) - with tf.io.gfile.GFile(cache_files.meta_data_file, 'r') as f: - meta_data_dict = yaml.load(f, Loader=yaml.FullLoader) - # Size is 3 because some examples are skipped for having poor bboxes - self.assertEqual(meta_data_dict['size'], expected_size) + # Checks the metadata file + self.assertTrue(os.path.isfile(cache_files.metadata_file)) + self.assertGreater(os.path.getsize(cache_files.metadata_file), 0) + metadata_dict = cache_files.load_metadata() + self.assertEqual(metadata_dict['size'], expected_size) def test_coco_cache_files_writer(self): tempdir = self.create_tempdir() diff --git a/mediapipe/model_maker/python/vision/object_detector/model.py b/mediapipe/model_maker/python/vision/object_detector/model.py index b1b4951f..ea78ca8c 100644 --- a/mediapipe/model_maker/python/vision/object_detector/model.py +++ b/mediapipe/model_maker/python/vision/object_detector/model.py @@ -74,8 +74,8 @@ class ObjectDetectorModel(tf.keras.Model): generator_config: configs.retinanet.DetectionGenerator = configs.retinanet.DetectionGenerator(), ) -> configs.retinanet.RetinaNet: model_config = configs.retinanet.RetinaNet( - min_level=3, - max_level=7, + min_level=self._model_spec.min_level, + max_level=self._model_spec.max_level, num_classes=self._num_classes, input_size=self._model_spec.input_image_shape, anchor=configs.retinanet.Anchor( diff --git a/mediapipe/model_maker/python/vision/object_detector/model_spec.py b/mediapipe/model_maker/python/vision/object_detector/model_spec.py index 9c89c4ed..ad043e87 100644 --- a/mediapipe/model_maker/python/vision/object_detector/model_spec.py +++ b/mediapipe/model_maker/python/vision/object_detector/model_spec.py @@ -20,18 +20,30 @@ from typing import List from mediapipe.model_maker.python.core.utils import file_util -MOBILENET_V2_FILES = file_util.DownloadedFiles( - 'object_detector/mobilenetv2', +MOBILENET_V2_I256_FILES = file_util.DownloadedFiles( + 'object_detector/mobilenetv2_i256', 'https://storage.googleapis.com/tf_model_garden/vision/qat/mobilenetv2_ssd_coco/mobilenetv2_ssd_i256_ckpt.tar.gz', is_folder=True, ) +MOBILENET_V2_I320_FILES = file_util.DownloadedFiles( + 'object_detector/mobilenetv2_i320', + 'https://storage.googleapis.com/tf_model_garden/vision/qat/mobilenetv2_ssd_coco/mobilenetv2_ssd_i320_ckpt.tar.gz', + is_folder=True, +) + MOBILENET_MULTI_AVG_FILES = file_util.DownloadedFiles( 'object_detector/mobilenetmultiavg', 'https://storage.googleapis.com/tf_model_garden/vision/qat/mobilenetv3.5_ssd_coco/mobilenetv3.5_ssd_i256_ckpt.tar.gz', is_folder=True, ) +MOBILENET_MULTI_AVG_I384_FILES = file_util.DownloadedFiles( + 'object_detector/mobilenetmultiavg_i384', + 'https://storage.googleapis.com/tf_model_garden/vision/qat/mobilenetv2_ssd_coco/mobilenetv3.5_ssd_i384_ckpt.tar.gz', + is_folder=True, +) + @dataclasses.dataclass class ModelSpec(object): @@ -48,30 +60,66 @@ class ModelSpec(object): input_image_shape: List[int] model_id: str + # Model Config values + min_level: int + max_level: int -mobilenet_v2_spec = functools.partial( + +mobilenet_v2_i256_spec = functools.partial( ModelSpec, - downloaded_files=MOBILENET_V2_FILES, + downloaded_files=MOBILENET_V2_I256_FILES, checkpoint_name='ckpt-277200', input_image_shape=[256, 256, 3], model_id='MobileNetV2', + min_level=3, + max_level=7, ) -mobilenet_multi_avg_spec = functools.partial( +mobilenet_v2_i320_spec = functools.partial( + ModelSpec, + downloaded_files=MOBILENET_V2_I320_FILES, + checkpoint_name='ckpt-277200', + input_image_shape=[320, 320, 3], + model_id='MobileNetV2', + min_level=3, + max_level=6, +) + +mobilenet_multi_avg_i256_spec = functools.partial( ModelSpec, downloaded_files=MOBILENET_MULTI_AVG_FILES, checkpoint_name='ckpt-277200', input_image_shape=[256, 256, 3], model_id='MobileNetMultiAVG', + min_level=3, + max_level=7, +) + +mobilenet_multi_avg_i384_spec = functools.partial( + ModelSpec, + downloaded_files=MOBILENET_MULTI_AVG_I384_FILES, + checkpoint_name='ckpt-277200', + input_image_shape=[384, 384, 3], + model_id='MobileNetMultiAVG', + min_level=3, + max_level=7, ) @enum.unique class SupportedModels(enum.Enum): - """Predefined object detector model specs supported by Model Maker.""" + """Predefined object detector model specs supported by Model Maker. - MOBILENET_V2 = mobilenet_v2_spec - MOBILENET_MULTI_AVG = mobilenet_multi_avg_spec + Supported models include the following: + - MOBILENET_V2: MobileNetV2 256x256 input + - MOBILENET_V2_I320: MobileNetV2 320x320 input + - MOBILENET_MULTI_AVG: MobileNet-MultiHW-AVG 256x256 input + - MOBILENET_MULTI_AVG_I384: MobileNet-MultiHW-AVG 384x384 input + """ + MOBILENET_V2 = mobilenet_v2_i256_spec + MOBILENET_V2_I320 = mobilenet_v2_i320_spec + MOBILENET_MULTI_AVG = mobilenet_multi_avg_i256_spec + MOBILENET_MULTI_AVG_I384 = mobilenet_multi_avg_i384_spec @classmethod def get(cls, spec: 'SupportedModels') -> 'ModelSpec': diff --git a/mediapipe/model_maker/python/vision/object_detector/object_detector.py b/mediapipe/model_maker/python/vision/object_detector/object_detector.py index 486c3ffa..6c7b9811 100644 --- a/mediapipe/model_maker/python/vision/object_detector/object_detector.py +++ b/mediapipe/model_maker/python/vision/object_detector/object_detector.py @@ -395,7 +395,7 @@ class ObjectDetector(classifier.Classifier): ) -> tf.keras.optimizers.Optimizer: """Creates an optimizer with learning rate schedule for regular training. - Uses Keras PiecewiseConstantDecay schedule by default. + Uses Keras CosineDecay schedule by default. Args: steps_per_epoch: Steps per epoch to calculate the step boundaries from the @@ -404,6 +404,8 @@ class ObjectDetector(classifier.Classifier): Returns: A tf.keras.optimizer.Optimizer for model training. """ + total_steps = steps_per_epoch * self._hparams.epochs + warmup_steps = int(total_steps * 0.1) init_lr = self._hparams.learning_rate * self._hparams.batch_size / 256 decay_epochs = ( self._hparams.cosine_decay_epochs @@ -415,6 +417,11 @@ class ObjectDetector(classifier.Classifier): steps_per_epoch * decay_epochs, self._hparams.cosine_decay_alpha, ) + learning_rate = model_util.WarmUp( + initial_learning_rate=init_lr, + decay_schedule_fn=learning_rate, + warmup_steps=warmup_steps, + ) return tf.keras.optimizers.experimental.SGD( learning_rate=learning_rate, momentum=0.9 ) diff --git a/mediapipe/model_maker/python/vision/object_detector/preprocessor.py b/mediapipe/model_maker/python/vision/object_detector/preprocessor.py index ebea6a07..1388cc7d 100644 --- a/mediapipe/model_maker/python/vision/object_detector/preprocessor.py +++ b/mediapipe/model_maker/python/vision/object_detector/preprocessor.py @@ -32,8 +32,8 @@ class Preprocessor(object): self._mean_norm = model_spec.mean_norm self._stddev_norm = model_spec.stddev_norm self._output_size = model_spec.input_image_shape[:2] - self._min_level = 3 - self._max_level = 7 + self._min_level = model_spec.min_level + self._max_level = model_spec.max_level self._num_scales = 3 self._aspect_ratios = [0.5, 1, 2] self._anchor_size = 3 diff --git a/mediapipe/model_maker/requirements.txt b/mediapipe/model_maker/requirements.txt index 5c78dc58..a1c975c1 100644 --- a/mediapipe/model_maker/requirements.txt +++ b/mediapipe/model_maker/requirements.txt @@ -3,6 +3,7 @@ mediapipe>=0.10.0 numpy opencv-python tensorflow>=2.10 +tensorflow-addons tensorflow-datasets tensorflow-hub -tf-models-official==2.11.6 +tf-models-official>=2.13.1 diff --git a/mediapipe/modules/objectron/calculators/BUILD b/mediapipe/modules/objectron/calculators/BUILD index 2e33ebf6..05b25475 100644 --- a/mediapipe/modules/objectron/calculators/BUILD +++ b/mediapipe/modules/objectron/calculators/BUILD @@ -135,6 +135,7 @@ cc_library( "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/util/tracking:box_tracker_cc_proto", + "@com_google_absl//absl/log:absl_check", ], ) @@ -146,10 +147,11 @@ cc_library( ":annotation_cc_proto", ":box_util", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/util/tracking:box_tracker_cc_proto", "@com_google_absl//absl/container:btree", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -163,6 +165,7 @@ cc_library( ], deps = [ "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/strings:str_format", "@eigen_archive//:eigen3", @@ -182,10 +185,11 @@ cc_library( ":belief_decoder_config_cc_proto", ":box", ":epnp", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@eigen_archive//:eigen3", ], @@ -203,6 +207,7 @@ cc_library( "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", + "@com_google_absl//absl/log:absl_check", "@org_tensorflow//tensorflow/lite:framework", ], ) @@ -223,6 +228,7 @@ cc_library( ":annotation_cc_proto", ":object_cc_proto", "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@eigen_archive//:eigen3", ], ) @@ -277,6 +283,8 @@ cc_library( "//mediapipe/framework/deps:file_path", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", @@ -301,6 +309,7 @@ cc_library( "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", @@ -322,6 +331,7 @@ cc_library( "//mediapipe/framework/deps:file_path", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", @@ -369,11 +379,11 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:map_util", "//mediapipe/framework/port:re2", "//mediapipe/framework/port:status", "@com_google_absl//absl/container:node_hash_set", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], alwayslink = 1, @@ -417,5 +427,6 @@ cc_test( "//mediapipe/framework/port:logging", "//mediapipe/util/tracking:box_tracker_cc_proto", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", ], ) diff --git a/mediapipe/modules/objectron/calculators/box.cc b/mediapipe/modules/objectron/calculators/box.cc index bd2ce57f..9b3e4348 100644 --- a/mediapipe/modules/objectron/calculators/box.cc +++ b/mediapipe/modules/objectron/calculators/box.cc @@ -15,6 +15,7 @@ #include "mediapipe/modules/objectron/calculators/box.h" #include "Eigen/Core" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -107,12 +108,12 @@ void Box::Adjust(const std::vector& variables) { } float* Box::GetVertex(size_t vertex_id) { - CHECK_LT(vertex_id, kNumKeypoints); + ABSL_CHECK_LT(vertex_id, kNumKeypoints); return bounding_box_[vertex_id].data(); } const float* Box::GetVertex(size_t vertex_id) const { - CHECK_LT(vertex_id, kNumKeypoints); + ABSL_CHECK_LT(vertex_id, kNumKeypoints); return bounding_box_[vertex_id].data(); } @@ -135,7 +136,7 @@ bool Box::InsideTest(const Eigen::Vector3f& point, int check_axis) const { } void Box::Deserialize(const Object& obj) { - CHECK_EQ(obj.keypoints_size(), kNumKeypoints); + ABSL_CHECK_EQ(obj.keypoints_size(), kNumKeypoints); Model::Deserialize(obj); } @@ -222,7 +223,7 @@ std::pair Box::GetGroundPlane() const { template void Box::Fit(const std::vector& vertices) { - CHECK_EQ(vertices.size(), kNumKeypoints); + ABSL_CHECK_EQ(vertices.size(), kNumKeypoints); scale_.setZero(); // The scale would remain invariant under rotation and translation. // We can safely estimate the scale from the oriented box. diff --git a/mediapipe/modules/objectron/calculators/box_util.cc b/mediapipe/modules/objectron/calculators/box_util.cc index 0663b5bd..c19fa5be 100644 --- a/mediapipe/modules/objectron/calculators/box_util.cc +++ b/mediapipe/modules/objectron/calculators/box_util.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" @@ -24,7 +25,7 @@ namespace mediapipe { void ComputeBoundingRect(const std::vector& points, mediapipe::TimedBoxProto* box) { - CHECK(box != nullptr); + ABSL_CHECK(box != nullptr); float top = 1.0f; float bottom = 0.0f; float left = 1.0f; diff --git a/mediapipe/modules/objectron/calculators/decoder.cc b/mediapipe/modules/objectron/calculators/decoder.cc index 0af34585..b823490d 100644 --- a/mediapipe/modules/objectron/calculators/decoder.cc +++ b/mediapipe/modules/objectron/calculators/decoder.cc @@ -19,9 +19,10 @@ #include "Eigen/Core" #include "Eigen/Dense" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "mediapipe/framework/port/canonical_errors.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/modules/objectron/calculators/annotation_data.pb.h" @@ -46,10 +47,10 @@ inline void SetPoint3d(const Eigen::Vector3f& point_vec, Point3D* point_3d) { FrameAnnotation Decoder::DecodeBoundingBoxKeypoints( const cv::Mat& heatmap, const cv::Mat& offsetmap) const { - CHECK_EQ(1, heatmap.channels()); - CHECK_EQ(kNumOffsetmaps, offsetmap.channels()); - CHECK_EQ(heatmap.cols, offsetmap.cols); - CHECK_EQ(heatmap.rows, offsetmap.rows); + ABSL_CHECK_EQ(1, heatmap.channels()); + ABSL_CHECK_EQ(kNumOffsetmaps, offsetmap.channels()); + ABSL_CHECK_EQ(heatmap.cols, offsetmap.cols); + ABSL_CHECK_EQ(heatmap.rows, offsetmap.rows); const float offset_scale = std::min(offsetmap.cols, offsetmap.rows); const std::vector center_points = ExtractCenterKeypoints(heatmap); @@ -201,10 +202,10 @@ std::vector Decoder::ExtractCenterKeypoints( absl::Status Decoder::Lift2DTo3D( const Eigen::Matrix& projection_matrix, bool portrait, FrameAnnotation* estimated_box) const { - CHECK(estimated_box != nullptr); + ABSL_CHECK(estimated_box != nullptr); for (auto& annotation : *estimated_box->mutable_annotations()) { - CHECK_EQ(kNumKeypoints, annotation.keypoints_size()); + ABSL_CHECK_EQ(kNumKeypoints, annotation.keypoints_size()); // Fill input 2D Points; std::vector input_points_2d; @@ -220,7 +221,7 @@ absl::Status Decoder::Lift2DTo3D( auto status = SolveEpnp(projection_matrix, portrait, input_points_2d, &output_points_3d); if (!status.ok()) { - LOG(ERROR) << status; + ABSL_LOG(ERROR) << status; return status; } diff --git a/mediapipe/modules/objectron/calculators/epnp.cc b/mediapipe/modules/objectron/calculators/epnp.cc index 8bd7151f..03b78c72 100644 --- a/mediapipe/modules/objectron/calculators/epnp.cc +++ b/mediapipe/modules/objectron/calculators/epnp.cc @@ -14,6 +14,8 @@ #include "mediapipe/modules/objectron/calculators/epnp.h" +#include "absl/log/absl_check.h" + namespace mediapipe { namespace { @@ -126,7 +128,7 @@ absl::Status SolveEpnp(const float focal_x, const float focal_y, if (eigen_solver.info() != Eigen::Success) { return absl::AbortedError("Eigen decomposition failed."); } - CHECK_EQ(12, eigen_solver.eigenvalues().size()); + ABSL_CHECK_EQ(12, eigen_solver.eigenvalues().size()); // Eigenvalues are sorted in increasing order for SelfAdjointEigenSolver // only! If you use other Eigen Solvers, it's not guaranteed to be in diff --git a/mediapipe/modules/objectron/calculators/filter_detection_calculator.cc b/mediapipe/modules/objectron/calculators/filter_detection_calculator.cc index 29f4c79d..3ac91c7c 100644 --- a/mediapipe/modules/objectron/calculators/filter_detection_calculator.cc +++ b/mediapipe/modules/objectron/calculators/filter_detection_calculator.cc @@ -17,13 +17,13 @@ #include #include "absl/container/node_hash_set.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/map_util.h" #include "mediapipe/framework/port/re2.h" #include "mediapipe/framework/port/status.h" @@ -264,11 +264,11 @@ bool FilterDetectionCalculator::IsValidLabel(const std::string& label) { bool FilterDetectionCalculator::IsValidScore(float score) { if (options_.has_min_score() && score < options_.min_score()) { - LOG(ERROR) << "Filter out detection with low score " << score; + ABSL_LOG(ERROR) << "Filter out detection with low score " << score; return false; } if (options_.has_max_score() && score > options_.max_score()) { - LOG(ERROR) << "Filter out detection with high score " << score; + ABSL_LOG(ERROR) << "Filter out detection with high score " << score; return false; } return true; diff --git a/mediapipe/modules/objectron/calculators/frame_annotation_tracker.cc b/mediapipe/modules/objectron/calculators/frame_annotation_tracker.cc index 1685a4f6..d060af35 100644 --- a/mediapipe/modules/objectron/calculators/frame_annotation_tracker.cc +++ b/mediapipe/modules/objectron/calculators/frame_annotation_tracker.cc @@ -15,7 +15,8 @@ #include "mediapipe/modules/objectron/calculators/frame_annotation_tracker.h" #include "absl/container/flat_hash_set.h" -#include "mediapipe/framework/port/logging.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/modules/objectron/calculators/annotation_data.pb.h" #include "mediapipe/modules/objectron/calculators/box_util.h" #include "mediapipe/util/tracking/box_tracker.pb.h" @@ -35,7 +36,7 @@ void FrameAnnotationTracker::AddDetectionResult( FrameAnnotation FrameAnnotationTracker::ConsolidateTrackingResult( const TimedBoxProtoList& tracked_boxes, absl::flat_hash_set* cancel_object_ids) { - CHECK(cancel_object_ids != nullptr); + ABSL_CHECK(cancel_object_ids != nullptr); FrameAnnotation frame_annotation; std::vector keys_to_be_deleted; for (const auto& detected_obj : detected_objects_) { @@ -53,8 +54,8 @@ FrameAnnotation FrameAnnotationTracker::ConsolidateTrackingResult( } } if (!ref_box.has_id() || ref_box.id() < 0) { - LOG(ERROR) << "Can't find matching tracked box for object id: " - << object_id << ". Likely lost tracking of it."; + ABSL_LOG(ERROR) << "Can't find matching tracked box for object id: " + << object_id << ". Likely lost tracking of it."; keys_to_be_deleted.push_back(detected_obj.first); continue; } diff --git a/mediapipe/modules/objectron/calculators/frame_annotation_tracker_test.cc b/mediapipe/modules/objectron/calculators/frame_annotation_tracker_test.cc index d155f8e7..df6ffd40 100644 --- a/mediapipe/modules/objectron/calculators/frame_annotation_tracker_test.cc +++ b/mediapipe/modules/objectron/calculators/frame_annotation_tracker_test.cc @@ -15,6 +15,7 @@ #include "mediapipe/modules/objectron/calculators/frame_annotation_tracker.h" #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/logging.h" @@ -53,7 +54,7 @@ ObjectAnnotation ConstructFixedObject( ObjectAnnotation obj; for (const auto& point : points) { auto* keypoint = obj.add_keypoints(); - CHECK_EQ(2, point.size()); + ABSL_CHECK_EQ(2, point.size()); keypoint->mutable_point_2d()->set_x(point[0]); keypoint->mutable_point_2d()->set_y(point[1]); } diff --git a/mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.cc b/mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.cc index 5e5df78b..652c5103 100644 --- a/mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.cc +++ b/mediapipe/modules/objectron/calculators/lift_2d_frame_annotation_to_3d_calculator.cc @@ -17,6 +17,7 @@ #include #include "Eigen/Dense" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" @@ -137,7 +138,7 @@ absl::Status Lift2DFrameAnnotationTo3DCalculator::ProcessCPU( auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ false, output_objects); if (!status.ok()) { - LOG(ERROR) << status; + ABSL_LOG(ERROR) << status; return status; } AssignObjectIdAndTimestamp(cc->InputTimestamp().Microseconds(), diff --git a/mediapipe/modules/objectron/calculators/model.cc b/mediapipe/modules/objectron/calculators/model.cc index 40aca39d..d6fe9ed6 100644 --- a/mediapipe/modules/objectron/calculators/model.cc +++ b/mediapipe/modules/objectron/calculators/model.cc @@ -14,6 +14,7 @@ #include "mediapipe/modules/objectron/calculators/model.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -66,9 +67,9 @@ const Eigen::Ref Model::GetRotation() const { const std::string& Model::GetCategory() const { return category_; } void Model::Deserialize(const Object& obj) { - CHECK_EQ(obj.rotation_size(), 9); - CHECK_EQ(obj.translation_size(), 3); - CHECK_EQ(obj.scale_size(), 3); + ABSL_CHECK_EQ(obj.rotation_size(), 9); + ABSL_CHECK_EQ(obj.translation_size(), 3); + ABSL_CHECK_EQ(obj.scale_size(), 3); category_ = obj.category(); using RotationMatrix = Eigen::Matrix; diff --git a/mediapipe/modules/objectron/calculators/tensor_util.cc b/mediapipe/modules/objectron/calculators/tensor_util.cc index 0004edd8..c6fa74b2 100644 --- a/mediapipe/modules/objectron/calculators/tensor_util.cc +++ b/mediapipe/modules/objectron/calculators/tensor_util.cc @@ -14,14 +14,16 @@ #include "mediapipe/modules/objectron/calculators/tensor_util.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { cv::Mat ConvertTfliteTensorToCvMat(const TfLiteTensor& tensor) { // Check tensor is BxCxWxH (size = 4) and the batch size is one(data[0] = 1) - CHECK(tensor.dims->size == 4 && tensor.dims->data[0] == 1); - CHECK_EQ(kTfLiteFloat32, tensor.type) << "tflite_tensor type is not float"; + ABSL_CHECK(tensor.dims->size == 4 && tensor.dims->data[0] == 1); + ABSL_CHECK_EQ(kTfLiteFloat32, tensor.type) + << "tflite_tensor type is not float"; const size_t num_output_channels = tensor.dims->data[3]; const int dims = 2; @@ -32,9 +34,9 @@ cv::Mat ConvertTfliteTensorToCvMat(const TfLiteTensor& tensor) { cv::Mat ConvertTensorToCvMat(const mediapipe::Tensor& tensor) { // Check tensor is BxCxWxH (size = 4) and the batch size is one(data[0] = 1) - CHECK(tensor.shape().dims.size() == 4 && tensor.shape().dims[0] == 1); - CHECK_EQ(mediapipe::Tensor::ElementType::kFloat32 == tensor.element_type(), - true) + ABSL_CHECK(tensor.shape().dims.size() == 4 && tensor.shape().dims[0] == 1); + ABSL_CHECK_EQ( + mediapipe::Tensor::ElementType::kFloat32 == tensor.element_type(), true) << "tensor type is not float"; const size_t num_output_channels = tensor.shape().dims[3]; diff --git a/mediapipe/modules/objectron/calculators/tensors_to_objects_calculator.cc b/mediapipe/modules/objectron/calculators/tensors_to_objects_calculator.cc index 6989c34c..c5ccf1d1 100644 --- a/mediapipe/modules/objectron/calculators/tensors_to_objects_calculator.cc +++ b/mediapipe/modules/objectron/calculators/tensors_to_objects_calculator.cc @@ -17,6 +17,8 @@ #include #include "Eigen/Dense" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" @@ -148,7 +150,7 @@ absl::Status TensorsToObjectsCalculator::ProcessCPU( auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ true, output_objects); if (!status.ok()) { - LOG(ERROR) << status; + ABSL_LOG(ERROR) << status; return status; } Project3DTo2D(/*portrait*/ true, output_objects); @@ -170,7 +172,7 @@ absl::Status TensorsToObjectsCalculator::LoadOptions(CalculatorContext* cc) { num_keypoints_ = options_.num_keypoints(); // Currently only support 2D when num_values_per_keypoint equals to 2. - CHECK_EQ(options_.num_values_per_keypoint(), 2); + ABSL_CHECK_EQ(options_.num_values_per_keypoint(), 2); return absl::OkStatus(); } diff --git a/mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.cc b/mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.cc index d74b59a2..1aefd467 100644 --- a/mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.cc +++ b/mediapipe/modules/objectron/calculators/tflite_tensors_to_objects_calculator.cc @@ -17,6 +17,8 @@ #include #include "Eigen/Dense" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" @@ -154,7 +156,7 @@ absl::Status TfLiteTensorsToObjectsCalculator::ProcessCPU( auto status = decoder_->Lift2DTo3D(projection_matrix_, /*portrait*/ true, output_objects); if (!status.ok()) { - LOG(ERROR) << status; + ABSL_LOG(ERROR) << status; return status; } Project3DTo2D(/*portrait*/ true, output_objects); @@ -178,7 +180,7 @@ absl::Status TfLiteTensorsToObjectsCalculator::LoadOptions( num_keypoints_ = options_.num_keypoints(); // Currently only support 2D when num_values_per_keypoint equals to 2. - CHECK_EQ(options_.num_values_per_keypoint(), 2); + ABSL_CHECK_EQ(options_.num_values_per_keypoint(), 2); return absl::OkStatus(); } diff --git a/mediapipe/objc/BUILD b/mediapipe/objc/BUILD index 83567a4d..df6c8db0 100644 --- a/mediapipe/objc/BUILD +++ b/mediapipe/objc/BUILD @@ -39,6 +39,8 @@ cc_library( "//mediapipe/framework/port:source_location", "//mediapipe/framework/port:status", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", ], ) diff --git a/mediapipe/objc/util.cc b/mediapipe/objc/util.cc index 36ad4e19..684dc181 100644 --- a/mediapipe/objc/util.cc +++ b/mediapipe/objc/util.cc @@ -15,6 +15,8 @@ #include "mediapipe/objc/util.h" #include "absl/base/macros.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" @@ -504,7 +506,7 @@ absl::Status CreateCGImageFromCVPixelBuffer(CVPixelBufferRef image_buffer, break; default: - LOG(FATAL) << "Unsupported pixelFormat " << pixel_format; + ABSL_LOG(FATAL) << "Unsupported pixelFormat " << pixel_format; break; } @@ -571,7 +573,7 @@ std::unique_ptr CreateImageFrameForCVPixelBuffer( CVPixelBufferRef image_buffer, bool can_overwrite, bool bgr_as_rgb) { CVReturn status = CVPixelBufferLockBaseAddress(image_buffer, kCVPixelBufferLock_ReadOnly); - CHECK_EQ(status, kCVReturnSuccess) + ABSL_CHECK_EQ(status, kCVReturnSuccess) << "CVPixelBufferLockBaseAddress failed: " << status; void* base_address = CVPixelBufferGetBaseAddress(image_buffer); @@ -601,7 +603,7 @@ std::unique_ptr CreateImageFrameForCVPixelBuffer( const uint8_t permute_map[4] = {2, 1, 0, 3}; vImage_Error vError = vImagePermuteChannels_ARGB8888( &v_image, &v_dest, permute_map, kvImageNoFlags); - CHECK(vError == kvImageNoError) + ABSL_CHECK(vError == kvImageNoError) << "vImagePermuteChannels failed: " << vError; } } break; @@ -623,7 +625,7 @@ std::unique_ptr CreateImageFrameForCVPixelBuffer( static_cast(pixel_format >> 16 & 0xFF), static_cast(pixel_format >> 8 & 0xFF), static_cast(pixel_format & 0xFF), 0}; - LOG(FATAL) << "unsupported pixel format: " << format_str; + ABSL_LOG(FATAL) << "unsupported pixel format: " << format_str; } break; } @@ -631,7 +633,7 @@ std::unique_ptr CreateImageFrameForCVPixelBuffer( // We have already created a new frame that does not reference the buffer. status = CVPixelBufferUnlockBaseAddress(image_buffer, kCVPixelBufferLock_ReadOnly); - CHECK_EQ(status, kCVReturnSuccess) + ABSL_CHECK_EQ(status, kCVReturnSuccess) << "CVPixelBufferUnlockBaseAddress failed: " << status; CVPixelBufferRelease(image_buffer); } else { diff --git a/mediapipe/platforms.bzl b/mediapipe/platforms.bzl new file mode 100644 index 00000000..fe2cbbd6 --- /dev/null +++ b/mediapipe/platforms.bzl @@ -0,0 +1,38 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build rule to generate 'config_setting' and 'platform' with the same constraints.""" + +def config_setting_and_platform( + name, + constraint_values = [], + visibility = None): + """Defines a 'config_setting' and 'platform' with the same constraints. + + Args: + name: the name for the 'config_setting'. The platform will be suffixed with '_platform'. + constraint_values: the constraints to meet. + visibility: the target visibility. + """ + native.config_setting( + name = name, + constraint_values = constraint_values, + visibility = visibility, + ) + + native.platform( + name = name + "_platform", + constraint_values = constraint_values, + visibility = visibility, + ) diff --git a/mediapipe/python/solutions/drawing_utils.py b/mediapipe/python/solutions/drawing_utils.py index 1b8b173f..a1acc0be 100644 --- a/mediapipe/python/solutions/drawing_utils.py +++ b/mediapipe/python/solutions/drawing_utils.py @@ -13,17 +13,17 @@ # limitations under the License. """MediaPipe solution drawing utils.""" +import dataclasses import math from typing import List, Mapping, Optional, Tuple, Union import cv2 -import dataclasses import matplotlib.pyplot as plt import numpy as np from mediapipe.framework.formats import detection_pb2 -from mediapipe.framework.formats import location_data_pb2 from mediapipe.framework.formats import landmark_pb2 +from mediapipe.framework.formats import location_data_pb2 _PRESENCE_THRESHOLD = 0.5 _VISIBILITY_THRESHOLD = 0.5 diff --git a/mediapipe/python/solutions/drawing_utils_test.py b/mediapipe/python/solutions/drawing_utils_test.py index 0039f9a9..8943a058 100644 --- a/mediapipe/python/solutions/drawing_utils_test.py +++ b/mediapipe/python/solutions/drawing_utils_test.py @@ -20,7 +20,6 @@ import cv2 import numpy as np from google.protobuf import text_format - from mediapipe.framework.formats import detection_pb2 from mediapipe.framework.formats import landmark_pb2 from mediapipe.python.solutions import drawing_utils diff --git a/mediapipe/tasks/c/components/containers/BUILD b/mediapipe/tasks/c/components/containers/BUILD new file mode 100644 index 00000000..0f55d18d --- /dev/null +++ b/mediapipe/tasks/c/components/containers/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "category", + srcs = ["category.cc"], + hdrs = ["category.h"], + deps = ["//mediapipe/tasks/cc/components/containers:category"], +) + +cc_library( + name = "classification_result", + srcs = ["classification_result.cc"], + hdrs = ["classification_result.h"], + deps = [ + ":category", + "//mediapipe/tasks/cc/components/containers:classification_result", + ], +) diff --git a/mediapipe/tasks/c/components/containers/category.cc b/mediapipe/tasks/c/components/containers/category.cc new file mode 100644 index 00000000..2311f637 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/category.cc @@ -0,0 +1,30 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/category.h" + +namespace mediapie::tasks::c::components::containers { + +void CppConvertToCategory(mediapipe::tasks::components::containers::Category in, + Category* out) { + out->index = in.index; + out->score = in.score; + out->category_name = + in.category_name.has_value() ? in.category_name->c_str() : nullptr; + out->display_name = + in.display_name.has_value() ? in.display_name->c_str() : nullptr; +} + +} // namespace mediapie::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/category.h b/mediapipe/tasks/c/components/containers/category.h new file mode 100644 index 00000000..c83140af --- /dev/null +++ b/mediapipe/tasks/c/components/containers/category.h @@ -0,0 +1,53 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ + +#include "mediapipe/tasks/cc/components/containers/category.h" + +extern "C" { +// Defines a single classification result. +// +// The label maps packed into the TFLite Model Metadata [1] are used to populate +// the 'category_name' and 'display_name' fields. +// +// [1]: https://www.tensorflow.org/lite/convert/metadata +struct Category { + // The index of the category in the classification model output. + int index; + + // The score for this category, e.g. (but not necessarily) a probability in + // [0,1]. + float score; + + // The optional ID for the category, read from the label map packed in the + // TFLite Model Metadata if present. Not necessarily human-readable. + const char* category_name; + + // The optional human-readable name for the category, read from the label map + // packed in the TFLite Model Metadata if present. + const char* display_name; +}; +} + +namespace mediapie::tasks::c::components::containers { + +void CppConvertToCategory(mediapipe::tasks::components::containers::Category in, + Category* out); + +} // namespace mediapie::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ diff --git a/mediapipe/tasks/c/components/containers/classification_result.cc b/mediapipe/tasks/c/components/containers/classification_result.cc new file mode 100644 index 00000000..4e6b1036 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/classification_result.cc @@ -0,0 +1,57 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/classification_result.h" + +#include "mediapipe/tasks/c/components/containers/category.h" + +namespace mediapipe::tasks::c::components::containers { + +namespace { +using mediapie::tasks::c::components::containers::CppConvertToCategory; +} // namespace + +void CppConvertToClassificationResult( + mediapipe::tasks::components::containers::ClassificationResult in, + ClassificationResult* out) { + out->has_timestamp_ms = in.timestamp_ms.has_value(); + if (out->has_timestamp_ms) { + out->timestamp_ms = in.timestamp_ms.value(); + } + + out->classifications_count = in.classifications.size(); + out->classifications = new Classifications[out->classifications_count]; + + for (uint32_t i = 0; i <= out->classifications_count; ++i) { + auto classification_in = in.classifications[i]; + auto classification_out = out->classifications[i]; + + classification_out.categories_count = classification_in.categories.size(); + classification_out.categories = + new Category[classification_out.categories_count]; + for (uint32_t j = 0; j <= classification_out.categories_count; ++j) { + CppConvertToCategory(classification_in.categories[j], + &(classification_out.categories[j])); + } + + classification_out.head_index = classification_in.head_index; + classification_out.head_name = + classification_in.head_name.has_value() + ? classification_in.head_name.value().c_str() + : nullptr; + } +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/classification_result.h b/mediapipe/tasks/c/components/containers/classification_result.h new file mode 100644 index 00000000..77ec4ba8 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/classification_result.h @@ -0,0 +1,73 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ + +#include +#include + +#include "mediapipe/tasks/cc/components/containers/classification_result.h" + +extern "C" { + +// Defines classification results for a given classifier head. +struct Classifications { + // The array of predicted categories, usually sorted by descending scores, + // e.g. from high to low probability. + struct Category* categories; + // The number of elements in the categories array. + uint32_t categories_count; + + // The index of the classifier head (i.e. output tensor) these categories + // refer to. This is useful for multi-head models. + int head_index; + + // The optional name of the classifier head, as provided in the TFLite Model + // Metadata [1] if present. This is useful for multi-head models. + // + // [1]: https://www.tensorflow.org/lite/convert/metadata + const char* head_name; +}; + +// Defines classification results of a model. +struct ClassificationResult { + // The classification results for each head of the model. + struct Classifications* classifications; + // The number of classifications in the classifications array. + uint32_t classifications_count; + + // The optional timestamp (in milliseconds) of the start of the chunk of data + // corresponding to these results. + // + // This is only used for classification on time series (e.g. audio + // classification). In these use cases, the amount of data to process might + // exceed the maximum size that the model can process: to solve this, the + // input data is split into multiple chunks starting at different timestamps. + int64_t timestamp_ms; + // Specifies whether the timestamp contains a valid value. + bool has_timestamp_ms; +}; +} + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToClassificationResult( + mediapipe::tasks::components::containers::ClassificationResult in, + ClassificationResult* out); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ diff --git a/mediapipe/tasks/c/components/processors/BUILD b/mediapipe/tasks/c/components/processors/BUILD new file mode 100644 index 00000000..397e149d --- /dev/null +++ b/mediapipe/tasks/c/components/processors/BUILD @@ -0,0 +1,24 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "classifier_options", + srcs = ["classifier_options.cc"], + hdrs = ["classifier_options.h"], + deps = ["//mediapipe/tasks/cc/components/processors:classifier_options"], +) diff --git a/mediapipe/tasks/c/components/processors/classifier_options.cc b/mediapipe/tasks/c/components/processors/classifier_options.cc new file mode 100644 index 00000000..7c84e7a0 --- /dev/null +++ b/mediapipe/tasks/c/components/processors/classifier_options.cc @@ -0,0 +1,42 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/processors/classifier_options.h" + +#include +#include + +#include "mediapipe/tasks/cc/components/processors/classifier_options.h" + +namespace mediapie::c::components::processors { + +void CppConvertToClassifierOptions( + ClassifierOptions in, + mediapipe::tasks::components::processors::ClassifierOptions* out) { + out->display_names_locale = in.display_names_locale; + out->max_results = in.max_results; + out->score_threshold = in.score_threshold; + out->category_allowlist = + std::vector(in.category_allowlist_count); + for (uint32_t i = 0; i < in.category_allowlist_count; ++i) { + out->category_allowlist[i] = in.category_allowlist[i]; + } + out->category_denylist = std::vector(in.category_denylist_count); + for (uint32_t i = 0; i < in.category_denylist_count; ++i) { + out->category_denylist[i] = in.category_denylist[i]; + } +} + +} // namespace mediapie::c::components::processors diff --git a/mediapipe/tasks/c/components/processors/classifier_options.h b/mediapipe/tasks/c/components/processors/classifier_options.h new file mode 100644 index 00000000..78197433 --- /dev/null +++ b/mediapipe/tasks/c/components/processors/classifier_options.h @@ -0,0 +1,61 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ + +#include + +#include "mediapipe/tasks/cc/components/processors/classifier_options.h" + +// Classifier options for MediaPipe C classification Tasks. +struct ClassifierOptions { + // The locale to use for display names specified through the TFLite Model + // Metadata, if any. Defaults to English. + char* display_names_locale; + + // The maximum number of top-scored classification results to return. If < 0, + // all available results will be returned. If 0, an invalid argument error is + // returned. + int max_results; + + // Score threshold to override the one provided in the model metadata (if + // any). Results below this value are rejected. + float score_threshold; + + // The allowlist of category names. If non-empty, detection results whose + // category name is not in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_denylist. + char** category_allowlist; + // The number of elements in the category allowlist. + uint32_t category_allowlist_count; + + // The denylist of category names. If non-empty, detection results whose + // category name is in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_allowlist. + char** category_denylist = {}; + // The number of elements in the category denylist. + uint32_t category_denylist_count; +}; + +namespace mediapipe::tasks::c::components::processors { + +void CppConvertToClassifierOptions( + ClassifierOptions in, + mediapipe::tasks::components::processors::ClassifierOptions* out); + +} // namespace mediapipe::tasks::c::components::processors + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ diff --git a/mediapipe/model_maker/python/core/utils/testdata/BUILD b/mediapipe/tasks/c/core/BUILD similarity index 60% rename from mediapipe/model_maker/python/core/utils/testdata/BUILD rename to mediapipe/tasks/c/core/BUILD index ea45f614..adf6c81a 100644 --- a/mediapipe/model_maker/python/core/utils/testdata/BUILD +++ b/mediapipe/tasks/c/core/BUILD @@ -1,10 +1,10 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -package( - default_visibility = ["//mediapipe/model_maker/python/core/utils:__subpackages__"], - licenses = ["notice"], # Apache 2.0 -) +package(default_visibility = ["//mediapipe/tasks:internal"]) -filegroup( - name = "testdata", - srcs = ["test.txt"], +licenses(["notice"]) + +cc_library( + name = "base_options", + srcs = ["base_options.cc"], + hdrs = ["base_options.h"], + deps = ["//mediapipe/tasks/cc/core:base_options"], ) diff --git a/mediapipe/tasks/c/core/base_options.cc b/mediapipe/tasks/c/core/base_options.cc new file mode 100644 index 00000000..d8fcfdb9 --- /dev/null +++ b/mediapipe/tasks/c/core/base_options.cc @@ -0,0 +1,29 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/core/base_options.h" + +#include "mediapipe/tasks/cc/core/base_options.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToBaseOptions(BaseOptions in, + mediapipe::tasks::core::BaseOptions* out) { + out->model_asset_buffer = + std::make_unique(in.model_asset_buffer); + out->model_asset_path = in.model_asset_path; +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/core/base_options.h b/mediapipe/tasks/c/core/base_options.h new file mode 100644 index 00000000..1707c9fa --- /dev/null +++ b/mediapipe/tasks/c/core/base_options.h @@ -0,0 +1,41 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ +#define MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ + +#include "mediapipe/tasks/cc/core/base_options.h" + +extern "C" { + +// Base options for MediaPipe C Tasks. +struct BaseOptions { + // The model asset file contents as a string. + char* model_asset_buffer; + + // The path to the model asset to open and mmap in memory. + char* model_asset_path; +}; + +} // extern C + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToBaseOptions(BaseOptions in, + mediapipe::tasks::core::BaseOptions* out); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ diff --git a/mediapipe/tasks/c/text/text_classifier/BUILD b/mediapipe/tasks/c/text/text_classifier/BUILD new file mode 100644 index 00000000..93ea468d --- /dev/null +++ b/mediapipe/tasks/c/text/text_classifier/BUILD @@ -0,0 +1,30 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "text_classifier", + srcs = ["text_classifier.cc"], + hdrs = ["text_classifier.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/tasks/c/components/containers:classification_result", + "//mediapipe/tasks/c/components/processors:classifier_options", + "//mediapipe/tasks/c/core:base_options", + "//mediapipe/tasks/cc/text/text_classifier", + ], +) diff --git a/mediapipe/tasks/c/text/text_classifier/text_classifier.cc b/mediapipe/tasks/c/text/text_classifier/text_classifier.cc new file mode 100644 index 00000000..b88a66bc --- /dev/null +++ b/mediapipe/tasks/c/text/text_classifier/text_classifier.cc @@ -0,0 +1,94 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/text/text_classifier/text_classifier.h" + +#include + +#include "mediapipe/tasks/c/components/containers/classification_result.h" +#include "mediapipe/tasks/c/components/processors/classifier_options.h" +#include "mediapipe/tasks/c/core/base_options.h" +#include "mediapipe/tasks/cc/text/text_classifier/text_classifier.h" + +namespace mediapipe::tasks::c::text::text_classifier { + +namespace { + +using ::mediapipe::tasks::c::components::containers::CppConvertToBaseOptions; +using ::mediapipe::tasks::c::components::containers:: + CppConvertToClassificationResult; +using ::mediapipe::tasks::c::components::processors:: + CppConvertToClassifierOptions; +using ::mediapipe::tasks::text::text_classifier::TextClassifier; +} // namespace + +TextClassifier* CppTextClassifierCreate(TextClassifierOptions options) { + auto cpp_options = std::make_unique< + ::mediapipe::tasks::text::text_classifier::TextClassifierOptions>(); + + CppConvertToBaseOptions(options.base_options, &cpp_options->base_options); + CppConvertToClassifierOptions(options.classifier_options, + &cpp_options->classifier_options); + + auto classifier = TextClassifier::Create(std::move(cpp_options)); + if (!classifier.ok()) { + LOG(ERROR) << "Failed to create TextClassifier: " << classifier.status(); + return nullptr; + } + return classifier->release(); +} + +bool CppTextClassifierClassify(void* classifier, char* utf8_str, + TextClassifierResult* result) { + auto cpp_classifier = static_cast(classifier); + auto cpp_result = cpp_classifier->Classify(utf8_str); + if (!cpp_result.ok()) { + LOG(ERROR) << "Classification failed: " << cpp_result.status(); + return false; + } + CppConvertToClassificationResult(*cpp_result, result); + return true; +} + +void CppTextClassifierClose(void* classifier) { + auto cpp_classifier = static_cast(classifier); + auto result = cpp_classifier->Close(); + if (!result.ok()) { + LOG(ERROR) << "Failed to close TextClassifier: " << result; + } + delete cpp_classifier; +} + +} // namespace mediapipe::tasks::c::text::text_classifier + +extern "C" { + +void* text_classifier_create(struct TextClassifierOptions options) { + return mediapipe::tasks::c::text::text_classifier::CppTextClassifierCreate( + options); +} + +bool text_classifier_classify(void* classifier, char* utf8_str, + TextClassifierResult* result) { + return mediapipe::tasks::c::text::text_classifier::CppTextClassifierClassify( + classifier, utf8_str, result); +} + +void text_classifier_close(void* classifier) { + mediapipe::tasks::c::text::text_classifier::CppTextClassifierClose( + classifier); +} + +} // extern "C" diff --git a/mediapipe/tasks/c/text/text_classifier/text_classifier.h b/mediapipe/tasks/c/text/text_classifier/text_classifier.h new file mode 100644 index 00000000..9ec9682d --- /dev/null +++ b/mediapipe/tasks/c/text/text_classifier/text_classifier.h @@ -0,0 +1,49 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_TEXT_TEXT_CLASSIFIER_TEXT_CLASSIFIER_H_ +#define MEDIAPIPE_TASKS_C_TEXT_TEXT_CLASSIFIER_TEXT_CLASSIFIER_H_ + +#include "mediapipe/tasks/c/components/containers/classification_result.h" +#include "mediapipe/tasks/c/components/processors/classifier_options.h" +#include "mediapipe/tasks/c/core/base_options.h" + +extern "C" { +typedef ClassificationResult TextClassifierResult; + +// The options for configuring a MediaPipe text classifier task. +struct TextClassifierOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + + // Options for configuring the classifier behavior, such as score threshold, + // number of results, etc. + struct ClassifierOptions classifier_options; +}; + +// Creates a TextClassifier from the provided `options`. +void* text_classifier_create(struct TextClassifierOptions options); + +// Performs classification on the input `text`. +bool text_classifier_classify(void* classifier, char* utf8_str, + TextClassifierResult* result); + +// Shuts down the TextClassifier when all the work is done. Frees all memory. +void text_classifier_close(void* classifier); + +} // extern C + +#endif // MEDIAPIPE_TASKS_C_TEXT_TEXT_CLASSIFIER_TEXT_CLASSIFIER_H_ diff --git a/mediapipe/tasks/cc/components/calculators/BUILD b/mediapipe/tasks/cc/components/calculators/BUILD index fb4b66b3..9046a280 100644 --- a/mediapipe/tasks/cc/components/calculators/BUILD +++ b/mediapipe/tasks/cc/components/calculators/BUILD @@ -133,6 +133,7 @@ cc_test( "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/tasks/metadata:metadata_schema_cc", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/tasks/cc/components/processors/BUILD b/mediapipe/tasks/cc/components/processors/BUILD index e8f9f57f..dc5aca48 100644 --- a/mediapipe/tasks/cc/components/processors/BUILD +++ b/mediapipe/tasks/cc/components/processors/BUILD @@ -199,6 +199,7 @@ cc_library( "//mediapipe/util:label_map_cc_proto", "//mediapipe/util:label_map_util", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/mediapipe/tasks/cc/components/processors/classification_postprocessing_graph.cc b/mediapipe/tasks/cc/components/processors/classification_postprocessing_graph.cc index 5534cb96..525b3d4e 100644 --- a/mediapipe/tasks/cc/components/processors/classification_postprocessing_graph.cc +++ b/mediapipe/tasks/cc/components/processors/classification_postprocessing_graph.cc @@ -296,7 +296,7 @@ void ConfigureClassificationAggregationCalculator( if (output_tensors_metadata == nullptr) { return; } - for (const auto& metadata : *output_tensors_metadata) { + for (const auto metadata : *output_tensors_metadata) { options->add_head_names(metadata->name()->str()); } } diff --git a/mediapipe/tasks/cc/components/processors/detection_postprocessing_graph.cc b/mediapipe/tasks/cc/components/processors/detection_postprocessing_graph.cc index d7fc1892..813a23ae 100644 --- a/mediapipe/tasks/cc/components/processors/detection_postprocessing_graph.cc +++ b/mediapipe/tasks/cc/components/processors/detection_postprocessing_graph.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" @@ -336,7 +337,7 @@ absl::StatusOr> GetOutputTensorIndices( int output_index = output_indices[i]; // If tensor name is not found, set the default output indices. if (output_index == -1) { - LOG(WARNING) << absl::StrFormat( + ABSL_LOG(WARNING) << absl::StrFormat( "You don't seem to be matching tensor names in metadata list. The " "tensor name \"%s\" at index %d in the model metadata doesn't " "match " @@ -360,7 +361,7 @@ absl::StatusOr> GetOutputTensorIndices( int output_index = output_indices[i]; // If tensor name is not found, set the default output indices. if (output_index == -1) { - LOG(WARNING) << absl::StrFormat( + ABSL_LOG(WARNING) << absl::StrFormat( "You don't seem to be matching tensor names in metadata list. The " "tensor name \"%s\" at index %d in the model metadata doesn't " "match " diff --git a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto b/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto index 8c1daf27..b2d13c3a 100644 --- a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto +++ b/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto @@ -43,4 +43,7 @@ message TransformerParameters { // Number of stacked transformers, `N` in the paper. int32 num_stacks = 7; + + // Whether to use Multi-Query-Attention (MQA). + bool use_mqa = 8; } diff --git a/mediapipe/tasks/cc/core/BUILD b/mediapipe/tasks/cc/core/BUILD index dad9cdf1..ce9181d5 100644 --- a/mediapipe/tasks/cc/core/BUILD +++ b/mediapipe/tasks/cc/core/BUILD @@ -29,6 +29,7 @@ cc_library( "//mediapipe/tasks/cc/core/proto:acceleration_cc_proto", "//mediapipe/tasks/cc/core/proto:base_options_cc_proto", "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@org_tensorflow//tensorflow/lite/core/api:op_resolver", "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", @@ -79,6 +80,7 @@ cc_library( "//mediapipe/tasks/cc/text/custom_ops/sentencepiece:sentencepiece_tokenizer_tflite", "//mediapipe/tasks/cc/text/language_detector/custom_ops:kmeans_embedding_lookup", "//mediapipe/tasks/cc/text/language_detector/custom_ops:ngram_hash", + "//mediapipe/tasks/cc/vision/custom_ops:fused_batch_norm", "//mediapipe/util/tflite/operations:landmarks_to_transform_matrix", "//mediapipe/util/tflite/operations:max_pool_argmax", "//mediapipe/util/tflite/operations:max_unpooling", @@ -106,13 +108,13 @@ cc_library( "//mediapipe/framework:subgraph", "//mediapipe/framework/api2:builder", "//mediapipe/framework/api2:port", - "//mediapipe/framework/port:logging", "//mediapipe/tasks/cc:common", "//mediapipe/tasks/cc/core/proto:acceleration_cc_proto", "//mediapipe/tasks/cc/core/proto:base_options_cc_proto", "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", "//mediapipe/tasks/cc/core/proto:inference_subgraph_cc_proto", "//mediapipe/tasks/cc/core/proto:model_resources_calculator_cc_proto", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/mediapipe/tasks/cc/core/base_options.cc b/mediapipe/tasks/cc/core/base_options.cc index a34c2316..7f7db525 100644 --- a/mediapipe/tasks/cc/core/base_options.cc +++ b/mediapipe/tasks/cc/core/base_options.cc @@ -17,15 +17,62 @@ limitations under the License. #include #include +#include +#include "absl/log/absl_log.h" #include "mediapipe/calculators/tensor/inference_calculator.pb.h" #include "mediapipe/tasks/cc/core/proto/acceleration.pb.h" +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" #include "mediapipe/tasks/cc/core/proto/external_file.pb.h" namespace mediapipe { namespace tasks { namespace core { +proto::Acceleration ConvertDelegateOptionsToAccelerationProto( + const BaseOptions::CpuOptions& options) { + proto::Acceleration acceleration_proto = proto::Acceleration(); + acceleration_proto.mutable_tflite(); + return acceleration_proto; +} + +proto::Acceleration ConvertDelegateOptionsToAccelerationProto( + const BaseOptions::GpuOptions& options) { + proto::Acceleration acceleration_proto = proto::Acceleration(); + auto* gpu = acceleration_proto.mutable_gpu(); + gpu->set_use_advanced_gpu_api(true); + if (!options.cached_kernel_path.empty()) { + gpu->set_cached_kernel_path(options.cached_kernel_path); + } + if (!options.serialized_model_dir.empty()) { + gpu->set_serialized_model_dir(options.serialized_model_dir); + } + if (!options.model_token.empty()) { + gpu->set_model_token(options.model_token); + } + return acceleration_proto; +} + +template +void SetDelegateOptionsOrDie(const BaseOptions* base_options, + proto::BaseOptions& base_options_proto) { + if (base_options->delegate_options.has_value()) { + if (!std::holds_alternative(*base_options->delegate_options)) { + ABSL_LOG(FATAL) << "Specified Delegate type does not match the provided " + "delegate options."; + } else { + std::visit( + [&base_options_proto](const auto& delegate_options) { + proto::Acceleration acceleration_proto = + ConvertDelegateOptionsToAccelerationProto(delegate_options); + base_options_proto.mutable_acceleration()->Swap( + &acceleration_proto); + }, + *base_options->delegate_options); + } + } +} + proto::BaseOptions ConvertBaseOptionsToProto(BaseOptions* base_options) { proto::BaseOptions base_options_proto; if (!base_options->model_asset_path.empty()) { @@ -53,11 +100,15 @@ proto::BaseOptions ConvertBaseOptionsToProto(BaseOptions* base_options) { switch (base_options->delegate) { case BaseOptions::Delegate::CPU: base_options_proto.mutable_acceleration()->mutable_tflite(); + SetDelegateOptionsOrDie(base_options, + base_options_proto); break; case BaseOptions::Delegate::GPU: base_options_proto.mutable_acceleration() ->mutable_gpu() ->set_use_advanced_gpu_api(true); + SetDelegateOptionsOrDie(base_options, + base_options_proto); break; case BaseOptions::Delegate::EDGETPU_NNAPI: base_options_proto.mutable_acceleration() @@ -65,7 +116,6 @@ proto::BaseOptions ConvertBaseOptionsToProto(BaseOptions* base_options) { ->set_accelerator_name("google-edgetpu"); break; } - return base_options_proto; } } // namespace core diff --git a/mediapipe/tasks/cc/core/base_options.h b/mediapipe/tasks/cc/core/base_options.h index 021aebbe..6cfc8a7a 100644 --- a/mediapipe/tasks/cc/core/base_options.h +++ b/mediapipe/tasks/cc/core/base_options.h @@ -17,7 +17,10 @@ limitations under the License. #define MEDIAPIPE_TASKS_CC_CORE_BASE_OPTIONS_H_ #include +#include #include +#include +#include #include "absl/memory/memory.h" #include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" @@ -38,7 +41,8 @@ struct BaseOptions { std::string model_asset_path = ""; // The delegate to run MediaPipe. If the delegate is not set, the default - // delegate CPU is used. + // delegate CPU is used. Use `delegate_options` to configure advanced + // features of the selected delegate." enum Delegate { CPU = 0, GPU = 1, @@ -48,6 +52,30 @@ struct BaseOptions { Delegate delegate = CPU; + // Options for CPU. + struct CpuOptions {}; + + // Options for GPU. + struct GpuOptions { + // Load pre-compiled serialized binary cache to accelerate init process. + // Only available on Android. Kernel caching will only be enabled if this + // path is set. NOTE: binary cache usage may be skipped if valid serialized + // model, specified by "serialized_model_dir", exists. + std::string cached_kernel_path; + + // A dir to load from and save to a pre-compiled serialized model used to + // accelerate init process. + // NOTE: serialized model takes precedence over binary cache + // specified by "cached_kernel_path", which still can be used if + // serialized model is invalid or missing. + std::string serialized_model_dir; + + // Unique token identifying the model. Used in conjunction with + // "serialized_model_dir". It is the caller's responsibility to ensure + // there is no clash of the tokens. + std::string model_token; + }; + // The file descriptor to a file opened with open(2), with optional additional // offset and length information. struct FileDescriptorMeta { @@ -67,6 +95,10 @@ struct BaseOptions { // built-in Ops. std::unique_ptr op_resolver = absl::make_unique(); + + // Options for the chosen delegate. If not set, the default delegate options + // is used. + std::optional> delegate_options; }; // Converts a BaseOptions to a BaseOptionsProto. diff --git a/mediapipe/tasks/cc/core/base_options_test.cc b/mediapipe/tasks/cc/core/base_options_test.cc index dce95050..39066351 100644 --- a/mediapipe/tasks/cc/core/base_options_test.cc +++ b/mediapipe/tasks/cc/core/base_options_test.cc @@ -1,6 +1,9 @@ #include "mediapipe/tasks/cc/core/base_options.h" +#include +#include #include +#include #include "mediapipe/calculators/tensor/inference_calculator.pb.h" #include "mediapipe/framework/port/gmock.h" @@ -11,6 +14,8 @@ constexpr char kTestModelBundlePath[] = "mediapipe/tasks/testdata/core/dummy_gesture_recognizer.task"; +constexpr char kCachedModelDir[] = "/data/local/tmp"; +constexpr char kModelToken[] = "dummy_model_token"; namespace mediapipe { namespace tasks { @@ -40,6 +45,45 @@ TEST(BaseOptionsTest, ConvertBaseOptionsToProtoWithAcceleration) { EXPECT_EQ(proto.acceleration().nnapi().accelerator_name(), "google-edgetpu"); } +TEST(DelegateOptionsTest, SucceedCpuOptions) { + BaseOptions base_options; + base_options.delegate = BaseOptions::Delegate::CPU; + BaseOptions::CpuOptions cpu_options; + base_options.delegate_options = cpu_options; + proto::BaseOptions proto = ConvertBaseOptionsToProto(&base_options); + EXPECT_TRUE(proto.acceleration().has_tflite()); + ASSERT_FALSE(proto.acceleration().has_gpu()); +} + +TEST(DelegateOptionsTest, SucceedGpuOptions) { + BaseOptions base_options; + base_options.delegate = BaseOptions::Delegate::GPU; + BaseOptions::GpuOptions gpu_options; + gpu_options.serialized_model_dir = kCachedModelDir; + gpu_options.model_token = kModelToken; + base_options.delegate_options = gpu_options; + proto::BaseOptions proto = ConvertBaseOptionsToProto(&base_options); + ASSERT_TRUE(proto.acceleration().has_gpu()); + ASSERT_FALSE(proto.acceleration().has_tflite()); + EXPECT_TRUE(proto.acceleration().gpu().use_advanced_gpu_api()); + EXPECT_FALSE(proto.acceleration().gpu().has_cached_kernel_path()); + EXPECT_EQ(proto.acceleration().gpu().serialized_model_dir(), kCachedModelDir); + EXPECT_EQ(proto.acceleration().gpu().model_token(), kModelToken); +} + +TEST(DelegateOptionsDeathTest, FailWrongDelegateOptionsType) { + BaseOptions base_options; + base_options.delegate = BaseOptions::Delegate::CPU; + BaseOptions::GpuOptions gpu_options; + gpu_options.cached_kernel_path = kCachedModelDir; + gpu_options.model_token = kModelToken; + base_options.delegate_options = gpu_options; + ASSERT_DEATH( + { proto::BaseOptions proto = ConvertBaseOptionsToProto(&base_options); }, + "Specified Delegate type does not match the provided " + "delegate options."); +} + } // namespace } // namespace core } // namespace tasks diff --git a/mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.cc b/mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.cc index b816d885..04bc7505 100644 --- a/mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.cc +++ b/mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.cc @@ -19,6 +19,7 @@ limitations under the License. #include "mediapipe/tasks/cc/text/custom_ops/sentencepiece/sentencepiece_tokenizer_tflite.h" #include "mediapipe/tasks/cc/text/language_detector/custom_ops/kmeans_embedding_lookup.h" #include "mediapipe/tasks/cc/text/language_detector/custom_ops/ngram_hash.h" +#include "mediapipe/tasks/cc/vision/custom_ops/fused_batch_norm.h" #include "mediapipe/util/tflite/operations/landmarks_to_transform_matrix.h" #include "mediapipe/util/tflite/operations/max_pool_argmax.h" #include "mediapipe/util/tflite/operations/max_unpooling.h" @@ -56,6 +57,8 @@ MediaPipeBuiltinOpResolver::MediaPipeBuiltinOpResolver() { mediapipe::tflite_operations::Register_SENTENCEPIECE_TOKENIZER()); AddCustom("RaggedTensorToTensor", mediapipe::tflite_operations::Register_RAGGED_TENSOR_TO_TENSOR()); + AddCustom("FusedBatchNormV3", + mediapipe::tflite_operations::Register_FusedBatchNorm()); } } // namespace core } // namespace tasks diff --git a/mediapipe/tasks/cc/core/model_task_graph.cc b/mediapipe/tasks/cc/core/model_task_graph.cc index 46cc088f..a68d40ae 100644 --- a/mediapipe/tasks/cc/core/model_task_graph.cc +++ b/mediapipe/tasks/cc/core/model_task_graph.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/ascii.h" @@ -30,7 +31,6 @@ limitations under the License. #include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/api2/port.h" #include "mediapipe/framework/calculator.pb.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/tasks/cc/common.h" #include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" #include "mediapipe/tasks/cc/core/model_resources.h" @@ -165,7 +165,7 @@ absl::StatusOr ModelTaskGraph::CreateModelResources( if (!model_resources_cache_service.IsAvailable()) { ASSIGN_OR_RETURN(auto local_model_resource, ModelResources::Create("", std::move(external_file))); - LOG(WARNING) + ABSL_LOG(WARNING) << "A local ModelResources object is created. Please consider using " "ModelResourcesCacheService to cache the created ModelResources " "object in the CalculatorGraph."; @@ -186,6 +186,21 @@ absl::StatusOr ModelTaskGraph::CreateModelResources( return model_resources_cache_service.GetObject().GetModelResources(tag); } +absl::StatusOr ModelTaskGraph::GetOrCreateModelResources( + SubgraphContext* sc, std::unique_ptr external_file, + std::string tag_suffix) { + auto model_resources_cache_service = sc->Service(kModelResourcesCacheService); + if (model_resources_cache_service.IsAvailable()) { + std::string tag = + absl::StrCat(CreateModelResourcesTag(sc->OriginalNode()), tag_suffix); + if (model_resources_cache_service.GetObject().Exists(tag)) { + return model_resources_cache_service.GetObject().GetModelResources(tag); + } + } + return ModelTaskGraph::CreateModelResources(sc, std::move(external_file), + tag_suffix); +} + absl::StatusOr ModelTaskGraph::CreateModelAssetBundleResources( SubgraphContext* sc, std::unique_ptr external_file, @@ -200,7 +215,7 @@ ModelTaskGraph::CreateModelAssetBundleResources( auto local_model_asset_bundle_resource, ModelAssetBundleResources::Create("", std::move(external_file))); if (!has_file_pointer_meta) { - LOG(WARNING) + ABSL_LOG(WARNING) << "A local ModelResources object is created. Please consider using " "ModelResourcesCacheService to cache the created ModelResources " "object in the CalculatorGraph."; diff --git a/mediapipe/tasks/cc/core/model_task_graph.h b/mediapipe/tasks/cc/core/model_task_graph.h index 10634d3d..38367da8 100644 --- a/mediapipe/tasks/cc/core/model_task_graph.h +++ b/mediapipe/tasks/cc/core/model_task_graph.h @@ -87,6 +87,20 @@ class ModelTaskGraph : public Subgraph { SubgraphContext* sc, std::unique_ptr external_file, std::string tag_suffix = ""); + template + absl::StatusOr GetOrCreateModelResources( + SubgraphContext* sc, std::string tag_suffix = "") { + auto external_file = std::make_unique(); + external_file->Swap(sc->MutableOptions() + ->mutable_base_options() + ->mutable_model_asset()); + return GetOrCreateModelResources(sc, std::move(external_file), tag_suffix); + } + + absl::StatusOr GetOrCreateModelResources( + SubgraphContext* sc, std::unique_ptr external_file, + std::string tag_suffix = ""); + // If the model resources graph service is available, creates a model asset // bundle resources object from the subgraph context, and caches the created // model asset bundle resources into the model resources graph service on diff --git a/mediapipe/tasks/cc/metadata/utils/BUILD b/mediapipe/tasks/cc/metadata/utils/BUILD index 881b8896..9e912c92 100644 --- a/mediapipe/tasks/cc/metadata/utils/BUILD +++ b/mediapipe/tasks/cc/metadata/utils/BUILD @@ -36,6 +36,7 @@ cc_library( "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@zlib//:zlib_minizip", diff --git a/mediapipe/tasks/cc/metadata/utils/zip_utils.cc b/mediapipe/tasks/cc/metadata/utils/zip_utils.cc index e0cc3d77..b9dd784c 100644 --- a/mediapipe/tasks/cc/metadata/utils/zip_utils.cc +++ b/mediapipe/tasks/cc/metadata/utils/zip_utils.cc @@ -19,6 +19,7 @@ limitations under the License. #include "absl/cleanup/cleanup.h" #include "absl/container/flat_hash_map.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "contrib/minizip/ioapi.h" @@ -63,7 +64,7 @@ absl::StatusOr GetCurrentZipFileInfo(const unzFile& zf) { absl::Cleanup unzipper_closer = [zf]() { auto status = UnzipErrorToStatus(unzCloseCurrentFile(zf)); if (!status.ok()) { - LOG(ERROR) << "Failed to close the current zip file: " << status; + ABSL_LOG(ERROR) << "Failed to close the current zip file: " << status; } }; if (method != Z_NO_COMPRESSION) { @@ -125,7 +126,7 @@ absl::Status ExtractFilesfromZipFile( } absl::Cleanup unzipper_closer = [zf]() { if (unzClose(zf) != UNZ_OK) { - LOG(ERROR) << "Unable to close zip archive."; + ABSL_LOG(ERROR) << "Unable to close zip archive."; } }; // Get number of files. diff --git a/mediapipe/tasks/cc/text/custom_ops/ragged/ragged_tensor_to_tensor_tflite.cc b/mediapipe/tasks/cc/text/custom_ops/ragged/ragged_tensor_to_tensor_tflite.cc index a0eadd71..1894dfa8 100644 --- a/mediapipe/tasks/cc/text/custom_ops/ragged/ragged_tensor_to_tensor_tflite.cc +++ b/mediapipe/tasks/cc/text/custom_ops/ragged/ragged_tensor_to_tensor_tflite.cc @@ -357,7 +357,7 @@ void CalculateOutputIndexValueRowID(const TfLiteTensor& value_rowids, }; int current_output_column = 0; int current_value_rowid = value_rowids_val(0); - // DCHECK_LT(current_value_rowid, parent_output_index.size()); + // ABSL_DCHECK_LT(current_value_rowid, parent_output_index.size()); int current_output_index = parent_output_index[current_value_rowid]; result->push_back(current_output_index); for (int i = 1; i < index_size; ++i) { @@ -374,12 +374,12 @@ void CalculateOutputIndexValueRowID(const TfLiteTensor& value_rowids, } else { current_output_column = 0; current_value_rowid = next_value_rowid; - // DCHECK_LT(next_value_rowid, parent_output_index.size()); + // ABSL_DCHECK_LT(next_value_rowid, parent_output_index.size()); current_output_index = parent_output_index[next_value_rowid]; } result->push_back(current_output_index); } - // DCHECK_EQ(result->size(), value_rowids.size()); + // ABSL_DCHECK_EQ(result->size(), value_rowids.size()); } void CalculateOutputIndexRowSplit(const TfLiteTensor& row_split, @@ -420,7 +420,7 @@ void CalculateOutputIndexRowSplit(const TfLiteTensor& row_split, } } // if (row_split_size > 0) { - // DCHECK_EQ(result->size(), row_split(row_split_size - 1)); + // ABSL_DCHECK_EQ(result->size(), row_split(row_split_size - 1)); //} } diff --git a/mediapipe/tasks/cc/text/language_detector/custom_ops/BUILD b/mediapipe/tasks/cc/text/language_detector/custom_ops/BUILD index 26eee18c..bb33bd20 100644 --- a/mediapipe/tasks/cc/text/language_detector/custom_ops/BUILD +++ b/mediapipe/tasks/cc/text/language_detector/custom_ops/BUILD @@ -37,6 +37,7 @@ cc_test( deps = [ ":kmeans_embedding_lookup", "//mediapipe/framework/port:gtest_main", + "@com_google_absl//absl/log:absl_check", "@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite/c:common", "@org_tensorflow//tensorflow/lite/kernels:test_util", @@ -66,6 +67,7 @@ cc_test( ":ngram_hash", "//mediapipe/framework/port:gtest_main", "//mediapipe/tasks/cc/text/language_detector/custom_ops/utils/hash:murmur", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/types:optional", "@flatbuffers", "@org_tensorflow//tensorflow/lite:framework", diff --git a/mediapipe/tasks/cc/text/language_detector/custom_ops/kmeans_embedding_lookup_test.cc b/mediapipe/tasks/cc/text/language_detector/custom_ops/kmeans_embedding_lookup_test.cc index f1ee661d..54b5161f 100644 --- a/mediapipe/tasks/cc/text/language_detector/custom_ops/kmeans_embedding_lookup_test.cc +++ b/mediapipe/tasks/cc/text/language_detector/custom_ops/kmeans_embedding_lookup_test.cc @@ -6,6 +6,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "tensorflow/lite/c/common.h" @@ -45,8 +46,8 @@ class KmeansEmbeddingLookupModel : public tflite::SingleOpModel { void Invoke(const std::vector& input, const std::vector& encoding_table, const std::vector& codebook) { - CHECK_EQ(SetUpInputTensor(input, encoding_table, codebook), kTfLiteOk); - CHECK_EQ(SingleOpModel::Invoke(), kTfLiteOk); + ABSL_CHECK_EQ(SetUpInputTensor(input, encoding_table, codebook), kTfLiteOk); + ABSL_CHECK_EQ(SingleOpModel::Invoke(), kTfLiteOk); } TfLiteStatus InvokeUnchecked(const std::vector& input, diff --git a/mediapipe/tasks/cc/text/language_detector/custom_ops/ngram_hash_test.cc b/mediapipe/tasks/cc/text/language_detector/custom_ops/ngram_hash_test.cc index d8b6ce3d..1e348bdd 100644 --- a/mediapipe/tasks/cc/text/language_detector/custom_ops/ngram_hash_test.cc +++ b/mediapipe/tasks/cc/text/language_detector/custom_ops/ngram_hash_test.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include +#include "absl/log/absl_check.h" #include "absl/types/optional.h" #include "flatbuffers/flexbuffers.h" #include "mediapipe/framework/port/gmock.h" @@ -78,13 +79,13 @@ class NGramHashModel : public tflite::SingleOpModel { void SetupInputTensor(const std::string& input) { PopulateStringTensor(input_, {input}); - CHECK(interpreter_->AllocateTensors() == kTfLiteOk) + ABSL_CHECK(interpreter_->AllocateTensors() == kTfLiteOk) << "Cannot allocate tensors"; } void Invoke(const std::string& input) { SetupInputTensor(input); - CHECK_EQ(SingleOpModel::Invoke(), kTfLiteOk); + ABSL_CHECK_EQ(SingleOpModel::Invoke(), kTfLiteOk); } TfLiteStatus InvokeUnchecked(const std::string& input) { diff --git a/mediapipe/tasks/cc/text/text_classifier/BUILD b/mediapipe/tasks/cc/text/text_classifier/BUILD index 28b5d709..121b4f5e 100644 --- a/mediapipe/tasks/cc/text/text_classifier/BUILD +++ b/mediapipe/tasks/cc/text/text_classifier/BUILD @@ -86,10 +86,9 @@ cc_test( "//mediapipe/tasks/cc/components/containers:classification_result", "@com_google_absl//absl/flags:flag", "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", - "@com_google_sentencepiece//src:sentencepiece_processor", + "@com_google_sentencepiece//src:sentencepiece_processor", # fixdeps: keep "@org_tensorflow//tensorflow/lite:test_util", ], ) diff --git a/mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc b/mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc index e10bd53f..dfb78c07 100644 --- a/mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc +++ b/mediapipe/tasks/cc/text/text_classifier/text_classifier_test.cc @@ -15,8 +15,6 @@ limitations under the License. #include "mediapipe/tasks/cc/text/text_classifier/text_classifier.h" -#include -#include #include #include #include @@ -24,7 +22,6 @@ limitations under the License. #include "absl/flags/flag.h" #include "absl/status/status.h" -#include "absl/status/statusor.h" #include "absl/strings/cord.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" diff --git a/mediapipe/tasks/cc/text/text_embedder/BUILD b/mediapipe/tasks/cc/text/text_embedder/BUILD index 76025b3c..c925abcb 100644 --- a/mediapipe/tasks/cc/text/text_embedder/BUILD +++ b/mediapipe/tasks/cc/text/text_embedder/BUILD @@ -66,6 +66,7 @@ cc_library( "//mediapipe/tasks/cc/core/proto:model_resources_calculator_cc_proto", "//mediapipe/tasks/cc/text/text_embedder/proto:text_embedder_graph_options_cc_proto", "//mediapipe/tasks/cc/text/utils:text_model_utils", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/mediapipe/tasks/cc/text/text_embedder/text_embedder_graph.cc b/mediapipe/tasks/cc/text/text_embedder/text_embedder_graph.cc index 9c812e9f..d5bdda4f 100644 --- a/mediapipe/tasks/cc/text/text_embedder/text_embedder_graph.cc +++ b/mediapipe/tasks/cc/text/text_embedder/text_embedder_graph.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -86,7 +87,7 @@ class TextEmbedderGraph : public core::ModelTaskGraph { public: absl::StatusOr GetConfig( SubgraphContext* sc) override { - CHECK(sc != nullptr); + ABSL_CHECK(sc != nullptr); ASSIGN_OR_RETURN(const ModelResources* model_resources, CreateModelResources(sc)); Graph graph; diff --git a/mediapipe/tasks/cc/text/tokenizers/BUILD b/mediapipe/tasks/cc/text/tokenizers/BUILD index 01908cd2..b299f1c7 100644 --- a/mediapipe/tasks/cc/text/tokenizers/BUILD +++ b/mediapipe/tasks/cc/text/tokenizers/BUILD @@ -71,6 +71,7 @@ cc_library( deps = [ ":tokenizer", "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", "@com_google_sentencepiece//src:sentencepiece_processor", ], @@ -86,6 +87,7 @@ cc_test( ":sentencepiece_tokenizer", "//mediapipe/framework/port:gtest_main", "//mediapipe/tasks/cc/core:utils", + "@com_google_absl//absl/log:absl_check", "@com_google_sentencepiece//src:sentencepiece_processor", ], ) @@ -105,6 +107,7 @@ cc_library( "//mediapipe/tasks/cc:common", "//mediapipe/tasks/cc/metadata:metadata_extractor", "//mediapipe/tasks/metadata:metadata_schema_cc", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -119,6 +122,7 @@ cc_test( "//mediapipe/tasks/testdata/text:albert_model", "//mediapipe/tasks/testdata/text:mobile_bert_model", "//mediapipe/tasks/testdata/text:text_classifier_models", + "@com_google_absl//absl/log:absl_check", ], linkopts = ["-ldl"], deps = [ diff --git a/mediapipe/tasks/cc/text/tokenizers/sentencepiece_tokenizer.h b/mediapipe/tasks/cc/text/tokenizers/sentencepiece_tokenizer.h index e1aab0ec..97ef1848 100644 --- a/mediapipe/tasks/cc/text/tokenizers/sentencepiece_tokenizer.h +++ b/mediapipe/tasks/cc/text/tokenizers/sentencepiece_tokenizer.h @@ -21,6 +21,7 @@ limitations under the License. #include #include +#include "absl/log/absl_check.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/tasks/cc/text/tokenizers/tokenizer.h" @@ -36,20 +37,27 @@ class SentencePieceTokenizer : public Tokenizer { public: // Initialize the SentencePiece tokenizer from model file path. explicit SentencePieceTokenizer(const std::string& path_to_model) { - CHECK_OK(sp_.Load(path_to_model)); + // Can't use ABSL_CHECK_OK here because in internal builds + // the return type is absl::Status while the open source builds + // use sentencepiece/src/deps/status.h's util::Status which + // doesn't work with the absl CHECK macros. + const auto status = sp_.Load(path_to_model); + ABSL_CHECK(status.ok()) << status.ToString(); } explicit SentencePieceTokenizer(const char* spmodel_buffer_data, size_t spmodel_buffer_size) { absl::string_view buffer_binary(spmodel_buffer_data, spmodel_buffer_size); - CHECK_OK(sp_.LoadFromSerializedProto(buffer_binary)); + const auto status = sp_.LoadFromSerializedProto(buffer_binary); + ABSL_CHECK(status.ok()) << status.ToString(); } // Perform tokenization, return tokenized results. TokenizerResult Tokenize(const std::string& input) override { TokenizerResult result; std::vector& subwords = result.subwords; - CHECK_OK(sp_.Encode(input, &subwords)); + const auto status = sp_.Encode(input, &subwords); + ABSL_CHECK(status.ok()) << status.ToString(); return result; } diff --git a/mediapipe/tasks/cc/vision/custom_ops/BUILD b/mediapipe/tasks/cc/vision/custom_ops/BUILD new file mode 100644 index 00000000..71eda50d --- /dev/null +++ b/mediapipe/tasks/cc/vision/custom_ops/BUILD @@ -0,0 +1,35 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "fused_batch_norm", + srcs = ["fused_batch_norm.cc"], + hdrs = ["fused_batch_norm.h"], + visibility = [ + "//visibility:public", + ], + deps = + [ + "@eigen_archive//:eigen3", + "@org_tensorflow//tensorflow/lite:framework", + "@org_tensorflow//tensorflow/lite/c:common", + "@org_tensorflow//tensorflow/lite/core/c:private_common", + "@org_tensorflow//tensorflow/lite/kernels:kernel_util", + "@org_tensorflow//tensorflow/lite/kernels/internal:tensor", + ], +) diff --git a/mediapipe/tasks/cc/vision/custom_ops/fused_batch_norm.cc b/mediapipe/tasks/cc/vision/custom_ops/fused_batch_norm.cc new file mode 100644 index 00000000..650f723e --- /dev/null +++ b/mediapipe/tasks/cc/vision/custom_ops/fused_batch_norm.cc @@ -0,0 +1,293 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/custom_ops/fused_batch_norm.h" + +#include + +#include "Eigen/Core" +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" + +namespace mediapipe::tflite_operations { +namespace vision::batch_norm { +namespace { + +using tflite::GetTensorData; + +constexpr int kInputIndex = 0; +constexpr int kInputScaleIndex = 1; +constexpr int kInputOffsetIndex = 2; +constexpr int kInputEstimatedMeanIndex = 3; +constexpr int kInputEstimatedVarIndex = 4; + +constexpr int kOutputIndex = 0; +constexpr int kOutputBatchMeanIndex = 1; +constexpr int kOutputBatchVarIndex = 2; +constexpr int kOutputSavedMeanIndex = 3; +constexpr int kOutputSavedVarIndex = 4; + +template +struct TTypes { + // Rank- tensor of scalar type T. + typedef Eigen::TensorMap> + Tensor; + + // Rank-1 tensor (vector) of scalar type T. + typedef Eigen::TensorMap> Vec; + typedef Eigen::TensorMap< + Eigen::Tensor> + ConstVec; +}; + +template +void FusedBarchNorm(TfLiteContext* context, TfLiteTensor* x_input, + TfLiteTensor* scale_input, TfLiteTensor* offset_input, + TfLiteTensor* running_mean_input, + TfLiteTensor* running_variance_input, + TfLiteTensor* y_output, TfLiteTensor* running_mean_output, + TfLiteTensor* running_var_output, + TfLiteTensor* saved_batch_mean_output, + TfLiteTensor* saved_batch_var_output, + U exponential_avg_factor, U epsilon) { + const int batches = x_input->dims->data[0]; + const int height = x_input->dims->data[1]; + const int width = x_input->dims->data[2]; + const int depth = x_input->dims->data[3]; + + Eigen::array x_dims = {batches, height, width, depth}; + Eigen::array depth_dims = {depth}; + + const int rest_size = batches * height * width; + + typename TTypes::Tensor x(GetTensorData(x_input), x_dims); + typename TTypes::ConstVec scale(GetTensorData(scale_input), depth_dims); + typename TTypes::ConstVec offset(GetTensorData(offset_input), + depth_dims); + typename TTypes::ConstVec old_mean(GetTensorData(running_mean_input), + depth_dims); + typename TTypes::ConstVec old_variance( + GetTensorData(running_variance_input), depth_dims); + typename TTypes::Tensor y(GetTensorData(y_output), x_dims); + typename TTypes::Vec new_mean(GetTensorData(running_mean_output), + depth_dims); + typename TTypes::Vec new_variance(GetTensorData(running_var_output), + depth_dims); + typename TTypes::Vec saved_batch_mean( + GetTensorData(saved_batch_mean_output), depth_dims); + typename TTypes::Vec saved_batch_var( + GetTensorData(saved_batch_var_output), depth_dims); + + Eigen::DSizes rest_by_depth(rest_size, depth); + Eigen::DSizes tensor_shape(batches, height, width, depth); + + Eigen::IndexList, Eigen::Index> one_by_depth; + one_by_depth.set(1, depth); + Eigen::IndexList> reduce_dims; + Eigen::IndexList> bcast_spec; + bcast_spec.set(0, rest_size); + + auto x_rest_by_depth = x.reshape(rest_by_depth).template cast(); + const int rest_size_minus_one = (rest_size > 1) ? (rest_size - 1) : 1; + U rest_size_inv = static_cast(1.0f / static_cast(rest_size)); + // This adjustment is for Bessel's correction + U rest_size_adjust = + static_cast(rest_size) / static_cast(rest_size_minus_one); + + Eigen::Tensor batch_mean(depth); + Eigen::Tensor batch_variance(depth); + + batch_mean = (x_rest_by_depth.sum(reduce_dims) * rest_size_inv); + auto x_centered = + x_rest_by_depth - batch_mean.reshape(one_by_depth).broadcast(bcast_spec); + + batch_variance = x_centered.square().sum(reduce_dims) * rest_size_inv; + auto scaling_factor = ((batch_variance + epsilon).rsqrt() * scale) + .eval() + .reshape(one_by_depth) + .broadcast(bcast_spec); + auto x_scaled = x_centered * scaling_factor; + auto x_shifted = + (x_scaled + offset.reshape(one_by_depth).broadcast(bcast_spec)) + .template cast(); + + y.reshape(rest_by_depth) = x_shifted; + if (exponential_avg_factor == U(1.0)) { + saved_batch_var = batch_variance; + saved_batch_mean = batch_mean; + new_variance = batch_variance * rest_size_adjust; + new_mean = batch_mean; + } else { + U one_minus_factor = U(1) - exponential_avg_factor; + saved_batch_var = batch_variance; + saved_batch_mean = batch_mean; + new_variance = one_minus_factor * old_variance + + (exponential_avg_factor * rest_size_adjust) * batch_variance; + new_mean = + one_minus_factor * old_mean + exponential_avg_factor * batch_mean; + } +} + +} // namespace + +// Initializes FusedBatchNorm object from serialized parameters. +void* Initialize(TfLiteContext* /*context*/, const char* /*buffer*/, + size_t /*length*/) { + return nullptr; +} + +void Free(TfLiteContext* /*context*/, void* /*buffer*/) {} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, tflite::NumInputs(node), 5); + TF_LITE_ENSURE_EQ(context, tflite::NumOutputs(node), 6); + + TfLiteTensor* output = tflite::GetOutput(context, node, kOutputIndex); + TF_LITE_ENSURE(context, output != nullptr); + TfLiteTensor* batch_mean = + tflite::GetOutput(context, node, kOutputBatchMeanIndex); + TF_LITE_ENSURE(context, batch_mean != nullptr); + TfLiteTensor* batch_var = + tflite::GetOutput(context, node, kOutputBatchVarIndex); + TF_LITE_ENSURE(context, batch_var != nullptr); + TfLiteTensor* saved_mean = + tflite::GetOutput(context, node, kOutputSavedMeanIndex); + TF_LITE_ENSURE(context, saved_mean != nullptr); + TfLiteTensor* saved_var = + tflite::GetOutput(context, node, kOutputSavedVarIndex); + TF_LITE_ENSURE(context, saved_var != nullptr); + TfLiteTensor* dummy_reserve_space = tflite::GetOutput(context, node, 5); + TF_LITE_ENSURE(context, dummy_reserve_space != nullptr); + + const TfLiteTensor* input = tflite::GetInput(context, node, kInputIndex); + TF_LITE_ENSURE(context, input != nullptr); + const TfLiteTensor* scale = tflite::GetInput(context, node, kInputScaleIndex); + TF_LITE_ENSURE(context, scale != nullptr); + const TfLiteTensor* offset = + tflite::GetInput(context, node, kInputOffsetIndex); + TF_LITE_ENSURE(context, offset != nullptr); + const TfLiteTensor* estimated_mean = + tflite::GetInput(context, node, kInputEstimatedMeanIndex); + TF_LITE_ENSURE(context, estimated_mean != nullptr); + const TfLiteTensor* estimated_var = + tflite::GetInput(context, node, kInputEstimatedVarIndex); + TF_LITE_ENSURE(context, estimated_var != nullptr); + + TF_LITE_ENSURE_EQ(context, tflite::NumDimensions(input), 4); + TF_LITE_ENSURE_EQ(context, tflite::NumDimensions(scale), 1); + TF_LITE_ENSURE_EQ(context, tflite::NumDimensions(offset), 1); + TF_LITE_ENSURE_EQ(context, tflite::NumDimensions(estimated_mean), 1); + TF_LITE_ENSURE_EQ(context, tflite::NumDimensions(estimated_var), 1); + TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32); + TF_LITE_ENSURE_EQ(context, output->type, kTfLiteFloat32); + TF_LITE_ENSURE_EQ(context, scale->type, kTfLiteFloat32); + TF_LITE_ENSURE_EQ(context, offset->type, kTfLiteFloat32); + + int batches = input->dims->data[0]; + int height = input->dims->data[1]; + int width = input->dims->data[2]; + int depth = input->dims->data[3]; + TfLiteIntArray* output_size = TfLiteIntArrayCreate(4); + output_size->data[0] = batches; + output_size->data[1] = height; + output_size->data[2] = width; + output_size->data[3] = depth; + if (context->ResizeTensor(context, output, output_size) != kTfLiteOk) { + return kTfLiteError; + } + TfLiteIntArray* batch_mean_size = TfLiteIntArrayCreate(1); + batch_mean_size->data[0] = depth; + if (context->ResizeTensor(context, batch_mean, batch_mean_size) != + kTfLiteOk) { + return kTfLiteError; + } + TfLiteIntArray* batch_var_size = TfLiteIntArrayCreate(1); + batch_var_size->data[0] = depth; + if (context->ResizeTensor(context, batch_var, batch_var_size) != kTfLiteOk) { + return kTfLiteError; + } + TfLiteIntArray* saved_mean_size = TfLiteIntArrayCreate(1); + saved_mean_size->data[0] = depth; + if (context->ResizeTensor(context, saved_mean, saved_mean_size) != + kTfLiteOk) { + return kTfLiteError; + } + TfLiteIntArray* saved_var_size = TfLiteIntArrayCreate(1); + saved_var_size->data[0] = depth; + if (context->ResizeTensor(context, saved_var, saved_var_size) != kTfLiteOk) { + return kTfLiteError; + } + TfLiteIntArray* dummy_reserve_size = TfLiteIntArrayCreate(1); + dummy_reserve_size->data[0] = 1; + if (context->ResizeTensor(context, dummy_reserve_space, dummy_reserve_size) != + kTfLiteOk) { + return kTfLiteError; + } + + return kTfLiteOk; +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + const TfLiteTensor* input = tflite::GetInput(context, node, kInputIndex); + TF_LITE_ENSURE(context, input != nullptr); + const TfLiteTensor* scale = tflite::GetInput(context, node, kInputScaleIndex); + TF_LITE_ENSURE(context, scale != nullptr); + const TfLiteTensor* offset = + tflite::GetInput(context, node, kInputOffsetIndex); + TF_LITE_ENSURE(context, offset != nullptr); + const TfLiteTensor* estimated_mean = + tflite::GetInput(context, node, kInputEstimatedMeanIndex); + TF_LITE_ENSURE(context, estimated_mean != nullptr); + const TfLiteTensor* estimated_var = + tflite::GetInput(context, node, kInputEstimatedVarIndex); + TF_LITE_ENSURE(context, estimated_var != nullptr); + + TfLiteTensor* output = tflite::GetOutput(context, node, kOutputIndex); + TF_LITE_ENSURE(context, output != nullptr); + TfLiteTensor* batch_mean = + tflite::GetOutput(context, node, kOutputBatchMeanIndex); + TF_LITE_ENSURE(context, batch_mean != nullptr); + TfLiteTensor* batch_var = + tflite::GetOutput(context, node, kOutputBatchVarIndex); + TF_LITE_ENSURE(context, batch_var != nullptr); + TfLiteTensor* saved_mean = + tflite::GetOutput(context, node, kOutputSavedMeanIndex); + TF_LITE_ENSURE(context, saved_mean != nullptr); + TfLiteTensor* saved_var = + tflite::GetOutput(context, node, kOutputSavedVarIndex); + TF_LITE_ENSURE(context, saved_var != nullptr); + + FusedBarchNorm( + context, const_cast(input), + const_cast(scale), const_cast(offset), + const_cast(estimated_mean), + const_cast(estimated_var), output, batch_mean, batch_var, + saved_mean, saved_var, /*exponential_avg_factor=*/0.001f, + /*epsilon=*/0.001f); + + return kTfLiteOk; +} +} // namespace vision::batch_norm + +TfLiteRegistration* Register_FusedBatchNorm() { + static TfLiteRegistration r = { + vision::batch_norm::Initialize, vision::batch_norm::Free, + vision::batch_norm::Prepare, vision::batch_norm::Eval}; + return &r; +} + +} // namespace mediapipe::tflite_operations diff --git a/mediapipe/tasks/cc/vision/custom_ops/fused_batch_norm.h b/mediapipe/tasks/cc/vision/custom_ops/fused_batch_norm.h new file mode 100644 index 00000000..98e16ff9 --- /dev/null +++ b/mediapipe/tasks/cc/vision/custom_ops/fused_batch_norm.h @@ -0,0 +1,28 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_CUSTOM_OPS_FUSED_BATCH_NORM_H_ +#define MEDIAPIPE_TASKS_CC_VISION_CUSTOM_OPS_FUSED_BATCH_NORM_H_ + +#include "tensorflow/lite/core/c/common.h" + +namespace mediapipe::tflite_operations { + +// The FusedBatchNorm op resolver is CPU-friendly only. +TfLiteRegistration* Register_FusedBatchNorm(); + +} // namespace mediapipe::tflite_operations + +#endif // MEDIAPIPE_TASKS_CC_VISION_CUSTOM_OPS_FUSED_BATCH_NORM_H_ diff --git a/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc index 72eb4cb5..651ad722 100644 --- a/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc +++ b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -119,8 +120,9 @@ absl::StatusOr> CreateTaskRunner( Detection GetExpectedFaceDetectionResult(absl::string_view file_name) { Detection detection; - CHECK_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), - &detection, Defaults())) + ABSL_CHECK_OK( + GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), + &detection, Defaults())) << "Expected face detection result does not exist."; return detection; } diff --git a/mediapipe/tasks/cc/vision/face_detector/face_detector_test.cc b/mediapipe/tasks/cc/vision/face_detector/face_detector_test.cc index 97c64ac1..fcb32a7d 100644 --- a/mediapipe/tasks/cc/vision/face_detector/face_detector_test.cc +++ b/mediapipe/tasks/cc/vision/face_detector/face_detector_test.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/port/file_helpers.h" @@ -57,8 +58,9 @@ constexpr float kKeypointErrorThreshold = 1e-2; FaceDetectorResult GetExpectedFaceDetectorResult(absl::string_view file_name) { mediapipe::Detection detection; - CHECK_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), - &detection, Defaults())) + ABSL_CHECK_OK( + GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), + &detection, Defaults())) << "Expected face detection result does not exist."; return components::containers::ConvertToDetectionResult({detection}); } diff --git a/mediapipe/tasks/cc/vision/face_landmarker/BUILD b/mediapipe/tasks/cc/vision/face_landmarker/BUILD index 16de2271..04e33c14 100644 --- a/mediapipe/tasks/cc/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/face_landmarker/BUILD @@ -213,7 +213,13 @@ cc_library( "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto", "//mediapipe/util:graph_builder_utils", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings:str_format", ], alwayslink = 1, ) + +cc_library( + name = "face_landmarks_connections", + hdrs = ["face_landmarks_connections.h"], +) diff --git a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc index 643f4062..54092b73 100644 --- a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc +++ b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h" #include "mediapipe/calculators/core/concatenate_vector_calculator.h" @@ -165,8 +166,8 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources, ->mutable_base_options() ->mutable_acceleration() ->mutable_xnnpack(); - LOG(WARNING) << "Face blendshape model contains CPU only ops. Sets " - << "FaceBlendshapesGraph acceleration to Xnnpack."; + ABSL_LOG(WARNING) << "Face blendshape model contains CPU only ops. Sets " + << "FaceBlendshapesGraph acceleration to Xnnpack."; } return absl::OkStatus(); diff --git a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_connections.h b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_connections.h new file mode 100644 index 00000000..360083a7 --- /dev/null +++ b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_connections.h @@ -0,0 +1,651 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_FACE_LANDMARKER_FACE_LANDMARKS_CONNECTIONS_H_ +#define MEDIAPIPE_TASKS_CC_VISION_FACE_LANDMARKER_FACE_LANDMARKS_CONNECTIONS_H_ + +#include + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace face_landmarker { + +struct FaceLandmarksConnections { + static constexpr std::array, 40> kFaceLandmarksLips{ + {{61, 146}, {146, 91}, {91, 181}, {181, 84}, {84, 17}, {17, 314}, + {314, 405}, {405, 321}, {321, 375}, {375, 291}, {61, 185}, {185, 40}, + {40, 39}, {39, 37}, {37, 0}, {0, 267}, {267, 269}, {269, 270}, + {270, 409}, {409, 291}, {78, 95}, {95, 88}, {88, 178}, {178, 87}, + {87, 14}, {14, 317}, {317, 402}, {402, 318}, {318, 324}, {324, 308}, + {78, 191}, {191, 80}, {80, 81}, {81, 82}, {82, 13}, {13, 312}, + {312, 311}, {311, 310}, {310, 415}, {415, 308}}}; + + static constexpr std::array, 16> kFaceLandmarksLeftEye{ + {{263, 249}, + {249, 390}, + {390, 373}, + {373, 374}, + {374, 380}, + {380, 381}, + {381, 382}, + {382, 362}, + {263, 466}, + {466, 388}, + {388, 387}, + {387, 386}, + {386, 385}, + {385, 384}, + {384, 398}, + {398, 362}}}; + + static constexpr std::array, 8> kFaceLandmarksLeftEyeBrow{ + {{276, 283}, + {283, 282}, + {282, 295}, + {295, 285}, + {300, 293}, + {293, 334}, + {334, 296}, + {296, 336}}}; + + static constexpr std::array, 4> kFaceLandmarksLeftIris{ + {{474, 475}, {475, 476}, {476, 477}, {477, 474}}}; + + static constexpr std::array, 16> kFaceLandmarksRightEye{ + {{33, 7}, + {7, 163}, + {163, 144}, + {144, 145}, + {145, 153}, + {153, 154}, + {154, 155}, + {155, 133}, + {33, 246}, + {246, 161}, + {161, 160}, + {160, 159}, + {159, 158}, + {158, 157}, + {157, 173}, + {173, 133}}}; + + static constexpr std::array, 8> kFaceLandmarksRightEyeBrow{ + {{46, 53}, + {53, 52}, + {52, 65}, + {65, 55}, + {70, 63}, + {63, 105}, + {105, 66}, + {66, 107}}}; + + static constexpr std::array, 4> kFaceLandmarksRightIris{ + {{469, 470}, {470, 471}, {471, 472}, {472, 469}}}; + + static constexpr std::array, 36> kFaceLandmarksFaceOval{ + {{10, 338}, {338, 297}, {297, 332}, {332, 284}, {284, 251}, {251, 389}, + {389, 356}, {356, 454}, {454, 323}, {323, 361}, {361, 288}, {288, 397}, + {397, 365}, {365, 379}, {379, 378}, {378, 400}, {400, 377}, {377, 152}, + {152, 148}, {148, 176}, {176, 149}, {149, 150}, {150, 136}, {136, 172}, + {172, 58}, {58, 132}, {132, 93}, {93, 234}, {234, 127}, {127, 162}, + {162, 21}, {21, 54}, {54, 103}, {103, 67}, {67, 109}, {109, 10}}}; + + // Lips + Left Eye + Left Eye Brows + Right Eye + Right Eye Brows + Face Oval. + static constexpr std::array, 132> kFaceLandmarksConnectors{ + {{61, 146}, {146, 91}, {91, 181}, {181, 84}, {84, 17}, {17, 314}, + {314, 405}, {405, 321}, {321, 375}, {375, 291}, {61, 185}, {185, 40}, + {40, 39}, {39, 37}, {37, 0}, {0, 267}, {267, 269}, {269, 270}, + {270, 409}, {409, 291}, {78, 95}, {95, 88}, {88, 178}, {178, 87}, + {87, 14}, {14, 317}, {317, 402}, {402, 318}, {318, 324}, {324, 308}, + {78, 191}, {191, 80}, {80, 81}, {81, 82}, {82, 13}, {13, 312}, + {312, 311}, {311, 310}, {310, 415}, {415, 30}, {263, 249}, {249, 390}, + {390, 373}, {373, 374}, {374, 380}, {380, 381}, {381, 382}, {382, 362}, + {263, 466}, {466, 388}, {388, 387}, {387, 386}, {386, 385}, {385, 384}, + {384, 398}, {398, 362}, {276, 283}, {283, 282}, {282, 295}, {295, 285}, + {300, 293}, {293, 334}, {334, 296}, {296, 336}, {33, 7}, {7, 163}, + {163, 144}, {144, 145}, {145, 153}, {153, 154}, {154, 155}, {155, 133}, + {33, 246}, {246, 161}, {161, 160}, {160, 159}, {159, 158}, {158, 157}, + {157, 173}, {173, 13}, {46, 53}, {53, 52}, {52, 65}, {65, 55}, + {70, 63}, {63, 105}, {105, 66}, {66, 107}, {10, 338}, {338, 297}, + {297, 332}, {332, 284}, {284, 251}, {251, 389}, {389, 356}, {356, 454}, + {454, 323}, {323, 361}, {361, 288}, {288, 397}, {397, 365}, {365, 379}, + {379, 378}, {378, 400}, {400, 377}, {377, 152}, {152, 148}, {148, 176}, + {176, 149}, {149, 150}, {150, 136}, {136, 172}, {172, 58}, {58, 132}, + {132, 93}, {93, 234}, {234, 127}, {127, 162}, {162, 21}, {21, 54}, + {54, 103}, {103, 67}, {67, 109}, {109, 10}}}; + + static constexpr std::array, 2556> + kFaceLandmarksTesselation{ + {{127, 34}, {34, 139}, {139, 127}, {11, 0}, {0, 37}, + {37, 11}, {232, 231}, {231, 120}, {120, 232}, {72, 37}, + {37, 39}, {39, 72}, {128, 121}, {121, 47}, {47, 128}, + {232, 121}, {121, 128}, {128, 232}, {104, 69}, {69, 67}, + {67, 104}, {175, 171}, {171, 148}, {148, 175}, {118, 50}, + {50, 101}, {101, 118}, {73, 39}, {39, 40}, {40, 73}, + {9, 151}, {151, 108}, {108, 9}, {48, 115}, {115, 131}, + {131, 48}, {194, 204}, {204, 211}, {211, 194}, {74, 40}, + {40, 185}, {185, 74}, {80, 42}, {42, 183}, {183, 80}, + {40, 92}, {92, 186}, {186, 40}, {230, 229}, {229, 118}, + {118, 230}, {202, 212}, {212, 214}, {214, 202}, {83, 18}, + {18, 17}, {17, 83}, {76, 61}, {61, 146}, {146, 76}, + {160, 29}, {29, 30}, {30, 160}, {56, 157}, {157, 173}, + {173, 56}, {106, 204}, {204, 194}, {194, 106}, {135, 214}, + {214, 192}, {192, 135}, {203, 165}, {165, 98}, {98, 203}, + {21, 71}, {71, 68}, {68, 21}, {51, 45}, {45, 4}, + {4, 51}, {144, 24}, {24, 23}, {23, 144}, {77, 146}, + {146, 91}, {91, 77}, {205, 50}, {50, 187}, {187, 205}, + {201, 200}, {200, 18}, {18, 201}, {91, 106}, {106, 182}, + {182, 91}, {90, 91}, {91, 181}, {181, 90}, {85, 84}, + {84, 17}, {17, 85}, {206, 203}, {203, 36}, {36, 206}, + {148, 171}, {171, 140}, {140, 148}, {92, 40}, {40, 39}, + {39, 92}, {193, 189}, {189, 244}, {244, 193}, {159, 158}, + {158, 28}, {28, 159}, {247, 246}, {246, 161}, {161, 247}, + {236, 3}, {3, 196}, {196, 236}, {54, 68}, {68, 104}, + {104, 54}, {193, 168}, {168, 8}, {8, 193}, {117, 228}, + {228, 31}, {31, 117}, {189, 193}, {193, 55}, {55, 189}, + {98, 97}, {97, 99}, {99, 98}, {126, 47}, {47, 100}, + {100, 126}, {166, 79}, {79, 218}, {218, 166}, {155, 154}, + {154, 26}, {26, 155}, {209, 49}, {49, 131}, {131, 209}, + {135, 136}, {136, 150}, {150, 135}, {47, 126}, {126, 217}, + {217, 47}, {223, 52}, {52, 53}, {53, 223}, {45, 51}, + {51, 134}, {134, 45}, {211, 170}, {170, 140}, {140, 211}, + {67, 69}, {69, 108}, {108, 67}, {43, 106}, {106, 91}, + {91, 43}, {230, 119}, {119, 120}, {120, 230}, {226, 130}, + {130, 247}, {247, 226}, {63, 53}, {53, 52}, {52, 63}, + {238, 20}, {20, 242}, {242, 238}, {46, 70}, {70, 156}, + {156, 46}, {78, 62}, {62, 96}, {96, 78}, {46, 53}, + {53, 63}, {63, 46}, {143, 34}, {34, 227}, {227, 143}, + {123, 117}, {117, 111}, {111, 123}, {44, 125}, {125, 19}, + {19, 44}, {236, 134}, {134, 51}, {51, 236}, {216, 206}, + {206, 205}, {205, 216}, {154, 153}, {153, 22}, {22, 154}, + {39, 37}, {37, 167}, {167, 39}, {200, 201}, {201, 208}, + {208, 200}, {36, 142}, {142, 100}, {100, 36}, {57, 212}, + {212, 202}, {202, 57}, {20, 60}, {60, 99}, {99, 20}, + {28, 158}, {158, 157}, {157, 28}, {35, 226}, {226, 113}, + {113, 35}, {160, 159}, {159, 27}, {27, 160}, {204, 202}, + {202, 210}, {210, 204}, {113, 225}, {225, 46}, {46, 113}, + {43, 202}, {202, 204}, {204, 43}, {62, 76}, {76, 77}, + {77, 62}, {137, 123}, {123, 116}, {116, 137}, {41, 38}, + {38, 72}, {72, 41}, {203, 129}, {129, 142}, {142, 203}, + {64, 98}, {98, 240}, {240, 64}, {49, 102}, {102, 64}, + {64, 49}, {41, 73}, {73, 74}, {74, 41}, {212, 216}, + {216, 207}, {207, 212}, {42, 74}, {74, 184}, {184, 42}, + {169, 170}, {170, 211}, {211, 169}, {170, 149}, {149, 176}, + {176, 170}, {105, 66}, {66, 69}, {69, 105}, {122, 6}, + {6, 168}, {168, 122}, {123, 147}, {147, 187}, {187, 123}, + {96, 77}, {77, 90}, {90, 96}, {65, 55}, {55, 107}, + {107, 65}, {89, 90}, {90, 180}, {180, 89}, {101, 100}, + {100, 120}, {120, 101}, {63, 105}, {105, 104}, {104, 63}, + {93, 137}, {137, 227}, {227, 93}, {15, 86}, {86, 85}, + {85, 15}, {129, 102}, {102, 49}, {49, 129}, {14, 87}, + {87, 86}, {86, 14}, {55, 8}, {8, 9}, {9, 55}, + {100, 47}, {47, 121}, {121, 100}, {145, 23}, {23, 22}, + {22, 145}, {88, 89}, {89, 179}, {179, 88}, {6, 122}, + {122, 196}, {196, 6}, {88, 95}, {95, 96}, {96, 88}, + {138, 172}, {172, 136}, {136, 138}, {215, 58}, {58, 172}, + {172, 215}, {115, 48}, {48, 219}, {219, 115}, {42, 80}, + {80, 81}, {81, 42}, {195, 3}, {3, 51}, {51, 195}, + {43, 146}, {146, 61}, {61, 43}, {171, 175}, {175, 199}, + {199, 171}, {81, 82}, {82, 38}, {38, 81}, {53, 46}, + {46, 225}, {225, 53}, {144, 163}, {163, 110}, {110, 144}, + {52, 65}, {65, 66}, {66, 52}, {229, 228}, {228, 117}, + {117, 229}, {34, 127}, {127, 234}, {234, 34}, {107, 108}, + {108, 69}, {69, 107}, {109, 108}, {108, 151}, {151, 109}, + {48, 64}, {64, 235}, {235, 48}, {62, 78}, {78, 191}, + {191, 62}, {129, 209}, {209, 126}, {126, 129}, {111, 35}, + {35, 143}, {143, 111}, {117, 123}, {123, 50}, {50, 117}, + {222, 65}, {65, 52}, {52, 222}, {19, 125}, {125, 141}, + {141, 19}, {221, 55}, {55, 65}, {65, 221}, {3, 195}, + {195, 197}, {197, 3}, {25, 7}, {7, 33}, {33, 25}, + {220, 237}, {237, 44}, {44, 220}, {70, 71}, {71, 139}, + {139, 70}, {122, 193}, {193, 245}, {245, 122}, {247, 130}, + {130, 33}, {33, 247}, {71, 21}, {21, 162}, {162, 71}, + {170, 169}, {169, 150}, {150, 170}, {188, 174}, {174, 196}, + {196, 188}, {216, 186}, {186, 92}, {92, 216}, {2, 97}, + {97, 167}, {167, 2}, {141, 125}, {125, 241}, {241, 141}, + {164, 167}, {167, 37}, {37, 164}, {72, 38}, {38, 12}, + {12, 72}, {38, 82}, {82, 13}, {13, 38}, {63, 68}, + {68, 71}, {71, 63}, {226, 35}, {35, 111}, {111, 226}, + {101, 50}, {50, 205}, {205, 101}, {206, 92}, {92, 165}, + {165, 206}, {209, 198}, {198, 217}, {217, 209}, {165, 167}, + {167, 97}, {97, 165}, {220, 115}, {115, 218}, {218, 220}, + {133, 112}, {112, 243}, {243, 133}, {239, 238}, {238, 241}, + {241, 239}, {214, 135}, {135, 169}, {169, 214}, {190, 173}, + {173, 133}, {133, 190}, {171, 208}, {208, 32}, {32, 171}, + {125, 44}, {44, 237}, {237, 125}, {86, 87}, {87, 178}, + {178, 86}, {85, 86}, {86, 179}, {179, 85}, {84, 85}, + {85, 180}, {180, 84}, {83, 84}, {84, 181}, {181, 83}, + {201, 83}, {83, 182}, {182, 201}, {137, 93}, {93, 132}, + {132, 137}, {76, 62}, {62, 183}, {183, 76}, {61, 76}, + {76, 184}, {184, 61}, {57, 61}, {61, 185}, {185, 57}, + {212, 57}, {57, 186}, {186, 212}, {214, 207}, {207, 187}, + {187, 214}, {34, 143}, {143, 156}, {156, 34}, {79, 239}, + {239, 237}, {237, 79}, {123, 137}, {137, 177}, {177, 123}, + {44, 1}, {1, 4}, {4, 44}, {201, 194}, {194, 32}, + {32, 201}, {64, 102}, {102, 129}, {129, 64}, {213, 215}, + {215, 138}, {138, 213}, {59, 166}, {166, 219}, {219, 59}, + {242, 99}, {99, 97}, {97, 242}, {2, 94}, {94, 141}, + {141, 2}, {75, 59}, {59, 235}, {235, 75}, {24, 110}, + {110, 228}, {228, 24}, {25, 130}, {130, 226}, {226, 25}, + {23, 24}, {24, 229}, {229, 23}, {22, 23}, {23, 230}, + {230, 22}, {26, 22}, {22, 231}, {231, 26}, {112, 26}, + {26, 232}, {232, 112}, {189, 190}, {190, 243}, {243, 189}, + {221, 56}, {56, 190}, {190, 221}, {28, 56}, {56, 221}, + {221, 28}, {27, 28}, {28, 222}, {222, 27}, {29, 27}, + {27, 223}, {223, 29}, {30, 29}, {29, 224}, {224, 30}, + {247, 30}, {30, 225}, {225, 247}, {238, 79}, {79, 20}, + {20, 238}, {166, 59}, {59, 75}, {75, 166}, {60, 75}, + {75, 240}, {240, 60}, {147, 177}, {177, 215}, {215, 147}, + {20, 79}, {79, 166}, {166, 20}, {187, 147}, {147, 213}, + {213, 187}, {112, 233}, {233, 244}, {244, 112}, {233, 128}, + {128, 245}, {245, 233}, {128, 114}, {114, 188}, {188, 128}, + {114, 217}, {217, 174}, {174, 114}, {131, 115}, {115, 220}, + {220, 131}, {217, 198}, {198, 236}, {236, 217}, {198, 131}, + {131, 134}, {134, 198}, {177, 132}, {132, 58}, {58, 177}, + {143, 35}, {35, 124}, {124, 143}, {110, 163}, {163, 7}, + {7, 110}, {228, 110}, {110, 25}, {25, 228}, {356, 389}, + {389, 368}, {368, 356}, {11, 302}, {302, 267}, {267, 11}, + {452, 350}, {350, 349}, {349, 452}, {302, 303}, {303, 269}, + {269, 302}, {357, 343}, {343, 277}, {277, 357}, {452, 453}, + {453, 357}, {357, 452}, {333, 332}, {332, 297}, {297, 333}, + {175, 152}, {152, 377}, {377, 175}, {347, 348}, {348, 330}, + {330, 347}, {303, 304}, {304, 270}, {270, 303}, {9, 336}, + {336, 337}, {337, 9}, {278, 279}, {279, 360}, {360, 278}, + {418, 262}, {262, 431}, {431, 418}, {304, 408}, {408, 409}, + {409, 304}, {310, 415}, {415, 407}, {407, 310}, {270, 409}, + {409, 410}, {410, 270}, {450, 348}, {348, 347}, {347, 450}, + {422, 430}, {430, 434}, {434, 422}, {313, 314}, {314, 17}, + {17, 313}, {306, 307}, {307, 375}, {375, 306}, {387, 388}, + {388, 260}, {260, 387}, {286, 414}, {414, 398}, {398, 286}, + {335, 406}, {406, 418}, {418, 335}, {364, 367}, {367, 416}, + {416, 364}, {423, 358}, {358, 327}, {327, 423}, {251, 284}, + {284, 298}, {298, 251}, {281, 5}, {5, 4}, {4, 281}, + {373, 374}, {374, 253}, {253, 373}, {307, 320}, {320, 321}, + {321, 307}, {425, 427}, {427, 411}, {411, 425}, {421, 313}, + {313, 18}, {18, 421}, {321, 405}, {405, 406}, {406, 321}, + {320, 404}, {404, 405}, {405, 320}, {315, 16}, {16, 17}, + {17, 315}, {426, 425}, {425, 266}, {266, 426}, {377, 400}, + {400, 369}, {369, 377}, {322, 391}, {391, 269}, {269, 322}, + {417, 465}, {465, 464}, {464, 417}, {386, 257}, {257, 258}, + {258, 386}, {466, 260}, {260, 388}, {388, 466}, {456, 399}, + {399, 419}, {419, 456}, {284, 332}, {332, 333}, {333, 284}, + {417, 285}, {285, 8}, {8, 417}, {346, 340}, {340, 261}, + {261, 346}, {413, 441}, {441, 285}, {285, 413}, {327, 460}, + {460, 328}, {328, 327}, {355, 371}, {371, 329}, {329, 355}, + {392, 439}, {439, 438}, {438, 392}, {382, 341}, {341, 256}, + {256, 382}, {429, 420}, {420, 360}, {360, 429}, {364, 394}, + {394, 379}, {379, 364}, {277, 343}, {343, 437}, {437, 277}, + {443, 444}, {444, 283}, {283, 443}, {275, 440}, {440, 363}, + {363, 275}, {431, 262}, {262, 369}, {369, 431}, {297, 338}, + {338, 337}, {337, 297}, {273, 375}, {375, 321}, {321, 273}, + {450, 451}, {451, 349}, {349, 450}, {446, 342}, {342, 467}, + {467, 446}, {293, 334}, {334, 282}, {282, 293}, {458, 461}, + {461, 462}, {462, 458}, {276, 353}, {353, 383}, {383, 276}, + {308, 324}, {324, 325}, {325, 308}, {276, 300}, {300, 293}, + {293, 276}, {372, 345}, {345, 447}, {447, 372}, {352, 345}, + {345, 340}, {340, 352}, {274, 1}, {1, 19}, {19, 274}, + {456, 248}, {248, 281}, {281, 456}, {436, 427}, {427, 425}, + {425, 436}, {381, 256}, {256, 252}, {252, 381}, {269, 391}, + {391, 393}, {393, 269}, {200, 199}, {199, 428}, {428, 200}, + {266, 330}, {330, 329}, {329, 266}, {287, 273}, {273, 422}, + {422, 287}, {250, 462}, {462, 328}, {328, 250}, {258, 286}, + {286, 384}, {384, 258}, {265, 353}, {353, 342}, {342, 265}, + {387, 259}, {259, 257}, {257, 387}, {424, 431}, {431, 430}, + {430, 424}, {342, 353}, {353, 276}, {276, 342}, {273, 335}, + {335, 424}, {424, 273}, {292, 325}, {325, 307}, {307, 292}, + {366, 447}, {447, 345}, {345, 366}, {271, 303}, {303, 302}, + {302, 271}, {423, 266}, {266, 371}, {371, 423}, {294, 455}, + {455, 460}, {460, 294}, {279, 278}, {278, 294}, {294, 279}, + {271, 272}, {272, 304}, {304, 271}, {432, 434}, {434, 427}, + {427, 432}, {272, 407}, {407, 408}, {408, 272}, {394, 430}, + {430, 431}, {431, 394}, {395, 369}, {369, 400}, {400, 395}, + {334, 333}, {333, 299}, {299, 334}, {351, 417}, {417, 168}, + {168, 351}, {352, 280}, {280, 411}, {411, 352}, {325, 319}, + {319, 320}, {320, 325}, {295, 296}, {296, 336}, {336, 295}, + {319, 403}, {403, 404}, {404, 319}, {330, 348}, {348, 349}, + {349, 330}, {293, 298}, {298, 333}, {333, 293}, {323, 454}, + {454, 447}, {447, 323}, {15, 16}, {16, 315}, {315, 15}, + {358, 429}, {429, 279}, {279, 358}, {14, 15}, {15, 316}, + {316, 14}, {285, 336}, {336, 9}, {9, 285}, {329, 349}, + {349, 350}, {350, 329}, {374, 380}, {380, 252}, {252, 374}, + {318, 402}, {402, 403}, {403, 318}, {6, 197}, {197, 419}, + {419, 6}, {318, 319}, {319, 325}, {325, 318}, {367, 364}, + {364, 365}, {365, 367}, {435, 367}, {367, 397}, {397, 435}, + {344, 438}, {438, 439}, {439, 344}, {272, 271}, {271, 311}, + {311, 272}, {195, 5}, {5, 281}, {281, 195}, {273, 287}, + {287, 291}, {291, 273}, {396, 428}, {428, 199}, {199, 396}, + {311, 271}, {271, 268}, {268, 311}, {283, 444}, {444, 445}, + {445, 283}, {373, 254}, {254, 339}, {339, 373}, {282, 334}, + {334, 296}, {296, 282}, {449, 347}, {347, 346}, {346, 449}, + {264, 447}, {447, 454}, {454, 264}, {336, 296}, {296, 299}, + {299, 336}, {338, 10}, {10, 151}, {151, 338}, {278, 439}, + {439, 455}, {455, 278}, {292, 407}, {407, 415}, {415, 292}, + {358, 371}, {371, 355}, {355, 358}, {340, 345}, {345, 372}, + {372, 340}, {346, 347}, {347, 280}, {280, 346}, {442, 443}, + {443, 282}, {282, 442}, {19, 94}, {94, 370}, {370, 19}, + {441, 442}, {442, 295}, {295, 441}, {248, 419}, {419, 197}, + {197, 248}, {263, 255}, {255, 359}, {359, 263}, {440, 275}, + {275, 274}, {274, 440}, {300, 383}, {383, 368}, {368, 300}, + {351, 412}, {412, 465}, {465, 351}, {263, 467}, {467, 466}, + {466, 263}, {301, 368}, {368, 389}, {389, 301}, {395, 378}, + {378, 379}, {379, 395}, {412, 351}, {351, 419}, {419, 412}, + {436, 426}, {426, 322}, {322, 436}, {2, 164}, {164, 393}, + {393, 2}, {370, 462}, {462, 461}, {461, 370}, {164, 0}, + {0, 267}, {267, 164}, {302, 11}, {11, 12}, {12, 302}, + {268, 12}, {12, 13}, {13, 268}, {293, 300}, {300, 301}, + {301, 293}, {446, 261}, {261, 340}, {340, 446}, {330, 266}, + {266, 425}, {425, 330}, {426, 423}, {423, 391}, {391, 426}, + {429, 355}, {355, 437}, {437, 429}, {391, 327}, {327, 326}, + {326, 391}, {440, 457}, {457, 438}, {438, 440}, {341, 382}, + {382, 362}, {362, 341}, {459, 457}, {457, 461}, {461, 459}, + {434, 430}, {430, 394}, {394, 434}, {414, 463}, {463, 362}, + {362, 414}, {396, 369}, {369, 262}, {262, 396}, {354, 461}, + {461, 457}, {457, 354}, {316, 403}, {403, 402}, {402, 316}, + {315, 404}, {404, 403}, {403, 315}, {314, 405}, {405, 404}, + {404, 314}, {313, 406}, {406, 405}, {405, 313}, {421, 418}, + {418, 406}, {406, 421}, {366, 401}, {401, 361}, {361, 366}, + {306, 408}, {408, 407}, {407, 306}, {291, 409}, {409, 408}, + {408, 291}, {287, 410}, {410, 409}, {409, 287}, {432, 436}, + {436, 410}, {410, 432}, {434, 416}, {416, 411}, {411, 434}, + {264, 368}, {368, 383}, {383, 264}, {309, 438}, {438, 457}, + {457, 309}, {352, 376}, {376, 401}, {401, 352}, {274, 275}, + {275, 4}, {4, 274}, {421, 428}, {428, 262}, {262, 421}, + {294, 327}, {327, 358}, {358, 294}, {433, 416}, {416, 367}, + {367, 433}, {289, 455}, {455, 439}, {439, 289}, {462, 370}, + {370, 326}, {326, 462}, {2, 326}, {326, 370}, {370, 2}, + {305, 460}, {460, 455}, {455, 305}, {254, 449}, {449, 448}, + {448, 254}, {255, 261}, {261, 446}, {446, 255}, {253, 450}, + {450, 449}, {449, 253}, {252, 451}, {451, 450}, {450, 252}, + {256, 452}, {452, 451}, {451, 256}, {341, 453}, {453, 452}, + {452, 341}, {413, 464}, {464, 463}, {463, 413}, {441, 413}, + {413, 414}, {414, 441}, {258, 442}, {442, 441}, {441, 258}, + {257, 443}, {443, 442}, {442, 257}, {259, 444}, {444, 443}, + {443, 259}, {260, 445}, {445, 444}, {444, 260}, {467, 342}, + {342, 445}, {445, 467}, {459, 458}, {458, 250}, {250, 459}, + {289, 392}, {392, 290}, {290, 289}, {290, 328}, {328, 460}, + {460, 290}, {376, 433}, {433, 435}, {435, 376}, {250, 290}, + {290, 392}, {392, 250}, {411, 416}, {416, 433}, {433, 411}, + {341, 463}, {463, 464}, {464, 341}, {453, 464}, {464, 465}, + {465, 453}, {357, 465}, {465, 412}, {412, 357}, {343, 412}, + {412, 399}, {399, 343}, {360, 363}, {363, 440}, {440, 360}, + {437, 399}, {399, 456}, {456, 437}, {420, 456}, {456, 363}, + {363, 420}, {401, 435}, {435, 288}, {288, 401}, {372, 383}, + {383, 353}, {353, 372}, {339, 255}, {255, 249}, {249, 339}, + {448, 261}, {261, 255}, {255, 448}, {133, 243}, {243, 190}, + {190, 133}, {133, 155}, {155, 112}, {112, 133}, {33, 246}, + {246, 247}, {247, 33}, {33, 130}, {130, 25}, {25, 33}, + {398, 384}, {384, 286}, {286, 398}, {362, 398}, {398, 414}, + {414, 362}, {362, 463}, {463, 341}, {341, 362}, {263, 359}, + {359, 467}, {467, 263}, {263, 249}, {249, 255}, {255, 263}, + {466, 467}, {467, 260}, {260, 466}, {75, 60}, {60, 166}, + {166, 75}, {238, 239}, {239, 79}, {79, 238}, {162, 127}, + {127, 139}, {139, 162}, {72, 11}, {11, 37}, {37, 72}, + {121, 232}, {232, 120}, {120, 121}, {73, 72}, {72, 39}, + {39, 73}, {114, 128}, {128, 47}, {47, 114}, {233, 232}, + {232, 128}, {128, 233}, {103, 104}, {104, 67}, {67, 103}, + {152, 175}, {175, 148}, {148, 152}, {119, 118}, {118, 101}, + {101, 119}, {74, 73}, {73, 40}, {40, 74}, {107, 9}, + {9, 108}, {108, 107}, {49, 48}, {48, 131}, {131, 49}, + {32, 194}, {194, 211}, {211, 32}, {184, 74}, {74, 185}, + {185, 184}, {191, 80}, {80, 183}, {183, 191}, {185, 40}, + {40, 186}, {186, 185}, {119, 230}, {230, 118}, {118, 119}, + {210, 202}, {202, 214}, {214, 210}, {84, 83}, {83, 17}, + {17, 84}, {77, 76}, {76, 146}, {146, 77}, {161, 160}, + {160, 30}, {30, 161}, {190, 56}, {56, 173}, {173, 190}, + {182, 106}, {106, 194}, {194, 182}, {138, 135}, {135, 192}, + {192, 138}, {129, 203}, {203, 98}, {98, 129}, {54, 21}, + {21, 68}, {68, 54}, {5, 51}, {51, 4}, {4, 5}, + {145, 144}, {144, 23}, {23, 145}, {90, 77}, {77, 91}, + {91, 90}, {207, 205}, {205, 187}, {187, 207}, {83, 201}, + {201, 18}, {18, 83}, {181, 91}, {91, 182}, {182, 181}, + {180, 90}, {90, 181}, {181, 180}, {16, 85}, {85, 17}, + {17, 16}, {205, 206}, {206, 36}, {36, 205}, {176, 148}, + {148, 140}, {140, 176}, {165, 92}, {92, 39}, {39, 165}, + {245, 193}, {193, 244}, {244, 245}, {27, 159}, {159, 28}, + {28, 27}, {30, 247}, {247, 161}, {161, 30}, {174, 236}, + {236, 196}, {196, 174}, {103, 54}, {54, 104}, {104, 103}, + {55, 193}, {193, 8}, {8, 55}, {111, 117}, {117, 31}, + {31, 111}, {221, 189}, {189, 55}, {55, 221}, {240, 98}, + {98, 99}, {99, 240}, {142, 126}, {126, 100}, {100, 142}, + {219, 166}, {166, 218}, {218, 219}, {112, 155}, {155, 26}, + {26, 112}, {198, 209}, {209, 131}, {131, 198}, {169, 135}, + {135, 150}, {150, 169}, {114, 47}, {47, 217}, {217, 114}, + {224, 223}, {223, 53}, {53, 224}, {220, 45}, {45, 134}, + {134, 220}, {32, 211}, {211, 140}, {140, 32}, {109, 67}, + {67, 108}, {108, 109}, {146, 43}, {43, 91}, {91, 146}, + {231, 230}, {230, 120}, {120, 231}, {113, 226}, {226, 247}, + {247, 113}, {105, 63}, {63, 52}, {52, 105}, {241, 238}, + {238, 242}, {242, 241}, {124, 46}, {46, 156}, {156, 124}, + {95, 78}, {78, 96}, {96, 95}, {70, 46}, {46, 63}, + {63, 70}, {116, 143}, {143, 227}, {227, 116}, {116, 123}, + {123, 111}, {111, 116}, {1, 44}, {44, 19}, {19, 1}, + {3, 236}, {236, 51}, {51, 3}, {207, 216}, {216, 205}, + {205, 207}, {26, 154}, {154, 22}, {22, 26}, {165, 39}, + {39, 167}, {167, 165}, {199, 200}, {200, 208}, {208, 199}, + {101, 36}, {36, 100}, {100, 101}, {43, 57}, {57, 202}, + {202, 43}, {242, 20}, {20, 99}, {99, 242}, {56, 28}, + {28, 157}, {157, 56}, {124, 35}, {35, 113}, {113, 124}, + {29, 160}, {160, 27}, {27, 29}, {211, 204}, {204, 210}, + {210, 211}, {124, 113}, {113, 46}, {46, 124}, {106, 43}, + {43, 204}, {204, 106}, {96, 62}, {62, 77}, {77, 96}, + {227, 137}, {137, 116}, {116, 227}, {73, 41}, {41, 72}, + {72, 73}, {36, 203}, {203, 142}, {142, 36}, {235, 64}, + {64, 240}, {240, 235}, {48, 49}, {49, 64}, {64, 48}, + {42, 41}, {41, 74}, {74, 42}, {214, 212}, {212, 207}, + {207, 214}, {183, 42}, {42, 184}, {184, 183}, {210, 169}, + {169, 211}, {211, 210}, {140, 170}, {170, 176}, {176, 140}, + {104, 105}, {105, 69}, {69, 104}, {193, 122}, {122, 168}, + {168, 193}, {50, 123}, {123, 187}, {187, 50}, {89, 96}, + {96, 90}, {90, 89}, {66, 65}, {65, 107}, {107, 66}, + {179, 89}, {89, 180}, {180, 179}, {119, 101}, {101, 120}, + {120, 119}, {68, 63}, {63, 104}, {104, 68}, {234, 93}, + {93, 227}, {227, 234}, {16, 15}, {15, 85}, {85, 16}, + {209, 129}, {129, 49}, {49, 209}, {15, 14}, {14, 86}, + {86, 15}, {107, 55}, {55, 9}, {9, 107}, {120, 100}, + {100, 121}, {121, 120}, {153, 145}, {145, 22}, {22, 153}, + {178, 88}, {88, 179}, {179, 178}, {197, 6}, {6, 196}, + {196, 197}, {89, 88}, {88, 96}, {96, 89}, {135, 138}, + {138, 136}, {136, 135}, {138, 215}, {215, 172}, {172, 138}, + {218, 115}, {115, 219}, {219, 218}, {41, 42}, {42, 81}, + {81, 41}, {5, 195}, {195, 51}, {51, 5}, {57, 43}, + {43, 61}, {61, 57}, {208, 171}, {171, 199}, {199, 208}, + {41, 81}, {81, 38}, {38, 41}, {224, 53}, {53, 225}, + {225, 224}, {24, 144}, {144, 110}, {110, 24}, {105, 52}, + {52, 66}, {66, 105}, {118, 229}, {229, 117}, {117, 118}, + {227, 34}, {34, 234}, {234, 227}, {66, 107}, {107, 69}, + {69, 66}, {10, 109}, {109, 151}, {151, 10}, {219, 48}, + {48, 235}, {235, 219}, {183, 62}, {62, 191}, {191, 183}, + {142, 129}, {129, 126}, {126, 142}, {116, 111}, {111, 143}, + {143, 116}, {118, 117}, {117, 50}, {50, 118}, {223, 222}, + {222, 52}, {52, 223}, {94, 19}, {19, 141}, {141, 94}, + {222, 221}, {221, 65}, {65, 222}, {196, 3}, {3, 197}, + {197, 196}, {45, 220}, {220, 44}, {44, 45}, {156, 70}, + {70, 139}, {139, 156}, {188, 122}, {122, 245}, {245, 188}, + {139, 71}, {71, 162}, {162, 139}, {149, 170}, {170, 150}, + {150, 149}, {122, 188}, {188, 196}, {196, 122}, {206, 216}, + {216, 92}, {92, 206}, {164, 2}, {2, 167}, {167, 164}, + {242, 141}, {141, 241}, {241, 242}, {0, 164}, {164, 37}, + {37, 0}, {11, 72}, {72, 12}, {12, 11}, {12, 38}, + {38, 13}, {13, 12}, {70, 63}, {63, 71}, {71, 70}, + {31, 226}, {226, 111}, {111, 31}, {36, 101}, {101, 205}, + {205, 36}, {203, 206}, {206, 165}, {165, 203}, {126, 209}, + {209, 217}, {217, 126}, {98, 165}, {165, 97}, {97, 98}, + {237, 220}, {220, 218}, {218, 237}, {237, 239}, {239, 241}, + {241, 237}, {210, 214}, {214, 169}, {169, 210}, {140, 171}, + {171, 32}, {32, 140}, {241, 125}, {125, 237}, {237, 241}, + {179, 86}, {86, 178}, {178, 179}, {180, 85}, {85, 179}, + {179, 180}, {181, 84}, {84, 180}, {180, 181}, {182, 83}, + {83, 181}, {181, 182}, {194, 201}, {201, 182}, {182, 194}, + {177, 137}, {137, 132}, {132, 177}, {184, 76}, {76, 183}, + {183, 184}, {185, 61}, {61, 184}, {184, 185}, {186, 57}, + {57, 185}, {185, 186}, {216, 212}, {212, 186}, {186, 216}, + {192, 214}, {214, 187}, {187, 192}, {139, 34}, {34, 156}, + {156, 139}, {218, 79}, {79, 237}, {237, 218}, {147, 123}, + {123, 177}, {177, 147}, {45, 44}, {44, 4}, {4, 45}, + {208, 201}, {201, 32}, {32, 208}, {98, 64}, {64, 129}, + {129, 98}, {192, 213}, {213, 138}, {138, 192}, {235, 59}, + {59, 219}, {219, 235}, {141, 242}, {242, 97}, {97, 141}, + {97, 2}, {2, 141}, {141, 97}, {240, 75}, {75, 235}, + {235, 240}, {229, 24}, {24, 228}, {228, 229}, {31, 25}, + {25, 226}, {226, 31}, {230, 23}, {23, 229}, {229, 230}, + {231, 22}, {22, 230}, {230, 231}, {232, 26}, {26, 231}, + {231, 232}, {233, 112}, {112, 232}, {232, 233}, {244, 189}, + {189, 243}, {243, 244}, {189, 221}, {221, 190}, {190, 189}, + {222, 28}, {28, 221}, {221, 222}, {223, 27}, {27, 222}, + {222, 223}, {224, 29}, {29, 223}, {223, 224}, {225, 30}, + {30, 224}, {224, 225}, {113, 247}, {247, 225}, {225, 113}, + {99, 60}, {60, 240}, {240, 99}, {213, 147}, {147, 215}, + {215, 213}, {60, 20}, {20, 166}, {166, 60}, {192, 187}, + {187, 213}, {213, 192}, {243, 112}, {112, 244}, {244, 243}, + {244, 233}, {233, 245}, {245, 244}, {245, 128}, {128, 188}, + {188, 245}, {188, 114}, {114, 174}, {174, 188}, {134, 131}, + {131, 220}, {220, 134}, {174, 217}, {217, 236}, {236, 174}, + {236, 198}, {198, 134}, {134, 236}, {215, 177}, {177, 58}, + {58, 215}, {156, 143}, {143, 124}, {124, 156}, {25, 110}, + {110, 7}, {7, 25}, {31, 228}, {228, 25}, {25, 31}, + {264, 356}, {356, 368}, {368, 264}, {0, 11}, {11, 267}, + {267, 0}, {451, 452}, {452, 349}, {349, 451}, {267, 302}, + {302, 269}, {269, 267}, {350, 357}, {357, 277}, {277, 350}, + {350, 452}, {452, 357}, {357, 350}, {299, 333}, {333, 297}, + {297, 299}, {396, 175}, {175, 377}, {377, 396}, {280, 347}, + {347, 330}, {330, 280}, {269, 303}, {303, 270}, {270, 269}, + {151, 9}, {9, 337}, {337, 151}, {344, 278}, {278, 360}, + {360, 344}, {424, 418}, {418, 431}, {431, 424}, {270, 304}, + {304, 409}, {409, 270}, {272, 310}, {310, 407}, {407, 272}, + {322, 270}, {270, 410}, {410, 322}, {449, 450}, {450, 347}, + {347, 449}, {432, 422}, {422, 434}, {434, 432}, {18, 313}, + {313, 17}, {17, 18}, {291, 306}, {306, 375}, {375, 291}, + {259, 387}, {387, 260}, {260, 259}, {424, 335}, {335, 418}, + {418, 424}, {434, 364}, {364, 416}, {416, 434}, {391, 423}, + {423, 327}, {327, 391}, {301, 251}, {251, 298}, {298, 301}, + {275, 281}, {281, 4}, {4, 275}, {254, 373}, {373, 253}, + {253, 254}, {375, 307}, {307, 321}, {321, 375}, {280, 425}, + {425, 411}, {411, 280}, {200, 421}, {421, 18}, {18, 200}, + {335, 321}, {321, 406}, {406, 335}, {321, 320}, {320, 405}, + {405, 321}, {314, 315}, {315, 17}, {17, 314}, {423, 426}, + {426, 266}, {266, 423}, {396, 377}, {377, 369}, {369, 396}, + {270, 322}, {322, 269}, {269, 270}, {413, 417}, {417, 464}, + {464, 413}, {385, 386}, {386, 258}, {258, 385}, {248, 456}, + {456, 419}, {419, 248}, {298, 284}, {284, 333}, {333, 298}, + {168, 417}, {417, 8}, {8, 168}, {448, 346}, {346, 261}, + {261, 448}, {417, 413}, {413, 285}, {285, 417}, {326, 327}, + {327, 328}, {328, 326}, {277, 355}, {355, 329}, {329, 277}, + {309, 392}, {392, 438}, {438, 309}, {381, 382}, {382, 256}, + {256, 381}, {279, 429}, {429, 360}, {360, 279}, {365, 364}, + {364, 379}, {379, 365}, {355, 277}, {277, 437}, {437, 355}, + {282, 443}, {443, 283}, {283, 282}, {281, 275}, {275, 363}, + {363, 281}, {395, 431}, {431, 369}, {369, 395}, {299, 297}, + {297, 337}, {337, 299}, {335, 273}, {273, 321}, {321, 335}, + {348, 450}, {450, 349}, {349, 348}, {359, 446}, {446, 467}, + {467, 359}, {283, 293}, {293, 282}, {282, 283}, {250, 458}, + {458, 462}, {462, 250}, {300, 276}, {276, 383}, {383, 300}, + {292, 308}, {308, 325}, {325, 292}, {283, 276}, {276, 293}, + {293, 283}, {264, 372}, {372, 447}, {447, 264}, {346, 352}, + {352, 340}, {340, 346}, {354, 274}, {274, 19}, {19, 354}, + {363, 456}, {456, 281}, {281, 363}, {426, 436}, {436, 425}, + {425, 426}, {380, 381}, {381, 252}, {252, 380}, {267, 269}, + {269, 393}, {393, 267}, {421, 200}, {200, 428}, {428, 421}, + {371, 266}, {266, 329}, {329, 371}, {432, 287}, {287, 422}, + {422, 432}, {290, 250}, {250, 328}, {328, 290}, {385, 258}, + {258, 384}, {384, 385}, {446, 265}, {265, 342}, {342, 446}, + {386, 387}, {387, 257}, {257, 386}, {422, 424}, {424, 430}, + {430, 422}, {445, 342}, {342, 276}, {276, 445}, {422, 273}, + {273, 424}, {424, 422}, {306, 292}, {292, 307}, {307, 306}, + {352, 366}, {366, 345}, {345, 352}, {268, 271}, {271, 302}, + {302, 268}, {358, 423}, {423, 371}, {371, 358}, {327, 294}, + {294, 460}, {460, 327}, {331, 279}, {279, 294}, {294, 331}, + {303, 271}, {271, 304}, {304, 303}, {436, 432}, {432, 427}, + {427, 436}, {304, 272}, {272, 408}, {408, 304}, {395, 394}, + {394, 431}, {431, 395}, {378, 395}, {395, 400}, {400, 378}, + {296, 334}, {334, 299}, {299, 296}, {6, 351}, {351, 168}, + {168, 6}, {376, 352}, {352, 411}, {411, 376}, {307, 325}, + {325, 320}, {320, 307}, {285, 295}, {295, 336}, {336, 285}, + {320, 319}, {319, 404}, {404, 320}, {329, 330}, {330, 349}, + {349, 329}, {334, 293}, {293, 333}, {333, 334}, {366, 323}, + {323, 447}, {447, 366}, {316, 15}, {15, 315}, {315, 316}, + {331, 358}, {358, 279}, {279, 331}, {317, 14}, {14, 316}, + {316, 317}, {8, 285}, {285, 9}, {9, 8}, {277, 329}, + {329, 350}, {350, 277}, {253, 374}, {374, 252}, {252, 253}, + {319, 318}, {318, 403}, {403, 319}, {351, 6}, {6, 419}, + {419, 351}, {324, 318}, {318, 325}, {325, 324}, {397, 367}, + {367, 365}, {365, 397}, {288, 435}, {435, 397}, {397, 288}, + {278, 344}, {344, 439}, {439, 278}, {310, 272}, {272, 311}, + {311, 310}, {248, 195}, {195, 281}, {281, 248}, {375, 273}, + {273, 291}, {291, 375}, {175, 396}, {396, 199}, {199, 175}, + {312, 311}, {311, 268}, {268, 312}, {276, 283}, {283, 445}, + {445, 276}, {390, 373}, {373, 339}, {339, 390}, {295, 282}, + {282, 296}, {296, 295}, {448, 449}, {449, 346}, {346, 448}, + {356, 264}, {264, 454}, {454, 356}, {337, 336}, {336, 299}, + {299, 337}, {337, 338}, {338, 151}, {151, 337}, {294, 278}, + {278, 455}, {455, 294}, {308, 292}, {292, 415}, {415, 308}, + {429, 358}, {358, 355}, {355, 429}, {265, 340}, {340, 372}, + {372, 265}, {352, 346}, {346, 280}, {280, 352}, {295, 442}, + {442, 282}, {282, 295}, {354, 19}, {19, 370}, {370, 354}, + {285, 441}, {441, 295}, {295, 285}, {195, 248}, {248, 197}, + {197, 195}, {457, 440}, {440, 274}, {274, 457}, {301, 300}, + {300, 368}, {368, 301}, {417, 351}, {351, 465}, {465, 417}, + {251, 301}, {301, 389}, {389, 251}, {394, 395}, {395, 379}, + {379, 394}, {399, 412}, {412, 419}, {419, 399}, {410, 436}, + {436, 322}, {322, 410}, {326, 2}, {2, 393}, {393, 326}, + {354, 370}, {370, 461}, {461, 354}, {393, 164}, {164, 267}, + {267, 393}, {268, 302}, {302, 12}, {12, 268}, {312, 268}, + {268, 13}, {13, 312}, {298, 293}, {293, 301}, {301, 298}, + {265, 446}, {446, 340}, {340, 265}, {280, 330}, {330, 425}, + {425, 280}, {322, 426}, {426, 391}, {391, 322}, {420, 429}, + {429, 437}, {437, 420}, {393, 391}, {391, 326}, {326, 393}, + {344, 440}, {440, 438}, {438, 344}, {458, 459}, {459, 461}, + {461, 458}, {364, 434}, {434, 394}, {394, 364}, {428, 396}, + {396, 262}, {262, 428}, {274, 354}, {354, 457}, {457, 274}, + {317, 316}, {316, 402}, {402, 317}, {316, 315}, {315, 403}, + {403, 316}, {315, 314}, {314, 404}, {404, 315}, {314, 313}, + {313, 405}, {405, 314}, {313, 421}, {421, 406}, {406, 313}, + {323, 366}, {366, 361}, {361, 323}, {292, 306}, {306, 407}, + {407, 292}, {306, 291}, {291, 408}, {408, 306}, {291, 287}, + {287, 409}, {409, 291}, {287, 432}, {432, 410}, {410, 287}, + {427, 434}, {434, 411}, {411, 427}, {372, 264}, {264, 383}, + {383, 372}, {459, 309}, {309, 457}, {457, 459}, {366, 352}, + {352, 401}, {401, 366}, {1, 274}, {274, 4}, {4, 1}, + {418, 421}, {421, 262}, {262, 418}, {331, 294}, {294, 358}, + {358, 331}, {435, 433}, {433, 367}, {367, 435}, {392, 289}, + {289, 439}, {439, 392}, {328, 462}, {462, 326}, {326, 328}, + {94, 2}, {2, 370}, {370, 94}, {289, 305}, {305, 455}, + {455, 289}, {339, 254}, {254, 448}, {448, 339}, {359, 255}, + {255, 446}, {446, 359}, {254, 253}, {253, 449}, {449, 254}, + {253, 252}, {252, 450}, {450, 253}, {252, 256}, {256, 451}, + {451, 252}, {256, 341}, {341, 452}, {452, 256}, {414, 413}, + {413, 463}, {463, 414}, {286, 441}, {441, 414}, {414, 286}, + {286, 258}, {258, 441}, {441, 286}, {258, 257}, {257, 442}, + {442, 258}, {257, 259}, {259, 443}, {443, 257}, {259, 260}, + {260, 444}, {444, 259}, {260, 467}, {467, 445}, {445, 260}, + {309, 459}, {459, 250}, {250, 309}, {305, 289}, {289, 290}, + {290, 305}, {305, 290}, {290, 460}, {460, 305}, {401, 376}, + {376, 435}, {435, 401}, {309, 250}, {250, 392}, {392, 309}, + {376, 411}, {411, 433}, {433, 376}, {453, 341}, {341, 464}, + {464, 453}, {357, 453}, {453, 465}, {465, 357}, {343, 357}, + {357, 412}, {412, 343}, {437, 343}, {343, 399}, {399, 437}, + {344, 360}, {360, 440}, {440, 344}, {420, 437}, {437, 456}, + {456, 420}, {360, 420}, {420, 363}, {363, 360}, {361, 401}, + {401, 288}, {288, 361}, {265, 372}, {372, 353}, {353, 265}, + {390, 339}, {339, 249}, {249, 390}, {339, 448}, {448, 255}, + {255, 339}}}; +}; + +} // namespace face_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_FACE_LANDMARKER_FACE_LANDMARKS_CONNECTIONS_H_ diff --git a/mediapipe/tasks/cc/vision/face_stylizer/calculators/BUILD b/mediapipe/tasks/cc/vision/face_stylizer/calculators/BUILD index 74b17401..46f8944a 100644 --- a/mediapipe/tasks/cc/vision/face_stylizer/calculators/BUILD +++ b/mediapipe/tasks/cc/vision/face_stylizer/calculators/BUILD @@ -65,6 +65,7 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", "//mediapipe/gpu:gpu_origin_cc_proto", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", ] + select({ diff --git a/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.cc b/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.cc index d9825b15..651b7efc 100644 --- a/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.cc +++ b/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.cc @@ -16,6 +16,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" #include "mediapipe/calculators/tensor/image_to_tensor_utils.h" @@ -111,6 +112,7 @@ class TensorsToImageCalculator : public Node { private: TensorsToImageCalculatorOptions options_; absl::Status CpuProcess(CalculatorContext* cc); + int tensor_position_; #if !MEDIAPIPE_DISABLE_GPU #if MEDIAPIPE_METAL_ENABLED @@ -161,11 +163,12 @@ absl::Status TensorsToImageCalculator::Open(CalculatorContext* cc) { #endif // MEDIAPIPE_METAL_ENABLED #endif // !MEDIAPIPE_DISABLE_GPU } else { - CHECK(options_.has_input_tensor_float_range() ^ - options_.has_input_tensor_uint_range()) + ABSL_CHECK(options_.has_input_tensor_float_range() ^ + options_.has_input_tensor_uint_range()) << "Must specify either `input_tensor_float_range` or " "`input_tensor_uint_range` in the calculator options"; } + tensor_position_ = options_.tensor_position(); return absl::OkStatus(); } @@ -202,17 +205,23 @@ absl::Status TensorsToImageCalculator::CpuProcess(CalculatorContext* cc) { return absl::OkStatus(); } const auto& input_tensors = kInputTensors(cc).Get(); - RET_CHECK_EQ(input_tensors.size(), 1) - << "Expect 1 input tensor, but have " << input_tensors.size(); + RET_CHECK_GT(input_tensors.size(), tensor_position_) + << "Expect input tensor at position " << tensor_position_ + << ", but have tensors of size " << input_tensors.size(); - const auto& input_tensor = input_tensors[0]; + const auto& input_tensor = input_tensors[tensor_position_]; const int tensor_in_height = input_tensor.shape().dims[1]; const int tensor_in_width = input_tensor.shape().dims[2]; const int tensor_in_channels = input_tensor.shape().dims[3]; - RET_CHECK_EQ(tensor_in_channels, 3); + RET_CHECK(tensor_in_channels == 3 || tensor_in_channels == 1); - auto output_frame = std::make_shared( - mediapipe::ImageFormat::SRGB, tensor_in_width, tensor_in_height); + auto format = mediapipe::ImageFormat::SRGB; + if (tensor_in_channels == 1) { + format = mediapipe::ImageFormat::GRAY8; + } + + auto output_frame = + std::make_shared(format, tensor_in_width, tensor_in_height); cv::Mat output_matview = mediapipe::formats::MatView(output_frame.get()); constexpr float kOutputImageRangeMin = 0.0f; @@ -227,8 +236,9 @@ absl::Status TensorsToImageCalculator::CpuProcess(CalculatorContext* cc) { GetValueRangeTransformation( input_range.min(), input_range.max(), kOutputImageRangeMin, kOutputImageRangeMax)); - tensor_matview.convertTo(output_matview, CV_8UC3, transform.scale, - transform.offset); + tensor_matview.convertTo(output_matview, + CV_MAKETYPE(CV_8U, tensor_in_channels), + transform.scale, transform.offset); } else if (input_tensor.element_type() == Tensor::ElementType::kUInt8) { cv::Mat tensor_matview( cv::Size(tensor_in_width, tensor_in_height), @@ -239,8 +249,9 @@ absl::Status TensorsToImageCalculator::CpuProcess(CalculatorContext* cc) { GetValueRangeTransformation( input_range.min(), input_range.max(), kOutputImageRangeMin, kOutputImageRangeMax)); - tensor_matview.convertTo(output_matview, CV_8UC3, transform.scale, - transform.offset); + tensor_matview.convertTo(output_matview, + CV_MAKETYPE(CV_8U, tensor_in_channels), + transform.scale, transform.offset); } else { return absl::InvalidArgumentError( absl::Substitute("Type of tensor must be kFloat32 or kUInt8, got: $0", @@ -264,10 +275,14 @@ absl::Status TensorsToImageCalculator::MetalProcess(CalculatorContext* cc) { return absl::OkStatus(); } const auto& input_tensors = kInputTensors(cc).Get(); - RET_CHECK_EQ(input_tensors.size(), 1) - << "Expect 1 input tensor, but have " << input_tensors.size(); - const int tensor_width = input_tensors[0].shape().dims[2]; - const int tensor_height = input_tensors[0].shape().dims[1]; + RET_CHECK_GT(input_tensors.size(), tensor_position_) + << "Expect input tensor at position " << tensor_position_ + << ", but have tensors of size " << input_tensors.size(); + const int tensor_width = input_tensors[tensor_position_].shape().dims[2]; + const int tensor_height = input_tensors[tensor_position_].shape().dims[1]; + const int tensor_channels = input_tensors[tensor_position_].shape().dims[3]; + // TODO: Add 1 channel support. + RET_CHECK(tensor_channels == 3); // TODO: Fix unused variable [[maybe_unused]] id device = gpu_helper_.mtlDevice; @@ -277,8 +292,8 @@ absl::Status TensorsToImageCalculator::MetalProcess(CalculatorContext* cc) { [command_buffer computeCommandEncoder]; [compute_encoder setComputePipelineState:to_buffer_program_]; - auto input_view = - mediapipe::MtlBufferView::GetReadView(input_tensors[0], command_buffer); + auto input_view = mediapipe::MtlBufferView::GetReadView( + input_tensors[tensor_position_], command_buffer); [compute_encoder setBuffer:input_view.buffer() offset:0 atIndex:0]; mediapipe::GpuBuffer output = @@ -355,7 +370,7 @@ absl::Status TensorsToImageCalculator::GlSetup(CalculatorContext* cc) { absl::StrCat(tflite::gpu::gl::GetShaderHeader(workgroup_size_), R"( precision highp float; layout(rgba8, binding = 0) writeonly uniform highp image2D output_texture; - uniform ivec2 out_size; + uniform ivec3 out_size; )"); const std::string shader_body = R"( @@ -366,10 +381,11 @@ absl::Status TensorsToImageCalculator::GlSetup(CalculatorContext* cc) { void main() { int out_width = out_size.x; int out_height = out_size.y; + int out_channels = out_size.z; ivec2 gid = ivec2(gl_GlobalInvocationID.xy); if (gid.x >= out_width || gid.y >= out_height) { return; } - int linear_index = 3 * (gid.y * out_width + gid.x); + int linear_index = out_channels * (gid.y * out_width + gid.x); #ifdef FLIP_Y_COORD int y_coord = out_height - gid.y - 1; @@ -377,8 +393,14 @@ absl::Status TensorsToImageCalculator::GlSetup(CalculatorContext* cc) { int y_coord = gid.y; #endif // defined(FLIP_Y_COORD) + vec4 out_value; ivec2 out_coordinate = ivec2(gid.x, y_coord); - vec4 out_value = vec4(input_data.elements[linear_index], input_data.elements[linear_index + 1], input_data.elements[linear_index + 2], 1.0); + if (out_channels == 3) { + out_value = vec4(input_data.elements[linear_index], input_data.elements[linear_index + 1], input_data.elements[linear_index + 2], 1.0); + } else { + float in_value = input_data.elements[linear_index]; + out_value = vec4(in_value, in_value, in_value, 1.0); + } imageStore(output_texture, out_coordinate, out_value); })"; @@ -438,10 +460,15 @@ absl::Status TensorsToImageCalculator::GlProcess(CalculatorContext* cc) { return absl::OkStatus(); } const auto& input_tensors = kInputTensors(cc).Get(); - RET_CHECK_EQ(input_tensors.size(), 1) - << "Expect 1 input tensor, but have " << input_tensors.size(); - const int tensor_width = input_tensors[0].shape().dims[2]; - const int tensor_height = input_tensors[0].shape().dims[1]; + RET_CHECK_GT(input_tensors.size(), tensor_position_) + << "Expect input tensor at position " << tensor_position_ + << ", but have tensors of size " << input_tensors.size(); + + const auto& input_tensor = input_tensors[tensor_position_]; + const int tensor_width = input_tensor.shape().dims[2]; + const int tensor_height = input_tensor.shape().dims[1]; + const int tensor_in_channels = input_tensor.shape().dims[3]; + RET_CHECK(tensor_in_channels == 3 || tensor_in_channels == 1); #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 @@ -454,7 +481,7 @@ absl::Status TensorsToImageCalculator::GlProcess(CalculatorContext* cc) { glBindImageTexture(output_index, out_texture->id(), 0, GL_FALSE, 0, GL_WRITE_ONLY, GL_RGBA8); - auto read_view = input_tensors[0].GetOpenGlBufferReadView(); + auto read_view = input_tensor.GetOpenGlBufferReadView(); glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, read_view.name()); const tflite::gpu::uint3 workload = {tensor_width, tensor_height, 1}; @@ -462,8 +489,8 @@ absl::Status TensorsToImageCalculator::GlProcess(CalculatorContext* cc) { tflite::gpu::DivideRoundUp(workload, workgroup_size_); glUseProgram(gl_compute_program_->id()); - glUniform2i(glGetUniformLocation(gl_compute_program_->id(), "out_size"), - tensor_width, tensor_height); + glUniform3i(glGetUniformLocation(gl_compute_program_->id(), "out_size"), + tensor_width, tensor_height, tensor_in_channels); MP_RETURN_IF_ERROR(gl_compute_program_->Dispatch(workgroups)); @@ -481,8 +508,8 @@ absl::Status TensorsToImageCalculator::GlProcess(CalculatorContext* cc) { #else - if (!input_tensors[0].ready_as_opengl_texture_2d()) { - (void)input_tensors[0].GetCpuReadView(); + if (!input_tensor.ready_as_opengl_texture_2d()) { + (void)input_tensor.GetCpuReadView(); } auto output_texture = @@ -490,7 +517,7 @@ absl::Status TensorsToImageCalculator::GlProcess(CalculatorContext* cc) { gl_helper_.BindFramebuffer(output_texture); // GL_TEXTURE0 glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, - input_tensors[0].GetOpenGlTexture2dReadView().name()); + input_tensor.GetOpenGlTexture2dReadView().name()); MP_RETURN_IF_ERROR(gl_renderer_->GlRender( tensor_width, tensor_height, output_texture.width(), diff --git a/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.proto b/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.proto index 6bca8626..b0ecb8b5 100644 --- a/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.proto +++ b/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.proto @@ -48,4 +48,8 @@ message TensorsToImageCalculatorOptions { FloatRange input_tensor_float_range = 2; UIntRange input_tensor_uint_range = 3; } + + // Determines which output tensor to slice when there are multiple output + // tensors available (e.g. network has multiple heads) + optional int32 tensor_position = 4 [default = 0]; } diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD b/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD index fe925db5..11e484e9 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/BUILD @@ -124,6 +124,7 @@ cc_library( "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/hand_landmarker:hand_landmarks_detector_graph", "//mediapipe/tasks/metadata:metadata_schema_cc", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", ], @@ -161,6 +162,7 @@ cc_library( "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_cc_proto", "//mediapipe/tasks/metadata:metadata_schema_cc", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", ], diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/calculators/handedness_to_matrix_calculator.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/calculators/handedness_to_matrix_calculator.cc index 0d0419d0..c806d589 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/calculators/handedness_to_matrix_calculator.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/calculators/handedness_to_matrix_calculator.cc @@ -34,15 +34,15 @@ namespace api2 { namespace { -using ::mediapipe::tasks::vision::gesture_recognizer::GetLeftHandScore; +using ::mediapipe::tasks::vision::gesture_recognizer::GetRightHandScore; constexpr char kHandednessTag[] = "HANDEDNESS"; constexpr char kHandednessMatrixTag[] = "HANDEDNESS_MATRIX"; absl::StatusOr> HandednessToMatrix( const mediapipe::ClassificationList& classification_list) { - // Feature value is the probability that the hand is a left hand. - ASSIGN_OR_RETURN(float score, GetLeftHandScore(classification_list)); + // Feature value is the probability that the hand is a right hand. + ASSIGN_OR_RETURN(float score, GetRightHandScore(classification_list)); auto matrix = Matrix(1, 1); matrix(0, 0) = score; auto result = std::make_unique(); diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/calculators/handedness_to_matrix_calculator_test.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/calculators/handedness_to_matrix_calculator_test.cc index 70012aa5..f0858e10 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/calculators/handedness_to_matrix_calculator_test.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/calculators/handedness_to_matrix_calculator_test.cc @@ -38,10 +38,10 @@ mediapipe::ClassificationList ClassificationForHandedness(float handedness) { mediapipe::ClassificationList result; auto* h = result.add_classification(); if (handedness < 0.5f) { - h->set_label("Right"); + h->set_label("Left"); h->set_score(1.0f - handedness); } else { - h->set_label("Left"); + h->set_label("Right"); h->set_score(handedness); } return result; @@ -84,8 +84,8 @@ TEST_P(HandednessToMatrixCalculatorTest, OutputsCorrectResult) { INSTANTIATE_TEST_CASE_P( HandednessToMatrixCalculatorTests, HandednessToMatrixCalculatorTest, testing::ValuesIn( - {{/* test_name= */ "TestWithRightHand", /* handedness= */ 0.01f}, - {/* test_name= */ "TestWithLeftHand", /* handedness= */ 0.99f}}), + {{/* test_name= */ "TestWithLeftHand", /* handedness= */ 0.01f}, + {/* test_name= */ "TestWithRightHand", /* handedness= */ 0.99f}}), [](const testing::TestParamInfo< HandednessToMatrixCalculatorTest::ParamType>& info) { return info.param.test_name; diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc index 0f4c88f5..9550112b 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "mediapipe/framework/api2/builder.h" @@ -125,8 +126,8 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources, hand_gesture_recognizer_graph_options->mutable_base_options() ->mutable_acceleration() ->mutable_xnnpack(); - LOG(WARNING) << "Hand Gesture Recognizer contains CPU only ops. Sets " - << "HandGestureRecognizerGraph acceleration to Xnnpack."; + ABSL_LOG(WARNING) << "Hand Gesture Recognizer contains CPU only ops. Sets " + << "HandGestureRecognizerGraph acceleration to Xnnpack."; } hand_gesture_recognizer_graph_options->mutable_base_options() ->set_use_stream_mode(options->base_options().use_stream_mode()); diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc index 097318be..fbe05b07 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/hand_gesture_recognizer_graph.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "mediapipe/calculators/tensor/tensors_to_classification_calculator.pb.h" @@ -246,7 +247,7 @@ class SingleHandGestureRecognizerGraph : public core::ModelTaskGraph { options->base_options(), custom_gesture_classifier_graph_options->mutable_base_options()); } else { - LOG(INFO) << "Custom gesture classifier is not defined."; + ABSL_LOG(INFO) << "Custom gesture classifier is not defined."; } return absl::OkStatus(); } diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util.cc index 52c67967..aeb01602 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util.cc @@ -37,7 +37,7 @@ bool IsRightHand(const Classification& c) { return absl::EqualsIgnoreCase(c.label(), "Right"); } -absl::StatusOr GetLeftHandScore( +absl::StatusOr GetRightHandScore( const ClassificationList& classification_list) { auto classifications = classification_list.classification(); auto iter_max = @@ -50,9 +50,9 @@ absl::StatusOr GetLeftHandScore( RET_CHECK_GE(h.score(), 0.5f); RET_CHECK_LE(h.score(), 1.0f); if (IsLeftHand(h)) { - return h.score(); - } else if (IsRightHand(h)) { return 1.0f - h.score(); + } else if (IsRightHand(h)) { + return h.score(); } else { // Unrecognized handedness label. RET_CHECK_FAIL() << "Unrecognized handedness: " << h.label(); diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util.h b/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util.h index 9abb6e59..077fbf1b 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util.h +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util.h @@ -28,7 +28,7 @@ bool IsLeftHand(const mediapipe::Classification& c); bool IsRightHand(const mediapipe::Classification& c); -absl::StatusOr GetLeftHandScore( +absl::StatusOr GetRightHandScore( const mediapipe::ClassificationList& classification_list); } // namespace gesture_recognizer diff --git a/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util_test.cc b/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util_test.cc index 01e21445..ae1a5c6e 100644 --- a/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util_test.cc +++ b/mediapipe/tasks/cc/vision/gesture_recognizer/handedness_util_test.cc @@ -26,49 +26,49 @@ namespace vision { namespace gesture_recognizer { namespace { -TEST(GetLeftHandScore, SingleLeftHandClassification) { - ClassificationList classifications; - auto& c = *classifications.add_classification(); - c.set_label("Left"); - c.set_score(0.6f); - - MP_ASSERT_OK_AND_ASSIGN(float score, GetLeftHandScore(classifications)); - EXPECT_FLOAT_EQ(score, 0.6f); -} - -TEST(GetLeftHandScore, SingleRightHandClassification) { +TEST(GetRightHandScore, SingleRightHandClassification) { ClassificationList classifications; auto& c = *classifications.add_classification(); c.set_label("Right"); + c.set_score(0.6f); + + MP_ASSERT_OK_AND_ASSIGN(float score, GetRightHandScore(classifications)); + EXPECT_FLOAT_EQ(score, 0.6f); +} + +TEST(GetRightHandScore, SingleLeftHandClassification) { + ClassificationList classifications; + auto& c = *classifications.add_classification(); + c.set_label("Left"); c.set_score(0.9f); - MP_ASSERT_OK_AND_ASSIGN(float score, GetLeftHandScore(classifications)); + MP_ASSERT_OK_AND_ASSIGN(float score, GetRightHandScore(classifications)); EXPECT_FLOAT_EQ(score, 0.1f); } -TEST(GetLeftHandScore, LeftAndRightHandClassification) { +TEST(GetRightHandScore, LeftAndRightHandClassification) { ClassificationList classifications; auto& right = *classifications.add_classification(); - right.set_label("Right"); + right.set_label("Left"); right.set_score(0.9f); auto& left = *classifications.add_classification(); - left.set_label("Left"); + left.set_label("Right"); left.set_score(0.1f); - MP_ASSERT_OK_AND_ASSIGN(float score, GetLeftHandScore(classifications)); + MP_ASSERT_OK_AND_ASSIGN(float score, GetRightHandScore(classifications)); EXPECT_FLOAT_EQ(score, 0.1f); } -TEST(GetLeftHandScore, LeftAndRightLowerCaseHandClassification) { +TEST(GetRightHandScore, LeftAndRightLowerCaseHandClassification) { ClassificationList classifications; auto& right = *classifications.add_classification(); - right.set_label("right"); + right.set_label("Left"); right.set_score(0.9f); auto& left = *classifications.add_classification(); - left.set_label("left"); + left.set_label("Right"); left.set_score(0.1f); - MP_ASSERT_OK_AND_ASSIGN(float score, GetLeftHandScore(classifications)); + MP_ASSERT_OK_AND_ASSIGN(float score, GetRightHandScore(classifications)); EXPECT_FLOAT_EQ(score, 0.1f); } diff --git a/mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph_test.cc b/mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph_test.cc index 2e47f693..eadc26ad 100644 --- a/mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph_test.cc +++ b/mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph_test.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -76,8 +77,8 @@ using ::testing::proto::Partially; constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; constexpr char kPalmDetectionModel[] = "palm_detection_full.tflite"; -constexpr char kTestRightHandsImage[] = "right_hands.jpg"; -constexpr char kTestRightHandsRotatedImage[] = "right_hands_rotated.jpg"; +constexpr char kTestLeftHandsImage[] = "left_hands.jpg"; +constexpr char kTestLeftHandsRotatedImage[] = "left_hands_rotated.jpg"; constexpr char kTestModelResourcesTag[] = "test_model_resources"; constexpr char kOneHandResultFile[] = "hand_detector_result_one_hand.pbtxt"; @@ -138,8 +139,8 @@ absl::StatusOr> CreateTaskRunner( HandDetectorResult GetExpectedHandDetectorResult(absl::string_view file_name) { HandDetectorResult result; - CHECK_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), - &result, Defaults())) + ABSL_CHECK_OK(GetTextProto( + file::JoinPath("./", kTestDataDirectory, file_name), &result, Defaults())) << "Expected hand detector result does not exist."; return result; } @@ -207,21 +208,21 @@ INSTANTIATE_TEST_SUITE_P( HandDetectionTest, HandDetectionTest, Values(TestParams{.test_name = "DetectOneHand", .hand_detection_model_name = kPalmDetectionModel, - .test_image_name = kTestRightHandsImage, + .test_image_name = kTestLeftHandsImage, .rotation = 0, .num_hands = 1, .expected_result = GetExpectedHandDetectorResult(kOneHandResultFile)}, TestParams{.test_name = "DetectTwoHands", .hand_detection_model_name = kPalmDetectionModel, - .test_image_name = kTestRightHandsImage, + .test_image_name = kTestLeftHandsImage, .rotation = 0, .num_hands = 2, .expected_result = GetExpectedHandDetectorResult(kTwoHandsResultFile)}, TestParams{.test_name = "DetectOneHandWithRotation", .hand_detection_model_name = kPalmDetectionModel, - .test_image_name = kTestRightHandsRotatedImage, + .test_image_name = kTestLeftHandsRotatedImage, .rotation = M_PI / 2.0f, .num_hands = 1, .expected_result = GetExpectedHandDetectorResult( diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD index 2eecb61b..1e24256d 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD @@ -153,6 +153,13 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "hand_landmarks_connections", + hdrs = ["hand_landmarks_connections.h"], +) + +# TODO: open source hand joints graph + cc_library( name = "hand_landmarker_result", srcs = ["hand_landmarker_result.cc"], diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/calculators/BUILD b/mediapipe/tasks/cc/vision/hand_landmarker/calculators/BUILD index a30cc555..15806b51 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/calculators/BUILD +++ b/mediapipe/tasks/cc/vision/hand_landmarker/calculators/BUILD @@ -42,6 +42,7 @@ cc_library( "//mediapipe/framework/port:rectangle", "//mediapipe/framework/port:status", "//mediapipe/util:rectangle_util", + "@com_google_absl//absl/log:absl_check", ], alwayslink = 1, ) diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/calculators/hand_association_calculator.cc b/mediapipe/tasks/cc/vision/hand_landmarker/calculators/hand_association_calculator.cc index 060e7a2d..5cbd72c3 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/calculators/hand_association_calculator.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/calculators/hand_association_calculator.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/collection_item_id.h" @@ -89,8 +90,8 @@ class HandAssociationCalculator : public CalculatorBase { cc->SetOffset(TimestampDiff(0)); options_ = cc->Options(); - CHECK_GT(options_.min_similarity_threshold(), 0.0); - CHECK_LE(options_.min_similarity_threshold(), 1.0); + ABSL_CHECK_GT(options_.min_similarity_threshold(), 0.0); + ABSL_CHECK_LE(options_.min_similarity_threshold(), 1.0); return absl::OkStatus(); } diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc index 673d0136..f08e2b86 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph_test.cc @@ -69,8 +69,8 @@ using ::testing::proto::Partially; constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; constexpr char kHandLandmarkerModelBundle[] = "hand_landmarker.task"; -constexpr char kLeftHandsImage[] = "left_hands.jpg"; -constexpr char kLeftHandsRotatedImage[] = "left_hands_rotated.jpg"; +constexpr char kRightHandsImage[] = "right_hands.jpg"; +constexpr char kRightHandsRotatedImage[] = "right_hands_rotated.jpg"; constexpr char kImageTag[] = "IMAGE"; constexpr char kImageName[] = "image_in"; @@ -86,15 +86,15 @@ constexpr char kHandednessTag[] = "HANDEDNESS"; constexpr char kHandednessName[] = "handedness"; // Expected hand landmarks positions, in text proto format. -constexpr char kExpectedLeftUpHandLandmarksFilename[] = - "expected_left_up_hand_landmarks.prototxt"; -constexpr char kExpectedLeftDownHandLandmarksFilename[] = - "expected_left_down_hand_landmarks.prototxt"; +constexpr char kExpectedRightUpHandLandmarksFilename[] = + "expected_right_up_hand_landmarks.prototxt"; +constexpr char kExpectedRightDownHandLandmarksFilename[] = + "expected_right_down_hand_landmarks.prototxt"; // Same but for the rotated image. -constexpr char kExpectedLeftUpHandRotatedLandmarksFilename[] = - "expected_left_up_hand_rotated_landmarks.prototxt"; -constexpr char kExpectedLeftDownHandRotatedLandmarksFilename[] = - "expected_left_down_hand_rotated_landmarks.prototxt"; +constexpr char kExpectedRightUpHandRotatedLandmarksFilename[] = + "expected_right_up_hand_rotated_landmarks.prototxt"; +constexpr char kExpectedRightDownHandRotatedLandmarksFilename[] = + "expected_right_down_hand_rotated_landmarks.prototxt"; constexpr float kFullModelFractionDiff = 0.03; // percentage constexpr float kAbsMargin = 0.03; @@ -141,8 +141,8 @@ class HandLandmarkerTest : public tflite::testing::Test {}; TEST_F(HandLandmarkerTest, Succeeds) { MP_ASSERT_OK_AND_ASSIGN( - Image image, - DecodeImageFromFile(JoinPath("./", kTestDataDirectory, kLeftHandsImage))); + Image image, DecodeImageFromFile( + JoinPath("./", kTestDataDirectory, kRightHandsImage))); NormalizedRect input_norm_rect; input_norm_rect.set_x_center(0.5); input_norm_rect.set_y_center(0.5); @@ -157,8 +157,8 @@ TEST_F(HandLandmarkerTest, Succeeds) { .Get>(); ASSERT_EQ(landmarks.size(), kMaxNumHands); std::vector expected_landmarks = { - GetExpectedLandmarkList(kExpectedLeftUpHandLandmarksFilename), - GetExpectedLandmarkList(kExpectedLeftDownHandLandmarksFilename)}; + GetExpectedLandmarkList(kExpectedRightUpHandLandmarksFilename), + GetExpectedLandmarkList(kExpectedRightDownHandLandmarksFilename)}; EXPECT_THAT(landmarks[0], Approximately(Partially(EqualsProto(expected_landmarks[0])), @@ -173,7 +173,7 @@ TEST_F(HandLandmarkerTest, Succeeds) { TEST_F(HandLandmarkerTest, SucceedsWithRotation) { MP_ASSERT_OK_AND_ASSIGN( Image image, DecodeImageFromFile(JoinPath("./", kTestDataDirectory, - kLeftHandsRotatedImage))); + kRightHandsRotatedImage))); NormalizedRect input_norm_rect; input_norm_rect.set_x_center(0.5); input_norm_rect.set_y_center(0.5); @@ -189,8 +189,8 @@ TEST_F(HandLandmarkerTest, SucceedsWithRotation) { .Get>(); ASSERT_EQ(landmarks.size(), kMaxNumHands); std::vector expected_landmarks = { - GetExpectedLandmarkList(kExpectedLeftUpHandRotatedLandmarksFilename), - GetExpectedLandmarkList(kExpectedLeftDownHandRotatedLandmarksFilename)}; + GetExpectedLandmarkList(kExpectedRightUpHandRotatedLandmarksFilename), + GetExpectedLandmarkList(kExpectedRightDownHandRotatedLandmarksFilename)}; EXPECT_THAT(landmarks[0], Approximately(Partially(EqualsProto(expected_landmarks[0])), diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_connections.h b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_connections.h new file mode 100644 index 00000000..51082029 --- /dev/null +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_connections.h @@ -0,0 +1,54 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKS_CONNECTIONS_H_ +#define MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKS_CONNECTIONS_H_ + +#include + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace hand_landmarker { + +static constexpr std::array, 6> kHandPalmConnections{ + {{0, 1}, {0, 5}, {9, 13}, {13, 17}, {5, 9}, {0, 17}}}; + +static constexpr std::array, 3> kHandThumbConnections{ + {{1, 2}, {2, 3}, {3, 4}}}; + +static constexpr std::array, 3> kHandIndexFingerConnections{ + {{5, 6}, {6, 7}, {7, 8}}}; + +static constexpr std::array, 3> kHandMiddleFingerConnections{ + {{9, 10}, {10, 11}, {11, 12}}}; + +static constexpr std::array, 3> kHandRingFingerConnections{ + {{13, 14}, {14, 15}, {15, 16}}}; + +static constexpr std::array, 3> kHandPinkyFingerConnections{ + {{17, 18}, {18, 19}, {19, 20}}}; + +static constexpr std::array, 21> kHandConnections{ + {{0, 1}, {0, 5}, {9, 13}, {13, 17}, {5, 9}, {0, 17}, {1, 2}, + {2, 3}, {3, 4}, {5, 6}, {6, 7}, {7, 8}, {9, 10}, {10, 11}, + {11, 12}, {13, 14}, {14, 15}, {15, 16}, {17, 18}, {18, 19}, {19, 20}}}; + +} // namespace hand_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKS_CONNECTIONS_H_ diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc index c27322b9..c3a4edec 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc @@ -142,8 +142,8 @@ void ConfigureTensorsToHandednessCalculator( LabelMapItem right_hand = LabelMapItem(); right_hand.set_name("Right"); right_hand.set_display_name("Right"); - (*options->mutable_label_items())[0] = std::move(left_hand); - (*options->mutable_label_items())[1] = std::move(right_hand); + (*options->mutable_label_items())[0] = std::move(right_hand); + (*options->mutable_label_items())[1] = std::move(left_hand); } void ConfigureHandRectTransformationCalculator( diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph_test.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph_test.cc index 62e466f6..b51381b1 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph_test.cc +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph_test.cc @@ -342,7 +342,7 @@ INSTANTIATE_TEST_SUITE_P( .test_name = "HandLandmarkerLiteModelRightUpHand", .input_model_name = kHandLandmarkerLiteModel, .test_image_name = kRightHandsImage, - .hand_rect = MakeHandRect(0.25, 0.5, 0.5, 1.0, 0), + .hand_rect = MakeHandRect(0.75, 0.5, 0.5, 1.0, 0), .expected_presence = true, .expected_landmarks = GetExpectedLandmarkList(kExpectedRightUpHandLandmarksFilename), @@ -352,7 +352,7 @@ INSTANTIATE_TEST_SUITE_P( .test_name = "HandLandmarkerLiteModelRightDownHand", .input_model_name = kHandLandmarkerLiteModel, .test_image_name = kRightHandsImage, - .hand_rect = MakeHandRect(0.75, 0.5, 0.5, 1.0, M_PI), + .hand_rect = MakeHandRect(0.25, 0.5, 0.5, 1.0, M_PI), .expected_presence = true, .expected_landmarks = GetExpectedLandmarkList( kExpectedRightDownHandLandmarksFilename), @@ -362,7 +362,7 @@ INSTANTIATE_TEST_SUITE_P( .test_name = "HandLandmarkerFullModelRightUpHand", .input_model_name = kHandLandmarkerFullModel, .test_image_name = kRightHandsImage, - .hand_rect = MakeHandRect(0.25, 0.5, 0.5, 1.0, 0), + .hand_rect = MakeHandRect(0.75, 0.5, 0.5, 1.0, 0), .expected_presence = true, .expected_landmarks = GetExpectedLandmarkList(kExpectedRightUpHandLandmarksFilename), @@ -372,7 +372,7 @@ INSTANTIATE_TEST_SUITE_P( .test_name = "HandLandmarkerFullModelRightDownHand", .input_model_name = kHandLandmarkerFullModel, .test_image_name = kRightHandsImage, - .hand_rect = MakeHandRect(0.75, 0.5, 0.5, 1.0, M_PI), + .hand_rect = MakeHandRect(0.25, 0.5, 0.5, 1.0, M_PI), .expected_presence = true, .expected_landmarks = GetExpectedLandmarkList( kExpectedRightDownHandLandmarksFilename), @@ -382,7 +382,7 @@ INSTANTIATE_TEST_SUITE_P( .test_name = "HandLandmarkerLiteModelLeftUpHand", .input_model_name = kHandLandmarkerLiteModel, .test_image_name = kLeftHandsImage, - .hand_rect = MakeHandRect(0.75, 0.5, 0.5, 1.0, 0), + .hand_rect = MakeHandRect(0.25, 0.5, 0.5, 1.0, 0), .expected_presence = true, .expected_landmarks = GetExpectedLandmarkList(kExpectedLeftUpHandLandmarksFilename), @@ -392,7 +392,7 @@ INSTANTIATE_TEST_SUITE_P( .test_name = "HandLandmarkerLiteModelLeftDownHand", .input_model_name = kHandLandmarkerLiteModel, .test_image_name = kLeftHandsImage, - .hand_rect = MakeHandRect(0.25, 0.5, 0.5, 1.0, M_PI), + .hand_rect = MakeHandRect(0.75, 0.5, 0.5, 1.0, M_PI), .expected_presence = true, .expected_landmarks = GetExpectedLandmarkList(kExpectedLeftDownHandLandmarksFilename), @@ -402,7 +402,7 @@ INSTANTIATE_TEST_SUITE_P( .test_name = "HandLandmarkerFullModelLeftUpHand", .input_model_name = kHandLandmarkerFullModel, .test_image_name = kLeftHandsImage, - .hand_rect = MakeHandRect(0.75, 0.5, 0.5, 1.0, 0), + .hand_rect = MakeHandRect(0.25, 0.5, 0.5, 1.0, 0), .expected_presence = true, .expected_landmarks = GetExpectedLandmarkList(kExpectedLeftUpHandLandmarksFilename), @@ -412,7 +412,7 @@ INSTANTIATE_TEST_SUITE_P( .test_name = "HandLandmarkerFullModelLeftDownHand", .input_model_name = kHandLandmarkerFullModel, .test_image_name = kLeftHandsImage, - .hand_rect = MakeHandRect(0.25, 0.5, 0.5, 1.0, M_PI), + .hand_rect = MakeHandRect(0.75, 0.5, 0.5, 1.0, M_PI), .expected_presence = true, .expected_landmarks = GetExpectedLandmarkList(kExpectedLeftDownHandLandmarksFilename), @@ -431,8 +431,8 @@ INSTANTIATE_TEST_SUITE_P( .test_image_name = kRightHandsImage, .hand_rects = { - MakeHandRect(0.25, 0.5, 0.5, 1.0, 0), - MakeHandRect(0.75, 0.5, 0.5, 1.0, M_PI), + MakeHandRect(0.75, 0.5, 0.5, 1.0, 0), + MakeHandRect(0.25, 0.5, 0.5, 1.0, M_PI), }, .expected_presences = {true, true}, .expected_landmark_lists = @@ -449,8 +449,8 @@ INSTANTIATE_TEST_SUITE_P( .test_image_name = kLeftHandsImage, .hand_rects = { - MakeHandRect(0.75, 0.5, 0.5, 1.0, 0), - MakeHandRect(0.25, 0.5, 0.5, 1.0, M_PI), + MakeHandRect(0.25, 0.5, 0.5, 1.0, 0), + MakeHandRect(0.75, 0.5, 0.5, 1.0, M_PI), }, .expected_presences = {true, true}, .expected_landmark_lists = diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/proto/BUILD b/mediapipe/tasks/cc/vision/hand_landmarker/proto/BUILD index d13f0afd..8097d7ab 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/proto/BUILD +++ b/mediapipe/tasks/cc/vision/hand_landmarker/proto/BUILD @@ -41,3 +41,5 @@ mediapipe_proto_library( "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_proto", ], ) + +# TODO: open source hand joints graph diff --git a/mediapipe/tasks/cc/vision/image_generator/BUILD b/mediapipe/tasks/cc/vision/image_generator/BUILD new file mode 100644 index 00000000..71b8230a --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/BUILD @@ -0,0 +1,136 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +cc_library( + name = "conditioned_image_graph", + srcs = ["conditioned_image_graph.cc"], + deps = [ + "//mediapipe/calculators/core:get_vector_item_calculator", + "//mediapipe/calculators/core:get_vector_item_calculator_cc_proto", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:flat_color_image_calculator", + "//mediapipe/calculators/util:flat_color_image_calculator_cc_proto", + "//mediapipe/calculators/util:landmarks_to_render_data_calculator", + "//mediapipe/calculators/util:landmarks_to_render_data_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_format_cc_proto", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgcodecs", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/tasks/cc/core:model_task_graph", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarks_connections", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_generator/proto:conditioned_image_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_segmenter:image_segmenter_graph", + "//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_graph_options_cc_proto", + "//mediapipe/util:color_cc_proto", + "//mediapipe/util:image_frame_util", + "//mediapipe/util:render_data_cc_proto", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + ], + alwayslink = 1, +) + +cc_library( + name = "image_generator_graph", + srcs = ["image_generator_graph.cc"], + deps = [ + ":conditioned_image_graph", + "//mediapipe/calculators/core:pass_through_calculator", + "//mediapipe/calculators/image:image_transformation_calculator", + "//mediapipe/calculators/image:image_transformation_calculator_cc_proto", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:image_to_tensor_calculator_cc_proto", + "//mediapipe/calculators/tensor:inference_calculator", + "//mediapipe/calculators/tensor:inference_calculator_cc_proto", + "//mediapipe/calculators/util:from_image_calculator", + "//mediapipe/calculators/util:to_image_calculator", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:stream_handler_cc_proto", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:status", + "//mediapipe/framework/tool:switch_container", + "//mediapipe/framework/tool:switch_container_cc_proto", + "//mediapipe/tasks/cc/core:model_asset_bundle_resources", + "//mediapipe/tasks/cc/core:model_resources", + "//mediapipe/tasks/cc/core:model_task_graph", + "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", + "//mediapipe/tasks/cc/vision/image_generator/diffuser:diffusion_plugins_output_calculator", + "//mediapipe/tasks/cc/vision/image_generator/diffuser:stable_diffusion_iterate_calculator", + "//mediapipe/tasks/cc/vision/image_generator/diffuser:stable_diffusion_iterate_calculator_cc_proto", + "//mediapipe/tasks/cc/vision/image_generator/proto:conditioned_image_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_generator/proto:control_plugin_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_generator/proto:image_generator_graph_options_cc_proto", + "//mediapipe/util:graph_builder_utils", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + ], + alwayslink = 1, +) + +cc_library( + name = "image_generator_result", + hdrs = ["image_generator_result.h"], + deps = ["//mediapipe/framework/formats:image"], +) + +cc_library( + name = "image_generator", + srcs = ["image_generator.cc"], + hdrs = ["image_generator.h"], + deps = [ + ":image_generator_graph", + ":image_generator_result", + "//mediapipe/framework:packet", + "//mediapipe/framework:timestamp", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:tensor", + "//mediapipe/tasks/cc/core:base_options", + "//mediapipe/tasks/cc/core:task_runner", + "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", + "//mediapipe/tasks/cc/vision/core:base_vision_task_api", + "//mediapipe/tasks/cc/vision/core:vision_task_api_factory", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_landmarker", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_generator/proto:conditioned_image_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_generator/proto:control_plugin_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_generator/proto:image_generator_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_segmenter", + "//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_graph_options_cc_proto", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", + ], +) diff --git a/mediapipe/tasks/cc/vision/image_generator/conditioned_image_graph.cc b/mediapipe/tasks/cc/vision/image_generator/conditioned_image_graph.cc new file mode 100644 index 00000000..c85fe981 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/conditioned_image_graph.cc @@ -0,0 +1,458 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "mediapipe/calculators/core/get_vector_item_calculator.h" +#include "mediapipe/calculators/core/get_vector_item_calculator.pb.h" +#include "mediapipe/calculators/util/flat_color_image_calculator.pb.h" +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.h" +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/tasks/cc/core/model_task_graph.h" +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_connections.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/proto/conditioned_image_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_segmenter/proto/image_segmenter_graph_options.pb.h" +#include "mediapipe/util/color.pb.h" +#include "mediapipe/util/image_frame_util.h" +#include "mediapipe/util/render_data.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace image_generator { + +namespace internal { + +// Helper postprocessing calculator for depth condition type to scale raw depth +// inference result to 0-255 uint8. +class DepthImagePostprocessingCalculator : public api2::Node { + public: + static constexpr api2::Input kImageIn{"IMAGE"}; + static constexpr api2::Output kImageOut{"IMAGE"}; + + MEDIAPIPE_NODE_CONTRACT(kImageIn, kImageOut); + + absl::Status Process(CalculatorContext* cc) final { + if (kImageIn(cc).IsEmpty()) { + return absl::OkStatus(); + } + Image raw_depth_image = kImageIn(cc).Get(); + cv::Mat raw_depth_mat = mediapipe::formats::MatView( + raw_depth_image.GetImageFrameSharedPtr().get()); + cv::Mat depth_mat; + cv::normalize(raw_depth_mat, depth_mat, 255, 0, cv::NORM_MINMAX); + depth_mat.convertTo(depth_mat, CV_8UC3, 1, 0); + cv::cvtColor(depth_mat, depth_mat, cv::COLOR_GRAY2RGB); + // Acquires the cv::Mat data and assign to the image frame. + ImageFrameSharedPtr depth_image_frame_ptr = std::make_shared( + mediapipe::ImageFormat::SRGB, depth_mat.cols, depth_mat.rows, + depth_mat.step, depth_mat.data, + [depth_mat](uint8_t[]) { depth_mat.~Mat(); }); + Image depth_image(depth_image_frame_ptr); + kImageOut(cc).Send(depth_image); + return absl::OkStatus(); + } +}; + +// NOLINTBEGIN: Node registration doesn't work when part of calculator name is +// moved to next line. +// clang-format off +MEDIAPIPE_REGISTER_NODE(::mediapipe::tasks::vision::image_generator::internal::DepthImagePostprocessingCalculator); +// clang-format on +// NOLINTEND + +// Calculator to detect edges in the image with OpenCV Canny edge detection. +class CannyEdgeCalculator : public api2::Node { + public: + static constexpr api2::Input kImageIn{"IMAGE"}; + static constexpr api2::Output kImageOut{"IMAGE"}; + + MEDIAPIPE_NODE_CONTRACT(kImageIn, kImageOut); + + absl::Status Process(CalculatorContext* cc) final { + if (kImageIn(cc).IsEmpty()) { + return absl::OkStatus(); + } + Image input_image = kImageIn(cc).Get(); + cv::Mat input_image_mat = + mediapipe::formats::MatView(input_image.GetImageFrameSharedPtr().get()); + const auto& options = cc->Options< + proto::ConditionedImageGraphOptions::EdgeConditionTypeOptions>(); + cv::Mat lumincance; + cv::cvtColor(input_image_mat, lumincance, cv::COLOR_RGB2GRAY); + cv::Mat edges_mat; + cv::Canny(lumincance, edges_mat, options.threshold_1(), + options.threshold_2(), options.aperture_size(), + options.l2_gradient()); + cv::normalize(edges_mat, edges_mat, 255, 0, cv::NORM_MINMAX); + edges_mat.convertTo(edges_mat, CV_8UC3, 1, 0); + cv::cvtColor(edges_mat, edges_mat, cv::COLOR_GRAY2RGB); + // Acquires the cv::Mat data and assign to the image frame. + ImageFrameSharedPtr edges_image_frame_ptr = std::make_shared( + mediapipe::ImageFormat::SRGB, edges_mat.cols, edges_mat.rows, + edges_mat.step, edges_mat.data, + [edges_mat](uint8_t[]) { edges_mat.~Mat(); }); + Image edges_image(edges_image_frame_ptr); + kImageOut(cc).Send(edges_image); + return absl::OkStatus(); + } +}; + +// NOLINTBEGIN: Node registration doesn't work when part of calculator name is +// moved to next line. +// clang-format off +MEDIAPIPE_REGISTER_NODE(::mediapipe::tasks::vision::image_generator::internal::CannyEdgeCalculator); +// clang-format on +// NOLINTEND + +} // namespace internal + +namespace { + +using ::mediapipe::api2::Input; +using ::mediapipe::api2::Output; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Source; + +constexpr absl::string_view kImageTag = "IMAGE"; +constexpr absl::string_view kUImageTag = "UIMAGE"; +constexpr absl::string_view kNormLandmarksTag = "NORM_LANDMARKS"; +constexpr absl::string_view kVectorTag = "VECTOR"; +constexpr absl::string_view kItemTag = "ITEM"; +constexpr absl::string_view kRenderDataTag = "RENDER_DATA"; +constexpr absl::string_view kConfidenceMaskTag = "CONFIDENCE_MASK:0"; + +enum ColorType { + WHITE = 0, + GREEN = 1, + RED = 2, + BLACK = 3, + BLUE = 4, +}; + +mediapipe::Color GetColor(ColorType color_type) { + mediapipe::Color color; + switch (color_type) { + case WHITE: + color.set_b(255); + color.set_g(255); + color.set_r(255); + break; + case GREEN: + color.set_b(0); + color.set_g(255); + color.set_r(0); + break; + case RED: + color.set_b(0); + color.set_g(0); + color.set_r(255); + break; + case BLACK: + color.set_b(0); + color.set_g(0); + color.set_r(0); + break; + case BLUE: + color.set_b(255); + color.set_g(0); + color.set_r(0); + break; + } + return color; +} + +// Get LandmarksToRenderDataCalculatorOptions for rendering face landmarks +// connections. +mediapipe::LandmarksToRenderDataCalculatorOptions +GetFaceLandmarksRenderDataOptions( + absl::Span> connections, ColorType color_type) { + mediapipe::LandmarksToRenderDataCalculatorOptions render_options; + render_options.set_thickness(1); + render_options.set_visualize_landmark_depth(false); + render_options.set_render_landmarks(false); + *render_options.mutable_connection_color() = GetColor(color_type); + for (const auto& connection : connections) { + render_options.add_landmark_connections(connection[0]); + render_options.add_landmark_connections(connection[1]); + } + return render_options; +} + +Source GetFaceLandmarksRenderData( + Source face_landmarks, + const mediapipe::LandmarksToRenderDataCalculatorOptions& + landmarks_to_render_data_options, + Graph& graph) { + auto& landmarks_to_render_data = + graph.AddNode("LandmarksToRenderDataCalculator"); + landmarks_to_render_data + .GetOptions() + .CopyFrom(landmarks_to_render_data_options); + face_landmarks >> landmarks_to_render_data.In(kNormLandmarksTag); + return landmarks_to_render_data.Out(kRenderDataTag) + .Cast(); +} + +// Add FaceLandmarkerGraph to detect the face landmarks in the given face image, +// and generate a face mesh guidance image for the diffusion plugin model. +absl::StatusOr> GetFaceLandmarksImage( + Source face_image, + const proto::ConditionedImageGraphOptions::FaceConditionTypeOptions& + face_condition_type_options, + Graph& graph) { + if (face_condition_type_options.face_landmarker_graph_options() + .face_detector_graph_options() + .num_faces() != 1) { + return absl::InvalidArgumentError( + "Only supports face landmarks of a single face as the guidance image."); + } + + // Detect face landmarks. + auto& face_landmarker_graph = graph.AddNode( + "mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph"); + face_landmarker_graph + .GetOptions() + .CopyFrom(face_condition_type_options.face_landmarker_graph_options()); + face_image >> face_landmarker_graph.In(kImageTag); + auto face_landmarks_lists = + face_landmarker_graph.Out(kNormLandmarksTag) + .Cast>(); + + // Get the single face landmarks. + auto& get_vector_item = + graph.AddNode("GetNormalizedLandmarkListVectorItemCalculator"); + get_vector_item.GetOptions() + .set_item_index(0); + face_landmarks_lists >> get_vector_item.In(kVectorTag); + auto single_face_landmarks = + get_vector_item.Out(kItemTag).Cast(); + + // Convert face landmarks to render data. + auto face_oval = GetFaceLandmarksRenderData( + single_face_landmarks, + GetFaceLandmarksRenderDataOptions( + absl::Span>( + face_landmarker::FaceLandmarksConnections::kFaceLandmarksFaceOval + .data(), + face_landmarker::FaceLandmarksConnections::kFaceLandmarksFaceOval + .size()), + ColorType::WHITE), + graph); + auto lips = GetFaceLandmarksRenderData( + single_face_landmarks, + GetFaceLandmarksRenderDataOptions( + absl::Span>( + face_landmarker::FaceLandmarksConnections::kFaceLandmarksLips + .data(), + face_landmarker::FaceLandmarksConnections::kFaceLandmarksLips + .size()), + ColorType::WHITE), + graph); + auto left_eye = GetFaceLandmarksRenderData( + single_face_landmarks, + GetFaceLandmarksRenderDataOptions( + absl::Span>( + face_landmarker::FaceLandmarksConnections::kFaceLandmarksLeftEye + .data(), + face_landmarker::FaceLandmarksConnections::kFaceLandmarksLeftEye + .size()), + ColorType::GREEN), + graph); + auto left_eye_brow = GetFaceLandmarksRenderData( + single_face_landmarks, + GetFaceLandmarksRenderDataOptions( + absl::Span>( + face_landmarker::FaceLandmarksConnections:: + kFaceLandmarksLeftEyeBrow.data(), + face_landmarker::FaceLandmarksConnections:: + kFaceLandmarksLeftEyeBrow.size()), + ColorType::GREEN), + graph); + auto left_iris = GetFaceLandmarksRenderData( + single_face_landmarks, + GetFaceLandmarksRenderDataOptions( + absl::Span>( + face_landmarker::FaceLandmarksConnections::kFaceLandmarksLeftIris + .data(), + face_landmarker::FaceLandmarksConnections::kFaceLandmarksLeftIris + .size()), + ColorType::GREEN), + graph); + + auto right_eye = GetFaceLandmarksRenderData( + single_face_landmarks, + GetFaceLandmarksRenderDataOptions( + absl::Span>( + face_landmarker::FaceLandmarksConnections::kFaceLandmarksRightEye + .data(), + face_landmarker::FaceLandmarksConnections::kFaceLandmarksRightEye + .size()), + ColorType::BLUE), + graph); + auto right_eye_brow = GetFaceLandmarksRenderData( + single_face_landmarks, + GetFaceLandmarksRenderDataOptions( + absl::Span>( + face_landmarker::FaceLandmarksConnections:: + kFaceLandmarksRightEyeBrow.data(), + face_landmarker::FaceLandmarksConnections:: + kFaceLandmarksRightEyeBrow.size()), + ColorType::BLUE), + graph); + auto right_iris = GetFaceLandmarksRenderData( + single_face_landmarks, + GetFaceLandmarksRenderDataOptions( + absl::Span>( + face_landmarker::FaceLandmarksConnections::kFaceLandmarksRightIris + .data(), + face_landmarker::FaceLandmarksConnections::kFaceLandmarksRightIris + .size()), + ColorType::BLUE), + graph); + + // Create a black canvas image with same size as face image. + auto& flat_color = graph.AddNode("FlatColorImageCalculator"); + flat_color.GetOptions() + .mutable_color() + ->set_r(0); + face_image >> flat_color.In(kImageTag); + auto blank_canvas = flat_color.Out(kImageTag); + + // Draw render data on the canvas image. + auto& annotation_overlay = graph.AddNode("AnnotationOverlayCalculator"); + blank_canvas >> annotation_overlay.In(kUImageTag); + face_oval >> annotation_overlay.In(0); + lips >> annotation_overlay.In(1); + left_eye >> annotation_overlay.In(2); + left_eye_brow >> annotation_overlay.In(3); + left_iris >> annotation_overlay.In(4); + right_eye >> annotation_overlay.In(5); + right_eye_brow >> annotation_overlay.In(6); + right_iris >> annotation_overlay.In(7); + return annotation_overlay.Out(kUImageTag).Cast(); +} + +absl::StatusOr> GetDepthImage( + Source image, + const image_generator::proto::ConditionedImageGraphOptions:: + DepthConditionTypeOptions& depth_condition_type_options, + Graph& graph) { + auto& image_segmenter_graph = graph.AddNode( + "mediapipe.tasks.vision.image_segmenter.ImageSegmenterGraph"); + image_segmenter_graph + .GetOptions() + .CopyFrom(depth_condition_type_options.image_segmenter_graph_options()); + image >> image_segmenter_graph.In(kImageTag); + auto raw_depth_image = image_segmenter_graph.Out(kConfidenceMaskTag); + + auto& depth_postprocessing = graph.AddNode( + "mediapipe.tasks.vision.image_generator.internal." + "DepthImagePostprocessingCalculator"); + raw_depth_image >> depth_postprocessing.In(kImageTag); + return depth_postprocessing.Out(kImageTag).Cast(); +} + +absl::StatusOr> GetEdgeImage( + Source image, + const image_generator::proto::ConditionedImageGraphOptions:: + EdgeConditionTypeOptions& edge_condition_type_options, + Graph& graph) { + auto& edge_detector = graph.AddNode( + "mediapipe.tasks.vision.image_generator.internal." + "CannyEdgeCalculator"); + edge_detector + .GetOptions< + proto::ConditionedImageGraphOptions::EdgeConditionTypeOptions>() + .CopyFrom(edge_condition_type_options); + image >> edge_detector.In(kImageTag); + return edge_detector.Out(kImageTag).Cast(); +} + +} // namespace + +// A mediapipe.tasks.vision.image_generator.ConditionedImageGraph converts the +// input image to an image of condition type. The output image can be used as +// input for the diffusion model with control plugin. +// Inputs: +// IMAGE - Image +// Conditioned image to generate the image for diffusion plugin model. +// +// Outputs: +// IMAGE - Image +// The guidance image used as input for the diffusion plugin model. +class ConditionedImageGraph : public core::ModelTaskGraph { + public: + absl::StatusOr GetConfig( + SubgraphContext* sc) override { + Graph graph; + auto& graph_options = + *sc->MutableOptions(); + Source conditioned_image = graph.In(kImageTag).Cast(); + // Configure the guidance graph and get the guidance image if guidance graph + // options is set. + switch (graph_options.condition_type_options_case()) { + case proto::ConditionedImageGraphOptions::CONDITION_TYPE_OPTIONS_NOT_SET: + return absl::InvalidArgumentError( + "Conditioned type options is not set."); + break; + case proto::ConditionedImageGraphOptions::kFaceConditionTypeOptions: { + ASSIGN_OR_RETURN( + auto face_landmarks_image, + GetFaceLandmarksImage(conditioned_image, + graph_options.face_condition_type_options(), + graph)); + face_landmarks_image >> graph.Out(kImageTag); + } break; + case proto::ConditionedImageGraphOptions::kDepthConditionTypeOptions: { + ASSIGN_OR_RETURN( + auto depth_image, + GetDepthImage(conditioned_image, + graph_options.depth_condition_type_options(), graph)); + depth_image >> graph.Out(kImageTag); + } break; + case proto::ConditionedImageGraphOptions::kEdgeConditionTypeOptions: { + ASSIGN_OR_RETURN( + auto edges_image, + GetEdgeImage(conditioned_image, + graph_options.edge_condition_type_options(), graph)); + edges_image >> graph.Out(kImageTag); + } break; + } + return graph.GetConfig(); + } +}; + +REGISTER_MEDIAPIPE_GRAPH( + ::mediapipe::tasks::vision::image_generator::ConditionedImageGraph); + +} // namespace image_generator +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/image_generator/conditioned_image_graph_test.cc b/mediapipe/tasks/cc/vision/image_generator/conditioned_image_graph_test.cc new file mode 100644 index 00000000..c67ae2fe --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/conditioned_image_graph_test.cc @@ -0,0 +1,147 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/tool/test_util.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/proto/conditioned_image_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_segmenter/proto/image_segmenter_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace image_generator { + +namespace { + +using ::mediapipe::Image; +using ::mediapipe::api2::Input; +using ::mediapipe::api2::Output; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Source; +using ::mediapipe::tasks::core::TaskRunner; +using ::mediapipe::tasks::vision::DecodeImageFromFile; + +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kFaceLandmarkerModel[] = "face_landmarker_v2.task"; +constexpr char kDepthModel[] = + "mobilenetsweep_dptrigmqn384_unit_384_384_fp16quant_fp32input_opt.tflite"; +constexpr char kPortraitImage[] = "portrait.jpg"; +constexpr char kImageTag[] = "IMAGE"; +constexpr char kImageInStream[] = "image_in"; +constexpr char kImageOutStream[] = "image_out"; + +// Helper function to create a ConditionedImageGraphTaskRunner TaskRunner. +absl::StatusOr> +CreateConditionedImageGraphTaskRunner( + std::unique_ptr options) { + Graph graph; + auto& conditioned_image_graph = graph.AddNode( + "mediapipe.tasks.vision.image_generator.ConditionedImageGraph"); + conditioned_image_graph.GetOptions() + .Swap(options.get()); + graph.In(kImageTag).Cast().SetName(kImageInStream) >> + conditioned_image_graph.In(kImageTag); + conditioned_image_graph.Out(kImageTag).SetName(kImageOutStream) >> + graph.Out(kImageTag).Cast(); + return core::TaskRunner::Create( + graph.GetConfig(), + absl::make_unique()); +} + +TEST(ConditionedImageGraphTest, SucceedsFaceLandmarkerConditionType) { + auto options = std::make_unique(); + options->mutable_face_condition_type_options() + ->mutable_face_landmarker_graph_options() + ->mutable_base_options() + ->mutable_model_asset() + ->set_file_name( + file::JoinPath("./", kTestDataDirectory, kFaceLandmarkerModel)); + options->mutable_face_condition_type_options() + ->mutable_face_landmarker_graph_options() + ->mutable_face_detector_graph_options() + ->set_num_faces(1); + MP_ASSERT_OK_AND_ASSIGN( + auto runner, CreateConditionedImageGraphTaskRunner(std::move(options))); + MP_ASSERT_OK_AND_ASSIGN( + Image image, DecodeImageFromFile(file::JoinPath("./", kTestDataDirectory, + kPortraitImage))); + MP_ASSERT_OK_AND_ASSIGN( + auto output_packets, + runner->Process({{kImageInStream, MakePacket(std::move(image))}})); + const auto& output_image = output_packets[kImageOutStream].Get(); + MP_EXPECT_OK(SavePngTestOutput(*output_image.GetImageFrameSharedPtr(), + "face_landmarks_image")); +} + +TEST(ConditionedImageGraphTest, SucceedsDepthConditionType) { + auto options = std::make_unique(); + options->mutable_depth_condition_type_options() + ->mutable_image_segmenter_graph_options() + ->mutable_base_options() + ->mutable_model_asset() + ->set_file_name(file::JoinPath("./", kTestDataDirectory, kDepthModel)); + MP_ASSERT_OK_AND_ASSIGN( + Image image, DecodeImageFromFile(file::JoinPath("./", kTestDataDirectory, + kPortraitImage))); + MP_ASSERT_OK_AND_ASSIGN( + auto runner, CreateConditionedImageGraphTaskRunner(std::move(options))); + MP_ASSERT_OK_AND_ASSIGN( + auto output_packets, + runner->Process({{kImageInStream, MakePacket(std::move(image))}})); + const auto& output_image = output_packets[kImageOutStream].Get(); + MP_EXPECT_OK( + SavePngTestOutput(*output_image.GetImageFrameSharedPtr(), "depth_image")); +} + +TEST(ConditionedImageGraphTest, SucceedsEdgeConditionType) { + auto options = std::make_unique(); + auto edge_condition_type_options = + options->mutable_edge_condition_type_options(); + edge_condition_type_options->set_threshold_1(100); + edge_condition_type_options->set_threshold_2(200); + edge_condition_type_options->set_aperture_size(3); + MP_ASSERT_OK_AND_ASSIGN( + Image image, DecodeImageFromFile(file::JoinPath("./", kTestDataDirectory, + kPortraitImage))); + MP_ASSERT_OK_AND_ASSIGN( + auto runner, CreateConditionedImageGraphTaskRunner(std::move(options))); + MP_ASSERT_OK_AND_ASSIGN( + auto output_packets, + runner->Process({{kImageInStream, MakePacket(std::move(image))}})); + const auto& output_image = output_packets[kImageOutStream].Get(); + MP_EXPECT_OK( + SavePngTestOutput(*output_image.GetImageFrameSharedPtr(), "edges_image")); +} + +} // namespace +} // namespace image_generator +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/image_generator/diffuser/BUILD b/mediapipe/tasks/cc/vision/image_generator/diffuser/BUILD new file mode 100644 index 00000000..fe10affa --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/diffuser/BUILD @@ -0,0 +1,70 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") + +licenses(["notice"]) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +cc_library( + name = "diffuser_gpu_header", + hdrs = ["diffuser_gpu.h"], + visibility = [ + "//mediapipe/tasks/cc/vision/image_generator/diffuser:__pkg__", + ], +) + +mediapipe_proto_library( + name = "stable_diffusion_iterate_calculator_proto", + srcs = ["stable_diffusion_iterate_calculator.proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], +) + +cc_library( + name = "stable_diffusion_iterate_calculator", + srcs = ["stable_diffusion_iterate_calculator.cc"], + deps = [ + ":diffuser_gpu_header", + ":stable_diffusion_iterate_calculator_cc_proto", + "//mediapipe/framework:calculator_context", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/deps:file_helpers", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:tensor", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + ], + alwayslink = 1, +) + +cc_library( + name = "diffusion_plugins_output_calculator", + srcs = ["diffusion_plugins_output_calculator.cc"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/formats:tensor", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], + alwayslink = 1, +) diff --git a/mediapipe/tasks/cc/vision/image_generator/diffuser/diffuser_gpu.h b/mediapipe/tasks/cc/vision/image_generator/diffuser/diffuser_gpu.h new file mode 100644 index 00000000..85738b80 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/diffuser/diffuser_gpu.h @@ -0,0 +1,87 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_DIFFUSER_DIFFUSER_GPU_H_ +#define MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_DIFFUSER_DIFFUSER_GPU_H_ + +#include +#include + +#ifndef DG_EXPORT +#define DG_EXPORT __attribute__((visibility("default"))) +#endif // DG_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + +enum DiffuserModelType { + kDiffuserModelTypeSd1, + kDiffuserModelTypeGldm, + kDiffuserModelTypeDistilledGldm, + kDiffuserModelTypeSd2Base, + kDiffuserModelTypeTigo, +}; + +enum DiffuserPriorityHint { + kDiffuserPriorityHintHigh, + kDiffuserPriorityHintNormal, + kDiffuserPriorityHintLow, +}; + +enum DiffuserPerformanceHint { + kDiffuserPerformanceHintHigh, + kDiffuserPerformanceHintNormal, + kDiffuserPerformanceHintLow, +}; + +typedef struct { + DiffuserPriorityHint priority_hint; + DiffuserPerformanceHint performance_hint; +} DiffuserEnvironmentOptions; + +typedef struct { + DiffuserModelType model_type; + char model_dir[PATH_MAX]; + char lora_dir[PATH_MAX]; + const void* lora_weights_layer_mapping; + int lora_rank; + int seed; + int image_width; + int image_height; + int run_unet_with_plugins; + DiffuserEnvironmentOptions env_options; +} DiffuserConfig; + +typedef struct { + void* diffuser; +} DiffuserContext; + +typedef struct { + int shape[4]; + const float* data; +} DiffuserPluginTensor; + +DG_EXPORT DiffuserContext* DiffuserCreate(const DiffuserConfig*); // NOLINT +DG_EXPORT int DiffuserReset(DiffuserContext*, // NOLINT + const char*, int, int, float, const void*); +DG_EXPORT int DiffuserIterate(DiffuserContext*, int, int); // NOLINT +DG_EXPORT int DiffuserDecode(DiffuserContext*, uint8_t*); // NOLINT +DG_EXPORT void DiffuserDelete(DiffuserContext*); // NOLINT + +#ifdef __cplusplus +} +#endif // __cplusplus + +#endif // MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_DIFFUSER_DIFFUSER_GPU_H_ diff --git a/mediapipe/tasks/cc/vision/image_generator/diffuser/diffusion_plugins_output_calculator.cc b/mediapipe/tasks/cc/vision/image_generator/diffuser/diffusion_plugins_output_calculator.cc new file mode 100644 index 00000000..a2282b90 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/diffuser/diffusion_plugins_output_calculator.cc @@ -0,0 +1,66 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/tensor.h" + +namespace mediapipe { +namespace api2 { + +// In iteration mode, output the image guidance tensors at the current timestamp +// and advance the output stream timestamp bound by the number of steps. +// Otherwise, output the image guidance tensors at the current timestamp only. +class DiffusionPluginsOutputCalculator : public Node { + public: + static constexpr Input> kTensorsIn{"TENSORS"}; + static constexpr Input kStepsIn{"STEPS"}; + static constexpr Input::Optional kIterationIn{"ITERATION"}; + static constexpr Output> kTensorsOut{"TENSORS"}; + MEDIAPIPE_NODE_CONTRACT(kTensorsIn, kStepsIn, kIterationIn, kTensorsOut); + + absl::Status Process(CalculatorContext* cc) override { + if (kTensorsIn(cc).IsEmpty()) { + return absl::OkStatus(); + } + // Consumes the tensor vector to avoid data copy. + absl::StatusOr>> status_or_tensor = + cc->Inputs().Tag("TENSORS").Value().Consume>(); + if (!status_or_tensor.ok()) { + return absl::InternalError("Input tensor vector is not consumable."); + } + if (kIterationIn(cc).IsConnected()) { + ABSL_CHECK_EQ(kIterationIn(cc).Get(), 0); + kTensorsOut(cc).Send(std::move(*status_or_tensor.value())); + kTensorsOut(cc).SetNextTimestampBound(cc->InputTimestamp() + + kStepsIn(cc).Get()); + } else { + kTensorsOut(cc).Send(std::move(*status_or_tensor.value())); + } + return absl::OkStatus(); + } +}; + +MEDIAPIPE_REGISTER_NODE(DiffusionPluginsOutputCalculator); + +} // namespace api2 +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/image_generator/diffuser/stable_diffusion_iterate_calculator.cc b/mediapipe/tasks/cc/vision/image_generator/diffuser/stable_diffusion_iterate_calculator.cc new file mode 100644 index 00000000..91c64450 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/diffuser/stable_diffusion_iterate_calculator.cc @@ -0,0 +1,280 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/calculator_context.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/deps/file_helpers.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/tasks/cc/vision/image_generator/diffuser/diffuser_gpu.h" +#include "mediapipe/tasks/cc/vision/image_generator/diffuser/stable_diffusion_iterate_calculator.pb.h" + +namespace mediapipe { +namespace api2 { +namespace { + +DiffuserPriorityHint ToDiffuserPriorityHint( + StableDiffusionIterateCalculatorOptions::ClPriorityHint priority) { + switch (priority) { + case StableDiffusionIterateCalculatorOptions::PRIORITY_HINT_LOW: + return kDiffuserPriorityHintLow; + case StableDiffusionIterateCalculatorOptions::PRIORITY_HINT_NORMAL: + return kDiffuserPriorityHintNormal; + case StableDiffusionIterateCalculatorOptions::PRIORITY_HINT_HIGH: + return kDiffuserPriorityHintHigh; + } + return kDiffuserPriorityHintNormal; +} + +DiffuserModelType ToDiffuserModelType( + StableDiffusionIterateCalculatorOptions::ModelType model_type) { + switch (model_type) { + case StableDiffusionIterateCalculatorOptions::DEFAULT: + case StableDiffusionIterateCalculatorOptions::SD_1: + return kDiffuserModelTypeSd1; + } + return kDiffuserModelTypeSd1; +} + +} // namespace + +// Runs diffusion models including, but not limited to, Stable Diffusion & gLDM. +// +// Inputs: +// PROMPT - std::string +// The prompt used to generate the image. +// STEPS - int +// The number of steps to run the UNet. +// ITERATION - int +// The iteration of the current run. +// PLUGIN_TENSORS - std::vector @Optional +// The output tensor vector of the diffusion plugins model. +// +// Outputs: +// IMAGE - mediapipe::ImageFrame +// The image generated by the Stable Diffusion model from the input prompt. +// The output image is in RGB format. +// +// Example: +// node { +// calculator: "StableDiffusionIterateCalculator" +// input_stream: "PROMPT:prompt" +// input_stream: "STEPS:steps" +// output_stream: "IMAGE:result" +// options { +// [mediapipe.StableDiffusionIterateCalculatorOptions.ext] { +// base_seed: 0 +// model_type: SD_1 +// } +// } +// } +class StableDiffusionIterateCalculator : public Node { + public: + static constexpr Input kPromptIn{"PROMPT"}; + static constexpr Input kStepsIn{"STEPS"}; + static constexpr Input::Optional kIterationIn{"ITERATION"}; + static constexpr Input::Optional kRandSeedIn{"RAND_SEED"}; + static constexpr SideInput::Optional + kOptionsIn{"OPTIONS"}; + static constexpr Input>::Optional kPlugInTensorsIn{ + "PLUGIN_TENSORS"}; + static constexpr Output kImageOut{"IMAGE"}; + MEDIAPIPE_NODE_CONTRACT(kPromptIn, kStepsIn, kIterationIn, kRandSeedIn, + kPlugInTensorsIn, kOptionsIn, kImageOut); + + ~StableDiffusionIterateCalculator() { + if (context_) DiffuserDelete(); + if (handle_) dlclose(handle_); + } + + absl::Status Open(CalculatorContext* cc) override; + absl::Status Process(CalculatorContext* cc) override; + + private: + std::vector GetPluginTensors( + CalculatorContext* cc) const { + if (!kPlugInTensorsIn(cc).IsConnected()) return {}; + std::vector diffuser_tensors; + diffuser_tensors.reserve(kPlugInTensorsIn(cc)->size()); + for (const auto& mp_tensor : *kPlugInTensorsIn(cc)) { + DiffuserPluginTensor diffuser_tensor; + diffuser_tensor.shape[0] = mp_tensor.shape().dims[0]; + diffuser_tensor.shape[1] = mp_tensor.shape().dims[1]; + diffuser_tensor.shape[2] = mp_tensor.shape().dims[2]; + diffuser_tensor.shape[3] = mp_tensor.shape().dims[3]; + diffuser_tensor.data = mp_tensor.GetCpuReadView().buffer(); + diffuser_tensors.push_back(diffuser_tensor); + } + return diffuser_tensors; + } + + absl::Status LoadDiffuser() { + handle_ = dlopen("libimagegenerator_gpu.so", RTLD_NOW | RTLD_LOCAL); + RET_CHECK(handle_) << dlerror(); + create_ptr_ = reinterpret_cast( + dlsym(handle_, "DiffuserCreate")); + RET_CHECK(create_ptr_) << dlerror(); + reset_ptr_ = + reinterpret_cast(dlsym(handle_, "DiffuserReset")); + RET_CHECK(reset_ptr_) << dlerror(); + iterate_ptr_ = reinterpret_cast( + dlsym(handle_, "DiffuserIterate")); + RET_CHECK(iterate_ptr_) << dlerror(); + decode_ptr_ = reinterpret_cast( + dlsym(handle_, "DiffuserDecode")); + RET_CHECK(decode_ptr_) << dlerror(); + delete_ptr_ = reinterpret_cast( + dlsym(handle_, "DiffuserDelete")); + RET_CHECK(delete_ptr_) << dlerror(); + return absl::OkStatus(); + } + + DiffuserContext* DiffuserCreate(const DiffuserConfig* a) { + return (*create_ptr_)(a); + } + bool DiffuserReset(const char* a, int b, int c, float d, + const std::vector* e) { + return (*reset_ptr_)(context_, a, b, c, d, e); + } + bool DiffuserIterate(int a, int b) { return (*iterate_ptr_)(context_, a, b); } + bool DiffuserDecode(uint8_t* a) { return (*decode_ptr_)(context_, a); } + void DiffuserDelete() { (*delete_ptr_)(context_); } + + void* handle_ = nullptr; + DiffuserContext* context_ = nullptr; + DiffuserContext* (*create_ptr_)(const DiffuserConfig*); + int (*reset_ptr_)(DiffuserContext*, const char*, int, int, float, + const void*); + int (*iterate_ptr_)(DiffuserContext*, int, int); + int (*decode_ptr_)(DiffuserContext*, uint8_t*); + void (*delete_ptr_)(DiffuserContext*); + + int show_every_n_iteration_; + bool emit_empty_packet_; +}; + +absl::Status StableDiffusionIterateCalculator::Open(CalculatorContext* cc) { + StableDiffusionIterateCalculatorOptions options; + if (kOptionsIn(cc).IsEmpty()) { + options = cc->Options(); + } else { + options = kOptionsIn(cc).Get(); + } + show_every_n_iteration_ = options.show_every_n_iteration(); + emit_empty_packet_ = options.emit_empty_packet(); + + MP_RETURN_IF_ERROR(LoadDiffuser()); + + DiffuserConfig config; + config.model_type = ToDiffuserModelType(options.model_type()); + if (options.file_folder().empty()) { + std::strcpy(config.model_dir, "bins/"); // NOLINT + } else { + std::strcpy(config.model_dir, options.file_folder().c_str()); // NOLINT + } + MP_RETURN_IF_ERROR(mediapipe::file::Exists(config.model_dir)) + << config.model_dir; + RET_CHECK(options.lora_file_folder().empty() || + options.lora_weights_layer_mapping().empty()) + << "Can't set both lora_file_folder and lora_weights_layer_mapping."; + std::strcpy(config.lora_dir, options.lora_file_folder().c_str()); // NOLINT + std::map lora_weights_layer_mapping; + for (auto& layer_name_and_weights : options.lora_weights_layer_mapping()) { + lora_weights_layer_mapping[layer_name_and_weights.first] = + (char*)layer_name_and_weights.second; + } + config.lora_weights_layer_mapping = !lora_weights_layer_mapping.empty() + ? &lora_weights_layer_mapping + : nullptr; + config.lora_rank = options.lora_rank(); + config.seed = options.base_seed(); + config.image_width = options.output_image_width(); + config.image_height = options.output_image_height(); + config.run_unet_with_plugins = kPlugInTensorsIn(cc).IsConnected(); + config.env_options = { + .priority_hint = ToDiffuserPriorityHint(options.cl_priority_hint()), + .performance_hint = kDiffuserPerformanceHintHigh, + }; + RET_CHECK(options.plugins_strength() >= 0.0f || + options.plugins_strength() <= 1.0f) + << "The value of plugins_strength must be in the range of [0, 1]."; + context_ = DiffuserCreate(&config); + RET_CHECK(context_); + return absl::OkStatus(); +} + +absl::Status StableDiffusionIterateCalculator::Process(CalculatorContext* cc) { + const auto& options = + cc->Options().GetExtension(StableDiffusionIterateCalculatorOptions::ext); + const std::string& prompt = *kPromptIn(cc); + const int steps = *kStepsIn(cc); + const int rand_seed = !kRandSeedIn(cc).IsEmpty() ? std::abs(*kRandSeedIn(cc)) + : options.base_seed(); + + if (kIterationIn(cc).IsEmpty()) { + const auto plugin_tensors = GetPluginTensors(cc); + RET_CHECK(DiffuserReset(prompt.c_str(), steps, rand_seed, + options.plugins_strength(), &plugin_tensors)); + for (int i = 0; i < steps; i++) RET_CHECK(DiffuserIterate(steps, i)); + ImageFrame image_out(ImageFormat::SRGB, options.output_image_width(), + options.output_image_height()); + RET_CHECK(DiffuserDecode(image_out.MutablePixelData())); + kImageOut(cc).Send(std::move(image_out)); + } else { + const int iteration = *kIterationIn(cc); + RET_CHECK_LT(iteration, steps); + + // Extract text embedding on first iteration. + if (iteration == 0) { + const auto plugin_tensors = GetPluginTensors(cc); + RET_CHECK(DiffuserReset(prompt.c_str(), steps, rand_seed, + options.plugins_strength(), &plugin_tensors)); + } + + RET_CHECK(DiffuserIterate(steps, iteration)); + + // Decode the output and send out the image for visualization. + if ((iteration + 1) % show_every_n_iteration_ == 0 || + iteration == steps - 1) { + ImageFrame image_out(ImageFormat::SRGB, options.output_image_width(), + options.output_image_height()); + RET_CHECK(DiffuserDecode(image_out.MutablePixelData())); + kImageOut(cc).Send(std::move(image_out)); + } else if (emit_empty_packet_) { + kImageOut(cc).Send(Packet()); + } + } + return absl::OkStatus(); +} + +MEDIAPIPE_REGISTER_NODE(StableDiffusionIterateCalculator); + +} // namespace api2 +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/image_generator/diffuser/stable_diffusion_iterate_calculator.proto b/mediapipe/tasks/cc/vision/image_generator/diffuser/stable_diffusion_iterate_calculator.proto new file mode 100644 index 00000000..ce6dcefd --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/diffuser/stable_diffusion_iterate_calculator.proto @@ -0,0 +1,84 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +option java_package = "com.google.mediapipe.calculator.proto"; +option java_outer_classname = "StableDiffusionIterateCalculatorOptionsProto"; + +message StableDiffusionIterateCalculatorOptions { + extend mediapipe.CalculatorOptions { + optional StableDiffusionIterateCalculatorOptions ext = 510855836; + } + + // The random seed that is fed into the calculator to control the randomness + // of the generated image. + optional uint32 base_seed = 1 [default = 0]; + + // The target output image size. Must be a multiple of 8 and larger than 384. + optional int32 output_image_width = 2 [default = 512]; + optional int32 output_image_height = 3 [default = 512]; + + // The folder name must end of '/'. + optional string file_folder = 4 [default = "bins/"]; + + // Note: only one of lora_file_folder and lora_weights_layer_mapping should be + // set. + // The LoRA file folder. The folder name must end of '/'. + optional string lora_file_folder = 9 [default = ""]; + + // The LoRA layer name mapping to the weight buffer position in the file. + map lora_weights_layer_mapping = 10; + + // The LoRA rank. + optional int32 lora_rank = 12 [default = 4]; + + // Determine when to run image decoding for how many every iterations. + // Setting this to 1 means we run the image decoding for every iteration for + // displaying the intermediate result, but it will also introduce much higher + // overall latency. + // Setting this to be the targeted number of iterations will only run the + // image decoding at the end, giving the best overall latency. + optional int32 show_every_n_iteration = 5 [default = 1]; + + // If set to be True, the calculator will perform a GPU-CPU sync and emit an + // empty packet. It is used to provide the signal of which iterations it is + // currently at, typically used to create a progress bar. Note that this also + // introduce overhead, but not significanly based on our experiments (~1ms). + optional bool emit_empty_packet = 6 [default = false]; + + enum ClPriorityHint { + PRIORITY_HINT_NORMAL = 0; // Default, must be first. + PRIORITY_HINT_LOW = 1; + PRIORITY_HINT_HIGH = 2; + } + + // OpenCL priority hint. Set this to LOW to yield to other GPU contexts. + // This lowers inference speed, but helps keeping the UI responsive. + optional ClPriorityHint cl_priority_hint = 7; + + enum ModelType { + DEFAULT = 0; + SD_1 = 1; // Stable Diffusion v1 models, including SD 1.4 and 1.5. + } + // Stable Diffusion model type. Default to Stable Diffusion v1. + optional ModelType model_type = 8 [default = SD_1]; + // The strength of the diffusion plugins inputs. + optional float plugins_strength = 11 [default = 1.0]; +} diff --git a/mediapipe/tasks/cc/vision/image_generator/image_generator.cc b/mediapipe/tasks/cc/vision/image_generator/image_generator.cc new file mode 100644 index 00000000..e4464d84 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/image_generator.cc @@ -0,0 +1,397 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/image_generator/image_generator.h" + +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/timestamp.h" +#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "mediapipe/tasks/cc/vision/core/vision_task_api_factory.h" +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/image_generator_result.h" +#include "mediapipe/tasks/cc/vision/image_generator/proto/conditioned_image_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/proto/control_plugin_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/proto/image_generator_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_segmenter/proto/image_segmenter_graph_options.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace image_generator { +namespace { + +using ImageGeneratorGraphOptionsProto = ::mediapipe::tasks::vision:: + image_generator::proto::ImageGeneratorGraphOptions; +using ConditionedImageGraphOptionsProto = ::mediapipe::tasks::vision:: + image_generator::proto::ConditionedImageGraphOptions; +using ControlPluginGraphOptionsProto = ::mediapipe::tasks::vision:: + image_generator::proto::ControlPluginGraphOptions; +using FaceLandmarkerGraphOptionsProto = ::mediapipe::tasks::vision:: + face_landmarker::proto::FaceLandmarkerGraphOptions; + +constexpr absl::string_view kImageTag = "IMAGE"; +constexpr absl::string_view kImageOutName = "image_out"; +constexpr absl::string_view kConditionImageTag = "CONDITION_IMAGE"; +constexpr absl::string_view kConditionImageName = "condition_image"; +constexpr absl::string_view kSourceConditionImageName = + "source_condition_image"; +constexpr absl::string_view kStepsTag = "STEPS"; +constexpr absl::string_view kStepsName = "steps"; +constexpr absl::string_view kIterationTag = "ITERATION"; +constexpr absl::string_view kIterationName = "iteration"; +constexpr absl::string_view kPromptTag = "PROMPT"; +constexpr absl::string_view kPromptName = "prompt"; +constexpr absl::string_view kRandSeedTag = "RAND_SEED"; +constexpr absl::string_view kRandSeedName = "rand_seed"; +constexpr absl::string_view kSelectTag = "SELECT"; +constexpr absl::string_view kSelectName = "select"; + +constexpr char kImageGeneratorGraphTypeName[] = + "mediapipe.tasks.vision.image_generator.ImageGeneratorGraph"; + +constexpr char kConditionedImageGraphContainerTypeName[] = + "mediapipe.tasks.vision.image_generator.ConditionedImageGraphContainer"; + +// Creates a MediaPipe graph config that contains a subgraph node of +// "mediapipe.tasks.vision.image_generator.ImageGeneratorGraph". +CalculatorGraphConfig CreateImageGeneratorGraphConfig( + std::unique_ptr options, + bool use_condition_image) { + api2::builder::Graph graph; + auto& subgraph = graph.AddNode(kImageGeneratorGraphTypeName); + subgraph.GetOptions().CopyFrom(*options); + graph.In(kStepsTag).SetName(kStepsName) >> subgraph.In(kStepsTag); + graph.In(kIterationTag).SetName(kIterationName) >> subgraph.In(kIterationTag); + graph.In(kPromptTag).SetName(kPromptName) >> subgraph.In(kPromptTag); + graph.In(kRandSeedTag).SetName(kRandSeedName) >> subgraph.In(kRandSeedTag); + if (use_condition_image) { + graph.In(kConditionImageTag).SetName(kConditionImageName) >> + subgraph.In(kConditionImageTag); + graph.In(kSelectTag).SetName(kSelectName) >> subgraph.In(kSelectTag); + } + subgraph.Out(kImageTag).SetName(kImageOutName) >> + graph[api2::Output::Optional(kImageTag)]; + return graph.GetConfig(); +} + +// Creates a MediaPipe graph config that contains a subgraph node of +// "mediapipe.tasks.vision.image_generator.ConditionedImageGraphContainer". +CalculatorGraphConfig CreateConditionedImageGraphContainerConfig( + std::unique_ptr options) { + api2::builder::Graph graph; + auto& subgraph = graph.AddNode(kConditionedImageGraphContainerTypeName); + subgraph.GetOptions().CopyFrom(*options); + graph.In(kImageTag).SetName(kSourceConditionImageName) >> + subgraph.In(kImageTag); + graph.In(kSelectTag).SetName(kSelectName) >> subgraph.In(kSelectTag); + subgraph.Out(kConditionImageTag).SetName(kConditionImageName) >> + graph.Out(kConditionImageTag).Cast(); + return graph.GetConfig(); +} + +absl::Status SetFaceConditionOptionsToProto( + FaceConditionOptions& face_condition_options, + ControlPluginGraphOptionsProto& options_proto) { + // Configure face plugin model. + auto plugin_base_options_proto = + std::make_unique( + tasks::core::ConvertBaseOptionsToProto( + &(face_condition_options.base_options))); + options_proto.mutable_base_options()->Swap(plugin_base_options_proto.get()); + + // Configure face landmarker graph. + auto& face_landmarker_options = + face_condition_options.face_landmarker_options; + auto& face_landmarker_options_proto = + *options_proto.mutable_conditioned_image_graph_options() + ->mutable_face_condition_type_options() + ->mutable_face_landmarker_graph_options(); + + auto base_options_proto = std::make_unique( + tasks::core::ConvertBaseOptionsToProto( + &(face_landmarker_options.base_options))); + face_landmarker_options_proto.mutable_base_options()->Swap( + base_options_proto.get()); + face_landmarker_options_proto.mutable_base_options()->set_use_stream_mode( + false); + + // Configure face detector options. + auto* face_detector_graph_options = + face_landmarker_options_proto.mutable_face_detector_graph_options(); + face_detector_graph_options->set_num_faces(face_landmarker_options.num_faces); + face_detector_graph_options->set_min_detection_confidence( + face_landmarker_options.min_face_detection_confidence); + + // Configure face landmark detector options. + face_landmarker_options_proto.set_min_tracking_confidence( + face_landmarker_options.min_tracking_confidence); + auto* face_landmarks_detector_graph_options = + face_landmarker_options_proto + .mutable_face_landmarks_detector_graph_options(); + face_landmarks_detector_graph_options->set_min_detection_confidence( + face_landmarker_options.min_face_presence_confidence); + return absl::OkStatus(); +} + +absl::Status SetDepthConditionOptionsToProto( + DepthConditionOptions& depth_condition_options, + ControlPluginGraphOptionsProto& options_proto) { + // Configure face plugin model. + auto plugin_base_options_proto = + std::make_unique( + tasks::core::ConvertBaseOptionsToProto( + &(depth_condition_options.base_options))); + options_proto.mutable_base_options()->Swap(plugin_base_options_proto.get()); + + auto& image_segmenter_graph_options = + *options_proto.mutable_conditioned_image_graph_options() + ->mutable_depth_condition_type_options() + ->mutable_image_segmenter_graph_options(); + + auto depth_base_options_proto = + std::make_unique( + tasks::core::ConvertBaseOptionsToProto( + &(depth_condition_options.image_segmenter_options.base_options))); + image_segmenter_graph_options.mutable_base_options()->Swap( + depth_base_options_proto.get()); + image_segmenter_graph_options.mutable_base_options()->set_use_stream_mode( + false); + image_segmenter_graph_options.set_display_names_locale( + depth_condition_options.image_segmenter_options.display_names_locale); + return absl::OkStatus(); +} + +absl::Status SetEdgeConditionOptionsToProto( + EdgeConditionOptions& edge_condition_options, + ControlPluginGraphOptionsProto& options_proto) { + auto plugin_base_options_proto = + std::make_unique( + tasks::core::ConvertBaseOptionsToProto( + &(edge_condition_options.base_options))); + options_proto.mutable_base_options()->Swap(plugin_base_options_proto.get()); + + auto& edge_options_proto = + *options_proto.mutable_conditioned_image_graph_options() + ->mutable_edge_condition_type_options(); + edge_options_proto.set_threshold_1(edge_condition_options.threshold_1); + edge_options_proto.set_threshold_2(edge_condition_options.threshold_2); + edge_options_proto.set_aperture_size(edge_condition_options.aperture_size); + edge_options_proto.set_l2_gradient(edge_condition_options.l2_gradient); + return absl::OkStatus(); +} + +// Helper holder struct of image generator graph options and condition type +// index mapping. +struct ImageGeneratorOptionsProtoAndConditionTypeIndex { + std::unique_ptr options_proto; + std::unique_ptr> + condition_type_index; +}; + +// Converts the user-facing ImageGeneratorOptions struct to the internal +// ImageGeneratorOptions proto. +absl::StatusOr +ConvertImageGeneratorGraphOptionsProto( + ImageGeneratorOptions* image_generator_options, + ConditionOptions* condition_options) { + ImageGeneratorOptionsProtoAndConditionTypeIndex + options_proto_and_condition_index; + + // Configure base image generator options. + options_proto_and_condition_index.options_proto = + std::make_unique(); + auto& options_proto = *options_proto_and_condition_index.options_proto; + options_proto.set_text2image_model_directory( + image_generator_options->text2image_model_directory); + if (image_generator_options->lora_weights_file_path.has_value()) { + options_proto.mutable_lora_weights_file()->set_file_name( + *image_generator_options->lora_weights_file_path); + } + + // Configure optional condition type options. + if (condition_options != nullptr) { + options_proto_and_condition_index.condition_type_index = + std::make_unique>(); + auto& condition_type_index = + *options_proto_and_condition_index.condition_type_index; + if (condition_options->face_condition_options.has_value()) { + condition_type_index[ConditionOptions::FACE] = + condition_type_index.size(); + auto& face_plugin_graph_options = + *options_proto.add_control_plugin_graphs_options(); + RET_CHECK_OK(SetFaceConditionOptionsToProto( + *condition_options->face_condition_options, + face_plugin_graph_options)); + } + if (condition_options->depth_condition_options.has_value()) { + condition_type_index[ConditionOptions::DEPTH] = + condition_type_index.size(); + auto& depth_plugin_graph_options = + *options_proto.add_control_plugin_graphs_options(); + RET_CHECK_OK(SetDepthConditionOptionsToProto( + *condition_options->depth_condition_options, + depth_plugin_graph_options)); + } + if (condition_options->edge_condition_options.has_value()) { + condition_type_index[ConditionOptions::EDGE] = + condition_type_index.size(); + auto& edge_plugin_graph_options = + *options_proto.add_control_plugin_graphs_options(); + RET_CHECK_OK(SetEdgeConditionOptionsToProto( + *condition_options->edge_condition_options, + edge_plugin_graph_options)); + } + if (condition_type_index.empty()) { + return absl::InvalidArgumentError( + "At least one condition type must be set."); + } + } + return options_proto_and_condition_index; +} + +} // namespace + +absl::StatusOr> ImageGenerator::Create( + std::unique_ptr image_generator_options, + std::unique_ptr condition_options) { + bool use_condition_image = condition_options != nullptr; + ASSIGN_OR_RETURN(auto options_proto_and_condition_index, + ConvertImageGeneratorGraphOptionsProto( + image_generator_options.get(), condition_options.get())); + std::unique_ptr + options_proto_for_condition_image_graphs_container; + if (use_condition_image) { + options_proto_for_condition_image_graphs_container = + std::make_unique(); + options_proto_for_condition_image_graphs_container->CopyFrom( + *options_proto_and_condition_index.options_proto); + } + ASSIGN_OR_RETURN( + auto image_generator, + (core::VisionTaskApiFactory::Create( + CreateImageGeneratorGraphConfig( + std::move(options_proto_and_condition_index.options_proto), + use_condition_image), + std::make_unique(), + core::RunningMode::IMAGE, + /*result_callback=*/nullptr))); + image_generator->use_condition_image_ = use_condition_image; + if (use_condition_image) { + image_generator->condition_type_index_ = + std::move(options_proto_and_condition_index.condition_type_index); + ASSIGN_OR_RETURN( + image_generator->condition_image_graphs_container_task_runner_, + tasks::core::TaskRunner::Create( + CreateConditionedImageGraphContainerConfig( + std::move(options_proto_for_condition_image_graphs_container)), + absl::make_unique())); + } + image_generator->init_timestamp_ = absl::Now(); + return image_generator; +} + +absl::StatusOr ImageGenerator::CreateConditionImage( + Image source_condition_image, + ConditionOptions::ConditionType condition_type) { + if (condition_type_index_->find(condition_type) == + condition_type_index_->end()) { + return absl::InvalidArgumentError( + "The condition type is not created during initialization."); + } + ASSIGN_OR_RETURN( + auto output_packets, + condition_image_graphs_container_task_runner_->Process({ + {std::string(kSourceConditionImageName), + MakePacket(std::move(source_condition_image))}, + {std::string(kSelectName), + MakePacket(condition_type_index_->at(condition_type))}, + })); + return output_packets.at(std::string(kConditionImageName)).Get(); +} + +absl::StatusOr ImageGenerator::Generate( + const std::string& prompt, int iterations, int seed) { + if (use_condition_image_) { + return absl::InvalidArgumentError( + "ImageGenerator is created to use with conditioned image."); + } + return RunIterations(prompt, iterations, seed, std::nullopt); +} + +absl::StatusOr ImageGenerator::Generate( + const std::string& prompt, Image condition_image, + ConditionOptions::ConditionType condition_type, int iterations, int seed) { + if (!use_condition_image_) { + return absl::InvalidArgumentError( + "ImageGenerator is created to use without conditioned image."); + } + ASSIGN_OR_RETURN(auto plugin_model_image, + CreateConditionImage(condition_image, condition_type)); + return RunIterations( + prompt, iterations, seed, + ConditionInputs{plugin_model_image, + condition_type_index_->at(condition_type)}); +} + +absl::StatusOr ImageGenerator::RunIterations( + const std::string& prompt, int steps, int rand_seed, + std::optional condition_inputs) { + tasks::core::PacketMap output_packets; + ImageGeneratorResult result; + auto timestamp = (absl::Now() - init_timestamp_) / absl::Milliseconds(1); + for (int i = 0; i < steps; ++i) { + tasks::core::PacketMap input_packets; + if (i == 0 && condition_inputs.has_value()) { + input_packets[std::string(kConditionImageName)] = + MakePacket(condition_inputs->condition_image) + .At(Timestamp(timestamp)); + input_packets[std::string(kSelectName)] = + MakePacket(condition_inputs->select).At(Timestamp(timestamp)); + } + input_packets[std::string(kStepsName)] = + MakePacket(steps).At(Timestamp(timestamp)); + input_packets[std::string(kIterationName)] = + MakePacket(i).At(Timestamp(timestamp)); + input_packets[std::string(kPromptName)] = + MakePacket(prompt).At(Timestamp(timestamp)); + input_packets[std::string(kRandSeedName)] = + MakePacket(rand_seed).At(Timestamp(timestamp)); + ASSIGN_OR_RETURN(output_packets, ProcessImageData(input_packets)); + timestamp += 1; + } + result.generated_image = + output_packets.at(std::string(kImageOutName)).Get(); + if (condition_inputs.has_value()) { + result.condition_image = condition_inputs->condition_image; + } + return result; +} + +} // namespace image_generator +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/image_generator/image_generator.h b/mediapipe/tasks/cc/vision/image_generator/image_generator.h new file mode 100644 index 00000000..52599c02 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/image_generator.h @@ -0,0 +1,157 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_H_ +#define MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/tasks/cc/core/base_options.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "mediapipe/tasks/cc/vision/core/base_vision_task_api.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker.h" +#include "mediapipe/tasks/cc/vision/image_generator/image_generator_result.h" +#include "mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace image_generator { + +// Options for drawing face landmarks image. +struct FaceConditionOptions { + // The base options for plugin model. + tasks::core::BaseOptions base_options; + + // Face landmarker options used to detect face landmarks in the condition + // image. + face_landmarker::FaceLandmarkerOptions face_landmarker_options; +}; + +// Options for detecting edges image. +struct EdgeConditionOptions { + // The base options for plugin model. + tasks::core::BaseOptions base_options; + + // These parameters are used to config Canny edge algorithm of OpenCV. + // See more details: + // https://docs.opencv.org/3.4/dd/d1a/group__imgproc__feature.html#ga04723e007ed888ddf11d9ba04e2232de + + // First threshold for the hysteresis procedure. + float threshold_1 = 100; + + // Second threshold for the hysteresis procedure. + float threshold_2 = 200; + + // Aperture size for the Sobel operator. Typical range is 3~7. + int aperture_size = 3; + + // A flag, indicating whether a more accurate L2 norm should be used to + // calculate the image gradient magnitude ( L2gradient=true ), or whether + // the default L1 norm is enough ( L2gradient=false ). + bool l2_gradient = false; +}; + +// Options for detecting depth image. +struct DepthConditionOptions { + // The base options for plugin model. + tasks::core::BaseOptions base_options; + + // Image segmenter options used to detect depth in the condition image. + image_segmenter::ImageSegmenterOptions image_segmenter_options; +}; + +struct ConditionOptions { + enum ConditionType { FACE, EDGE, DEPTH }; + std::optional face_condition_options; + std::optional edge_condition_options; + std::optional depth_condition_options; +}; + +// Note: The API is experimental and subject to change. +// The options for configuring a mediapipe image generator task. +struct ImageGeneratorOptions { + // The text to image model directory storing the model weights. + std::string text2image_model_directory; + + // The path to LoRA weights file. + std::optional lora_weights_file_path; +}; + +class ImageGenerator : tasks::vision::core::BaseVisionTaskApi { + public: + using BaseVisionTaskApi::BaseVisionTaskApi; + + // Creates an ImageGenerator from the provided options. + // image_generator_options: options to create the image generator. + // condition_options: optional options if plugin models are used to generate + // an image based on the condition image. + static absl::StatusOr> Create( + std::unique_ptr image_generator_options, + std::unique_ptr condition_options = nullptr); + + // Create the condition image of specified condition type from the source + // condition image. Currently support face landmarks, depth image and edge + // image as the condition image. + absl::StatusOr CreateConditionImage( + Image source_condition_image, + ConditionOptions::ConditionType condition_type); + + // Generates an image for iterations and the given random seed. Only valid + // when the ImageGenerator is created without condition options. + absl::StatusOr Generate(const std::string& prompt, + int iterations, int seed = 0); + + // Generates an image based on the condition image for iterations and the + // given random seed. + // A detailed introduction to the condition image: + // https://ai.googleblog.com/2023/06/on-device-diffusion-plugins-for.html + absl::StatusOr Generate( + const std::string& prompt, Image condition_image, + ConditionOptions::ConditionType condition_type, int iterations, + int seed = 0); + + private: + struct ConditionInputs { + Image condition_image; + int select; + }; + + bool use_condition_image_ = false; + + absl::Time init_timestamp_; + + std::unique_ptr + condition_image_graphs_container_task_runner_; + + std::unique_ptr> + condition_type_index_; + + absl::StatusOr RunIterations( + const std::string& prompt, int steps, int rand_seed, + std::optional condition_inputs); +}; + +} // namespace image_generator +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_H_ diff --git a/mediapipe/tasks/cc/vision/image_generator/image_generator_graph.cc b/mediapipe/tasks/cc/vision/image_generator/image_generator_graph.cc new file mode 100644 index 00000000..639a73e3 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/image_generator_graph.cc @@ -0,0 +1,361 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h" +#include "mediapipe/calculators/tensor/inference_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/framework/tool/switch_container.pb.h" +#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" +#include "mediapipe/tasks/cc/core/model_resources.h" +#include "mediapipe/tasks/cc/core/model_task_graph.h" +#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/diffuser/stable_diffusion_iterate_calculator.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/proto/conditioned_image_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/proto/control_plugin_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/image_generator/proto/image_generator_graph_options.pb.h" +#include "mediapipe/util/graph_builder_utils.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace image_generator { + +namespace { + +using ::mediapipe::api2::Input; +using ::mediapipe::api2::Output; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Source; + +constexpr int kPluginsOutputSize = 512; +constexpr absl::string_view kTensorsTag = "TENSORS"; +constexpr absl::string_view kImageTag = "IMAGE"; +constexpr absl::string_view kImageCpuTag = "IMAGE_CPU"; +constexpr absl::string_view kStepsTag = "STEPS"; +constexpr absl::string_view kIterationTag = "ITERATION"; +constexpr absl::string_view kPromptTag = "PROMPT"; +constexpr absl::string_view kRandSeedTag = "RAND_SEED"; +constexpr absl::string_view kPluginTensorsTag = "PLUGIN_TENSORS"; +constexpr absl::string_view kConditionImageTag = "CONDITION_IMAGE"; +constexpr absl::string_view kSelectTag = "SELECT"; +constexpr absl::string_view kMetadataFilename = "metadata"; +constexpr absl::string_view kLoraRankStr = "lora_rank"; + +struct ImageGeneratorInputs { + Source prompt; + Source steps; + Source iteration; + Source rand_seed; + std::optional> condition_image; + std::optional> select_condition_type; +}; + +struct ImageGeneratorOutputs { + Source generated_image; +}; + +} // namespace + +// A container graph containing several ConditionedImageGraph from which to +// choose specified condition type. +// Inputs: +// IMAGE - Image +// The source condition image, used to generate the condition image. +// SELECT - int +// The index of the selected conditioned image graph. +// Outputs: +// CONDITION_IMAGE - Image +// The condition image created from the specified condition type. +class ConditionedImageGraphContainer : public core::ModelTaskGraph { + public: + absl::StatusOr GetConfig( + SubgraphContext* sc) override { + Graph graph; + auto& graph_options = + *sc->MutableOptions(); + auto source_condition_image = graph.In(kImageTag).Cast(); + auto select_condition_type = graph.In(kSelectTag).Cast(); + auto& switch_container = graph.AddNode("SwitchContainer"); + auto& switch_options = + switch_container.GetOptions(); + for (auto& control_plugin_graph_options : + *graph_options.mutable_control_plugin_graphs_options()) { + auto& node = *switch_options.add_contained_node(); + node.set_calculator( + "mediapipe.tasks.vision.image_generator.ConditionedImageGraph"); + node.mutable_node_options()->Add()->PackFrom( + control_plugin_graph_options.conditioned_image_graph_options()); + } + source_condition_image >> switch_container.In(kImageTag); + select_condition_type >> switch_container.In(kSelectTag); + auto condition_image = switch_container.Out(kImageTag).Cast(); + condition_image >> graph.Out(kConditionImageTag); + return graph.GetConfig(); + } +}; + +// clang-format off +REGISTER_MEDIAPIPE_GRAPH( + ::mediapipe::tasks::vision::image_generator::ConditionedImageGraphContainer); // NOLINT +// clang-format on + +// A helper graph to convert condition image to Tensor using the control plugin +// model. +// Inputs: +// CONDITION_IMAGE - Image +// The condition image input to the control plugin model. +// Outputs: +// PLUGIN_TENSORS - std::vector +// The output tensors from the control plugin model. The tensors are used as +// inputs to the image generation model. +class ControlPluginGraph : public core::ModelTaskGraph { + public: + absl::StatusOr GetConfig( + SubgraphContext* sc) override { + Graph graph; + auto& graph_options = + *sc->MutableOptions(); + + auto condition_image = graph.In(kConditionImageTag).Cast(); + + // Convert Image to ImageFrame. + auto& from_image = graph.AddNode("FromImageCalculator"); + condition_image >> from_image.In(kImageTag); + auto image_frame = from_image.Out(kImageCpuTag); + + // Convert ImageFrame to Tensor. + auto& image_to_tensor = graph.AddNode("ImageToTensorCalculator"); + auto& image_to_tensor_options = + image_to_tensor.GetOptions(); + image_to_tensor_options.set_output_tensor_width(kPluginsOutputSize); + image_to_tensor_options.set_output_tensor_height(kPluginsOutputSize); + image_to_tensor_options.mutable_output_tensor_float_range()->set_min(-1); + image_to_tensor_options.mutable_output_tensor_float_range()->set_max(1); + image_to_tensor_options.set_keep_aspect_ratio(true); + image_frame >> image_to_tensor.In(kImageTag); + + // Create the plugin model resource. + ASSIGN_OR_RETURN( + const core::ModelResources* plugin_model_resources, + CreateModelResources( + sc, + std::make_unique( + *graph_options.mutable_base_options()->mutable_model_asset()))); + + // Add control plugin model inference. + auto& plugins_inference = + AddInference(*plugin_model_resources, + graph_options.base_options().acceleration(), graph); + image_to_tensor.Out(kTensorsTag) >> plugins_inference.In(kTensorsTag); + // The plugins model is not runnable on OpenGL. Error message: + // TfLiteGpuDelegate Prepare: Batch size mismatch, expected 1 but got 64 + // Node number 67 (TfLiteGpuDelegate) failed to prepare. + plugins_inference.GetOptions() + .mutable_delegate() + ->mutable_xnnpack(); + plugins_inference.Out(kTensorsTag).Cast>() >> + graph.Out(kPluginTensorsTag); + return graph.GetConfig(); + } +}; + +REGISTER_MEDIAPIPE_GRAPH( + ::mediapipe::tasks::vision::image_generator::ControlPluginGraph); + +// A "mediapipe.tasks.vision.image_generator.ImageGeneratorGraph" performs image +// generation from a text prompt, and a optional condition image. +// +// Inputs: +// PROMPT - std::string +// The prompt describing the image to be generated. +// STEPS - int +// The total steps to generate the image. +// ITERATION - int +// The current iteration in the generating steps. Must be less than STEPS. +// RAND_SEED - int +// The randaom seed input to the image generation model. +// CONDITION_IMAGE - Image +// The condition image used as a guidance for the image generation. Only +// valid, if condtrol plugin graph options are set in the graph options. +// SELECT - int +// The index of the selected the control plugin graph. +// +// Outputs: +// IMAGE - Image +// The generated image. +// STEPS - int @optional +// The total steps to generate the image. The same as STEPS input. +// ITERATION - int @optional +// The current iteration in the generating steps. The same as ITERATION +// input. +class ImageGeneratorGraph : public core::ModelTaskGraph { + public: + absl::StatusOr GetConfig( + SubgraphContext* sc) override { + Graph graph; + auto* subgraph_options = + sc->MutableOptions(); + std::optional lora_resources; + // Create LoRA weights asset bundle resources. + if (subgraph_options->has_lora_weights_file()) { + auto external_file = std::make_unique(); + external_file->Swap(subgraph_options->mutable_lora_weights_file()); + ASSIGN_OR_RETURN(lora_resources, CreateModelAssetBundleResources( + sc, std::move(external_file))); + } + std::optional> condition_image; + std::optional> select_condition_type; + if (!subgraph_options->control_plugin_graphs_options().empty()) { + condition_image = graph.In(kConditionImageTag).Cast(); + select_condition_type = graph.In(kSelectTag).Cast(); + } + ASSIGN_OR_RETURN( + auto outputs, + BuildImageGeneratorGraph( + *sc->MutableOptions(), + lora_resources, + ImageGeneratorInputs{ + /*prompt=*/graph.In(kPromptTag).Cast(), + /*steps=*/graph.In(kStepsTag).Cast(), + /*iteration=*/graph.In(kIterationTag).Cast(), + /*rand_seed=*/graph.In(kRandSeedTag).Cast(), + /*condition_image*/ condition_image, + /*select_condition_type*/ select_condition_type, + }, + graph)); + outputs.generated_image >> graph.Out(kImageTag).Cast(); + + // Optional outputs to provide the current iteration. + auto& pass_through = graph.AddNode("PassThroughCalculator"); + graph.In(kIterationTag) >> pass_through.In(0); + graph.In(kStepsTag) >> pass_through.In(1); + pass_through.Out(0) >> graph[Output::Optional(kIterationTag)]; + pass_through.Out(1) >> graph[Output::Optional(kStepsTag)]; + return graph.GetConfig(); + } + + absl::StatusOr BuildImageGeneratorGraph( + proto::ImageGeneratorGraphOptions& subgraph_options, + std::optional lora_resources, + ImageGeneratorInputs inputs, Graph& graph) { + auto& stable_diff = graph.AddNode("StableDiffusionIterateCalculator"); + if (inputs.condition_image.has_value()) { + // Add switch container for multiple control plugin graphs. + auto& switch_container = graph.AddNode("SwitchContainer"); + auto& switch_options = + switch_container.GetOptions(); + for (auto& control_plugin_graph_options : + *subgraph_options.mutable_control_plugin_graphs_options()) { + auto& node = *switch_options.add_contained_node(); + node.set_calculator( + "mediapipe.tasks.vision.image_generator.ControlPluginGraph"); + node.mutable_node_options()->Add()->PackFrom( + control_plugin_graph_options); + } + *inputs.condition_image >> switch_container.In(kConditionImageTag); + *inputs.select_condition_type >> switch_container.In(kSelectTag); + auto plugin_tensors = switch_container.Out(kPluginTensorsTag); + + // Additional diffusion plugins calculator to pass tensors to diffusion + // iterator. + auto& plugins_output = graph.AddNode("DiffusionPluginsOutputCalculator"); + plugin_tensors >> plugins_output.In(kTensorsTag); + inputs.steps >> plugins_output.In(kStepsTag); + inputs.iteration >> plugins_output.In(kIterationTag); + plugins_output.Out(kTensorsTag) >> stable_diff.In(kPluginTensorsTag); + } + + inputs.prompt >> stable_diff.In(kPromptTag); + inputs.steps >> stable_diff.In(kStepsTag); + inputs.iteration >> stable_diff.In(kIterationTag); + inputs.rand_seed >> stable_diff.In(kRandSeedTag); + mediapipe::StableDiffusionIterateCalculatorOptions& options = + stable_diff + .GetOptions(); + options.set_base_seed(0); + options.set_output_image_height(kPluginsOutputSize); + options.set_output_image_width(kPluginsOutputSize); + options.set_file_folder(subgraph_options.text2image_model_directory()); + options.set_show_every_n_iteration(100); + options.set_emit_empty_packet(true); + if (lora_resources.has_value()) { + auto& lora_layer_weights_mapping = + *options.mutable_lora_weights_layer_mapping(); + for (const auto& file_path : (*lora_resources)->ListFiles()) { + auto basename = file::Basename(file_path); + ASSIGN_OR_RETURN(auto file_content, + (*lora_resources)->GetFile(std::string(file_path))); + if (file_path == kMetadataFilename) { + MP_RETURN_IF_ERROR( + ParseLoraMetadataAndConfigOptions(file_content, options)); + } else { + lora_layer_weights_mapping[basename] = + reinterpret_cast(file_content.data()); + } + } + } + + auto& to_image = graph.AddNode("ToImageCalculator"); + stable_diff.Out(kImageTag) >> to_image.In(kImageCpuTag); + + return {{to_image.Out(kImageTag).Cast()}}; + } + + private: + absl::Status ParseLoraMetadataAndConfigOptions( + absl::string_view contents, + mediapipe::StableDiffusionIterateCalculatorOptions& options) { + std::vector lines = + absl::StrSplit(contents, '\n', absl::SkipEmpty()); + for (const auto& line : lines) { + std::vector values = absl::StrSplit(line, ','); + if (values[0] == kLoraRankStr) { + int lora_rank; + if (values.size() != 2 || !absl::SimpleAtoi(values[1], &lora_rank)) { + return absl::InvalidArgumentError( + absl::StrCat("Error parsing LoRA weights metadata. ", line)); + } + options.set_lora_rank(lora_rank); + } + } + return absl::OkStatus(); + } +}; + +REGISTER_MEDIAPIPE_GRAPH( + ::mediapipe::tasks::vision::image_generator::ImageGeneratorGraph); + +} // namespace image_generator +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/image_generator/image_generator_result.h b/mediapipe/tasks/cc/vision/image_generator/image_generator_result.h new file mode 100644 index 00000000..7b7054d7 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/image_generator_result.h @@ -0,0 +1,41 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_RESULT_H_ +#define MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_RESULT_H_ + +#include "mediapipe/framework/formats/image.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace image_generator { + +// The result of ImageGenerator task. +struct ImageGeneratorResult { + // The generated image. + Image generated_image; + + // The condition_image used in the plugin model, only available if the + // condition type is set in ImageGeneratorOptions. + std::optional condition_image = std::nullopt; +}; + +} // namespace image_generator +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_IMAGE_GENERATOR_IMAGE_GENERATOR_RESULT_H_ diff --git a/mediapipe/tasks/cc/vision/image_generator/proto/BUILD b/mediapipe/tasks/cc/vision/image_generator/proto/BUILD new file mode 100644 index 00000000..38e1048c --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/proto/BUILD @@ -0,0 +1,52 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") + +package(default_visibility = [ + "//mediapipe/tasks:internal", +]) + +licenses(["notice"]) + +mediapipe_proto_library( + name = "conditioned_image_graph_options_proto", + srcs = ["conditioned_image_graph_options.proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_proto", + "//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_graph_options_proto", + ], +) + +mediapipe_proto_library( + name = "control_plugin_graph_options_proto", + srcs = ["control_plugin_graph_options.proto"], + deps = [ + ":conditioned_image_graph_options_proto", + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + "//mediapipe/tasks/cc/core/proto:base_options_proto", + ], +) + +mediapipe_proto_library( + name = "image_generator_graph_options_proto", + srcs = ["image_generator_graph_options.proto"], + deps = [ + ":control_plugin_graph_options_proto", + "//mediapipe/tasks/cc/core/proto:external_file_proto", + ], +) diff --git a/mediapipe/tasks/cc/vision/image_generator/proto/conditioned_image_graph_options.proto b/mediapipe/tasks/cc/vision/image_generator/proto/conditioned_image_graph_options.proto new file mode 100644 index 00000000..8d0798d7 --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/proto/conditioned_image_graph_options.proto @@ -0,0 +1,66 @@ + +/* Copyright 2023 The MediaPipe Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +syntax = "proto3"; + +package mediapipe.tasks.vision.image_generator.proto; + +import "mediapipe/framework/calculator.proto"; +import "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.proto"; +import "mediapipe/tasks/cc/vision/image_segmenter/proto/image_segmenter_graph_options.proto"; + +option java_package = "com.google.mediapipe.tasks.vision.imagegenerator.proto"; +option java_outer_classname = "ConditionedImageGraphOptionsProto"; + +message ConditionedImageGraphOptions { + // For conditioned image graph based on face landmarks. + message FaceConditionTypeOptions { + // Options for the face landmarker used in the face landmarks type graph. + face_landmarker.proto.FaceLandmarkerGraphOptions + face_landmarker_graph_options = 1; + } + + // For conditioned image graph base on edges detection. + message EdgeConditionTypeOptions { + // These parameters are used to config Canny edge algorithm of OpenCV. + // See more details: + // https://docs.opencv.org/3.4/dd/d1a/group__imgproc__feature.html#ga04723e007ed888ddf11d9ba04e2232de + + // First threshold for the hysteresis procedure. + float threshold_1 = 1; + + // Second threshold for the hysteresis procedure. + float threshold_2 = 2; + + // Aperture size for the Sobel operator. Typical range is 3~7. + int32 aperture_size = 3; + + // A flag, indicating whether a more accurate L2 norm should be used to + // calculate the image gradient magnitude ( L2gradient=true ), or whether + // the default L1 norm is enough ( L2gradient=false ). + bool l2_gradient = 4; + } + + // For conditioned image graph base on depth map. + message DepthConditionTypeOptions { + // Options for the image segmenter used in the depth condition type graph. + image_segmenter.proto.ImageSegmenterGraphOptions + image_segmenter_graph_options = 1; + } + + // The options for configuring the conditioned image graph. + oneof condition_type_options { + FaceConditionTypeOptions face_condition_type_options = 2; + EdgeConditionTypeOptions edge_condition_type_options = 3; + DepthConditionTypeOptions depth_condition_type_options = 4; + } +} diff --git a/mediapipe/tasks/cc/vision/image_generator/proto/control_plugin_graph_options.proto b/mediapipe/tasks/cc/vision/image_generator/proto/control_plugin_graph_options.proto new file mode 100644 index 00000000..52d94efb --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/proto/control_plugin_graph_options.proto @@ -0,0 +1,34 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto3"; + +package mediapipe.tasks.vision.image_generator.proto; + +import "mediapipe/framework/calculator.proto"; +import "mediapipe/tasks/cc/core/proto/base_options.proto"; +import "mediapipe/tasks/cc/vision/image_generator/proto/conditioned_image_graph_options.proto"; + +option java_package = "com.google.mediapipe.tasks.vision.imagegenerator.proto"; +option java_outer_classname = "ControlPluginGraphOptionsProto"; + +message ControlPluginGraphOptions { + // The base options for the control plugin model. + core.proto.BaseOptions base_options = 1; + + // The options for the ConditionedImageGraphOptions to generate control plugin + // model input image. + proto.ConditionedImageGraphOptions conditioned_image_graph_options = 2; +} diff --git a/mediapipe/tasks/cc/vision/image_generator/proto/image_generator_graph_options.proto b/mediapipe/tasks/cc/vision/image_generator/proto/image_generator_graph_options.proto new file mode 100644 index 00000000..867080dc --- /dev/null +++ b/mediapipe/tasks/cc/vision/image_generator/proto/image_generator_graph_options.proto @@ -0,0 +1,35 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto3"; + +package mediapipe.tasks.vision.image_generator.proto; + +import "mediapipe/tasks/cc/core/proto/external_file.proto"; +import "mediapipe/tasks/cc/vision/image_generator/proto/control_plugin_graph_options.proto"; + +option java_package = "com.google.mediapipe.tasks.vision.imagegenerator.proto"; +option java_outer_classname = "ImageGeneratorGraphOptionsProto"; + +message ImageGeneratorGraphOptions { + // The directory containing the models weight of the text to image model. + string text2image_model_directory = 1; + + // An optional LoRA weights file. If set, the diffusion model will be created + // with LoRA weights. + core.proto.ExternalFile lora_weights_file = 2; + + repeated proto.ControlPluginGraphOptions control_plugin_graphs_options = 3; +} diff --git a/mediapipe/tasks/cc/vision/image_segmenter/BUILD b/mediapipe/tasks/cc/vision/image_segmenter/BUILD index fc977c0b..fa67d9af 100644 --- a/mediapipe/tasks/cc/vision/image_segmenter/BUILD +++ b/mediapipe/tasks/cc/vision/image_segmenter/BUILD @@ -37,6 +37,7 @@ cc_library( "//mediapipe/framework/api2:builder", "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:status", "//mediapipe/tasks/cc/core:base_options", "//mediapipe/tasks/cc/core:utils", "//mediapipe/tasks/cc/vision/core:base_vision_task_api", @@ -95,6 +96,7 @@ cc_library( "//mediapipe/util:graph_builder_utils", "//mediapipe/util:label_map_cc_proto", "//mediapipe/util:label_map_util", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings:str_format", diff --git a/mediapipe/tasks/cc/vision/image_segmenter/calculators/segmentation_postprocessor_gl.cc b/mediapipe/tasks/cc/vision/image_segmenter/calculators/segmentation_postprocessor_gl.cc index 3c086183..b1791fc0 100644 --- a/mediapipe/tasks/cc/vision/image_segmenter/calculators/segmentation_postprocessor_gl.cc +++ b/mediapipe/tasks/cc/vision/image_segmenter/calculators/segmentation_postprocessor_gl.cc @@ -5,6 +5,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/strings/str_format.h" #include "mediapipe/framework/port/status_macros.h" @@ -367,19 +368,20 @@ absl::Status SegmentationPostprocessorGl::GlInit( // TODO: We could skip this entirely if no confidence masks // are being produced AND num_classes > 1, but num_classes is only // known at runtime, so this would take a little extra refactoring. - LOG(INFO) << "SIGMOID activation function chosen on GPU"; + ABSL_LOG(INFO) << "SIGMOID activation function chosen on GPU"; activation_fn = "vec4 out_value = 1.0 / (exp(-in_value) + 1.0);"; break; case SegmenterOptions::SOFTMAX: if (produce_confidence_masks) { - LOG(INFO) << "SOFTMAX activation function chosen on GPU"; + ABSL_LOG(INFO) << "SOFTMAX activation function chosen on GPU"; } else { - LOG(INFO) << "SOFTMAX activation function chosen on GPU, but only " - << "category mask produced, so not applying."; + ABSL_LOG(INFO) + << "SOFTMAX activation function chosen on GPU, but only " + << "category mask produced, so not applying."; } break; case SegmenterOptions::NONE: - LOG(INFO) << "NONE activation function chosen on GPU"; + ABSL_LOG(INFO) << "NONE activation function chosen on GPU"; break; } @@ -490,7 +492,7 @@ SegmentationPostprocessorGl::GetSegmentationResultGpu( int input_width, input_height; if (!tensor.ready_on_gpu()) { - LOG(WARNING) << "Tensor wasn't ready on GPU; using slow workaround."; + ABSL_LOG(WARNING) << "Tensor wasn't ready on GPU; using slow workaround."; (void)tensor.GetCpuReadView(); } @@ -507,7 +509,7 @@ SegmentationPostprocessorGl::GetSegmentationResultGpu( const auto layout = tensor.GetOpenGlTexture2dReadView().GetLayoutDimensions( tensor.shape(), &input_width, &input_height); if (layout != Tensor::OpenGlTexture2dView::Layout::kAligned) { - LOG(ERROR) << "Tensor layout not kAligned! Cannot handle."; + ABSL_LOG(ERROR) << "Tensor layout not kAligned! Cannot handle."; } #endif // TASK_SEGMENTATION_USE_GLES_31_POSTPROCESSING @@ -853,7 +855,7 @@ SegmentationPostprocessorGl::GetSegmentationResultGpu( }); if (!status.ok()) { - LOG(ERROR) << "Error with rendering: " << status; + ABSL_LOG(ERROR) << "Error with rendering: " << status; } return image_outputs; diff --git a/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.cc b/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.cc index 99faa106..74d8047d 100644 --- a/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.cc +++ b/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.cc @@ -16,12 +16,14 @@ limitations under the License. #include "mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.h" #include +#include #include "absl/strings/str_format.h" #include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/tasks/cc/core/utils.h" #include "mediapipe/tasks/cc/vision/core/image_processing_options.h" #include "mediapipe/tasks/cc/vision/core/running_mode.h" @@ -41,6 +43,8 @@ constexpr char kConfidenceMasksTag[] = "CONFIDENCE_MASKS"; constexpr char kConfidenceMasksStreamName[] = "confidence_masks"; constexpr char kCategoryMaskTag[] = "CATEGORY_MASK"; constexpr char kCategoryMaskStreamName[] = "category_mask"; +constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; +constexpr char kOutputSizeStreamName[] = "output_size"; constexpr char kImageInStreamName[] = "image_in"; constexpr char kImageOutStreamName[] = "image_out"; constexpr char kImageTag[] = "IMAGE"; @@ -70,6 +74,7 @@ CalculatorGraphConfig CreateGraphConfig( options.get()); graph.In(kImageTag).SetName(kImageInStreamName); graph.In(kNormRectTag).SetName(kNormRectStreamName); + graph.In(kOutputSizeTag).SetName(kOutputSizeStreamName); if (output_confidence_masks) { task_subgraph.Out(kConfidenceMasksTag) .SetName(kConfidenceMasksStreamName) >> @@ -85,10 +90,12 @@ CalculatorGraphConfig CreateGraphConfig( graph.Out(kImageTag); if (enable_flow_limiting) { return tasks::core::AddFlowLimiterCalculator( - graph, task_subgraph, {kImageTag, kNormRectTag}, kConfidenceMasksTag); + graph, task_subgraph, {kImageTag, kNormRectTag, kOutputSizeTag}, + kConfidenceMasksTag); } graph.In(kImageTag) >> task_subgraph.In(kImageTag); graph.In(kNormRectTag) >> task_subgraph.In(kNormRectTag); + graph.In(kOutputSizeTag) >> task_subgraph.In(kOutputSizeTag); return graph.GetConfig(); } @@ -211,6 +218,16 @@ absl::StatusOr> ImageSegmenter::Create( absl::StatusOr ImageSegmenter::Segment( mediapipe::Image image, std::optional image_processing_options) { + return Segment(image, { + /*output_width=*/image.width(), + /*output_height=*/image.height(), + std::move(image_processing_options), + }); +} + +absl::StatusOr ImageSegmenter::Segment( + mediapipe::Image image, SegmentationOptions segmentation_options) { + MP_RETURN_IF_ERROR(ValidateSegmentationOptions(segmentation_options)); if (image.UsesGpu()) { return CreateStatusWithPayload( absl::StatusCode::kInvalidArgument, @@ -218,14 +235,19 @@ absl::StatusOr ImageSegmenter::Segment( MediaPipeTasksStatus::kRunnerUnexpectedInputError); } ASSIGN_OR_RETURN(NormalizedRect norm_rect, - ConvertToNormalizedRect(image_processing_options, image, - /*roi_allowed=*/false)); + ConvertToNormalizedRect( + segmentation_options.image_processing_options, image, + /*roi_allowed=*/false)); ASSIGN_OR_RETURN( auto output_packets, ProcessImageData( {{kImageInStreamName, mediapipe::MakePacket(std::move(image))}, {kNormRectStreamName, - MakePacket(std::move(norm_rect))}})); + MakePacket(std::move(norm_rect))}, + {kOutputSizeStreamName, + MakePacket>( + std::make_pair(segmentation_options.output_width, + segmentation_options.output_height))}})); std::optional> confidence_masks; if (output_confidence_masks_) { confidence_masks = @@ -243,6 +265,18 @@ absl::StatusOr ImageSegmenter::Segment( absl::StatusOr ImageSegmenter::SegmentForVideo( mediapipe::Image image, int64_t timestamp_ms, std::optional image_processing_options) { + return SegmentForVideo(image, timestamp_ms, + { + /*output_width=*/image.width(), + /*output_height=*/image.height(), + std::move(image_processing_options), + }); +} + +absl::StatusOr ImageSegmenter::SegmentForVideo( + mediapipe::Image image, int64_t timestamp_ms, + SegmentationOptions segmentation_options) { + MP_RETURN_IF_ERROR(ValidateSegmentationOptions(segmentation_options)); if (image.UsesGpu()) { return CreateStatusWithPayload( absl::StatusCode::kInvalidArgument, @@ -250,8 +284,9 @@ absl::StatusOr ImageSegmenter::SegmentForVideo( MediaPipeTasksStatus::kRunnerUnexpectedInputError); } ASSIGN_OR_RETURN(NormalizedRect norm_rect, - ConvertToNormalizedRect(image_processing_options, image, - /*roi_allowed=*/false)); + ConvertToNormalizedRect( + segmentation_options.image_processing_options, image, + /*roi_allowed=*/false)); ASSIGN_OR_RETURN( auto output_packets, ProcessVideoData( @@ -260,6 +295,11 @@ absl::StatusOr ImageSegmenter::SegmentForVideo( .At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}, {kNormRectStreamName, MakePacket(std::move(norm_rect)) + .At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}, + {kOutputSizeStreamName, + MakePacket>( + std::make_pair(segmentation_options.output_width, + segmentation_options.output_height)) .At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}})); std::optional> confidence_masks; if (output_confidence_masks_) { @@ -278,6 +318,18 @@ absl::StatusOr ImageSegmenter::SegmentForVideo( absl::Status ImageSegmenter::SegmentAsync( Image image, int64_t timestamp_ms, std::optional image_processing_options) { + return SegmentAsync(image, timestamp_ms, + { + /*output_width=*/image.width(), + /*output_height=*/image.height(), + std::move(image_processing_options), + }); +} + +absl::Status ImageSegmenter::SegmentAsync( + Image image, int64_t timestamp_ms, + SegmentationOptions segmentation_options) { + MP_RETURN_IF_ERROR(ValidateSegmentationOptions(segmentation_options)); if (image.UsesGpu()) { return CreateStatusWithPayload( absl::StatusCode::kInvalidArgument, @@ -285,14 +337,20 @@ absl::Status ImageSegmenter::SegmentAsync( MediaPipeTasksStatus::kRunnerUnexpectedInputError); } ASSIGN_OR_RETURN(NormalizedRect norm_rect, - ConvertToNormalizedRect(image_processing_options, image, - /*roi_allowed=*/false)); + ConvertToNormalizedRect( + segmentation_options.image_processing_options, image, + /*roi_allowed=*/false)); return SendLiveStreamData( {{kImageInStreamName, MakePacket(std::move(image)) .At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}, {kNormRectStreamName, MakePacket(std::move(norm_rect)) + .At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}, + {kOutputSizeStreamName, + MakePacket>( + std::make_pair(segmentation_options.output_width, + segmentation_options.output_height)) .At(Timestamp(timestamp_ms * kMicroSecondsPerMilliSecond))}}); } diff --git a/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.h b/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.h index 0546cef3..82bb3a3a 100644 --- a/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.h +++ b/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter.h @@ -67,6 +67,22 @@ struct ImageSegmenterOptions { result_callback = nullptr; }; +// Options for configuring runtime behavior of ImageSegmenter. +struct SegmentationOptions { + // The width of the output segmentation masks. + int output_width; + + // The height of the output segmentation masks. + int output_height; + + // The optional 'image_processing_options' parameter can be used to specify + // the rotation to apply to the image before performing segmentation, by + // setting its 'rotation_degrees' field. Note that specifying a + // region-of-interest using the 'region_of_interest' field is NOT supported + // and will result in an invalid argument error being returned. + std::optional image_processing_options; +}; + // Performs segmentation on images. // // The API expects a TFLite model with mandatory TFLite Model Metadata. @@ -102,17 +118,46 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi { // // The image can be of any size with format RGB or RGBA. // + // The output size is the same as the input image size. + // // The optional 'image_processing_options' parameter can be used to specify // the rotation to apply to the image before performing segmentation, by // setting its 'rotation_degrees' field. Note that specifying a // region-of-interest using the 'region_of_interest' field is NOT supported // and will result in an invalid argument error being returned. - absl::StatusOr Segment( mediapipe::Image image, std::optional image_processing_options = std::nullopt); + // Performs image segmentation on the provided single image. + // Only use this method when the ImageSegmenter is created with the image + // running mode. + // + // The image can be of any size with format RGB or RGBA. + absl::StatusOr Segment( + mediapipe::Image image, SegmentationOptions segmentation_options); + + // Performs image segmentation on the provided video frame. + // Only use this method when the ImageSegmenter is created with the video + // running mode. + // + // The image can be of any size with format RGB or RGBA. It's required to + // provide the video frame's timestamp (in milliseconds). The input timestamps + // must be monotonically increasing. + // + // The output size is the same as the input image size. + // + // The optional 'image_processing_options' parameter can be used + // to specify the rotation to apply to the image before performing + // segmentation, by setting its 'rotation_degrees' field. Note that specifying + // a region-of-interest using the 'region_of_interest' field is NOT supported + // and will result in an invalid argument error being returned. + absl::StatusOr SegmentForVideo( + mediapipe::Image image, int64_t timestamp_ms, + std::optional image_processing_options = + std::nullopt); + // Performs image segmentation on the provided video frame. // Only use this method when the ImageSegmenter is created with the video // running mode. @@ -120,16 +165,9 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi { // The image can be of any size with format RGB or RGBA. It's required to // provide the video frame's timestamp (in milliseconds). The input timestamps // must be monotonically increasing. - // - // The optional 'image_processing_options' parameter can be used to specify - // the rotation to apply to the image before performing segmentation, by - // setting its 'rotation_degrees' field. Note that specifying a - // region-of-interest using the 'region_of_interest' field is NOT supported - // and will result in an invalid argument error being returned. absl::StatusOr SegmentForVideo( mediapipe::Image image, int64_t timestamp_ms, - std::optional image_processing_options = - std::nullopt); + SegmentationOptions segmentation_options); // Sends live image data to perform image segmentation, and the results will // be available via the "result_callback" provided in the @@ -141,13 +179,15 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi { // sent to the image segmenter. The input timestamps must be monotonically // increasing. // + // The output size is the same as the input image size. + // // The optional 'image_processing_options' parameter can be used to specify // the rotation to apply to the image before performing segmentation, by // setting its 'rotation_degrees' field. Note that specifying a // region-of-interest using the 'region_of_interest' field is NOT supported // and will result in an invalid argument error being returned. // - // The "result_callback" prvoides + // The "result_callback" provides // - An ImageSegmenterResult. // - The const reference to the corresponding input image that the image // segmentation runs on. Note that the const reference to the image will @@ -158,6 +198,26 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi { std::optional image_processing_options = std::nullopt); + // Sends live image data to perform image segmentation, and the results will + // be available via the "result_callback" provided in the + // ImageSegmenterOptions. Only use this method when the ImageSegmenter is + // created with the live stream running mode. + // + // The image can be of any size with format RGB or RGBA. It's required to + // provide a timestamp (in milliseconds) to indicate when the input image is + // sent to the image segmenter. The input timestamps must be monotonically + // increasing. + // + // The "result_callback" provides + // - An ImageSegmenterResult. + // - The const reference to the corresponding input image that the image + // segmentation runs on. Note that the const reference to the image will + // no longer be valid when the callback returns. To access the image data + // outside of the callback, callers need to make a copy of the image. + // - The input timestamp in milliseconds. + absl::Status SegmentAsync(mediapipe::Image image, int64_t timestamp_ms, + SegmentationOptions segmentation_options); + // Shuts down the ImageSegmenter when all works are done. absl::Status Close() { return runner_->Close(); } @@ -174,6 +234,14 @@ class ImageSegmenter : tasks::vision::core::BaseVisionTaskApi { std::vector labels_; bool output_confidence_masks_; bool output_category_mask_; + + absl::Status ValidateSegmentationOptions(const SegmentationOptions& options) { + if (options.output_width <= 0 || options.output_height <= 0) { + return absl::InvalidArgumentError( + "Both output_width and output_height must be larger than 0."); + } + return absl::OkStatus(); + } }; } // namespace image_segmenter diff --git a/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_graph.cc b/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_graph.cc index 0ae47ffd..b49f22ca 100644 --- a/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_graph.cc +++ b/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_graph.cc @@ -19,6 +19,7 @@ limitations under the License. #include #include +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" @@ -82,6 +83,7 @@ constexpr char kImageGpuTag[] = "IMAGE_GPU"; constexpr char kNormRectTag[] = "NORM_RECT"; constexpr char kTensorsTag[] = "TENSORS"; constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; +constexpr char kSizeTag[] = "SIZE"; constexpr char kQualityScoresTag[] = "QUALITY_SCORES"; constexpr char kSegmentationMetadataName[] = "SEGMENTER_METADATA"; @@ -183,7 +185,7 @@ absl::Status ConfigureTensorsToSegmentationCalculator( } } if (!found_activation_in_metadata) { - LOG(WARNING) + ABSL_LOG(WARNING) << "No activation type is found in model metadata. Use NONE for " "ImageSegmenterGraph."; } @@ -356,6 +358,9 @@ absl::StatusOr ConvertImageToTensors( // Describes image rotation and region of image to perform detection // on. // @Optional: rect covering the whole image is used if not specified. +// OUTPUT_SIZE - std::pair @Optional +// The output size of the mask, in width and height. If not specified, the +// output size of the input image is used. // // Outputs: // CONFIDENCE_MASK - mediapipe::Image @Multiple @@ -400,11 +405,16 @@ class ImageSegmenterGraph : public core::ModelTaskGraph { if (!options.segmenter_options().has_output_type()) { MP_RETURN_IF_ERROR(SanityCheck(sc)); } + std::optional>> output_size; + if (HasInput(sc->OriginalNode(), kOutputSizeTag)) { + output_size = graph.In(kOutputSizeTag).Cast>(); + } ASSIGN_OR_RETURN( auto output_streams, BuildSegmentationTask( options, *model_resources, graph[Input(kImageTag)], - graph[Input::Optional(kNormRectTag)], graph)); + graph[Input::Optional(kNormRectTag)], output_size, + graph)); // TODO: remove deprecated output type support. if (options.segmenter_options().has_output_type()) { @@ -469,7 +479,8 @@ class ImageSegmenterGraph : public core::ModelTaskGraph { absl::StatusOr BuildSegmentationTask( const ImageSegmenterGraphOptions& task_options, const core::ModelResources& model_resources, Source image_in, - Source norm_rect_in, Graph& graph) { + Source norm_rect_in, + std::optional>> output_size, Graph& graph) { MP_RETURN_IF_ERROR(SanityCheckOptions(task_options)); // Adds preprocessing calculators and connects them to the graph input image @@ -514,10 +525,14 @@ class ImageSegmenterGraph : public core::ModelTaskGraph { image_and_tensors.tensors >> inference.In(kTensorsTag); inference.Out(kTensorsTag) >> tensor_to_images.In(kTensorsTag); - // Adds image property calculator for output size. - auto& image_properties = graph.AddNode("ImagePropertiesCalculator"); - image_in >> image_properties.In("IMAGE"); - image_properties.Out("SIZE") >> tensor_to_images.In(kOutputSizeTag); + if (output_size.has_value()) { + *output_size >> tensor_to_images.In(kOutputSizeTag); + } else { + // Adds image property calculator for output size. + auto& image_properties = graph.AddNode("ImagePropertiesCalculator"); + image_in >> image_properties.In(kImageTag); + image_properties.Out(kSizeTag) >> tensor_to_images.In(kOutputSizeTag); + } // Exports multiple segmented masks. // TODO: remove deprecated output type support. diff --git a/mediapipe/tasks/cc/vision/pose_detector/pose_detector_graph_test.cc b/mediapipe/tasks/cc/vision/pose_detector/pose_detector_graph_test.cc index 71113110..4d15583a 100644 --- a/mediapipe/tasks/cc/vision/pose_detector/pose_detector_graph_test.cc +++ b/mediapipe/tasks/cc/vision/pose_detector/pose_detector_graph_test.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" @@ -114,16 +115,18 @@ absl::StatusOr> CreateTaskRunner( Detection GetExpectedPoseDetectionResult(absl::string_view file_name) { Detection detection; - CHECK_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), - &detection, Defaults())) + ABSL_CHECK_OK( + GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), + &detection, Defaults())) << "Expected pose detection result does not exist."; return detection; } NormalizedRect GetExpectedExpandedPoseRect(absl::string_view file_name) { NormalizedRect expanded_rect; - CHECK_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), - &expanded_rect, Defaults())) + ABSL_CHECK_OK( + GetTextProto(file::JoinPath("./", kTestDataDirectory, file_name), + &expanded_rect, Defaults())) << "Expected expanded pose rect does not exist."; return expanded_rect; } diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/BUILD b/mediapipe/tasks/cc/vision/pose_landmarker/BUILD index f97857dd..f9bdb561 100644 --- a/mediapipe/tasks/cc/vision/pose_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/pose_landmarker/BUILD @@ -155,3 +155,13 @@ cc_library( "//mediapipe/tasks/cc/components/containers:landmark", ], ) + +cc_library( + name = "pose_landmarks_connections", + hdrs = ["pose_landmarks_connections.h"], +) + +cc_library( + name = "pose_landmark", + hdrs = ["pose_landmark.h"], +) diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmark.h b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmark.h new file mode 100644 index 00000000..36c62814 --- /dev/null +++ b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmark.h @@ -0,0 +1,68 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARK_H_ +#define MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARK_H_ + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace pose_landmarker { + +static constexpr int kNumPoseLandmarks = 33; + +// BlazePose 33 landmark names. +enum class PoseLandmark { + kNose = 0, + kLeftEyeInner, + kLeftEye, + kLeftEyeOuter, + kRightEyeInner, + kRightEye, + kRightEyeOuter, + kLeftEar, + kRightEar, + kMouthLeft, + kMouthRight, + kLeftShoulder, + kRightShoulder, + kLeftElbow, + kRightElbow, + kLeftWrist, + kRightWrist, + kLeftPinky1, + kRightPinky1, + kLeftIndex1, + kRightIndex1, + kLeftThumb2, + kRightThumb2, + kLeftHip, + kRightHip, + kLeftKnee, + kRightKnee, + kLeftAnkle, + kRightAnkle, + kLeftHeel, + kRightHeel, + kLeftFootIndex, + kRightFootIndex, +}; + +} // namespace pose_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARK_H_ diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_result.h b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_result.h index 8978e514..27314b6c 100644 --- a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_result.h +++ b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_result.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKER_RESULT_H_ -#define MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKER_RESULT_H_ +#ifndef MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARKER_RESULT_H_ +#define MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARKER_RESULT_H_ #include @@ -49,4 +49,4 @@ PoseLandmarkerResult ConvertToPoseLandmarkerResult( } // namespace tasks } // namespace mediapipe -#endif // MEDIAPIPE_TASKS_CC_VISION_HAND_LANDMARKER_HAND_LANDMARKER_RESULT_H_ +#endif // MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARKER_RESULT_H_ diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_test.cc b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_test.cc index afc58b1d..239851b5 100644 --- a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_test.cc +++ b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_test.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include "absl/flags/flag.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/string_view.h" @@ -105,17 +106,17 @@ MATCHER_P2(LandmarksMatches, expected_landmarks, toleration, "") { for (int i = 0; i < arg.size(); i++) { for (int j = 0; j < arg[i].landmarks.size(); j++) { if (arg[i].landmarks.size() != expected_landmarks[i].landmarks.size()) { - LOG(INFO) << "sizes not equal"; + ABSL_LOG(INFO) << "sizes not equal"; return false; } if (std::abs(arg[i].landmarks[j].x - expected_landmarks[i].landmarks[j].x) > toleration || std::abs(arg[i].landmarks[j].y - expected_landmarks[i].landmarks[j].y) > toleration) { - LOG(INFO) << DUMP_VARS(arg[i].landmarks[j].x, - expected_landmarks[i].landmarks[j].x); - LOG(INFO) << DUMP_VARS(arg[i].landmarks[j].y, - expected_landmarks[i].landmarks[j].y); + ABSL_LOG(INFO) << DUMP_VARS(arg[i].landmarks[j].x, + expected_landmarks[i].landmarks[j].x); + ABSL_LOG(INFO) << DUMP_VARS(arg[i].landmarks[j].y, + expected_landmarks[i].landmarks[j].y); return false; } } @@ -316,7 +317,7 @@ TEST_P(VideoModeTest, Succeeds) { MP_ASSERT_OK_AND_ASSIGN(pose_landmarker_results, pose_landmarker->DetectForVideo(image, i)); } - LOG(INFO) << i; + ABSL_LOG(INFO) << i; ExpectPoseLandmarkerResultsCorrect( pose_landmarker_results, expected_results, kLandmarksOnVideoAbsMargin); } diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h new file mode 100644 index 00000000..4b79215a --- /dev/null +++ b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h @@ -0,0 +1,39 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARKS_CONNECTIONS_H_ +#define MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARKS_CONNECTIONS_H_ + +#include + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace pose_landmarker { + +static constexpr std::array, 34> kPoseLandmarksConnections{{ + {1, 2}, {0, 1}, {2, 3}, {3, 7}, {0, 4}, {4, 5}, {5, 6}, + {6, 8}, {9, 10}, {11, 12}, {11, 13}, {13, 15}, {15, 17}, {15, 19}, + {15, 21}, {17, 19}, {12, 14}, {14, 16}, {16, 18}, {16, 20}, {16, 22}, + {18, 20}, {11, 23}, {12, 24}, {23, 24}, {23, 25}, {24, 26}, {25, 27}, + {26, 28}, {27, 29}, {28, 30}, {29, 31}, {30, 32}, {27, 31}, +}}; + +} // namespace pose_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_POSE_LANDMARKER_POSE_LANDMARKS_CONNECTIONS_H_ diff --git a/mediapipe/tasks/cc/vision/utils/BUILD b/mediapipe/tasks/cc/vision/utils/BUILD index ae303441..bb84cf3f 100644 --- a/mediapipe/tasks/cc/vision/utils/BUILD +++ b/mediapipe/tasks/cc/vision/utils/BUILD @@ -33,6 +33,7 @@ cc_library_with_tflite( "//mediapipe/tasks/cc/metadata:metadata_extractor", "//mediapipe/tasks/metadata:metadata_schema_cc", "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -60,6 +61,7 @@ cc_test_with_tflite( "//mediapipe/tasks/cc/metadata:metadata_extractor", "//mediapipe/tasks/metadata:metadata_schema_cc", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", @@ -110,3 +112,23 @@ cc_test( "//mediapipe/tasks/cc/components/containers:rect", ], ) + +cc_library( + name = "data_renderer", + srcs = ["data_renderer.cc"], + hdrs = ["data_renderer.h"], + deps = [ + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:landmarks_to_render_data_calculator", + "//mediapipe/calculators/util:landmarks_to_render_data_calculator_cc_proto", + "//mediapipe/calculators/util:rect_to_render_data_calculator_cc_proto", + "//mediapipe/calculators/util:rect_to_render_scale_calculator", + "//mediapipe/calculators/util:rect_to_render_scale_calculator_cc_proto", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/util:render_data_cc_proto", + "@com_google_absl//absl/types:span", + ], +) diff --git a/mediapipe/tasks/cc/vision/utils/data_renderer.cc b/mediapipe/tasks/cc/vision/utils/data_renderer.cc new file mode 100644 index 00000000..aeefbba2 --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/data_renderer.cc @@ -0,0 +1,88 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/utils/data_renderer.h" + +#include +#include +#include + +#include "absl/types/span.h" +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/calculators/util/rect_to_render_data_calculator.pb.h" +#include "mediapipe/calculators/util/rect_to_render_scale_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/util/render_data.pb.h" + +namespace mediapipe::tasks::vision::utils { + +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Stream; + +Stream Render(Stream image, + absl::Span> render_data_list, + Graph& graph) { + auto& annotation_overlay = graph.AddNode("AnnotationOverlayCalculator"); + image >> annotation_overlay.In("UIMAGE"); + for (int i = 0; i < render_data_list.size(); ++i) { + render_data_list[i] >> annotation_overlay.In(i); + } + return annotation_overlay.Out("UIMAGE").Cast(); +} + +Stream RenderLandmarks( + Stream landmarks, + std::optional> render_scale, + const mediapipe::LandmarksToRenderDataCalculatorOptions& renderer_options, + Graph& graph) { + auto& landmarks_render = graph.AddNode("LandmarksToRenderDataCalculator"); + landmarks_render + .GetOptions() + .CopyFrom(renderer_options); + landmarks >> landmarks_render.In("NORM_LANDMARKS"); + if (render_scale.has_value()) { + *render_scale >> landmarks_render.In("RENDER_SCALE"); + } + auto render_data = landmarks_render.Out("RENDER_DATA"); + return render_data.Cast(); +} + +Stream GetRenderScale(Stream> image_size, + Stream roi, float multiplier, + Graph& graph) { + auto& to_render_scale = graph.AddNode("RectToRenderScaleCalculator"); + to_render_scale.GetOptions() + .set_multiplier(multiplier); + roi >> to_render_scale.In("NORM_RECT"); + image_size >> to_render_scale.In("IMAGE_SIZE"); + return to_render_scale.Out("RENDER_SCALE").Cast(); +} + +Stream RenderRect( + Stream rect, + const mediapipe::RectToRenderDataCalculatorOptions& renderer_options, + Graph& graph) { + auto& rect_render = graph.AddNode("RectToRenderDataCalculator"); + rect_render.GetOptions() + .CopyFrom(renderer_options); + rect >> rect_render.In("NORM_RECT"); + auto render_data = rect_render.Out("RENDER_DATA"); + return render_data.Cast(); +} + +} // namespace mediapipe::tasks::vision::utils diff --git a/mediapipe/tasks/cc/vision/utils/data_renderer.h b/mediapipe/tasks/cc/vision/utils/data_renderer.h new file mode 100644 index 00000000..f58f94ee --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/data_renderer.h @@ -0,0 +1,69 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_UTILS_DATA_RENDERER_H_ +#define MEDIAPIPE_TASKS_CC_VISION_UTILS_DATA_RENDERER_H_ + +#include +#include + +#include "absl/types/span.h" +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/calculators/util/rect_to_render_data_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/util/render_data.pb.h" + +namespace mediapipe::tasks::vision::utils { + +// Adds a node to the provided graph that renders the render_data_list on the +// given image, and returns the rendered image. +api2::builder::Stream Render( + api2::builder::Stream image, + absl::Span> render_data_list, + api2::builder::Graph& graph); + +// Adds a node to the provided graph that infers the render scale from the image +// size and the object RoI. It will give you bigger rendered primitives for +// bigger/closer objects and smaller primitives for smaller/far objects. The +// primitives scale is proportional to `roi_size * multiplier`. +// +// See more details in +// mediapipe/calculators/util/rect_to_render_scale_calculator.cc +api2::builder::Stream GetRenderScale( + api2::builder::Stream> image_size, + api2::builder::Stream roi, float multiplier, + api2::builder::Graph& graph); + +// Adds a node to the provided graph that gets the landmarks render data +// according to the renderer_options. +api2::builder::Stream RenderLandmarks( + api2::builder::Stream landmarks, + std::optional> render_scale, + const mediapipe::LandmarksToRenderDataCalculatorOptions& renderer_options, + api2::builder::Graph& graph); + +// Adds a node to the provided graph that gets the rect render data according to +// the renderer_options. +api2::builder::Stream RenderRect( + api2::builder::Stream rect, + const mediapipe::RectToRenderDataCalculatorOptions& renderer_options, + api2::builder::Graph& graph); + +} // namespace mediapipe::tasks::vision::utils + +#endif // MEDIAPIPE_TASKS_CC_VISION_UTILS_DATA_RENDERER_H_ diff --git a/mediapipe/tasks/cc/vision/utils/data_renderer_test.cc b/mediapipe/tasks/cc/vision/utils/data_renderer_test.cc new file mode 100644 index 00000000..b42c335b --- /dev/null +++ b/mediapipe/tasks/cc/vision/utils/data_renderer_test.cc @@ -0,0 +1,133 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/utils/data_renderer.h" + +#include +#include + +#include "absl/types/span.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/util/render_data.pb.h" + +namespace mediapipe::tasks::vision::utils { +namespace { + +using ::mediapipe::CalculatorGraphConfig; +using ::mediapipe::EqualsProto; +using ::mediapipe::NormalizedRect; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Stream; + +TEST(DataRenderer, Render) { + Graph graph; + Stream image_in = graph.In("IMAGE").Cast(); + Stream render_data_in = + graph.In("RENDER_DATA").Cast(); + std::vector> render_data_list = {render_data_in}; + Stream image_out = + Render(image_in, absl::Span>(render_data_list), graph); + image_out.SetName("image_out"); + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "AnnotationOverlayCalculator" + input_stream: "__stream_1" + input_stream: "UIMAGE:__stream_0" + output_stream: "UIMAGE:image_out" + } + input_stream: "IMAGE:__stream_0" + input_stream: "RENDER_DATA:__stream_1" + )pb"))); +} + +TEST(DataRenderer, RenderLandmarks) { + Graph graph; + Stream rect = + graph.In("NORM_LANDMARKS").Cast(); + Stream render_data = + RenderLandmarks(rect, std::nullopt, {}, graph); + render_data.SetName("render_data"); + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "LandmarksToRenderDataCalculator" + input_stream: "NORM_LANDMARKS:__stream_0" + output_stream: "RENDER_DATA:render_data" + options { + [mediapipe.LandmarksToRenderDataCalculatorOptions.ext] {} + } + } + input_stream: "NORM_LANDMARKS:__stream_0" + )pb"))); +} + +TEST(DataRenderer, GetRenderScale) { + Graph graph; + Stream> image_size = + graph.In("IMAGE_SIZE").Cast>(); + Stream roi = graph.In("ROI").Cast(); + Stream render_scale = GetRenderScale(image_size, roi, 0.0001, graph); + render_scale.SetName("render_scale"); + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "RectToRenderScaleCalculator" + input_stream: "IMAGE_SIZE:__stream_0" + input_stream: "NORM_RECT:__stream_1" + output_stream: "RENDER_SCALE:render_scale" + options { + [mediapipe.RectToRenderScaleCalculatorOptions.ext] { + multiplier: 0.0001 + } + } + } + input_stream: "IMAGE_SIZE:__stream_0" + input_stream: "ROI:__stream_1" + )pb"))); +} + +TEST(DataRenderer, RenderRect) { + Graph graph; + Stream rect = graph.In("NORM_RECT").Cast(); + Stream render_data = RenderRect(rect, {}, graph); + render_data.SetName("render_data"); + EXPECT_THAT( + graph.GetConfig(), + EqualsProto(mediapipe::ParseTextProtoOrDie(R"pb( + node { + calculator: "RectToRenderDataCalculator" + input_stream: "NORM_RECT:__stream_0" + output_stream: "RENDER_DATA:render_data" + options { + [mediapipe.RectToRenderDataCalculatorOptions.ext] {} + } + } + input_stream: "NORM_RECT:__stream_0" + )pb"))); +} + +} // namespace +} // namespace mediapipe::tasks::vision::utils diff --git a/mediapipe/tasks/cc/vision/utils/image_tensor_specs.cc b/mediapipe/tasks/cc/vision/utils/image_tensor_specs.cc index 1041dd1f..690cd6e5 100644 --- a/mediapipe/tasks/cc/vision/utils/image_tensor_specs.cc +++ b/mediapipe/tasks/cc/vision/utils/image_tensor_specs.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include "absl/algorithm/container.h" +#include "absl/log/absl_log.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" @@ -241,11 +242,12 @@ absl::StatusOr BuildInputImageTensorSpecs( absl::StatusOr BuildInputImageTensorSpecs( const core::ModelResources& model_resources) { const tflite::Model& model = *model_resources.GetTfLiteModel(); + // TODO: Investigate if there is any better solutions support + // running inference with multiple subgraphs. if (model.subgraphs()->size() != 1) { - return CreateStatusWithPayload( - absl::StatusCode::kInvalidArgument, - "Image tflite models are assumed to have a single subgraph.", - MediaPipeTasksStatus::kInvalidArgumentError); + ABSL_LOG(WARNING) + << "TFLite model has more than 1 subgraphs. Use subrgaph 0 as " + "the primary subgraph for inference"; } const auto* primary_subgraph = (*model.subgraphs())[0]; if (primary_subgraph->inputs()->size() != 1) { diff --git a/mediapipe/tasks/cc/vision/utils/image_tensor_specs_test.cc b/mediapipe/tasks/cc/vision/utils/image_tensor_specs_test.cc index 7293d58b..a10d1281 100644 --- a/mediapipe/tasks/cc/vision/utils/image_tensor_specs_test.cc +++ b/mediapipe/tasks/cc/vision/utils/image_tensor_specs_test.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/cord.h" @@ -179,7 +180,7 @@ TEST_F(ImageTensorSpecsTest, BuildInputImageTensorSpecsFromModelResources) { core::ModelResources::Create(kTestModelResourcesTag, std::move(model_file))); const tflite::Model* model = model_resources->GetTfLiteModel(); - CHECK(model != nullptr); + ABSL_CHECK(model != nullptr); absl::StatusOr input_specs_or = BuildInputImageTensorSpecs(*model_resources); MP_ASSERT_OK(input_specs_or); diff --git a/mediapipe/tasks/ios/BUILD b/mediapipe/tasks/ios/BUILD index 29b0dd65..7f3db7f7 100644 --- a/mediapipe/tasks/ios/BUILD +++ b/mediapipe/tasks/ios/BUILD @@ -54,6 +54,8 @@ CALCULATORS_AND_GRAPHS = [ "//mediapipe/tasks/cc/text/text_embedder:text_embedder_graph", "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", + "//mediapipe/tasks/cc/vision/hand_landmarker:hand_landmarker_graph", + "//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_graph", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", ] @@ -66,7 +68,10 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/components/containers:sources/MPPClassificationResult.h", "//mediapipe/tasks/ios/components/containers:sources/MPPEmbedding.h", "//mediapipe/tasks/ios/components/containers:sources/MPPEmbeddingResult.h", + "//mediapipe/tasks/ios/components/containers:sources/MPPConnection.h", "//mediapipe/tasks/ios/components/containers:sources/MPPDetection.h", + "//mediapipe/tasks/ios/components/containers:sources/MPPLandmark.h", + "//mediapipe/tasks/ios/components/processors:sources/MPPClassifierOptions.h", "//mediapipe/tasks/ios/core:sources/MPPBaseOptions.h", "//mediapipe/tasks/ios/core:sources/MPPTaskOptions.h", "//mediapipe/tasks/ios/core:sources/MPPTaskResult.h", @@ -84,6 +89,12 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarker.h", "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarkerOptions.h", "//mediapipe/tasks/ios/vision/face_landmarker:sources/MPPFaceLandmarkerResult.h", + "//mediapipe/tasks/ios/vision/hand_landmarker:sources/MPPHandLandmarker.h", + "//mediapipe/tasks/ios/vision/hand_landmarker:sources/MPPHandLandmarkerOptions.h", + "//mediapipe/tasks/ios/vision/hand_landmarker:sources/MPPHandLandmarkerResult.h", + "//mediapipe/tasks/ios/vision/gesture_recognizer:sources/MPPGestureRecognizer.h", + "//mediapipe/tasks/ios/vision/gesture_recognizer:sources/MPPGestureRecognizerOptions.h", + "//mediapipe/tasks/ios/vision/gesture_recognizer:sources/MPPGestureRecognizerResult.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifier.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierOptions.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierResult.h", @@ -159,7 +170,10 @@ apple_static_xcframework( ":MPPBaseOptions.h", ":MPPCategory.h", ":MPPClassificationResult.h", + ":MPPClassifierOptions.h", ":MPPDetection.h", + ":MPPLandmark.h", + ":MPPConnection.h", ":MPPCommon.h", ":MPPTaskOptions.h", ":MPPTaskResult.h", @@ -174,6 +188,12 @@ apple_static_xcframework( ":MPPImageClassifier.h", ":MPPImageClassifierOptions.h", ":MPPImageClassifierResult.h", + ":MPPHandLandmarker.h", + ":MPPHandLandmarkerOptions.h", + ":MPPHandLandmarkerResult.h", + ":MPPGestureRecognizer.h", + ":MPPGestureRecognizerOptions.h", + ":MPPGestureRecognizerResult.h", ":MPPObjectDetector.h", ":MPPObjectDetectorOptions.h", ":MPPObjectDetectorResult.h", @@ -181,6 +201,8 @@ apple_static_xcframework( deps = [ "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetector", "//mediapipe/tasks/ios/vision/face_landmarker:MPPFaceLandmarker", + "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizer", + "//mediapipe/tasks/ios/vision/hand_landmarker:MPPHandLandmarker", "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifier", "//mediapipe/tasks/ios/vision/object_detector:MPPObjectDetector", ], diff --git a/mediapipe/tasks/ios/MediaPipeTasksCommon.podspec.template b/mediapipe/tasks/ios/MediaPipeTasksCommon.podspec.template index 1e622469..cf01e99c 100644 --- a/mediapipe/tasks/ios/MediaPipeTasksCommon.podspec.template +++ b/mediapipe/tasks/ios/MediaPipeTasksCommon.podspec.template @@ -8,7 +8,7 @@ Pod::Spec.new do |s| s.summary = 'MediaPipe Task Library - Text' s.description = 'The common libraries of the MediaPipe Task Library' - s.ios.deployment_target = '11.0' + s.ios.deployment_target = '12.0' s.module_name = 'MediaPipeTasksCommon' s.static_framework = true diff --git a/mediapipe/tasks/ios/MediaPipeTasksText.podspec.template b/mediapipe/tasks/ios/MediaPipeTasksText.podspec.template index f2f04bf7..261baf7b 100644 --- a/mediapipe/tasks/ios/MediaPipeTasksText.podspec.template +++ b/mediapipe/tasks/ios/MediaPipeTasksText.podspec.template @@ -8,7 +8,7 @@ Pod::Spec.new do |s| s.summary = 'MediaPipe Task Library - Text' s.description = 'The Natural Language APIs of the MediaPipe Task Library' - s.ios.deployment_target = '11.0' + s.ios.deployment_target = '12.0' s.module_name = 'MediaPipeTasksText' s.static_framework = true diff --git a/mediapipe/tasks/ios/MediaPipeTasksVision.podspec.template b/mediapipe/tasks/ios/MediaPipeTasksVision.podspec.template index af63ba94..62698dfd 100644 --- a/mediapipe/tasks/ios/MediaPipeTasksVision.podspec.template +++ b/mediapipe/tasks/ios/MediaPipeTasksVision.podspec.template @@ -8,7 +8,7 @@ Pod::Spec.new do |s| s.summary = 'MediaPipe Task Library - Vision' s.description = 'The Vision APIs of the MediaPipe Task Library' - s.ios.deployment_target = '11.0' + s.ios.deployment_target = '12.0' s.module_name = 'MediaPipeTasksVision' s.static_framework = true diff --git a/mediapipe/tasks/ios/build_ios_framework.sh b/mediapipe/tasks/ios/build_ios_framework.sh index 1142f08a..50f5797f 100755 --- a/mediapipe/tasks/ios/build_ios_framework.sh +++ b/mediapipe/tasks/ios/build_ios_framework.sh @@ -112,7 +112,7 @@ function build_ios_frameworks_and_libraries { IOS_GRAPHS_SIMULATOR_LIBRARY_PATH="$(get_output_file_path "${IOS_SIM_FAT_LIBRARY_CQUERY_COMMAND}")" # Build static library for iOS devices with arch ios_arm64. We don't need to build for armv7 since - # our deployment target is iOS 11.0. iOS 11.0 and upwards is not supported by old armv7 devices. + # our deployment target is iOS 12.0. iOS 12.0 and upwards is not supported by old armv7 devices. local IOS_DEVICE_LIBRARY_CQUERY_COMMAND="-c opt --config=ios_arm64 --apple_generate_dsym=false --define OPENCV=source //mediapipe/tasks/ios:MediaPipeTaskGraphs_library" ${BAZEL} build ${IOS_DEVICE_LIBRARY_CQUERY_COMMAND} IOS_GRAPHS_DEVICE_LIBRARY_PATH="$(get_output_file_path "${IOS_DEVICE_LIBRARY_CQUERY_COMMAND}")" @@ -124,7 +124,7 @@ function build_ios_frameworks_and_libraries { function create_framework_archive { # Change to the Bazel iOS output directory. - pushd "${BAZEL_IOS_OUTDIR}" + pushd "${MPP_ROOT_DIR}" # Create the temporary directory for the given framework. local ARCHIVE_NAME="${FRAMEWORK_NAME}-${MPP_BUILD_VERSION}" @@ -165,9 +165,9 @@ function create_framework_archive { #----- (3) Move the framework to the destination ----- if [[ "${ARCHIVE_FRAMEWORK}" == true ]]; then - local TARGET_DIR="$(realpath "${FRAMEWORK_NAME}")" - # Create the framework archive directory. + mkdir -p "${FRAMEWORK_NAME}" + local TARGET_DIR="$(realpath "${FRAMEWORK_NAME}")" local FRAMEWORK_ARCHIVE_DIR if [[ "${IS_RELEASE_BUILD}" == true ]]; then @@ -186,8 +186,11 @@ function create_framework_archive { mv "${MPP_ARCHIVE_FILE}" "${FRAMEWORK_ARCHIVE_DIR}" popd - # Move the target directory to the Kokoro artifacts directory. - mv "${TARGET_DIR}" "$(realpath "${DEST_DIR}")"/ + # Move the target directory to the Kokoro artifacts directory and clean up + # the artifacts directory in the mediapipe root directory even if the + # move command fails. + mv "${TARGET_DIR}" "$(realpath "${DEST_DIR}")"/ || true + rm -rf "${TARGET_DIR}" else rsync -r "${MPP_TMPDIR}/" "$(realpath "${DEST_DIR}")/" fi diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h index f360d46d..5753c4d3 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -44,14 +44,14 @@ NS_SWIFT_NAME(ResultCategory) @property(nonatomic, readonly, nullable) NSString *displayName; /** - * Initializes a new `MPPCategory` with the given index, score, category name and display name. + * Initializes a new `Category` with the given index, score, category name and display name. * * @param index The index of the label in the corresponding label file. * @param score The probability score of this label category. * @param categoryName The label of this category object. * @param displayName The display name of the label. * - * @return An instance of `MPPCategory` initialized with the given index, score, category name and + * @return An instance of `Category` initialized with the given index, score, category name and * display name. */ - (instancetype)initWithIndex:(NSInteger)index diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h index bbc9aa8a..43507105 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -32,32 +32,32 @@ NS_SWIFT_NAME(Classifications) /** The optional name of the classifier head, which is the corresponding tensor metadata name. */ @property(nonatomic, readonly, nullable) NSString *headName; -/** An array of `MPPCategory` objects containing the predicted categories. */ +/** An array of `Category` objects containing the predicted categories. */ @property(nonatomic, readonly) NSArray *categories; /** - * Initializes a new `MPPClassifications` object with the given head index and array of categories. + * Initializes a new `Classifications` object with the given head index and array of categories. * Head name is initialized to `nil`. * * @param headIndex The index of the classifier head. - * @param categories An array of `MPPCategory` objects containing the predicted categories. + * @param categories An array of `Category` objects containing the predicted categories. * - * @return An instance of `MPPClassifications` initialized with the given head index and + * @return An instance of `Classifications` initialized with the given head index and * array of categories. */ - (instancetype)initWithHeadIndex:(NSInteger)headIndex categories:(NSArray *)categories; /** - * Initializes a new `MPPClassifications` with the given head index, head name and array of + * Initializes a new `Classifications` with the given head index, head name and array of * categories. * * @param headIndex The index of the classifier head. * @param headName The name of the classifier head, which is the corresponding tensor metadata * name. - * @param categories An array of `MPPCategory` objects containing the predicted categories. + * @param categories An array of `Category` objects containing the predicted categories. * - * @return An object of `MPPClassifications` initialized with the given head index, head name and + * @return An object of `Classifications` initialized with the given head index, head name and * array of categories. */ - (instancetype)initWithHeadIndex:(NSInteger)headIndex @@ -78,7 +78,7 @@ NS_SWIFT_NAME(ClassificationResult) @interface MPPClassificationResult : NSObject /** - * An Array of `MPPClassifications` objects containing the predicted categories for each head of + * An Array of `Classifications` objects containing the predicted categories for each head of * the model. */ @property(nonatomic, readonly) NSArray *classifications; @@ -93,15 +93,15 @@ NS_SWIFT_NAME(ClassificationResult) @property(nonatomic, readonly) NSInteger timestampInMilliseconds; /** - * Initializes a new `MPPClassificationResult` with the given array of classifications and time + * Initializes a new `ClassificationResult` with the given array of classifications and time * stamp (in milliseconds). * - * @param classifications An Array of `MPPClassifications` objects containing the predicted + * @param classifications An Array of `Classifications` objects containing the predicted * categories for each head of the model. * @param timestampInMilliseconds The timestamp (in milliseconds) of the start of the chunk of data * corresponding to these results. * - * @return An instance of `MPPClassificationResult` initialized with the given array of + * @return An instance of `ClassificationResult` initialized with the given array of * classifications and timestamp (in milliseconds). */ - (instancetype)initWithClassifications:(NSArray *)classifications diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPDetection.h b/mediapipe/tasks/ios/components/containers/sources/MPPDetection.h index e085007a..0aaad19e 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPDetection.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPDetection.h @@ -35,7 +35,7 @@ NS_SWIFT_NAME(NormalizedKeypoint) @property(nonatomic, readonly) float score; /** - * Initializes a new `MPPNormalizedKeypoint` object with the given location, label and score. + * Initializes a new `NormalizedKeypoint` object with the given location, label and score. * You must pass 0.0 for `score` if it is not present. * * @param location The (x,y) coordinates location of the normalized keypoint. @@ -43,7 +43,7 @@ NS_SWIFT_NAME(NormalizedKeypoint) * @param score The optional score of the normalized keypoint. You must pass 0.0 for score if it * is not present. * - * @return An instance of `MPPNormalizedKeypoint` initialized with the given given location, label + * @return An instance of `NormalizedKeypoint` initialized with the given given location, label * and score. */ - (instancetype)initWithLocation:(CGPoint)location @@ -56,18 +56,18 @@ NS_SWIFT_NAME(NormalizedKeypoint) @end -/** Represents one detected object in the results of `MPPObjectDetector`. */ +/** Represents one detected object in the results of `ObjectDetector`. */ NS_SWIFT_NAME(Detection) @interface MPPDetection : NSObject -/** An array of `MPPCategory` objects containing the predicted categories. */ +/** An array of `Category` objects containing the predicted categories. */ @property(nonatomic, readonly) NSArray *categories; /** The bounding box of the detected object. */ @property(nonatomic, readonly) CGRect boundingBox; /** - * An optional array of `MPPNormalizedKeypoint` objects associated with the detection. Keypoints + * An optional array of `NormalizedKeypoint` objects associated with the detection. Keypoints * represent interesting points related to the detection. For example, the keypoints represent the * eyes, ear and mouth from the from detection model. In template matching detection, e.g. KNIFT, * they can instead represent the feature points for template matching. @@ -75,18 +75,18 @@ NS_SWIFT_NAME(Detection) @property(nonatomic, readonly, nullable) NSArray *keypoints; /** - * Initializes a new `MPPDetection` object with the given array of categories, bounding box and + * Initializes a new `Detection` object with the given array of categories, bounding box and * optional array of keypoints; * - * @param categories A list of `MPPCategory` objects that contain category name, display name, + * @param categories A list of `Category` objects that contain category name, display name, * score, and the label index. * @param boundingBox A `CGRect` that represents the bounding box. - * @param keypoints: An optional array of `MPPNormalizedKeypoint` objects associated with the + * @param keypoints: An optional array of `NormalizedKeypoint` objects associated with the * detection. Keypoints represent interesting points related to the detection. For example, the * keypoints represent the eyes, ear and mouth from the face detection model. In template matching * detection, e.g. KNIFT, they can instead represent the feature points for template matching. * - * @return An instance of `MPPDetection` initialized with the given array of categories, bounding + * @return An instance of `Detection` initialized with the given array of categories, bounding * box and `nil` keypoints. */ - (instancetype)initWithCategories:(NSArray *)categories diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h b/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h index f47602fc..703124d3 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h @@ -49,13 +49,13 @@ NS_SWIFT_NAME(Landmark) @property(nonatomic, readonly, nullable) NSNumber *presence; /** - * Initializes a new `MPPLandmark` object with the given x, y and z coordinates. + * Initializes a new `Landmark` object with the given x, y and z coordinates. * * @param x The x coordinates of the landmark. * @param y The y coordinates of the landmark. * @param z The z coordinates of the landmark. * - * @return An instance of `MPPLandmark` initialized with the given x, y and z coordinates. + * @return An instance of `Landmark` initialized with the given x, y and z coordinates. */ - (instancetype)initWithX:(float)x y:(float)y @@ -103,13 +103,13 @@ NS_SWIFT_NAME(NormalizedLandmark) @property(nonatomic, readonly, nullable) NSNumber *presence; /** - * Initializes a new `MPPNormalizedLandmark` object with the given x, y and z coordinates. + * Initializes a new `NormalizedLandmark` object with the given x, y and z coordinates. * * @param x The x coordinates of the landmark. * @param y The y coordinates of the landmark. * @param z The z coordinates of the landmark. * - * @return An instance of `MPPNormalizedLandmark` initialized with the given x, y and z coordinates. + * @return An instance of `NormalizedLandmark` initialized with the given x, y and z coordinates. */ - (instancetype)initWithX:(float)x y:(float)y diff --git a/mediapipe/tasks/ios/test/vision/core/BUILD b/mediapipe/tasks/ios/test/vision/core/BUILD index 5932968e..c1512fcb 100644 --- a/mediapipe/tasks/ios/test/vision/core/BUILD +++ b/mediapipe/tasks/ios/test/vision/core/BUILD @@ -54,3 +54,20 @@ ios_unit_test( ":MPPImageObjcTestLibrary", ], ) + +objc_library( + name = "MPPMaskObjcTestLibrary", + testonly = 1, + srcs = ["MPPMaskTests.m"], + deps = ["//mediapipe/tasks/ios/vision/core:MPPMask"], +) + +ios_unit_test( + name = "MPPMaskObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPMaskObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/core/MPPMaskTests.m b/mediapipe/tasks/ios/test/vision/core/MPPMaskTests.m new file mode 100644 index 00000000..ea416eb2 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/core/MPPMaskTests.m @@ -0,0 +1,127 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h" + +#import + +/** Unit tests for `MPPMask`. */ +@interface MPPMaskTests : XCTestCase + +@end + +@implementation MPPMaskTests + +#pragma mark - Tests + +- (void)testInitWithUInt8ArrayNoCopySucceeds { + + NSInteger width = 2; + NSInteger height = 3; + + UInt8 uint8Data[] = {128, 128, 128, 128, 128, 128}; + float float32Data[] = {0.501f, 0.501f, 0.501f, 0.501f, 0.501f, 0.501f}; + + MPPMask *mask = [[MPPMask alloc] initWithUInt8Data:uint8Data width:width height:height shouldCopy:NO]; + + XCTAssertEqual(mask.width, width); + XCTAssertEqual(mask.height, height); + + // Test if UInt8 mask is not copied. + XCTAssertEqual(mask.uint8Data, (const UInt8*)uint8Data); + XCTAssertNotEqual(mask.float32Data, NULL); + + for (int i = 0 ; i < width * height ; i ++) { + XCTAssertEqualWithAccuracy(mask.float32Data[i], float32Data[i], 1e-3f, @"index i = %d", i); + } + + // Test if repeated Float32 mask accesses return the same array in memory. + XCTAssertEqual(mask.float32Data, mask.float32Data); +} + +- (void)testInitWithUInt8ArrayCopySucceeds { + + NSInteger width = 2; + NSInteger height = 3; + + UInt8 uint8Data[] = {128, 128, 128, 128, 128, 128}; + float float32Data[] = {0.501f, 0.501f, 0.501f, 0.501f, 0.501f, 0.501f}; + + MPPMask *mask = [[MPPMask alloc] initWithUInt8Data:uint8Data width:width height:height shouldCopy:YES]; + + XCTAssertEqual(mask.width, width); + XCTAssertEqual(mask.height, height); + + // Test if UInt8 mask is copied. + XCTAssertNotEqual(mask.uint8Data, (const UInt8*)uint8Data); + XCTAssertNotEqual(mask.float32Data, NULL); + + for (int i = 0 ; i < width * height ; i ++) { + XCTAssertEqualWithAccuracy(mask.float32Data[i], float32Data[i], 1e-3f); + } + + // Test if repeated Float32 mask accesses return the same array in memory. + XCTAssertEqual(mask.float32Data, mask.float32Data); +} + +- (void)testInitWithFloat32ArrayNoCopySucceeds { + + NSInteger width = 2; + NSInteger height = 3; + + UInt8 uint8Data[] = {132, 132, 132, 132, 132, 132}; + float float32Data[] = {0.52f, 0.52f, 0.52f, 0.52f, 0.52f, 0.52f}; + MPPMask *mask = [[MPPMask alloc] initWithFloat32Data:float32Data width:width height:height shouldCopy:NO]; + + XCTAssertEqual(mask.width, width); + XCTAssertEqual(mask.height, height); + + // Test if Float32 mask is not copied. + XCTAssertEqual(mask.float32Data, (const float*)float32Data); + XCTAssertNotEqual(mask.uint8Data, NULL); + + for (int i = 0 ; i < width * height ; i ++) { + XCTAssertEqual(mask.uint8Data[i], uint8Data[i]); + } + + // Test if repeated UInt8 mask accesses return the same array in memory. + XCTAssertEqual(mask.uint8Data, mask.uint8Data); +} + +- (void)testInitWithFloat32ArrayCopySucceeds { + + NSInteger width = 2; + NSInteger height = 3; + + UInt8 uint8Data[] = {132, 132, 132, 132, 132, 132}; + float float32Data[] = {0.52f, 0.52f, 0.52f, 0.52f, 0.52f, 0.52f}; + + MPPMask *mask = [[MPPMask alloc] initWithFloat32Data:float32Data width:width height:height shouldCopy:YES]; + + XCTAssertEqual(mask.width, width); + XCTAssertEqual(mask.height, height); + + // Test if Float32 mask is copied. + XCTAssertNotEqual(mask.float32Data, (const float*)float32Data); + XCTAssertNotEqual(mask.uint8Data, NULL); + + for (int i = 0 ; i < width * height ; i ++) { + XCTAssertEqual(mask.uint8Data[i], uint8Data[i]); + } + + // Test if repeated UInt8 mask accesses return the same array in memory. + XCTAssertEqual(mask.uint8Data, mask.uint8Data); +} + +@end diff --git a/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm b/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm index ea066440..548c4bdb 100644 --- a/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm +++ b/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm @@ -25,7 +25,7 @@ static NSDictionary *const kPortraitImage = @{@"name" : @"portrait", @"type" : @"jpg", @"orientation" : @(UIImageOrientationUp)}; static NSDictionary *const kPortraitRotatedImage = - @{@"name" : @"portrait_rotated", @"type" : @"jpg", @"orientation" : @(UIImageOrientationRight)}; + @{@"name" : @"portrait_rotated", @"type" : @"jpg", @"orientation" : @(UIImageOrientationLeft)}; static NSDictionary *const kCatImage = @{@"name" : @"cat", @"type" : @"jpg"}; static NSString *const kShortRangeBlazeFaceModel = @"face_detection_short_range"; static NSArray *const kPortraitExpectedKeypoints = @[ @@ -155,12 +155,12 @@ static const float kKeypointErrorThreshold = 1e-2; NSInteger iterationCount = 100; // Because of flow limiting, the callback might be invoked fewer than `iterationCount` times. An - // normal expectation will fail if expectation.fullfill() is not called + // normal expectation will fail if expectation.fulfill() is not called // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will - // only succeed if expectation is not fullfilled for the specified `expectedFulfillmentCount`. + // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`. // Since it is not possible to predict how many times the expectation is supposed to be - // fullfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and - // `expectation.isInverted = true` ensures that test succeeds if expectation is fullfilled <= + // fulfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds if expectation is fulfilled <= // `iterationCount` times. XCTestExpectation *expectation = [[XCTestExpectation alloc] initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; @@ -385,13 +385,13 @@ static const float kKeypointErrorThreshold = 1e-2; NSInteger iterationCount = 100; // Because of flow limiting, the callback might be invoked fewer than `iterationCount` times. An - // normal expectation will fail if expectation.fullfill() is not called times. An normal - // expectation will fail if expectation.fullfill() is not called + // normal expectation will fail if expectation.fulfill() is not called times. An normal + // expectation will fail if expectation.fulfill() is not called // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will - // only succeed if expectation is not fullfilled for the specified `expectedFulfillmentCount`. + // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`. // Since it it not possible to determine how many times the expectation is supposed to be - // fullfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and - // `expectation.isInverted = true` ensures that test succeeds if expectation is fullfilled <= + // fulfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds if expectation is fulfilled <= // `iterationCount` times. XCTestExpectation *expectation = [[XCTestExpectation alloc] initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; diff --git a/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm b/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm index f1d6033a..3ebc8946 100644 --- a/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm +++ b/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm @@ -174,12 +174,12 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; NSInteger iterationCount = 100; // Because of flow limiting, the callback might be invoked fewer than `iterationCount` times. An - // normal expectation will fail if expectation.fullfill() is not called + // normal expectation will fail if expectation.fulfill() is not called // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will - // only succeed if expectation is not fullfilled for the specified `expectedFulfillmentCount`. + // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`. // Since it is not possible to predict how many times the expectation is supposed to be - // fullfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and - // `expectation.isInverted = true` ensures that test succeeds if expectation is fullfilled <= + // fulfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds if expectation is fulfilled <= // `iterationCount` times. XCTestExpectation *expectation = [[XCTestExpectation alloc] initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index dcd5683f..4b4eceed 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -98,18 +98,17 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; [MPPGestureRecognizerTests filePathWithFileInfo:kExpectedThumbUpLandmarksFile]; return [MPPGestureRecognizerResult - gestureRecognizerResultsFromTextEncodedProtobufFileWithName:filePath - gestureLabel:kExpectedThumbUpLabel - shouldRemoveZPosition:YES]; + gestureRecognizerResultsFromProtobufFileWithName:filePath + gestureLabel:kExpectedThumbUpLabel + shouldRemoveZPosition:YES]; } + (MPPGestureRecognizerResult *)fistGestureRecognizerResultWithLabel:(NSString *)gestureLabel { NSString *filePath = [MPPGestureRecognizerTests filePathWithFileInfo:kExpectedFistLandmarksFile]; - return [MPPGestureRecognizerResult - gestureRecognizerResultsFromTextEncodedProtobufFileWithName:filePath - gestureLabel:gestureLabel - shouldRemoveZPosition:YES]; + return [MPPGestureRecognizerResult gestureRecognizerResultsFromProtobufFileWithName:filePath + gestureLabel:gestureLabel + shouldRemoveZPosition:YES]; } #pragma mark Assert Gesture Recognizer Results @@ -344,7 +343,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPGestureRecognizer *gestureRecognizer = [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; MPPImage *mppImage = [self imageWithFileInfo:kPointingUpRotatedImage - orientation:UIImageOrientationRight]; + orientation:UIImageOrientationLeft]; MPPGestureRecognizerResult *gestureRecognizerResult = [gestureRecognizer recognizeImage:mppImage error:nil]; @@ -655,9 +654,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; // times. An normal expectation will fail if expectation.fulfill() is not called // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`. - // Since in our case we cannot predict how many times the expectation is supposed to be fullfilled + // Since in our case we cannot predict how many times the expectation is supposed to be fulfilled // setting, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and - // `expectation.isInverted = true` ensures that test succeeds ifexpectation is fullfilled <= + // `expectation.isInverted = true` ensures that test succeeds ifexpectation is fulfilled <= // `iterationCount` times. XCTestExpectation *expectation = [[XCTestExpectation alloc] initWithDescription:@"recognizeWithLiveStream"]; diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h index cfa0a5e5..069b90b9 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h @@ -1,4 +1,4 @@ -// Copyright 2022 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,9 +19,9 @@ NS_ASSUME_NONNULL_BEGIN @interface MPPGestureRecognizerResult (ProtobufHelpers) + (MPPGestureRecognizerResult *) - gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName - gestureLabel:(NSString *)gestureLabel - shouldRemoveZPosition:(BOOL)removeZPosition; + gestureRecognizerResultsFromProtobufFileWithName:(NSString *)fileName + gestureLabel:(NSString *)gestureLabel + shouldRemoveZPosition:(BOOL)removeZPosition; @end diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm index f628499d..c6349857 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm @@ -32,9 +32,9 @@ using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; @implementation MPPGestureRecognizerResult (ProtobufHelpers) + (MPPGestureRecognizerResult *) - gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName - gestureLabel:(NSString *)gestureLabel - shouldRemoveZPosition:(BOOL)removeZPosition { + gestureRecognizerResultsFromProtobufFileWithName:(NSString *)fileName + gestureLabel:(NSString *)gestureLabel + shouldRemoveZPosition:(BOOL)removeZPosition { LandmarksDetectionResultProto landmarkDetectionResultProto; if (!get_proto_from_pbtxt(fileName.cppString, landmarkDetectionResultProto).ok()) { diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/BUILD b/mediapipe/tasks/ios/test/vision/hand_landmarker/BUILD new file mode 100644 index 00000000..1ea324b3 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/BUILD @@ -0,0 +1,62 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPHandLandmarkerObjcTestLibrary", + testonly = 1, + srcs = ["MPPHandLandmarkerTests.m"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:hand_landmarker.task", + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/test/vision/hand_landmarker/utils:MPPHandLandmarkerResultProtobufHelpers", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/vision/hand_landmarker:MPPHandLandmarker", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPHandLandmarkerObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPHandLandmarkerObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m b/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m new file mode 100644 index 00000000..36ad2ba9 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m @@ -0,0 +1,557 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h" + +static NSString *const kPbFileExtension = @"pbtxt"; + +typedef NSDictionary ResourceFileInfo; + +static ResourceFileInfo *const kHandLandmarkerBundleAssetFile = + @{@"name" : @"hand_landmarker", @"type" : @"task"}; + +static ResourceFileInfo *const kTwoHandsImage = @{@"name" : @"right_hands", @"type" : @"jpg"}; +static ResourceFileInfo *const kNoHandsImage = @{@"name" : @"cats_and_dogs", @"type" : @"jpg"}; +static ResourceFileInfo *const kThumbUpImage = @{@"name" : @"thumb_up", @"type" : @"jpg"}; +static ResourceFileInfo *const kPointingUpRotatedImage = + @{@"name" : @"pointing_up_rotated", @"type" : @"jpg"}; + +static ResourceFileInfo *const kExpectedThumbUpLandmarksFile = + @{@"name" : @"thumb_up_landmarks", @"type" : kPbFileExtension}; +static ResourceFileInfo *const kExpectedPointingUpRotatedLandmarksFile = + @{@"name" : @"pointing_up_rotated_landmarks", @"type" : kPbFileExtension}; + +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +static const float kLandmarksErrorTolerance = 0.03f; + +static NSString *const kLiveStreamTestsDictHandLandmarkerKey = @"gesture_recognizer"; +static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) + +#define AssertApproximatelyEqualLandmarks(landmark, expectedLandmark, handIndex, landmarkIndex) \ + XCTAssertEqualWithAccuracy(landmark.x, expectedLandmark.x, kLandmarksErrorTolerance, \ + @"hand index = %d landmark index j = %d", handIndex, landmarkIndex); \ + XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \ + @"hand index = %d landmark index j = %d", handIndex, landmarkIndex); + +#define AssertHandLandmarkerResultIsEmpty(handLandmarkerResult) \ + XCTAssertTrue(handLandmarkerResult.handedness.count == 0); \ + XCTAssertTrue(handLandmarkerResult.landmarks.count == 0); \ + XCTAssertTrue(handLandmarkerResult.worldLandmarks.count == 0); + +@interface MPPHandLandmarkerTests : XCTestCase { + NSDictionary *_liveStreamSucceedsTestDict; + NSDictionary *_outOfOrderTimestampTestDict; +} +@end + +@implementation MPPHandLandmarkerTests + +#pragma mark Results + ++ (MPPHandLandmarkerResult *)emptyHandLandmarkerResult { + return [[MPPHandLandmarkerResult alloc] initWithLandmarks:@[] + worldLandmarks:@[] + handedness:@[] + + timestampInMilliseconds:0]; +} + ++ (MPPHandLandmarkerResult *)thumbUpHandLandmarkerResult { + NSString *filePath = [MPPHandLandmarkerTests filePathWithFileInfo:kExpectedThumbUpLandmarksFile]; + + return [MPPHandLandmarkerResult handLandmarkerResultFromProtobufFileWithName:filePath + shouldRemoveZPosition:YES]; +} + ++ (MPPHandLandmarkerResult *)pointingUpRotatedHandLandmarkerResult { + NSString *filePath = + [MPPHandLandmarkerTests filePathWithFileInfo:kExpectedPointingUpRotatedLandmarksFile]; + + return [MPPHandLandmarkerResult handLandmarkerResultFromProtobufFileWithName:filePath + shouldRemoveZPosition:YES]; +} + +- (void)assertMultiHandLandmarks:(NSArray *> *)multiHandLandmarks + areApproximatelyEqualToExpectedMultiHandLandmarks: + (NSArray *> *)expectedMultiHandLandmarks { + XCTAssertEqual(multiHandLandmarks.count, expectedMultiHandLandmarks.count); + if (multiHandLandmarks.count == 0) { + return; + } + + NSArray *topHandLandmarks = multiHandLandmarks[0]; + NSArray *expectedTopHandLandmarks = expectedMultiHandLandmarks[0]; + + XCTAssertEqual(topHandLandmarks.count, expectedTopHandLandmarks.count); + for (int i = 0; i < expectedTopHandLandmarks.count; i++) { + MPPNormalizedLandmark *landmark = topHandLandmarks[i]; + XCTAssertNotNil(landmark); + AssertApproximatelyEqualLandmarks(landmark, expectedTopHandLandmarks[i], 0, i); + } +} + +- (void)assertMultiHandWorldLandmarks:(NSArray *> *)multiHandWorldLandmarks + areApproximatelyEqualToExpectedMultiHandWorldLandmarks: + (NSArray *> *)expectedMultiHandWorldLandmarks { + XCTAssertEqual(multiHandWorldLandmarks.count, expectedMultiHandWorldLandmarks.count); + if (expectedMultiHandWorldLandmarks.count == 0) { + return; + } + + NSArray *topHandWorldLandmarks = multiHandWorldLandmarks[0]; + NSArray *expectedTopHandWorldLandmarks = expectedMultiHandWorldLandmarks[0]; + + XCTAssertEqual(topHandWorldLandmarks.count, expectedTopHandWorldLandmarks.count); + for (int i = 0; i < expectedTopHandWorldLandmarks.count; i++) { + MPPLandmark *landmark = topHandWorldLandmarks[i]; + XCTAssertNotNil(landmark); + AssertApproximatelyEqualLandmarks(landmark, expectedTopHandWorldLandmarks[i], 0, i); + } +} + +- (void)assertHandLandmarkerResult:(MPPHandLandmarkerResult *)handLandmarkerResult + isApproximatelyEqualToExpectedResult:(MPPHandLandmarkerResult *)expectedHandLandmarkerResult { + [self assertMultiHandLandmarks:handLandmarkerResult.landmarks + areApproximatelyEqualToExpectedMultiHandLandmarks:expectedHandLandmarkerResult.landmarks]; + [self assertMultiHandWorldLandmarks:handLandmarkerResult.worldLandmarks + areApproximatelyEqualToExpectedMultiHandWorldLandmarks:expectedHandLandmarkerResult + .worldLandmarks]; +} + +#pragma mark File + ++ (NSString *)filePathWithFileInfo:(ResourceFileInfo *)fileInfo { + NSString *filePath = [MPPHandLandmarkerTests filePathWithName:fileInfo[@"name"] + extension:fileInfo[@"type"]]; + return filePath; +} + ++ (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = [[NSBundle bundleForClass:self.class] pathForResource:fileName + ofType:extension]; + return filePath; +} + +#pragma mark Hand Landmarker Initializers + +- (MPPHandLandmarkerOptions *)handLandmarkerOptionsWithModelFileInfo: + (ResourceFileInfo *)modelFileInfo { + NSString *modelPath = [MPPHandLandmarkerTests filePathWithFileInfo:modelFileInfo]; + MPPHandLandmarkerOptions *handLandmarkerOptions = [[MPPHandLandmarkerOptions alloc] init]; + handLandmarkerOptions.baseOptions.modelAssetPath = modelPath; + + return handLandmarkerOptions; +} + +- (MPPHandLandmarker *)createHandLandmarkerWithOptionsSucceeds: + (MPPHandLandmarkerOptions *)handLandmarkerOptions { + NSError *error; + MPPHandLandmarker *handLandmarker = + [[MPPHandLandmarker alloc] initWithOptions:handLandmarkerOptions error:&error]; + XCTAssertNotNil(handLandmarker); + XCTAssertNil(error); + + return handLandmarker; +} + +- (void)assertCreateHandLandmarkerWithOptions:(MPPHandLandmarkerOptions *)handLandmarkerOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPHandLandmarker *handLandmarker = + [[MPPHandLandmarker alloc] initWithOptions:handLandmarkerOptions error:&error]; + + XCTAssertNil(handLandmarker); + AssertEqualErrors(error, expectedError); +} + +#pragma mark Assert Hand Landmarker Results + +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPHandLandmarkerTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"]]; + XCTAssertNotNil(image); + + return image; +} + +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo + orientation:(UIImageOrientation)orientation { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPHandLandmarkerTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + + return image; +} + +- (MPPHandLandmarkerResult *)detectInImageWithFileInfo:(ResourceFileInfo *)imageFileInfo + usingHandLandmarker:(MPPHandLandmarker *)handLandmarker { + MPPImage *mppImage = [self imageWithFileInfo:imageFileInfo]; + MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInImage:mppImage error:nil]; + XCTAssertNotNil(handLandmarkerResult); + + return handLandmarkerResult; +} + +- (void)assertResultsOfDetectInImageWithFileInfo:(ResourceFileInfo *)fileInfo + usingHandLandmarker:(MPPHandLandmarker *)handLandmarker + approximatelyEqualsHandLandmarkerResult: + (MPPHandLandmarkerResult *)expectedHandLandmarkerResult { + MPPHandLandmarkerResult *handLandmarkerResult = [self detectInImageWithFileInfo:fileInfo + usingHandLandmarker:handLandmarker]; + [self assertHandLandmarkerResult:handLandmarkerResult + isApproximatelyEqualToExpectedResult:expectedHandLandmarkerResult]; +} + +#pragma mark General Tests + +- (void)testDetectWithModelPathSucceeds { + NSString *modelPath = + [MPPHandLandmarkerTests filePathWithFileInfo:kHandLandmarkerBundleAssetFile]; + MPPHandLandmarker *handLandmarker = [[MPPHandLandmarker alloc] initWithModelPath:modelPath + error:nil]; + XCTAssertNotNil(handLandmarker); + + [self assertResultsOfDetectInImageWithFileInfo:kThumbUpImage + usingHandLandmarker:handLandmarker + approximatelyEqualsHandLandmarkerResult:[MPPHandLandmarkerTests + thumbUpHandLandmarkerResult]]; +} + +- (void)testDetectWithEmptyResultsSucceeds { + MPPHandLandmarkerOptions *handLandmarkerOptions = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + MPPHandLandmarker *handLandmarker = + [self createHandLandmarkerWithOptionsSucceeds:handLandmarkerOptions]; + + MPPHandLandmarkerResult *handLandmarkerResult = [self detectInImageWithFileInfo:kNoHandsImage + usingHandLandmarker:handLandmarker]; + AssertHandLandmarkerResultIsEmpty(handLandmarkerResult); +} + +- (void)testDetectWithNumHandsSucceeds { + MPPHandLandmarkerOptions *handLandmarkerOptions = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + const NSInteger numHands = 2; + handLandmarkerOptions.numHands = numHands; + + MPPHandLandmarker *handLandmarker = + [self createHandLandmarkerWithOptionsSucceeds:handLandmarkerOptions]; + + MPPHandLandmarkerResult *handLandmarkerResult = [self detectInImageWithFileInfo:kTwoHandsImage + usingHandLandmarker:handLandmarker]; + + XCTAssertTrue(handLandmarkerResult.handedness.count == numHands); +} + +- (void)testDetectWithRotationSucceeds { + MPPHandLandmarkerOptions *handLandmarkerOptions = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + MPPHandLandmarker *handLandmarker = + [self createHandLandmarkerWithOptionsSucceeds:handLandmarkerOptions]; + + MPPImage *mppImage = [self imageWithFileInfo:kPointingUpRotatedImage + orientation:UIImageOrientationRight]; + + MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInImage:mppImage error:nil]; + + [self assertHandLandmarkerResult:handLandmarkerResult + isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests + pointingUpRotatedHandLandmarkerResult]]; +} + +#pragma mark Running Mode Tests + +- (void)testCreateHandLandmarkerFailsWithDelegateInNonLiveStreamMode { + MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; + for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { + MPPHandLandmarkerOptions *options = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + options.runningMode = runningModesToTest[i]; + options.handLandmarkerLiveStreamDelegate = self; + + [self + assertCreateHandLandmarkerWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in image or video mode. The " + @"delegate must not be set in the task's options." + }]]; + } +} + +- (void)testCreateHandLandmarkerFailsWithMissingDelegateInLiveStreamMode { + MPPHandLandmarkerOptions *options = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + options.runningMode = MPPRunningModeLiveStream; + + [self assertCreateHandLandmarkerWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in live stream mode. An " + @"object must be set as the delegate of the task " + @"in its options to ensure asynchronous delivery " + @"of results." + }]]; +} + +- (void)testDetectFailsWithCallingWrongApiInImageMode { + MPPHandLandmarkerOptions *options = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([handLandmarker detectAsyncInImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Image" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([handLandmarker detectInVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Image" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testDetectFailsWithCallingWrongApiInVideoMode { + MPPHandLandmarkerOptions *options = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + options.runningMode = MPPRunningModeVideo; + + MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([handLandmarker detectAsyncInImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Video" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *imageApiCallError; + XCTAssertFalse([handLandmarker detectInImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Video" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); +} + +- (void)testDetectFailsWithCallingWrongApiInLiveStreamMode { + MPPHandLandmarkerOptions *options = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + options.runningMode = MPPRunningModeLiveStream; + options.handLandmarkerLiveStreamDelegate = self; + + MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; + + NSError *imageApiCallError; + XCTAssertFalse([handLandmarker detectInImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([handLandmarker detectInVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testDetectWithVideoModeSucceeds { + MPPHandLandmarkerOptions *options = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + options.runningMode = MPPRunningModeVideo; + + MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; + + for (int i = 0; i < 3; i++) { + MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInVideoFrame:image + timestampInMilliseconds:i + error:nil]; + [self assertHandLandmarkerResult:handLandmarkerResult + isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests thumbUpHandLandmarkerResult]]; + } +} + +- (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails { + MPPHandLandmarkerOptions *options = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + options.runningMode = MPPRunningModeLiveStream; + options.handLandmarkerLiveStreamDelegate = self; + + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWiththOutOfOrderTimestampsAndLiveStream"]; + + expectation.expectedFulfillmentCount = 1; + + MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options]; + + _outOfOrderTimestampTestDict = @{ + kLiveStreamTestsDictHandLandmarkerKey : handLandmarker, + kLiveStreamTestsDictExpectationKey : expectation + }; + + MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; + + XCTAssertTrue([handLandmarker detectAsyncInImage:image timestampInMilliseconds:1 error:nil]); + + NSError *error; + XCTAssertFalse([handLandmarker detectAsyncInImage:image timestampInMilliseconds:0 error:&error]); + + NSError *expectedError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." + }]; + AssertEqualErrors(error, expectedError); + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +- (void)testDetectWithLiveStreamModeSucceeds { + MPPHandLandmarkerOptions *options = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + options.runningMode = MPPRunningModeLiveStream; + options.handLandmarkerLiveStreamDelegate = self; + + NSInteger iterationCount = 100; + + // Because of flow limiting, we cannot ensure that the callback will be invoked `iterationCount` + // times. An normal expectation will fail if expectation.fulfill() is not called + // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will + // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`. + // Since in our case we cannot predict how many times the expectation is supposed to be fullfilled + // setting, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds ifexpectation is fullfilled <= + // `iterationCount` times. + XCTestExpectation *expectation = + [[XCTestExpectation alloc] initWithDescription:@"detectWithLiveStream"]; + + expectation.expectedFulfillmentCount = iterationCount + 1; + expectation.inverted = YES; + + MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options]; + + _liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictHandLandmarkerKey : handLandmarker, + kLiveStreamTestsDictExpectationKey : expectation + }; + + // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used + // with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type + // `CMSampleBuffer`. + MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; + + for (int i = 0; i < iterationCount; i++) { + XCTAssertTrue([handLandmarker detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + } + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +- (void)handLandmarker:(MPPHandLandmarker *)handLandmarker + didFinishDetectionWithResult:(MPPHandLandmarkerResult *)handLandmarkerResult + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError *)error { + [self assertHandLandmarkerResult:handLandmarkerResult + isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests thumbUpHandLandmarkerResult]]; + + if (handLandmarker == _outOfOrderTimestampTestDict[kLiveStreamTestsDictHandLandmarkerKey]) { + [_outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } else if (handLandmarker == _liveStreamSucceedsTestDict[kLiveStreamTestsDictHandLandmarkerKey]) { + [_liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } +} + +@end diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/BUILD b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/BUILD new file mode 100644 index 00000000..9ded5386 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/BUILD @@ -0,0 +1,22 @@ +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPHandLandmarkerResultProtobufHelpers", + srcs = ["sources/MPPHandLandmarkerResult+ProtobufHelpers.mm"], + hdrs = ["sources/MPPHandLandmarkerResult+ProtobufHelpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/tasks/cc/components/containers/proto:landmarks_detection_result_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/test/vision/utils:parse_proto_utils", + "//mediapipe/tasks/ios/vision/hand_landmarker:MPPHandLandmarkerResult", + "//mediapipe/tasks/ios/vision/hand_landmarker/utils:MPPHandLandmarkerResultHelpers", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h new file mode 100644 index 00000000..d391e05e --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h @@ -0,0 +1,26 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerResult.h" + +NS_ASSUME_NONNULL_BEGIN +@interface MPPHandLandmarkerResult (ProtobufHelpers) + ++ (MPPHandLandmarkerResult *)handLandmarkerResultFromProtobufFileWithName:(NSString *)fileName + shouldRemoveZPosition:(BOOL)removeZPosition; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.mm b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.mm new file mode 100644 index 00000000..63e3f06a --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.mm @@ -0,0 +1,58 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+Helpers.h" + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/tasks/cc/components/containers/proto/landmarks_detection_result.pb.h" +#include "mediapipe/tasks/ios/test/vision/utils/sources/parse_proto_utils.h" + +namespace { +using ClassificationListProto = ::mediapipe::ClassificationList; +using ClassificationProto = ::mediapipe::Classification; +using LandmarksDetectionResultProto = + ::mediapipe::tasks::containers::proto::LandmarksDetectionResult; +using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; +} // anonymous namespace + +@implementation MPPHandLandmarkerResult (ProtobufHelpers) + ++ (MPPHandLandmarkerResult *)handLandmarkerResultFromProtobufFileWithName:(NSString *)fileName + shouldRemoveZPosition:(BOOL)removeZPosition { + LandmarksDetectionResultProto landmarkDetectionResultProto; + + if (!get_proto_from_pbtxt(fileName.cppString, landmarkDetectionResultProto).ok()) { + return nil; + } + + if (removeZPosition) { + // Remove z position of landmarks, because they are not used in correctness testing. For video + // or live stream mode, the z positions varies a lot during tracking from frame to frame. + for (int i = 0; i < landmarkDetectionResultProto.landmarks().landmark().size(); i++) { + auto &landmark = *landmarkDetectionResultProto.mutable_landmarks()->mutable_landmark(i); + landmark.clear_z(); + } + } + + return [MPPHandLandmarkerResult + handLandmarkerResultWithLandmarksProto:{landmarkDetectionResultProto.landmarks()} + worldLandmarksProto:{landmarkDetectionResultProto.world_landmarks()} + handednessProto:{landmarkDetectionResultProto.classifications()} + timestampInMilliSeconds:0]; +} + +@end diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m index 59383dad..e1bd9f6c 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m +++ b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m @@ -402,7 +402,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; ]; MPPImage *image = [self imageWithFileInfo:kBurgerRotatedImage - orientation:UIImageOrientationRight]; + orientation:UIImageOrientationLeft]; [self assertResultsOfClassifyImage:image usingImageClassifier:imageClassifier @@ -425,7 +425,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; displayName:nil] ]; MPPImage *image = [self imageWithFileInfo:kMultiObjectsRotatedImage - orientation:UIImageOrientationRight]; + orientation:UIImageOrientationLeft]; // roi around folding chair MPPImageClassifierResult *imageClassifierResult = @@ -673,10 +673,10 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; // If `expectation.isInverted = true`, the test will only succeed if // expectation is not fulfilled for the specified `expectedFulfillmentCount`. // Since in our case we cannot predict how many times the expectation is - // supposed to be fullfilled setting, + // supposed to be fulfilled setting, // `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and // `expectation.isInverted = true` ensures that test succeeds if - // expectation is fullfilled <= `iterationCount` times. + // expectation is fulfilled <= `iterationCount` times. XCTestExpectation *expectation = [[XCTestExpectation alloc] initWithDescription:@"classifyWithLiveStream"]; diff --git a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m index 2ef5a095..079682df 100644 --- a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m +++ b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m @@ -438,7 +438,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; [[MPPObjectDetectorResult alloc] initWithDetections:detections timestampInMilliseconds:0]; MPPImage *image = [self imageWithFileInfo:kCatsAndDogsRotatedImage - orientation:UIImageOrientationRight]; + orientation:UIImageOrientationLeft]; [self assertResultsOfDetectInImage:image usingObjectDetector:objectDetector diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index a97410e1..7c312456 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -64,3 +64,41 @@ objc_library( "@com_google_absl//absl/status:statusor", ], ) + +objc_library( + name = "MPPVisionTaskRunnerRefactored", + srcs = ["sources/MPPVisionTaskRunnerRefactored.mm"], + hdrs = ["sources/MPPVisionTaskRunnerRefactored.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + ":MPPImage", + ":MPPRunningMode", + ":MPPVisionPacketCreator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/core:MPPTaskRunner", + "//third_party/apple_frameworks:UIKit", + "@com_google_absl//absl/status:statusor", + ], +) + +objc_library( + name = "MPPMask", + srcs = ["sources/MPPMask.mm"], + hdrs = ["sources/MPPMask.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + ], +) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h index deffc97e..0e819cbf 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h @@ -40,10 +40,10 @@ NS_SWIFT_NAME(MPImage) @property(nonatomic, readonly) CGFloat height; /** - * The display orientation of the image. If `imageSourceType` is `MPPImageSourceTypeImage`, the + * The display orientation of the image. If `imageSourceType` is `.image`, the * default value is `image.imageOrientation`; otherwise the default value is - * `UIImageOrientationUp`. If the `MPPImage` is being used as input for any MediaPipe vision tasks - * and is set to any orientation other than `UIImageOrientationUp`, inference will be performed on + * `UIImage.Orientation.up`. If the `MPImage` is being used as input for any MediaPipe vision tasks + * and is set to any orientation other than `UIImage.Orientation.up`, inference will be performed on * a rotated copy of the image according to the orientation. */ @property(nonatomic, readonly) UIImageOrientation orientation; @@ -54,41 +54,48 @@ NS_SWIFT_NAME(MPImage) /** The source image. `nil` if `imageSourceType` is not `.image`. */ @property(nonatomic, readonly, nullable) UIImage *image; -/** The source pixel buffer. `nil` if `imageSourceType` is not `.pixelBuffer`. */ +/** The source pixel buffer. `nil` if ``imageSourceType`` is not `.pixelBuffer`. */ @property(nonatomic, readonly, nullable) CVPixelBufferRef pixelBuffer; -/** The source sample buffer. `nil` if `imageSourceType` is not `.sampleBuffer`. */ +/** The source sample buffer. `nil` if ``imageSourceType`` is not `.sampleBuffer`. */ @property(nonatomic, readonly, nullable) CMSampleBufferRef sampleBuffer; /** - * Initializes an `MPPImage` object with the given `UIImage`. - * The orientation of the newly created `MPPImage` will be `UIImageOrientationUp`. - * Hence, if this image is used as input for any MediaPipe vision tasks, inference will be - * performed on the it without any rotation. To create an `MPPImage` with a different orientation, - * please use `[MPPImage initWithImage:orientation:error:]`. + * Initializes an `MPImage` object with the given `UIImage`. + * + * The orientation of the newly created `MPImage` will be equal to the `imageOrientation` of + * `UIImage` and when sent to the vision tasks for inference, rotation will be applied accordingly. + * To create an `MPImage` with an orientation different from its `imageOrientation`, please use + * `MPImage(uiImage:orientation:)s`. * * @param image The image to use as the source. Its `CGImage` property must not be `NULL`. * @param error An optional error parameter populated when there is an error in initializing the - * `MPPImage`. + * `MPImage`. * - * @return A new `MPPImage` instance with the given image as the source. `nil` if the given + * @return A new `MPImage` instance with the given image as the source. `nil` if the given * `image` is `nil` or invalid. */ - (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error; /** - * Initializes an `MPPImage` object with the given `UIImabe` and orientation. + * Initializes an `MPImage` object with the given `UIImage` and orientation. * - * If the newly created `MPPImage` is used as input for any MediaPipe vision tasks, inference + * The given orientation will be used to calculate the rotation to be applied to the `UIImage` + * before inference is performed on it by the vision tasks. The `imageOrientation` stored in the + * `UIImage` is ignored when `MPImage` objects created by this method are sent to the vision tasks + * for inference. Use `MPImage(uiImage:)` to initialize images with the `imageOrientation` of + * `UIImage`. + * + * If the newly created `MPImage` is used as input for any MediaPipe vision tasks, inference * will be performed on a copy of the image rotated according to the orientation. * * @param image The image to use as the source. Its `CGImage` property must not be `NULL`. * @param orientation The display orientation of the image. This will be stored in the property - * `orientation`. `MPPImage`. + * `orientation` `MPImage` and will override the `imageOrientation` of the passed in `UIImage`. * @param error An optional error parameter populated when there is an error in initializing the - * `MPPImage`. + * `MPImage`. * - * @return A new `MPPImage` instance with the given image as the source. `nil` if the given + * @return A new `MPImage` instance with the given image as the source. `nil` if the given * `image` is `nil` or invalid. */ - (nullable instancetype)initWithUIImage:(UIImage *)image @@ -96,36 +103,36 @@ NS_SWIFT_NAME(MPImage) error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** - * Initializes an `MPPImage` object with the given pixel buffer. + * Initializes an `MPImage` object with the given pixel buffer. * - * The orientation of the newly created `MPPImage` will be `UIImageOrientationUp`. + * The orientation of the newly created `MPImage` will be `UIImageOrientationUp`. * Hence, if this image is used as input for any MediaPipe vision tasks, inference will be - * performed on the it without any rotation. To create an `MPPImage` with a different - * orientation, please use `[MPPImage initWithPixelBuffer:orientation:error:]`. + * performed on the it without any rotation. To create an `MPImage` with a different + * orientation, please use `MPImage(pixelBuffer:orientation:)`. * * @param pixelBuffer The pixel buffer to use as the source. It will be retained by the new - * `MPPImage` instance for the duration of its lifecycle. + * `MPImage` instance for the duration of its lifecycle. * @param error An optional error parameter populated when there is an error in initializing the - * `MPPImage`. + * `MPImage`. * - * @return A new `MPPImage` instance with the given pixel buffer as the source. `nil` if the + * @return A new `MPImage` instance with the given pixel buffer as the source. `nil` if the * given pixel buffer is `nil` or invalid. */ - (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error; /** - * Initializes an `MPPImage` object with the given pixel buffer and orientation. + * Initializes an `MPImage` object with the given pixel buffer and orientation. * - * If the newly created `MPPImage` is used as input for any MediaPipe vision tasks, inference + * If the newly created `MPImage` is used as input for any MediaPipe vision tasks, inference * will be performed on a copy of the image rotated according to the orientation. * * @param pixelBuffer The pixel buffer to use as the source. It will be retained by the new - * `MPPImage` instance for the duration of its lifecycle. + * `MPImage` instance for the duration of its lifecycle. * @param orientation The display orientation of the image. * @param error An optional error parameter populated when there is an error in initializing the - * `MPPImage`. + * `MPImage`. * - * @return A new `MPPImage` instance with the given orientation and pixel buffer as the source. + * @return A new `MPImage` instance with the given orientation and pixel buffer as the source. * `nil` if the given pixel buffer is `nil` or invalid. */ - (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer @@ -133,35 +140,35 @@ NS_SWIFT_NAME(MPImage) error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** - * Initializes an `MPPImage` object with the given sample buffer. + * Initializes an `MPImage` object with the given sample buffer. * - * The orientation of the newly created `MPPImage` will be `UIImageOrientationUp`. + * The orientation of the newly created `MPImage` will be `UIImageOrientationUp`. * Hence, if this image is used as input for any MediaPipe vision tasks, inference will be - * performed on the it without any rotation. To create an `MPPImage` with a different orientation, - * please use `[MPPImage initWithSampleBuffer:orientation:error:]`. + * performed on the it without any rotation. To create an `MPImage` with a different orientation, + * please use `MPImage(sampleBuffer:orientation:)`. * * @param sampleBuffer The sample buffer to use as the source. It will be retained by the new - * `MPPImage` instance for the duration of its lifecycle. The sample buffer must be based on + * `MPImage` instance for the duration of its lifecycle. The sample buffer must be based on * a pixel buffer (not compressed data). In practice, it should be the video output of the * camera on an iOS device, not other arbitrary types of `CMSampleBuffer`s. - * @return A new `MPPImage` instance with the given sample buffer as the source. `nil` if the + * @return A new `MPImage` instance with the given sample buffer as the source. `nil` if the * given sample buffer is `nil` or invalid. */ - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer error:(NSError **)error; /** - * Initializes an `MPPImage` object with the given sample buffer and orientation. + * Initializes an `MPImage` object with the given sample buffer and orientation. * - * If the newly created `MPPImage` is used as input for any MediaPipe vision tasks, inference + * If the newly created `MPImage` is used as input for any MediaPipe vision tasks, inference * will be performed on a copy of the image rotated according to the orientation. * * @param sampleBuffer The sample buffer to use as the source. It will be retained by the new - * `MPPImage` instance for the duration of its lifecycle. The sample buffer must be based on + * `MPImage` instance for the duration of its lifecycle. The sample buffer must be based on * a pixel buffer (not compressed data). In practice, it should be the video output of the * camera on an iOS device, not other arbitrary types of `CMSampleBuffer`s. * @param orientation The display orientation of the image. - * @return A new `MPPImage` instance with the given orientation and sample buffer as the source. + * @return A new `MPImage` instance with the given orientation and sample buffer as the source. * `nil` if the given sample buffer is `nil` or invalid. */ - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPMask.h b/mediapipe/tasks/ios/vision/core/sources/MPPMask.h new file mode 100644 index 00000000..2227baaa --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/sources/MPPMask.h @@ -0,0 +1,118 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** The underlying type of the segmentation mask. */ +typedef NS_ENUM(NSUInteger, MPPMaskDataType) { + + /** Represents the native `UInt8 *` type. */ + MPPMaskDataTypeUInt8, + + /** Represents the native `float *` type. */ + MPPMaskDataTypeFloat32, + +} NS_SWIFT_NAME(MaskDataType); + +/** + * The wrapper class for MediaPipe segmentation masks. + * + * Masks are stored as `UInt8 *` or `float *` objects. + * Every mask has an underlying type which can be accessed using `dataType`. You can access the + * mask as any other type using the appropriate properties. For example, if the underlying type is + * `MPPMaskDataTypeUInt8`, in addition to accessing the mask using `uint8Data`, you can access + * `float32Data` to get the 32 bit float data (with values ranging from 0.0 to 1.0). The first + * time you access the data as a type different from the underlying type, an expensive type + * conversion is performed. Subsequent accesses return a pointer to the memory location fo the same + * type converted array. As type conversions can be expensive, it is recommended to limit the + * accesses to data of types different from the underlying type. + * + * Masks that are returned from a MediaPipe Tasks are owned by by the underlying C++ Task. If you + * need to extend the lifetime of these objects, you can invoke the `[MPPMask copy:]` method. + */ +NS_SWIFT_NAME(Mask) +@interface MPPMask : NSObject + +/** The width of the mask. */ +@property(nonatomic, readonly) NSInteger width; + +/** The height of the mask. */ +@property(nonatomic, readonly) NSInteger height; + +/** The data type of the mask. */ +@property(nonatomic, readonly) MPPMaskDataType dataType; + +/** + * The pointer to the memory location where the underlying mask as a single channel `UInt8` array is + * stored. Uint8 values use the full value range and range from 0 to 255. + */ +@property(nonatomic, readonly, assign) const UInt8 *uint8Data; + +/** + * The pointer to the memory location where the underlying mask as a single channel float 32 array + * is stored. Float values range from 0.0 to 1.0. + */ +@property(nonatomic, readonly, assign) const float *float32Data; + +/** + * Initializes an `MPPMask` object of type `MPPMaskDataTypeUInt8` with the given `UInt8*` data, + * width and height. + * + * If `shouldCopy` is set to `YES`, the newly created `MPPMask` stores a reference to a deep copied + * `uint8Data`. Since deep copies are expensive, it is recommended to not set `shouldCopy` unless + * the `MPPMask` must outlive the passed in `uint8Data`. + * + * @param uint8Data A pointer to the memory location of the `UInt8` data array. + * @param width The width of the mask. + * @param height The height of the mask. + * @param shouldCopy The height of the mask. + * + * @return A new `MPPMask` instance with the given `UInt8*` data, width and height. + */ +- (nullable instancetype)initWithUInt8Data:(const UInt8 *)uint8Data + width:(NSInteger)width + height:(NSInteger)height + shouldCopy:(BOOL)shouldCopy NS_DESIGNATED_INITIALIZER; + +/** + * Initializes an `MPPMask` object of type `MPPMaskDataTypeFloat32` with the given `float*` data, + * width and height. + * + * If `shouldCopy` is set to `YES`, the newly created `MPPMask` stores a reference to a deep copied + * `float32Data`. Since deep copies are expensive, it is recommended to not set `shouldCopy` unless + * the `MPPMask` must outlive the passed in `float32Data`. + * + * @param float32Data A pointer to the memory location of the `float` data array. + * @param width The width of the mask. + * @param height The height of the mask. + * + * @return A new `MPPMask` instance with the given `float*` data, width and height. + */ +- (nullable instancetype)initWithFloat32Data:(const float *)float32Data + width:(NSInteger)width + height:(NSInteger)height + shouldCopy:(BOOL)shouldCopy NS_DESIGNATED_INITIALIZER; + +// TODO: Add methods for CVPixelBuffer conversion. + +/** Unavailable. */ +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPMask.mm b/mediapipe/tasks/ios/vision/core/sources/MPPMask.mm new file mode 100644 index 00000000..0d78e11d --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/sources/MPPMask.mm @@ -0,0 +1,135 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +@interface MPPMask () { + const UInt8 *_uint8Data; + const float *_float32Data; + std::unique_ptr _uint8DataPtr; + std::unique_ptr _float32DataPtr; +} +@end + +@implementation MPPMask + +- (nullable instancetype)initWithUInt8Data:(const UInt8 *)uint8Data + width:(NSInteger)width + height:(NSInteger)height + shouldCopy:(BOOL)shouldCopy { + + self = [super init]; + if (self) { + _width = width; + _height = height; + _dataType = MPPMaskDataTypeUInt8; + + if (shouldCopy) { + size_t length = _width * _height; + _uint8DataPtr = std::unique_ptr(new UInt8[length]); + _uint8Data = _uint8DataPtr.get(); + memcpy((UInt8 *)_uint8Data, uint8Data, length * sizeof(UInt8)); + } else { + _uint8Data = uint8Data; + } + } + return self; +} + +- (nullable instancetype)initWithFloat32Data:(const float *)float32Data + width:(NSInteger)width + height:(NSInteger)height + shouldCopy:(BOOL)shouldCopy { + self = [super init]; + if (self) { + _width = width; + _height = height; + _dataType = MPPMaskDataTypeFloat32; + + if (shouldCopy) { + size_t length = _width * _height; + _float32DataPtr = std::unique_ptr(new float[length]); + _float32Data = _float32DataPtr.get(); + memcpy((float *)_float32Data, float32Data, length * sizeof(float)); + } else { + _float32Data = float32Data; + } + } + return self; +} + +- (const UInt8 *)uint8Data { + switch (_dataType) { + case MPPMaskDataTypeUInt8: { + return _uint8Data; + } + case MPPMaskDataTypeFloat32: { + if (_uint8DataPtr) { + return _uint8DataPtr.get(); + } + + size_t length = _width * _height; + _uint8DataPtr = std::unique_ptr(new UInt8[length]); + UInt8 *data = _uint8DataPtr.get(); + for (int i = 0; i < length; i++) { + data[i] = _float32Data[i] * 255; + } + return data; + } + default: + return NULL; + } +} + +- (const float *)float32Data { + switch (_dataType) { + case MPPMaskDataTypeUInt8: { + if (_float32DataPtr) { + return _float32DataPtr.get(); + } + + size_t length = _width * _height; + _float32DataPtr = std::unique_ptr(new float[length]); + float *data = _float32DataPtr.get(); + for (int i = 0; i < length; i++) { + data[i] = (float)_uint8Data[i] / 255; + } + return data; + } + case MPPMaskDataTypeFloat32: { + return _float32Data; + } + default: + return NULL; + } +} + +- (id)copyWithZone:(NSZone *)zone { + switch (_dataType) { + case MPPMaskDataTypeUInt8: + return [[MPPMask alloc] initWithUInt8Data:self.uint8Data + width:self.width + height:self.height + shouldCopy:YES]; + case MPPMaskDataTypeFloat32: + return [[MPPMask alloc] initWithFloat32Data:self.float32Data + width:self.width + height:self.height + shouldCopy:YES]; + } +} + +@end diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index c1b5d058..ae5e1d64 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -30,13 +30,13 @@ using ::mediapipe::tasks::core::PacketsCallback; } // namespace /** Rotation degrees for a 90 degree rotation to the right. */ -static const NSInteger kMPPOrientationDegreesRight = -90; +static const NSInteger kMPPOrientationDegreesRight = -270; /** Rotation degrees for a 180 degree rotation. */ static const NSInteger kMPPOrientationDegreesDown = -180; /** Rotation degrees for a 90 degree rotation to the left. */ -static const NSInteger kMPPOrientationDegreesLeft = -270; +static const NSInteger kMPPOrientationDegreesLeft = -90; static NSString *const kTaskPrefix = @"com.mediapipe.tasks.vision"; @@ -165,7 +165,7 @@ static NSString *const kTaskPrefix = @"com.mediapipe.tasks.vision"; // For 90° and 270° rotations, we need to swap width and height. // This is due to the internal behavior of ImageToTensorCalculator, which: // - first denormalizes the provided rect by multiplying the rect width or height by the image - // width or height, repectively. + // width or height, respectively. // - then rotates this by denormalized rect by the provided rotation, and uses this for cropping, // - then finally rotates this back. if (rotationDegrees % 180 == 0) { diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h new file mode 100644 index 00000000..aa0307d7 --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h @@ -0,0 +1,218 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, + * execute and terminate any MediaPipe vision task. + */ +@interface MPPVisionTaskRunner : MPPTaskRunner + +/** + * Initializes a new `MPPVisionTaskRunner` with the taskInfo, running mode, whether task supports + * region of interest, packets callback, image and norm rect input stream names. Make sure that the + * packets callback is set properly based on the vision task's running mode. In case of live stream + * running mode, a C++ packets callback that is intended to deliver inference results must be + * provided. In case of image or video running mode, packets callback must be set to nil. + * + * @param taskInfo A `MPPTaskInfo` initialized by the task. + * @param runningMode MediaPipe vision task running mode. + * @param roiAllowed A `BOOL` indicating if the task supports region of interest. + * @param packetsCallback An optional C++ callback function that takes a list of output packets as + * the input argument. If provided, the callback must in turn call the block provided by the user in + * the appropriate task options. Make sure that the packets callback is set properly based on the + * vision task's running mode. In case of live stream running mode, a C++ packets callback that is + * intended to deliver inference results must be provided. In case of image or video running mode, + * packets callback must be set to nil. + * @param imageInputStreamName Name of the image input stream of the task. + * @param normRectInputStreamName Name of the norm rect input stream of the task. + * + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return An instance of `MPPVisionTaskRunner` initialized with the given the taskInfo, running + * mode, whether task supports region of interest, packets callback, image and norm rect input + * stream names. + */ + +- (nullable instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo + runningMode:(MPPRunningMode)runningMode + roiAllowed:(BOOL)roiAllowed + packetsCallback:(mediapipe::tasks::core::PacketsCallback)packetsCallback + imageInputStreamName:(NSString *)imageInputStreamName + normRectInputStreamName:(NSString *)normRectInputStreamName + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * A synchronous method to invoke the C++ task runner to process single image inputs. The call + * blocks the current thread until a failure status or a successful result is returned. + * + * This method must be used by tasks when region of interest must not be factored in for inference. + * + * @param image An `MPPImage` input to the task. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional)processImage:(MPPImage *)image + error:(NSError **)error; + +/** + * A synchronous method to invoke the C++ task runner to process single image inputs. The call + * blocks the current thread until a failure status or a successful result is returned. + * + * This method must be used by tasks when region of interest must be factored in for inference. + * When tasks which do not support region of interest calls this method in combination with any roi + * other than `CGRectZero` an error is returned. + * + * @param image An `MPPImage` input to the task. + * @param regionOfInterest A `CGRect` specifying the region of interest within the given image data + * of type `MPPImage`, on which inference should be performed. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional)processImage:(MPPImage *)image + regionOfInterest:(CGRect)regionOfInterest + error:(NSError **)error; + +/** + * A synchronous method to invoke the C++ task runner to process continuous video frames. The call + * blocks the current thread until a failure status or a successful result is returned. + * + * This method must be used by tasks when region of interest must not be factored in for inference. + * + * @param videoFrame An `MPPImage` input to the task. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional)processVideoFrame:(MPPImage *)videoFrame + timestampInMilliseconds: + (NSInteger)timeStampInMilliseconds + error:(NSError **)error; + +/** + * A synchronous method to invoke the C++ task runner to process continuous video frames. The call + * blocks the current thread until a failure status or a successful result is returned. + * + * This method must be used by tasks when region of interest must be factored in for inference. + * When tasks which do not support region of interest calls this method in combination with any roi + * other than `CGRectZero` an error is returned. + * + * @param videoFrame An `MPPImage` input to the task. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param regionOfInterest A `CGRect` specifying the region of interest within the given image data + * of type `MPPImage`, on which inference should be performed. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return An optional `PacketMap` containing pairs of output stream name and data packet. + */ +- (std::optional)processVideoFrame:(MPPImage *)videoFrame + regionOfInterest:(CGRect)regionOfInterest + timestampInMilliseconds: + (NSInteger)timeStampInMilliseconds + error:(NSError **)error; + +/** + * An asynchronous method to send live stream data to the C++ task runner. The call blocks the + * current thread until a failure status or a successful result is returned. The results will be + * available in the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. + * + * This method must be used by tasks when region of interest must not be factored in for inference. + * + * @param image An `MPPImage` input to the task. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `BOOL` indicating if the live stream data was sent to the C++ task runner successfully. + * Please note that any errors during processing of the live stream packet map will only be + * available in the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. + */ +- (BOOL)processLiveStreamImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timeStampInMilliseconds + error:(NSError **)error; + +/** + * An asynchronous method to send live stream data to the C++ task runner. The call blocks the + * current thread until a failure status or a successful result is returned. The results will be + * available in the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. + * + * This method must be used by tasks when region of interest must not be factored in for inference. + * + * @param image An `MPPImage` input to the task. + * @param regionOfInterest A `CGRect` specifying the region of interest within the given image data + * of type `MPPImage`, on which inference should be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `BOOL` indicating if the live stream data was sent to the C++ task runner successfully. + * Please note that any errors during processing of the live stream packet map will only be + * available in the user-defined `packetsCallback` that was provided during initialization of the + * `MPPVisionTaskRunner`. + */ +- (BOOL)processLiveStreamImage:(MPPImage *)image + regionOfInterest:(CGRect)regionOfInterest + timestampInMilliseconds:(NSInteger)timeStampInMilliseconds + error:(NSError **)error; + +/** + * This method returns a unique dispatch queue name by adding the given suffix and a `UUID` to the + * pre-defined queue name prefix for vision tasks. The vision tasks can use this method to get + * unique dispatch queue names which are consistent with other vision tasks. + * Dispatch queue names need not be unique, but for easy debugging we ensure that the queue names + * are unique. + * + * @param suffix A suffix that identifies a dispatch queue's functionality. + * + * @return A unique dispatch queue name by adding the given suffix and a `UUID` to the pre-defined + * queue name prefix for vision tasks. + */ ++ (const char *)uniqueDispatchQueueNameWithSuffix:(NSString *)suffix; + +- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + packetsCallback: + (mediapipe::tasks::core::PacketsCallback)packetsCallback + error:(NSError **)error NS_UNAVAILABLE; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.mm new file mode 100644 index 00000000..8a42175c --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.mm @@ -0,0 +1,331 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" + +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/rect.pb.h" + +#include + +namespace { +using ::mediapipe::NormalizedRect; +using ::mediapipe::Packet; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; +} // namespace + +/** Rotation degrees for a 90 degree rotation to the right. */ +static const NSInteger kMPPOrientationDegreesRight = -270; + +/** Rotation degrees for a 180 degree rotation. */ +static const NSInteger kMPPOrientationDegreesDown = -180; + +/** Rotation degrees for a 90 degree rotation to the left. */ +static const NSInteger kMPPOrientationDegreesLeft = -90; + +static NSString *const kTaskPrefix = @"com.mediapipe.tasks.vision"; + +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {_imageInStreamName, imagePacket}, { _normRectInStreamName, normalizedRectPacket } \ + } + +@interface MPPVisionTaskRunner () { + MPPRunningMode _runningMode; + BOOL _roiAllowed; + std::string _imageInStreamName; + std::string _normRectInStreamName; +} +@end + +@implementation MPPVisionTaskRunner + +- (nullable instancetype)initWithTaskInfo:(MPPTaskInfo *)taskInfo + runningMode:(MPPRunningMode)runningMode + roiAllowed:(BOOL)roiAllowed + packetsCallback:(PacketsCallback)packetsCallback + imageInputStreamName:(NSString *)imageInputStreamName + normRectInputStreamName:(NSString *)normRectInputStreamName + error:(NSError **)error { + _roiAllowed = roiAllowed; + _imageInStreamName = imageInputStreamName.cppString; + _normRectInStreamName = normRectInputStreamName.cppString; + + switch (runningMode) { + case MPPRunningModeImage: + case MPPRunningModeVideo: { + if (packetsCallback) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"The vision task is in image or video mode. The " + @"delegate must not be set in the task's options."]; + return nil; + } + break; + } + case MPPRunningModeLiveStream: { + if (!packetsCallback) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is in live stream mode. An object must be set as the " + @"delegate of the task in its options to ensure asynchronous delivery of " + @"results."]; + return nil; + } + break; + } + default: { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Unrecognized running mode"]; + return nil; + } + } + + _runningMode = runningMode; + self = [super initWithCalculatorGraphConfig: [taskInfo generateGraphConfig] + packetsCallback:packetsCallback + error:error]; + return self; +} + +- (std::optional)normalizedRectWithRegionOfInterest:(CGRect)roi + imageSize:(CGSize)imageSize + imageOrientation: + (UIImageOrientation)imageOrientation + error:(NSError **)error { + if (!CGRectEqualToRect(roi, CGRectZero) && !_roiAllowed) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"This task doesn't support region-of-interest."]; + return std::nullopt; + } + + CGRect calculatedRoi = CGRectEqualToRect(roi, CGRectZero) ? CGRectMake(0.0, 0.0, 1.0, 1.0) : roi; + + NormalizedRect normalizedRect; + normalizedRect.set_x_center(CGRectGetMidX(calculatedRoi)); + normalizedRect.set_y_center(CGRectGetMidY(calculatedRoi)); + + int rotationDegrees = 0; + switch (imageOrientation) { + case UIImageOrientationUp: + break; + case UIImageOrientationRight: { + rotationDegrees = kMPPOrientationDegreesRight; + break; + } + case UIImageOrientationDown: { + rotationDegrees = kMPPOrientationDegreesDown; + break; + } + case UIImageOrientationLeft: { + rotationDegrees = kMPPOrientationDegreesLeft; + break; + } + default: + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"Unsupported UIImageOrientation. `imageOrientation` cannot be equal to " + @"any of the mirrored orientations " + @"(`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`,`" + @"UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`)"]; + } + + normalizedRect.set_rotation(rotationDegrees * M_PI / kMPPOrientationDegreesDown); + + // For 90° and 270° rotations, we need to swap width and height. + // This is due to the internal behavior of ImageToTensorCalculator, which: + // - first denormalizes the provided rect by multiplying the rect width or height by the image + // width or height, respectively. + // - then rotates this by denormalized rect by the provided rotation, and uses this for cropping, + // - then finally rotates this back. + if (rotationDegrees % 180 == 0) { + normalizedRect.set_width(CGRectGetWidth(calculatedRoi)); + normalizedRect.set_height(CGRectGetHeight(calculatedRoi)); + } else { + const float width = CGRectGetHeight(calculatedRoi) * imageSize.height / imageSize.width; + const float height = CGRectGetWidth(calculatedRoi) * imageSize.width / imageSize.height; + + normalizedRect.set_width(width); + normalizedRect.set_height(height); + } + + return normalizedRect; +} + +- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image + regionOfInterest:(CGRect)roi + error:(NSError **)error { + std::optional rect = + [self normalizedRectWithRegionOfInterest:roi + imageSize:CGSizeMake(image.width, image.height) + imageOrientation:image.orientation + error:error]; + if (!rect.has_value()) { + return std::nullopt; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; + if (imagePacket.IsEmpty()) { + return std::nullopt; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + return inputPacketMap; +} + +- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image + regionOfInterest:(CGRect)roi + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional rect = + [self normalizedRectWithRegionOfInterest:roi + imageSize:CGSizeMake(image.width, image.height) + imageOrientation:image.orientation + error:error]; + if (!rect.has_value()) { + return std::nullopt; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (imagePacket.IsEmpty()) { + return std::nullopt; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() + timestampInMilliseconds:timestampInMilliseconds]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + return inputPacketMap; +} + +- (std::optional)processImage:(MPPImage *)image + regionOfInterest:(CGRect)regionOfInterest + error:(NSError **)error { + if (_runningMode != MPPRunningModeImage) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:[NSString stringWithFormat:@"The vision task is not initialized with " + @"image mode. Current Running Mode: %@", + MPPRunningModeDisplayName(_runningMode)]]; + return std::nullopt; + } + + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + regionOfInterest:regionOfInterest + error:error]; + if (!inputPacketMap.has_value()) { + return std::nullopt; + } + + return [self processPacketMap:inputPacketMap.value() error:error]; +} + +- (std::optional)processImage:(MPPImage *)image error:(NSError **)error { + return [self processImage:image regionOfInterest:CGRectZero error:error]; +} + +- (std::optional)processVideoFrame:(MPPImage *)videoFrame + regionOfInterest:(CGRect)regionOfInterest + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + if (_runningMode != MPPRunningModeVideo) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:[NSString stringWithFormat:@"The vision task is not initialized with " + @"video mode. Current Running Mode: %@", + MPPRunningModeDisplayName(_runningMode)]]; + return std::nullopt; + } + + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:videoFrame + regionOfInterest:regionOfInterest + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return std::nullopt; + } + + return [self processPacketMap:inputPacketMap.value() error:error]; +} + +- (std::optional)processVideoFrame:(MPPImage *)videoFrame + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + return [self processVideoFrame:videoFrame + regionOfInterest:CGRectZero + timestampInMilliseconds:timestampInMilliseconds + error:error]; +} + +- (BOOL)processLiveStreamImage:(MPPImage *)image + regionOfInterest:(CGRect)regionOfInterest + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + if (_runningMode != MPPRunningModeLiveStream) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:[NSString stringWithFormat:@"The vision task is not initialized with " + @"live stream mode. Current Running Mode: %@", + MPPRunningModeDisplayName(_runningMode)]]; + return NO; + } + + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + regionOfInterest:regionOfInterest + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return NO; + } + + return [self sendPacketMap:inputPacketMap.value() error:error]; +} + +- (BOOL)processLiveStreamImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + return [self processLiveStreamImage:image + regionOfInterest:CGRectZero + timestampInMilliseconds:timestampInMilliseconds + error:error]; +} + ++ (const char *)uniqueDispatchQueueNameWithSuffix:(NSString *)suffix { + return [NSString stringWithFormat:@"%@.%@_%@", kTaskPrefix, suffix, [NSString uuidString]] + .UTF8String; +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/BUILD b/mediapipe/tasks/ios/vision/face_detector/BUILD index e4fc1561..eb34da1b 100644 --- a/mediapipe/tasks/ios/vision/face_detector/BUILD +++ b/mediapipe/tasks/ios/vision/face_detector/BUILD @@ -55,7 +55,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/face_detector/utils:MPPFaceDetectorOptionsHelpers", "//mediapipe/tasks/ios/vision/face_detector/utils:MPPFaceDetectorResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h index 78f2fafb..3dec361a 100644 --- a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h @@ -57,27 +57,23 @@ NS_SWIFT_NAME(FaceDetector) @interface MPPFaceDetector : NSObject /** - * Creates a new instance of `MPPFaceDetector` from an absolute path to a TensorFlow Lite model - * file stored locally on the device and the default `MPPFaceDetector`. + * Creates a new instance of `FaceDetector` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `FaceDetector`. * * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. - * @param error An optional error parameter populated when there is an error in initializing the - * face detector. * - * @return A new instance of `MPPFaceDetector` with the given model path. `nil` if there is an + * @return A new instance of `FaceDetector` with the given model path. `nil` if there is an * error in initializing the face detector. */ - (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; /** - * Creates a new instance of `MPPFaceDetector` from the given `MPPFaceDetectorOptions`. + * Creates a new instance of `FaceDetector` from the given `FaceDetectorOptions`. * - * @param options The options of type `MPPFaceDetectorOptions` to use for configuring the - * `MPPFaceDetector`. - * @param error An optional error parameter populated when there is an error in initializing the - * face detector. + * @param options The options of type `FaceDetectorOptions` to use for configuring the + * `FaceDetector`. * - * @return A new instance of `MPPFaceDetector` with the given options. `nil` if there is an error + * @return A new instance of `FaceDetector` with the given options. `nil` if there is an error * in initializing the face detector. */ - (nullable instancetype)initWithOptions:(MPPFaceDetectorOptions *)options @@ -86,23 +82,21 @@ NS_SWIFT_NAME(FaceDetector) /** * Performs face detection on the provided MPPImage using the whole image as region of * interest. Rotation will be applied according to the `orientation` property of the provided - * `MPPImage`. Only use this method when the `MPPFaceDetector` is created with - * `MPPRunningModeImage`. + * `MPImage`. Only use this method when the `MPPFaceDetector` is created with running mode + * `.image`. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports classification of RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the + * following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * If your `MPImage` has a source type of `.image` ensure that the color space is * RGB with an Alpha channel. * - * @param image The `MPPImage` on which face detection is to be performed. - * @param error An optional error parameter populated when there is an error in performing face - * detection on the input image. + * @param image The `MPImage` on which face detection is to be performed. * - * @return An `MPPFaceDetectorResult` face that contains a list of detections, each detection + * @return An `FaceDetectorResult` face that contains a list of detections, each detection * has a bounding box that is expressed in the unrotated input frame of reference coordinates * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying * image data. @@ -111,27 +105,25 @@ NS_SWIFT_NAME(FaceDetector) error:(NSError **)error NS_SWIFT_NAME(detect(image:)); /** - * Performs face detection on the provided video frame of type `MPPImage` using the whole + * Performs face detection on the provided video frame of type `MPImage` using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPFaceDetector` is created with - * `MPPRunningModeVideo`. + * the provided `MPImage`. Only use this method when the `FaceDetector` is created with running + * mode `.video`. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports classification of RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the + * following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image The `MPPImage` on which face detection is to be performed. + * @param image The `MPImage` on which face detection is to be performed. * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input * timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing face - * detection on the input image. * - * @return An `MPPFaceDetectorResult` face that contains a list of detections, each detection + * @return An `FaceDetectorResult` face that contains a list of detections, each detection * has a bounding box that is expressed in the unrotated input frame of reference coordinates * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying * image data. @@ -142,39 +134,37 @@ NS_SWIFT_NAME(FaceDetector) NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); /** - * Sends live stream image data of type `MPPImage` to perform face detection using the whole + * Sends live stream image data of type `MPImage` to perform face detection using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPFaceDetector` is created with - * `MPPRunningModeLiveStream`. + * the provided `MPImage`. Only use this method when the `FaceDetector` is created with + * `.liveStream`. * * The object which needs to be continuously notified of the available results of face - * detection must confirm to `MPPFaceDetectorLiveStreamDelegate` protocol and implement the - * `faceDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. + * detection must confirm to `FaceDetectorLiveStreamDelegate` protocol and implement the + * `faceDetector(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` delegate method. * * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the face detector. The input timestamps must be monotonically increasing. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports classification of RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the + * following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * If the input `MPImage` has a source type of `.image` ensure that the color * space is RGB with an Alpha channel. * * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its * `videoSettings` property. * - * @param image A live stream image data of type `MPPImage` on which face detection is to be + * @param image A live stream image data of type `MPImage` on which face detection is to be * performed. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image is sent to the face detector. The input timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing face - * detection on the input live stream image data. * - * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + * @return `true` if the image was sent to the task successfully, otherwise `false`. */ - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm index 7cb525fb..6da599fd 100644 --- a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm @@ -18,12 +18,10 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h" -using ::mediapipe::NormalizedRect; -using ::mediapipe::Packet; using ::mediapipe::Timestamp; using ::mediapipe::tasks::core::PacketMap; using ::mediapipe::tasks::core::PacketsCallback; @@ -49,6 +47,12 @@ static NSString *const kTaskName = @"faceDetector"; } \ } +#define FaceDetectorResultWithOutputPacketMap(outputPacketMap) \ + ( \ + [MPPFaceDetectorResult \ + faceDetectorResultWithDetectionsPacket:outputPacketMap[kDetectionsStreamName.cppString]] \ + ) + @interface MPPFaceDetector () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -102,11 +106,13 @@ static NSString *const kTaskName = @"faceDetector"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectStreamName + error:error]; if (!_visionTaskRunner) { return nil; @@ -124,95 +130,29 @@ static NSString *const kTaskName = @"faceDetector"; return [self initWithOptions:options error:error]; } -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; -} - - (nullable MPPFaceDetectorResult *)detectInImage:(MPPImage *)image error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPFaceDetectorResult - faceDetectorResultWithDetectionsPacket:outputPacketMap - .value()[kDetectionsStreamName.cppString]]; + return [MPPFaceDetector faceDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPFaceDetectorResult *)detectInVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPFaceDetectorResult - faceDetectorResultWithDetectionsPacket:outputPacketMap - .value()[kDetectionsStreamName.cppString]]; + return [MPPFaceDetector faceDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; - } - - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; } - (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { @@ -237,9 +177,7 @@ static NSString *const kTaskName = @"faceDetector"; return; } - MPPFaceDetectorResult *result = [MPPFaceDetectorResult - faceDetectorResultWithDetectionsPacket:liveStreamResult - .value()[kDetectionsStreamName.cppString]]; + MPPFaceDetectorResult *result = FaceDetectorResultWithOutputPacketMap(liveStreamResult.value()); NSInteger timeStampInMilliseconds = outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / @@ -252,4 +190,13 @@ static NSString *const kTaskName = @"faceDetector"; }); } ++ (nullable MPPFaceDetectorResult *)faceDetectorResultWithOptionalOutputPacketMap: + (std::optional)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return FaceDetectorResultWithOutputPacketMap(outputPacketMap.value()); +} + @end diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h index b5d65268..51d52c3a 100644 --- a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h @@ -23,11 +23,11 @@ NS_ASSUME_NONNULL_BEGIN @class MPPFaceDetector; /** - * This protocol defines an interface for the delegates of `MPPFaceDetector` face to receive + * This protocol defines an interface for the delegates of `FaceDetector` face to receive * results of performing asynchronous face detection on images (i.e, when `runningMode` = - * `MPPRunningModeLiveStream`). + * `.liveStream`). * - * The delegate of `MPPFaceDetector` must adopt `MPPFaceDetectorLiveStreamDelegate` protocol. + * The delegate of `FaceDetector` must adopt `FaceDetectorLiveStreamDelegate` protocol. * The methods in this protocol are optional. */ NS_SWIFT_NAME(FaceDetectorLiveStreamDelegate) @@ -37,14 +37,14 @@ NS_SWIFT_NAME(FaceDetectorLiveStreamDelegate) /** * This method notifies a delegate that the results of asynchronous face detection of - * an image submitted to the `MPPFaceDetector` is available. + * an image submitted to the `FaceDetector` is available. * - * This method is called on a private serial dispatch queue created by the `MPPFaceDetector` + * This method is called on a private serial dispatch queue created by the `FaceDetector` * for performing the asynchronous delegates calls. * * @param faceDetector The face detector which performed the face detection. - * This is useful to test equality when there are multiple instances of `MPPFaceDetector`. - * @param result The `MPPFaceDetectorResult` object that contains a list of detections, each + * This is useful to test equality when there are multiple instances of `FaceDetector`. + * @param result The `FaceDetectorResult` object that contains a list of detections, each * detection has a bounding box that is expressed in the unrotated input frame of reference * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the * underlying image data. @@ -60,26 +60,26 @@ NS_SWIFT_NAME(FaceDetectorLiveStreamDelegate) NS_SWIFT_NAME(faceDetector(_:didFinishDetection:timestampInMilliseconds:error:)); @end -/** Options for setting up a `MPPFaceDetector`. */ +/** Options for setting up a `FaceDetector`. */ NS_SWIFT_NAME(FaceDetectorOptions) @interface MPPFaceDetectorOptions : MPPTaskOptions /** - * Running mode of the face detector task. Defaults to `MPPRunningModeImage`. - * `MPPFaceDetector` can be created with one of the following running modes: - * 1. `MPPRunningModeImage`: The mode for performing face detection on single image inputs. - * 2. `MPPRunningModeVideo`: The mode for performing face detection on the decoded frames of a + * Running mode of the face detector task. Defaults to `.image`. + * `FaceDetector` can be created with one of the following running modes: + * 1. `.image`: The mode for performing face detection on single image inputs. + * 2. `.video`: The mode for performing face detection on the decoded frames of a * video. - * 3. `MPPRunningModeLiveStream`: The mode for performing face detection on a live stream of + * 3. `.liveStream`: The mode for performing face detection on a live stream of * input data, such as from the camera. */ @property(nonatomic) MPPRunningMode runningMode; /** - * An object that confirms to `MPPFaceDetectorLiveStreamDelegate` protocol. This object must - * implement `faceDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` to receive - * the results of performing asynchronous face detection on images (i.e, when `runningMode` = - * `MPPRunningModeLiveStream`). + * An object that confirms to `FaceDetectorLiveStreamDelegate` protocol. This object must + * implement `faceDetector(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` to + * receive the results of performing asynchronous face detection on images (i.e, when `runningMode` + * = `.liveStream`). */ @property(nonatomic, weak, nullable) id faceDetectorLiveStreamDelegate; diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m index 7d990aa6..9ea57395 100644 --- a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m @@ -28,6 +28,7 @@ - (id)copyWithZone:(NSZone *)zone { MPPFaceDetectorOptions *faceDetectorOptions = [super copyWithZone:zone]; + faceDetectorOptions.runningMode = self.runningMode; faceDetectorOptions.minDetectionConfidence = self.minDetectionConfidence; faceDetectorOptions.minSuppressionThreshold = self.minSuppressionThreshold; faceDetectorOptions.faceDetectorLiveStreamDelegate = self.faceDetectorLiveStreamDelegate; diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h index 67a9082a..e2986d06 100644 --- a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h @@ -18,27 +18,27 @@ NS_ASSUME_NONNULL_BEGIN -/** Represents the detection results generated by `MPPFaceDetector`. */ +/** Represents the detection results generated by `FaceDetector`. */ NS_SWIFT_NAME(FaceDetectorResult) @interface MPPFaceDetectorResult : MPPTaskResult /** - * The array of `MPPDetection` objects each of which has a bounding box that is expressed in the + * The array of `Detection` objects each of which has a bounding box that is expressed in the * unrotated input frame of reference coordinates system, i.e. in `[0,image_width) x * [0,image_height)`, which are the dimensions of the underlying image data. */ @property(nonatomic, readonly) NSArray *detections; /** - * Initializes a new `MPPFaceDetectorResult` with the given array of detections and timestamp (in + * Initializes a new `FaceDetectorResult` with the given array of detections and timestamp (in * milliseconds). * - * @param detections An array of `MPPDetection` objects each of which has a bounding box that is + * @param detections An array of `Detection` objects each of which has a bounding box that is * expressed in the unrotated input frame of reference coordinates system, i.e. in `[0,image_width) * x [0,image_height)`, which are the dimensions of the underlying image data. * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. * - * @return An instance of `MPPFaceDetectorResult` initialized with the given array of detections + * @return An instance of `FaceDetectorResult` initialized with the given array of detections * and timestamp (in milliseconds). */ - (instancetype)initWithDetections:(NSArray *)detections diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h index 02bb84ac..6c5c3751 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h @@ -30,27 +30,23 @@ NS_SWIFT_NAME(FaceLandmarker) @interface MPPFaceLandmarker : NSObject /** - * Creates a new instance of `MPPFaceLandmarker` from an absolute path to a TensorFlow Lite model - * file stored locally on the device and the default `MPPFaceLandmarker`. + * Creates a new instance of `FaceLandmarker` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `FaceLandmarker`. * * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. - * @param error An optional error parameter populated when there is an error in initializing the - * face landmaker. * - * @return A new instance of `MPPFaceLandmarker` with the given model path. `nil` if there is an + * @return A new instance of `FaceLandmarker` with the given model path. `nil` if there is an * error in initializing the face landmaker. */ - (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; /** - * Creates a new instance of `MPPFaceLandmarker` from the given `MPPFaceLandmarkerOptions`. + * Creates a new instance of `FaceLandmarker` from the given `FaceLandmarkerOptions`. * - * @param options The options of type `MPPFaceLandmarkerOptions` to use for configuring the + * @param options The options of type `FaceLandmarkerOptions` to use for configuring the * `MPPFaceLandmarker`. - * @param error An optional error parameter populated when there is an error in initializing the - * face landmaker. * - * @return A new instance of `MPPFaceLandmarker` with the given options. `nil` if there is an error + * @return A new instance of `FaceLandmarker` with the given options. `nil` if there is an error * in initializing the face landmaker. */ - (nullable instancetype)initWithOptions:(MPPFaceLandmarkerOptions *)options @@ -59,49 +55,45 @@ NS_SWIFT_NAME(FaceLandmarker) /** * Performs face landmark detection on the provided MPPImage using the whole image as region of * interest. Rotation will be applied according to the `orientation` property of the provided - * `MPPImage`. Only use this method when the `MPPFaceLandmarker` is created with - * `MPPRunningModeImage`. + * `MPImage`. Only use this method when the `FaceLandmarker` is created with `.image`. * - * This method supports RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports RGBA images. If your `MPPImage` has a source type of `.pixelBuffer` or + * `.sampleBuffer`, the underlying pixel buffer must have one of the following pixel format + * types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an + * Alpha channel. * - * @param image The `MPPImage` on which face landmark detection is to be performed. - * @param error An optional error parameter populated when there is an error in performing face - * landmark detection on the input image. + * @param image The `MPImage` on which face landmark detection is to be performed. * - * @return An `MPPFaceLandmarkerResult` that contains a list of landmarks. + * @return An `MPPFaceLandmarkerResult` that contains a list of landmarks. `nil` if there is an + * error in initializing the face landmaker. */ - (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error NS_SWIFT_NAME(detect(image:)); /** - * Performs face landmark detection on the provided video frame of type `MPPImage` using the whole + * Performs face landmark detection on the provided video frame of type `MPImage` using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPFaceLandmarker` is created with - * `MPPRunningModeVideo`. + * the provided `MPImage`. Only use this method when the `MPPFaceLandmarker` is created with + * running mode `.video`. * - * This method supports RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports RGBA images. If your `MPImage` has a source type of `.pixelBuffer` or + * `.sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image The `MPPImage` on which face landmark detection is to be performed. + * @param image The `MPImage` on which face landmark detection is to be performed. * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input * timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing face - * landmark detection on the input image. * - * @return An `MPPFaceLandmarkerResult` that contains a list of landmarks. + * @return An `FaceLandmarkerResult` that contains a list of landmarks. `nil` if there is an + * error in initializing the face landmaker. */ - (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds @@ -109,39 +101,36 @@ NS_SWIFT_NAME(FaceLandmarker) NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); /** - * Sends live stream image data of type `MPPImage` to perform face landmark detection using the + * Sends live stream image data of type `MPImage` to perform face landmark detection using the * whole image as region of interest. Rotation will be applied according to the `orientation` - * property of the provided `MPPImage`. Only use this method when the `MPPFaceLandmarker` is created - * with `MPPRunningModeLiveStream`. + * property of the provided `MPImage`. Only use this method when the `FaceLandmarker` is created + * with `.liveStream`. * * The object which needs to be continuously notified of the available results of face - * detection must confirm to `MPPFaceLandmarkerLiveStreamDelegate` protocol and implement the - * `faceLandmarker:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. + * detection must confirm to `FaceLandmarkerLiveStreamDelegate` protocol and implement the + * `faceLandmarker(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` delegate method. * * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the face detector. The input timestamps must be monotonically increasing. * - * This method supports RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports RGBA images. If your `MPImage` has a source type of `.pixelBuffer` or + * `.sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color - * space is RGB with an Alpha channel. + * If the input `MPImage` has a source type of `.image` ensure that the color space is RGB with an + * Alpha channel. * * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its * `videoSettings` property. * - * @param image A live stream image data of type `MPPImage` on which face landmark detection is to - * be performed. + * @param image A live stream image data of type `MPImage` on which face landmark detection is to be + * performed. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image is sent to the face detector. The input timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error when sending the input - * image to the graph. * - * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + * @return `true` if the image was sent to the task successfully, otherwise `false`. */ - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h index 23b423ad..8aa3a40c 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.h @@ -23,26 +23,26 @@ NS_ASSUME_NONNULL_BEGIN @class MPPFaceLandmarker; /** - * This protocol defines an interface for the delegates of `MPPFaceLandmarker` face to receive + * This protocol defines an interface for the delegates of `FaceLandmarker` face to receive * results of performing asynchronous face detection on images (i.e, when `runningMode` = - * `MPPRunningModeLiveStream`). + * `.liveStream`). * - * The delegate of `MPPFaceLandmarker` must adopt `MPPFaceLandmarkerLiveStreamDelegate` protocol. + * The delegate of `FaceLandmarker` must adopt `FaceLandmarkerLiveStreamDelegate` protocol. * The methods in this protocol are optional. */ -NS_SWIFT_NAME(FaceDetectorLiveStreamDelegate) +NS_SWIFT_NAME(FaceLandmarkerLiveStreamDelegate) @protocol MPPFaceLandmarkerLiveStreamDelegate /** * This method notifies a delegate that the results of asynchronous face detection of - * an image submitted to the `MPPFaceLandmarker` is available. + * an image submitted to the `FaceLandmarker` is available. * - * This method is called on a private serial dispatch queue created by the `MPPFaceLandmarker` + * This method is called on a private serial dispatch queue created by the `FaceLandmarker` * for performing the asynchronous delegates calls. * * @param faceLandmarker The face landmarker which performed the face landmark detctions. - * This is useful to test equality when there are multiple instances of `MPPFaceLandmarker`. - * @param result The `MPPFaceLandmarkerResult` object that contains a list of landmarks. + * This is useful to test equality when there are multiple instances of `FaceLandmarker`. + * @param result The `FaceLandmarkerResult` object that contains a list of landmarks. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image was sent to the face detector. * @param error An optional error parameter populated when there is an error in performing face @@ -55,26 +55,25 @@ NS_SWIFT_NAME(FaceDetectorLiveStreamDelegate) NS_SWIFT_NAME(faceLandmarker(_:didFinishDetection:timestampInMilliseconds:error:)); @end -/** Options for setting up a `MPPFaceLandmarker`. */ +/** Options for setting up a `FaceLandmarker`. */ NS_SWIFT_NAME(FaceLandmarkerOptions) @interface MPPFaceLandmarkerOptions : MPPTaskOptions /** - * Running mode of the face landmark dection task. Defaults to `MPPRunningModeImage`. - * `MPPFaceLandmarker` can be created with one of the following running modes: - * 1. `MPPRunningModeImage`: The mode for performing face detection on single image inputs. - * 2. `MPPRunningModeVideo`: The mode for performing face detection on the decoded frames of a - * video. - * 3. `MPPRunningModeLiveStream`: The mode for performing face detection on a live stream of - * input data, such as from the camera. + * Running mode of the face landmark dection task. Defaults to `.image`. `FaceLandmarker` can be + * created with one of the following running modes: + * 1. `.image`: The mode for performing face detection on single image inputs. + * 2. `.video`: The mode for performing face detection on the decoded frames of a video. + * 3. `.liveStream`: The mode for performing face detection on a live stream of input data, such as + * from the camera. */ @property(nonatomic) MPPRunningMode runningMode; /** - * An object that confirms to `MPPFaceLandmarkerLiveStreamDelegate` protocol. This object must - * implement `faceLandmarker:didFinishDetectionWithResult:timestampInMilliseconds:error:` to receive - * the results of performing asynchronous face landmark detection on images (i.e, when `runningMode` - * = `MPPRunningModeLiveStream`). + * An object that confirms to `FaceLandmarkerLiveStreamDelegate` protocol. This object must + * implement `faceLandmarker(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` to + * receive the results of performing asynchronous face landmark detection on images (i.e, when + * `runningMode` = `.liveStream`). */ @property(nonatomic, weak, nullable) id faceLandmarkerLiveStreamDelegate; diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m index ebef092f..3438ed8d 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerOptions.m @@ -33,6 +33,7 @@ - (id)copyWithZone:(NSZone *)zone { MPPFaceLandmarkerOptions *faceLandmarkerOptions = [super copyWithZone:zone]; + faceLandmarkerOptions.runningMode = self.runningMode; faceLandmarkerOptions.numFaces = self.numFaces; faceLandmarkerOptions.minFaceDetectionConfidence = self.minFaceDetectionConfidence; faceLandmarkerOptions.minFacePresenceConfidence = self.minFacePresenceConfidence; diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h index c517ec15..8ff8e984 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h @@ -54,7 +54,7 @@ NS_SWIFT_NAME(TransformMatrix) @end -/** Represents the detection results generated by `MPPFaceLandmarker`. */ +/** Represents the detection results generated by `FaceLandmarker`. */ NS_SWIFT_NAME(FaceLandmarkerResult) @interface MPPFaceLandmarkerResult : MPPTaskResult @@ -72,16 +72,16 @@ NS_SWIFT_NAME(FaceLandmarkerResult) @property(nonatomic, readonly) NSArray *facialTransformationMatrixes; /** - * Initializes a new `MPPFaceLandmarkerResult` with the given array of landmarks, blendshapes, + * Initializes a new `FaceLandmarkerResult` with the given array of landmarks, blendshapes, * facialTransformationMatrixes and timestamp (in milliseconds). * - * @param faceLandmarks An array of `MPPNormalizedLandmark` objects. - * @param faceBlendshapes An array of `MPPClassifications` objects. + * @param faceLandmarks An array of `NormalizedLandmark` objects. + * @param faceBlendshapes An array of `Classifications` objects. * @param facialTransformationMatrixes An array of flattended matrices. * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. * - * @return An instance of `MPPFaceLandmarkerResult` initialized with the given array of detections - * and timestamp (in milliseconds). + * @return An instance of `FaceLandmarkerResult` initialized with the given array of detections and + * timestamp (in milliseconds). */ - (instancetype)initWithFaceLandmarks:(NSArray *> *)faceLandmarks faceBlendshapes:(NSArray *)faceBlendshapes diff --git a/mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h b/mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h index 422e1bf0..b27bd267 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h +++ b/mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h @@ -32,7 +32,7 @@ NS_ASSUME_NONNULL_BEGIN * @param transformationMatrixesPacket a MediaPipe packet wrapping a * `std::vector`. * - * @return An `MPPFaceLandmarkerResult` object that contains the contenst of the provided packets. + * @return An `MPPFaceLandmarkerResult` object that contains the contents of the provided packets. */ + (MPPFaceLandmarkerResult *) faceLandmarkerResultWithLandmarksPacket:(const ::mediapipe::Packet &)landmarksPacket diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h index 5149ec0a..5a954af4 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h @@ -29,6 +29,24 @@ NS_ASSUME_NONNULL_BEGIN NS_SWIFT_NAME(HandLandmarker) @interface MPPHandLandmarker : NSObject +/** The array of connections between the landmarks in the palm. */ +@property(class, nonatomic, readonly) NSArray *handPalmConnections; + +/** The array of connections between the landmarks in the index finger. */ +@property(class, nonatomic, readonly) NSArray *handIndexFingerConnections; + +/** The array of connections between the landmarks in the middle finger. */ +@property(class, nonatomic, readonly) NSArray *handMiddleFingerConnections; + +/** The array of connections between the landmarks in the ring finger. */ +@property(class, nonatomic, readonly) NSArray *handRingFingerConnections; + +/** The array of connections between the landmarks in the pinky. */ +@property(class, nonatomic, readonly) NSArray *handPinkyConnections; + +/** The array of connections between all the landmarks in the hand. */ +@property(class, nonatomic, readonly) NSArray *handConnections; + /** * Creates a new instance of `MPPHandLandmarker` from an absolute path to a model asset bundle * stored locally on the device and the default `MPPHandLandmarkerOptions`. @@ -156,48 +174,6 @@ NS_SWIFT_NAME(HandLandmarker) - (instancetype)init NS_UNAVAILABLE; -/** - * Returns the connections between the landmarks in the palm. - * - * @return An array of connections between the landmarks in the palm. - */ -+ (NSArray *)handPalmConnections; - -/** - * Returns the connections between the landmarks in the index finger. - * - * @return An array of connections between the landmarks in the index finger. - */ -+ (NSArray *)handIndexFingerConnections; - -/** - * Returns the connections between the landmarks in the middle finger. - * - * @return An array of connections between the landmarks in the middle finger. - */ -+ (NSArray *)handMiddleFingerConnections; - -/** - * Returns the connections between the landmarks in the ring finger. - * - * @return An array of connections between the landmarks in the ring finger. - */ -+ (NSArray *)handRingFingerConnections; - -/** - * Returns the connections between the landmarks in the pinky. - * - * @return An array of connections between the landmarks in the pinky. - */ -+ (NSArray *)handPinkyConnections; - -/** - * Returns the connections between all the landmarks in the hand. - * - * @return An array of connections between all the landmarks in the hand. - */ -+ (NSArray *)handConnections; - + (instancetype)new NS_UNAVAILABLE; @end diff --git a/mediapipe/tasks/ios/vision/image_classifier/BUILD b/mediapipe/tasks/ios/vision/image_classifier/BUILD index cf89249c..daff017d 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/BUILD +++ b/mediapipe/tasks/ios/vision/image_classifier/BUILD @@ -57,7 +57,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/image_classifier/utils:MPPImageClassifierOptionsHelpers", "//mediapipe/tasks/ios/vision/image_classifier/utils:MPPImageClassifierResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h index 6b81a240..a22dc632 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h @@ -53,28 +53,24 @@ NS_SWIFT_NAME(ImageClassifier) @interface MPPImageClassifier : NSObject /** - * Creates a new instance of `MPPImageClassifier` from an absolute path to a TensorFlow Lite model - * file stored locally on the device and the default `MPPImageClassifierOptions`. + * Creates a new instance of `ImageClassifier` from an absolute path to a TensorFlow Lite model file + * stored locally on the device and the default `ImageClassifierOptions`. * * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. - * @param error An optional error parameter populated when there is an error in initializing the - * image classifier. * - * @return A new instance of `MPPImageClassifier` with the given model path. `nil` if there is an + * @return A new instance of `ImageClassifier` with the given model path. `nil` if there is an * error in initializing the image classifier. */ - (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; /** - * Creates a new instance of `MPPImageClassifier` from the given `MPPImageClassifierOptions`. + * Creates a new instance of `ImageClassifier` from the given `ImageClassifierOptions`. * - * @param options The options of type `MPPImageClassifierOptions` to use for configuring the - * `MPPImageClassifier`. - * @param error An optional error parameter populated when there is an error in initializing the - * image classifier. + * @param options The options of type `ImageClassifierOptions` to use for configuring the + * `ImageClassifier`. * - * @return A new instance of `MPPImageClassifier` with the given options. `nil` if there is an error - * in initializing the image classifier. + * @return A new instance of `ImageClassifier` with the given options. `nil` if there is an error in + * initializing the image classifier. */ - (nullable instancetype)initWithOptions:(MPPImageClassifierOptions *)options error:(NSError **)error NS_DESIGNATED_INITIALIZER; @@ -82,49 +78,46 @@ NS_SWIFT_NAME(ImageClassifier) /** * Performs image classification on the provided MPPImage using the whole image as region of * interest. Rotation will be applied according to the `orientation` property of the provided - * `MPPImage`. Only use this method when the `MPPImageClassifier` is created with - * `MPPRunningModeImage`. - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * `MPImage`. Only use this method when the `ImageClassifier` is created with running mode, + * `.image`. + * + * This method supports classification of RGBA images. If your `MPImage` has a source type + * ofm`.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. * * @param image The `MPPImage` on which image classification is to be performed. - * @param error An optional error parameter populated when there is an error in performing image - * classification on the input image. * - * @return An `MPPImageClassifierResult` object that contains a list of image classifications. + * @return An `ImageClassifierResult` object that contains a list of image classifications. */ - (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image error:(NSError **)error NS_SWIFT_NAME(classify(image:)); /** - * Performs image classification on the provided `MPPImage` cropped to the specified region of + * Performs image classification on the provided `MPImage` cropped to the specified region of * interest. Rotation will be applied on the cropped image according to the `orientation` property - * of the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with - * `MPPRunningModeImage`. + * of the provided `MPImage`. Only use this method when the `MPPImageClassifier` is created with + * running mode, `.image`. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports classification of RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image The `MPPImage` on which image classification is to be performed. - * @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which + * @param image The `MPImage` on which image classification is to be performed. + * @param roi A `CGRect` specifying the region of interest within the given `MPImage`, on which * image classification should be performed. - * @param error An optional error parameter populated when there is an error in performing image - * classification on the input image. * - * @return An `MPPImageClassifierResult` object that contains a list of image classifications. + * @return An `ImageClassifierResult` object that contains a list of image classifications. */ - (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image regionOfInterest:(CGRect)roi @@ -132,30 +125,28 @@ NS_SWIFT_NAME(ImageClassifier) NS_SWIFT_NAME(classify(image:regionOfInterest:)); /** - * Performs image classification on the provided video frame of type `MPPImage` using the whole + * Performs image classification on the provided video frame of type `MPImage` using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with - * `MPPRunningModeVideo`. + * the provided `MPImage`. Only use this method when the `MPPImageClassifier` is created with + * running mode `.video`. * * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must * be monotonically increasing. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports classification of RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. * * @param image The `MPPImage` on which image classification is to be performed. * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input * timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing image - * classification on the input video frame. * - * @return An `MPPImageClassifierResult` object that contains a list of image classifications. + * @return An `ImageClassifierResult` object that contains a list of image classifications. */ - (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds @@ -163,33 +154,30 @@ NS_SWIFT_NAME(ImageClassifier) NS_SWIFT_NAME(classify(videoFrame:timestampInMilliseconds:)); /** - * Performs image classification on the provided video frame of type `MPPImage` cropped to the + * Performs image classification on the provided video frame of type `MPImage` cropped to the * specified region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with - * `MPPRunningModeVideo`. + * the provided `MPImage`. Only use this method when the `ImageClassifier` is created with `.video`. * * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must * be monotonically increasing. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports classification of RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image A live stream image data of type `MPPImage` on which image classification is to be + * @param image A live stream image data of type `MPImage` on which image classification is to be * performed. * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input * timestamps must be monotonically increasing. * @param roi A `CGRect` specifying the region of interest within the video frame of type - * `MPPImage`, on which image classification should be performed. - * @param error An optional error parameter populated when there is an error in performing image - * classification on the input video frame. + * `MPImage`, on which image classification should be performed. * - * @return An `MPPImageClassifierResult` object that contains a list of image classifications. + * @return An `ImageClassifierResult` object that contains a list of image classifications. */ - (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds @@ -198,40 +186,38 @@ NS_SWIFT_NAME(ImageClassifier) NS_SWIFT_NAME(classify(videoFrame:timestampInMilliseconds:regionOfInterest:)); /** - * Sends live stream image data of type `MPPImage` to perform image classification using the whole + * Sends live stream image data of type `MPImage` to perform image classification using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with + * the provided `MPImage`. Only use this method when the `ImageClassifier` is created with * `MPPRunningModeLiveStream`. * * The object which needs to be continuously notified of the available results of image - * classification must confirm to `MPPImageClassifierLiveStreamDelegate` protocol and implement the - * `imageClassifier:didFinishClassificationWithResult:timestampInMilliseconds:error:` + * classification must confirm to `ImageClassifierLiveStreamDelegate` protocol and implement the + * `imageClassifier(_:didFinishClassificationWithResult:timestampInMilliseconds:error:)` * delegate method. * * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the image classifier. The input timestamps must be monotonically increasing. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports classification of RGBA images. If your `MPImage` has a source type of + * .pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color - * space is RGB with an Alpha channel. + * If the input `MPImage` has a source type of `.image` ensure that the color space is RGB with an + * Alpha channel. * * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its * `videoSettings` property. * - * @param image A live stream image data of type `MPPImage` on which image classification is to be + * @param image A live stream image data of type `MPImage` on which image classification is to be * performed. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image is sent to the image classifier. The input timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing image - * classification on the input live stream image data. * - * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + * @return `true` if the image was sent to the task successfully, otherwise `false`. */ - (BOOL)classifyAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds @@ -239,42 +225,40 @@ NS_SWIFT_NAME(ImageClassifier) NS_SWIFT_NAME(classifyAsync(image:timestampInMilliseconds:)); /** - * Sends live stream image data of type ``MPPImage`` to perform image classification, cropped to the + * Sends live stream image data of type `MPImage` to perform image classification, cropped to the * specified region of interest.. Rotation will be applied according to the `orientation` property - * of the provided `MPPImage`. Only use this method when the `MPPImageClassifier` is created with - * `MPPRunningModeLiveStream`. + * of the provided `MPImage`. Only use this method when the `ImageClassifier` is created with + * `.liveStream`. * * The object which needs to be continuously notified of the available results of image - * classification must confirm to `MPPImageClassifierLiveStreamDelegate` protocol and implement the - * `imageClassifier:didFinishClassificationWithResult:timestampInMilliseconds:error:` delegate + * classification must confirm to `ImageClassifierLiveStreamDelegate` protocol and implement the + * `imageClassifier(_:didFinishClassificationWithResult:timestampInMilliseconds:error:)` delegate * method. * * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the image classifier. The input timestamps must be monotonically increasing. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports classification of RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color - * space is RGB with an Alpha channel. + * If the input `MPImage` has a source type of `.image` ensure that the color space is RGB with an + * Alpha channel. * * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its * `videoSettings` property. * - * @param image A live stream image data of type `MPPImage` on which image classification is to be + * @param image A live stream image data of type `MPImage` on which image classification is to be * performed. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image is sent to the image classifier. The input timestamps must be monotonically increasing. * @param roi A `CGRect` specifying the region of interest within the given live stream image data - * of type `MPPImage`, on which image classification should be performed. - * @param error An optional error parameter populated when there is an error in performing image - * classification on the input live stream image data. + * of type `MPImage`, on which image classification should be performed. * - * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + * @return `true` if the image was sent to the task successfully, otherwise `false`. */ - (BOOL)classifyAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index 5d2595cd..3e1592e1 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -18,7 +18,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h" @@ -52,6 +52,13 @@ static const int kMicroSecondsPerMilliSecond = 1000; } \ } +#define ImageClassifierResultWithOutputPacketMap(outputPacketMap) \ + ( \ + [MPPImageClassifierResult \ + imageClassifierResultWithClassificationsPacket:outputPacketMap[kClassificationsStreamName \ + .cppString]] \ + ) + @interface MPPImageClassifier () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -63,43 +70,7 @@ static const int kMicroSecondsPerMilliSecond = 1000; @implementation MPPImageClassifier -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - if (![self.imageClassifierLiveStreamDelegate - respondsToSelector:@selector - (imageClassifier:didFinishClassificationWithResult:timestampInMilliseconds:error:)]) { - return; - } - - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [self.imageClassifierLiveStreamDelegate imageClassifier:self - didFinishClassificationWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = liveStreamResult.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPImageClassifierResult *result = [MPPImageClassifierResult - imageClassifierResultWithClassificationsPacket:outputPacketMap[kClassificationsStreamName - .cppString]]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(_callbackQueue, ^{ - [self.imageClassifierLiveStreamDelegate imageClassifier:self - didFinishClassificationWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); -} +#pragma mark - Public - (instancetype)initWithOptions:(MPPImageClassifierOptions *)options error:(NSError **)error { self = [super init]; @@ -143,11 +114,13 @@ static const int kMicroSecondsPerMilliSecond = 1000; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:YES + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectStreamName + error:error]; if (!_visionTaskRunner) { return nil; @@ -167,90 +140,28 @@ static const int kMicroSecondsPerMilliSecond = 1000; - (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image regionOfInterest:(CGRect)roi error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithRegionOfInterest:roi - imageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } + std::optional outputPacketMap = [_visionTaskRunner processImage:image + regionOfInterest:roi + error:error]; - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return - [MPPImageClassifierResult imageClassifierResultWithClassificationsPacket: - outputPacketMap.value()[kClassificationsStreamName.cppString]]; + return [MPPImageClassifier imageClassifierResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image error:(NSError **)error { return [self classifyImage:image regionOfInterest:CGRectZero error:error]; } -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - regionOfInterest:(CGRect)roi - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithRegionOfInterest:roi - imageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; -} - - (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds regionOfInterest:(CGRect)roi error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - regionOfInterest:roi - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + [_visionTaskRunner processVideoFrame:image + regionOfInterest:roi + timestampInMilliseconds:timestampInMilliseconds + error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return - [MPPImageClassifierResult imageClassifierResultWithClassificationsPacket: - outputPacketMap.value()[kClassificationsStreamName.cppString]]; + return [MPPImageClassifier imageClassifierResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image @@ -266,15 +177,10 @@ static const int kMicroSecondsPerMilliSecond = 1000; timestampInMilliseconds:(NSInteger)timestampInMilliseconds regionOfInterest:(CGRect)roi error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - regionOfInterest:roi - error:error]; - if (!inputPacketMap.has_value()) { - return NO; - } - - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; + return [_visionTaskRunner processLiveStreamImage:image + regionOfInterest:roi + timestampInMilliseconds:timestampInMilliseconds + error:error]; } - (BOOL)classifyAsyncImage:(MPPImage *)image @@ -286,4 +192,51 @@ static const int kMicroSecondsPerMilliSecond = 1000; error:error]; } +#pragma mark - Private + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.imageClassifierLiveStreamDelegate + respondsToSelector:@selector + (imageClassifier:didFinishClassificationWithResult:timestampInMilliseconds:error:)]) { + return; + } + + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.imageClassifierLiveStreamDelegate imageClassifier:self + didFinishClassificationWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPImageClassifierResult *result = ImageClassifierResultWithOutputPacketMap(outputPacketMap); + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.imageClassifierLiveStreamDelegate imageClassifier:self + didFinishClassificationWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + ++ (nullable MPPImageClassifierResult *)imageClassifierResultWithOptionalOutputPacketMap: + (std::optional)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return ImageClassifierResultWithOutputPacketMap(outputPacketMap.value()); +} + @end diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h index 058c21ae..72f8859b 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h @@ -23,11 +23,11 @@ NS_ASSUME_NONNULL_BEGIN @class MPPImageClassifier; /** - * This protocol defines an interface for the delegates of `MPPImageClassifier` object to receive + * This protocol defines an interface for the delegates of `ImageClassifier` object to receive * results of asynchronous classification of images (i.e, when `runningMode = - * MPPRunningModeLiveStream`). + * .liveStream`). * - * The delegate of `MPPImageClassifier` must adopt `MPPImageClassifierLiveStreamDelegate` protocol. + * The delegate of `ImageClassifier` must adopt `ImageClassifierLiveStreamDelegate` protocol. * The methods in this protocol are optional. */ NS_SWIFT_NAME(ImageClassifierLiveStreamDelegate) @@ -36,14 +36,14 @@ NS_SWIFT_NAME(ImageClassifierLiveStreamDelegate) @optional /** * This method notifies a delegate that the results of asynchronous classification of - * an image submitted to the `MPPImageClassifier` is available. + * an image submitted to the `ImageClassifier` is available. * - * This method is called on a private serial queue created by the `MPPImageClassifier` + * This method is called on a private serial queue created by the `ImageClassifier` * for performing the asynchronous delegates calls. * * @param imageClassifier The image classifier which performed the classification. - * This is useful to test equality when there are multiple instances of `MPPImageClassifier`. - * @param result An `MPPImageClassifierResult` object that contains a list of image classifications. + * This is useful to test equality when there are multiple instances of `ImageClassifier`. + * @param result An `ImageClassifierResult` object that contains a list of image classifications. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image was sent to the image classifier. * @param error An optional error parameter populated when there is an error in performing image @@ -57,27 +57,27 @@ NS_SWIFT_NAME(ImageClassifierLiveStreamDelegate) @end /** - * Options for setting up a `MPPImageClassifier`. + * Options for setting up a `ImageClassifier`. */ NS_SWIFT_NAME(ImageClassifierOptions) @interface MPPImageClassifierOptions : MPPTaskOptions /** - * Running mode of the image classifier task. Defaults to `MPPRunningModeImage`. - * `MPPImageClassifier` can be created with one of the following running modes: - * 1. `MPPRunningModeImage`: The mode for performing classification on single image inputs. - * 2. `MPPRunningModeVideo`: The mode for performing classification on the decoded frames of a + * Running mode of the image classifier task. Defaults to `.image`. + * `ImageClassifier` can be created with one of the following running modes: + * 1. `.image`: The mode for performing classification on single image inputs. + * 2. `.video`: The mode for performing classification on the decoded frames of a * video. - * 3. `MPPRunningModeLiveStream`: The mode for performing classification on a live stream of input + * 3. `.liveStream`: The mode for performing classification on a live stream of input * data, such as from the camera. */ @property(nonatomic) MPPRunningMode runningMode; /** - * An object that confirms to `MPPImageClassifierLiveStreamDelegate` protocol. This object must - * implement `objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` to receive - * the results of asynchronous classification on images (i.e, when `runningMode = - * MPPRunningModeLiveStream`). + * An object that confirms to `ImageClassifierLiveStreamDelegate` protocol. This object must + * implement `objectDetector(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` to + * receive the results of asynchronous classification on images (i.e, when `runningMode = + * .liveStream`). */ @property(nonatomic, weak, nullable) id imageClassifierLiveStreamDelegate; diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.m b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.m index 8d3815ff..99f08d50 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.m +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.m @@ -28,6 +28,7 @@ - (id)copyWithZone:(NSZone *)zone { MPPImageClassifierOptions *imageClassifierOptions = [super copyWithZone:zone]; + imageClassifierOptions.runningMode = self.runningMode; imageClassifierOptions.scoreThreshold = self.scoreThreshold; imageClassifierOptions.maxResults = self.maxResults; imageClassifierOptions.categoryDenylist = self.categoryDenylist; diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierResult.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierResult.h index 478bd452..0767072a 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierResult.h +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierResult.h @@ -18,23 +18,23 @@ NS_ASSUME_NONNULL_BEGIN -/** Represents the classification results generated by `MPPImageClassifier`. **/ +/** Represents the classification results generated by `ImageClassifier`. **/ NS_SWIFT_NAME(ImageClassifierResult) @interface MPPImageClassifierResult : MPPTaskResult -/** The `MPPClassificationResult` instance containing one set of results per classifier head. **/ +/** The `ClassificationResult` instance containing one set of results per classifier head. **/ @property(nonatomic, readonly) MPPClassificationResult *classificationResult; /** - * Initializes a new `MPPImageClassifierResult` with the given `MPPClassificationResult` and + * Initializes a new `ImageClassifierResult` with the given `ClassificationResult` and * timestamp (in milliseconds). * - * @param classificationResult The `MPPClassificationResult` instance containing one set of results + * @param classificationResult The `ClassificationResult` instance containing one set of results * per classifier head. * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. * - * @return An instance of `MPPImageClassifierResult` initialized with the given - * `MPPClassificationResult` and timestamp (in milliseconds). + * @return An instance of `ImageClassifierResult` initialized with the given + * `ClassificationResult` and timestamp (in milliseconds). */ - (instancetype)initWithClassificationResult:(MPPClassificationResult *)classificationResult timestampInMilliseconds:(NSInteger)timestampInMilliseconds; diff --git a/mediapipe/tasks/ios/vision/image_segmenter/BUILD b/mediapipe/tasks/ios/vision/image_segmenter/BUILD new file mode 100644 index 00000000..54031f24 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/BUILD @@ -0,0 +1,47 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPImageSegmenterResult", + srcs = ["sources/MPPImageSegmenterResult.m"], + hdrs = ["sources/MPPImageSegmenterResult.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskResult", + "//mediapipe/tasks/ios/vision/core:MPPMask", + ], +) + +objc_library( + name = "MPPImageSegmenterOptions", + srcs = ["sources/MPPImageSegmenterOptions.m"], + hdrs = ["sources/MPPImageSegmenterOptions.h"], + deps = [ + ":MPPImageSegmenterResult", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPRunningMode", + ], +) + +objc_library( + name = "MPPImageSegmenter", + hdrs = ["sources/MPPImageSegmenterOptions.h"], + deps = [ + ":MPPImageSegmenterOptions", + ":MPPImageSegmenterResult", + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], +) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h new file mode 100644 index 00000000..819b2012 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h @@ -0,0 +1,217 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Class that performs segmentation on images. + * + * The API expects a TFLite model with mandatory TFLite Model Metadata. + */ +NS_SWIFT_NAME(ImageSegmenter) +@interface MPPImageSegmenter : NSObject + +/** + * Creates a new instance of `MPPImageSegmenter` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPImageSegmenterOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * image segmenter. + * + * @return A new instance of `MPPImageSegmenter` with the given model path. `nil` if there is an + * error in initializing the image segmenter. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPImageSegmenter` from the given `MPPImageSegmenterOptions`. + * + * @param options The options of type `MPPImageSegmenterOptions` to use for configuring the + * `MPPImageSegmenter`. + * @param error An optional error parameter populated when there is an error in initializing the + * image segmenter. + * + * @return A new instance of `MPPImageSegmenter` with the given options. `nil` if there is an error + * in initializing the image segmenter. + */ +- (nullable instancetype)initWithOptions:(MPPImageSegmenterOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs segmentation on the provided MPPImage using the whole image as region of interest. + * Rotation will be applied according to the `orientation` property of the provided `MPPImage`. Only + * use this method when the `MPPImageSegmenter` is created with `MPPRunningModeImage`. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which segmentation is to be performed. + * @param error An optional error parameter populated when there is an error in performing + * segmentation on the input image. + * + * @return An `MPPImageSegmenterResult` that contains the segmented masks. + */ +- (nullable MPPImageSegmenterResult *)segmentImage:(MPPImage *)image + error:(NSError *)error NS_SWIFT_NAME(segment(image:)); + +/** + * Performs segmentation on the provided MPPImage using the whole image as region of interest and + * invokes the given completion handler block with the response. The method returns synchronously + * once the completion handler returns. + * + * Rotation will be applied according to the `orientation` property of the provided + * `MPPImage`. Only use this method when the `MPPImageSegmenter` is created with + * `MPPRunningModeImage`. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which segmentation is to be performed. + * @param completionHandler A block to be invoked with the results of performing segmentation on the + * image. The block takes two arguments, the optional `MPPImageSegmenterResult` that contains the + * segmented masks if the segmentation was successful and an optional error populated upon failure. + * The lifetime of the returned masks is only guaranteed for the duration of the block. + */ +- (void)segmentImage:(MPPImage *)image + withCompletionHandler:((void ^)(MPPImageSegmenterResult *_Nullable result, + NSError *_Nullable error))completionHandler + NS_SWIFT_NAME(segment(image:completion:)); + +/** + * Performs segmentation on the provided video frame of type `MPPImage` using the whole image as + * region of interest. + * + * Rotation will be applied according to the `orientation` property of the provided `MPPImage`. Only + * use this method when the `MPPImageSegmenter` is created with `MPPRunningModeVideo`. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which segmentation is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing + * segmentation on the input image. + * + * @return An `MPPImageSegmenterResult` that contains a the segmented masks. + */ +- (nullable MPPImageSegmenterResult *)segmentVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(segment(videoFrame:timestampInMilliseconds:)); + +/** + * Performs segmentation on the provided video frame of type `MPPImage` using the whole image as + * region of interest invokes the given completion handler block with the response. The method + * returns synchronously once the completion handler returns. + * + * Rotation will be applied according to the `orientation` property of the provided `MPPImage`. Only + * use this method when the `MPPImageSegmenter` is created with `MPPRunningModeVideo`. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which segmentation is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param completionHandler A block to be invoked with the results of performing segmentation on the + * image. The block takes two arguments, the optional `MPPImageSegmenterResult` that contains the + * segmented masks if the segmentation was successful and an optional error only populated upon + * failure. The lifetime of the returned masks is only guaranteed for the duration of the block. + */ +- (void)segmentVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + withCompletionHandler:((void ^)(MPPImageSegmenterResult *_Nullable result, + NSError *_Nullable error))completionHandler + NS_SWIFT_NAME(segment(videoFrame:timestampInMilliseconds:completion:)); + +/** + * Sends live stream image data of type `MPPImage` to perform segmentation using the whole image as + * region of interest. + * + * Rotation will be applied according to the `orientation` property of the provided `MPPImage`. Only + * use this method when the `MPPImageSegmenter` is created with`MPPRunningModeLiveStream`. + * + * The object which needs to be continuously notified of the available results of image segmentation + * must confirm to `MPPImageSegmenterLiveStreamDelegate` protocol and implement the + *`imageSegmenter:didFinishSegmentationWithResult:timestampInMilliseconds:error:` delegate method. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the segmenter. The input timestamps must be monotonically increasing. + * + * This method supports RGBA images. If your `MPPImage` has a source type of + *`MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * space is RGB with an Alpha channel. + * + * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you + * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its + * `videoSettings` property. + * + * @param image A live stream image data of type `MPPImage` on which segmentation is to be + * performed. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image is sent to the segmenter. The input timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error when sending the input + * image to the graph. + * + * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + */ +- (BOOL)segmentAsyncInImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(segmentAsync(image:timestampInMilliseconds:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h new file mode 100644 index 00000000..f1ba5411 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h @@ -0,0 +1,99 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@class MPPImageSegmenter; + +/** + * This protocol defines an interface for the delegates of `MPPImageSegmenter` object to receive + * results of performing asynchronous segmentation on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + * + * The delegate of `MPPImageSegmenter` must adopt `MPPImageSegmenterLiveStreamDelegate` protocol. + * The methods in this protocol are optional. + */ +NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) +@protocol MPPImageSegmenterLiveStreamDelegate + +@required + +/** + * This method notifies a delegate that the results of asynchronous segmentation of + * an image submitted to the `MPPImageSegmenter` is available. + * + * This method is called on a private serial dispatch queue created by the `MPPImageSegmenter` + * for performing the asynchronous delegates calls. + * + * @param imageSegmenter The image segmenter which performed the segmentation. This is useful to + * test equality when there are multiple instances of `MPPImageSegmenter`. + * @param result The `MPPImageSegmenterResult` object that contains a list of category or confidence + * masks and optional quality scores. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the image segmenter. + * @param error An optional error parameter populated when there is an error in performing + * segmentation on the input live stream image data. + */ +- (void)imageSegmenter:(MPPImageSegmenter *)imageSegmenter + didFinishSegmentationWithResult:(nullable MPPImageSegmenterResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(imageSegmenter(_:didFinishSegmentation:timestampInMilliseconds:error:)); +@end + +/** Options for setting up a `MPPImageSegmenter`. */ +NS_SWIFT_NAME(ImageSegmenterOptions) +@interface MPPImageSegmenterOptions : MPPTaskOptions + +/** + * Running mode of the image segmenter task. Defaults to `MPPRunningModeImage`. + * `MPPImageSegmenter` can be created with one of the following running modes: + * 1. `MPPRunningModeImage`: The mode for performing segmentation on single image inputs. + * 2. `MPPRunningModeVideo`: The mode for performing segmentation on the decoded frames of a + * video. + * 3. `MPPRunningModeLiveStream`: The mode for performing segmentation on a live stream of + * input data, such as from the camera. + */ +@property(nonatomic) MPPRunningMode runningMode; + +/** + * An object that confirms to `MPPImageSegmenterLiveStreamDelegate` protocol. This object must + * implement `imageSegmenter:didFinishSegmentationWithResult:timestampInMilliseconds:error:` to + * receive the results of performing asynchronous segmentation on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + */ +@property(nonatomic, weak, nullable) id + imageSegmenterLiveStreamDelegate; + +/** + * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults + * to English. + */ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** Represents whether to output confidence masks. */ +@property(nonatomic) BOOL shouldOutputConfidenceMasks; + +/** Represents whether to output category mask. */ +@property(nonatomic) BOOL shouldOutputCategoryMasks; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m new file mode 100644 index 00000000..282a729b --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m @@ -0,0 +1,40 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h" + +@implementation MPPImageSegmenterOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _displayNamesLocale = @"en"; + _shouldOutputConfidenceMasks = YES; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPImageSegmenterOptions *imageSegmenterOptions = [super copyWithZone:zone]; + + imageSegmenterOptions.runningMode = self.runningMode; + imageSegmenterOptions.shouldOutputConfidenceMasks = self.shouldOutputConfidenceMasks; + imageSegmenterOptions.shouldOutputCategoryMasks = self.shouldOutputConfidenceMasks; + imageSegmenterOptions.displayNamesLocale = self.displayNamesLocale; + imageSegmenterOptions.imageSegmenterLiveStreamDelegate = self.imageSegmenterLiveStreamDelegate; + + return imageSegmenterOptions; +} + +@end diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h new file mode 100644 index 00000000..20bf3fef --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h @@ -0,0 +1,68 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the segmentation results generated by `MPPImageSegmenter`. */ +NS_SWIFT_NAME(ImageSegmenterResult) +@interface MPPImageSegmenterResult : MPPTaskResult + +/** + * An optional array of `MPPMask` objects. Each `MPPMask` in the array holds a 32 bit float array of + * size `image width` * `image height` which represents the confidence mask for each category. Each + * element of the float array represents the confidence with which the model predicted that the + * corresponding pixel belongs to the category that the mask represents, usually in the range [0,1]. + */ +@property(nonatomic, readonly, nullable) NSArray *confidenceMasks; + +/** + * An optional `MPPMask` that holds a`UInt8` array of size `image width` * `image height`. Each + * element of this array represents the class to which the pixel in the original image was predicted + * to belong to. + */ +@property(nonatomic, readonly, nullable) MPPMask *categoryMask; + +/** + * The quality scores of the result masks, in the range of [0, 1]. Defaults to `1` if the model + * doesn't output quality scores. Each element corresponds to the score of the category in the model + * outputs. + */ +@property(nonatomic, readonly, nullable) NSArray *qualityScores; + +/** + * Initializes a new `MPPImageSegmenterResult` with the given array of confidence masks, category + * mask, quality scores and timestamp (in milliseconds). + * + * @param confidenceMasks An optional array of `MPPMask` objects. Each `MPPMask` in the array must + * be of type `MPPMaskDataTypeFloat32`. + * @param categoryMask An optional `MPMask` object of type `MPPMaskDataTypeUInt8`. + * @param qualityScores The quality scores of the result masks of type NSArray *. Each + * `NSNumber` in the array holds a `float`. + * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. + * + * @return An instance of `MPPImageSegmenterResult` initialized with the given array of confidence + * masks, category mask, quality scores and timestamp (in milliseconds). + */ +- (instancetype)initWithConfidenceMasks:(nullable NSArray *)confidenceMasks + categoryMask:(nullable MPPMask *)categoryMask + qualityScores:(nullable NSArray *)qualityScores + timestampInMilliseconds:(NSInteger)timestampInMilliseconds; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.m b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.m new file mode 100644 index 00000000..2b11fc16 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.m @@ -0,0 +1,32 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" + +@implementation MPPImageSegmenterResult + +- (instancetype)initWithConfidenceMasks:(NSArray *)confidenceMasks + categoryMask:(MPPMask *)categoryMask + qualityScores:(NSArray *)qualityScores + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { + self = [super initWithTimestampInMilliseconds:timestampInMilliseconds]; + if (self) { + _confidenceMasks = confidenceMasks; + _categoryMask = categoryMask; + _qualityScores = qualityScores; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/vision/image_segmenter/utils/BUILD b/mediapipe/tasks/ios/vision/image_segmenter/utils/BUILD new file mode 100644 index 00000000..7630dd7e --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/utils/BUILD @@ -0,0 +1,42 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPImageSegmenterOptionsHelpers", + srcs = ["sources/MPPImageSegmenterOptions+Helpers.mm"], + hdrs = ["sources/MPPImageSegmenterOptions+Helpers.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_graph_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenterOptions", + ], +) + +objc_library( + name = "MPPImageSegmenterResultHelpers", + srcs = ["sources/MPPImageSegmenterResult+Helpers.mm"], + hdrs = ["sources/MPPImageSegmenterResult+Helpers.h"], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:image", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenterResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.h b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.h new file mode 100644 index 00000000..4d3b222f --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.h @@ -0,0 +1,32 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPImageSegmenterOptions (Helpers) + +/** + * Populates the provided `CalculatorOptions` proto container with the current settings. + * + * @param optionsProto The `CalculatorOptions` proto object to copy the settings to. + */ +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.mm b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.mm new file mode 100644 index 00000000..d27bb91d --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.mm @@ -0,0 +1,41 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/vision/image_segmenter/proto/image_segmenter_graph_options.pb.h" + +namespace { +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using ImageSegmenterGraphOptionsProto = + ::mediapipe::tasks::vision::image_segmenter::proto::ImageSegmenterGraphOptions; +using SegmenterOptionsProto = ::mediapipe::tasks::vision::image_segmenter::proto::SegmenterOptions; +} // namespace + +@implementation MPPImageSegmenterOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + ImageSegmenterGraphOptionsProto *imageSegmenterGraphOptionsProto = + optionsProto->MutableExtension(ImageSegmenterGraphOptionsProto::ext); + imageSegmenterGraphOptionsProto->Clear(); + + [self.baseOptions copyToProto:imageSegmenterGraphOptionsProto->mutable_base_options() + withUseStreamMode:self.runningMode != MPPRunningModeImage]; + imageSegmenterGraphOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); +} + +@end diff --git a/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.h b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.h new file mode 100644 index 00000000..503fcd1d --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.h @@ -0,0 +1,48 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPImageSegmenterResult (Helpers) + +/** + * Creates an `MPPImageSegmenterResult` from confidence masks, category mask and quality scores + * packets. + * + * If `shouldCopyMaskPacketData` is set to `YES`, the confidence and catergory masks of the newly + * created `MPPImageSegmenterResult` holds references to deep copied pixel data of the output + * respective masks. + * + * @param confidenceMasksPacket A MediaPipe packet wrapping a `std::vector`. + * @param categoryMaskPacket A MediaPipe packet wrapping a ``. + * @param qualityScoresPacket A MediaPipe packet wrapping a `std::vector`. + * @param shouldCopyMaskPacketData A `BOOL` which indicates if the pixel data of the output masks + * must be deep copied to the newly created `MPPImageSegmenterResult`. + * + * @return An `MPPImageSegmenterResult` object that contains the image segmentation results. + */ ++ (MPPImageSegmenterResult *) + imageSegmenterResultWithConfidenceMasksPacket:(const mediapipe::Packet &)confidenceMasksPacket + categoryMaskPacket:(const mediapipe::Packet &)categoryMaskPacket + qualityScoresPacket:(const mediapipe::Packet &)qualityScoresPacket + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm new file mode 100644 index 00000000..d6e3b1be --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm @@ -0,0 +1,78 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.h" + +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/packet.h" + +namespace { +using ::mediapipe::Image; +using ::mediapipe::ImageFrameSharedPtr; +using ::mediapipe::Packet; +} // namespace + +@implementation MPPImageSegmenterResult (Helpers) + ++ (MPPImageSegmenterResult *) + imageSegmenterResultWithConfidenceMasksPacket:(const Packet &)confidenceMasksPacket + categoryMaskPacket:(const Packet &)categoryMaskPacket + qualityScoresPacket:(const Packet &)qualityScoresPacket + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData { + NSMutableArray *confidenceMasks; + MPPMask *categoryMask; + NSMutableArray *qualityScores; + + if (confidenceMasksPacket.ValidateAsType>().ok()) { + std::vector cppConfidenceMasks = confidenceMasksPacket.Get>(); + confidenceMasks = [NSMutableArray arrayWithCapacity:(NSUInteger)cppConfidenceMasks.size()]; + + for (const auto &confidenceMask : cppConfidenceMasks) { + [confidenceMasks + addObject:[[MPPMask alloc] + initWithFloat32Data:(float *)confidenceMask.GetImageFrameSharedPtr() + .get() + ->PixelData() + width:confidenceMask.width() + height:confidenceMask.height() + shouldCopy:shouldCopyMaskPacketData ? YES : NO]]; + } + } + + if (categoryMaskPacket.ValidateAsType().ok()) { + const Image &cppCategoryMask = confidenceMasksPacket.Get(); + categoryMask = [[MPPMask alloc] + initWithUInt8Data:(UInt8 *)cppCategoryMask.GetImageFrameSharedPtr().get()->PixelData() + width:cppCategoryMask.width() + height:cppCategoryMask.height() + shouldCopy:shouldCopyMaskPacketData ? YES : NO]; + } + + if (qualityScoresPacket.ValidateAsType>().ok()) { + std::vector cppQualityScores = qualityScoresPacket.Get>(); + qualityScores = [NSMutableArray arrayWithCapacity:(NSUInteger)cppQualityScores.size()]; + + for (const auto &qualityScore : cppQualityScores) { + [qualityScores addObject:[NSNumber numberWithFloat:qualityScore]]; + } + } + + return [[MPPImageSegmenterResult alloc] initWithConfidenceMasks:confidenceMasks + categoryMask:categoryMask + qualityScores:qualityScores + timestampInMilliseconds:timestampInMilliseconds]; +} + +@end diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h index f8cfcc91..851e8a35 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h @@ -64,52 +64,50 @@ NS_SWIFT_NAME(ObjectDetector) @interface MPPObjectDetector : NSObject /** - * Creates a new instance of `MPPObjectDetector` from an absolute path to a TensorFlow Lite model - * file stored locally on the device and the default `MPPObjectDetector`. + * Creates a new instance of `ObjectDetector` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `ObjectDetector`. * * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. * @param error An optional error parameter populated when there is an error in initializing the * object detector. * - * @return A new instance of `MPPObjectDetector` with the given model path. `nil` if there is an + * @return A new instance of `ObjectDetector` with the given model path. `nil` if there is an * error in initializing the object detector. */ - (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; /** - * Creates a new instance of `MPPObjectDetector` from the given `MPPObjectDetectorOptions`. + * Creates a new instance of `ObjectDetector` from the given `ObjectDetectorOptions`. * - * @param options The options of type `MPPObjectDetectorOptions` to use for configuring the - * `MPPObjectDetector`. + * @param options The options of type `ObjectDetectorOptions` to use for configuring the + * `ObjectDetector`. * @param error An optional error parameter populated when there is an error in initializing the * object detector. * - * @return A new instance of `MPPObjectDetector` with the given options. `nil` if there is an error + * @return A new instance of `ObjectDetector` with the given options. `nil` if there is an error * in initializing the object detector. */ - (nullable instancetype)initWithOptions:(MPPObjectDetectorOptions *)options error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** - * Performs object detection on the provided MPPImage using the whole image as region of + * Performs object detection on the provided MPImage using the whole image as region of * interest. Rotation will be applied according to the `orientation` property of the provided - * `MPPImage`. Only use this method when the `MPPObjectDetector` is created with - * `MPPRunningModeImage`. + * `MPImage`. Only use this method when the `ObjectDetector` is created with + * `.image`. * - * This method supports detecting objects in RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports detecting objects in RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * If your `MPImage` has a source type of `.image` ensure that the color space is * RGB with an Alpha channel. * - * @param image The `MPPImage` on which object detection is to be performed. - * @param error An optional error parameter populated when there is an error in performing object - * detection on the input image. + * @param image The `.image` on which object detection is to be performed. * - * @return An `MPPObjectDetectorResult` object that contains a list of detections, each detection + * @return An `ObjectDetectorResult` object that contains a list of detections, each detection * has a bounding box that is expressed in the unrotated input frame of reference coordinates * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying * image data. @@ -118,27 +116,25 @@ NS_SWIFT_NAME(ObjectDetector) error:(NSError **)error NS_SWIFT_NAME(detect(image:)); /** - * Performs object detection on the provided video frame of type `MPPImage` using the whole + * Performs object detection on the provided video frame of type `MPImage` using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with - * `MPPRunningModeVideo`. + * the provided `MPImage`. Only use this method when the `MPPObjectDetector` is created with + * `.video`. * - * This method supports detecting objects in of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports detecting objects in of RGBA images. If your `MPImage` has a source type of + * .pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image The `MPPImage` on which object detection is to be performed. + * @param image The `MPImage` on which object detection is to be performed. * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input * timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing object - * detection on the input image. * - * @return An `MPPObjectDetectorResult` object that contains a list of detections, each detection + * @return An `ObjectDetectorResult` object that contains a list of detections, each detection * has a bounding box that is expressed in the unrotated input frame of reference coordinates * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying * image data. @@ -149,26 +145,26 @@ NS_SWIFT_NAME(ObjectDetector) NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); /** - * Sends live stream image data of type `MPPImage` to perform object detection using the whole + * Sends live stream image data of type `MPImage` to perform object detection using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with - * `MPPRunningModeLiveStream`. + * the provided `MPImage`. Only use this method when the `ObjectDetector` is created with + * `.liveStream`. * * The object which needs to be continuously notified of the available results of object - * detection must confirm to `MPPObjectDetectorLiveStreamDelegate` protocol and implement the - * `objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. + * detection must confirm to `ObjectDetectorLiveStreamDelegate` protocol and implement the + * `objectDetector(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` delegate method. * * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the object detector. The input timestamps must be monotonically increasing. * - * This method supports detecting objects in RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports detecting objects in RGBA images. If your `MPImage` has a source type of + * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color - * space is RGB with an Alpha channel. + * If the input `MPImage` has a source type of `.image` ensure that the color space is RGB with an + * Alpha channel. * * If this method is used for detecting objects in live camera frames using `AVFoundation`, ensure * that you request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its @@ -178,10 +174,8 @@ NS_SWIFT_NAME(ObjectDetector) * performed. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image is sent to the object detector. The input timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing object - * detection on the input live stream image data. * - * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + * @return `true` if the image was sent to the task successfully, otherwise `false`. */ - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index 52648af5..e4704fbb 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -81,8 +81,7 @@ static NSString *const kTaskName = @"objectDetector"; } MPPObjectDetectorResult *result = [MPPObjectDetectorResult - objectDetectorResultWithDetectionsPacket: - outputPacketMap[kDetectionsStreamName.cppString]]; + objectDetectorResultWithDetectionsPacket:outputPacketMap[kDetectionsStreamName.cppString]]; NSInteger timeStampInMilliseconds = outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h index 33d7bdbb..0060d374 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h @@ -23,11 +23,11 @@ NS_ASSUME_NONNULL_BEGIN @class MPPObjectDetector; /** - * This protocol defines an interface for the delegates of `MPPObjectDetector` object to receive + * This protocol defines an interface for the delegates of `ObjectDetector` object to receive * results of performing asynchronous object detection on images (i.e, when `runningMode` = - * `MPPRunningModeLiveStream`). + * `.liveStream`). * - * The delegate of `MPPObjectDetector` must adopt `MPPObjectDetectorLiveStreamDelegate` protocol. + * The delegate of `ObjectDetector` must adopt `ObjectDetectorLiveStreamDelegate` protocol. * The methods in this protocol are optional. */ NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) @@ -37,14 +37,14 @@ NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) /** * This method notifies a delegate that the results of asynchronous object detection of - * an image submitted to the `MPPObjectDetector` is available. + * an image submitted to the `ObjectDetector` is available. * - * This method is called on a private serial dispatch queue created by the `MPPObjectDetector` + * This method is called on a private serial dispatch queue created by the `ObjectDetector` * for performing the asynchronous delegates calls. * * @param objectDetector The object detector which performed the object detection. - * This is useful to test equality when there are multiple instances of `MPPObjectDetector`. - * @param result The `MPPObjectDetectorResult` object that contains a list of detections, each + * This is useful to test equality when there are multiple instances of `ObjectDetector`. + * @param result The `ObjectDetectorResult` object that contains a list of detections, each * detection has a bounding box that is expressed in the unrotated input frame of reference * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the * underlying image data. @@ -60,26 +60,27 @@ NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) NS_SWIFT_NAME(objectDetector(_:didFinishDetection:timestampInMilliseconds:error:)); @end -/** Options for setting up a `MPPObjectDetector`. */ +/** Options for setting up a `ObjectDetector`. */ NS_SWIFT_NAME(ObjectDetectorOptions) @interface MPPObjectDetectorOptions : MPPTaskOptions /** - * Running mode of the object detector task. Defaults to `MPPRunningModeImage`. - * `MPPObjectDetector` can be created with one of the following running modes: - * 1. `MPPRunningModeImage`: The mode for performing object detection on single image inputs. - * 2. `MPPRunningModeVideo`: The mode for performing object detection on the decoded frames of a + * Running mode of the object detector task. Defaults to `.image`. + * `ObjectDetector` can be created with one of the following running modes: + * 1. `.image`: The mode for performing object detection on single image inputs. + * 2. `.video`: The mode for performing object detection on the decoded frames of a * video. - * 3. `MPPRunningModeLiveStream`: The mode for performing object detection on a live stream of + * 3. `.liveStream`: The mode for performing object detection on a live stream of * input data, such as from the camera. */ @property(nonatomic) MPPRunningMode runningMode; /** - * An object that confirms to `MPPObjectDetectorLiveStreamDelegate` protocol. This object must - * implement `objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` to receive - * the results of performing asynchronous object detection on images (i.e, when `runningMode` = - * `MPPRunningModeLiveStream`). + * An object that confirms to `ObjectDetectorLiveStreamDelegate` protocol. This object must + * implement `objectDetector(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` to + * receive the results of performing asynchronous object detection on images (i.e, when + * `runningMode` = + * `.liveStream`). */ @property(nonatomic, weak, nullable) id objectDetectorLiveStreamDelegate; diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m index b93a6b30..bb4605cd 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m @@ -28,6 +28,7 @@ - (id)copyWithZone:(NSZone *)zone { MPPObjectDetectorOptions *objectDetectorOptions = [super copyWithZone:zone]; + objectDetectorOptions.runningMode = self.runningMode; objectDetectorOptions.scoreThreshold = self.scoreThreshold; objectDetectorOptions.maxResults = self.maxResults; objectDetectorOptions.categoryDenylist = self.categoryDenylist; diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorResult.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorResult.h index 2641b6b4..e48cb4fb 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorResult.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorResult.h @@ -18,27 +18,27 @@ NS_ASSUME_NONNULL_BEGIN -/** Represents the detection results generated by `MPPObjectDetector`. */ +/** Represents the detection results generated by `ObjectDetector`. */ NS_SWIFT_NAME(ObjectDetectorResult) @interface MPPObjectDetectorResult : MPPTaskResult /** - * The array of `MPPDetection` objects each of which has a bounding box that is expressed in the + * The array of `Detection` objects each of which has a bounding box that is expressed in the * unrotated input frame of reference coordinates system, i.e. in `[0,image_width) x * [0,image_height)`, which are the dimensions of the underlying image data. */ @property(nonatomic, readonly) NSArray *detections; /** - * Initializes a new `MPPObjectDetectorResult` with the given array of detections and timestamp (in + * Initializes a new `ObjectDetectorResult` with the given array of detections and timestamp (in * milliseconds). * - * @param detections An array of `MPPDetection` objects each of which has a bounding box that is + * @param detections An array of `Detection` objects each of which has a bounding box that is * expressed in the unrotated input frame of reference coordinates system, i.e. in `[0,image_width) * x [0,image_height)`, which are the dimensions of the underlying image data. * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. * - * @return An instance of `MPPObjectDetectorResult` initialized with the given array of detections + * @return An instance of `ObjectDetectorResult` initialized with the given array of detections * and timestamp (in milliseconds). */ - (instancetype)initWithDetections:(NSArray *)detections diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD index 07106985..bcdc0e5e 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD @@ -92,6 +92,9 @@ android_library( android_library( name = "landmark", srcs = ["Landmark.java"], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], deps = [ "//third_party:autovalue", "@maven//:com_google_guava_guava", @@ -101,6 +104,9 @@ android_library( android_library( name = "normalized_landmark", srcs = ["NormalizedLandmark.java"], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], deps = [ "//third_party:autovalue", "@maven//:com_google_guava_guava", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java index c3e9f271..e23d9115 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java @@ -16,6 +16,7 @@ package com.google.mediapipe.tasks.components.containers; import com.google.auto.value.AutoValue; import java.util.Objects; +import java.util.Optional; /** * Landmark represents a point in 3D space with x, y, z coordinates. The landmark coordinates are in @@ -27,7 +28,12 @@ public abstract class Landmark { private static final float TOLERANCE = 1e-6f; public static Landmark create(float x, float y, float z) { - return new AutoValue_Landmark(x, y, z); + return new AutoValue_Landmark(x, y, z, Optional.empty(), Optional.empty()); + } + + public static Landmark create( + float x, float y, float z, Optional visibility, Optional presence) { + return new AutoValue_Landmark(x, y, z, visibility, presence); } // The x coordinates of the landmark. @@ -39,6 +45,12 @@ public abstract class Landmark { // The z coordinates of the landmark. public abstract float z(); + // Visibility of the normalized landmark. + public abstract Optional visibility(); + + // Presence of the normalized landmark. + public abstract Optional presence(); + @Override public final boolean equals(Object o) { if (!(o instanceof Landmark)) { @@ -57,6 +69,16 @@ public abstract class Landmark { @Override public final String toString() { - return ""; + return ""; } } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedLandmark.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedLandmark.java index f96e434c..50a95d56 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedLandmark.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedLandmark.java @@ -16,6 +16,7 @@ package com.google.mediapipe.tasks.components.containers; import com.google.auto.value.AutoValue; import java.util.Objects; +import java.util.Optional; /** * Normalized Landmark represents a point in 3D space with x, y, z coordinates. x and y are @@ -28,7 +29,12 @@ public abstract class NormalizedLandmark { private static final float TOLERANCE = 1e-6f; public static NormalizedLandmark create(float x, float y, float z) { - return new AutoValue_NormalizedLandmark(x, y, z); + return new AutoValue_NormalizedLandmark(x, y, z, Optional.empty(), Optional.empty()); + } + + public static NormalizedLandmark create( + float x, float y, float z, Optional visibility, Optional presence) { + return new AutoValue_NormalizedLandmark(x, y, z, visibility, presence); } // The x coordinates of the normalized landmark. @@ -40,6 +46,12 @@ public abstract class NormalizedLandmark { // The z coordinates of the normalized landmark. public abstract float z(); + // Visibility of the normalized landmark. + public abstract Optional visibility(); + + // Presence of the normalized landmark. + public abstract Optional presence(); + @Override public final boolean equals(Object o) { if (!(o instanceof NormalizedLandmark)) { @@ -58,6 +70,16 @@ public abstract class NormalizedLandmark { @Override public final String toString() { - return ""; + return ""; } } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD index d04fc425..e8d3b1c6 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD @@ -16,6 +16,20 @@ package(default_visibility = ["//visibility:public"]) android_library( name = "core", + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + manifest = "AndroidManifest.xml", + exports = [ + ":core_java", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni:model_resources_cache_jni", + ], + deps = ["@maven//:com_google_guava_guava"], +) + +android_library( + name = "core_java", srcs = glob(["*.java"]), javacopts = [ "-Xep:AndroidJdkLibsChecker:OFF", @@ -27,11 +41,11 @@ android_library( "//mediapipe/calculators/tensor:inference_calculator_java_proto_lite", "//mediapipe/framework:calculator_java_proto_lite", "//mediapipe/framework:calculator_options_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/framework:android_framework_no_mff", "//mediapipe/tasks/cc/core/proto:acceleration_java_proto_lite", "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite", "//mediapipe/tasks/cc/core/proto:external_file_java_proto_lite", - "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni:model_resources_cache_jni", + "//third_party:any_java_proto", "//third_party:autovalue", "@com_google_protobuf//:protobuf_javalite", "@maven//:com_google_guava_guava", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BaseOptions.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BaseOptions.java index 8eec72ef..dc2c001b 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BaseOptions.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BaseOptions.java @@ -54,6 +54,9 @@ public abstract class BaseOptions { */ public abstract Builder setDelegate(Delegate delegate); + /** Options for the chosen delegate. If not set, the default delegate options is used. */ + public abstract Builder setDelegateOptions(DelegateOptions delegateOptions); + abstract BaseOptions autoBuild(); /** @@ -79,6 +82,23 @@ public abstract class BaseOptions { throw new IllegalArgumentException( "The model buffer should be either a direct ByteBuffer or a MappedByteBuffer."); } + boolean delegateMatchesDelegateOptions = true; + if (options.delegateOptions().isPresent()) { + switch (options.delegate()) { + case CPU: + delegateMatchesDelegateOptions = + options.delegateOptions().get() instanceof DelegateOptions.CpuOptions; + break; + case GPU: + delegateMatchesDelegateOptions = + options.delegateOptions().get() instanceof DelegateOptions.GpuOptions; + break; + } + if (!delegateMatchesDelegateOptions) { + throw new IllegalArgumentException( + "Specified Delegate type does not match the provided delegate options."); + } + } return options; } } @@ -91,6 +111,67 @@ public abstract class BaseOptions { abstract Delegate delegate(); + abstract Optional delegateOptions(); + + /** Advanced config options for the used delegate. */ + public abstract static class DelegateOptions { + + /** Options for CPU. */ + @AutoValue + public abstract static class CpuOptions extends DelegateOptions { + + public static Builder builder() { + Builder builder = new AutoValue_BaseOptions_DelegateOptions_CpuOptions.Builder(); + return builder; + } + + /** Builder for {@link CpuOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract CpuOptions build(); + } + } + + /** Options for GPU. */ + @AutoValue + public abstract static class GpuOptions extends DelegateOptions { + // Load pre-compiled serialized binary cache to accelerate init process. + // Only available on Android. Kernel caching will only be enabled if this + // path is set. NOTE: binary cache usage may be skipped if valid serialized + // model, specified by "serialized_model_dir", exists. + abstract Optional cachedKernelPath(); + + // A dir to load from and save to a pre-compiled serialized model used to + // accelerate init process. + // NOTE: serialized model takes precedence over binary cache + // specified by "cached_kernel_path", which still can be used if + // serialized model is invalid or missing. + abstract Optional serializedModelDir(); + + // Unique token identifying the model. Used in conjunction with + // "serialized_model_dir". It is the caller's responsibility to ensure + // there is no clash of the tokens. + abstract Optional modelToken(); + + public static Builder builder() { + return new AutoValue_BaseOptions_DelegateOptions_GpuOptions.Builder(); + } + + /** Builder for {@link GpuOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setCachedKernelPath(String cachedKernelPath); + + public abstract Builder setSerializedModelDir(String serializedModelDir); + + public abstract Builder setModelToken(String modelToken); + + public abstract GpuOptions build(); + } + } + } + public static Builder builder() { return new AutoValue_BaseOptions.Builder().setDelegate(Delegate.CPU); } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskInfo.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskInfo.java index 3c422a8b..ad3d0111 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskInfo.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskInfo.java @@ -20,6 +20,8 @@ import com.google.mediapipe.proto.CalculatorProto.CalculatorGraphConfig; import com.google.mediapipe.proto.CalculatorProto.CalculatorGraphConfig.Node; import com.google.mediapipe.proto.CalculatorProto.InputStreamInfo; import com.google.mediapipe.calculator.proto.FlowLimiterCalculatorProto.FlowLimiterCalculatorOptions; +import com.google.mediapipe.framework.MediaPipeException; +import com.google.protobuf.Any; import java.util.ArrayList; import java.util.List; @@ -110,10 +112,21 @@ public abstract class TaskInfo { */ CalculatorGraphConfig generateGraphConfig() { CalculatorGraphConfig.Builder graphBuilder = CalculatorGraphConfig.newBuilder(); - Node.Builder taskSubgraphBuilder = - Node.newBuilder() - .setCalculator(taskGraphName()) - .setOptions(taskOptions().convertToCalculatorOptionsProto()); + CalculatorOptions options = taskOptions().convertToCalculatorOptionsProto(); + Any anyOptions = taskOptions().convertToAnyProto(); + if (!(options == null ^ anyOptions == null)) { + throw new MediaPipeException( + MediaPipeException.StatusCode.INVALID_ARGUMENT.ordinal(), + "Only one of convertTo*Proto() method should be implemented for " + + taskOptions().getClass()); + } + Node.Builder taskSubgraphBuilder = Node.newBuilder().setCalculator(taskGraphName()); + if (options != null) { + taskSubgraphBuilder.setOptions(options); + } + if (anyOptions != null) { + taskSubgraphBuilder.addNodeOptions(anyOptions); + } for (String outputStream : outputStreams()) { taskSubgraphBuilder.addOutputStream(outputStream); graphBuilder.addOutputStream(outputStream); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskOptions.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskOptions.java index 11330ac0..4ca25842 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskOptions.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskOptions.java @@ -20,18 +20,26 @@ import com.google.mediapipe.proto.CalculatorOptionsProto.CalculatorOptions; import com.google.mediapipe.tasks.core.proto.AccelerationProto; import com.google.mediapipe.tasks.core.proto.BaseOptionsProto; import com.google.mediapipe.tasks.core.proto.ExternalFileProto; +import com.google.protobuf.Any; import com.google.protobuf.ByteString; /** * MediaPipe Tasks options base class. Any MediaPipe task-specific options class should extend - * {@link TaskOptions}. + * {@link TaskOptions} and implement exactly one of converTo*Proto() methods. */ public abstract class TaskOptions { /** * Converts a MediaPipe Tasks task-specific options to a {@link CalculatorOptions} protobuf * message. */ - public abstract CalculatorOptions convertToCalculatorOptionsProto(); + public CalculatorOptions convertToCalculatorOptionsProto() { + return null; + } + + /** Converts a MediaPipe Tasks task-specific options to an proto3 {@link Any} message. */ + public Any convertToAnyProto() { + return null; + } /** * Converts a {@link BaseOptions} instance to a {@link BaseOptionsProto.BaseOptions} protobuf @@ -61,17 +69,51 @@ public abstract class TaskOptions { accelerationBuilder.setTflite( InferenceCalculatorProto.InferenceCalculatorOptions.Delegate.TfLite .getDefaultInstance()); + options + .delegateOptions() + .ifPresent( + delegateOptions -> + setDelegateOptions( + accelerationBuilder, + (BaseOptions.DelegateOptions.CpuOptions) delegateOptions)); break; case GPU: accelerationBuilder.setGpu( InferenceCalculatorProto.InferenceCalculatorOptions.Delegate.Gpu.newBuilder() .setUseAdvancedGpuApi(true) .build()); + options + .delegateOptions() + .ifPresent( + delegateOptions -> + setDelegateOptions( + accelerationBuilder, + (BaseOptions.DelegateOptions.GpuOptions) delegateOptions)); break; } + return BaseOptionsProto.BaseOptions.newBuilder() .setModelAsset(externalFileBuilder.build()) .setAcceleration(accelerationBuilder.build()) .build(); } + + private void setDelegateOptions( + AccelerationProto.Acceleration.Builder accelerationBuilder, + BaseOptions.DelegateOptions.CpuOptions options) { + accelerationBuilder.setTflite( + InferenceCalculatorProto.InferenceCalculatorOptions.Delegate.TfLite.getDefaultInstance()); + } + + private void setDelegateOptions( + AccelerationProto.Acceleration.Builder accelerationBuilder, + BaseOptions.DelegateOptions.GpuOptions options) { + InferenceCalculatorProto.InferenceCalculatorOptions.Delegate.Gpu.Builder gpuBuilder = + InferenceCalculatorProto.InferenceCalculatorOptions.Delegate.Gpu.newBuilder() + .setUseAdvancedGpuApi(true); + options.cachedKernelPath().ifPresent(gpuBuilder::setCachedKernelPath); + options.serializedModelDir().ifPresent(gpuBuilder::setSerializedModelDir); + options.modelToken().ifPresent(gpuBuilder::setModelToken); + accelerationBuilder.setGpu(gpuBuilder.build()); + } } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl index 9d4fd00f..0fc4a497 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl @@ -59,6 +59,22 @@ _VISION_TASKS_JAVA_PROTO_LITE_TARGETS = [ "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_java_proto_lite", ] +_VISION_TASKS_IMAGE_GENERATOR_JAVA_PROTO_LITE_TARGETS = [ + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_geometry/proto:mesh_3d_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/image_segmenter/proto:segmenter_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/image_segmenter/calculators:tensors_to_segmentation_calculator_java_proto_lite", + "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite", +] + _TEXT_TASKS_JAVA_PROTO_LITE_TARGETS = [ "//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_java_proto_lite", "//mediapipe/tasks/cc/text/text_embedder/proto:text_embedder_graph_options_java_proto_lite", @@ -249,6 +265,39 @@ EOF native_library = native_library, ) +def mediapipe_tasks_vision_image_generator_aar(name, srcs, native_library): + """Builds medaipipe tasks vision image generator AAR. + + Args: + name: The bazel target name. + srcs: MediaPipe Vision Tasks' source files. + native_library: The native library that contains image generator task's graph and calculators. + """ + + native.genrule( + name = name + "tasks_manifest_generator", + outs = ["AndroidManifest.xml"], + cmd = """ +cat > $(OUTS) < + + + +EOF +""", + ) + + _mediapipe_tasks_aar( + name = name, + srcs = srcs, + manifest = "AndroidManifest.xml", + java_proto_lite_targets = _CORE_TASKS_JAVA_PROTO_LITE_TARGETS + _VISION_TASKS_IMAGE_GENERATOR_JAVA_PROTO_LITE_TARGETS, + native_library = native_library, + ) + def mediapipe_tasks_text_aar(name, srcs, native_library): """Builds medaipipe tasks text AAR. @@ -300,7 +349,6 @@ def _mediapipe_tasks_aar(name, srcs, manifest, java_proto_lite_targets, native_l name = name + "_jni_opencv_cc_lib", srcs = select({ "//mediapipe:android_arm64": ["@android_opencv//:libopencv_java3_so_arm64-v8a"], - "//mediapipe:android_armeabi": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"], "//mediapipe:android_arm": ["@android_opencv//:libopencv_java3_so_armeabi-v7a"], "//mediapipe:android_x86": ["@android_opencv//:libopencv_java3_so_x86"], "//mediapipe:android_x86_64": ["@android_opencv//:libopencv_java3_so_x86_64"], @@ -345,6 +393,7 @@ def _mediapipe_tasks_aar(name, srcs, manifest, java_proto_lite_targets, native_l "//third_party:androidx_annotation", "//third_party:autovalue", "@maven//:com_google_guava_guava", + "@com_google_protobuf//:protobuf_javalite", ] + select({ "//conditions:default": [":" + name + "_jni_opencv_cc_lib"], "//mediapipe/framework/port:disable_opencv": [], diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD index cbb1797e..1ddcd46c 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD @@ -18,16 +18,27 @@ package(default_visibility = ["//visibility:public"]) android_library( name = "core", + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + exports = [ + ":core_java", + ":libmediapipe_tasks_vision_jni_lib", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", + ], +) + +android_library( + name = "core_java", srcs = glob(["core/*.java"]), javacopts = [ "-Xep:AndroidJdkLibsChecker:OFF", ], deps = [ - ":libmediapipe_tasks_vision_jni_lib", "//mediapipe/framework/formats:rect_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework_no_mff", "//mediapipe/java/com/google/mediapipe/framework/image", - "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core:core_java", "//third_party:autovalue", "@maven//:com_google_guava_guava", ], @@ -246,6 +257,20 @@ android_library( android_library( name = "imagesegmenter", + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + manifest = "imagesegmenter/AndroidManifest.xml", + exports = [ + ":core", + ":imagesegmenter_java", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", + ], + deps = ["@maven//:com_google_guava_guava"], +) + +android_library( + name = "imagesegmenter_java", srcs = [ "imagesegmenter/ImageSegmenter.java", "imagesegmenter/ImageSegmenterResult.java", @@ -255,15 +280,15 @@ android_library( ], manifest = "imagesegmenter/AndroidManifest.xml", deps = [ - ":core", + ":core_java", "//mediapipe/framework:calculator_options_java_proto_lite", - "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/framework:android_framework_no_mff", "//mediapipe/java/com/google/mediapipe/framework/image", "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite", "//mediapipe/tasks/cc/vision/image_segmenter/calculators:tensors_to_segmentation_calculator_java_proto_lite", "//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/image_segmenter/proto:segmenter_options_java_proto_lite", - "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core:core_java", "//third_party:autovalue", "@maven//:com_google_guava_guava", ], @@ -388,6 +413,9 @@ load("//mediapipe/tasks/java/com/google/mediapipe/tasks:mediapipe_tasks_aar.bzl" mediapipe_tasks_vision_aar( name = "tasks_vision", - srcs = glob(["**/*.java"]), + srcs = glob( + ["**/*.java"], + exclude = ["imagegenerator/**"], + ), native_library = ":libmediapipe_tasks_vision_jni_lib", ) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/core/BaseVisionTaskApi.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/core/BaseVisionTaskApi.java index 9ea057b0..0405e6db 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/core/BaseVisionTaskApi.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/core/BaseVisionTaskApi.java @@ -27,7 +27,7 @@ import java.util.Map; /** The base class of MediaPipe vision tasks. */ public class BaseVisionTaskApi implements AutoCloseable { - private static final long MICROSECONDS_PER_MILLISECOND = 1000; + protected static final long MICROSECONDS_PER_MILLISECOND = 1000; protected final TaskRunner runner; protected final RunningMode runningMode; protected final String imageStreamName; @@ -69,12 +69,6 @@ public class BaseVisionTaskApi implements AutoCloseable { */ protected TaskResult processImageData( MPImage image, ImageProcessingOptions imageProcessingOptions) { - if (runningMode != RunningMode.IMAGE) { - throw new MediaPipeException( - MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), - "Task is not initialized with the image mode. Current running mode:" - + runningMode.name()); - } Map inputPackets = new HashMap<>(); inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image)); if (!normRectStreamName.isEmpty()) { @@ -84,6 +78,23 @@ public class BaseVisionTaskApi implements AutoCloseable { .getPacketCreator() .createProto(convertToNormalizedRect(imageProcessingOptions, image))); } + return processImageData(inputPackets); + } + + /** + * A synchronous method to process single image inputs. The call blocks the current thread until a + * failure status or a successful result is returned. + * + * @param inputPackets the maps of input stream names to the input packets. + * @throws MediaPipeException if the task is not in the image mode. + */ + protected TaskResult processImageData(Map inputPackets) { + if (runningMode != RunningMode.IMAGE) { + throw new MediaPipeException( + MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), + "Task is not initialized with the image mode. Current running mode:" + + runningMode.name()); + } return runner.process(inputPackets); } @@ -99,12 +110,6 @@ public class BaseVisionTaskApi implements AutoCloseable { */ protected TaskResult processVideoData( MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { - if (runningMode != RunningMode.VIDEO) { - throw new MediaPipeException( - MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), - "Task is not initialized with the video mode. Current running mode:" - + runningMode.name()); - } Map inputPackets = new HashMap<>(); inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image)); if (!normRectStreamName.isEmpty()) { @@ -114,6 +119,24 @@ public class BaseVisionTaskApi implements AutoCloseable { .getPacketCreator() .createProto(convertToNormalizedRect(imageProcessingOptions, image))); } + return processVideoData(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND); + } + + /** + * A synchronous method to process continuous video frames. The call blocks the current thread + * until a failure status or a successful result is returned. + * + * @param inputPackets the maps of input stream names to the input packets. + * @param timestampMs the corresponding timestamp of the input image in milliseconds. + * @throws MediaPipeException if the task is not in the video mode. + */ + protected TaskResult processVideoData(Map inputPackets, long timestampMs) { + if (runningMode != RunningMode.VIDEO) { + throw new MediaPipeException( + MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), + "Task is not initialized with the video mode. Current running mode:" + + runningMode.name()); + } return runner.process(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND); } @@ -129,12 +152,6 @@ public class BaseVisionTaskApi implements AutoCloseable { */ protected void sendLiveStreamData( MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { - if (runningMode != RunningMode.LIVE_STREAM) { - throw new MediaPipeException( - MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), - "Task is not initialized with the live stream mode. Current running mode:" - + runningMode.name()); - } Map inputPackets = new HashMap<>(); inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image)); if (!normRectStreamName.isEmpty()) { @@ -144,6 +161,24 @@ public class BaseVisionTaskApi implements AutoCloseable { .getPacketCreator() .createProto(convertToNormalizedRect(imageProcessingOptions, image))); } + sendLiveStreamData(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND); + } + + /** + * An asynchronous method to send live stream data to the {@link TaskRunner}. The results will be + * available in the user-defined result listener. + * + * @param inputPackets the maps of input stream names to the input packets. + * @param timestampMs the corresponding timestamp of the input image in milliseconds. + * @throws MediaPipeException if the task is not in the stream mode. + */ + protected void sendLiveStreamData(Map inputPackets, long timestampMs) { + if (runningMode != RunningMode.LIVE_STREAM) { + throw new MediaPipeException( + MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), + "Task is not initialized with the live stream mode. Current running mode:" + + runningMode.name()); + } runner.send(inputPackets, timestampMs * MICROSECONDS_PER_MILLISECOND); } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java index c91477e1..0429ecac 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java @@ -53,7 +53,15 @@ public abstract class FaceLandmarkerResult implements TaskResult { faceLandmarksProto.getLandmarkList()) { faceLandmarks.add( NormalizedLandmark.create( - faceLandmarkProto.getX(), faceLandmarkProto.getY(), faceLandmarkProto.getZ())); + faceLandmarkProto.getX(), + faceLandmarkProto.getY(), + faceLandmarkProto.getZ(), + faceLandmarkProto.hasVisibility() + ? Optional.of(faceLandmarkProto.getVisibility()) + : Optional.empty(), + faceLandmarkProto.hasPresence() + ? Optional.of(faceLandmarkProto.getPresence()) + : Optional.empty())); } } Optional>> multiFaceBlendshapes = Optional.empty(); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java index 467e871b..b8b236d4 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java @@ -25,6 +25,7 @@ import com.google.mediapipe.tasks.core.TaskResult; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; /** Represents the hand landmarks deection results generated by {@link HandLandmarker}. */ @AutoValue @@ -53,7 +54,15 @@ public abstract class HandLandmarkerResult implements TaskResult { handLandmarksProto.getLandmarkList()) { handLandmarks.add( NormalizedLandmark.create( - handLandmarkProto.getX(), handLandmarkProto.getY(), handLandmarkProto.getZ())); + handLandmarkProto.getX(), + handLandmarkProto.getY(), + handLandmarkProto.getZ(), + handLandmarkProto.hasVisibility() + ? Optional.of(handLandmarkProto.getVisibility()) + : Optional.empty(), + handLandmarkProto.hasPresence() + ? Optional.of(handLandmarkProto.getPresence()) + : Optional.empty())); } } for (LandmarkProto.LandmarkList handWorldLandmarksProto : worldLandmarksProto) { @@ -65,7 +74,13 @@ public abstract class HandLandmarkerResult implements TaskResult { com.google.mediapipe.tasks.components.containers.Landmark.create( handWorldLandmarkProto.getX(), handWorldLandmarkProto.getY(), - handWorldLandmarkProto.getZ())); + handWorldLandmarkProto.getZ(), + handWorldLandmarkProto.hasVisibility() + ? Optional.of(handWorldLandmarkProto.getVisibility()) + : Optional.empty(), + handWorldLandmarkProto.hasPresence() + ? Optional.of(handWorldLandmarkProto.getPresence()) + : Optional.empty())); } } for (ClassificationList handednessProto : handednessesProto) { diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/AndroidManifest.xml b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/AndroidManifest.xml new file mode 100644 index 00000000..5645810d --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + + diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/BUILD new file mode 100644 index 00000000..3a55c602 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/BUILD @@ -0,0 +1,91 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "//mediapipe/framework/tool:mediapipe_files.bzl", + "mediapipe_files", +) + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +mediapipe_files(srcs = ["libimagegenerator_gpu.so"]) + +# The native library of MediaPipe vision image generator tasks. +cc_binary( + name = "libmediapipe_tasks_vision_image_generator_jni.so", + linkopts = [ + "-Wl,--no-undefined", + "-Wl,--version-script,$(location //mediapipe/tasks/java:version_script.lds)", + ], + linkshared = 1, + linkstatic = 1, + deps = [ + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", + "//mediapipe/tasks/cc/vision/image_generator:image_generator_graph", + "//mediapipe/tasks/cc/vision/image_segmenter:image_segmenter_graph", + "//mediapipe/tasks/java:version_script.lds", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core/jni:model_resources_cache_jni", + ], +) + +cc_library( + name = "libmediapipe_tasks_vision_image_generator_jni_lib", + srcs = [":libmediapipe_tasks_vision_image_generator_jni.so"], + alwayslink = 1, +) + +android_library( + name = "imagegenerator", + srcs = [ + "ImageGenerator.java", + "ImageGeneratorResult.java", + ], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + manifest = "AndroidManifest.xml", + deps = [ + "//mediapipe/framework:calculator_options_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/framework/image", + "//mediapipe/tasks/cc/core/proto:external_file_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/image_generator/proto:conditioned_image_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/image_generator/proto:control_plugin_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/image_generator/proto:image_generator_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_graph_options_java_proto_lite", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/vision:core_java", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/vision:facelandmarker", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/vision:imagesegmenter", + "//third_party:any_java_proto", + "//third_party:autovalue", + "//third_party/java/protobuf:protobuf_lite", + "@maven//:androidx_annotation_annotation", + "@maven//:com_google_guava_guava", + ], +) + +load("//mediapipe/tasks/java/com/google/mediapipe/tasks:mediapipe_tasks_aar.bzl", "mediapipe_tasks_vision_image_generator_aar") + +mediapipe_tasks_vision_image_generator_aar( + name = "tasks_vision_image_generator", + srcs = glob(["**/*.java"]), + native_library = ":libmediapipe_tasks_vision_image_generator_jni_lib", +) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/ImageGenerator.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/ImageGenerator.java new file mode 100644 index 00000000..1de8e4c4 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/ImageGenerator.java @@ -0,0 +1,660 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.imagegenerator; + +import android.content.Context; +import android.graphics.Bitmap; +import android.util.Log; +import androidx.annotation.Nullable; +import com.google.auto.value.AutoValue; +import com.google.mediapipe.proto.CalculatorOptionsProto.CalculatorOptions; +import com.google.mediapipe.framework.AndroidPacketGetter; +import com.google.mediapipe.framework.Packet; +import com.google.mediapipe.framework.PacketGetter; +import com.google.mediapipe.framework.image.BitmapImageBuilder; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.core.BaseOptions; +import com.google.mediapipe.tasks.core.ErrorListener; +import com.google.mediapipe.tasks.core.OutputHandler; +import com.google.mediapipe.tasks.core.OutputHandler.PureResultListener; +import com.google.mediapipe.tasks.core.OutputHandler.ResultListener; +import com.google.mediapipe.tasks.core.TaskInfo; +import com.google.mediapipe.tasks.core.TaskOptions; +import com.google.mediapipe.tasks.core.TaskResult; +import com.google.mediapipe.tasks.core.TaskRunner; +import com.google.mediapipe.tasks.core.proto.ExternalFileProto; +import com.google.mediapipe.tasks.vision.core.BaseVisionTaskApi; +import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.facelandmarker.FaceLandmarker.FaceLandmarkerOptions; +import com.google.mediapipe.tasks.vision.facelandmarker.proto.FaceLandmarkerGraphOptionsProto.FaceLandmarkerGraphOptions; +import com.google.mediapipe.tasks.vision.imagegenerator.proto.ConditionedImageGraphOptionsProto.ConditionedImageGraphOptions; +import com.google.mediapipe.tasks.vision.imagegenerator.proto.ControlPluginGraphOptionsProto; +import com.google.mediapipe.tasks.vision.imagegenerator.proto.ImageGeneratorGraphOptionsProto; +import com.google.mediapipe.tasks.vision.imagesegmenter.ImageSegmenter.ImageSegmenterOptions; +import com.google.mediapipe.tasks.vision.imagesegmenter.proto.ImageSegmenterGraphOptionsProto.ImageSegmenterGraphOptions; +import com.google.protobuf.Any; +import com.google.protobuf.ExtensionRegistryLite; +import com.google.protobuf.InvalidProtocolBufferException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Performs image generation from a text prompt. */ +public final class ImageGenerator extends BaseVisionTaskApi { + + private static final String STEPS_STREAM_NAME = "steps"; + private static final String ITERATION_STREAM_NAME = "iteration"; + private static final String PROMPT_STREAM_NAME = "prompt"; + private static final String RAND_SEED_STREAM_NAME = "rand_seed"; + private static final String SOURCE_CONDITION_IMAGE_STREAM_NAME = "source_condition_image"; + private static final String CONDITION_IMAGE_STREAM_NAME = "condition_image"; + private static final String SELECT_STREAM_NAME = "select"; + private static final int GENERATED_IMAGE_OUT_STREAM_INDEX = 0; + private static final int STEPS_OUT_STREAM_INDEX = 1; + private static final int ITERATION_OUT_STREAM_INDEX = 2; + private static final String TASK_GRAPH_NAME = + "mediapipe.tasks.vision.image_generator.ImageGeneratorGraph"; + private static final String CONDITION_IMAGE_GRAPHS_CONTAINER_NAME = + "mediapipe.tasks.vision.image_generator.ConditionedImageGraphContainer"; + private static final String TAG = "ImageGenerator"; + private TaskRunner conditionImageGraphsContainerTaskRunner; + private Map conditionTypeIndex; + private boolean useConditionImage = false; + + /** + * Creates an {@link ImageGenerator} instance from an {@link ImageGeneratorOptions}. + * + * @param context an Android {@link Context}. + * @param generatorOptions an {@link ImageGeneratorOptions} instance. + * @throws MediaPipeException if there is an error during {@link ImageGenerator} creation. + */ + public static ImageGenerator createFromOptions( + Context context, ImageGeneratorOptions generatorOptions) { + return createFromOptions(context, generatorOptions, null); + } + + /** + * Creates an {@link ImageGenerator} instance, from {@link ImageGeneratorOptions} and {@link + * ConditionOptions}, if plugin models are used to generate an image based on the condition image. + * + * @param context an Android {@link Context}. + * @param generatorOptions an {@link ImageGeneratorOptions} instance. + * @param conditionOptions an {@link ConditionOptions} instance. + * @throws MediaPipeException if there is an error during {@link ImageGenerator} creation. + */ + public static ImageGenerator createFromOptions( + Context context, + ImageGeneratorOptions generatorOptions, + @Nullable ConditionOptions conditionOptions) { + List inputStreams = new ArrayList<>(); + inputStreams.addAll( + Arrays.asList( + "STEPS:" + STEPS_STREAM_NAME, + "ITERATION:" + ITERATION_STREAM_NAME, + "PROMPT:" + PROMPT_STREAM_NAME, + "RAND_SEED:" + RAND_SEED_STREAM_NAME)); + final boolean useConditionImage = conditionOptions != null; + if (useConditionImage) { + inputStreams.add("SELECT:" + SELECT_STREAM_NAME); + inputStreams.add("CONDITION_IMAGE:" + CONDITION_IMAGE_STREAM_NAME); + generatorOptions.conditionOptions = Optional.of(conditionOptions); + } + List outputStreams = + Arrays.asList("IMAGE:image_out", "STEPS:steps_out", "ITERATION:iteration_out"); + + OutputHandler handler = new OutputHandler<>(); + handler.setOutputPacketConverter( + new OutputHandler.OutputPacketConverter() { + @Override + @Nullable + public ImageGeneratorResult convertToTaskResult(List packets) { + int iteration = PacketGetter.getInt32(packets.get(ITERATION_OUT_STREAM_INDEX)); + int steps = PacketGetter.getInt32(packets.get(STEPS_OUT_STREAM_INDEX)); + Log.i("ImageGenerator", "Iteration: " + iteration + ", Steps: " + steps); + if (iteration != steps - 1) { + return null; + } + Log.i("ImageGenerator", "processing generated image"); + Packet packet = packets.get(GENERATED_IMAGE_OUT_STREAM_INDEX); + Bitmap generatedBitmap = AndroidPacketGetter.getBitmapFromRgb(packet); + BitmapImageBuilder bitmapImageBuilder = new BitmapImageBuilder(generatedBitmap); + return ImageGeneratorResult.create( + bitmapImageBuilder.build(), packet.getTimestamp() / MICROSECONDS_PER_MILLISECOND); + } + + @Override + public Void convertToTaskInput(List packets) { + return null; + } + }); + handler.setHandleTimestampBoundChanges(true); + if (generatorOptions.resultListener().isPresent()) { + ResultListener resultListener = + new ResultListener() { + @Override + public void run(ImageGeneratorResult imageGeneratorResult, Void input) { + generatorOptions.resultListener().get().run(imageGeneratorResult); + } + }; + handler.setResultListener(resultListener); + } + generatorOptions.errorListener().ifPresent(handler::setErrorListener); + TaskRunner runner = + TaskRunner.create( + context, + TaskInfo.builder() + .setTaskName(ImageGenerator.class.getSimpleName()) + .setTaskRunningModeName(RunningMode.IMAGE.name()) + .setTaskGraphName(TASK_GRAPH_NAME) + .setInputStreams(inputStreams) + .setOutputStreams(outputStreams) + .setTaskOptions(generatorOptions) + .setEnableFlowLimiting(false) + .build(), + handler); + ImageGenerator imageGenerator = new ImageGenerator(runner); + if (useConditionImage) { + imageGenerator.useConditionImage = true; + inputStreams = + Arrays.asList( + "IMAGE:" + SOURCE_CONDITION_IMAGE_STREAM_NAME, "SELECT:" + SELECT_STREAM_NAME); + outputStreams = Arrays.asList("CONDITION_IMAGE:" + CONDITION_IMAGE_STREAM_NAME); + OutputHandler conditionImageHandler = new OutputHandler<>(); + conditionImageHandler.setOutputPacketConverter( + new OutputHandler.OutputPacketConverter() { + @Override + public ConditionImageResult convertToTaskResult(List packets) { + Packet packet = packets.get(0); + return new AutoValue_ImageGenerator_ConditionImageResult( + new BitmapImageBuilder(AndroidPacketGetter.getBitmapFromRgb(packet)).build(), + packet.getTimestamp() / MICROSECONDS_PER_MILLISECOND); + } + + @Override + public Void convertToTaskInput(List packets) { + return null; + } + }); + conditionImageHandler.setHandleTimestampBoundChanges(true); + imageGenerator.conditionImageGraphsContainerTaskRunner = + TaskRunner.create( + context, + TaskInfo.builder() + .setTaskName(ImageGenerator.class.getSimpleName()) + .setTaskRunningModeName(RunningMode.IMAGE.name()) + .setTaskGraphName(CONDITION_IMAGE_GRAPHS_CONTAINER_NAME) + .setInputStreams(inputStreams) + .setOutputStreams(outputStreams) + .setTaskOptions(generatorOptions) + .setEnableFlowLimiting(false) + .build(), + conditionImageHandler); + imageGenerator.conditionTypeIndex = new HashMap<>(); + if (conditionOptions.faceConditionOptions().isPresent()) { + imageGenerator.conditionTypeIndex.put( + ConditionOptions.ConditionType.FACE, imageGenerator.conditionTypeIndex.size()); + } + if (conditionOptions.edgeConditionOptions().isPresent()) { + imageGenerator.conditionTypeIndex.put( + ConditionOptions.ConditionType.EDGE, imageGenerator.conditionTypeIndex.size()); + } + if (conditionOptions.depthConditionOptions().isPresent()) { + imageGenerator.conditionTypeIndex.put( + ConditionOptions.ConditionType.DEPTH, imageGenerator.conditionTypeIndex.size()); + } + } + return imageGenerator; + } + + private ImageGenerator(TaskRunner taskRunner) { + super(taskRunner, RunningMode.IMAGE, "", ""); + } + + /** + * Generates an image for iterations and the given random seed. Only valid when the ImageGenerator + * is created without condition options. + * + * @param prompt The text prompt describing the image to be generated. + * @param iterations The total iterations to generate the image. + * @param seed The random seed used during image generation. + */ + public ImageGeneratorResult generate(String prompt, int iterations, int seed) { + return runIterations(prompt, iterations, seed, null, 0); + } + + /** + * Generates an image based on the source image for iterations and the given random seed. Only + * valid when the ImageGenerator is created with condition options. + * + * @param prompt The text prompt describing the image to be generated. + * @param sourceConditionImage The source image used to create the condition image, which is used + * as a guidance for the image generation. + * @param conditionType The {@link ConditionOptions.ConditionType} specifying the type of + * condition image. + * @param iterations The total iterations to generate the image. + * @param seed The random seed used during image generation. + */ + public ImageGeneratorResult generate( + String prompt, + MPImage sourceConditionImage, + ConditionOptions.ConditionType conditionType, + int iterations, + int seed) { + return runIterations( + prompt, + iterations, + seed, + createConditionImage(sourceConditionImage, conditionType), + conditionTypeIndex.get(conditionType)); + } + + /** + * Create the condition image of specified condition type from the source image. Currently support + * face landmarks, depth image and edge image as the condition image. + * + * @param sourceConditionImage The source image used to create the condition image. + * @param conditionType The {@link ConditionOptions.ConditionType} specifying the type of + * condition image. + */ + public MPImage createConditionImage( + MPImage sourceConditionImage, ConditionOptions.ConditionType conditionType) { + if (!conditionTypeIndex.containsKey(conditionType)) { + throw new IllegalArgumentException( + "The condition type " + conditionType.name() + " is not created during initialization."); + } + Map inputPackets = new HashMap<>(); + inputPackets.put( + SOURCE_CONDITION_IMAGE_STREAM_NAME, + conditionImageGraphsContainerTaskRunner + .getPacketCreator() + .createImage(sourceConditionImage)); + inputPackets.put( + SELECT_STREAM_NAME, + conditionImageGraphsContainerTaskRunner + .getPacketCreator() + .createInt32(conditionTypeIndex.get(conditionType))); + ConditionImageResult result = + (ConditionImageResult) conditionImageGraphsContainerTaskRunner.process(inputPackets); + return result.conditionImage(); + } + + private ImageGeneratorResult runIterations( + String prompt, int steps, int seed, @Nullable MPImage conditionImage, int select) { + ImageGeneratorResult result = null; + long timestamp = System.currentTimeMillis() * MICROSECONDS_PER_MILLISECOND; + for (int i = 0; i < steps; i++) { + Map inputPackets = new HashMap<>(); + if (i == 0 && useConditionImage) { + inputPackets.put( + CONDITION_IMAGE_STREAM_NAME, runner.getPacketCreator().createImage(conditionImage)); + inputPackets.put(SELECT_STREAM_NAME, runner.getPacketCreator().createInt32(select)); + } + inputPackets.put(PROMPT_STREAM_NAME, runner.getPacketCreator().createString(prompt)); + inputPackets.put(STEPS_STREAM_NAME, runner.getPacketCreator().createInt32(steps)); + inputPackets.put(ITERATION_STREAM_NAME, runner.getPacketCreator().createInt32(i)); + inputPackets.put(RAND_SEED_STREAM_NAME, runner.getPacketCreator().createInt32(seed)); + result = (ImageGeneratorResult) runner.process(inputPackets, timestamp++); + } + if (useConditionImage) { + // Add condition image to the ImageGeneratorResult. + return ImageGeneratorResult.create( + result.generatedImage(), conditionImage, result.timestampMs()); + } + return result; + } + + /** Closes and cleans up the task runners. */ + @Override + public void close() { + runner.close(); + conditionImageGraphsContainerTaskRunner.close(); + } + + /** A container class for the condition image. */ + @AutoValue + protected abstract static class ConditionImageResult implements TaskResult { + + public abstract MPImage conditionImage(); + + @Override + public abstract long timestampMs(); + } + + /** Options for setting up an {@link ImageGenerator}. */ + @AutoValue + public abstract static class ImageGeneratorOptions extends TaskOptions { + + /** Builder for {@link ImageGeneratorOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + + /** Sets the text to image model directory storing the model weights. */ + public abstract Builder setText2ImageModelDirectory(String modelDirectory); + + /** Sets the path to LoRA weights file. */ + public abstract Builder setLoraWeightsFilePath(String loraWeightsFilePath); + + public abstract Builder setResultListener( + PureResultListener resultListener); + + /** Sets an optional {@link ErrorListener}}. */ + public abstract Builder setErrorListener(ErrorListener value); + + abstract ImageGeneratorOptions autoBuild(); + + /** Validates and builds the {@link ImageGeneratorOptions} instance. */ + public final ImageGeneratorOptions build() { + return autoBuild(); + } + } + + abstract String text2ImageModelDirectory(); + + abstract Optional loraWeightsFilePath(); + + abstract Optional> resultListener(); + + abstract Optional errorListener(); + + private Optional conditionOptions; + + public static Builder builder() { + return new AutoValue_ImageGenerator_ImageGeneratorOptions.Builder() + .setText2ImageModelDirectory(""); + } + + /** Converts an {@link ImageGeneratorOptions} to a {@link Any} protobuf message. */ + @Override + public Any convertToAnyProto() { + ImageGeneratorGraphOptionsProto.ImageGeneratorGraphOptions.Builder taskOptionsBuilder = + ImageGeneratorGraphOptionsProto.ImageGeneratorGraphOptions.newBuilder(); + if (conditionOptions != null && conditionOptions.isPresent()) { + try { + taskOptionsBuilder.mergeFrom( + conditionOptions.get().convertToAnyProto().getValue(), + ExtensionRegistryLite.getGeneratedRegistry()); + } catch (InvalidProtocolBufferException e) { + Log.e(TAG, "Error converting ConditionOptions to proto. " + e.getMessage()); + e.printStackTrace(); + } + } + taskOptionsBuilder.setText2ImageModelDirectory(text2ImageModelDirectory()); + if (loraWeightsFilePath().isPresent()) { + ExternalFileProto.ExternalFile.Builder externalFileBuilder = + ExternalFileProto.ExternalFile.newBuilder(); + externalFileBuilder.setFileName(loraWeightsFilePath().get()); + taskOptionsBuilder.setLoraWeightsFile(externalFileBuilder.build()); + } + return Any.newBuilder() + .setTypeUrl( + "type.googleapis.com/mediapipe.tasks.vision.image_generator.proto.ImageGeneratorGraphOptions") + .setValue(taskOptionsBuilder.build().toByteString()) + .build(); + } + } + + /** Options for setting up the conditions types and the plugin models */ + @AutoValue + public abstract static class ConditionOptions extends TaskOptions { + + /** The supported condition type. */ + public enum ConditionType { + FACE, + EDGE, + DEPTH + } + + /** Builder for {@link ConditionOptions}. At least one type of condition options must be set. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setFaceConditionOptions(FaceConditionOptions faceConditionOptions); + + public abstract Builder setDepthConditionOptions(DepthConditionOptions depthConditionOptions); + + public abstract Builder setEdgeConditionOptions(EdgeConditionOptions edgeConditionOptions); + + abstract ConditionOptions autoBuild(); + + /** Validates and builds the {@link ConditionOptions} instance. */ + public final ConditionOptions build() { + ConditionOptions options = autoBuild(); + if (!options.faceConditionOptions().isPresent() + && !options.depthConditionOptions().isPresent() + && !options.edgeConditionOptions().isPresent()) { + throw new IllegalArgumentException( + "At least one of `faceConditionOptions`, `depthConditionOptions` and" + + " `edgeConditionOptions` must be set."); + } + return options; + } + } + + abstract Optional faceConditionOptions(); + + abstract Optional depthConditionOptions(); + + abstract Optional edgeConditionOptions(); + + public static Builder builder() { + return new AutoValue_ImageGenerator_ConditionOptions.Builder(); + } + + /** + * Converts an {@link ImageGeneratorOptions} to a {@link CalculatorOptions} protobuf message. + */ + @Override + public Any convertToAnyProto() { + ImageGeneratorGraphOptionsProto.ImageGeneratorGraphOptions.Builder taskOptionsBuilder = + ImageGeneratorGraphOptionsProto.ImageGeneratorGraphOptions.newBuilder(); + if (faceConditionOptions().isPresent()) { + taskOptionsBuilder.addControlPluginGraphsOptions( + ControlPluginGraphOptionsProto.ControlPluginGraphOptions.newBuilder() + .setBaseOptions( + convertBaseOptionsToProto(faceConditionOptions().get().baseOptions())) + .setConditionedImageGraphOptions( + ConditionedImageGraphOptions.newBuilder() + .setFaceConditionTypeOptions(faceConditionOptions().get().convertToProto()) + .build()) + .build()); + } + if (edgeConditionOptions().isPresent()) { + taskOptionsBuilder.addControlPluginGraphsOptions( + ControlPluginGraphOptionsProto.ControlPluginGraphOptions.newBuilder() + .setBaseOptions( + convertBaseOptionsToProto(edgeConditionOptions().get().baseOptions())) + .setConditionedImageGraphOptions( + ConditionedImageGraphOptions.newBuilder() + .setEdgeConditionTypeOptions(edgeConditionOptions().get().convertToProto()) + .build()) + .build()); + if (depthConditionOptions().isPresent()) { + taskOptionsBuilder.addControlPluginGraphsOptions( + ControlPluginGraphOptionsProto.ControlPluginGraphOptions.newBuilder() + .setBaseOptions( + convertBaseOptionsToProto(depthConditionOptions().get().baseOptions())) + .setConditionedImageGraphOptions( + ConditionedImageGraphOptions.newBuilder() + .setDepthConditionTypeOptions( + depthConditionOptions().get().convertToProto()) + .build()) + .build()); + } + } + return Any.newBuilder() + .setTypeUrl( + "type.googleapis.com/mediapipe.tasks.vision.image_generator.proto.ImageGeneratorGraphOptions") + .setValue(taskOptionsBuilder.build().toByteString()) + .build(); + } + + /** Options for drawing face landmarks image. */ + @AutoValue + public abstract static class FaceConditionOptions extends TaskOptions { + + /** Builder for {@link FaceConditionOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Set the base options for plugin model. */ + public abstract Builder setBaseOptions(BaseOptions baseOptions); + + /* {@link FaceLandmarkerOptions} used to detect face landmarks in the source image. */ + public abstract Builder setFaceLandmarkerOptions( + FaceLandmarkerOptions faceLandmarkerOptions); + + abstract FaceConditionOptions autoBuild(); + + /** Validates and builds the {@link FaceConditionOptions} instance. */ + public final FaceConditionOptions build() { + return autoBuild(); + } + } + + abstract BaseOptions baseOptions(); + + abstract FaceLandmarkerOptions faceLandmarkerOptions(); + + public static Builder builder() { + return new AutoValue_ImageGenerator_ConditionOptions_FaceConditionOptions.Builder(); + } + + ConditionedImageGraphOptions.FaceConditionTypeOptions convertToProto() { + return ConditionedImageGraphOptions.FaceConditionTypeOptions.newBuilder() + .setFaceLandmarkerGraphOptions( + FaceLandmarkerGraphOptions.newBuilder() + .mergeFrom( + faceLandmarkerOptions() + .convertToCalculatorOptionsProto() + .getExtension(FaceLandmarkerGraphOptions.ext)) + .build()) + .build(); + } + } + + /** Options for detecting depth image. */ + @AutoValue + public abstract static class DepthConditionOptions extends TaskOptions { + + /** Builder for {@link DepthConditionOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + + /** Set the base options for plugin model. */ + public abstract Builder setBaseOptions(BaseOptions baseOptions); + + /** {@link ImageSegmenterOptions} used to detect depth image from the source image. */ + public abstract Builder setImageSegmenterOptions( + ImageSegmenterOptions imageSegmenterOptions); + + abstract DepthConditionOptions autoBuild(); + + /** Validates and builds the {@link DepthConditionOptions} instance. */ + public final DepthConditionOptions build() { + DepthConditionOptions options = autoBuild(); + return options; + } + } + + abstract BaseOptions baseOptions(); + + abstract ImageSegmenterOptions imageSegmenterOptions(); + + public static Builder builder() { + return new AutoValue_ImageGenerator_ConditionOptions_DepthConditionOptions.Builder(); + } + + ConditionedImageGraphOptions.DepthConditionTypeOptions convertToProto() { + return ConditionedImageGraphOptions.DepthConditionTypeOptions.newBuilder() + .setImageSegmenterGraphOptions( + imageSegmenterOptions() + .convertToCalculatorOptionsProto() + .getExtension(ImageSegmenterGraphOptions.ext)) + .build(); + } + } + + /** Options for detecting edge image. */ + @AutoValue + public abstract static class EdgeConditionOptions { + + /** + * Builder for {@link EdgeConditionOptions}. + * + *

These parameters are used to config Canny edge algorithm of OpenCV. + * + *

See more details: + * https://docs.opencv.org/3.4/dd/d1a/group__imgproc__feature.html#ga04723e007ed888ddf11d9ba04e2232de + */ + @AutoValue.Builder + public abstract static class Builder { + + /** Set the base options for plugin model. */ + public abstract Builder setBaseOptions(BaseOptions baseOptions); + + /** First threshold for the hysteresis procedure. */ + public abstract Builder setThreshold1(Float threshold1); + + /** Second threshold for the hysteresis procedure. */ + public abstract Builder setThreshold2(Float threshold2); + + /** Aperture size for the Sobel operator. Typical range is 3~7. */ + public abstract Builder setApertureSize(Integer apertureSize); + + /** + * flag, indicating whether a more accurate L2 norm should be used to calculate the image + * gradient magnitude ( L2gradient=true ), or whether the default L1 norm is enough ( + * L2gradient=false ). + */ + public abstract Builder setL2Gradient(Boolean l2Gradient); + + abstract EdgeConditionOptions autoBuild(); + + /** Validates and builds the {@link EdgeConditionOptions} instance. */ + public final EdgeConditionOptions build() { + return autoBuild(); + } + } + + abstract BaseOptions baseOptions(); + + abstract Float threshold1(); + + abstract Float threshold2(); + + abstract Integer apertureSize(); + + abstract Boolean l2Gradient(); + + public static Builder builder() { + return new AutoValue_ImageGenerator_ConditionOptions_EdgeConditionOptions.Builder() + .setThreshold1(100f) + .setThreshold2(200f) + .setApertureSize(3) + .setL2Gradient(false); + } + + ConditionedImageGraphOptions.EdgeConditionTypeOptions convertToProto() { + return ConditionedImageGraphOptions.EdgeConditionTypeOptions.newBuilder() + .setThreshold1(threshold1()) + .setThreshold2(threshold2()) + .setApertureSize(apertureSize()) + .setL2Gradient(l2Gradient()) + .build(); + } + } + } +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/ImageGeneratorResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/ImageGeneratorResult.java new file mode 100644 index 00000000..6bb3ab60 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/ImageGeneratorResult.java @@ -0,0 +1,44 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.imagegenerator; + +import com.google.auto.value.AutoValue; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.core.TaskResult; +import java.util.Optional; + +/** Represents the image generation results generated by {@link ImageGenerator}. */ +@AutoValue +public abstract class ImageGeneratorResult implements TaskResult { + + /** Create an {@link ImageGeneratorResult} instance from the generated image. */ + public static ImageGeneratorResult create( + MPImage generatedImage, MPImage conditionImage, long timestampMs) { + return new AutoValue_ImageGeneratorResult( + generatedImage, Optional.of(conditionImage), timestampMs); + } + + /** Create an {@link ImageGeneratorResult} instance from the generated image. */ + public static ImageGeneratorResult create(MPImage generatedImage, long timestampMs) { + return new AutoValue_ImageGeneratorResult(generatedImage, Optional.empty(), timestampMs); + } + + public abstract MPImage generatedImage(); + + public abstract Optional conditionImage(); + + @Override + public abstract long timestampMs(); +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java index f977c015..2a64b588 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java @@ -43,7 +43,9 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.function.BiFunction; @@ -77,9 +79,13 @@ public final class ImageSegmenter extends BaseVisionTaskApi { private static final String TAG = ImageSegmenter.class.getSimpleName(); private static final String IMAGE_IN_STREAM_NAME = "image_in"; private static final String NORM_RECT_IN_STREAM_NAME = "norm_rect_in"; + private static final String OUTPUT_SIZE_IN_STREAM_NAME = "output_size_in"; private static final List INPUT_STREAMS = Collections.unmodifiableList( - Arrays.asList("IMAGE:" + IMAGE_IN_STREAM_NAME, "NORM_RECT:" + NORM_RECT_IN_STREAM_NAME)); + Arrays.asList( + "IMAGE:" + IMAGE_IN_STREAM_NAME, + "NORM_RECT:" + NORM_RECT_IN_STREAM_NAME, + "OUTPUT_SIZE:" + OUTPUT_SIZE_IN_STREAM_NAME)); private static final String TASK_GRAPH_NAME = "mediapipe.tasks.vision.image_segmenter.ImageSegmenterGraph"; private static final String TENSORS_TO_SEGMENTATION_CALCULATOR_NAME = @@ -238,6 +244,7 @@ public final class ImageSegmenter extends BaseVisionTaskApi { this.hasResultListener = hasResultListener; populateLabels(); } + /** * Populate the labelmap in TensorsToSegmentationCalculator to labels field. * @@ -275,9 +282,9 @@ public final class ImageSegmenter extends BaseVisionTaskApi { /** * Performs image segmentation on the provided single image with default image processing options, - * i.e. without any rotation applied. Only use this method when the {@link ImageSegmenter} is - * created with {@link RunningMode.IMAGE}. TODO update java doc for input image - * format. + * i.e. without any rotation applied. The output mask has the same size as the input image. Only + * use this method when the {@link ImageSegmenter} is created with {@link RunningMode.IMAGE}. + * TODO update java doc for input image format. * *

{@link ImageSegmenter} supports the following color space types: * @@ -294,9 +301,9 @@ public final class ImageSegmenter extends BaseVisionTaskApi { } /** - * Performs image segmentation on the provided single image. Only use this method when the {@link - * ImageSegmenter} is created with {@link RunningMode.IMAGE}. TODO update java doc - * for input image format. + * Performs image segmentation on the provided single image. The output mask has the same size as + * the input image. Only use this method when the {@link ImageSegmenter} is created with {@link + * RunningMode.IMAGE}. TODO update java doc for input image format. * *

{@link ImageSegmenter} supports the following color space types: * @@ -316,21 +323,47 @@ public final class ImageSegmenter extends BaseVisionTaskApi { */ public ImageSegmenterResult segment( MPImage image, ImageProcessingOptions imageProcessingOptions) { + return segment( + image, + SegmentationOptions.builder() + .setOutputWidth(image.getWidth()) + .setOutputHeight(image.getHeight()) + .setImageProcessingOptions(imageProcessingOptions) + .build()); + } + + /** + * Performs image segmentation on the provided single image. Only use this method when the {@link + * ImageSegmenter} is created with {@link RunningMode.IMAGE}. TODO update java doc + * for input image format. + * + *

{@link ImageSegmenter} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param segmentationOptions the {@link SegmentationOptions} used to configure the runtime + * behavior of the {@link ImageSegmenter}. + * @throws MediaPipeException if there is an internal error. Or if {@link ImageSegmenter} is + * created with a {@link ResultListener}. + */ + public ImageSegmenterResult segment(MPImage image, SegmentationOptions segmentationOptions) { if (hasResultListener) { throw new MediaPipeException( MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), "ResultListener is provided in the ImageSegmenterOptions, but this method will return an" + " ImageSegmentationResult."); } - validateImageProcessingOptions(imageProcessingOptions); - return (ImageSegmenterResult) processImageData(image, imageProcessingOptions); + return (ImageSegmenterResult) processImageData(buildInputPackets(image, segmentationOptions)); } /** * Performs image segmentation on the provided single image with default image processing options, * i.e. without any rotation applied, and provides zero-copied results via {@link ResultListener} - * in {@link ImageSegmenterOptions}. Only use this method when the {@link ImageSegmenter} is - * created with {@link RunningMode.IMAGE}. + * in {@link ImageSegmenterOptions}. The output mask has the same size as the input image. Only + * use this method when the {@link ImageSegmenter} is created with {@link RunningMode.IMAGE}. * *

TODO update java doc for input image format. * @@ -341,8 +374,6 @@ public final class ImageSegmenter extends BaseVisionTaskApi { * * * @param image a MediaPipe {@link MPImage} object for processing. - * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a - * region-of-interest. * @throws MediaPipeException if there is an internal error. Or if {@link ImageSegmenter} is not * created with {@link ResultListener} set in {@link ImageSegmenterOptions}. */ @@ -352,8 +383,9 @@ public final class ImageSegmenter extends BaseVisionTaskApi { /** * Performs image segmentation on the provided single image, and provides zero-copied results via - * {@link ResultListener} in {@link ImageSegmenterOptions}. Only use this method when the {@link - * ImageSegmenter} is created with {@link RunningMode.IMAGE}. + * {@link ResultListener} in {@link ImageSegmenterOptions}. The output mask has the same size as + * the input image. Only use this method when the {@link ImageSegmenter} is created with {@link + * RunningMode.IMAGE}. * *

TODO update java doc for input image format. * @@ -375,21 +407,53 @@ public final class ImageSegmenter extends BaseVisionTaskApi { */ public void segmentWithResultListener( MPImage image, ImageProcessingOptions imageProcessingOptions) { + segmentWithResultListener( + image, + SegmentationOptions.builder() + .setOutputWidth(image.getWidth()) + .setOutputHeight(image.getHeight()) + .setImageProcessingOptions(imageProcessingOptions) + .build()); + } + + /** + * Performs image segmentation on the provided single image, and provides zero-copied results via + * {@link ResultListener} in {@link ImageSegmenterOptions}. Only use this method when the {@link + * ImageSegmenter} is created with {@link RunningMode.IMAGE}. + * + *

TODO update java doc for input image format. + * + *

{@link ImageSegmenter} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param segmentationOptions the {@link SegmentationOptions} used to configure the runtime + * behavior of the {@link ImageSegmenter}. + * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the + * input image before running inference. Note that region-of-interest is not supported + * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in + * this method throwing an IllegalArgumentException. + * @throws MediaPipeException if there is an internal error. Or if {@link ImageSegmenter} is not + * created with {@link ResultListener} set in {@link ImageSegmenterOptions}. + */ + public void segmentWithResultListener(MPImage image, SegmentationOptions segmentationOptions) { if (!hasResultListener) { throw new MediaPipeException( MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), "ResultListener is not set in the ImageSegmenterOptions, but this method expects a" + " ResultListener to process ImageSegmentationResult."); } - validateImageProcessingOptions(imageProcessingOptions); ImageSegmenterResult unused = - (ImageSegmenterResult) processImageData(image, imageProcessingOptions); + (ImageSegmenterResult) processImageData(buildInputPackets(image, segmentationOptions)); } /** * Performs image segmentation on the provided video frame with default image processing options, - * i.e. without any rotation applied. Only use this method when the {@link ImageSegmenter} is - * created with {@link RunningMode.VIDEO}. + * i.e. without any rotation applied. The output mask has the same size as the input image. Only + * use this method when the {@link ImageSegmenter} is created with {@link RunningMode.VIDEO}. * *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps * must be monotonically increasing. @@ -410,8 +474,9 @@ public final class ImageSegmenter extends BaseVisionTaskApi { } /** - * Performs image segmentation on the provided video frame. Only use this method when the {@link - * ImageSegmenter} is created with {@link RunningMode.VIDEO}. + * Performs image segmentation on the provided video frame. The output mask has the same size as + * the input image. Only use this method when the {@link ImageSegmenter} is created with {@link + * RunningMode.VIDEO}. * *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps * must be monotonically increasing. @@ -435,21 +500,53 @@ public final class ImageSegmenter extends BaseVisionTaskApi { */ public ImageSegmenterResult segmentForVideo( MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { + return segmentForVideo( + image, + SegmentationOptions.builder() + .setOutputWidth(image.getWidth()) + .setOutputHeight(image.getHeight()) + .setImageProcessingOptions(imageProcessingOptions) + .build(), + timestampMs); + } + + /** + * Performs image segmentation on the provided video frame. Only use this method when the {@link + * ImageSegmenter} is created with {@link RunningMode.VIDEO}. + * + *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps + * must be monotonically increasing. + * + *

{@link ImageSegmenter} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param segmentationOptions the {@link SegmentationOptions} used to configure the runtime + * behavior of the {@link ImageSegmenter}. + * @param timestampMs the input timestamp (in milliseconds). + * @throws MediaPipeException if there is an internal error. Or if {@link ImageSegmenter} is + * created with a {@link ResultListener}. + */ + public ImageSegmenterResult segmentForVideo( + MPImage image, SegmentationOptions segmentationOptions, long timestampMs) { if (hasResultListener) { throw new MediaPipeException( MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), "ResultListener is provided in the ImageSegmenterOptions, but this method will return an" + " ImageSegmentationResult."); } - validateImageProcessingOptions(imageProcessingOptions); - return (ImageSegmenterResult) processVideoData(image, imageProcessingOptions, timestampMs); + return (ImageSegmenterResult) + processVideoData(buildInputPackets(image, segmentationOptions), timestampMs); } /** * Performs image segmentation on the provided video frame with default image processing options, * i.e. without any rotation applied, and provides zero-copied results via {@link ResultListener} - * in {@link ImageSegmenterOptions}. Only use this method when the {@link ImageSegmenter} is - * created with {@link RunningMode.VIDEO}. + * in {@link ImageSegmenterOptions}. The output mask has the same size as the input image. Only + * use this method when the {@link ImageSegmenter} is created with {@link RunningMode.VIDEO}. * *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps * must be monotonically increasing. @@ -469,6 +566,40 @@ public final class ImageSegmenter extends BaseVisionTaskApi { segmentForVideoWithResultListener(image, ImageProcessingOptions.builder().build(), timestampMs); } + /** + * Performs image segmentation on the provided video frame, and provides zero-copied results via + * {@link ResultListener} in {@link ImageSegmenterOptions}. The output mask has the same size as + * the input image. Only use this method when the {@link ImageSegmenter} is created with {@link + * RunningMode.VIDEO}. + * + *

It's required to provide the video frame's timestamp (in milliseconds). The input timestamps + * must be monotonically increasing. + * + *

{@link ImageSegmenter} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param timestampMs the input timestamp (in milliseconds). + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. Or if {@link ImageSegmenter} is not + * created with {@link ResultListener} set in {@link ImageSegmenterOptions}. + */ + public void segmentForVideoWithResultListener( + MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { + segmentForVideoWithResultListener( + image, + SegmentationOptions.builder() + .setOutputWidth(image.getWidth()) + .setOutputHeight(image.getHeight()) + .setImageProcessingOptions(imageProcessingOptions) + .build(), + timestampMs); + } + /** * Performs image segmentation on the provided video frame, and provides zero-copied results via * {@link ResultListener} in {@link ImageSegmenterOptions}. Only use this method when the {@link @@ -484,28 +615,31 @@ public final class ImageSegmenter extends BaseVisionTaskApi { * * * @param image a MediaPipe {@link MPImage} object for processing. + * @param segmentationOptions the {@link SegmentationOptions} used to configure the runtime + * behavior of the {@link ImageSegmenter}. * @param timestampMs the input timestamp (in milliseconds). * @throws MediaPipeException if there is an internal error. Or if {@link ImageSegmenter} is not * created with {@link ResultListener} set in {@link ImageSegmenterOptions}. */ public void segmentForVideoWithResultListener( - MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { + MPImage image, SegmentationOptions segmentationOptions, long timestampMs) { if (!hasResultListener) { throw new MediaPipeException( MediaPipeException.StatusCode.FAILED_PRECONDITION.ordinal(), "ResultListener is not set in the ImageSegmenterOptions, but this method expects a" + " ResultListener to process ImageSegmentationResult."); } - validateImageProcessingOptions(imageProcessingOptions); ImageSegmenterResult unused = - (ImageSegmenterResult) processVideoData(image, imageProcessingOptions, timestampMs); + (ImageSegmenterResult) + processVideoData(buildInputPackets(image, segmentationOptions), timestampMs); } /** * Sends live image data to perform image segmentation with default image processing options, i.e. * without any rotation applied, and the results will be available via the {@link ResultListener} - * provided in the {@link ImageSegmenterOptions}. Only use this method when the {@link - * ImageSegmenter } is created with {@link RunningMode.LIVE_STREAM}. + * provided in the {@link ImageSegmenterOptions}. The output mask has the same size as the input + * image. Only use this method when the {@link ImageSegmenter } is created with {@link + * RunningMode.LIVE_STREAM}. * *

It's required to provide a timestamp (in milliseconds) to indicate when the input image is * sent to the image segmenter. The input timestamps must be monotonically increasing. @@ -526,8 +660,9 @@ public final class ImageSegmenter extends BaseVisionTaskApi { /** * Sends live image data to perform image segmentation, and the results will be available via the - * {@link ResultListener} provided in the {@link ImageSegmenterOptions}. Only use this method when - * the {@link ImageSegmenter} is created with {@link RunningMode.LIVE_STREAM}. + * {@link ResultListener} provided in the {@link ImageSegmenterOptions}. The output mask has the + * same size as the input image. Only use this method when the {@link ImageSegmenter} is created + * with {@link RunningMode.LIVE_STREAM}. * *

It's required to provide a timestamp (in milliseconds) to indicate when the input image is * sent to the image segmenter. The input timestamps must be monotonically increasing. @@ -550,8 +685,39 @@ public final class ImageSegmenter extends BaseVisionTaskApi { */ public void segmentAsync( MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { - validateImageProcessingOptions(imageProcessingOptions); - sendLiveStreamData(image, imageProcessingOptions, timestampMs); + segmentAsync( + image, + SegmentationOptions.builder() + .setOutputWidth(image.getWidth()) + .setOutputHeight(image.getHeight()) + .setImageProcessingOptions(imageProcessingOptions) + .build(), + timestampMs); + } + + /** + * Sends live image data to perform image segmentation, and the results will be available via the + * {@link ResultListener} provided in the {@link ImageSegmenterOptions}. Only use this method when + * the {@link ImageSegmenter} is created with {@link RunningMode.LIVE_STREAM}. + * + *

It's required to provide a timestamp (in milliseconds) to indicate when the input image is + * sent to the image segmenter. The input timestamps must be monotonically increasing. + * + *

{@link ImageSegmenter} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param segmentationOptions the {@link SegmentationOptions} used to configure the runtime + * behavior of the {@link ImageSegmenter}. + * @param timestampMs the input timestamp (in milliseconds). + * @throws MediaPipeException if there is an internal error. + */ + public void segmentAsync( + MPImage image, SegmentationOptions segmentationOptions, long timestampMs) { + sendLiveStreamData(buildInputPackets(image, segmentationOptions), timestampMs); } /** @@ -565,6 +731,56 @@ public final class ImageSegmenter extends BaseVisionTaskApi { return labels; } + /** Options for configuring runtime behavior of {@link ImageSegmenter}. */ + @AutoValue + public abstract static class SegmentationOptions { + + /** Builder fo {@link SegmentationOptions} */ + @AutoValue.Builder + public abstract static class Builder { + + /** Set the width of the output segmentation masks. */ + public abstract Builder setOutputWidth(int value); + + /** Set the height of the output segmentation masks. */ + public abstract Builder setOutputHeight(int value); + + /** Set the image processing options. */ + public abstract Builder setImageProcessingOptions(ImageProcessingOptions value); + + abstract SegmentationOptions autoBuild(); + + /** + * Validates and builds the {@link SegmentationOptions} instance. + * + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + */ + public final SegmentationOptions build() { + SegmentationOptions options = autoBuild(); + if (options.outputWidth() <= 0 || options.outputHeight() <= 0) { + throw new IllegalArgumentException( + "Both outputWidth and outputHeight must be larger than 0."); + } + if (options.imageProcessingOptions().regionOfInterest().isPresent()) { + throw new IllegalArgumentException("ImageSegmenter doesn't support region-of-interest."); + } + return options; + } + } + + abstract int outputWidth(); + + abstract int outputHeight(); + + abstract ImageProcessingOptions imageProcessingOptions(); + + public static Builder builder() { + return new AutoValue_ImageSegmenter_SegmentationOptions.Builder() + .setImageProcessingOptions(ImageProcessingOptions.builder().build()); + } + } + /** Options for setting up an {@link ImageSegmenter}. */ @AutoValue public abstract static class ImageSegmenterOptions extends TaskOptions { @@ -680,14 +896,24 @@ public final class ImageSegmenter extends BaseVisionTaskApi { } } - /** - * Validates that the provided {@link ImageProcessingOptions} doesn't contain a - * region-of-interest. - */ - private static void validateImageProcessingOptions( - ImageProcessingOptions imageProcessingOptions) { - if (imageProcessingOptions.regionOfInterest().isPresent()) { - throw new IllegalArgumentException("ImageSegmenter doesn't support region-of-interest."); + private Map buildInputPackets( + MPImage image, SegmentationOptions segmentationOptions) { + Map inputPackets = new HashMap<>(); + inputPackets.put(imageStreamName, runner.getPacketCreator().createImage(image)); + inputPackets.put( + OUTPUT_SIZE_IN_STREAM_NAME, + runner + .getPacketCreator() + .createInt32Pair( + segmentationOptions.outputWidth(), segmentationOptions.outputHeight())); + if (!normRectStreamName.isEmpty()) { + inputPackets.put( + normRectStreamName, + runner + .getPacketCreator() + .createProto( + convertToNormalizedRect(segmentationOptions.imageProcessingOptions(), image))); } + return inputPackets; } } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java index 389e7826..0dde5670 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java @@ -58,7 +58,15 @@ public abstract class PoseLandmarkerResult implements TaskResult { poseLandmarksProto.getLandmarkList()) { poseLandmarks.add( NormalizedLandmark.create( - poseLandmarkProto.getX(), poseLandmarkProto.getY(), poseLandmarkProto.getZ())); + poseLandmarkProto.getX(), + poseLandmarkProto.getY(), + poseLandmarkProto.getZ(), + poseLandmarkProto.hasVisibility() + ? Optional.of(poseLandmarkProto.getVisibility()) + : Optional.empty(), + poseLandmarkProto.hasPresence() + ? Optional.of(poseLandmarkProto.getPresence()) + : Optional.empty())); } } for (LandmarkProto.LandmarkList poseWorldLandmarksProto : worldLandmarksProto) { @@ -70,7 +78,13 @@ public abstract class PoseLandmarkerResult implements TaskResult { Landmark.create( poseWorldLandmarkProto.getX(), poseWorldLandmarkProto.getY(), - poseWorldLandmarkProto.getZ())); + poseWorldLandmarkProto.getZ(), + poseWorldLandmarkProto.hasVisibility() + ? Optional.of(poseWorldLandmarkProto.getVisibility()) + : Optional.empty(), + poseWorldLandmarkProto.hasPresence() + ? Optional.of(poseWorldLandmarkProto.getPresence()) + : Optional.empty())); } } return new AutoValue_PoseLandmarkerResult( diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/AndroidManifest.xml b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/AndroidManifest.xml new file mode 100644 index 00000000..26310fc1 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/AndroidManifest.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/BUILD b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/BUILD index 01e7ad0f..ce7435d6 100644 --- a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/BUILD +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/BUILD @@ -23,3 +23,5 @@ android_library( "//third_party/java/android_libs/guava_jdk5:io", ], ) + +# TODO: Enable this in OSS diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/BaseOptionsTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/BaseOptionsTest.java new file mode 100644 index 00000000..939ecb40 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/core/BaseOptionsTest.java @@ -0,0 +1,159 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.core; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.mediapipe.proto.CalculatorOptionsProto.CalculatorOptions; +import com.google.mediapipe.tasks.core.proto.AccelerationProto; +import com.google.mediapipe.tasks.core.proto.BaseOptionsProto; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; + +/** Test for {@link BaseOptions} */ +@RunWith(Suite.class) +@SuiteClasses({BaseOptionsTest.General.class, BaseOptionsTest.ConvertProtoTest.class}) +public class BaseOptionsTest { + + static final String MODEL_ASSET_PATH = "dummy_model.tflite"; + static final String SERIALIZED_MODEL_DIR = "dummy_serialized_model_dir"; + static final String MODEL_TOKEN = "dummy_model_token"; + static final String CACHED_KERNEL_PATH = "dummy_cached_kernel_path"; + + @RunWith(AndroidJUnit4.class) + public static final class General extends BaseOptionsTest { + @Test + public void succeedsWithDefaultOptions() throws Exception { + BaseOptions options = BaseOptions.builder().setModelAssetPath(MODEL_ASSET_PATH).build(); + assertThat(options.modelAssetPath().isPresent()).isTrue(); + assertThat(options.modelAssetPath().get()).isEqualTo(MODEL_ASSET_PATH); + assertThat(options.delegate()).isEqualTo(Delegate.CPU); + } + + @Test + public void succeedsWithGpuOptions() throws Exception { + BaseOptions options = + BaseOptions.builder() + .setModelAssetPath(MODEL_ASSET_PATH) + .setDelegate(Delegate.GPU) + .setDelegateOptions( + BaseOptions.DelegateOptions.GpuOptions.builder() + .setSerializedModelDir(SERIALIZED_MODEL_DIR) + .setModelToken(MODEL_TOKEN) + .setCachedKernelPath(CACHED_KERNEL_PATH) + .build()) + .build(); + assertThat( + ((BaseOptions.DelegateOptions.GpuOptions) options.delegateOptions().get()) + .serializedModelDir() + .get()) + .isEqualTo(SERIALIZED_MODEL_DIR); + assertThat( + ((BaseOptions.DelegateOptions.GpuOptions) options.delegateOptions().get()) + .modelToken() + .get()) + .isEqualTo(MODEL_TOKEN); + assertThat( + ((BaseOptions.DelegateOptions.GpuOptions) options.delegateOptions().get()) + .cachedKernelPath() + .get()) + .isEqualTo(CACHED_KERNEL_PATH); + } + + @Test + public void failsWithInvalidDelegateOptions() throws Exception { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + BaseOptions.builder() + .setModelAssetPath(MODEL_ASSET_PATH) + .setDelegate(Delegate.CPU) + .setDelegateOptions( + BaseOptions.DelegateOptions.GpuOptions.builder() + .setSerializedModelDir(SERIALIZED_MODEL_DIR) + .setModelToken(MODEL_TOKEN) + .build()) + .build()); + assertThat(exception) + .hasMessageThat() + .contains("Specified Delegate type does not match the provided delegate options."); + } + } + + /** A mock TaskOptions class providing access to convertBaseOptionsToProto. */ + public static class MockTaskOptions extends TaskOptions { + + public MockTaskOptions(BaseOptions baseOptions) { + baseOptionsProto = convertBaseOptionsToProto(baseOptions); + } + + public BaseOptionsProto.BaseOptions getBaseOptionsProto() { + return baseOptionsProto; + } + + private BaseOptionsProto.BaseOptions baseOptionsProto; + + @Override + public CalculatorOptions convertToCalculatorOptionsProto() { + return CalculatorOptions.newBuilder().build(); + } + } + + /** Test for converting {@link BaseOptions} to {@link BaseOptionsProto} */ + @RunWith(AndroidJUnit4.class) + public static final class ConvertProtoTest extends BaseOptionsTest { + @Test + public void succeedsWithDefaultOptions() throws Exception { + BaseOptions options = + BaseOptions.builder() + .setModelAssetPath(MODEL_ASSET_PATH) + .setDelegate(Delegate.CPU) + .setDelegateOptions(BaseOptions.DelegateOptions.CpuOptions.builder().build()) + .build(); + MockTaskOptions taskOptions = new MockTaskOptions(options); + AccelerationProto.Acceleration acceleration = + taskOptions.getBaseOptionsProto().getAcceleration(); + assertThat(acceleration.hasTflite()).isTrue(); + } + + @Test + public void succeedsWithGpuOptions() throws Exception { + BaseOptions options = + BaseOptions.builder() + .setModelAssetPath(MODEL_ASSET_PATH) + .setDelegate(Delegate.GPU) + .setDelegateOptions( + BaseOptions.DelegateOptions.GpuOptions.builder() + .setModelToken(MODEL_TOKEN) + .setSerializedModelDir(SERIALIZED_MODEL_DIR) + .build()) + .build(); + MockTaskOptions taskOptions = new MockTaskOptions(options); + AccelerationProto.Acceleration acceleration = + taskOptions.getBaseOptionsProto().getAcceleration(); + assertThat(acceleration.hasTflite()).isFalse(); + assertThat(acceleration.hasGpu()).isTrue(); + assertThat(acceleration.getGpu().getUseAdvancedGpuApi()).isTrue(); + assertThat(acceleration.getGpu().hasCachedKernelPath()).isFalse(); + assertThat(acceleration.getGpu().getModelToken()).isEqualTo(MODEL_TOKEN); + assertThat(acceleration.getGpu().getSerializedModelDir()).isEqualTo(SERIALIZED_MODEL_DIR); + } + } +} diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/text/textembedder/TextEmbedderTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/text/textembedder/TextEmbedderTest.java index ed7573b2..20084ee7 100644 --- a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/text/textembedder/TextEmbedderTest.java +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/text/textembedder/TextEmbedderTest.java @@ -140,7 +140,7 @@ public class TextEmbedderTest { TextEmbedder.cosineSimilarity( result0.embeddingResult().embeddings().get(0), result1.embeddingResult().embeddings().get(0)); - assertThat(similarity).isWithin(DOUBLE_DIFF_TOLERANCE).of(0.3477488707202946); + assertThat(similarity).isWithin(DOUBLE_DIFF_TOLERANCE).of(0.3565317439544432); } @Test diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenterTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenterTest.java index 959f444c..49ab0be1 100644 --- a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenterTest.java +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenterTest.java @@ -31,6 +31,7 @@ import com.google.mediapipe.framework.image.MPImage; import com.google.mediapipe.tasks.core.BaseOptions; import com.google.mediapipe.tasks.vision.core.RunningMode; import com.google.mediapipe.tasks.vision.imagesegmenter.ImageSegmenter.ImageSegmenterOptions; +import com.google.mediapipe.tasks.vision.imagesegmenter.ImageSegmenter.SegmentationOptions; import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.FloatBuffer; diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerTest.java index 7adef9e2..508709ab 100644 --- a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerTest.java +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerTest.java @@ -15,6 +15,7 @@ package com.google.mediapipe.tasks.vision.poselandmarker; import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; import static org.junit.Assert.assertThrows; import android.content.res.AssetManager; @@ -26,6 +27,7 @@ import com.google.common.truth.Correspondence; import com.google.mediapipe.framework.MediaPipeException; import com.google.mediapipe.framework.image.BitmapImageBuilder; import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.components.containers.Landmark; import com.google.mediapipe.tasks.components.containers.NormalizedLandmark; import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult; import com.google.mediapipe.tasks.core.BaseOptions; @@ -34,6 +36,7 @@ import com.google.mediapipe.tasks.vision.core.RunningMode; import com.google.mediapipe.tasks.vision.poselandmarker.PoseLandmarker.PoseLandmarkerOptions; import java.io.InputStream; import java.util.Arrays; +import java.util.List; import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; @@ -50,6 +53,8 @@ public class PoseLandmarkerTest { private static final String NO_POSES_IMAGE = "burger.jpg"; private static final String TAG = "Pose Landmarker Test"; private static final float LANDMARKS_ERROR_TOLERANCE = 0.03f; + private static final float VISIBILITY_TOLERANCE = 0.9f; + private static final float PRESENCE_TOLERANCE = 0.9f; private static final int IMAGE_WIDTH = 1000; private static final int IMAGE_HEIGHT = 667; @@ -70,6 +75,8 @@ public class PoseLandmarkerTest { PoseLandmarkerResult actualResult = poseLandmarker.detect(getImageFromAsset(POSE_IMAGE)); PoseLandmarkerResult expectedResult = getExpectedPoseLandmarkerResult(POSE_LANDMARKS); assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult); + assertAllLandmarksAreVisibleAndPresent( + actualResult, VISIBILITY_TOLERANCE, PRESENCE_TOLERANCE); } @Test @@ -361,4 +368,40 @@ public class PoseLandmarkerTest { assertThat(inputImage.getWidth()).isEqualTo(IMAGE_WIDTH); assertThat(inputImage.getHeight()).isEqualTo(IMAGE_HEIGHT); } + + private static void assertAllLandmarksAreVisibleAndPresent( + PoseLandmarkerResult result, float visbilityThreshold, float presenceThreshold) { + for (int i = 0; i < result.landmarks().size(); i++) { + List landmarks = result.landmarks().get(i); + for (int j = 0; j < landmarks.size(); j++) { + NormalizedLandmark landmark = landmarks.get(j); + String landmarkMessage = "Landmark List " + i + " landmark " + j + ": " + landmark; + landmark + .visibility() + .ifPresent( + val -> + assertWithMessage(landmarkMessage).that(val).isAtLeast((visbilityThreshold))); + landmark + .presence() + .ifPresent( + val -> assertWithMessage(landmarkMessage).that(val).isAtLeast((presenceThreshold))); + } + } + for (int i = 0; i < result.worldLandmarks().size(); i++) { + List landmarks = result.worldLandmarks().get(i); + for (int j = 0; j < landmarks.size(); j++) { + Landmark landmark = landmarks.get(j); + String landmarkMessage = "World Landmark List " + i + " landmark " + j + ": " + landmark; + landmark + .visibility() + .ifPresent( + val -> + assertWithMessage(landmarkMessage).that(val).isAtLeast((visbilityThreshold))); + landmark + .presence() + .ifPresent( + val -> assertWithMessage(landmarkMessage).that(val).isAtLeast((presenceThreshold))); + } + } + } } diff --git a/mediapipe/tasks/python/core/BUILD b/mediapipe/tasks/python/core/BUILD index 76791c23..9d2dc3f0 100644 --- a/mediapipe/tasks/python/core/BUILD +++ b/mediapipe/tasks/python/core/BUILD @@ -29,7 +29,7 @@ py_library( name = "base_options", srcs = ["base_options.py"], visibility = [ - "//mediapipe/model_maker/python/vision/gesture_recognizer:__subpackages__", + "//mediapipe/model_maker:__subpackages__", "//mediapipe/tasks:users", ], deps = [ diff --git a/mediapipe/tasks/python/metadata/BUILD b/mediapipe/tasks/python/metadata/BUILD index 07805ec6..720cbad4 100644 --- a/mediapipe/tasks/python/metadata/BUILD +++ b/mediapipe/tasks/python/metadata/BUILD @@ -1,3 +1,5 @@ +# Placeholder: load py_library +# Placeholder: load py_binary load("//mediapipe/tasks/metadata:build_defs.bzl", "stamp_metadata_parser_version") package( diff --git a/mediapipe/tasks/python/metadata/flatbuffers_lib/flatbuffers_lib.cc b/mediapipe/tasks/python/metadata/flatbuffers_lib/flatbuffers_lib.cc index 0c251c69..cf6ddd9b 100644 --- a/mediapipe/tasks/python/metadata/flatbuffers_lib/flatbuffers_lib.cc +++ b/mediapipe/tasks/python/metadata/flatbuffers_lib/flatbuffers_lib.cc @@ -41,12 +41,12 @@ PYBIND11_MODULE(_pywrap_flatbuffers, m) { self->PushFlatBuffer(reinterpret_cast(contents.c_str()), contents.length()); }); - m.def("generate_text_file", &flatbuffers::GenerateTextFile); + m.def("generate_text_file", &flatbuffers::GenTextFile); m.def("generate_text", [](const flatbuffers::Parser& parser, const std::string& buffer) -> std::string { std::string text; - const char* result = flatbuffers::GenerateText( + const char* result = flatbuffers::GenText( parser, reinterpret_cast(buffer.c_str()), &text); if (result) { return ""; diff --git a/mediapipe/tasks/python/metadata/metadata.py b/mediapipe/tasks/python/metadata/metadata.py index e888a9d1..c7375232 100644 --- a/mediapipe/tasks/python/metadata/metadata.py +++ b/mediapipe/tasks/python/metadata/metadata.py @@ -737,7 +737,7 @@ class MetadataDisplayer(object): metadata_buffer = get_metadata_buffer(model_buffer) if not metadata_buffer: raise ValueError("The model does not have metadata.") - associated_file_list = cls._parse_packed_associted_file_list(model_buffer) + associated_file_list = cls._parse_packed_associated_file_list(model_buffer) return cls(model_buffer, metadata_buffer, associated_file_list) def get_associated_file_buffer(self, filename): @@ -775,8 +775,8 @@ class MetadataDisplayer(object): """ return copy.deepcopy(self._associated_file_list) - @staticmethod - def _parse_packed_associted_file_list(model_buf): + @classmethod + def _parse_packed_associated_file_list(cls, model_buf): """Gets a list of associated files packed to the model file. Args: diff --git a/mediapipe/tasks/python/metadata/metadata_writers/BUILD b/mediapipe/tasks/python/metadata/metadata_writers/BUILD index e86254f2..6528f5ce 100644 --- a/mediapipe/tasks/python/metadata/metadata_writers/BUILD +++ b/mediapipe/tasks/python/metadata/metadata_writers/BUILD @@ -1,3 +1,4 @@ +# Placeholder: load py_library # Placeholder for internal Python strict library and test compatibility macro. package( diff --git a/mediapipe/tasks/python/test/metadata/BUILD b/mediapipe/tasks/python/test/metadata/BUILD index 2cdc7e63..ba72daf9 100644 --- a/mediapipe/tasks/python/test/metadata/BUILD +++ b/mediapipe/tasks/python/test/metadata/BUILD @@ -1,3 +1,5 @@ +# Placeholder: load py_test + package( default_visibility = [ "//visibility:public", diff --git a/mediapipe/tasks/python/test/test_utils.py b/mediapipe/tasks/python/test/test_utils.py index 2dfc5a8c..e790b915 100644 --- a/mediapipe/tasks/python/test/test_utils.py +++ b/mediapipe/tasks/python/test/test_utils.py @@ -22,7 +22,6 @@ import six from google.protobuf import descriptor from google.protobuf import descriptor_pool from google.protobuf import text_format - from mediapipe.python._framework_bindings import image as image_module from mediapipe.python._framework_bindings import image_frame as image_frame_module @@ -44,18 +43,21 @@ def test_srcdir(): def get_test_data_path(file_or_dirname_path: str) -> str: """Returns full test data path.""" - for (directory, subdirs, files) in os.walk(test_srcdir()): + for directory, subdirs, files in os.walk(test_srcdir()): for f in subdirs + files: path = os.path.join(directory, f) if path.endswith(file_or_dirname_path): return path - raise ValueError("No %s in test directory: %s." % - (file_or_dirname_path, test_srcdir())) + raise ValueError( + "No %s in test directory: %s." % (file_or_dirname_path, test_srcdir()) + ) -def create_calibration_file(file_dir: str, - file_name: str = "score_calibration.txt", - content: str = "1.0,2.0,3.0,4.0") -> str: +def create_calibration_file( + file_dir: str, + file_name: str = "score_calibration.txt", + content: str = "1.0,2.0,3.0,4.0", +) -> str: """Creates the calibration file.""" calibration_file = os.path.join(file_dir, file_name) with open(calibration_file, mode="w") as file: @@ -63,12 +65,9 @@ def create_calibration_file(file_dir: str, return calibration_file -def assert_proto_equals(self, - a, - b, - check_initialized=True, - normalize_numbers=True, - msg=None): +def assert_proto_equals( + self, a, b, check_initialized=True, normalize_numbers=True, msg=None +): """assert_proto_equals() is useful for unit tests. It produces much more helpful output than assertEqual() for proto2 messages. @@ -113,7 +112,8 @@ def assert_proto_equals(self, self.assertMultiLineEqual(a_str, b_str, msg=msg) else: diff = "".join( - difflib.unified_diff(a_str.splitlines(True), b_str.splitlines(True))) + difflib.unified_diff(a_str.splitlines(True), b_str.splitlines(True)) + ) if diff: self.fail("%s :\n%s" % (msg, diff)) @@ -147,14 +147,18 @@ def _normalize_number_fields(pb): # We force 32-bit values to int and 64-bit values to long to make # alternate implementations where the distinction is more significant # (e.g. the C++ implementation) simpler. - if desc.type in (descriptor.FieldDescriptor.TYPE_INT64, - descriptor.FieldDescriptor.TYPE_UINT64, - descriptor.FieldDescriptor.TYPE_SINT64): + if desc.type in ( + descriptor.FieldDescriptor.TYPE_INT64, + descriptor.FieldDescriptor.TYPE_UINT64, + descriptor.FieldDescriptor.TYPE_SINT64, + ): normalized_values = [int(x) for x in values] - elif desc.type in (descriptor.FieldDescriptor.TYPE_INT32, - descriptor.FieldDescriptor.TYPE_UINT32, - descriptor.FieldDescriptor.TYPE_SINT32, - descriptor.FieldDescriptor.TYPE_ENUM): + elif desc.type in ( + descriptor.FieldDescriptor.TYPE_INT32, + descriptor.FieldDescriptor.TYPE_UINT32, + descriptor.FieldDescriptor.TYPE_SINT32, + descriptor.FieldDescriptor.TYPE_ENUM, + ): normalized_values = [int(x) for x in values] elif desc.type == descriptor.FieldDescriptor.TYPE_FLOAT: normalized_values = [round(x, 4) for x in values] @@ -168,14 +172,20 @@ def _normalize_number_fields(pb): else: setattr(pb, desc.name, normalized_values[0]) - if (desc.type == descriptor.FieldDescriptor.TYPE_MESSAGE or - desc.type == descriptor.FieldDescriptor.TYPE_GROUP): - if (desc.type == descriptor.FieldDescriptor.TYPE_MESSAGE and - desc.message_type.has_options and - desc.message_type.GetOptions().map_entry): + if ( + desc.type == descriptor.FieldDescriptor.TYPE_MESSAGE + or desc.type == descriptor.FieldDescriptor.TYPE_GROUP + ): + if ( + desc.type == descriptor.FieldDescriptor.TYPE_MESSAGE + and desc.message_type.has_options + and desc.message_type.GetOptions().map_entry + ): # This is a map, only recurse if the values have a message type. - if (desc.message_type.fields_by_number[2].type == - descriptor.FieldDescriptor.TYPE_MESSAGE): + if ( + desc.message_type.fields_by_number[2].type + == descriptor.FieldDescriptor.TYPE_MESSAGE + ): for v in six.itervalues(values): _normalize_number_fields(v) else: diff --git a/mediapipe/tasks/python/test/text/text_embedder_test.py b/mediapipe/tasks/python/test/text/text_embedder_test.py index 27726b70..9688ee91 100644 --- a/mediapipe/tasks/python/test/text/text_embedder_test.py +++ b/mediapipe/tasks/python/test/text/text_embedder_test.py @@ -37,7 +37,7 @@ _TEST_DATA_DIR = 'mediapipe/tasks/testdata/text' # Tolerance for embedding vector coordinate values. _EPSILON = 1e-4 # Tolerance for cosine similarity evaluation. -_SIMILARITY_TOLERANCE = 1e-6 +_SIMILARITY_TOLERANCE = 1e-3 class ModelFileType(enum.Enum): @@ -287,7 +287,7 @@ class TextEmbedderTest(parameterized.TestCase): @parameterized.parameters( # TODO: The similarity should likely be lower - (_BERT_MODEL_FILE, 0.980880), + (_BERT_MODEL_FILE, 0.98077), (_USE_MODEL_FILE, 0.780334), ) def test_embed_with_different_themes(self, model_file, expected_similarity): diff --git a/mediapipe/tasks/python/vision/BUILD b/mediapipe/tasks/python/vision/BUILD index 958cf0e0..0c1d4229 100644 --- a/mediapipe/tasks/python/vision/BUILD +++ b/mediapipe/tasks/python/vision/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Placeholder: load py_library # Placeholder for internal Python strict library and test compatibility macro. package(default_visibility = ["//visibility:public"]) diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index 4fde58e0..19c9e613 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -57,6 +57,7 @@ mediapipe_files(srcs = [ "hand_landmarker.task", "left_hands.jpg", "left_hands_rotated.jpg", + "leopard_bg_removal_result_512x512.png", "mobilenet_v1_0.25_192_quantized_1_default_1.tflite", "mobilenet_v1_0.25_224_1_default_1.tflite", "mobilenet_v1_0.25_224_1_metadata_1.tflite", @@ -65,6 +66,7 @@ mediapipe_files(srcs = [ "mobilenet_v1_0.25_224_quant_without_subgraph_metadata.tflite", "mobilenet_v2_1.0_224.tflite", "mobilenet_v3_small_100_224_embedder.tflite", + "mobilenetsweep_dptrigmqn384_unit_384_384_fp16quant_fp32input_opt.tflite", "mozart_square.jpg", "multi_objects.jpg", "multi_objects_rotated.jpg", @@ -105,7 +107,6 @@ exports_files( "expected_left_down_hand_landmarks.prototxt", "expected_left_down_hand_rotated_landmarks.prototxt", "expected_left_up_hand_landmarks.prototxt", - "expected_left_up_hand_rotated_landmarks.prototxt", "expected_right_down_hand_landmarks.prototxt", "expected_right_up_hand_landmarks.prototxt", "face_geometry_expected_out.pbtxt", @@ -136,6 +137,7 @@ filegroup( "hand_landmark_lite.tflite", "left_hands.jpg", "left_hands_rotated.jpg", + "leopard_bg_removal_result_512x512.png", "mozart_square.jpg", "multi_objects.jpg", "multi_objects_rotated.jpg", @@ -211,12 +213,12 @@ filegroup( name = "test_protos", srcs = [ "expected_left_down_hand_landmarks.prototxt", - "expected_left_down_hand_rotated_landmarks.prototxt", "expected_left_up_hand_landmarks.prototxt", - "expected_left_up_hand_rotated_landmarks.prototxt", "expected_pose_landmarks.prototxt", "expected_right_down_hand_landmarks.prototxt", + "expected_right_down_hand_rotated_landmarks.prototxt", "expected_right_up_hand_landmarks.prototxt", + "expected_right_up_hand_rotated_landmarks.prototxt", "face_geometry_expected_out.pbtxt", "fist_landmarks.pbtxt", "hand_detector_result_one_hand.pbtxt", diff --git a/mediapipe/tasks/testdata/vision/expected_left_down_hand_landmarks.prototxt b/mediapipe/tasks/testdata/vision/expected_left_down_hand_landmarks.prototxt index 9dada76e..b0848a31 100644 --- a/mediapipe/tasks/testdata/vision/expected_left_down_hand_landmarks.prototxt +++ b/mediapipe/tasks/testdata/vision/expected_left_down_hand_landmarks.prototxt @@ -1,84 +1,84 @@ landmark { - x: 0.19942205 - y: 0.09026158 + x: 0.8055556 + y: 0.08900524 } landmark { - x: 0.29673815 - y: 0.1236096 + x: 0.7 + y: 0.13089006 } landmark { - x: 0.35452557 - y: 0.24131873 + x: 0.6375 + y: 0.2460733 } landmark { - x: 0.39504135 - y: 0.3613678 + x: 0.59583336 + y: 0.38219896 } landmark { - x: 0.4381017 - y: 0.44257507 + x: 0.55138886 + y: 0.4764398 } landmark { - x: 0.30564976 - y: 0.43276948 + x: 0.70416665 + y: 0.43717277 } landmark { - x: 0.33376893 - y: 0.6287609 + x: 0.6652778 + y: 0.64136124 } landmark { - x: 0.34690586 - y: 0.7581718 + x: 0.6513889 + y: 0.7643979 } landmark { - x: 0.3569131 - y: 0.85597074 + x: 0.64444447 + y: 0.8638743 } landmark { - x: 0.24617499 - y: 0.4616468 + x: 0.7569444 + y: 0.4712042 } landmark { - x: 0.25602233 - y: 0.6825256 + x: 0.7416667 + y: 0.6937173 } landmark { - x: 0.25772986 - y: 0.8347353 + x: 0.74027777 + y: 0.83507854 } landmark { - x: 0.25762093 - y: 0.949471 + x: 0.74444443 + y: 0.9424084 } landmark { - x: 0.18984047 - y: 0.45083284 + x: 0.80694443 + y: 0.45026177 } landmark { - x: 0.18280011 - y: 0.65619284 + x: 0.81527776 + y: 0.65968585 } landmark { - x: 0.17377229 - y: 0.7914928 + x: 0.82361114 + y: 0.79581153 } landmark { - x: 0.16702436 - y: 0.89128083 + x: 0.83194447 + y: 0.90575916 } landmark { - x: 0.14224908 - y: 0.41272494 + x: 0.8541667 + y: 0.43979058 } landmark { - x: 0.119362295 - y: 0.5680165 + x: 0.87222224 + y: 0.5837696 } landmark { - x: 0.102372244 - y: 0.67237973 + x: 0.88611114 + y: 0.6753927 } landmark { - x: 0.08747025 - y: 0.7554076 + x: 0.9 + y: 0.7539267 } diff --git a/mediapipe/tasks/testdata/vision/expected_left_up_hand_landmarks.prototxt b/mediapipe/tasks/testdata/vision/expected_left_up_hand_landmarks.prototxt index 1d5aec4b..74fb3999 100644 --- a/mediapipe/tasks/testdata/vision/expected_left_up_hand_landmarks.prototxt +++ b/mediapipe/tasks/testdata/vision/expected_left_up_hand_landmarks.prototxt @@ -1,84 +1,84 @@ landmark { - x: 0.7977909 - y: 0.90771425 + x: 0.19166666 + y: 0.89790577 } landmark { - x: 0.7005595 - y: 0.87075 + x: 0.29305556 + y: 0.8638743 } landmark { - x: 0.6439954 - y: 0.7551088 + x: 0.35694444 + y: 0.7486911 } landmark { - x: 0.60334325 - y: 0.6363517 + x: 0.40138888 + y: 0.62041885 } landmark { - x: 0.5600122 - y: 0.55537516 + x: 0.44722223 + y: 0.5314136 } landmark { - x: 0.6928512 - y: 0.56547815 + x: 0.30416667 + y: 0.565445 } landmark { - x: 0.66476023 - y: 0.3680001 + x: 0.33055556 + y: 0.36125654 } landmark { - x: 0.6514839 - y: 0.23800957 + x: 0.34583333 + y: 0.2356021 } landmark { - x: 0.6416936 - y: 0.13911664 + x: 0.3513889 + y: 0.13350785 } landmark { - x: 0.75269383 - y: 0.53802305 + x: 0.24583334 + y: 0.5340314 } landmark { - x: 0.7422081 - y: 0.31609806 + x: 0.25555557 + y: 0.30104712 } landmark { - x: 0.74030703 - y: 0.16485286 + x: 0.25972223 + y: 0.15706806 } landmark { - x: 0.7408123 - y: 0.050073862 + x: 0.25694445 + y: 0.04973822 } landmark { - x: 0.80908364 - y: 0.548252 + x: 0.19166666 + y: 0.5445026 } landmark { - x: 0.8152498 - y: 0.34377483 + x: 0.18194444 + y: 0.33246073 } landmark { - x: 0.82466483 - y: 0.20964715 + x: 0.17222223 + y: 0.20157067 } landmark { - x: 0.832543 - y: 0.10994735 + x: 0.1625 + y: 0.09424084 } landmark { - x: 0.85659754 - y: 0.5847515 + x: 0.14722222 + y: 0.58115184 } landmark { - x: 0.8787856 - y: 0.42845485 + x: 0.12777779 + y: 0.41623038 } landmark { - x: 0.89572114 - y: 0.32542163 + x: 0.10972222 + y: 0.32460734 } landmark { - x: 0.9110377 - y: 0.24356759 + x: 0.094444446 + y: 0.2434555 } diff --git a/mediapipe/tasks/testdata/vision/expected_right_down_hand_landmarks.prototxt b/mediapipe/tasks/testdata/vision/expected_right_down_hand_landmarks.prototxt index b0848a31..9dada76e 100644 --- a/mediapipe/tasks/testdata/vision/expected_right_down_hand_landmarks.prototxt +++ b/mediapipe/tasks/testdata/vision/expected_right_down_hand_landmarks.prototxt @@ -1,84 +1,84 @@ landmark { - x: 0.8055556 - y: 0.08900524 + x: 0.19942205 + y: 0.09026158 } landmark { - x: 0.7 - y: 0.13089006 + x: 0.29673815 + y: 0.1236096 } landmark { - x: 0.6375 - y: 0.2460733 + x: 0.35452557 + y: 0.24131873 } landmark { - x: 0.59583336 - y: 0.38219896 + x: 0.39504135 + y: 0.3613678 } landmark { - x: 0.55138886 - y: 0.4764398 + x: 0.4381017 + y: 0.44257507 } landmark { - x: 0.70416665 - y: 0.43717277 + x: 0.30564976 + y: 0.43276948 } landmark { - x: 0.6652778 - y: 0.64136124 + x: 0.33376893 + y: 0.6287609 } landmark { - x: 0.6513889 - y: 0.7643979 + x: 0.34690586 + y: 0.7581718 } landmark { - x: 0.64444447 - y: 0.8638743 + x: 0.3569131 + y: 0.85597074 } landmark { - x: 0.7569444 - y: 0.4712042 + x: 0.24617499 + y: 0.4616468 } landmark { - x: 0.7416667 - y: 0.6937173 + x: 0.25602233 + y: 0.6825256 } landmark { - x: 0.74027777 - y: 0.83507854 + x: 0.25772986 + y: 0.8347353 } landmark { - x: 0.74444443 - y: 0.9424084 + x: 0.25762093 + y: 0.949471 } landmark { - x: 0.80694443 - y: 0.45026177 + x: 0.18984047 + y: 0.45083284 } landmark { - x: 0.81527776 - y: 0.65968585 + x: 0.18280011 + y: 0.65619284 } landmark { - x: 0.82361114 - y: 0.79581153 + x: 0.17377229 + y: 0.7914928 } landmark { - x: 0.83194447 - y: 0.90575916 + x: 0.16702436 + y: 0.89128083 } landmark { - x: 0.8541667 - y: 0.43979058 + x: 0.14224908 + y: 0.41272494 } landmark { - x: 0.87222224 - y: 0.5837696 + x: 0.119362295 + y: 0.5680165 } landmark { - x: 0.88611114 - y: 0.6753927 + x: 0.102372244 + y: 0.67237973 } landmark { - x: 0.9 - y: 0.7539267 + x: 0.08747025 + y: 0.7554076 } diff --git a/mediapipe/tasks/testdata/vision/expected_left_down_hand_rotated_landmarks.prototxt b/mediapipe/tasks/testdata/vision/expected_right_down_hand_rotated_landmarks.prototxt similarity index 100% rename from mediapipe/tasks/testdata/vision/expected_left_down_hand_rotated_landmarks.prototxt rename to mediapipe/tasks/testdata/vision/expected_right_down_hand_rotated_landmarks.prototxt diff --git a/mediapipe/tasks/testdata/vision/expected_right_up_hand_landmarks.prototxt b/mediapipe/tasks/testdata/vision/expected_right_up_hand_landmarks.prototxt index 74fb3999..1d5aec4b 100644 --- a/mediapipe/tasks/testdata/vision/expected_right_up_hand_landmarks.prototxt +++ b/mediapipe/tasks/testdata/vision/expected_right_up_hand_landmarks.prototxt @@ -1,84 +1,84 @@ landmark { - x: 0.19166666 - y: 0.89790577 + x: 0.7977909 + y: 0.90771425 } landmark { - x: 0.29305556 - y: 0.8638743 + x: 0.7005595 + y: 0.87075 } landmark { - x: 0.35694444 - y: 0.7486911 + x: 0.6439954 + y: 0.7551088 } landmark { - x: 0.40138888 - y: 0.62041885 + x: 0.60334325 + y: 0.6363517 } landmark { - x: 0.44722223 - y: 0.5314136 + x: 0.5600122 + y: 0.55537516 } landmark { - x: 0.30416667 - y: 0.565445 + x: 0.6928512 + y: 0.56547815 } landmark { - x: 0.33055556 - y: 0.36125654 + x: 0.66476023 + y: 0.3680001 } landmark { - x: 0.34583333 - y: 0.2356021 + x: 0.6514839 + y: 0.23800957 } landmark { - x: 0.3513889 - y: 0.13350785 + x: 0.6416936 + y: 0.13911664 } landmark { - x: 0.24583334 - y: 0.5340314 + x: 0.75269383 + y: 0.53802305 } landmark { - x: 0.25555557 - y: 0.30104712 + x: 0.7422081 + y: 0.31609806 } landmark { - x: 0.25972223 - y: 0.15706806 + x: 0.74030703 + y: 0.16485286 } landmark { - x: 0.25694445 - y: 0.04973822 + x: 0.7408123 + y: 0.050073862 } landmark { - x: 0.19166666 - y: 0.5445026 + x: 0.80908364 + y: 0.548252 } landmark { - x: 0.18194444 - y: 0.33246073 + x: 0.8152498 + y: 0.34377483 } landmark { - x: 0.17222223 - y: 0.20157067 + x: 0.82466483 + y: 0.20964715 } landmark { - x: 0.1625 - y: 0.09424084 + x: 0.832543 + y: 0.10994735 } landmark { - x: 0.14722222 - y: 0.58115184 + x: 0.85659754 + y: 0.5847515 } landmark { - x: 0.12777779 - y: 0.41623038 + x: 0.8787856 + y: 0.42845485 } landmark { - x: 0.10972222 - y: 0.32460734 + x: 0.89572114 + y: 0.32542163 } landmark { - x: 0.094444446 - y: 0.2434555 + x: 0.9110377 + y: 0.24356759 } diff --git a/mediapipe/tasks/testdata/vision/expected_left_up_hand_rotated_landmarks.prototxt b/mediapipe/tasks/testdata/vision/expected_right_up_hand_rotated_landmarks.prototxt similarity index 100% rename from mediapipe/tasks/testdata/vision/expected_left_up_hand_rotated_landmarks.prototxt rename to mediapipe/tasks/testdata/vision/expected_right_up_hand_rotated_landmarks.prototxt diff --git a/mediapipe/tasks/testdata/vision/fist_landmarks.pbtxt b/mediapipe/tasks/testdata/vision/fist_landmarks.pbtxt index a24358c3..b9b7ca40 100644 --- a/mediapipe/tasks/testdata/vision/fist_landmarks.pbtxt +++ b/mediapipe/tasks/testdata/vision/fist_landmarks.pbtxt @@ -1,8 +1,8 @@ classifications { classification { score: 1.0 - label: "Left" - display_name: "Left" + label: "Right" + display_name: "Right" } } diff --git a/mediapipe/tasks/testdata/vision/pointing_up_landmarks.pbtxt b/mediapipe/tasks/testdata/vision/pointing_up_landmarks.pbtxt index 05917af3..7ab095af 100644 --- a/mediapipe/tasks/testdata/vision/pointing_up_landmarks.pbtxt +++ b/mediapipe/tasks/testdata/vision/pointing_up_landmarks.pbtxt @@ -1,8 +1,8 @@ classifications { classification { score: 1.0 - label: "Left" - display_name: "Left" + label: "Right" + display_name: "Right" } } diff --git a/mediapipe/tasks/testdata/vision/pointing_up_rotated_landmarks.pbtxt b/mediapipe/tasks/testdata/vision/pointing_up_rotated_landmarks.pbtxt index 65bb11bc..ae905521 100644 --- a/mediapipe/tasks/testdata/vision/pointing_up_rotated_landmarks.pbtxt +++ b/mediapipe/tasks/testdata/vision/pointing_up_rotated_landmarks.pbtxt @@ -1,8 +1,8 @@ classifications { classification { score: 1.0 - label: "Left" - display_name: "Left" + label: "Right" + display_name: "Right" } } diff --git a/mediapipe/tasks/testdata/vision/thumb_up_landmarks.pbtxt b/mediapipe/tasks/testdata/vision/thumb_up_landmarks.pbtxt index e73a69d3..3407d4a6 100644 --- a/mediapipe/tasks/testdata/vision/thumb_up_landmarks.pbtxt +++ b/mediapipe/tasks/testdata/vision/thumb_up_landmarks.pbtxt @@ -1,8 +1,8 @@ classifications { classification { score: 1.0 - label: "Left" - display_name: "Left" + label: "Right" + display_name: "Right" } } diff --git a/mediapipe/tasks/testdata/vision/thumb_up_rotated_landmarks.pbtxt b/mediapipe/tasks/testdata/vision/thumb_up_rotated_landmarks.pbtxt index 3636e2e4..1bde54db 100644 --- a/mediapipe/tasks/testdata/vision/thumb_up_rotated_landmarks.pbtxt +++ b/mediapipe/tasks/testdata/vision/thumb_up_rotated_landmarks.pbtxt @@ -1,8 +1,8 @@ classifications { classification { score: 1.0 - label: "Left" - display_name: "Left" + label: "Right" + display_name: "Right" } } diff --git a/mediapipe/tasks/testdata/vision/victory_landmarks.pbtxt b/mediapipe/tasks/testdata/vision/victory_landmarks.pbtxt index 7a704ee3..a55b08b1 100644 --- a/mediapipe/tasks/testdata/vision/victory_landmarks.pbtxt +++ b/mediapipe/tasks/testdata/vision/victory_landmarks.pbtxt @@ -1,8 +1,8 @@ classifications { classification { score: 1.0 - label: "Left" - display_name: "Left" + label: "Right" + display_name: "Right" } } diff --git a/mediapipe/tasks/web/audio/BUILD b/mediapipe/tasks/web/audio/BUILD index 4dd5a2f6..3338d17b 100644 --- a/mediapipe/tasks/web/audio/BUILD +++ b/mediapipe/tasks/web/audio/BUILD @@ -38,7 +38,7 @@ mediapipe_files(srcs = [ ]) rollup_bundle( - name = "audio_bundle", + name = "audio_bundle_mjs", config_file = "//mediapipe/tasks/web:rollup.config.mjs", entry_point = "index.ts", format = "esm", @@ -69,6 +69,29 @@ rollup_bundle( ], ) +genrule( + name = "audio_sources", + srcs = [ + ":audio_bundle_cjs", + ":audio_bundle_mjs", + ], + outs = [ + "audio_bundle.cjs", + "audio_bundle.cjs.map", + "audio_bundle.mjs", + "audio_bundle.mjs.map", + ], + cmd = ( + "for FILE in $(SRCS); do " + + " OUT_FILE=$(GENDIR)/mediapipe/tasks/web/audio/$$(" + + " basename $$FILE | sed -E 's/_([cm])js\\.js/.\\1js/'" + + " ); " + + " echo $$FILE ; echo $$OUT_FILE ; " + + " cp $$FILE $$OUT_FILE ; " + + "done;" + ), +) + genrule( name = "package_json", srcs = ["//mediapipe/tasks/web:package.json"], @@ -91,8 +114,7 @@ pkg_npm( "wasm/audio_wasm_internal.wasm", "wasm/audio_wasm_nosimd_internal.js", "wasm/audio_wasm_nosimd_internal.wasm", - ":audio_bundle", - ":audio_bundle_cjs", + ":audio_sources", ":package_json", ], ) diff --git a/mediapipe/tasks/web/components/containers/bounding_box.d.ts b/mediapipe/tasks/web/components/containers/bounding_box.d.ts index 77f2837d..85811f44 100644 --- a/mediapipe/tasks/web/components/containers/bounding_box.d.ts +++ b/mediapipe/tasks/web/components/containers/bounding_box.d.ts @@ -24,4 +24,10 @@ export declare interface BoundingBox { width: number; /** The height of the bounding box, in pixels. */ height: number; + /** + * Angle of rotation of the original non-rotated box around the top left + * corner of the original non-rotated box, in clockwise degrees from the + * horizontal. + */ + angle: number; } diff --git a/mediapipe/tasks/web/components/processors/detection_result.test.ts b/mediapipe/tasks/web/components/processors/detection_result.test.ts index 0fa8156b..8e3e413e 100644 --- a/mediapipe/tasks/web/components/processors/detection_result.test.ts +++ b/mediapipe/tasks/web/components/processors/detection_result.test.ts @@ -58,7 +58,7 @@ describe('convertFromDetectionProto()', () => { categoryName: 'foo', displayName: 'bar', }], - boundingBox: {originX: 1, originY: 2, width: 3, height: 4}, + boundingBox: {originX: 1, originY: 2, width: 3, height: 4, angle: 0}, keypoints: [{ x: 5, y: 6, @@ -85,7 +85,7 @@ describe('convertFromDetectionProto()', () => { categoryName: '', displayName: '', }], - boundingBox: {originX: 0, originY: 0, width: 0, height: 0}, + boundingBox: {originX: 0, originY: 0, width: 0, height: 0, angle: 0}, keypoints: [] }); }); diff --git a/mediapipe/tasks/web/components/processors/detection_result.ts b/mediapipe/tasks/web/components/processors/detection_result.ts index 4999ed31..6cb5e623 100644 --- a/mediapipe/tasks/web/components/processors/detection_result.ts +++ b/mediapipe/tasks/web/components/processors/detection_result.ts @@ -42,7 +42,8 @@ export function convertFromDetectionProto(source: DetectionProto): Detection { originX: boundingBox.getXmin() ?? 0, originY: boundingBox.getYmin() ?? 0, width: boundingBox.getWidth() ?? 0, - height: boundingBox.getHeight() ?? 0 + height: boundingBox.getHeight() ?? 0, + angle: 0.0, }; } diff --git a/mediapipe/tasks/web/core/task_runner.ts b/mediapipe/tasks/web/core/task_runner.ts index 8c6aae6c..e2690cde 100644 --- a/mediapipe/tasks/web/core/task_runner.ts +++ b/mediapipe/tasks/web/core/task_runner.ts @@ -25,9 +25,6 @@ import {SupportModelResourcesGraphService} from '../../../web/graph_runner/regis import {WasmFileset} from './wasm_fileset'; -// None of the MP Tasks ship bundle assets. -const NO_ASSETS = undefined; - // Internal stream names for temporarily keeping memory alive, then freeing it. const FREE_MEMORY_STREAM = 'free_memory'; const UNUSED_STREAM_SUFFIX = '_unused_out'; @@ -54,14 +51,22 @@ export async function createTaskRunner( canvas: HTMLCanvasElement|OffscreenCanvas|null|undefined, fileset: WasmFileset, options: TaskRunnerOptions): Promise { const fileLocator: FileLocator = { - locateFile() { - // The only file loaded with this mechanism is the Wasm binary - return fileset.wasmBinaryPath.toString(); + locateFile(file): string { + const wasm = fileset.wasmBinaryPath.toString(); + if (wasm.includes(file)) { + return wasm; + } + const asset = fileset.assetBinaryPath?.toString(); + if (asset?.includes(file)) { + return asset; + } + return file; } }; const instance = await createMediaPipeLib( - type, fileset.wasmLoaderPath, NO_ASSETS, canvas, fileLocator); + type, fileset.wasmLoaderPath, fileset.assetLoaderPath, canvas, + fileLocator); await instance.setOptions(options); return instance; } @@ -96,65 +101,73 @@ export abstract class TaskRunner { abstract setOptions(options: TaskRunnerOptions): Promise; /** - * Applies the current set of options, including any base options that have - * not been processed by the task implementation. The options are applied - * synchronously unless a `modelAssetPath` is provided. This ensures that - * for most use cases options are applied directly and immediately affect + * Applies the current set of options, including optionally any base options + * that have not been processed by the task implementation. The options are + * applied synchronously unless a `modelAssetPath` is provided. This ensures + * that for most use cases options are applied directly and immediately affect * the next inference. + * + * @param options The options for the task. + * @param loadTfliteModel Whether to load the model specified in + * `options.baseOptions`. */ - protected applyOptions(options: TaskRunnerOptions): Promise { - const baseOptions: BaseOptions = options.baseOptions || {}; + protected applyOptions(options: TaskRunnerOptions, loadTfliteModel = true): + Promise { + if (loadTfliteModel) { + const baseOptions: BaseOptions = options.baseOptions || {}; - // Validate that exactly one model is configured - if (options.baseOptions?.modelAssetBuffer && - options.baseOptions?.modelAssetPath) { - throw new Error( - 'Cannot set both baseOptions.modelAssetPath and baseOptions.modelAssetBuffer'); - } else if (!(this.baseOptions.getModelAsset()?.hasFileContent() || - this.baseOptions.getModelAsset()?.hasFileName() || - options.baseOptions?.modelAssetBuffer || - options.baseOptions?.modelAssetPath)) { - throw new Error( - 'Either baseOptions.modelAssetPath or baseOptions.modelAssetBuffer must be set'); + // Validate that exactly one model is configured + if (options.baseOptions?.modelAssetBuffer && + options.baseOptions?.modelAssetPath) { + throw new Error( + 'Cannot set both baseOptions.modelAssetPath and baseOptions.modelAssetBuffer'); + } else if (!(this.baseOptions.getModelAsset()?.hasFileContent() || + this.baseOptions.getModelAsset()?.hasFileName() || + options.baseOptions?.modelAssetBuffer || + options.baseOptions?.modelAssetPath)) { + throw new Error( + 'Either baseOptions.modelAssetPath or baseOptions.modelAssetBuffer must be set'); + } + + this.setAcceleration(baseOptions); + if (baseOptions.modelAssetPath) { + // We don't use `await` here since we want to apply most settings + // synchronously. + return fetch(baseOptions.modelAssetPath.toString()) + .then(response => { + if (!response.ok) { + throw new Error(`Failed to fetch model: ${ + baseOptions.modelAssetPath} (${response.status})`); + } else { + return response.arrayBuffer(); + } + }) + .then(buffer => { + try { + // Try to delete file as we cannot overwite an existing file + // using our current API. + this.graphRunner.wasmModule.FS_unlink('/model.dat'); + } catch { + } + // TODO: Consider passing the model to the graph as an + // input side packet as this might reduce copies. + this.graphRunner.wasmModule.FS_createDataFile( + '/', 'model.dat', new Uint8Array(buffer), + /* canRead= */ true, /* canWrite= */ false, + /* canOwn= */ false); + this.setExternalFile('/model.dat'); + this.refreshGraph(); + this.onGraphRefreshed(); + }); + } else { + this.setExternalFile(baseOptions.modelAssetBuffer); + } } - this.setAcceleration(baseOptions); - if (baseOptions.modelAssetPath) { - // We don't use `await` here since we want to apply most settings - // synchronously. - return fetch(baseOptions.modelAssetPath.toString()) - .then(response => { - if (!response.ok) { - throw new Error(`Failed to fetch model: ${ - baseOptions.modelAssetPath} (${response.status})`); - } else { - return response.arrayBuffer(); - } - }) - .then(buffer => { - try { - // Try to delete file as we cannot overwite an existing file using - // our current API. - this.graphRunner.wasmModule.FS_unlink('/model.dat'); - } catch { - } - // TODO: Consider passing the model to the graph as an - // input side packet as this might reduce copies. - this.graphRunner.wasmModule.FS_createDataFile( - '/', 'model.dat', new Uint8Array(buffer), - /* canRead= */ true, /* canWrite= */ false, - /* canOwn= */ false); - this.setExternalFile('/model.dat'); - this.refreshGraph(); - this.onGraphRefreshed(); - }); - } else { - // Apply the setting synchronously. - this.setExternalFile(baseOptions.modelAssetBuffer); - this.refreshGraph(); - this.onGraphRefreshed(); - return Promise.resolve(); - } + // If there is no model to download, we can apply the setting synchronously. + this.refreshGraph(); + this.onGraphRefreshed(); + return Promise.resolve(); } /** Appliest the current options to the MediaPipe graph. */ diff --git a/mediapipe/tasks/web/core/task_runner_test.ts b/mediapipe/tasks/web/core/task_runner_test.ts index a68ba224..7419453c 100644 --- a/mediapipe/tasks/web/core/task_runner_test.ts +++ b/mediapipe/tasks/web/core/task_runner_test.ts @@ -20,11 +20,13 @@ import {InferenceCalculatorOptions} from '../../../calculators/tensor/inference_ import {BaseOptions as BaseOptionsProto} from '../../../tasks/cc/core/proto/base_options_pb'; import {TaskRunner} from '../../../tasks/web/core/task_runner'; import {createSpyWasmModule, SpyWasmModule} from '../../../tasks/web/core/task_runner_test_utils'; +import * as graphRunner from '../../../web/graph_runner/graph_runner'; import {ErrorListener} from '../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource URL builder -import {CachedGraphRunner} from './task_runner'; +import {CachedGraphRunner, createTaskRunner} from './task_runner'; import {TaskRunnerOptions} from './task_runner_options'; +import {WasmFileset} from './wasm_fileset'; type Writeable = { -readonly[P in keyof T]: T[P] @@ -120,6 +122,8 @@ describe('TaskRunner', () => { allowPrecisionLoss: true, cachedKernelPath: undefined, serializedModelDir: undefined, + cacheWritingBehavior: InferenceCalculatorOptions.Delegate.Gpu + .CacheWritingBehavior.WRITE_OR_ERROR, modelToken: undefined, usage: InferenceCalculatorOptions.Delegate.Gpu.InferenceUsage .SUSTAINED_SPEED, @@ -147,6 +151,9 @@ describe('TaskRunner', () => { let fetchSpy: jasmine.Spy; let taskRunner: TaskRunnerFake; let fetchStatus: number; + let locator: graphRunner.FileLocator|undefined; + + let oldCreate = graphRunner.createMediaPipeLib; beforeEach(() => { fetchStatus = 200; @@ -159,9 +166,70 @@ describe('TaskRunner', () => { }); global.fetch = fetchSpy; + // Monkeypatch an exported static method for testing! + oldCreate = graphRunner.createMediaPipeLib; + locator = undefined; + (graphRunner as {createMediaPipeLib: Function}).createMediaPipeLib = + jasmine.createSpy().and.callFake( + (type, wasmLoaderPath, assetLoaderPath, canvas, fileLocator) => { + locator = fileLocator; + // tslint:disable-next-line:no-any Monkeypatching for test mocks. + return Promise.resolve(taskRunner as any); + }); + taskRunner = TaskRunnerFake.createFake(); }); + afterEach(() => { + // Restore the monkeypatch. + (graphRunner as {createMediaPipeLib: Function}).createMediaPipeLib = + oldCreate; + }); + + it('constructs with useful file locators for asset.data files', () => { + const fileset: WasmFileset = { + wasmLoaderPath: `wasm.js`, + wasmBinaryPath: `a/b/c/wasm.wasm`, + assetLoaderPath: `asset.js`, + assetBinaryPath: `a/b/c/asset.data`, + }; + + const options = { + baseOptions: { + modelAssetPath: `modelAssetPath`, + } + }; + + const runner = createTaskRunner(TaskRunnerFake, null, fileset, options); + expect(runner).toBeDefined(); + expect(locator).toBeDefined(); + expect(locator?.locateFile('wasm.wasm')).toEqual('a/b/c/wasm.wasm'); + expect(locator?.locateFile('asset.data')).toEqual('a/b/c/asset.data'); + expect(locator?.locateFile('unknown')).toEqual('unknown'); + }); + + it('constructs without useful file locators with no asset.data file', () => { + const fileset: WasmFileset = { + wasmLoaderPath: `wasm.js`, + wasmBinaryPath: `a/b/c/wasm.wasm`, + assetLoaderPath: `asset.js`, + // No path to the assets binary. + }; + + const options = { + baseOptions: { + modelAssetPath: `modelAssetPath`, + } + }; + + const runner = createTaskRunner(TaskRunnerFake, null, fileset, options); + expect(runner).toBeDefined(); + expect(locator).toBeDefined(); + expect(locator?.locateFile('wasm.wasm')).toEqual('a/b/c/wasm.wasm'); + expect(locator?.locateFile('asset.data')).toEqual('asset.data'); + expect(locator?.locateFile('unknown')).toEqual('unknown'); + }); + it('handles errors during graph update', () => { taskRunner.enqueueError('Test error'); diff --git a/mediapipe/tasks/web/core/wasm_fileset.d.ts b/mediapipe/tasks/web/core/wasm_fileset.d.ts index 558aa3fa..e4cfbe1d 100644 --- a/mediapipe/tasks/web/core/wasm_fileset.d.ts +++ b/mediapipe/tasks/web/core/wasm_fileset.d.ts @@ -22,4 +22,8 @@ export declare interface WasmFileset { wasmLoaderPath: string; /** The path to the Wasm binary. */ wasmBinaryPath: string; + /** The optional path to the asset loader script. */ + assetLoaderPath?: string; + /** The optional path to the assets binary. */ + assetBinaryPath?: string; } diff --git a/mediapipe/tasks/web/package.json b/mediapipe/tasks/web/package.json index 3f495d15..3cb39947 100644 --- a/mediapipe/tasks/web/package.json +++ b/mediapipe/tasks/web/package.json @@ -2,11 +2,18 @@ "name": "@mediapipe/tasks-__NAME__", "version": "__VERSION__", "description": "__DESCRIPTION__", - "main": "__NAME___bundle_cjs.js", - "browser": "__NAME___bundle.js", - "module": "__NAME___bundle.js", + "main": "__NAME___bundle.cjs", + "browser": "__NAME___bundle.mjs", + "module": "__NAME___bundle.mjs", + "exports": { + "import": "./__NAME___bundle.mjs", + "require": "./__NAME___bundle.cjs", + "default": "./__NAME___bundle.mjs", + "types": "./__NAME___.d.ts" + }, "author": "mediapipe@google.com", "license": "Apache-2.0", + "type": "module", "types": "__TYPES__", "homepage": "http://mediapipe.dev", "keywords": [ "AR", "ML", "Augmented", "MediaPipe", "MediaPipe Tasks" ] diff --git a/mediapipe/tasks/web/text/BUILD b/mediapipe/tasks/web/text/BUILD index f68a8c9f..76d875ba 100644 --- a/mediapipe/tasks/web/text/BUILD +++ b/mediapipe/tasks/web/text/BUILD @@ -39,7 +39,7 @@ mediapipe_ts_library( ) rollup_bundle( - name = "text_bundle", + name = "text_bundle_mjs", config_file = "//mediapipe/tasks/web:rollup.config.mjs", entry_point = "index.ts", format = "esm", @@ -70,6 +70,29 @@ rollup_bundle( ], ) +genrule( + name = "text_sources", + srcs = [ + ":text_bundle_cjs", + ":text_bundle_mjs", + ], + outs = [ + "text_bundle.cjs", + "text_bundle.cjs.map", + "text_bundle.mjs", + "text_bundle.mjs.map", + ], + cmd = ( + "for FILE in $(SRCS); do " + + " OUT_FILE=$(GENDIR)/mediapipe/tasks/web/text/$$(" + + " basename $$FILE | sed -E 's/_([cm])js\\.js/.\\1js/'" + + " ); " + + " echo $$FILE ; echo $$OUT_FILE ; " + + " cp $$FILE $$OUT_FILE ; " + + "done;" + ), +) + genrule( name = "package_json", srcs = ["//mediapipe/tasks/web:package.json"], @@ -93,7 +116,6 @@ pkg_npm( "wasm/text_wasm_nosimd_internal.js", "wasm/text_wasm_nosimd_internal.wasm", ":package_json", - ":text_bundle", - ":text_bundle_cjs", + ":text_sources", ], ) diff --git a/mediapipe/tasks/web/vision/BUILD b/mediapipe/tasks/web/vision/BUILD index a7767fe5..58795b16 100644 --- a/mediapipe/tasks/web/vision/BUILD +++ b/mediapipe/tasks/web/vision/BUILD @@ -50,7 +50,7 @@ mediapipe_ts_library( ) rollup_bundle( - name = "vision_bundle", + name = "vision_bundle_mjs", config_file = "//mediapipe/tasks/web:rollup.config.mjs", entry_point = "index.ts", format = "esm", @@ -81,6 +81,29 @@ rollup_bundle( ], ) +genrule( + name = "vision_sources", + srcs = [ + ":vision_bundle_cjs", + ":vision_bundle_mjs", + ], + outs = [ + "vision_bundle.cjs", + "vision_bundle.cjs.map", + "vision_bundle.mjs", + "vision_bundle.mjs.map", + ], + cmd = ( + "for FILE in $(SRCS); do " + + " OUT_FILE=$(GENDIR)/mediapipe/tasks/web/vision/$$(" + + " basename $$FILE | sed -E 's/_([cm])js\\.js/.\\1js/'" + + " ); " + + " echo $$FILE ; echo $$OUT_FILE ; " + + " cp $$FILE $$OUT_FILE ; " + + "done;" + ), +) + genrule( name = "package_json", srcs = ["//mediapipe/tasks/web:package.json"], @@ -104,7 +127,6 @@ pkg_npm( "wasm/vision_wasm_nosimd_internal.js", "wasm/vision_wasm_nosimd_internal.wasm", ":package_json", - ":vision_bundle", - ":vision_bundle_cjs", + ":vision_sources", ], ) diff --git a/mediapipe/tasks/web/vision/core/vision_task_runner.ts b/mediapipe/tasks/web/vision/core/vision_task_runner.ts index f8f7826d..3ed15b97 100644 --- a/mediapipe/tasks/web/vision/core/vision_task_runner.ts +++ b/mediapipe/tasks/web/vision/core/vision_task_runner.ts @@ -70,7 +70,8 @@ export abstract class VisionTaskRunner extends TaskRunner { * @param imageStreamName the name of the input image stream. * @param normRectStreamName the name of the input normalized rect image * stream used to provide (mandatory) rotation and (optional) - * region-of-interest. + * region-of-interest. `null` if the graph does not support normalized + * rects. * @param roiAllowed Whether this task supports Region-Of-Interest * pre-processing * @@ -79,13 +80,20 @@ export abstract class VisionTaskRunner extends TaskRunner { constructor( protected override readonly graphRunner: VisionGraphRunner, private readonly imageStreamName: string, - private readonly normRectStreamName: string, + private readonly normRectStreamName: string|null, private readonly roiAllowed: boolean) { super(graphRunner); } - /** Configures the shared options of a vision task. */ - override applyOptions(options: VisionTaskOptions): Promise { + /** + * Configures the shared options of a vision task. + * + * @param options The options for the task. + * @param loadTfliteModel Whether to load the model specified in + * `options.baseOptions`. + */ + override applyOptions(options: VisionTaskOptions, loadTfliteModel = true): + Promise { if ('runningMode' in options) { const useStreamMode = !!options.runningMode && options.runningMode !== 'IMAGE'; @@ -98,7 +106,7 @@ export abstract class VisionTaskRunner extends TaskRunner { } } - return super.applyOptions(options); + return super.applyOptions(options, loadTfliteModel); } /** Sends a single image to the graph and awaits results. */ @@ -209,11 +217,13 @@ export abstract class VisionTaskRunner extends TaskRunner { imageSource: ImageSource, imageProcessingOptions: ImageProcessingOptions|undefined, timestamp: number): void { - const normalizedRect = - this.convertToNormalizedRect(imageSource, imageProcessingOptions); - this.graphRunner.addProtoToStream( - normalizedRect.serializeBinary(), 'mediapipe.NormalizedRect', - this.normRectStreamName, timestamp); + if (this.normRectStreamName) { + const normalizedRect = + this.convertToNormalizedRect(imageSource, imageProcessingOptions); + this.graphRunner.addProtoToStream( + normalizedRect.serializeBinary(), 'mediapipe.NormalizedRect', + this.normRectStreamName, timestamp); + } this.graphRunner.addGpuBufferAsImageToStream( imageSource, this.imageStreamName, timestamp ?? performance.now()); this.finishProcessing(); diff --git a/mediapipe/tasks/web/vision/face_detector/face_detector_test.ts b/mediapipe/tasks/web/vision/face_detector/face_detector_test.ts index dfe84bb1..049edefd 100644 --- a/mediapipe/tasks/web/vision/face_detector/face_detector_test.ts +++ b/mediapipe/tasks/web/vision/face_detector/face_detector_test.ts @@ -191,7 +191,7 @@ describe('FaceDetector', () => { categoryName: '', displayName: '', }], - boundingBox: {originX: 0, originY: 0, width: 0, height: 0}, + boundingBox: {originX: 0, originY: 0, width: 0, height: 0, angle: 0}, keypoints: [] }); }); diff --git a/mediapipe/tasks/web/vision/face_stylizer/karma.conf.ts b/mediapipe/tasks/web/vision/face_stylizer/karma.conf.ts new file mode 100644 index 00000000..0d1aa5ff --- /dev/null +++ b/mediapipe/tasks/web/vision/face_stylizer/karma.conf.ts @@ -0,0 +1,10 @@ +module.exports = config => { + config.files.push({ + pattern: 'mediapipe/tasks/**', + watched: false, + served: true, + nocache: false, + included: false, + }); + config.pingTimeout = 400000; +}; diff --git a/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts b/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts index 6d295aaa..2f35f667 100644 --- a/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts +++ b/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts @@ -308,7 +308,7 @@ export class ImageSegmenter extends VisionTaskRunner { /** * Performs image segmentation on the provided video frame and returns the * segmentation result. This method creates a copy of the resulting masks and - * should not be used in high-throughput applictions. Only use this method + * should not be used in high-throughput applications. Only use this method * when the ImageSegmenter is created with running mode `video`. * * @param videoFrame A video frame to process. diff --git a/mediapipe/tasks/web/vision/interactive_segmenter/interactive_segmenter.ts b/mediapipe/tasks/web/vision/interactive_segmenter/interactive_segmenter.ts index 662eaf09..acd7265c 100644 --- a/mediapipe/tasks/web/vision/interactive_segmenter/interactive_segmenter.ts +++ b/mediapipe/tasks/web/vision/interactive_segmenter/interactive_segmenter.ts @@ -230,7 +230,7 @@ export class InteractiveSegmenter extends VisionTaskRunner { /** * Performs interactive segmentation on the provided video frame and returns * the segmentation result. This method creates a copy of the resulting masks - * and should not be used in high-throughput applictions. The `roi` parameter + * and should not be used in high-throughput applications. The `roi` parameter * is used to represent a user's region of interest for segmentation. * * @param image An image to process. @@ -243,7 +243,7 @@ export class InteractiveSegmenter extends VisionTaskRunner { /** * Performs interactive segmentation on the provided video frame and returns * the segmentation result. This method creates a copy of the resulting masks - * and should not be used in high-throughput applictions. The `roi` parameter + * and should not be used in high-throughput applications. The `roi` parameter * is used to represent a user's region of interest for segmentation. * * The 'image_processing_options' parameter can be used to specify the diff --git a/mediapipe/tasks/web/vision/object_detector/object_detector_test.ts b/mediapipe/tasks/web/vision/object_detector/object_detector_test.ts index 9c63eaba..6437216b 100644 --- a/mediapipe/tasks/web/vision/object_detector/object_detector_test.ts +++ b/mediapipe/tasks/web/vision/object_detector/object_detector_test.ts @@ -210,7 +210,7 @@ describe('ObjectDetector', () => { categoryName: '', displayName: '', }], - boundingBox: {originX: 0, originY: 0, width: 0, height: 0}, + boundingBox: {originX: 0, originY: 0, width: 0, height: 0, angle: 0}, keypoints: [] }); }); diff --git a/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts b/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts index 927b3c24..d2cb9234 100644 --- a/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts +++ b/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts @@ -233,7 +233,7 @@ export class PoseLandmarker extends VisionTaskRunner { /** * Performs pose detection on the provided single image and waits * synchronously for the response. This method creates a copy of the resulting - * masks and should not be used in high-throughput applictions. Only + * masks and should not be used in high-throughput applications. Only * use this method when the PoseLandmarker is created with running mode * `image`. * @@ -246,7 +246,7 @@ export class PoseLandmarker extends VisionTaskRunner { /** * Performs pose detection on the provided single image and waits * synchronously for the response. This method creates a copy of the resulting - * masks and should not be used in high-throughput applictions. Only + * masks and should not be used in high-throughput applications. Only * use this method when the PoseLandmarker is created with running mode * `image`. * @@ -311,7 +311,7 @@ export class PoseLandmarker extends VisionTaskRunner { /** * Performs pose detection on the provided video frame and returns the result. * This method creates a copy of the resulting masks and should not be used - * in high-throughput applictions. Only use this method when the + * in high-throughput applications. Only use this method when the * PoseLandmarker is created with running mode `video`. * * @param videoFrame A video frame to process. @@ -324,7 +324,7 @@ export class PoseLandmarker extends VisionTaskRunner { /** * Performs pose detection on the provided video frame and returns the result. * This method creates a copy of the resulting masks and should not be used - * in high-throughput applictions. The method returns synchronously once the + * in high-throughput applications. The method returns synchronously once the * callback returns. Only use this method when the PoseLandmarker is created * with running mode `video`. * diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index b9fe8b0c..0316224f 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -11,7 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + +# Placeholder: load py_library load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test") @@ -68,6 +69,8 @@ cc_library( "//third_party:libffmpeg", "@com_google_absl//absl/base:endian", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/time", "@eigen_archive//:eigen3", @@ -121,10 +124,11 @@ cc_library( "//mediapipe/framework/port", "//mediapipe/framework/port:aligned_malloc_and_free", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:status_util", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@libyuv", ], @@ -154,6 +158,7 @@ cc_library( "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", + "@com_google_absl//absl/log:absl_log", ], ) @@ -169,6 +174,8 @@ cc_library( "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -218,6 +225,7 @@ cc_library( "//mediapipe/framework/port:singleton", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ] + select({ @@ -248,6 +256,7 @@ cc_library( "//mediapipe/framework/port:logging", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/functional:function_ref", + "@com_google_absl//absl/log:absl_check", ], ) @@ -297,8 +306,9 @@ cc_library( "//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) @@ -319,6 +329,8 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@eigen_archive//:eigen3", ], @@ -397,11 +409,13 @@ cc_library( "//mediapipe/framework:packet", "//mediapipe/framework:timestamp", "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgproc", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/util/android/BUILD b/mediapipe/util/android/BUILD index 726732ed..d66558d1 100644 --- a/mediapipe/util/android/BUILD +++ b/mediapipe/util/android/BUILD @@ -39,6 +39,8 @@ cc_library( "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/util/android/file/base", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ] + select({ "//conditions:default": [], diff --git a/mediapipe/util/android/asset_manager_util.cc b/mediapipe/util/android/asset_manager_util.cc index 8b5803d6..6e544ee8 100644 --- a/mediapipe/util/android/asset_manager_util.cc +++ b/mediapipe/util/android/asset_manager_util.cc @@ -16,6 +16,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/jni_util.h" @@ -56,7 +58,7 @@ bool AssetManager::InitializeFromAssetManager( // Finally get the pointer to the AAssetManager using native code. asset_manager_ = AAssetManager_fromJava(env, global_asset_manager); if (asset_manager_) { - LOG(INFO) << "Created global reference to asset manager."; + ABSL_LOG(INFO) << "Created global reference to asset manager."; return true; } return false; @@ -97,7 +99,7 @@ bool AssetManager::InitializeFromActivity(JNIEnv* env, jobject activity, bool AssetManager::FileExists(const std::string& filename, bool* is_dir) { if (!asset_manager_) { - LOG(ERROR) << "Asset manager was not initialized from JNI"; + ABSL_LOG(ERROR) << "Asset manager was not initialized from JNI"; return false; } @@ -132,9 +134,9 @@ bool AssetManager::FileExists(const std::string& filename, bool* is_dir) { } bool AssetManager::ReadFile(const std::string& filename, std::string* output) { - CHECK(output); + ABSL_CHECK(output); if (!asset_manager_) { - LOG(ERROR) << "Asset manager was not initialized from JNI"; + ABSL_LOG(ERROR) << "Asset manager was not initialized from JNI"; return false; } diff --git a/mediapipe/util/android/file/base/BUILD b/mediapipe/util/android/file/base/BUILD index f97bf271..9d014b2a 100644 --- a/mediapipe/util/android/file/base/BUILD +++ b/mediapipe/util/android/file/base/BUILD @@ -29,9 +29,9 @@ cc_library( visibility = ["//visibility:public"], deps = [ "//mediapipe/framework/port:file_helpers", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", "@com_google_absl//absl/base", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/util/android/file/base/file.cc b/mediapipe/util/android/file/base/file.cc index 83a34f15..ff58f9c3 100644 --- a/mediapipe/util/android/file/base/file.cc +++ b/mediapipe/util/android/file/base/file.cc @@ -19,11 +19,11 @@ #include #include "absl/base/call_once.h" +#include "absl/log/absl_log.h" #include "absl/strings/match.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" #include "absl/strings/strip.h" -#include "mediapipe/framework/port/logging.h" #ifdef __APPLE__ static_assert(sizeof(off_t) == 8, "Large file support is required"); @@ -95,7 +95,7 @@ void LocalHostInit() { buf[sizeof(buf) - 1] = '\0'; localhost_name_str = new std::string(buf); } else { - LOG(ERROR) << "Could not get local host name"; + ABSL_LOG(ERROR) << "Could not get local host name"; localhost_name_str = new std::string("localhost"); } } diff --git a/mediapipe/util/annotation_renderer.cc b/mediapipe/util/annotation_renderer.cc index d8516f9b..b83b4f71 100644 --- a/mediapipe/util/annotation_renderer.cc +++ b/mediapipe/util/annotation_renderer.cc @@ -19,6 +19,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" #include "mediapipe/util/color.pb.h" @@ -47,10 +49,10 @@ int ClampThickness(int thickness) { bool NormalizedtoPixelCoordinates(double normalized_x, double normalized_y, int image_width, int image_height, int* x_px, int* y_px) { - CHECK(x_px != nullptr); - CHECK(y_px != nullptr); - CHECK_GT(image_width, 0); - CHECK_GT(image_height, 0); + ABSL_CHECK(x_px != nullptr); + ABSL_CHECK(y_px != nullptr); + ABSL_CHECK_GT(image_width, 0); + ABSL_CHECK_GT(image_height, 0); if (normalized_x < 0 || normalized_x > 1.0 || normalized_y < 0 || normalized_y > 1.0) { @@ -116,7 +118,7 @@ void AnnotationRenderer::RenderDataOnImage(const RenderData& render_data) { } else if (annotation.data_case() == RenderAnnotation::kScribble) { DrawScribble(annotation); } else { - LOG(FATAL) << "Unknown annotation type: " << annotation.data_case(); + ABSL_LOG(FATAL) << "Unknown annotation type: " << annotation.data_case(); } } } @@ -147,12 +149,12 @@ void AnnotationRenderer::DrawRectangle(const RenderAnnotation& annotation) { int bottom = -1; const auto& rectangle = annotation.rectangle(); if (rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), - image_width_, image_height_, &left, - &top)); - CHECK(NormalizedtoPixelCoordinates(rectangle.right(), rectangle.bottom(), - image_width_, image_height_, &right, - &bottom)); + ABSL_CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), + image_width_, image_height_, &left, + &top)); + ABSL_CHECK(NormalizedtoPixelCoordinates(rectangle.right(), + rectangle.bottom(), image_width_, + image_height_, &right, &bottom)); } else { left = static_cast(rectangle.left() * scale_factor_); top = static_cast(rectangle.top() * scale_factor_); @@ -199,12 +201,12 @@ void AnnotationRenderer::DrawFilledRectangle( int bottom = -1; const auto& rectangle = annotation.filled_rectangle().rectangle(); if (rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), - image_width_, image_height_, &left, - &top)); - CHECK(NormalizedtoPixelCoordinates(rectangle.right(), rectangle.bottom(), - image_width_, image_height_, &right, - &bottom)); + ABSL_CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), + image_width_, image_height_, &left, + &top)); + ABSL_CHECK(NormalizedtoPixelCoordinates(rectangle.right(), + rectangle.bottom(), image_width_, + image_height_, &right, &bottom)); } else { left = static_cast(rectangle.left() * scale_factor_); top = static_cast(rectangle.top() * scale_factor_); @@ -239,12 +241,12 @@ void AnnotationRenderer::DrawRoundedRectangle( int bottom = -1; const auto& rectangle = annotation.rounded_rectangle().rectangle(); if (rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), - image_width_, image_height_, &left, - &top)); - CHECK(NormalizedtoPixelCoordinates(rectangle.right(), rectangle.bottom(), - image_width_, image_height_, &right, - &bottom)); + ABSL_CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), + image_width_, image_height_, &left, + &top)); + ABSL_CHECK(NormalizedtoPixelCoordinates(rectangle.right(), + rectangle.bottom(), image_width_, + image_height_, &right, &bottom)); } else { left = static_cast(rectangle.left() * scale_factor_); top = static_cast(rectangle.top() * scale_factor_); @@ -272,12 +274,12 @@ void AnnotationRenderer::DrawFilledRoundedRectangle( const auto& rectangle = annotation.filled_rounded_rectangle().rounded_rectangle().rectangle(); if (rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), - image_width_, image_height_, &left, - &top)); - CHECK(NormalizedtoPixelCoordinates(rectangle.right(), rectangle.bottom(), - image_width_, image_height_, &right, - &bottom)); + ABSL_CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), + image_width_, image_height_, &left, + &top)); + ABSL_CHECK(NormalizedtoPixelCoordinates(rectangle.right(), + rectangle.bottom(), image_width_, + image_height_, &right, &bottom)); } else { left = static_cast(rectangle.left() * scale_factor_); top = static_cast(rectangle.top() * scale_factor_); @@ -344,10 +346,10 @@ void AnnotationRenderer::DrawOval(const RenderAnnotation& annotation) { int bottom = -1; const auto& enclosing_rectangle = annotation.oval().rectangle(); if (enclosing_rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(enclosing_rectangle.left(), - enclosing_rectangle.top(), image_width_, - image_height_, &left, &top)); - CHECK(NormalizedtoPixelCoordinates( + ABSL_CHECK(NormalizedtoPixelCoordinates( + enclosing_rectangle.left(), enclosing_rectangle.top(), image_width_, + image_height_, &left, &top)); + ABSL_CHECK(NormalizedtoPixelCoordinates( enclosing_rectangle.right(), enclosing_rectangle.bottom(), image_width_, image_height_, &right, &bottom)); } else { @@ -373,10 +375,10 @@ void AnnotationRenderer::DrawFilledOval(const RenderAnnotation& annotation) { int bottom = -1; const auto& enclosing_rectangle = annotation.filled_oval().oval().rectangle(); if (enclosing_rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(enclosing_rectangle.left(), - enclosing_rectangle.top(), image_width_, - image_height_, &left, &top)); - CHECK(NormalizedtoPixelCoordinates( + ABSL_CHECK(NormalizedtoPixelCoordinates( + enclosing_rectangle.left(), enclosing_rectangle.top(), image_width_, + image_height_, &left, &top)); + ABSL_CHECK(NormalizedtoPixelCoordinates( enclosing_rectangle.right(), enclosing_rectangle.bottom(), image_width_, image_height_, &right, &bottom)); } else { @@ -402,12 +404,12 @@ void AnnotationRenderer::DrawArrow(const RenderAnnotation& annotation) { const auto& arrow = annotation.arrow(); if (arrow.normalized()) { - CHECK(NormalizedtoPixelCoordinates(arrow.x_start(), arrow.y_start(), - image_width_, image_height_, &x_start, - &y_start)); - CHECK(NormalizedtoPixelCoordinates(arrow.x_end(), arrow.y_end(), - image_width_, image_height_, &x_end, - &y_end)); + ABSL_CHECK(NormalizedtoPixelCoordinates(arrow.x_start(), arrow.y_start(), + image_width_, image_height_, + &x_start, &y_start)); + ABSL_CHECK(NormalizedtoPixelCoordinates(arrow.x_end(), arrow.y_end(), + image_width_, image_height_, &x_end, + &y_end)); } else { x_start = static_cast(arrow.x_start() * scale_factor_); y_start = static_cast(arrow.y_start() * scale_factor_); @@ -453,8 +455,8 @@ void AnnotationRenderer::DrawPoint(const RenderAnnotation::Point& point, int x = -1; int y = -1; if (point.normalized()) { - CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_, - image_height_, &x, &y)); + ABSL_CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_, + image_height_, &x, &y)); } else { x = static_cast(point.x() * scale_factor_); y = static_cast(point.y() * scale_factor_); @@ -481,11 +483,12 @@ void AnnotationRenderer::DrawLine(const RenderAnnotation& annotation) { const auto& line = annotation.line(); if (line.normalized()) { - CHECK(NormalizedtoPixelCoordinates(line.x_start(), line.y_start(), - image_width_, image_height_, &x_start, - &y_start)); - CHECK(NormalizedtoPixelCoordinates(line.x_end(), line.y_end(), image_width_, - image_height_, &x_end, &y_end)); + ABSL_CHECK(NormalizedtoPixelCoordinates(line.x_start(), line.y_start(), + image_width_, image_height_, + &x_start, &y_start)); + ABSL_CHECK(NormalizedtoPixelCoordinates(line.x_end(), line.y_end(), + image_width_, image_height_, &x_end, + &y_end)); } else { x_start = static_cast(line.x_start() * scale_factor_); y_start = static_cast(line.y_start() * scale_factor_); @@ -509,11 +512,12 @@ void AnnotationRenderer::DrawGradientLine(const RenderAnnotation& annotation) { const auto& line = annotation.gradient_line(); if (line.normalized()) { - CHECK(NormalizedtoPixelCoordinates(line.x_start(), line.y_start(), - image_width_, image_height_, &x_start, - &y_start)); - CHECK(NormalizedtoPixelCoordinates(line.x_end(), line.y_end(), image_width_, - image_height_, &x_end, &y_end)); + ABSL_CHECK(NormalizedtoPixelCoordinates(line.x_start(), line.y_start(), + image_width_, image_height_, + &x_start, &y_start)); + ABSL_CHECK(NormalizedtoPixelCoordinates(line.x_end(), line.y_end(), + image_width_, image_height_, &x_end, + &y_end)); } else { x_start = static_cast(line.x_start() * scale_factor_); y_start = static_cast(line.y_start() * scale_factor_); @@ -537,9 +541,9 @@ void AnnotationRenderer::DrawText(const RenderAnnotation& annotation) { const auto& text = annotation.text(); if (text.normalized()) { - CHECK(NormalizedtoPixelCoordinates(text.left(), text.baseline(), - image_width_, image_height_, &left, - &baseline)); + ABSL_CHECK(NormalizedtoPixelCoordinates(text.left(), text.baseline(), + image_width_, image_height_, &left, + &baseline)); font_size = static_cast(round(text.font_height() * image_height_)); } else { left = static_cast(text.left() * scale_factor_); diff --git a/mediapipe/util/audio_decoder.cc b/mediapipe/util/audio_decoder.cc index 569e8015..33d56887 100644 --- a/mediapipe/util/audio_decoder.cc +++ b/mediapipe/util/audio_decoder.cc @@ -22,6 +22,8 @@ #include "Eigen/Core" #include "absl/base/internal/endian.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" #include "absl/strings/substitute.h" @@ -196,7 +198,7 @@ absl::Status LogStatus(const absl::Status& status, << (packet.flags & AV_PKT_FLAG_KEY ? " Key Frame." : ""); if (always_return_ok_status) { - LOG(WARNING) << status.message(); + ABSL_LOG(WARNING) << status.message(); return absl::OkStatus(); } else { return status; @@ -227,8 +229,8 @@ BasePacketProcessor::~BasePacketProcessor() { Close(); } bool BasePacketProcessor::HasData() { return !buffer_.empty(); } absl::Status BasePacketProcessor::GetData(Packet* packet) { - CHECK(packet); - CHECK(!buffer_.empty()); + ABSL_CHECK(packet); + ABSL_CHECK(!buffer_.empty()); *packet = buffer_.front(); buffer_.pop_front(); @@ -335,7 +337,7 @@ inline float PcmEncodedSampleInt32ToFloat(const char* data) { AudioPacketProcessor::AudioPacketProcessor(const AudioStreamOptions& options) : sample_time_base_{0, 0}, options_(options) { - DCHECK(absl::little_endian::IsLittleEndian()); + ABSL_DCHECK(absl::little_endian::IsLittleEndian()); } absl::Status AudioPacketProcessor::Open(int id, AVStream* stream) { @@ -349,7 +351,7 @@ absl::Status AudioPacketProcessor::Open(int id, AVStream* stream) { if (avcodec_open2(avcodec_ctx_, avcodec_, &avcodec_opts_) < 0) { return UnknownError("avcodec_open() failed."); } - CHECK(avcodec_ctx_->codec); + ABSL_CHECK(avcodec_ctx_->codec); source_time_base_ = stream->time_base; source_frame_rate_ = stream->r_frame_rate; @@ -411,7 +413,7 @@ int64_t AudioPacketProcessor::SampleNumberToMicroseconds( } absl::Status AudioPacketProcessor::ProcessPacket(AVPacket* packet) { - CHECK(packet); + ABSL_CHECK(packet); if (flushed_) { return UnknownError( "ProcessPacket was called, but AudioPacketProcessor is already " @@ -450,17 +452,18 @@ absl::Status AudioPacketProcessor::ProcessDecodedFrame(const AVPacket& packet) { if (absl::Microseconds(std::abs(expected_us - actual_us)) > absl::Seconds( absl::GetFlag(FLAGS_media_decoder_allowed_audio_gap_merge))) { - LOG(ERROR) << "The expected time based on how many samples we have seen (" - << expected_us - << " microseconds) no longer matches the time based " - "on what the audio stream is telling us (" - << actual_us - << " microseconds). The difference is more than " - "--media_decoder_allowed_audio_gap_merge (" - << absl::FormatDuration(absl::Seconds(absl::GetFlag( - FLAGS_media_decoder_allowed_audio_gap_merge))) - << " microseconds). Resetting the timestamps to track what " - "the audio stream is telling us."; + ABSL_LOG(ERROR) + << "The expected time based on how many samples we have seen (" + << expected_us + << " microseconds) no longer matches the time based " + "on what the audio stream is telling us (" + << actual_us + << " microseconds). The difference is more than " + "--media_decoder_allowed_audio_gap_merge (" + << absl::FormatDuration(absl::Seconds( + absl::GetFlag(FLAGS_media_decoder_allowed_audio_gap_merge))) + << " microseconds). Resetting the timestamps to track what " + "the audio stream is telling us."; expected_sample_number_ = TimestampToSampleNumber(pts); } } @@ -560,14 +563,15 @@ absl::Status AudioPacketProcessor::AddAudioDataToBuffer( last_timestamp_ = output_timestamp; if (last_frame_time_regression_detected_) { last_frame_time_regression_detected_ = false; - LOG(INFO) << "Processor " << this << " resumed audio packet processing."; + ABSL_LOG(INFO) << "Processor " << this + << " resumed audio packet processing."; } } else if (!last_frame_time_regression_detected_) { last_frame_time_regression_detected_ = true; - LOG(ERROR) << "Processor " << this - << " is dropping an audio packet because the timestamps " - "regressed. Was " - << last_timestamp_ << " but got " << output_timestamp; + ABSL_LOG(ERROR) << "Processor " << this + << " is dropping an audio packet because the timestamps " + "regressed. Was " + << last_timestamp_ << " but got " << output_timestamp; } expected_sample_number_ += num_samples; @@ -575,7 +579,7 @@ absl::Status AudioPacketProcessor::AddAudioDataToBuffer( } absl::Status AudioPacketProcessor::FillHeader(TimeSeriesHeader* header) const { - CHECK(header); + ABSL_CHECK(header); header->set_sample_rate(sample_rate_); header->set_num_channels(num_channels_); return absl::OkStatus(); @@ -592,8 +596,8 @@ AudioDecoder::AudioDecoder() { av_register_all(); } AudioDecoder::~AudioDecoder() { absl::Status status = Close(); if (!status.ok()) { - LOG(ERROR) << "Encountered error while closing media file: " - << status.message(); + ABSL_LOG(ERROR) << "Encountered error while closing media file: " + << status.message(); } } @@ -615,8 +619,8 @@ absl::Status AudioDecoder::Initialize( Cleanup> decoder_closer([this]() { absl::Status status = Close(); if (!status.ok()) { - LOG(ERROR) << "Encountered error while closing media file: " - << status.message(); + ABSL_LOG(ERROR) << "Encountered error while closing media file: " + << status.message(); } }); @@ -645,24 +649,24 @@ absl::Status AudioDecoder::Initialize( absl::make_unique( options.audio_stream(*options_index_ptr)); if (!ContainsKey(audio_processor_, stream_id)) { - LOG(INFO) << "Created audio processor " << processor.get() - << " for file \"" << input_file << "\""; + ABSL_LOG(INFO) << "Created audio processor " << processor.get() + << " for file \"" << input_file << "\""; } else { - LOG(ERROR) << "Stream " << stream_id - << " already mapped to audio processor " - << audio_processor_[stream_id].get(); + ABSL_LOG(ERROR) << "Stream " << stream_id + << " already mapped to audio processor " + << audio_processor_[stream_id].get(); } MP_RETURN_IF_ERROR(processor->Open(stream_id, stream)); audio_processor_.emplace(stream_id, std::move(processor)); - CHECK(InsertIfNotPresent( + ABSL_CHECK(InsertIfNotPresent( &stream_index_to_stream_id_, options.audio_stream(*options_index_ptr).stream_index(), stream_id)); - CHECK(InsertIfNotPresent(&stream_id_to_audio_options_index_, - stream_id, *options_index_ptr)); - CHECK(InsertIfNotPresent(&audio_options_index_to_stream_id, - *options_index_ptr, stream_id)); + ABSL_CHECK(InsertIfNotPresent(&stream_id_to_audio_options_index_, + stream_id, *options_index_ptr)); + ABSL_CHECK(InsertIfNotPresent(&audio_options_index_to_stream_id, + *options_index_ptr, stream_id)); } ++current_audio_index; break; @@ -703,10 +707,10 @@ absl::Status AudioDecoder::GetData(int* options_index, Packet* data) { // Ignore packets which are out of the requested timestamp range. if (start_time_ != Timestamp::Unset()) { if (is_first_packet && data->Timestamp() > start_time_) { - LOG(ERROR) << "First packet in audio stream " << *options_index - << " has timestamp " << data->Timestamp() - << " which is after start time of " << start_time_ - << "."; + ABSL_LOG(ERROR) + << "First packet in audio stream " << *options_index + << " has timestamp " << data->Timestamp() + << " which is after start time of " << start_time_ << "."; } if (data->Timestamp() < start_time_) { VLOG(1) << "Skipping audio frame with timestamp " @@ -772,8 +776,8 @@ absl::Status AudioDecoder::ProcessPacket() { av_packet->data = nullptr; int ret = av_read_frame(avformat_ctx_, av_packet.get()); if (ret >= 0) { - CHECK(av_packet->data) << "AVPacket does not include any data but " - "av_read_frame was successful."; + ABSL_CHECK(av_packet->data) << "AVPacket does not include any data but " + "av_read_frame was successful."; const int stream_id = av_packet->stream_index; auto audio_iterator = audio_processor_.find(stream_id); if (audio_iterator != audio_processor_.end()) { diff --git a/mediapipe/util/cpu_util.cc b/mediapipe/util/cpu_util.cc index 052eabb8..74e6debd 100644 --- a/mediapipe/util/cpu_util.cc +++ b/mediapipe/util/cpu_util.cc @@ -26,7 +26,6 @@ #include #include "absl/algorithm/container.h" -#include "absl/flags/flag.h" #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "absl/strings/str_cat.h" @@ -35,23 +34,14 @@ #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/statusor.h" -ABSL_FLAG(std::string, system_cpu_max_freq_file, - "/sys/devices/system/cpu/cpu$0/cpufreq/cpuinfo_max_freq", - "The file pattern for CPU max frequencies, where $0 will be replaced " - "with the CPU id."); - namespace mediapipe { namespace { constexpr uint32_t kBufferLength = 64; absl::StatusOr GetFilePath(int cpu) { - if (!absl::StrContains(absl::GetFlag(FLAGS_system_cpu_max_freq_file), "$0")) { - return absl::InvalidArgumentError( - absl::StrCat("Invalid frequency file: ", - absl::GetFlag(FLAGS_system_cpu_max_freq_file))); - } - return absl::Substitute(absl::GetFlag(FLAGS_system_cpu_max_freq_file), cpu); + return absl::Substitute( + "/sys/devices/system/cpu/cpu$0/cpufreq/cpuinfo_max_freq", cpu); } absl::StatusOr GetCpuMaxFrequency(int cpu) { diff --git a/mediapipe/util/filtering/BUILD b/mediapipe/util/filtering/BUILD index 6bd6bc36..4acb83f6 100644 --- a/mediapipe/util/filtering/BUILD +++ b/mediapipe/util/filtering/BUILD @@ -23,7 +23,7 @@ cc_library( srcs = ["low_pass_filter.cc"], hdrs = ["low_pass_filter.h"], deps = [ - "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", ], ) @@ -45,7 +45,7 @@ cc_library( deps = [ ":low_pass_filter", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/time", ], @@ -57,7 +57,8 @@ cc_library( hdrs = ["relative_velocity_filter.h"], deps = [ ":low_pass_filter", - "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/time", ], @@ -70,6 +71,7 @@ cc_test( ":relative_velocity_filter", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/time", ], diff --git a/mediapipe/util/filtering/low_pass_filter.cc b/mediapipe/util/filtering/low_pass_filter.cc index 91ef1560..670fab57 100644 --- a/mediapipe/util/filtering/low_pass_filter.cc +++ b/mediapipe/util/filtering/low_pass_filter.cc @@ -14,8 +14,8 @@ #include "mediapipe/util/filtering/low_pass_filter.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -49,7 +49,7 @@ float LowPassFilter::LastValue() { return stored_value_; } void LowPassFilter::SetAlpha(float alpha) { if (alpha < 0.0f || alpha > 1.0f) { - LOG(ERROR) << "alpha: " << alpha << " should be in [0.0, 1.0] range"; + ABSL_LOG(ERROR) << "alpha: " << alpha << " should be in [0.0, 1.0] range"; return; } alpha_ = alpha; diff --git a/mediapipe/util/filtering/one_euro_filter.cc b/mediapipe/util/filtering/one_euro_filter.cc index e7893edf..954477bc 100644 --- a/mediapipe/util/filtering/one_euro_filter.cc +++ b/mediapipe/util/filtering/one_euro_filter.cc @@ -2,9 +2,9 @@ #include +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/util/filtering/low_pass_filter.h" namespace mediapipe { @@ -28,7 +28,7 @@ double OneEuroFilter::Apply(absl::Duration timestamp, double value_scale, if (last_time_ >= new_timestamp) { // Results are unpredictable in this case, so nothing to do but // return same value - LOG(WARNING) << "New timestamp is equal or less than the last one."; + ABSL_LOG(WARNING) << "New timestamp is equal or less than the last one."; return value; } @@ -59,7 +59,7 @@ double OneEuroFilter::GetAlpha(double cutoff) { void OneEuroFilter::SetFrequency(double frequency) { if (frequency <= kEpsilon) { - LOG(ERROR) << "frequency should be > 0"; + ABSL_LOG(ERROR) << "frequency should be > 0"; return; } frequency_ = frequency; @@ -67,7 +67,7 @@ void OneEuroFilter::SetFrequency(double frequency) { void OneEuroFilter::SetMinCutoff(double min_cutoff) { if (min_cutoff <= kEpsilon) { - LOG(ERROR) << "min_cutoff should be > 0"; + ABSL_LOG(ERROR) << "min_cutoff should be > 0"; return; } min_cutoff_ = min_cutoff; @@ -77,7 +77,7 @@ void OneEuroFilter::SetBeta(double beta) { beta_ = beta; } void OneEuroFilter::SetDerivateCutoff(double derivate_cutoff) { if (derivate_cutoff <= kEpsilon) { - LOG(ERROR) << "derivate_cutoff should be > 0"; + ABSL_LOG(ERROR) << "derivate_cutoff should be > 0"; return; } derivate_cutoff_ = derivate_cutoff; diff --git a/mediapipe/util/filtering/relative_velocity_filter.cc b/mediapipe/util/filtering/relative_velocity_filter.cc index ab88ad59..a10b1c5a 100644 --- a/mediapipe/util/filtering/relative_velocity_filter.cc +++ b/mediapipe/util/filtering/relative_velocity_filter.cc @@ -17,8 +17,9 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -28,7 +29,7 @@ float RelativeVelocityFilter::Apply(absl::Duration timestamp, float value_scale, if (last_timestamp_ >= new_timestamp) { // Results are unpredictable in this case, so nothing to do but // return same value - LOG(WARNING) << "New timestamp is equal or less than the last one."; + ABSL_LOG(WARNING) << "New timestamp is equal or less than the last one."; return value; } @@ -36,8 +37,8 @@ float RelativeVelocityFilter::Apply(absl::Duration timestamp, float value_scale, if (last_timestamp_ == -1) { alpha = 1.0; } else { - DCHECK(distance_mode_ == DistanceEstimationMode::kLegacyTransition || - distance_mode_ == DistanceEstimationMode::kForceCurrentScale); + ABSL_DCHECK(distance_mode_ == DistanceEstimationMode::kLegacyTransition || + distance_mode_ == DistanceEstimationMode::kForceCurrentScale); const float distance = distance_mode_ == DistanceEstimationMode::kLegacyTransition ? value * value_scale - diff --git a/mediapipe/util/filtering/relative_velocity_filter_test.cc b/mediapipe/util/filtering/relative_velocity_filter_test.cc index 717237bb..4589f833 100644 --- a/mediapipe/util/filtering/relative_velocity_filter_test.cc +++ b/mediapipe/util/filtering/relative_velocity_filter_test.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/memory/memory.h" #include "absl/time/time.h" #include "mediapipe/framework/port/gtest.h" @@ -268,7 +269,7 @@ void TestTranslationInvariance(DistanceEstimationMode distance_mode) { ++times_largely_diverged; } } else { - CHECK(distance_mode == DistanceEstimationMode::kForceCurrentScale); + ABSL_CHECK(distance_mode == DistanceEstimationMode::kForceCurrentScale); EXPECT_NEAR(difference, 0.0f, kForceCurrentScaleAbsoluteError); } } diff --git a/mediapipe/util/frame_buffer/BUILD b/mediapipe/util/frame_buffer/BUILD index 5dfffbac..c42c9643 100644 --- a/mediapipe/util/frame_buffer/BUILD +++ b/mediapipe/util/frame_buffer/BUILD @@ -41,6 +41,7 @@ cc_test( "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", ], ) @@ -86,7 +87,7 @@ cc_test( deps = [ ":buffer", "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/log", + "@com_google_absl//absl/log:absl_log", ], ) @@ -96,7 +97,7 @@ cc_test( deps = [ ":buffer", "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/log", + "@com_google_absl//absl/log:absl_log", ], ) @@ -106,6 +107,6 @@ cc_test( deps = [ ":buffer", "//mediapipe/framework/port:gtest_main", - "@com_google_absl//absl/log", + "@com_google_absl//absl/log:absl_log", ], ) diff --git a/mediapipe/util/frame_buffer/frame_buffer_util_test.cc b/mediapipe/util/frame_buffer/frame_buffer_util_test.cc index 8e86f02d..aed03962 100644 --- a/mediapipe/util/frame_buffer/frame_buffer_util_test.cc +++ b/mediapipe/util/frame_buffer/frame_buffer_util_test.cc @@ -18,6 +18,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/formats/frame_buffer.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port/gmock.h" @@ -784,7 +785,7 @@ TEST(FrameBufferUtil, RgbRotate) { absl::StatusOr> CreateYuvBuffer( uint8_t* buffer, FrameBuffer::Dimension dimension, int plane_count, FrameBuffer::Format format) { - DCHECK(plane_count > 0 && plane_count < 4); + ABSL_DCHECK(plane_count > 0 && plane_count < 4); ASSIGN_OR_RETURN(auto uv_dimension, GetUvPlaneDimension(dimension, format)); if (plane_count == 1) { @@ -793,8 +794,8 @@ absl::StatusOr> CreateYuvBuffer( /*pixel_stride_bytes=*/1}}}; return std::make_shared(planes, dimension, format); } else if (plane_count == 2) { - CHECK(format == FrameBuffer::Format::kNV12 || - format == FrameBuffer::Format::kNV21); + ABSL_CHECK(format == FrameBuffer::Format::kNV12 || + format == FrameBuffer::Format::kNV21); const std::vector planes = { {buffer, /*stride=*/{/*row_stride_bytes=*/dimension.width, diff --git a/mediapipe/util/frame_buffer/gray_buffer_test.cc b/mediapipe/util/frame_buffer/gray_buffer_test.cc index f6f9e9e3..43719d26 100644 --- a/mediapipe/util/frame_buffer/gray_buffer_test.cc +++ b/mediapipe/util/frame_buffer/gray_buffer_test.cc @@ -16,14 +16,14 @@ #include -#include "absl/log/log.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" // The default implementation of halide_error calls abort(), which we don't // want. Instead, log the error and let the filter invocation fail. extern "C" void halide_error(void*, const char* message) { - LOG(ERROR) << "Halide Error: " << message; + ABSL_LOG(ERROR) << "Halide Error: " << message; } namespace mediapipe { diff --git a/mediapipe/util/frame_buffer/rgb_buffer_test.cc b/mediapipe/util/frame_buffer/rgb_buffer_test.cc index 8ade0b92..88043e47 100644 --- a/mediapipe/util/frame_buffer/rgb_buffer_test.cc +++ b/mediapipe/util/frame_buffer/rgb_buffer_test.cc @@ -17,7 +17,7 @@ #include #include -#include "absl/log/log.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/util/frame_buffer/float_buffer.h" @@ -27,7 +27,7 @@ // The default implementation of halide_error calls abort(), which we don't // want. Instead, log the error and let the filter invocation fail. extern "C" void halide_error(void*, const char* message) { - LOG(ERROR) << "Halide Error: " << message; + ABSL_LOG(ERROR) << "Halide Error: " << message; } namespace mediapipe { diff --git a/mediapipe/util/frame_buffer/yuv_buffer_test.cc b/mediapipe/util/frame_buffer/yuv_buffer_test.cc index a18b19a9..b1e7b68d 100644 --- a/mediapipe/util/frame_buffer/yuv_buffer_test.cc +++ b/mediapipe/util/frame_buffer/yuv_buffer_test.cc @@ -16,7 +16,7 @@ #include -#include "absl/log/log.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/util/frame_buffer/rgb_buffer.h" @@ -24,7 +24,7 @@ // The default implementation of halide_error calls abort(), which we don't // want. Instead, log the error and let the filter invocation fail. extern "C" void halide_error(void*, const char* message) { - LOG(ERROR) << "Halide Error: " << message; + ABSL_LOG(ERROR) << "Halide Error: " << message; } namespace mediapipe { diff --git a/mediapipe/util/image_frame_util.cc b/mediapipe/util/image_frame_util.cc index bf2773fd..418a6b09 100644 --- a/mediapipe/util/image_frame_util.cc +++ b/mediapipe/util/image_frame_util.cc @@ -20,6 +20,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" @@ -34,7 +36,6 @@ #include "mediapipe/framework/formats/yuv_image.h" #include "mediapipe/framework/port/aligned_malloc_and_free.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/port.h" #include "mediapipe/framework/port/status_macros.h" @@ -46,8 +47,8 @@ void RescaleImageFrame(const ImageFrame& source_frame, const int width, const int height, const int alignment_boundary, const int open_cv_interpolation_algorithm, ImageFrame* destination_frame) { - CHECK(destination_frame); - CHECK_EQ(ImageFormat::SRGB, source_frame.Format()); + ABSL_CHECK(destination_frame); + ABSL_CHECK_EQ(ImageFormat::SRGB, source_frame.Format()); cv::Mat source_mat = ::mediapipe::formats::MatView(&source_frame); destination_frame->Reset(source_frame.Format(), width, height, @@ -61,7 +62,7 @@ void RescaleImageFrame(const ImageFrame& source_frame, const int width, void RescaleSrgbImage(const cv::Mat& source, const int width, const int height, const int open_cv_interpolation_algorithm, cv::Mat* destination) { - CHECK(destination); + ABSL_CHECK(destination); // Convert input_mat into 16 bit per channel linear RGB space. cv::Mat input_mat16; @@ -106,7 +107,7 @@ void ImageFrameToYUVImage(const ImageFrame& image_frame, YUVImage* yuv_image) { u, uv_stride, // v, uv_stride, // width, height); - CHECK_EQ(0, rv); + ABSL_CHECK_EQ(0, rv); } void ImageFrameToYUVNV12Image(const ImageFrame& image_frame, @@ -136,12 +137,12 @@ void ImageFrameToYUVNV12Image(const ImageFrame& image_frame, yuv_i420_image.stride(2), yuv_nv12_image->mutable_data(0), yuv_nv12_image->stride(0), yuv_nv12_image->mutable_data(1), yuv_nv12_image->stride(1), width, height); - CHECK_EQ(0, rv); + ABSL_CHECK_EQ(0, rv); } void YUVImageToImageFrame(const YUVImage& yuv_image, ImageFrame* image_frame, bool use_bt709) { - CHECK(image_frame); + ABSL_CHECK(image_frame); int width = yuv_image.width(); int height = yuv_image.height(); image_frame->Reset(ImageFormat::SRGB, width, height, 16); @@ -161,12 +162,12 @@ void YUVImageToImageFrame(const YUVImage& yuv_image, ImageFrame* image_frame, image_frame->MutablePixelData(), image_frame->WidthStep(), width, height); } - CHECK_EQ(0, rv); + ABSL_CHECK_EQ(0, rv); } void YUVImageToImageFrameFromFormat(const YUVImage& yuv_image, ImageFrame* image_frame) { - CHECK(image_frame); + ABSL_CHECK(image_frame); int width = yuv_image.width(); int height = yuv_image.height(); image_frame->Reset(ImageFormat::SRGB, width, height, 16); @@ -207,7 +208,7 @@ void YUVImageToImageFrameFromFormat(const YUVImage& yuv_image, yuv_image.width(), yuv_image.height()); break; default: - LOG(FATAL) << "Unsupported YUVImage format."; + ABSL_LOG(FATAL) << "Unsupported YUVImage format."; } } diff --git a/mediapipe/util/image_test_utils.cc b/mediapipe/util/image_test_utils.cc index 77b75595..9e10f40c 100644 --- a/mediapipe/util/image_test_utils.cc +++ b/mediapipe/util/image_test_utils.cc @@ -1,7 +1,15 @@ #include "mediapipe/util/image_test_utils.h" +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" @@ -38,20 +46,29 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) { } else if (image_channels == 1) { return ImageFormat::GRAY8; } - LOG(FATAL) << "Unsupported input image channles: " << image_channels; + ABSL_LOG(FATAL) << "Unsupported input image channles: " << image_channels; } Packet MakeImageFramePacket(cv::Mat input, int timestamp) { ImageFrame input_image(GetImageFormat(input.channels()), input.cols, - input.rows, input.step, input.data, [](uint8_t*) {}); - return MakePacket(std::move(input_image)).At(Timestamp(0)); + input.rows, input.step, input.data, + [input](uint8_t*) mutable { input.release(); }); + return MakePacket(std::move(input_image)) + .At(Timestamp(timestamp)); } Packet MakeImagePacket(cv::Mat input, int timestamp) { mediapipe::Image input_image(std::make_shared( GetImageFormat(input.channels()), input.cols, input.rows, input.step, - input.data, [](uint8_t*) {})); - return MakePacket(std::move(input_image)).At(Timestamp(0)); + input.data, [input](uint8_t*) mutable { input.release(); })); + return MakePacket(std::move(input_image)) + .At(Timestamp(timestamp)); +} + +cv::Mat RgbaToBgr(cv::Mat rgba) { + cv::Mat bgra; + cv::cvtColor(rgba, bgra, cv::COLOR_RGBA2BGR); + return bgra; } } // namespace mediapipe diff --git a/mediapipe/util/image_test_utils.h b/mediapipe/util/image_test_utils.h index 6df9644d..15a21c5b 100644 --- a/mediapipe/util/image_test_utils.h +++ b/mediapipe/util/image_test_utils.h @@ -3,7 +3,7 @@ #include -#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/opencv_core_inc.h" @@ -27,6 +27,9 @@ Packet MakeImageFramePacket(cv::Mat input, int timestamp = 0); // Converts the cv::Mat into Image packet. Packet MakeImagePacket(cv::Mat input, int timestamp = 0); +// Converts RGBA Mat to BGR. +cv::Mat RgbaToBgr(cv::Mat rgba); + } // namespace mediapipe #endif // MEDIAPIPE_UTIL_IMAGE_TEST_UTILS_H_ diff --git a/mediapipe/util/log_fatal_to_breakpad.cc b/mediapipe/util/log_fatal_to_breakpad.cc index 45087f2e..555b13df 100644 --- a/mediapipe/util/log_fatal_to_breakpad.cc +++ b/mediapipe/util/log_fatal_to_breakpad.cc @@ -2,7 +2,6 @@ #import -#include "absl/log/log.h" #include "absl/log/log_sink.h" #include "absl/log/log_sink_registry.h" #import "googlemac/iPhone/Shared/GoogleIOSBreakpad/Classes/GoogleBreakpadController.h" diff --git a/mediapipe/util/pose_util.cc b/mediapipe/util/pose_util.cc index 3a9c1e97..6c9af9bf 100644 --- a/mediapipe/util/pose_util.cc +++ b/mediapipe/util/pose_util.cc @@ -1,5 +1,6 @@ #include "mediapipe/util/pose_util.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" namespace { @@ -108,9 +109,35 @@ const int kFaceMeshFaceOval[36][2] = { {172, 58}, {58, 132}, {132, 93}, {93, 234}, {234, 127}, {127, 162}, {162, 21}, {21, 54}, {54, 103}, {103, 67}, {67, 109}, {109, 10}}; -const cv::Scalar kRightEyeColor = cv::Scalar(255.0, 48.0, 48.0); -const cv::Scalar kLeftEyeColor = cv::Scalar(48.0, 255.0, 48.0); -const cv::Scalar kFaceContourColor = cv::Scalar(224.0, 224.0, 224.0); +const int kFaceMeshNose[25][2] = { + {168, 6}, {6, 197}, {197, 195}, {195, 5}, {5, 4}, + {4, 1}, {1, 19}, {19, 94}, {94, 2}, {98, 97}, + {97, 2}, {2, 326}, {326, 327}, {327, 294}, {294, 278}, + {278, 344}, {344, 440}, {440, 275}, {275, 4}, {4, 45}, + {45, 220}, {220, 115}, {115, 48}, {48, 64}, {64, 98}}; + +const cv::Scalar kRedColor = cv::Scalar{255, 48, 48}; +const cv::Scalar kGreenColor = cv::Scalar{48, 255, 48}; +const cv::Scalar kGreenColor2 = cv::Scalar{0, 128, 0}; +const cv::Scalar kBlueColor = cv::Scalar{21, 101, 192}; +const cv::Scalar kBlueColor2 = cv::Scalar{0, 204, 255}; +const cv::Scalar kYellowColor = cv::Scalar{255, 204, 0}; +const cv::Scalar kYellowColor2 = cv::Scalar{192, 255, 48}; +const cv::Scalar kGrayColor = cv::Scalar{128, 128, 128}; +const cv::Scalar kPurpleColor = cv::Scalar{128, 64, 128}; +const cv::Scalar kPeachColor = cv::Scalar{255, 229, 180}; +const cv::Scalar kWhiteColor = cv::Scalar(224, 224, 224); +const cv::Scalar kCyanColor = cv::Scalar{48, 255, 192}; +const cv::Scalar kCyanColor2 = cv::Scalar{48, 48, 255}; +const cv::Scalar kMagentaColor = cv::Scalar{255, 48, 192}; +const cv::Scalar kPinkColor = cv::Scalar{255, 0, 255}; +const cv::Scalar kOrangeColor = cv::Scalar{192, 101, 21}; + +void ReverseRGB(cv::Scalar* color) { + int tmp = color->val[0]; + color->val[0] = color->val[2]; + color->val[2] = tmp; +} } // namespace namespace mediapipe { @@ -171,62 +198,131 @@ void DrawPose(const mediapipe::NormalizedLandmarkList& pose, bool flip_y, } } -void DrawFace(const mediapipe::NormalizedLandmarkList& face, bool flip_y, - cv::Mat* image) { - const int target_width = image->cols; - const int target_height = image->rows; - std::vector landmarks; +void DrawFace(const mediapipe::NormalizedLandmarkList& face, + const std::pair& image_size, const cv::Mat& affine, + bool flip_y, bool draw_nose, int color_style, bool reverse_color, + int draw_line_width, cv::Mat* image) { + std::vector landmarks; for (const auto& lm : face.landmark()) { - landmarks.emplace_back(lm.x() * target_width, - (flip_y ? 1.0f - lm.y() : lm.y()) * target_height); + float ori_x = lm.x() * image_size.first; + float ori_y = (flip_y ? 1.0f - lm.y() : lm.y()) * image_size.second; + + landmarks.emplace_back( + affine.at(0, 0) * ori_x + affine.at(0, 1) * ori_y + + affine.at(0, 2), + affine.at(1, 0) * ori_x + affine.at(1, 1) * ori_y + + affine.at(1, 2)); + } + + cv::Scalar kFaceOvalColor; + cv::Scalar kLipsColor; + cv::Scalar kLeftEyeColor; + cv::Scalar kLeftEyebrowColor; + cv::Scalar kLeftEyeIrisColor; + cv::Scalar kRightEyeColor; + cv::Scalar kRightEyebrowColor; + cv::Scalar kRightEyeIrisColor; + cv::Scalar kNoseColor; + if (color_style == 0) { + kFaceOvalColor = kWhiteColor; + kLipsColor = kWhiteColor; + kLeftEyeColor = kGreenColor; + kLeftEyebrowColor = kGreenColor; + kLeftEyeIrisColor = kGreenColor; + kRightEyeColor = kRedColor; + kRightEyebrowColor = kRedColor; + kRightEyeIrisColor = kRedColor; + kNoseColor = kWhiteColor; + } else if (color_style == 1) { + kFaceOvalColor = kWhiteColor; + kLipsColor = kBlueColor; + kLeftEyeColor = kCyanColor; + kLeftEyebrowColor = kGreenColor; + kLeftEyeIrisColor = kGreenColor; + kRightEyeColor = kMagentaColor; + kRightEyebrowColor = kRedColor; + kRightEyeIrisColor = kRedColor; + kNoseColor = kYellowColor; + } else if (color_style == 2) { + kFaceOvalColor = kWhiteColor; + kLipsColor = kRedColor; + kLeftEyeColor = kYellowColor2; + kLeftEyebrowColor = kGreenColor; + kLeftEyeIrisColor = kBlueColor2; + kRightEyeColor = kPinkColor; + kRightEyebrowColor = kGreenColor2; + kRightEyeIrisColor = kCyanColor2; + kNoseColor = kOrangeColor; + } else { + ABSL_LOG(ERROR) << "color_style not supported."; + } + + if (reverse_color) { + ReverseRGB(&kFaceOvalColor); + ReverseRGB(&kLipsColor); + ReverseRGB(&kLeftEyeColor); + ReverseRGB(&kLeftEyebrowColor); + ReverseRGB(&kLeftEyeIrisColor); + ReverseRGB(&kRightEyeColor); + ReverseRGB(&kRightEyebrowColor); + ReverseRGB(&kRightEyeIrisColor); + ReverseRGB(&kNoseColor); } - constexpr int draw_line_width = 2; for (int j = 0; j < 36; ++j) { cv::line(*image, landmarks[kFaceMeshFaceOval[j][0]], - landmarks[kFaceMeshFaceOval[j][1]], kFaceContourColor, - draw_line_width); + landmarks[kFaceMeshFaceOval[j][1]], kFaceOvalColor, + draw_line_width, cv::LINE_AA); } for (int j = 0; j < 40; ++j) { cv::line(*image, landmarks[kFaceMeshLips[j][0]], - landmarks[kFaceMeshLips[j][1]], kFaceContourColor, - draw_line_width); + landmarks[kFaceMeshLips[j][1]], kLipsColor, draw_line_width, + cv::LINE_AA); } for (int j = 0; j < 16; ++j) { cv::line(*image, landmarks[kFaceMeshLeftEye[j][0]], - landmarks[kFaceMeshLeftEye[j][1]], kLeftEyeColor, draw_line_width); + landmarks[kFaceMeshLeftEye[j][1]], kLeftEyeColor, draw_line_width, + cv::LINE_AA); } for (int j = 0; j < 8; ++j) { cv::line(*image, landmarks[kFaceMeshLeftEyebrow[j][0]], - landmarks[kFaceMeshLeftEyebrow[j][1]], kLeftEyeColor, - draw_line_width); + landmarks[kFaceMeshLeftEyebrow[j][1]], kLeftEyebrowColor, + draw_line_width, cv::LINE_AA); } for (int j = 0; j < 4; ++j) { cv::line(*image, landmarks[kFaceMeshLeftIris[j][0]], - landmarks[kFaceMeshLeftIris[j][1]], kLeftEyeColor, - draw_line_width); + landmarks[kFaceMeshLeftIris[j][1]], kLeftEyeIrisColor, + draw_line_width, cv::LINE_AA); } for (int j = 0; j < 16; ++j) { cv::line(*image, landmarks[kFaceMeshRightEye[j][0]], landmarks[kFaceMeshRightEye[j][1]], kRightEyeColor, - draw_line_width); + draw_line_width, cv::LINE_AA); } for (int j = 0; j < 8; ++j) { cv::line(*image, landmarks[kFaceMeshRightEyebrow[j][0]], - landmarks[kFaceMeshRightEyebrow[j][1]], kRightEyeColor, - draw_line_width); + landmarks[kFaceMeshRightEyebrow[j][1]], kRightEyebrowColor, + draw_line_width, cv::LINE_AA); } for (int j = 0; j < 4; ++j) { cv::line(*image, landmarks[kFaceMeshRightIris[j][0]], - landmarks[kFaceMeshRightIris[j][1]], kRightEyeColor, - draw_line_width); + landmarks[kFaceMeshRightIris[j][1]], kRightEyeIrisColor, + draw_line_width, cv::LINE_AA); + } + + if (draw_nose) { + for (int j = 0; j < 25; ++j) { + cv::line(*image, landmarks[kFaceMeshNose[j][0]], + landmarks[kFaceMeshNose[j][1]], kNoseColor, draw_line_width, + cv::LINE_AA); + } } } } // namespace mediapipe diff --git a/mediapipe/util/pose_util.h b/mediapipe/util/pose_util.h index ed271e2e..aeb2b922 100644 --- a/mediapipe/util/pose_util.h +++ b/mediapipe/util/pose_util.h @@ -23,8 +23,10 @@ namespace mediapipe { void DrawPose(const mediapipe::NormalizedLandmarkList& pose, bool flip_y, cv::Mat* image); -void DrawFace(const mediapipe::NormalizedLandmarkList& face, bool flip_y, - cv::Mat* image); +void DrawFace(const mediapipe::NormalizedLandmarkList& face, + const std::pair& image_size, const cv::Mat& affine, + bool flip_y, bool draw_nose, int color_style, bool reverse_color, + int draw_line_width, cv::Mat* image); } // namespace mediapipe diff --git a/mediapipe/util/resource_cache.h b/mediapipe/util/resource_cache.h index 2b3ccbc7..517182f1 100644 --- a/mediapipe/util/resource_cache.h +++ b/mediapipe/util/resource_cache.h @@ -19,6 +19,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/functional/function_ref.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { @@ -40,10 +41,10 @@ class ResourceCache { std::tie(map_it, std::ignore) = map_.try_emplace(key, std::make_unique(key)); entry = map_it->second.get(); - CHECK_EQ(entry->request_count, 0); + ABSL_CHECK_EQ(entry->request_count, 0); entry->request_count = 1; entry_list_.Append(entry); - if (entry->prev != nullptr) CHECK_GE(entry->prev->request_count, 1); + if (entry->prev != nullptr) ABSL_CHECK_GE(entry->prev->request_count, 1); } else { entry = map_it->second.get(); ++entry->request_count; diff --git a/mediapipe/util/resource_util_android.cc b/mediapipe/util/resource_util_android.cc index 1e970f21..8678b973 100644 --- a/mediapipe/util/resource_util_android.cc +++ b/mediapipe/util/resource_util_android.cc @@ -14,6 +14,7 @@ #include +#include "absl/log/absl_log.h" #include "absl/strings/match.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/ret_check.h" @@ -36,7 +37,7 @@ absl::Status DefaultGetResourceContents(const std::string& path, std::string* output, bool read_as_binary) { if (!read_as_binary) { - LOG(WARNING) + ABSL_LOG(WARNING) << "Setting \"read_as_binary\" to false is a no-op on Android."; } if (absl::StartsWith(path, "/")) { @@ -74,7 +75,7 @@ absl::StatusOr PathToResourceAsFile(const std::string& path) { { auto status_or_path = PathToResourceAsFileInternal(path); if (status_or_path.ok()) { - LOG(INFO) << "Successfully loaded: " << path; + ABSL_LOG(INFO) << "Successfully loaded: " << path; return status_or_path; } } @@ -87,7 +88,7 @@ absl::StatusOr PathToResourceAsFile(const std::string& path) { auto base_name = path.substr(last_slash_idx + 1); auto status_or_path = PathToResourceAsFileInternal(base_name); if (status_or_path.ok()) { - LOG(INFO) << "Successfully loaded: " << base_name; + ABSL_LOG(INFO) << "Successfully loaded: " << base_name; return status_or_path; } } diff --git a/mediapipe/util/resource_util_apple.cc b/mediapipe/util/resource_util_apple.cc index f6471834..d6ca2c36 100644 --- a/mediapipe/util/resource_util_apple.cc +++ b/mediapipe/util/resource_util_apple.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/log/absl_log.h" #include "absl/strings/match.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/ret_check.h" @@ -46,7 +47,8 @@ absl::Status DefaultGetResourceContents(const std::string& path, std::string* output, bool read_as_binary) { if (!read_as_binary) { - LOG(WARNING) << "Setting \"read_as_binary\" to false is a no-op on ios."; + ABSL_LOG(WARNING) + << "Setting \"read_as_binary\" to false is a no-op on ios."; } ASSIGN_OR_RETURN(std::string full_path, PathToResourceAsFile(path)); return file::GetContents(full_path, output, read_as_binary); @@ -63,7 +65,7 @@ absl::StatusOr PathToResourceAsFile(const std::string& path) { { auto status_or_path = PathToResourceAsFileInternal(path); if (status_or_path.ok()) { - LOG(INFO) << "Successfully loaded: " << path; + ABSL_LOG(INFO) << "Successfully loaded: " << path; return status_or_path; } } @@ -76,7 +78,7 @@ absl::StatusOr PathToResourceAsFile(const std::string& path) { auto base_name = path.substr(last_slash_idx + 1); auto status_or_path = PathToResourceAsFileInternal(base_name); if (status_or_path.ok()) { - LOG(INFO) << "Successfully loaded: " << base_name; + ABSL_LOG(INFO) << "Successfully loaded: " << base_name; return status_or_path; } } @@ -90,7 +92,7 @@ absl::StatusOr PathToResourceAsFile(const std::string& path) { if ([[NSFileManager defaultManager] fileExistsAtPath:[NSString stringWithUTF8String:test_path.c_str()]]) { - LOG(INFO) << "Successfully loaded: " << test_path; + ABSL_LOG(INFO) << "Successfully loaded: " << test_path; return test_path; } } diff --git a/mediapipe/util/sequence/BUILD b/mediapipe/util/sequence/BUILD index ac7c2ba5..23858304 100644 --- a/mediapipe/util/sequence/BUILD +++ b/mediapipe/util/sequence/BUILD @@ -13,6 +13,9 @@ # limitations under the License. # +# Placeholder: load py_library +# Placeholder: load py_test + licenses(["notice"]) package(default_visibility = ["//visibility:private"]) @@ -28,6 +31,7 @@ cc_library( "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -47,6 +51,7 @@ cc_library( "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:protos_all_cc", @@ -72,7 +77,6 @@ cc_test( "//mediapipe/framework/formats:location", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:opencv_imgcodecs", - "//mediapipe/framework/port:status", "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) diff --git a/mediapipe/util/sequence/README.md b/mediapipe/util/sequence/README.md index e5b5ed91..960a0d9b 100644 --- a/mediapipe/util/sequence/README.md +++ b/mediapipe/util/sequence/README.md @@ -555,9 +555,9 @@ without timestamps, use the `context`. |`PREFIX/feature/dimensions`|context int list|`set_feature_dimensions` / `SetFeatureDimensions`|A list of integer dimensions for each feature.| |`PREFIX/feature/rate`|context float|`set_feature_rate` / `SetFeatureRate`|The rate that features are calculated as features per second.| |`PREFIX/feature/bytes/format`|context bytes|`set_feature_bytes_format` / `SetFeatureBytesFormat`|The encoding format if any for features stored as bytes.| -|`PREFIX/context_feature/floats`|context float list|`add_context_feature_floats` / `AddContextFeatureFloats`|A list of floats for the entire example.| -|`PREFIX/context_feature/bytes`|context bytes list|`add_context_feature_bytes` / `AddContextFeatureBytes`|A list of bytes for the entire example. Maybe be encoded.| -|`PREFIX/context_feature/ints`|context int list|`add_context_feature_ints` / `AddContextFeatureInts`|A list of ints for the entire example.| +|`PREFIX/context_feature/floats`|context float list|`set_context_feature_floats` / `AddContextFeatureFloats`|A list of floats for the entire example.| +|`PREFIX/context_feature/bytes`|context bytes list|`set_context_feature_bytes` / `AddContextFeatureBytes`|A list of bytes for the entire example. Maybe be encoded.| +|`PREFIX/context_feature/ints`|context int list|`set_context_feature_ints` / `AddContextFeatureInts`|A list of ints for the entire example.| ### Keys related to audio Audio is a special subtype of generic features with additional data about the @@ -593,6 +593,8 @@ ground truth transcripts. |-----|------|------------------------|-------------| |`text/language`|context bytes|`set_text_langage` / `SetTextLanguage`|The language for the corresponding text.| |`text/context/content`|context bytes|`set_text_context_content` / `SetTextContextContent`|Storage for large blocks of text in the context.| +|`text/context/token_id`|context int list|`set_text_context_token_id` / `SetTextContextTokenId`|Storage for large blocks of text in the context as token ids.| +|`text/context/embedding`|context float list|`set_text_context_embedding` / `SetTextContextEmbedding`|Storage for large blocks of text in the context as embeddings.| |`text/content`|feature list bytes|`add_text_content` / `AddTextContent`|One (or a few) text tokens that occur at one timestamp.| |`text/timestamp`|feature list int|`add_text_timestamp` / `AddTextTimestamp`|When a text token occurs in microseconds.| |`text/duration`|feature list int|`add_text_duration` / `SetTextDuration`|The duration in microseconds for the corresponding text tokens.| diff --git a/mediapipe/util/sequence/media_sequence.cc b/mediapipe/util/sequence/media_sequence.cc index 287db618..9cff193c 100644 --- a/mediapipe/util/sequence/media_sequence.cc +++ b/mediapipe/util/sequence/media_sequence.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_split.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h" #include "mediapipe/framework/port/ret_check.h" @@ -147,6 +148,22 @@ absl::Status ReconcileMetadataImages(const std::string& prefix, return absl::OkStatus(); } +// Reconciles metadata for all images. +absl::Status ReconcileMetadataImages(tensorflow::SequenceExample* sequence) { + RET_CHECK_OK(ReconcileMetadataImages("", sequence)); + for (const auto& key_value : sequence->feature_lists().feature_list()) { + const auto& key = key_value.first; + if (::absl::StrContains(key, kImageTimestampKey)) { + std::string prefix = ""; + if (key != kImageTimestampKey) { + prefix = key.substr(0, key.size() - sizeof(kImageTimestampKey)); + } + RET_CHECK_OK(ReconcileMetadataImages(prefix, sequence)); + } + } + return absl::OkStatus(); +} + // Sets the values of "feature/${TAG}/dimensions", and // "feature/${TAG}/frame_rate" for each float list feature TAG. If the // dimensions are already present as a context feature, this method verifies @@ -514,11 +531,11 @@ std::unique_ptr GetAudioFromFeatureAt( const std::string& prefix, const tensorflow::SequenceExample& sequence, int index) { const auto& flat_data = GetFeatureFloatsAt(prefix, sequence, index); - CHECK(HasFeatureNumChannels(prefix, sequence)) + ABSL_CHECK(HasFeatureNumChannels(prefix, sequence)) << "GetAudioAt requires num_channels context to be specified as key: " << merge_prefix(prefix, kFeatureNumChannelsKey); int num_channels = GetFeatureNumChannels(prefix, sequence); - CHECK_EQ(flat_data.size() % num_channels, 0) + ABSL_CHECK_EQ(flat_data.size() % num_channels, 0) << "The data size is not a multiple of the number of channels: " << flat_data.size() << " % " << num_channels << " = " << flat_data.size() % num_channels << " for sequence index " << index; @@ -545,10 +562,7 @@ absl::Status ReconcileMetadata(bool reconcile_bbox_annotations, bool reconcile_region_annotations, tensorflow::SequenceExample* sequence) { RET_CHECK_OK(ReconcileAnnotationIndicesByImageTimestamps(sequence)); - RET_CHECK_OK(ReconcileMetadataImages("", sequence)); - RET_CHECK_OK(ReconcileMetadataImages(kForwardFlowPrefix, sequence)); - RET_CHECK_OK(ReconcileMetadataImages(kClassSegmentationPrefix, sequence)); - RET_CHECK_OK(ReconcileMetadataImages(kInstanceSegmentationPrefix, sequence)); + RET_CHECK_OK(ReconcileMetadataImages(sequence)); RET_CHECK_OK(ReconcileMetadataFeatureFloats(sequence)); if (reconcile_bbox_annotations) { RET_CHECK_OK(ReconcileMetadataBoxAnnotations("", sequence)); diff --git a/mediapipe/util/sequence/media_sequence.h b/mediapipe/util/sequence/media_sequence.h index 620d6d48..e4bfcf5a 100644 --- a/mediapipe/util/sequence/media_sequence.h +++ b/mediapipe/util/sequence/media_sequence.h @@ -634,6 +634,10 @@ PREFIXED_IMAGE(InstanceSegmentation, kInstanceSegmentationPrefix); const char kTextLanguageKey[] = "text/language"; // A large block of text that applies to the media. const char kTextContextContentKey[] = "text/context/content"; +// A large block of text that applies to the media as token ids. +const char kTextContextTokenIdKey[] = "text/context/token_id"; +// A large block of text that applies to the media as embeddings. +const char kTextContextEmbeddingKey[] = "text/context/embedding"; // Feature list keys: // The text contents for a given time. @@ -651,6 +655,8 @@ const char kTextTokenIdKey[] = "text/token/id"; BYTES_CONTEXT_FEATURE(TextLanguage, kTextLanguageKey); BYTES_CONTEXT_FEATURE(TextContextContent, kTextContextContentKey); +VECTOR_INT64_CONTEXT_FEATURE(TextContextTokenId, kTextContextTokenIdKey); +VECTOR_FLOAT_CONTEXT_FEATURE(TextContextEmbedding, kTextContextEmbeddingKey); BYTES_FEATURE_LIST(TextContent, kTextContentKey); INT64_FEATURE_LIST(TextTimestamp, kTextTimestampKey); INT64_FEATURE_LIST(TextDuration, kTextDurationKey); diff --git a/mediapipe/util/sequence/media_sequence.py b/mediapipe/util/sequence/media_sequence.py index 1b96383d..e87d8c21 100644 --- a/mediapipe/util/sequence/media_sequence.py +++ b/mediapipe/util/sequence/media_sequence.py @@ -601,6 +601,10 @@ _create_image_with_prefix("instance_segmentation", INSTANCE_SEGMENTATION_PREFIX) TEXT_LANGUAGE_KEY = "text/language" # A large block of text that applies to the media. TEXT_CONTEXT_CONTENT_KEY = "text/context/content" +# A large block of text that applies to the media as token ids. +TEXT_CONTEXT_TOKEN_ID_KEY = "text/context/token_id" +# A large block of text that applies to the media as embeddings. +TEXT_CONTEXT_EMBEDDING_KEY = "text/context/embedding" # The text contents for a given time. TEXT_CONTENT_KEY = "text/content" @@ -619,6 +623,10 @@ msu.create_bytes_context_feature( "text_language", TEXT_LANGUAGE_KEY, module_dict=globals()) msu.create_bytes_context_feature( "text_context_content", TEXT_CONTEXT_CONTENT_KEY, module_dict=globals()) +msu.create_int_list_context_feature( + "text_context_token_id", TEXT_CONTEXT_TOKEN_ID_KEY, module_dict=globals()) +msu.create_float_list_context_feature( + "text_context_embedding", TEXT_CONTEXT_EMBEDDING_KEY, module_dict=globals()) msu.create_bytes_feature_list( "text_content", TEXT_CONTENT_KEY, module_dict=globals()) msu.create_int_feature_list( diff --git a/mediapipe/util/sequence/media_sequence_test.cc b/mediapipe/util/sequence/media_sequence_test.cc index e220eace..17365fae 100644 --- a/mediapipe/util/sequence/media_sequence_test.cc +++ b/mediapipe/util/sequence/media_sequence_test.cc @@ -16,6 +16,7 @@ #include #include +#include #include "mediapipe/framework/formats/location.h" #include "mediapipe/framework/port/gmock.h" @@ -711,6 +712,30 @@ TEST(MediaSequenceTest, RoundTripTextContextContent) { ASSERT_FALSE(HasTextContextContent(sequence)); } +TEST(MediaSequenceTest, RoundTripTextContextTokenId) { + tensorflow::SequenceExample sequence; + ASSERT_FALSE(HasTextContextTokenId(sequence)); + std::vector vi = {47, 35}; + SetTextContextTokenId(vi, &sequence); + ASSERT_TRUE(HasTextContextTokenId(sequence)); + ASSERT_EQ(GetTextContextTokenId(sequence).size(), vi.size()); + ASSERT_EQ(GetTextContextTokenId(sequence)[1], vi[1]); + ClearTextContextTokenId(&sequence); + ASSERT_FALSE(HasTextContextTokenId(sequence)); +} + +TEST(MediaSequenceTest, RoundTripTextContextEmbedding) { + tensorflow::SequenceExample sequence; + ASSERT_FALSE(HasTextContextEmbedding(sequence)); + std::vector vi = {47., 35.}; + SetTextContextEmbedding(vi, &sequence); + ASSERT_TRUE(HasTextContextEmbedding(sequence)); + ASSERT_EQ(GetTextContextEmbedding(sequence).size(), vi.size()); + ASSERT_EQ(GetTextContextEmbedding(sequence)[1], vi[1]); + ClearTextContextEmbedding(&sequence); + ASSERT_FALSE(HasTextContextEmbedding(sequence)); +} + TEST(MediaSequenceTest, RoundTripTextContent) { tensorflow::SequenceExample sequence; std::vector text = {"test", "again"}; diff --git a/mediapipe/util/sequence/media_sequence_test.py b/mediapipe/util/sequence/media_sequence_test.py index 5a5c61c7..5c4ff382 100644 --- a/mediapipe/util/sequence/media_sequence_test.py +++ b/mediapipe/util/sequence/media_sequence_test.py @@ -129,6 +129,8 @@ class MediaSequenceTest(tf.test.TestCase): ms.add_bbox_embedding_confidence((0.47, 0.49), example) ms.set_text_language(b"test", example) ms.set_text_context_content(b"text", example) + ms.set_text_context_token_id([47, 49], example) + ms.set_text_context_embedding([0.47, 0.49], example) ms.add_text_content(b"one", example) ms.add_text_timestamp(47, example) ms.add_text_confidence(0.47, example) @@ -260,6 +262,29 @@ class MediaSequenceTest(tf.test.TestCase): self.assertFalse(ms.has_feature_dimensions(example, "1")) self.assertFalse(ms.has_feature_dimensions(example, "2")) + def test_text_context_round_trip(self): + example = tf.train.SequenceExample() + text_content = b"text content" + text_token_ids = np.array([1, 2, 3, 4]) + text_embeddings = np.array([0.1, 0.2, 0.3, 0.4]) + self.assertFalse(ms.has_text_context_embedding(example)) + self.assertFalse(ms.has_text_context_token_id(example)) + self.assertFalse(ms.has_text_context_content(example)) + ms.set_text_context_content(text_content, example) + ms.set_text_context_token_id(text_token_ids, example) + ms.set_text_context_embedding(text_embeddings, example) + self.assertEqual(text_content, ms.get_text_context_content(example)) + self.assertAllClose(text_token_ids, ms.get_text_context_token_id(example)) + self.assertAllClose(text_embeddings, ms.get_text_context_embedding(example)) + self.assertTrue(ms.has_text_context_embedding(example)) + self.assertTrue(ms.has_text_context_token_id(example)) + self.assertTrue(ms.has_text_context_content(example)) + ms.clear_text_context_content(example) + ms.clear_text_context_token_id(example) + ms.clear_text_context_embedding(example) + self.assertFalse(ms.has_text_context_embedding(example)) + self.assertFalse(ms.has_text_context_token_id(example)) + self.assertFalse(ms.has_text_context_content(example)) if __name__ == "__main__": tf.test.main() diff --git a/mediapipe/util/sequence/media_sequence_util.h b/mediapipe/util/sequence/media_sequence_util.h index 1737f91a..5b765f13 100644 --- a/mediapipe/util/sequence/media_sequence_util.h +++ b/mediapipe/util/sequence/media_sequence_util.h @@ -92,6 +92,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/proto_ns.h" @@ -124,7 +125,7 @@ inline const tensorflow::Feature& GetContext( // proto map's at function also checks whether key is present, but it doesn't // print the missing key when it check-fails. const auto it = sequence.context().feature().find(key); - CHECK(it != sequence.context().feature().end()) + ABSL_CHECK(it != sequence.context().feature().end()) << "Could not find context key " << key << ". Sequence: \n" << sequence.DebugString(); return it->second; @@ -220,7 +221,7 @@ inline const proto_ns::RepeatedField& GetFloatsAt( const tensorflow::SequenceExample& sequence, const std::string& key, const int index) { const tensorflow::FeatureList& fl = GetFeatureList(sequence, key); - CHECK_LT(index, fl.feature_size()) + ABSL_CHECK_LT(index, fl.feature_size()) << "Sequence: \n " << sequence.DebugString(); return fl.feature().Get(index).float_list().value(); } @@ -231,7 +232,7 @@ inline const proto_ns::RepeatedField& GetInt64sAt( const tensorflow::SequenceExample& sequence, const std::string& key, const int index) { const tensorflow::FeatureList& fl = GetFeatureList(sequence, key); - CHECK_LT(index, fl.feature_size()) + ABSL_CHECK_LT(index, fl.feature_size()) << "Sequence: \n " << sequence.DebugString(); return fl.feature().Get(index).int64_list().value(); } @@ -242,7 +243,7 @@ inline const proto_ns::RepeatedPtrField& GetBytesAt( const tensorflow::SequenceExample& sequence, const std::string& key, const int index) { const tensorflow::FeatureList& fl = GetFeatureList(sequence, key); - CHECK_LT(index, fl.feature_size()) + ABSL_CHECK_LT(index, fl.feature_size()) << "Sequence: \n " << sequence.DebugString(); return fl.feature().Get(index).bytes_list().value(); } diff --git a/mediapipe/util/tflite/BUILD b/mediapipe/util/tflite/BUILD index f31c2369..b34d0e08 100644 --- a/mediapipe/util/tflite/BUILD +++ b/mediapipe/util/tflite/BUILD @@ -49,6 +49,7 @@ cc_library_with_tflite( "//mediapipe/util/tflite/operations:transform_landmarks", "//mediapipe/util/tflite/operations:transform_tensor_bilinear", "//mediapipe/util/tflite/operations:transpose_conv_bias", + "@com_google_absl//absl/log:absl_check", "@org_tensorflow//tensorflow/lite:builtin_op_data", ], # For using the symbol `MediaPipe_RegisterTfLiteOpResolver` in Python @@ -100,6 +101,7 @@ cc_library( "//mediapipe:ios": [], "//mediapipe:macos": [], "//conditions:default": [ + "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", diff --git a/mediapipe/util/tflite/cpu_op_resolver.cc b/mediapipe/util/tflite/cpu_op_resolver.cc index 588a237b..3b5ab308 100644 --- a/mediapipe/util/tflite/cpu_op_resolver.cc +++ b/mediapipe/util/tflite/cpu_op_resolver.cc @@ -14,6 +14,7 @@ #include "mediapipe/util/tflite/cpu_op_resolver.h" +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/util/tflite/operations/landmarks_to_transform_matrix.h" #include "mediapipe/util/tflite/operations/max_pool_argmax.h" @@ -27,7 +28,7 @@ namespace mediapipe { void MediaPipe_RegisterTfLiteOpResolver(tflite::MutableOpResolver *resolver) { - CHECK(resolver != nullptr); + ABSL_CHECK(resolver != nullptr); resolver->AddCustom("MaxPoolingWithArgmax2D", tflite_operations::RegisterMaxPoolingWithArgmax2D()); resolver->AddCustom("MaxUnpooling2D", diff --git a/mediapipe/util/tflite/op_resolver.cc b/mediapipe/util/tflite/op_resolver.cc index 44eff456..dc872833 100644 --- a/mediapipe/util/tflite/op_resolver.cc +++ b/mediapipe/util/tflite/op_resolver.cc @@ -58,7 +58,8 @@ TfLiteRegistration* RegisterMaxPoolingWithArgmax2D() { }); return r; }(); - static TfLiteRegistration reg = {.registration_external = reg_external}; + static TfLiteRegistration reg{}; + reg.registration_external = reg_external; return ® } @@ -68,7 +69,8 @@ TfLiteRegistration* RegisterMaxUnpooling2D() { TfLiteRegistrationExternalCreate(kTfLiteBuiltinCustom, kMaxUnpooling2DOpName, kMaxUnpooling2DOpVersion); - static TfLiteRegistration reg = {.registration_external = reg_external}; + static TfLiteRegistration reg{}; + reg.registration_external = reg_external; return ® } @@ -78,7 +80,8 @@ TfLiteRegistration* RegisterConvolution2DTransposeBias() { TfLiteRegistrationExternalCreate(kTfLiteBuiltinCustom, kConvolution2DTransposeBiasOpName, kConvolution2DTransposeBiasOpVersion); - static TfLiteRegistration reg = {.registration_external = reg_external}; + static TfLiteRegistration reg{}; + reg.registration_external = reg_external; return ® } diff --git a/mediapipe/util/tflite/tflite_gpu_runner.cc b/mediapipe/util/tflite/tflite_gpu_runner.cc index 4e40975c..6a132ac6 100644 --- a/mediapipe/util/tflite/tflite_gpu_runner.cc +++ b/mediapipe/util/tflite/tflite_gpu_runner.cc @@ -21,6 +21,7 @@ #include "absl/status/status.h" #include "absl/strings/substitute.h" #include "mediapipe/framework/port/canonical_errors.h" +#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_macros.h" @@ -234,6 +235,11 @@ absl::Status TFLiteGPURunner::InitializeOpenCL( MP_RETURN_IF_ERROR( cl::NewInferenceEnvironment(env_options, &cl_environment_, &properties)); + if (serialized_model_.empty() && + opencl_init_from_serialized_model_is_forced_) { + ASSIGN_OR_RETURN(serialized_model_, GetSerializedModel()); + } + // Try to initialize from serialized model first. if (!serialized_model_.empty()) { absl::Status init_status = InitializeOpenCLFromSerializedModel(builder); @@ -270,7 +276,6 @@ absl::Status TFLiteGPURunner::InitializeOpenCLFromSerializedModel( } absl::StatusOr> TFLiteGPURunner::GetSerializedModel() { - RET_CHECK(runner_) << "Runner is in invalid state."; if (serialized_model_used_) { return serialized_model_; } diff --git a/mediapipe/util/tflite/tflite_gpu_runner.h b/mediapipe/util/tflite/tflite_gpu_runner.h index 5eeaa230..c64981ef 100644 --- a/mediapipe/util/tflite/tflite_gpu_runner.h +++ b/mediapipe/util/tflite/tflite_gpu_runner.h @@ -62,6 +62,9 @@ class TFLiteGPURunner { void ForceOpenGL() { opengl_is_forced_ = true; } void ForceOpenCL() { opencl_is_forced_ = true; } + void ForceOpenCLInitFromSerializedModel() { + opencl_init_from_serialized_model_is_forced_ = true; + } absl::Status BindSSBOToInputTensor(GLuint ssbo_id, int input_id); absl::Status BindSSBOToOutputTensor(GLuint ssbo_id, int output_id); @@ -141,6 +144,7 @@ class TFLiteGPURunner { bool opencl_is_forced_ = false; bool opengl_is_forced_ = false; + bool opencl_init_from_serialized_model_is_forced_ = false; }; } // namespace gpu diff --git a/mediapipe/util/time_series_test_util.h b/mediapipe/util/time_series_test_util.h index 7e31aeff..50fe3260 100644 --- a/mediapipe/util/time_series_test_util.h +++ b/mediapipe/util/time_series_test_util.h @@ -20,6 +20,8 @@ #include #include "Eigen/Core" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "absl/strings/substitute.h" @@ -139,7 +141,7 @@ class TimeSeriesCalculatorTest : public ::testing::Test { // _, _, etc. std::vector MakeNames(const std::vector& base_names, const std::vector& ids) { - CHECK_EQ(base_names.size(), ids.size()); + ABSL_CHECK_EQ(base_names.size(), ids.size()); std::vector names; for (int i = 0; i < base_names.size(); ++i) { const std::string name_template = R"($0_$1)"; @@ -186,7 +188,8 @@ class TimeSeriesCalculatorTest : public ::testing::Test { void InitializeGraph(const CalculatorOptions& options) { if (num_external_inputs_ != -1) { - LOG(WARNING) << "Use num_side_packets_ instead of num_external_inputs_."; + ABSL_LOG(WARNING) + << "Use num_side_packets_ instead of num_external_inputs_."; num_side_packets_ = num_external_inputs_; } diff --git a/mediapipe/util/time_series_util.cc b/mediapipe/util/time_series_util.cc index 87f69475..f978280a 100644 --- a/mediapipe/util/time_series_util.cc +++ b/mediapipe/util/time_series_util.cc @@ -19,10 +19,11 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/time_series_header.pb.h" -#include "mediapipe/framework/port/logging.h" namespace mediapipe { namespace time_series_util { @@ -36,8 +37,8 @@ bool LogWarningIfTimestampIsInconsistent(const Timestamp& current_timestamp, // Don't accept other special timestamp values. We may need to change this // depending on how they're used in practice. if (!current_timestamp.IsRangeValue()) { - LOG(WARNING) << "Unexpected special timestamp: " - << current_timestamp.DebugString(); + ABSL_LOG(WARNING) << "Unexpected special timestamp: " + << current_timestamp.DebugString(); return false; } @@ -48,7 +49,7 @@ bool LogWarningIfTimestampIsInconsistent(const Timestamp& current_timestamp, initial_timestamp.Seconds() + cumulative_samples / sample_rate; if (fabs(current_timestamp.Seconds() - expected_timestamp_seconds) > 0.5 / sample_rate) { - LOG_EVERY_N(WARNING, 20) + ABSL_LOG_EVERY_N(WARNING, 20) << std::fixed << "Timestamp " << current_timestamp.Seconds() << " not consistent with number of samples " << cumulative_samples << " and initial timestamp " << initial_timestamp @@ -79,7 +80,7 @@ absl::Status IsTimeSeriesHeaderValid(const TimeSeriesHeader& header) { absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, TimeSeriesHeader* header) { - CHECK(header); + ABSL_CHECK(header); if (header_packet.IsEmpty()) { return tool::StatusFail("No header found."); } @@ -92,7 +93,7 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, absl::Status FillMultiStreamTimeSeriesHeaderIfValid( const Packet& header_packet, MultiStreamTimeSeriesHeader* header) { - CHECK(header); + ABSL_CHECK(header); if (header_packet.IsEmpty()) { return tool::StatusFail("No header found."); } @@ -127,7 +128,7 @@ int64_t SecondsToSamples(double time_in_seconds, double sample_rate) { } double SamplesToSeconds(int64_t num_samples, double sample_rate) { - DCHECK_NE(sample_rate, 0.0); + ABSL_DCHECK_NE(sample_rate, 0.0); return (num_samples / sample_rate); } diff --git a/mediapipe/util/time_series_util.h b/mediapipe/util/time_series_util.h index afa66acc..be5838df 100644 --- a/mediapipe/util/time_series_util.h +++ b/mediapipe/util/time_series_util.h @@ -25,7 +25,6 @@ #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/time_series_header.pb.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/status.h" namespace mediapipe { diff --git a/mediapipe/util/tracking/BUILD b/mediapipe/util/tracking/BUILD index 5a271ffa..96972398 100644 --- a/mediapipe/util/tracking/BUILD +++ b/mediapipe/util/tracking/BUILD @@ -143,6 +143,8 @@ cc_library( "//mediapipe/framework/port:singleton", "//mediapipe/framework/port:vector", "@com_google_absl//absl/container:node_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings:str_format", "@eigen_archive//:eigen3", ], @@ -156,6 +158,7 @@ cc_library( ":motion_models", ":motion_models_cc_proto", "//mediapipe/framework/port:opencv_core", + "@com_google_absl//absl/log:absl_check", ], ) @@ -169,10 +172,11 @@ cc_library( ":parallel_invoker", ":region_flow_cc_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:vector", "@com_google_absl//absl/container:node_hash_map", "@com_google_absl//absl/container:node_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) @@ -186,6 +190,8 @@ cc_library( ":motion_models", ":region_flow", ":region_flow_cc_proto", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings:str_format", ], ) @@ -204,7 +210,8 @@ cc_library( hdrs = ["measure_time.h"], deps = [ "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/time", @@ -219,8 +226,9 @@ cc_library( linkopts = PARALLEL_LINKOPTS, deps = [ ":parallel_invoker_forbid_mixed_active", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:threadpool", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/synchronization", ], ) @@ -241,10 +249,11 @@ cc_library( ":motion_models_cc_proto", ":region_flow", ":region_flow_cc_proto", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -253,9 +262,10 @@ cc_library( srcs = ["streaming_buffer.cc"], hdrs = ["streaming_buffer.h"], deps = [ - "//mediapipe/framework/port:logging", "//mediapipe/framework/tool:type_util", "@com_google_absl//absl/container:node_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:any", ], @@ -278,10 +288,11 @@ cc_library( ":region_flow", ":region_flow_cc_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:vector", "@com_google_absl//absl/container:node_hash_map", "@com_google_absl//absl/container:node_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@eigen_archive//:eigen3", ], @@ -297,8 +308,9 @@ cc_library( ":motion_saliency_cc_proto", ":region_flow", ":region_flow_cc_proto", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -309,8 +321,9 @@ cc_library( ":image_util", ":push_pull_filtering_cc_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -321,9 +334,10 @@ cc_library( deps = [ ":tone_models_cc_proto", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings:str_format", ], ) @@ -344,6 +358,8 @@ cc_library( "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", ], ) @@ -378,6 +394,8 @@ cc_library( "//mediapipe/framework/port:vector", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:node_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@eigen_archive//:eigen3", ], @@ -398,6 +416,7 @@ cc_library( "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings", ], ) @@ -424,10 +443,11 @@ cc_library( ":region_flow_visualization", ":streaming_buffer", "//mediapipe/framework/port:integral_types", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:vector", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings:str_format", ], ) @@ -450,6 +470,8 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:vector", "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", ], ) @@ -476,6 +498,8 @@ cc_library( "//mediapipe/framework/port:vector", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@eigen_archive//:eigen3", ], @@ -495,6 +519,8 @@ cc_library( "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", "//mediapipe/framework/port:threadpool", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", @@ -519,6 +545,8 @@ cc_library( "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_video", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/synchronization", ], @@ -536,6 +564,8 @@ cc_library( ":tracking_cc_proto", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/strings:str_format", ], ) @@ -589,13 +619,14 @@ cc_test( "//mediapipe/framework/deps:file_path", "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/time", ], ) diff --git a/mediapipe/util/tracking/box_detector.cc b/mediapipe/util/tracking/box_detector.cc index 58d85553..e477d7cd 100644 --- a/mediapipe/util/tracking/box_detector.cc +++ b/mediapipe/util/tracking/box_detector.cc @@ -16,6 +16,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/port/opencv_calib3d_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" @@ -43,10 +45,10 @@ void ScaleBox(float scale_x, float scale_y, TimedBoxProto *box) { } cv::Mat ConvertDescriptorsToMat(const std::vector &descriptors) { - CHECK(!descriptors.empty()) << "empty descriptors."; + ABSL_CHECK(!descriptors.empty()) << "empty descriptors."; const int descriptors_dims = descriptors[0].size(); - CHECK_GT(descriptors_dims, 0); + ABSL_CHECK_GT(descriptors_dims, 0); cv::Mat mat(descriptors.size(), descriptors_dims, CV_8U); @@ -59,13 +61,13 @@ cv::Mat ConvertDescriptorsToMat(const std::vector &descriptors) { cv::Mat GetDescriptorsWithIndices(const cv::Mat &frame_descriptors, const std::vector &indices) { - CHECK_GT(frame_descriptors.rows, 0); + ABSL_CHECK_GT(frame_descriptors.rows, 0); const int num_inlier_descriptors = indices.size(); - CHECK_GT(num_inlier_descriptors, 0); + ABSL_CHECK_GT(num_inlier_descriptors, 0); const int descriptors_dims = frame_descriptors.cols; - CHECK_GT(descriptors_dims, 0); + ABSL_CHECK_GT(descriptors_dims, 0); cv::Mat mat(num_inlier_descriptors, descriptors_dims, CV_32F); @@ -97,7 +99,7 @@ std::unique_ptr BoxDetectorInterface::Create( if (options.index_type() == BoxDetectorOptions::OPENCV_BF) { return absl::make_unique(options); } else { - LOG(FATAL) << "index type undefined."; + ABSL_LOG(FATAL) << "index type undefined."; } } @@ -186,7 +188,8 @@ void BoxDetectorInterface::DetectAndAddBox( if (features_from_tracking_data.empty() || descriptors_from_tracking_data.empty()) { - LOG(WARNING) << "Detection skipped due to empty features or descriptors."; + ABSL_LOG(WARNING) + << "Detection skipped due to empty features or descriptors."; return; } @@ -301,7 +304,7 @@ void BoxDetectorInterface::DetectAndAddBox( orb_extractor_->detect(resize_image, keypoints); orb_extractor_->compute(resize_image, keypoints, descriptors); - CHECK_EQ(keypoints.size(), descriptors.rows); + ABSL_CHECK_EQ(keypoints.size(), descriptors.rows); float inv_scale = 1.0f / std::max(resize_image.cols, resize_image.rows); std::vector v_keypoints(keypoints.size()); @@ -395,9 +398,9 @@ TimedBoxProtoList BoxDetectorInterface::FindQuadFromFeatureCorrespondence( TimedBoxProtoList result_list; if (matches.points_frame.size() != matches.points_index.size()) { - LOG(ERROR) << matches.points_frame.size() << " vs " - << matches.points_index.size() - << ". Correpondence size doesn't match."; + ABSL_LOG(ERROR) << matches.points_frame.size() << " vs " + << matches.points_index.size() + << ". Correpondence size doesn't match."; return result_list; } @@ -681,15 +684,15 @@ void BoxDetectorInterface::AddBoxDetectorIndex(const BoxDetectorIndex &index) { continue; } - CHECK_EQ(frame_entry.keypoints_size(), - frame_entry.descriptors_size() * 2); + ABSL_CHECK_EQ(frame_entry.keypoints_size(), + frame_entry.descriptors_size() * 2); const int num_features = frame_entry.descriptors_size(); - CHECK_GT(num_features, 0); + ABSL_CHECK_GT(num_features, 0); std::vector features(num_features); const int descriptors_dims = frame_entry.descriptors(0).data().size(); - CHECK_GT(descriptors_dims, 0); + ABSL_CHECK_GT(descriptors_dims, 0); cv::Mat descriptors_mat(num_features, descriptors_dims / sizeof(float), CV_32F); @@ -713,7 +716,7 @@ std::vector BoxDetectorOpencvBfImpl::MatchFeatureDescriptors( const std::vector &features, const cv::Mat &descriptors, int box_idx) { - CHECK_EQ(features.size(), descriptors.rows); + ABSL_CHECK_EQ(features.size(), descriptors.rows); std::vector correspondence_result( frame_box_[box_idx].size()); diff --git a/mediapipe/util/tracking/box_tracker.cc b/mediapipe/util/tracking/box_tracker.cc index 2d1af779..47986516 100644 --- a/mediapipe/util/tracking/box_tracker.cc +++ b/mediapipe/util/tracking/box_tracker.cc @@ -19,6 +19,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/synchronization/mutex.h" #include "absl/time/clock.h" @@ -36,8 +38,8 @@ static constexpr int kInitCheckpoint = -1; void MotionBoxStateQuadToVertices(const MotionBoxState::Quad& quad, std::vector* vertices) { - CHECK_EQ(TimedBox::kNumQuadVertices * 2, quad.vertices_size()); - CHECK(vertices != nullptr); + ABSL_CHECK_EQ(TimedBox::kNumQuadVertices * 2, quad.vertices_size()); + ABSL_CHECK(vertices != nullptr); vertices->clear(); for (int i = 0; i < TimedBox::kNumQuadVertices; ++i) { vertices->push_back( @@ -47,8 +49,8 @@ void MotionBoxStateQuadToVertices(const MotionBoxState::Quad& quad, void VerticesToMotionBoxStateQuad(const std::vector& vertices, MotionBoxState::Quad* quad) { - CHECK_EQ(TimedBox::kNumQuadVertices, vertices.size()); - CHECK(quad != nullptr); + ABSL_CHECK_EQ(TimedBox::kNumQuadVertices, vertices.size()); + ABSL_CHECK(quad != nullptr); for (const Vector2_f& vertex : vertices) { quad->add_vertices(vertex.x()); quad->add_vertices(vertex.y()); @@ -56,7 +58,7 @@ void VerticesToMotionBoxStateQuad(const std::vector& vertices, } void MotionBoxStateFromTimedBox(const TimedBox& box, MotionBoxState* state) { - CHECK(state); + ABSL_CHECK(state); state->set_pos_x(box.left); state->set_pos_y(box.top); state->set_width(box.right - box.left); @@ -90,7 +92,7 @@ void MotionBoxStateFromTimedBox(const TimedBox& box, MotionBoxState* state) { } void TimedBoxFromMotionBoxState(const MotionBoxState& state, TimedBox* box) { - CHECK(box); + ABSL_CHECK(box); const float scale_dx = state.width() * (state.scale() - 1.0f) * 0.5f; const float scale_dy = state.height() * (state.scale() - 1.0f) * 0.5f; box->left = state.pos_x() - scale_dx; @@ -113,7 +115,7 @@ namespace { TimedBox BlendTimedBoxes(const TimedBox& lhs, const TimedBox& rhs, int64_t time_msec) { - CHECK_LT(lhs.time_msec, rhs.time_msec); + ABSL_CHECK_LT(lhs.time_msec, rhs.time_msec); const double alpha = (time_msec - lhs.time_msec) * 1.0 / (rhs.time_msec - lhs.time_msec); return TimedBox::Blend(lhs, rhs, alpha); @@ -245,12 +247,12 @@ BoxTracker::BoxTracker( void BoxTracker::AddTrackingDataChunk(const TrackingDataChunk* chunk, bool copy_data) { - CHECK_GT(chunk->item_size(), 0) << "Empty chunk."; + ABSL_CHECK_GT(chunk->item_size(), 0) << "Empty chunk."; int64_t chunk_time_msec = chunk->item(0).timestamp_usec() / 1000; int chunk_idx = ChunkIdxFromTime(chunk_time_msec); - CHECK_GE(chunk_idx, tracking_data_.size()) << "Chunk is out of order."; + ABSL_CHECK_GE(chunk_idx, tracking_data_.size()) << "Chunk is out of order."; if (chunk_idx > tracking_data_.size()) { - LOG(INFO) << "Resize tracking_data_ to " << chunk_idx; + ABSL_LOG(INFO) << "Resize tracking_data_ to " << chunk_idx; tracking_data_.resize(chunk_idx); } if (copy_data) { @@ -278,7 +280,7 @@ void BoxTracker::NewBoxTrack(const TimedBox& initial_pos, int id, absl::MutexLock lock(&status_mutex_); if (canceling_) { - LOG(WARNING) << "Box Tracker is in cancel state. Refusing request."; + ABSL_LOG(WARNING) << "Box Tracker is in cancel state. Refusing request."; return; } ++track_status_[id][kInitCheckpoint].tracks_ongoing; @@ -319,8 +321,8 @@ void BoxTracker::NewBoxTrackAsync(const TimedBox& initial_pos, int id, if (!tracking_chunk.first) { absl::MutexLock lock(&status_mutex_); --track_status_[id][kInitCheckpoint].tracks_ongoing; - LOG(ERROR) << "Could not read tracking chunk from file: " << chunk_idx - << " for start position: " << initial_pos.ToString(); + ABSL_LOG(ERROR) << "Could not read tracking chunk from file: " << chunk_idx + << " for start position: " << initial_pos.ToString(); return; } @@ -485,12 +487,12 @@ void BoxTracker::CancelTracking(int id, int checkpoint) { bool BoxTracker::GetTimedPosition(int id, int64_t time_msec, TimedBox* result, std::vector* states) { - CHECK(result); + ABSL_CHECK(result); MotionBoxState* lhs_box_state = nullptr; MotionBoxState* rhs_box_state = nullptr; if (states) { - CHECK(options_.record_path_states()) + ABSL_CHECK(options_.record_path_states()) << "Requesting corresponding tracking states requires option " << "record_path_states to be set"; states->resize(1); @@ -502,7 +504,7 @@ bool BoxTracker::GetTimedPosition(int id, int64_t time_msec, TimedBox* result, absl::MutexLock lock(&path_mutex_); const Path& path = paths_[id]; if (path.empty()) { - LOG(ERROR) << "Empty path!"; + ABSL_LOG(ERROR) << "Empty path!"; return false; } @@ -586,7 +588,7 @@ BoxTracker::AugmentedChunkPtr BoxTracker::ReadChunk(int id, int checkpoint, if (chunk_idx < tracking_data_.size()) { return std::make_pair(tracking_data_[chunk_idx], false); } else { - LOG(ERROR) << "chunk_idx >= tracking_data_.size()"; + ABSL_LOG(ERROR) << "chunk_idx >= tracking_data_.size()"; return std::make_pair(nullptr, false); } } else { @@ -607,7 +609,7 @@ std::unique_ptr BoxTracker::ReadChunkFromCache( if (format_runtime) { chunk_file = cache_dir_ + "/" + absl::StrFormat(*format_runtime, chunk_idx); } else { - LOG(ERROR) << "chache_file_format wrong. fall back to chunk_%04d."; + ABSL_LOG(ERROR) << "chache_file_format wrong. fall back to chunk_%04d."; chunk_file = cache_dir_ + "/" + absl::StrFormat("chunk_%04d", chunk_idx); } @@ -625,7 +627,7 @@ std::unique_ptr BoxTracker::ReadChunkFromCache( std::ifstream in(chunk_file, std::ios::in | std::ios::binary); if (!in) { - LOG(ERROR) << "Could not read chunk file: " << chunk_file; + ABSL_LOG(ERROR) << "Could not read chunk file: " << chunk_file; return nullptr; } @@ -688,7 +690,7 @@ bool BoxTracker::WaitForChunkFile(int id, int checkpoint, int BoxTracker::ClosestFrameIndex(int64_t msec, const TrackingDataChunk& chunk) const { - CHECK_GT(chunk.item_size(), 0); + ABSL_CHECK_GT(chunk.item_size(), 0); typedef TrackingDataChunk::Item Item; Item item_to_find; item_to_find.set_timestamp_usec(msec * 1000); @@ -712,7 +714,8 @@ int BoxTracker::ClosestFrameIndex(int64_t msec, const int64_t rhs_diff = chunk.item(pos).timestamp_usec() / 1000 - msec; if (std::min(lhs_diff, rhs_diff) >= 67) { - LOG(ERROR) << "No frame found within 67ms, probably using wrong chunk."; + ABSL_LOG(ERROR) + << "No frame found within 67ms, probably using wrong chunk."; } if (lhs_diff < rhs_diff) { @@ -749,8 +752,8 @@ void BoxTracker::TrackingImpl(const TrackingImplArgs& a) { MotionBox motion_box(track_step_options); const int chunk_data_size = a.chunk_data->item_size(); - CHECK_GE(a.start_frame, 0); - CHECK_LT(a.start_frame, chunk_data_size); + ABSL_CHECK_GE(a.start_frame, 0); + ABSL_CHECK_LT(a.start_frame, chunk_data_size); VLOG(1) << " a.start_frame = " << a.start_frame << " @" << a.chunk_data->item(a.start_frame).timestamp_usec() << " with " @@ -831,7 +834,7 @@ void BoxTracker::TrackingImpl(const TrackingImplArgs& a) { TrackingImpl(next_args); } else { cleanup_func(); - LOG(ERROR) << "Can't read expected chunk file!"; + ABSL_LOG(ERROR) << "Can't read expected chunk file!"; } } } @@ -892,10 +895,10 @@ void BoxTracker::TrackingImpl(const TrackingImplArgs& a) { TrackingImpl(prev_args); } else { cleanup_func(); - LOG(ERROR) << "Can't read expected chunk file! " << a.chunk_idx - 1 - << " while tracking @" - << a.chunk_data->item(f).timestamp_usec() / 1000 - << " with cutoff " << a.min_msec; + ABSL_LOG(ERROR) << "Can't read expected chunk file! " + << a.chunk_idx - 1 << " while tracking @" + << a.chunk_data->item(f).timestamp_usec() / 1000 + << " with cutoff " << a.min_msec; return; } } @@ -907,7 +910,7 @@ void BoxTracker::TrackingImpl(const TrackingImplArgs& a) { bool TimedBoxAtTime(const PathSegment& segment, int64_t time_msec, TimedBox* box, MotionBoxState* state) { - CHECK(box); + ABSL_CHECK(box); if (segment.empty()) { return false; @@ -1031,7 +1034,7 @@ bool BoxTracker::WaitForAllOngoingTracks(int timeout_us) { bool BoxTracker::GetTrackingData(int id, int64_t request_time_msec, TrackingData* tracking_data, int* tracking_data_msec) { - CHECK(tracking_data); + ABSL_CHECK(tracking_data); int chunk_idx = ChunkIdxFromTime(request_time_msec); @@ -1039,7 +1042,7 @@ bool BoxTracker::GetTrackingData(int id, int64_t request_time_msec, if (!tracking_chunk.first) { absl::MutexLock lock(&status_mutex_); --track_status_[id][kInitCheckpoint].tracks_ongoing; - LOG(ERROR) << "Could not read tracking chunk from file."; + ABSL_LOG(ERROR) << "Could not read tracking chunk from file."; return false; } diff --git a/mediapipe/util/tracking/camera_motion.cc b/mediapipe/util/tracking/camera_motion.cc index e753be71..21924a9d 100644 --- a/mediapipe/util/tracking/camera_motion.cc +++ b/mediapipe/util/tracking/camera_motion.cc @@ -16,6 +16,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "mediapipe/util/tracking/region_flow.h" @@ -76,8 +78,8 @@ void CameraMotionToMixtureHomography(const CameraMotion& camera_motion, CameraMotion ComposeCameraMotion(const CameraMotion& lhs, const CameraMotion& rhs) { - CHECK_EQ(lhs.frame_width(), rhs.frame_width()); - CHECK_EQ(lhs.frame_height(), rhs.frame_height()); + ABSL_CHECK_EQ(lhs.frame_width(), rhs.frame_width()); + ABSL_CHECK_EQ(lhs.frame_height(), rhs.frame_height()); CameraMotion result = rhs; if (lhs.has_translation() || rhs.has_translation()) { @@ -106,9 +108,10 @@ CameraMotion ComposeCameraMotion(const CameraMotion& lhs, if (rhs.has_mixture_homography()) { if (lhs.has_mixture_homography()) { - LOG(ERROR) << "Mixture homographies are not closed under composition, " - << "Only rhs mixtures composed with lhs homographies " - << "are supported."; + ABSL_LOG(ERROR) + << "Mixture homographies are not closed under composition, " + << "Only rhs mixtures composed with lhs homographies " + << "are supported."; } else if (lhs.type() <= CameraMotion::UNSTABLE_SIM) { // We only composit base model when stability is sufficient. *result.mutable_mixture_homography() = @@ -116,7 +119,7 @@ CameraMotion ComposeCameraMotion(const CameraMotion& lhs, lhs.homography()); } } else if (lhs.has_mixture_homography()) { - LOG(ERROR) << "Only rhs mixtures supported."; + ABSL_LOG(ERROR) << "Only rhs mixtures supported."; } // Select max unstable type. @@ -175,7 +178,7 @@ CameraMotion InvertCameraMotion(const CameraMotion& motion) { } if (motion.has_mixture_homography()) { - LOG(ERROR) << "Mixture homographies are not closed under inversion."; + ABSL_LOG(ERROR) << "Mixture homographies are not closed under inversion."; } return inverted; @@ -184,8 +187,8 @@ CameraMotion InvertCameraMotion(const CameraMotion& motion) { void SubtractCameraMotionFromFeatures( const std::vector& camera_motions, std::vector* feature_lists) { - CHECK(feature_lists != nullptr); - CHECK_GE(camera_motions.size(), feature_lists->size()); + ABSL_CHECK(feature_lists != nullptr); + ABSL_CHECK_GE(camera_motions.size(), feature_lists->size()); if (feature_lists->empty()) { return; } @@ -227,8 +230,9 @@ void SubtractCameraMotionFromFeatures( float ForegroundMotion(const CameraMotion& camera_motion, const RegionFlowFeatureList& feature_list) { if (camera_motion.has_mixture_homography()) { - LOG(WARNING) << "Mixture homographies are present but function is only " - << "using homographies. Truncation error likely."; + ABSL_LOG(WARNING) + << "Mixture homographies are present but function is only " + << "using homographies. Truncation error likely."; } Homography background_motion; @@ -327,7 +331,7 @@ template CameraMotion FirstCameraMotionForLooping( const CameraMotionContainer& camera_motions) { if (camera_motions.size() < 2) { - LOG(ERROR) << "Not enough camera motions for refinement."; + ABSL_LOG(ERROR) << "Not enough camera motions for refinement."; return CameraMotion(); } @@ -346,8 +350,8 @@ CameraMotion FirstCameraMotionForLooping( const CameraMotion& motion = camera_motions[i]; if (motion.has_mixture_homography()) { // TODO: Implement - LOG(WARNING) << "This function does not validly apply mixtures; " - << "which are currently not closed under composition. "; + ABSL_LOG(WARNING) << "This function does not validly apply mixtures; " + << "which are currently not closed under composition. "; } switch (motion.type()) { @@ -367,7 +371,7 @@ CameraMotion FirstCameraMotionForLooping( case CameraMotion::UNSTABLE_HOMOG: break; default: - LOG(FATAL) << "Unknown CameraMotion::type."; + ABSL_LOG(FATAL) << "Unknown CameraMotion::type."; } // Only accumulate motions which are valid for the entire chain, otherwise diff --git a/mediapipe/util/tracking/camera_motion.h b/mediapipe/util/tracking/camera_motion.h index cadee78c..cfe6b250 100644 --- a/mediapipe/util/tracking/camera_motion.h +++ b/mediapipe/util/tracking/camera_motion.h @@ -17,6 +17,8 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/util/tracking/camera_motion.pb.h" #include "mediapipe/util/tracking/motion_models.h" #include "mediapipe/util/tracking/region_flow.pb.h" @@ -165,7 +167,7 @@ Model UnstableCameraMotionToModel(const CameraMotion& camera_motion, } case CameraMotion::VALID: - LOG(FATAL) << "Specify a type != VALID"; + ABSL_LOG(FATAL) << "Specify a type != VALID"; return Model(); } } @@ -225,7 +227,7 @@ Model ProjectToTypeModel(const Model& model, float frame_width, template <> inline MixtureHomography ProjectToTypeModel(const MixtureHomography&, float, float, CameraMotion::Type) { - LOG(FATAL) << "Projection not supported for mixtures."; + ABSL_LOG(FATAL) << "Projection not supported for mixtures."; return MixtureHomography(); } @@ -236,11 +238,11 @@ void DownsampleMotionModels( std::vector* downsampled_models, std::vector* downsampled_types) { if (model_type) { - CHECK_EQ(models.size(), model_type->size()); - CHECK(downsampled_models) << "Expecting output models."; + ABSL_CHECK_EQ(models.size(), model_type->size()); + ABSL_CHECK(downsampled_models) << "Expecting output models."; } - CHECK(downsampled_models); + ABSL_CHECK(downsampled_models); downsampled_models->clear(); if (downsampled_types) { downsampled_types->clear(); @@ -276,7 +278,7 @@ void DownsampleMotionModels( template void SubsampleEntities(const Container& input, int downsample_factor, Container* output) { - CHECK(output); + ABSL_CHECK(output); output->clear(); if (input.empty()) { diff --git a/mediapipe/util/tracking/flow_packager.cc b/mediapipe/util/tracking/flow_packager.cc index dceacbcd..1f358860 100644 --- a/mediapipe/util/tracking/flow_packager.cc +++ b/mediapipe/util/tracking/flow_packager.cc @@ -20,6 +20,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/port/logging.h" @@ -36,8 +38,8 @@ namespace mediapipe { FlowPackager::FlowPackager(const FlowPackagerOptions& options) : options_(options) { if (options_.binary_tracking_data_support()) { - CHECK_LE(options.domain_width(), 256); - CHECK_LE(options.domain_height(), 256); + ABSL_CHECK_LE(options.domain_width(), 256); + ABSL_CHECK_LE(options.domain_height(), 256); } } @@ -105,7 +107,7 @@ inline std::string EncodeVectorToString(const std::vector& vec) { template inline bool DecodeFromStringView(absl::string_view str, T* result) { - CHECK(result != nullptr); + ABSL_CHECK(result != nullptr); if (sizeof(*result) != str.size()) { return false; } @@ -116,7 +118,7 @@ inline bool DecodeFromStringView(absl::string_view str, T* result) { template inline bool DecodeVectorFromStringView(absl::string_view str, std::vector* result) { - CHECK(result != nullptr); + ABSL_CHECK(result != nullptr); if (str.size() % sizeof(T) != 0) return false; result->clear(); result->reserve(str.size() / sizeof(T)); @@ -134,9 +136,9 @@ inline bool DecodeVectorFromStringView(absl::string_view str, void FlowPackager::PackFlow(const RegionFlowFeatureList& feature_list, const CameraMotion* camera_motion, TrackingData* tracking_data) const { - CHECK(tracking_data); - CHECK_GT(feature_list.frame_width(), 0); - CHECK_GT(feature_list.frame_height(), 0); + ABSL_CHECK(tracking_data); + ABSL_CHECK_GT(feature_list.frame_width(), 0); + ABSL_CHECK_GT(feature_list.frame_height(), 0); // Scale flow to output domain. const float dim_x_scale = @@ -231,12 +233,12 @@ void FlowPackager::PackFlow(const RegionFlowFeatureList& feature_list, const int curr_col = loc.x(); if (curr_col != last_col) { - CHECK_LT(last_col, curr_col); - CHECK_EQ(-1, col_start[curr_col]); + ABSL_CHECK_LT(last_col, curr_col); + ABSL_CHECK_EQ(-1, col_start[curr_col]); col_start[curr_col] = data->row_indices_size() - 1; last_col = curr_col; } else { - CHECK_LE(last_row, loc.y()); + ABSL_CHECK_LE(last_row, loc.y()); } last_row = loc.y(); } @@ -247,7 +249,7 @@ void FlowPackager::PackFlow(const RegionFlowFeatureList& feature_list, // Fill unset values with previously set value. Propagate end value. for (int i = options_.domain_width() - 1; i > 0; --i) { if (col_start[i] < 0) { - DCHECK_GE(col_start[i + 1], 0); + ABSL_DCHECK_GE(col_start[i + 1], 0); col_start[i] = col_start[i + 1]; } } @@ -261,11 +263,11 @@ void FlowPackager::PackFlow(const RegionFlowFeatureList& feature_list, const int r_start = data->col_starts(c); const int r_end = data->col_starts(c + 1); for (int r = r_start; r < r_end - 1; ++r) { - CHECK_LE(data->row_indices(r), data->row_indices(r + 1)); + ABSL_CHECK_LE(data->row_indices(r), data->row_indices(r + 1)); } } - CHECK_EQ(data->vector_data_size(), 2 * data->row_indices_size()); + ABSL_CHECK_EQ(data->vector_data_size(), 2 * data->row_indices_size()); *data->mutable_actively_discarded_tracked_ids() = feature_list.actively_discarded_tracked_ids(); @@ -273,8 +275,8 @@ void FlowPackager::PackFlow(const RegionFlowFeatureList& feature_list, void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, BinaryTrackingData* binary_data) const { - CHECK(options_.binary_tracking_data_support()); - CHECK(binary_data != nullptr); + ABSL_CHECK(options_.binary_tracking_data_support()); + ABSL_CHECK(binary_data != nullptr); int32_t frame_flags = 0; const bool high_profile = options_.use_high_profile(); @@ -313,7 +315,7 @@ void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, const int32_t domain_width = tracking_data.domain_width(); const int32_t domain_height = tracking_data.domain_height(); - CHECK_LT(domain_height, 256) << "Only heights below 256 are supported."; + ABSL_CHECK_LT(domain_height, 256) << "Only heights below 256 are supported."; const float frame_aspect = tracking_data.frame_aspect(); // Limit vector value from above (to 20% frame diameter) and below (small @@ -321,9 +323,9 @@ void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, const float max_vector_threshold = hypot(domain_width, domain_height) * 0.2f; // Warn if too much truncation. if (max_vector_value > max_vector_threshold * 1.5f) { - LOG(WARNING) << "A lot of truncation will occur during encoding. " - << "Vector magnitudes are larger than 20% of the " - << "frame diameter."; + ABSL_LOG(WARNING) << "A lot of truncation will occur during encoding. " + << "Vector magnitudes are larger than 20% of the " + << "frame diameter."; } max_vector_value = @@ -393,7 +395,7 @@ void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, flow_compressed_8.push_back(flow_y); } - DCHECK_LT(motion_data.row_indices(r), 256); + ABSL_DCHECK_LT(motion_data.row_indices(r), 256); row_idx.push_back(motion_data.row_indices(r)); } } @@ -470,7 +472,7 @@ void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, // Delta compress. int delta_row = motion_data.row_indices(r) - (r == r_start ? 0 : motion_data.row_indices(r - 1)); - CHECK_GE(delta_row, 0); + ABSL_CHECK_GE(delta_row, 0); bool combined = false; if (r > r_start) { @@ -520,9 +522,9 @@ void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, } if (options_.high_fidelity_16bit_encode()) { - CHECK_EQ(2 * encoded, flow_compressed_16.size()); + ABSL_CHECK_EQ(2 * encoded, flow_compressed_16.size()); } else { - CHECK_EQ(2 * encoded, flow_compressed_8.size()); + ABSL_CHECK_EQ(2 * encoded, flow_compressed_8.size()); } // Adjust column start by compressions. @@ -530,11 +532,11 @@ void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, for (int k = 0; k < domain_width; ++k) { curr_adjust -= compressions_per_column[k]; col_starts[k + 1] += curr_adjust; - CHECK_LE(col_starts[k], col_starts[k + 1]); + ABSL_CHECK_LE(col_starts[k], col_starts[k + 1]); } - CHECK_EQ(row_idx.size(), col_starts.back()); - CHECK_EQ(num_vectors, row_idx.size() + compressible); + ABSL_CHECK_EQ(row_idx.size(), col_starts.back()); + ABSL_CHECK_EQ(num_vectors, row_idx.size() + compressible); } // Delta compress col_starts. @@ -542,7 +544,7 @@ void FlowPackager::EncodeTrackingData(const TrackingData& tracking_data, col_start_delta[0] = col_starts[0]; for (int k = 1; k < domain_width + 1; ++k) { const int delta = col_starts[k] - col_starts[k - 1]; - CHECK_LT(delta, 256) << "Only up to 255 items per column supported."; + ABSL_CHECK_LT(delta, 256) << "Only up to 255 items per column supported."; col_start_delta[k] = delta; } @@ -602,7 +604,7 @@ std::string PopSubstring(int len, absl::string_view* piece) { void FlowPackager::DecodeTrackingData(const BinaryTrackingData& container_data, TrackingData* tracking_data) const { - CHECK(tracking_data != nullptr); + ABSL_CHECK(tracking_data != nullptr); absl::string_view data(container_data.data()); int32_t frame_flags = 0; @@ -618,8 +620,8 @@ void FlowPackager::DecodeTrackingData(const BinaryTrackingData& container_data, DecodeFromStringView(PopSubstring(4, &data), &domain_height); DecodeFromStringView(PopSubstring(4, &data), &frame_aspect); - CHECK_LE(domain_width, 256); - CHECK_LE(domain_height, 256); + ABSL_CHECK_LE(domain_width, 256); + ABSL_CHECK_LE(domain_height, 256); DecodeVectorFromStringView( PopSubstring(4 * HomographyAdapter::NumParameters(), &data), @@ -662,7 +664,7 @@ void FlowPackager::DecodeTrackingData(const BinaryTrackingData& container_data, // Should not have more row indices than vectors. (One for each in baseline // profile, less in high profile). - CHECK_LE(row_idx_size, num_vectors); + ABSL_CHECK_LE(row_idx_size, num_vectors); DecodeVectorFromStringView(PopSubstring(row_idx_size, &data), &row_idx); // Records for each vector whether to advance pointer in the vector data array @@ -707,7 +709,7 @@ void FlowPackager::DecodeTrackingData(const BinaryTrackingData& container_data, } } row_idx.swap(row_idx_unpacked); - CHECK_EQ(num_vectors, row_idx.size()); + ABSL_CHECK_EQ(num_vectors, row_idx.size()); // Adjust column start by expansions. int curr_adjust = 0; @@ -717,7 +719,7 @@ void FlowPackager::DecodeTrackingData(const BinaryTrackingData& container_data, } } - CHECK_EQ(num_vectors, col_starts.back()); + ABSL_CHECK_EQ(num_vectors, col_starts.back()); int vector_data_size; DecodeFromStringView(PopSubstring(4, &data), &vector_data_size); @@ -749,7 +751,7 @@ void FlowPackager::DecodeTrackingData(const BinaryTrackingData& container_data, motion_data->add_vector_data(prev_flow_y * flow_denom); } } - CHECK_EQ(vector_data_size, counter); + ABSL_CHECK_EQ(vector_data_size, counter); } else { std::vector vector_data; DecodeVectorFromStringView( @@ -775,7 +777,7 @@ void FlowPackager::DecodeTrackingData(const BinaryTrackingData& container_data, motion_data->add_vector_data(prev_flow_y * flow_denom); } } - CHECK_EQ(vector_data_size, counter); + ABSL_CHECK_EQ(vector_data_size, counter); } for (auto idx : row_idx) { @@ -789,7 +791,7 @@ void FlowPackager::DecodeTrackingData(const BinaryTrackingData& container_data, void FlowPackager::BinaryTrackingDataToContainer( const BinaryTrackingData& binary_data, TrackingContainer* container) const { - CHECK(container != nullptr); + ABSL_CHECK(container != nullptr); container->Clear(); container->set_header("TRAK"); container->set_version(1); @@ -799,17 +801,17 @@ void FlowPackager::BinaryTrackingDataToContainer( void FlowPackager::BinaryTrackingDataFromContainer( const TrackingContainer& container, BinaryTrackingData* binary_data) const { - CHECK_EQ("TRAK", container.header()); - CHECK_EQ(1, container.version()) << "Unsupported version."; + ABSL_CHECK_EQ("TRAK", container.header()); + ABSL_CHECK_EQ(1, container.version()) << "Unsupported version."; *binary_data->mutable_data() = container.data(); } void FlowPackager::DecodeMetaData(const TrackingContainer& container_data, MetaData* meta_data) const { - CHECK(meta_data != nullptr); + ABSL_CHECK(meta_data != nullptr); - CHECK_EQ("META", container_data.header()); - CHECK_EQ(1, container_data.version()) << "Unsupported version."; + ABSL_CHECK_EQ("META", container_data.header()); + ABSL_CHECK_EQ(1, container_data.version()) << "Unsupported version."; absl::string_view data(container_data.data()); @@ -833,14 +835,14 @@ void FlowPackager::DecodeMetaData(const TrackingContainer& container_data, void FlowPackager::FinalizeTrackingContainerFormat( std::vector* timestamps, TrackingContainerFormat* container_format) { - CHECK(container_format != nullptr); + ABSL_CHECK(container_format != nullptr); // Compute binary sizes of track_data. const int num_frames = container_format->track_data_size(); std::vector msecs(num_frames, 0); if (timestamps) { - CHECK_EQ(num_frames, timestamps->size()); + ABSL_CHECK_EQ(num_frames, timestamps->size()); msecs = *timestamps; } std::vector sizes(num_frames, 0); @@ -877,14 +879,14 @@ void FlowPackager::FinalizeTrackingContainerFormat( void FlowPackager::FinalizeTrackingContainerProto( std::vector* timestamps, TrackingContainerProto* proto) { - CHECK(proto != nullptr); + ABSL_CHECK(proto != nullptr); // Compute binary sizes of track_data. const int num_frames = proto->track_data_size(); std::vector msecs(num_frames, 0); if (timestamps) { - CHECK_EQ(num_frames, timestamps->size()); + ABSL_CHECK_EQ(num_frames, timestamps->size()); msecs = *timestamps; } @@ -909,8 +911,8 @@ void FlowPackager::InitializeMetaData(int num_frames, const std::vector& data_sizes, MetaData* meta_data) const { meta_data->set_num_frames(num_frames); - CHECK_EQ(num_frames, msecs.size()); - CHECK_EQ(num_frames, data_sizes.size()); + ABSL_CHECK_EQ(num_frames, msecs.size()); + ABSL_CHECK_EQ(num_frames, data_sizes.size()); int curr_offset = 0; for (int f = 0; f < num_frames; ++f) { @@ -923,9 +925,9 @@ void FlowPackager::InitializeMetaData(int num_frames, void FlowPackager::AddContainerToString(const TrackingContainer& container, std::string* binary_data) { - CHECK(binary_data != nullptr); + ABSL_CHECK(binary_data != nullptr); std::string header_string(container.header()); - CHECK_EQ(4, header_string.size()); + ABSL_CHECK_EQ(4, header_string.size()); std::vector header{header_string[0], header_string[1], header_string[2], header_string[3]}; @@ -936,10 +938,10 @@ void FlowPackager::AddContainerToString(const TrackingContainer& container, std::string FlowPackager::SplitContainerFromString( absl::string_view* binary_data, TrackingContainer* container) { - CHECK(binary_data != nullptr); - CHECK(container != nullptr); - CHECK_GE(binary_data->size(), 12) << "Data does not contain " - << "valid container"; + ABSL_CHECK(binary_data != nullptr); + ABSL_CHECK(container != nullptr); + ABSL_CHECK_GE(binary_data->size(), 12) << "Data does not contain " + << "valid container"; container->set_header(PopSubstring(4, binary_data)); @@ -961,7 +963,7 @@ std::string FlowPackager::SplitContainerFromString( void FlowPackager::TrackingContainerFormatToBinary( const TrackingContainerFormat& container_format, std::string* binary) { - CHECK(binary != nullptr); + ABSL_CHECK(binary != nullptr); binary->clear(); AddContainerToString(container_format.meta_data(), binary); @@ -974,28 +976,28 @@ void FlowPackager::TrackingContainerFormatToBinary( void FlowPackager::TrackingContainerFormatFromBinary( const std::string& binary, TrackingContainerFormat* container_format) { - CHECK(container_format != nullptr); + ABSL_CHECK(container_format != nullptr); container_format->Clear(); absl::string_view data(binary); - CHECK_EQ("META", SplitContainerFromString( - &data, container_format->mutable_meta_data())); + ABSL_CHECK_EQ("META", SplitContainerFromString( + &data, container_format->mutable_meta_data())); MetaData meta_data; DecodeMetaData(container_format->meta_data(), &meta_data); for (int f = 0; f < meta_data.num_frames(); ++f) { TrackingContainer* container = container_format->add_track_data(); - CHECK_EQ("TRAK", SplitContainerFromString(&data, container)); + ABSL_CHECK_EQ("TRAK", SplitContainerFromString(&data, container)); } - CHECK_EQ("TERM", SplitContainerFromString( - &data, container_format->mutable_term_data())); + ABSL_CHECK_EQ("TERM", SplitContainerFromString( + &data, container_format->mutable_term_data())); } void FlowPackager::SortRegionFlowFeatureList( float scale_x, float scale_y, RegionFlowFeatureList* feature_list) const { - CHECK(feature_list != nullptr); + ABSL_CHECK(feature_list != nullptr); // Sort features lexicographically. std::sort(feature_list->mutable_feature()->begin(), feature_list->mutable_feature()->end(), diff --git a/mediapipe/util/tracking/image_util.cc b/mediapipe/util/tracking/image_util.cc index a44c00b0..d376ca30 100644 --- a/mediapipe/util/tracking/image_util.cc +++ b/mediapipe/util/tracking/image_util.cc @@ -17,7 +17,8 @@ #include #include -#include "mediapipe/framework/port/logging.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/util/tracking/motion_models.h" #include "mediapipe/util/tracking/region_flow.h" @@ -25,8 +26,8 @@ namespace mediapipe { // Returns median of the L1 color distance between img_1 and img_2 float FrameDifferenceMedian(const cv::Mat& img_1, const cv::Mat& img_2) { - CHECK(img_1.size() == img_2.size()); - CHECK_EQ(img_1.channels(), img_2.channels()); + ABSL_CHECK(img_1.size() == img_2.size()); + ABSL_CHECK_EQ(img_1.channels(), img_2.channels()); std::vector color_diffs; color_diffs.reserve(img_1.cols * img_1.rows); @@ -52,7 +53,7 @@ float FrameDifferenceMedian(const cv::Mat& img_1, const cv::Mat& img_2) { } void JetColoring(int steps, std::vector* color_map) { - CHECK(color_map != nullptr); + ABSL_CHECK(color_map != nullptr); color_map->resize(steps); for (int i = 0; i < steps; ++i) { const float frac = 2.0f * (i * (1.0f / steps) - 0.5f); @@ -71,7 +72,7 @@ void JetColoring(int steps, std::vector* color_map) { (*color_map)[i] = Vector3_f(1.0f + (frac - 0.8f) * -2.0f, 0.0f, 0.0f) * 255.0f; } else { - LOG(ERROR) << "Out of bound value. Should not occur."; + ABSL_LOG(ERROR) << "Out of bound value. Should not occur."; } } } diff --git a/mediapipe/util/tracking/image_util.h b/mediapipe/util/tracking/image_util.h index ba58d343..f1e7eda3 100644 --- a/mediapipe/util/tracking/image_util.h +++ b/mediapipe/util/tracking/image_util.h @@ -17,6 +17,7 @@ #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/vector.h" @@ -75,7 +76,7 @@ void CopyMatBorder(cv::Mat* mat) { } // src and dst should point to same column from here. - DCHECK_EQ(0, (src_ptr - dst_ptr) * sizeof(T) % mat->step[0]); + ABSL_DCHECK_EQ(0, (src_ptr - dst_ptr) * sizeof(T) % mat->step[0]); // Top row copy. memcpy(dst_ptr, src_ptr, width * channels * sizeof(dst_ptr[0])); @@ -122,7 +123,7 @@ void CopyMatBorder(cv::Mat* mat) { } // src and dst should point to same column from here. - DCHECK_EQ(0, (dst_ptr - src_ptr) * sizeof(T) % mat->step[0]); + ABSL_DCHECK_EQ(0, (dst_ptr - src_ptr) * sizeof(T) % mat->step[0]); memcpy(dst_ptr, src_ptr, width * channels * sizeof(dst_ptr[0])); src_ptr += width * channels; // Points one behind the end. dst_ptr += width * channels; diff --git a/mediapipe/util/tracking/measure_time.cc b/mediapipe/util/tracking/measure_time.cc index e628dff0..f4c9b290 100644 --- a/mediapipe/util/tracking/measure_time.cc +++ b/mediapipe/util/tracking/measure_time.cc @@ -15,7 +15,7 @@ #include "mediapipe/util/tracking/measure_time.h" #ifdef SET_FLAG_MEASURE_TIME -// If set to true, outputs time measurements to LOG(INFO). +// If set to true, outputs time measurements to ABSL_LOG(INFO). bool flags_measure_time = true; #else bool flags_measure_time = false; diff --git a/mediapipe/util/tracking/measure_time.h b/mediapipe/util/tracking/measure_time.h index 0351f465..20b859b4 100644 --- a/mediapipe/util/tracking/measure_time.h +++ b/mediapipe/util/tracking/measure_time.h @@ -13,7 +13,7 @@ // limitations under the License. // // Helper class and macro to take time measurements within current scope. -// Takes time measurement within current scope. Outputs to LOG(INFO) if +// Takes time measurement within current scope. Outputs to ABSL_LOG(INFO) if // flag --measure_time is set or if build flag SET_FLAG_MEASURE_TIME is // defined (add --copt=-DSET_FLAG_MEASURE_TIME to your build command). // Additionally you can limit time measurements to specific files, @@ -31,12 +31,13 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/synchronization/mutex.h" #include "absl/time/clock.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" extern bool flags_measure_time; @@ -101,7 +102,7 @@ class ScopedWallTimer { show_output_(show_output), accumulator_(accumulator) { if (show_output_) { - CHECK(accumulator_); + ABSL_CHECK(accumulator_); start_time_ = GetWallTime(); } } @@ -115,10 +116,10 @@ class ScopedWallTimer { double accum_time = 0.0; int count = 0; accumulator_->Accumulate(passed_time, &accum_time, &count); - LOG(INFO) << stream_.str() << " TIMES: [Curr: " << passed_time * 1e-6 - << " ms, " - << "Avg: " << accum_time * 1e-6 / std::max(1, count) << " ms, " - << count << " calls]"; + ABSL_LOG(INFO) << stream_.str() << " TIMES: [Curr: " << passed_time * 1e-6 + << " ms, " + << "Avg: " << accum_time * 1e-6 / std::max(1, count) + << " ms, " << count << " calls]"; } } diff --git a/mediapipe/util/tracking/motion_analysis.cc b/mediapipe/util/tracking/motion_analysis.cc index 67973cbc..6d35a3e3 100644 --- a/mediapipe/util/tracking/motion_analysis.cc +++ b/mediapipe/util/tracking/motion_analysis.cc @@ -20,9 +20,10 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" #include "mediapipe/util/tracking/camera_motion.h" #include "mediapipe/util/tracking/camera_motion.pb.h" @@ -92,8 +93,8 @@ MotionAnalysis::MotionAnalysis(const MotionAnalysisOptions& options, use_spatial_bias; if (compute_feature_descriptors_) { - CHECK_EQ(RegionFlowComputationOptions::FORMAT_RGB, - options_.flow_options().image_format()) + ABSL_CHECK_EQ(RegionFlowComputationOptions::FORMAT_RGB, + options_.flow_options().image_format()) << "Feature descriptors only support RGB currently."; prev_frame_.reset(new cv::Mat(frame_height_, frame_width_, CV_8UC3)); } @@ -362,15 +363,15 @@ bool MotionAnalysis::AddFrameGeneric( RegionFlowFeatureList* output_feature_list) { // Don't check input sizes here, RegionFlowComputation does that based // on its internal options. - CHECK(feature_computation_) << "Calls to AddFrame* can NOT be mixed " - << "with AddFeatures"; + ABSL_CHECK(feature_computation_) << "Calls to AddFrame* can NOT be mixed " + << "with AddFeatures"; // Compute RegionFlow. { MEASURE_TIME << "CALL RegionFlowComputation::AddImage"; if (!region_flow_computation_->AddImageWithSeed(frame, timestamp_usec, initial_transform)) { - LOG(ERROR) << "Error while computing region flow."; + ABSL_LOG(ERROR) << "Error while computing region flow."; return false; } } @@ -401,7 +402,7 @@ bool MotionAnalysis::AddFrameGeneric( compute_feature_match_descriptors ? prev_frame_.get() : nullptr)); if (feature_list == nullptr) { - LOG(ERROR) << "Error retrieving feature list."; + ABSL_LOG(ERROR) << "Error retrieving feature list."; return false; } } @@ -461,7 +462,7 @@ void MotionAnalysis::AddFeatures(const RegionFlowFeatureList& features) { void MotionAnalysis::EnqueueFeaturesAndMotions( const RegionFlowFeatureList& features, const CameraMotion& motion) { feature_computation_ = false; - CHECK(buffer_->HaveEqualSize({"motion", "features"})) + ABSL_CHECK(buffer_->HaveEqualSize({"motion", "features"})) << "Can not be mixed with other Add* calls"; buffer_->EmplaceDatum("features", new RegionFlowFeatureList(features)); buffer_->EmplaceDatum("motion", new CameraMotion(motion)); @@ -479,7 +480,7 @@ int MotionAnalysis::GetResults( const int num_features_lists = buffer_->BufferSize("features"); const int num_new_feature_lists = num_features_lists - overlap_start_; - CHECK_GE(num_new_feature_lists, 0); + ABSL_CHECK_GE(num_new_feature_lists, 0); if (!flush && num_new_feature_lists < options_.estimation_clip_size()) { // Nothing to compute, return. @@ -487,7 +488,7 @@ int MotionAnalysis::GetResults( } const bool compute_saliency = options_.compute_motion_saliency(); - CHECK_EQ(compute_saliency, saliency != nullptr) + ABSL_CHECK_EQ(compute_saliency, saliency != nullptr) << "Computing saliency requires saliency output and vice versa"; // Estimate motions for newly buffered RegionFlowFeatureLists, which also @@ -514,7 +515,7 @@ int MotionAnalysis::GetResults( } } - CHECK(buffer_->HaveEqualSize({"features", "motion"})); + ABSL_CHECK(buffer_->HaveEqualSize({"features", "motion"})); if (compute_saliency) { ComputeSaliency(); @@ -528,9 +529,9 @@ int MotionAnalysis::OutputResults( std::vector>* camera_motion, std::vector>* saliency) { const bool compute_saliency = options_.compute_motion_saliency(); - CHECK_EQ(compute_saliency, saliency != nullptr) + ABSL_CHECK_EQ(compute_saliency, saliency != nullptr) << "Computing saliency requires saliency output and vice versa"; - CHECK(buffer_->HaveEqualSize({"features", "motion"})); + ABSL_CHECK(buffer_->HaveEqualSize({"features", "motion"})); // Discard prev. overlap (already output, just used for filtering here). buffer_->DiscardData(buffer_->AllTags(), prev_overlap_start_); @@ -598,9 +599,9 @@ int MotionAnalysis::OutputResults( // Reset for next chunk. prev_overlap_start_ = num_output_frames - new_overlap_start; - CHECK_GE(prev_overlap_start_, 0); + ABSL_CHECK_GE(prev_overlap_start_, 0); - CHECK(buffer_->TruncateBuffer(flush)); + ABSL_CHECK(buffer_->TruncateBuffer(flush)); overlap_start_ = buffer_->MaxBufferSize(); return num_output_frames; @@ -611,9 +612,9 @@ void MotionAnalysis::RenderResults(const RegionFlowFeatureList& feature_list, const SalientPointFrame* saliency, cv::Mat* rendered_results) { #ifndef NO_RENDERING - CHECK(rendered_results != nullptr); - CHECK_EQ(frame_width_, rendered_results->cols); - CHECK_EQ(frame_height_, rendered_results->rows); + ABSL_CHECK(rendered_results != nullptr); + ABSL_CHECK_EQ(frame_width_, rendered_results->cols); + ABSL_CHECK_EQ(frame_height_, rendered_results->rows); const auto viz_options = options_.visualization_options(); @@ -670,7 +671,7 @@ void MotionAnalysis::RenderResults(const RegionFlowFeatureList& feature_list, text_scale * 3, cv::LINE_AA); } #else - LOG(FATAL) << "Code stripped out because of NO_RENDERING"; + ABSL_LOG(FATAL) << "Code stripped out because of NO_RENDERING"; #endif } @@ -698,10 +699,10 @@ void MotionAnalysis::ComputeDenseForeground( &foreground_weights); // Setup push pull map (with border). Ensure constructor used the right type. - CHECK(foreground_push_pull_->filter_type() == - PushPullFilteringC1::BINOMIAL_5X5 || - foreground_push_pull_->filter_type() == - PushPullFilteringC1::GAUSSIAN_5X5); + ABSL_CHECK(foreground_push_pull_->filter_type() == + PushPullFilteringC1::BINOMIAL_5X5 || + foreground_push_pull_->filter_type() == + PushPullFilteringC1::GAUSSIAN_5X5); cv::Mat foreground_map(frame_height_ + 4, frame_width_ + 4, CV_32FC2); std::vector feature_locations; @@ -741,8 +742,8 @@ void MotionAnalysis::ComputeDenseForeground( void MotionAnalysis::VisualizeDenseForeground(const cv::Mat& foreground_mask, cv::Mat* output) { - CHECK(output != nullptr); - CHECK(foreground_mask.size() == output->size()); + ABSL_CHECK(output != nullptr); + ABSL_CHECK(foreground_mask.size() == output->size()); // Map foreground measure to color (green by default). std::vector color_map; if (options_.visualization_options().foreground_jet_coloring()) { @@ -780,7 +781,7 @@ void MotionAnalysis::VisualizeDenseForeground(const cv::Mat& foreground_mask, } void MotionAnalysis::VisualizeBlurAnalysisRegions(cv::Mat* input_view) { - CHECK(input_view != nullptr); + ABSL_CHECK(input_view != nullptr); cv::Mat intensity; cv::cvtColor(*input_view, intensity, cv::COLOR_RGB2GRAY); @@ -797,7 +798,7 @@ void MotionAnalysis::VisualizeBlurAnalysisRegions(cv::Mat* input_view) { void MotionAnalysis::ComputeSaliency() { MEASURE_TIME << "Saliency computation."; - CHECK_EQ(overlap_start_, buffer_->BufferSize("saliency")); + ABSL_CHECK_EQ(overlap_start_, buffer_->BufferSize("saliency")); const int num_features_lists = buffer_->BufferSize("features"); @@ -821,7 +822,7 @@ void MotionAnalysis::ComputeSaliency() { buffer_->AddDatum("saliency", std::move(saliency)); } - CHECK(buffer_->HaveEqualSize({"features", "motion", "saliency"})); + ABSL_CHECK(buffer_->HaveEqualSize({"features", "motion", "saliency"})); // Clear output saliency and copy from saliency. buffer_->DiscardDatum("output_saliency", diff --git a/mediapipe/util/tracking/motion_estimation.cc b/mediapipe/util/tracking/motion_estimation.cc index b608b470..4406359a 100644 --- a/mediapipe/util/tracking/motion_estimation.cc +++ b/mediapipe/util/tracking/motion_estimation.cc @@ -31,8 +31,9 @@ #include "Eigen/SVD" #include "absl/container/node_hash_map.h" #include "absl/container/node_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/util/tracking/camera_motion.h" #include "mediapipe/util/tracking/measure_time.h" #include "mediapipe/util/tracking/motion_models.h" @@ -173,9 +174,9 @@ class InlierMask { // estimated translation. void MotionPrior(const RegionFlowFeatureList& feature_list, std::vector* motion_prior) { - CHECK(motion_prior != nullptr); + ABSL_CHECK(motion_prior != nullptr); const int num_features = feature_list.feature_size(); - CHECK_EQ(num_features, motion_prior->size()); + ABSL_CHECK_EQ(num_features, motion_prior->size()); // Return, if prior is too low. const float kMinTranslationPrior = 0.5f; @@ -185,7 +186,7 @@ class InlierMask { } const float prev_magnitude = translation_.Norm(); - CHECK_EQ(num_features, motion_prior->size()); + ABSL_CHECK_EQ(num_features, motion_prior->size()); const float inv_prev_magnitude = prev_magnitude < options_.min_translation_norm() ? (1.0f / options_.min_translation_norm()) @@ -350,7 +351,7 @@ struct MotionEstimation::SingleTrackClipData { // feature's irls weight. If weight_backup is set, allocates storage // to backup and reset irls weights. void AllocateIRLSWeightStorage(bool weight_backup) { - CHECK(feature_lists != nullptr); + ABSL_CHECK(feature_lists != nullptr); const int num_frames = feature_lists->size(); if (weight_backup) { irls_weight_backup = &irls_backup_storage; @@ -380,7 +381,7 @@ struct MotionEstimation::SingleTrackClipData { // Returns number of frames in this clip. int num_frames() const { - DCHECK(feature_lists); + ABSL_DCHECK(feature_lists); return feature_lists->size(); } @@ -396,23 +397,23 @@ struct MotionEstimation::SingleTrackClipData { // Checks that SingleTrackClipData is properly initialized. void CheckInitialization() const { - CHECK(feature_lists != nullptr); - CHECK(camera_motions != nullptr); - CHECK_EQ(feature_lists->size(), camera_motions->size()); + ABSL_CHECK(feature_lists != nullptr); + ABSL_CHECK(camera_motions != nullptr); + ABSL_CHECK_EQ(feature_lists->size(), camera_motions->size()); if (feature_lists->empty()) { return; } - CHECK_EQ(num_frames(), irls_weight_input.size()); - CHECK_EQ(num_frames(), homog_irls_weight_input.size()); + ABSL_CHECK_EQ(num_frames(), irls_weight_input.size()); + ABSL_CHECK_EQ(num_frames(), homog_irls_weight_input.size()); if (irls_weight_backup) { - CHECK_EQ(num_frames(), irls_weight_backup->size()); + ABSL_CHECK_EQ(num_frames(), irls_weight_backup->size()); } for (int k = 0; k < num_frames(); ++k) { const int num_features = (*feature_lists)[k]->feature_size(); - CHECK_EQ(num_features, irls_weight_input[k].size()); - CHECK_EQ(num_features, homog_irls_weight_input[k].size()); + ABSL_CHECK_EQ(num_features, irls_weight_input[k].size()); + ABSL_CHECK_EQ(num_features, homog_irls_weight_input[k].size()); } } @@ -486,29 +487,31 @@ void MotionEstimation::InitializeWithOptions( MotionEstimationOptions::ESTIMATION_HOMOG_NONE && options.linear_similarity_estimation() == MotionEstimationOptions::ESTIMATION_LS_NONE) { - LOG(FATAL) << "Invalid MotionEstimationOptions. " - << "Homography estimation requires similarity to be estimated"; + ABSL_LOG(FATAL) + << "Invalid MotionEstimationOptions. " + << "Homography estimation requires similarity to be estimated"; } if (options.mix_homography_estimation() != MotionEstimationOptions::ESTIMATION_HOMOG_MIX_NONE && options.homography_estimation() == MotionEstimationOptions::ESTIMATION_HOMOG_NONE) { - LOG(FATAL) << "Invalid MotionEstimationOptions. " - << "Mixture homography estimation requires homography to be " - << "estimated."; + ABSL_LOG(FATAL) + << "Invalid MotionEstimationOptions. " + << "Mixture homography estimation requires homography to be " + << "estimated."; } // Check for deprecated options. - CHECK_NE(options.estimate_similarity(), true) + ABSL_CHECK_NE(options.estimate_similarity(), true) << "Option estimate_similarity is deprecated, use static function " << "EstimateSimilarityModelL2 instead."; - CHECK_NE(options.linear_similarity_estimation(), - MotionEstimationOptions::ESTIMATION_LS_L2_RANSAC) + ABSL_CHECK_NE(options.linear_similarity_estimation(), + MotionEstimationOptions::ESTIMATION_LS_L2_RANSAC) << "Option ESTIMATION_LS_L2_RANSAC is deprecated, use " << "ESTIMATION_LS_IRLS instead."; - CHECK_NE(options.linear_similarity_estimation(), - MotionEstimationOptions::ESTIMATION_LS_L1) + ABSL_CHECK_NE(options.linear_similarity_estimation(), + MotionEstimationOptions::ESTIMATION_LS_L1) << "Option ESTIMATION_LS_L1 is deprecated, use static function " << "EstimateLinearSimilarityL1 instead."; @@ -563,7 +566,7 @@ void MotionEstimation::InitializeWithOptions( } case MotionEstimationOptions::TEMPORAL_IRLS_MASK: - CHECK(options.irls_initialization().activated()) + ABSL_CHECK(options.irls_initialization().activated()) << "To use dependent_initialization, irls_initialization has to " << "be activated. "; inlier_mask_.reset(new InlierMask(options.irls_mask_options(), @@ -578,11 +581,11 @@ void MotionEstimation::EstimateMotion(const RegionFlowFrame& region_flow_frame, const int* intensity_frame, // null const int* prev_intensity_frame, // null CameraMotion* camera_motion) const { - CHECK(camera_motion); + ABSL_CHECK(camera_motion); - CHECK(intensity_frame == NULL) + ABSL_CHECK(intensity_frame == NULL) << "Parameter intensity_frame is deprecated, must be NULL."; - CHECK(prev_intensity_frame == NULL) + ABSL_CHECK(prev_intensity_frame == NULL) << "Parameter prev_intensity_frame is deprecated, must be NULL."; RegionFlowFeatureList feature_list; @@ -796,7 +799,7 @@ class EstimateMotionIRLSInvoker { break; case MotionEstimation::MODEL_NUM_VALUES: - LOG(FATAL) << "Function should not be called with this value"; + ABSL_LOG(FATAL) << "Function should not be called with this value"; break; } } @@ -821,11 +824,11 @@ void MotionEstimation::EstimateMotionsParallelImpl( std::vector* camera_motions) const { MEASURE_TIME << "Estimate motions: " << feature_lists->size(); - CHECK(feature_lists != nullptr); - CHECK(camera_motions != nullptr); + ABSL_CHECK(feature_lists != nullptr); + ABSL_CHECK(camera_motions != nullptr); const int num_frames = feature_lists->size(); - CHECK_EQ(num_frames, camera_motions->size()); + ABSL_CHECK_EQ(num_frames, camera_motions->size()); // Initialize camera_motions. for (int f = 0; f < num_frames; ++f) { @@ -866,7 +869,7 @@ void MotionEstimation::EstimateMotionsParallelImpl( const int num_motion_models = use_joint_tracks ? options_.joint_track_estimation().num_motion_models() : 1; - CHECK_GT(num_motion_models, 0); + ABSL_CHECK_GT(num_motion_models, 0); // Several single track clip datas, we seek to process. std::vector clip_datas(num_motion_models); @@ -941,8 +944,8 @@ void MotionEstimation::EstimateMotionsParallelImpl( if (options_.long_feature_initialization().activated()) { if (!feature_list.long_tracks()) { - LOG(ERROR) << "Requesting long feature initialization but " - << "input is not computed with long features."; + ABSL_LOG(ERROR) << "Requesting long feature initialization but " + << "input is not computed with long features."; } else { LongFeatureInitialization(feature_list, long_feature_info, track_length_importance, &irls_weight_input); @@ -1080,7 +1083,8 @@ void MotionEstimation::EstimateMotionsParallelImpl( // Estimate mixtures across a spectrum a different regularizers, from the // weakest to the most regularized one. const int num_mixture_levels = options_.mixture_regularizer_levels(); - CHECK_LE(num_mixture_levels, 10) << "Only up to 10 mixtures are supported."; + ABSL_CHECK_LE(num_mixture_levels, 10) + << "Only up to 10 mixtures are supported."; // Initialize to weakest regularizer. float regularizer = options_.mixture_regularizer(); @@ -1124,8 +1128,8 @@ void MotionEstimation::EstimateMotionsParallelImpl( // Check that mixture spectrum has sufficient entries. for (const CameraMotion& motion : *camera_motions) { if (motion.mixture_homography_spectrum_size() > 0) { - CHECK_EQ(motion.mixture_homography_spectrum_size(), - options_.mixture_regularizer_levels()); + ABSL_CHECK_EQ(motion.mixture_homography_spectrum_size(), + options_.mixture_regularizer_levels()); } } @@ -1164,7 +1168,7 @@ bool MotionEstimation::EstimateMotionModels( const EstimateModelOptions& model_options, const MotionEstimationThreadStorage* thread_storage, std::vector* clip_datas) const { - CHECK(clip_datas != nullptr); + ABSL_CHECK(clip_datas != nullptr); const int num_datas = clip_datas->size(); if (num_datas == 0) { @@ -1266,7 +1270,7 @@ bool MotionEstimation::EstimateMotionModels( // Traverse frames in order. for (int k = 0; k < clip_data.num_frames(); ++k) { if (clip_data.feature_lists->at(k)->feature_size() > 0) { - CHECK(clip_data.feature_lists->at(k)->long_tracks()) + ABSL_CHECK(clip_data.feature_lists->at(k)->long_tracks()) << "Estimation policy TEMPORAL_LONG_FEATURE_BIAS requires " << "tracking with long tracks."; } @@ -1281,7 +1285,7 @@ bool MotionEstimation::EstimateMotionModels( } if (clip_data.camera_motions->at(k).type() <= max_unstable_type) { - CHECK(clip_data.prior_weights[k].use_full_prior); + ABSL_CHECK(clip_data.prior_weights[k].use_full_prior); clip_data.prior_weights[k].alphas.assign(irls_per_round, 1.0f); clip_data.prior_weights[k].alphas.back() = 0.0; } @@ -1570,7 +1574,7 @@ class IrlsInitializationInvoker { // Initialize priors from irls weights. if (use_prior_weights) { - CHECK_LT(frame, clip_data_->prior_weights.size()); + ABSL_CHECK_LT(frame, clip_data_->prior_weights.size()); if (clip_data_->prior_weights[frame].priors.empty()) { clip_data_->prior_weights[frame].priors.resize( @@ -1604,13 +1608,13 @@ void MotionEstimation::LongFeatureInitialization( const LongFeatureInfo& feature_info, const std::vector& track_length_importance, std::vector* irls_weights) const { - CHECK(irls_weights); + ABSL_CHECK(irls_weights); const int num_features = feature_list.feature_size(); if (num_features == 0) { return; } - CHECK_EQ(num_features, irls_weights->size()); + ABSL_CHECK_EQ(num_features, irls_weights->size()); // Determine actual scale to be applied to each feature. std::vector feature_scales(num_features); @@ -1642,9 +1646,9 @@ void MotionEstimation::LongFeatureInitialization( void MotionEstimation::FeatureDensityNormalization( const RegionFlowFeatureList& feature_list, std::vector* irls_weights) const { - CHECK(irls_weights); + ABSL_CHECK(irls_weights); const int num_features = feature_list.feature_size(); - CHECK_EQ(num_features, irls_weights->size()); + ABSL_CHECK_EQ(num_features, irls_weights->size()); // Compute mask index for each feature. std::vector bin_indices; @@ -1707,13 +1711,13 @@ void MotionEstimation::FeatureDensityNormalization( float normalizer = 0; int bin_idx = int_grid_y * mask_size + int_grid_x; - CHECK_LT(bin_idx, max_bins); + ABSL_CHECK_LT(bin_idx, max_bins); // See above. normalizer += bin_normalizer[bin_idx] * (1 - dx_plus_dy + dxdy); normalizer += bin_normalizer[bin_idx + inc_x] * (dx - dxdy); bin_idx += mask_size * inc_y; - CHECK_LT(bin_idx, max_bins); + ABSL_CHECK_LT(bin_idx, max_bins); normalizer += bin_normalizer[bin_idx] * (dy - dxdy); normalizer += bin_normalizer[bin_idx + inc_x] * dxdy; @@ -1738,8 +1742,8 @@ void MotionEstimation::IrlsInitialization( SingleTrackClipData* clip_data) const { if (options_.estimation_policy() == MotionEstimationOptions::TEMPORAL_LONG_FEATURE_BIAS) { - CHECK_NE(frame, -1) << "Only per frame processing for this policy " - << "supported."; + ABSL_CHECK_NE(frame, -1) << "Only per frame processing for this policy " + << "supported."; } IrlsInitializationInvoker invoker(type, max_unstable_type, model_options, @@ -1763,8 +1767,8 @@ void MotionEstimation::IrlsInitialization( for_function(0, clip_data->num_frames(), 1, invoker); } else { - CHECK_GE(frame, 0); - CHECK_LT(frame, clip_data->num_frames()); + ABSL_CHECK_GE(frame, 0); + ABSL_CHECK_LT(frame, clip_data->num_frames()); invoker(BlockedRange(frame, frame + 1, 1)); } } @@ -1843,7 +1847,7 @@ void MotionEstimation::MinFilterIrlsWeightByTrack( void MotionEstimation::EnforceTrackConsistency( std::vector* clip_datas) const { - CHECK(clip_datas != nullptr); + ABSL_CHECK(clip_datas != nullptr); if (clip_datas->empty()) { return; } @@ -1888,7 +1892,7 @@ void MotionEstimation::EnforceTrackConsistency( void MotionEstimation::BiasFromFeatures( const RegionFlowFeatureList& feature_list, MotionType type, const EstimateModelOptions& model_options, std::vector* bias) const { - CHECK(bias); + ABSL_CHECK(bias); const int num_features = feature_list.feature_size(); bias->resize(num_features); @@ -1928,8 +1932,8 @@ void MotionEstimation::BiasLongFeatures( RegionFlowFeatureList* feature_list, MotionType type, const EstimateModelOptions& model_options, PriorFeatureWeights* prior_weights) const { - CHECK(prior_weights); - CHECK(feature_list); + ABSL_CHECK(prior_weights); + ABSL_CHECK(feature_list); // Don't bias duplicated frames -> should be identity transform. if (feature_list->is_duplicated()) { @@ -1943,11 +1947,11 @@ void MotionEstimation::BiasLongFeatures( // Bias along long tracks. if (!prior_weights->use_full_prior) { - LOG_IF(WARNING, - []() { - static int k = 0; - return k++ < 2; - }()) + ABSL_LOG_IF(WARNING, + []() { + static int k = 0; + return k++ < 2; + }()) << "Use full prior overridden to true, no initialization used. " << "Atypical usage."; prior_weights->use_full_prior = true; @@ -1955,12 +1959,13 @@ void MotionEstimation::BiasLongFeatures( const int num_features = feature_list->feature_size(); if (prior_weights->priors.empty() && num_features > 0) { - LOG(WARNING) << "BiasLongFeatures without using IrlsOutlierInitialization " - << "or LongFeatureInitialization."; + ABSL_LOG(WARNING) + << "BiasLongFeatures without using IrlsOutlierInitialization " + << "or LongFeatureInitialization."; prior_weights->priors.resize(num_features, 1.0f); } - CHECK_EQ(num_features, prior_weights->priors.size()); + ABSL_CHECK_EQ(num_features, prior_weights->priors.size()); for (int k = 0; k < num_features; ++k) { prior_weights->priors[k] *= bias[k]; auto* feature = feature_list->mutable_feature(k); @@ -1993,7 +1998,7 @@ void MotionEstimation::ComputeSpatialBias( BuildFeatureGrid(NormalizedDomain().x(), NormalizedDomain().y(), bias_options.grid_size(), {feature_view}, FeatureLocation, &feature_taps_3, nullptr, nullptr, &feature_grids); - CHECK_EQ(1, feature_grids.size()); + ABSL_CHECK_EQ(1, feature_grids.size()); const FeatureGrid& single_grid = feature_grids[0]; const float long_track_threshold = bias_options.long_track_threshold(); @@ -2059,8 +2064,8 @@ void MotionEstimation::ComputeSpatialBias( } } - DCHECK(spatial_bias->find(feature_ptr->track_id()) == - spatial_bias->end()); + ABSL_DCHECK(spatial_bias->find(feature_ptr->track_id()) == + spatial_bias->end()); // Threshold such that few similar tracks do not count. // Set to 0.25% of features. @@ -2118,7 +2123,7 @@ void MotionEstimation::UpdateLongFeatureBias( const auto& bias_options = options_.long_feature_bias_options(); const int num_irls_observations = bias_options.num_irls_observations(); - CHECK_GT(num_irls_observations, 0) << "Specify value > 0"; + ABSL_CHECK_GT(num_irls_observations, 0) << "Specify value > 0"; const float inv_num_irls_observations = 1.0f / num_irls_observations; SpatialBiasMap spatial_bias; @@ -2137,7 +2142,7 @@ void MotionEstimation::UpdateLongFeatureBias( // Scale applied to irls weight for linear interpolation between inlier and // outlier bias. - CHECK_GT(bias_options.inlier_irls_weight(), 0); + ABSL_CHECK_GT(bias_options.inlier_irls_weight(), 0); const float irls_scale = 1.0f / bias_options.inlier_irls_weight(); const float long_track_scale = 1.0f / bias_options.long_track_confidence_fraction(); @@ -2231,7 +2236,8 @@ void MotionEstimation::UpdateLongFeatureBias( // Update feature's weight as well. feature.set_irls_weight(1.0f / (biased_weight + kIrlsEps)); } else { - CHECK(!update_irls_observation) << "Should never happen on >= 2nd round"; + ABSL_CHECK(!update_irls_observation) + << "Should never happen on >= 2nd round"; // Not present, reset to spatial bias. const float biased_weight = spatial_bias[feature.track_id()].first; @@ -2256,7 +2262,7 @@ void MotionEstimation::UpdateLongFeatureBias( } void MotionEstimation::SmoothIRLSWeights(std::deque* irls) const { - CHECK(irls != nullptr); + ABSL_CHECK(irls != nullptr); if (irls->empty()) { return; } @@ -2316,7 +2322,7 @@ int MotionEstimation::IRLSRoundsFromSettings(const MotionType& type) const { const int irls_rounds = options_.irls_rounds(); switch (type) { case MODEL_AVERAGE_MAGNITUDE: - LOG(WARNING) << "Called with irls free motion type. Returning zero."; + ABSL_LOG(WARNING) << "Called with irls free motion type. Returning zero."; return 0; case MODEL_TRANSLATION: @@ -2340,7 +2346,8 @@ int MotionEstimation::IRLSRoundsFromSettings(const MotionType& type) const { case MotionEstimationOptions::ESTIMATION_LS_L2_RANSAC: case MotionEstimationOptions::ESTIMATION_LS_L1: - LOG(FATAL) << "Deprecated options, use ESTIMATION_LS_IRLS instead."; + ABSL_LOG(FATAL) + << "Deprecated options, use ESTIMATION_LS_IRLS instead."; return -1; } break; @@ -2385,18 +2392,19 @@ int MotionEstimation::IRLSRoundsFromSettings(const MotionType& type) const { break; case MODEL_NUM_VALUES: - LOG(FATAL) << "Function should never be called with this value"; + ABSL_LOG(FATAL) << "Function should never be called with this value"; break; } - LOG(FATAL) << "All branches above return, execution can not reach this point"; + ABSL_LOG(FATAL) + << "All branches above return, execution can not reach this point"; return -1; } void MotionEstimation::PolicyToIRLSRounds(int irls_rounds, int* total_rounds, int* irls_per_round) const { - CHECK(total_rounds != nullptr); - CHECK(irls_per_round != nullptr); + ABSL_CHECK(total_rounds != nullptr); + ABSL_CHECK(irls_per_round != nullptr); // Small optimization: irls_rounds == 0 -> total_rounds = 0 regardless of // settings. @@ -2430,13 +2438,13 @@ void MotionEstimation::CheckModelStability( const std::vector>* reset_irls_weights, std::vector* feature_lists, std::vector* camera_motions) const { - CHECK(feature_lists != nullptr); - CHECK(camera_motions != nullptr); + ABSL_CHECK(feature_lists != nullptr); + ABSL_CHECK(camera_motions != nullptr); const int num_frames = feature_lists->size(); if (reset_irls_weights) { - DCHECK_EQ(num_frames, reset_irls_weights->size()); + ABSL_DCHECK_EQ(num_frames, reset_irls_weights->size()); } - DCHECK_EQ(num_frames, camera_motions->size()); + ABSL_DCHECK_EQ(num_frames, camera_motions->size()); for (int f = 0; f < num_frames; ++f) { CameraMotion& camera_motion = (*camera_motions)[f]; @@ -2462,7 +2470,7 @@ void MotionEstimation::CheckSingleModelStability( switch (type) { case MODEL_AVERAGE_MAGNITUDE: - LOG(WARNING) << "Nothing to check for requested model type."; + ABSL_LOG(WARNING) << "Nothing to check for requested model type."; return; case MODEL_TRANSLATION: @@ -2470,7 +2478,7 @@ void MotionEstimation::CheckSingleModelStability( camera_motion->translation_variance(), *feature_list)) { // Translation can never be singular. - CHECK_EQ( + ABSL_CHECK_EQ( 0, camera_motion->flags() & CameraMotion::FLAG_SINGULAR_ESTIMATION); } else { // Invalid model. @@ -2551,8 +2559,8 @@ void MotionEstimation::CheckSingleModelStability( case CameraMotion::INVALID: case CameraMotion::UNSTABLE_HOMOG: - LOG(FATAL) << "Unexpected CameraMotion::Type: " - << camera_motion->type(); + ABSL_LOG(FATAL) + << "Unexpected CameraMotion::Type: " << camera_motion->type(); break; } @@ -2575,21 +2583,21 @@ void MotionEstimation::CheckSingleModelStability( } case MODEL_NUM_VALUES: - LOG(FATAL) << "Function should not be called with this value"; + ABSL_LOG(FATAL) << "Function should not be called with this value"; break; } } void MotionEstimation::ProjectMotionsDown( const MotionType& type, std::vector* camera_motions) const { - CHECK(camera_motions != nullptr); + ABSL_CHECK(camera_motions != nullptr); for (auto& camera_motion : *camera_motions) { switch (type) { case MODEL_AVERAGE_MAGNITUDE: case MODEL_TRANSLATION: case MODEL_MIXTURE_HOMOGRAPHY: case MODEL_AFFINE: - LOG(WARNING) << "Nothing to project for requested model type"; + ABSL_LOG(WARNING) << "Nothing to project for requested model type"; return; case MODEL_HOMOGRAPHY: @@ -2620,7 +2628,7 @@ void MotionEstimation::ProjectMotionsDown( break; case MODEL_NUM_VALUES: - LOG(FATAL) << "Function should not be called with this value"; + ABSL_LOG(FATAL) << "Function should not be called with this value"; break; } } @@ -2628,7 +2636,7 @@ void MotionEstimation::ProjectMotionsDown( void MotionEstimation::IRLSWeightFilter( std::vector* feature_lists) const { - CHECK(feature_lists != nullptr); + ABSL_CHECK(feature_lists != nullptr); for (auto feature_ptr : *feature_lists) { switch (options_.irls_weight_filter()) { case MotionEstimationOptions::IRLS_FILTER_TEXTURE: @@ -2655,7 +2663,7 @@ void MotionEstimation::EstimateMotionsParallel( bool post_irls_weight_smoothing, std::vector* feature_lists, std::vector* camera_motions) const { - CHECK(camera_motions != nullptr); + ABSL_CHECK(camera_motions != nullptr); camera_motions->clear(); camera_motions->resize(feature_lists->size()); @@ -2696,8 +2704,8 @@ void MotionEstimation::EstimateMotionsParallel( void MotionEstimation::DetermineShotBoundaries( const std::vector& feature_lists, std::vector* camera_motions) const { - CHECK(camera_motions != nullptr); - CHECK_EQ(feature_lists.size(), camera_motions->size()); + ABSL_CHECK(camera_motions != nullptr); + ABSL_CHECK_EQ(feature_lists.size(), camera_motions->size()); const auto& shot_options = options_.shot_boundary_options(); // Verify empty feature frames and invalid models via visual consistency. @@ -2758,7 +2766,7 @@ void MotionEstimation::DetermineShotBoundaries( void MotionEstimation::ResetMotionModels(const MotionEstimationOptions& options, CameraMotion* camera_motion) { - CHECK(camera_motion); + ABSL_CHECK(camera_motion); // Clear models. camera_motion->clear_translation(); @@ -3011,8 +3019,8 @@ Vector2_f EstimateTranslationModelDouble( void MotionEstimation::ComputeFeatureMask( const RegionFlowFeatureList& feature_list, std::vector* mask_indices, std::vector* bin_normalizer) const { - CHECK(mask_indices != nullptr); - CHECK(bin_normalizer != nullptr); + ABSL_CHECK(mask_indices != nullptr); + ABSL_CHECK(bin_normalizer != nullptr); const int num_features = feature_list.feature_size(); mask_indices->clear(); @@ -3047,7 +3055,7 @@ bool MotionEstimation::GetTranslationIrlsInitialization( RegionFlowFeatureList* feature_list, const EstimateModelOptions& model_options, float avg_camera_motion, InlierMask* inlier_mask, TranslationModel* best_model) const { - CHECK(best_model != nullptr); + ABSL_CHECK(best_model != nullptr); const int num_features = feature_list->feature_size(); if (!num_features) { @@ -3163,7 +3171,7 @@ void MotionEstimation::EstimateTranslationModelIRLS( CameraMotion* camera_motion) const { if (prior_weights && !prior_weights->HasCorrectDimension( irls_rounds, flow_feature_list->feature_size())) { - LOG(ERROR) << "Prior weights incorrectly initialized, ignoring."; + ABSL_LOG(ERROR) << "Prior weights incorrectly initialized, ignoring."; prior_weights = nullptr; } @@ -3269,9 +3277,9 @@ LinearSimilarityModel LinearSimilarityL2SolveSystem( const RegionFlowFeatureList& feature_list, Eigen::Matrix* matrix, Eigen::Matrix* rhs, Eigen::Matrix* solution, bool* success) { - CHECK(matrix != nullptr); - CHECK(rhs != nullptr); - CHECK(solution != nullptr); + ABSL_CHECK(matrix != nullptr); + ABSL_CHECK(rhs != nullptr); + ABSL_CHECK(solution != nullptr); *matrix = Eigen::Matrix::Zero(); *rhs = Eigen::Matrix::Zero(); @@ -3352,7 +3360,7 @@ bool MotionEstimation::GetSimilarityIrlsInitialization( RegionFlowFeatureList* feature_list, const EstimateModelOptions& model_options, float avg_camera_motion, InlierMask* inlier_mask, LinearSimilarityModel* best_model) const { - CHECK(best_model != nullptr); + ABSL_CHECK(best_model != nullptr); const int num_features = feature_list->feature_size(); if (!num_features) { @@ -3483,8 +3491,8 @@ bool MotionEstimation::GetSimilarityIrlsInitialization( void MotionEstimation::ComputeSimilarityInliers( const RegionFlowFeatureList& feature_list, int* num_inliers, int* num_strict_inliers) const { - CHECK(num_inliers); - CHECK(num_strict_inliers); + ABSL_CHECK(num_inliers); + ABSL_CHECK(num_strict_inliers); const auto& similarity_bounds = options_.stable_similarity_bounds(); @@ -3493,11 +3501,11 @@ void MotionEstimation::ComputeSimilarityInliers( float threshold = std::max(similarity_bounds.inlier_threshold(), similarity_bounds.frac_inlier_threshold() * hypot(frame_width_, frame_height_)); - CHECK_GT(threshold, 0); + ABSL_CHECK_GT(threshold, 0); threshold = 1.0f / threshold; float strict_threshold = similarity_bounds.strict_inlier_threshold(); - CHECK_GT(strict_threshold, 0); + ABSL_CHECK_GT(strict_threshold, 0); strict_threshold = 1.0f / strict_threshold; if (!options_.irls_use_l0_norm()) { @@ -3524,7 +3532,7 @@ bool MotionEstimation::EstimateLinearSimilarityModelIRLS( CameraMotion* camera_motion) const { if (prior_weights && !prior_weights->HasCorrectDimension( irls_rounds, flow_feature_list->feature_size())) { - LOG(ERROR) << "Prior weights incorrectly initialized, ignoring."; + ABSL_LOG(ERROR) << "Prior weights incorrectly initialized, ignoring."; prior_weights = nullptr; } @@ -3759,14 +3767,14 @@ bool HomographyL2QRSolve( float perspective_regularizer, Eigen::Matrix* matrix, // tmp matrix Eigen::Matrix* solution) { - CHECK(matrix); - CHECK(solution); - CHECK_EQ(8, matrix->cols()); + ABSL_CHECK(matrix); + ABSL_CHECK(solution); + ABSL_CHECK_EQ(8, matrix->cols()); const int num_rows = 2 * feature_list.feature_size() + (perspective_regularizer == 0 ? 0 : 1); - CHECK_EQ(num_rows, matrix->rows()); - CHECK_EQ(1, solution->cols()); - CHECK_EQ(8, solution->rows()); + ABSL_CHECK_EQ(num_rows, matrix->rows()); + ABSL_CHECK_EQ(1, solution->cols()); + ABSL_CHECK_EQ(8, solution->rows()); // Compute homography from features (H * location = prev_location). *matrix = Eigen::Matrix::Zero(matrix->rows(), 8); @@ -3848,9 +3856,9 @@ Homography HomographyL2NormalEquationSolve( float perspective_regularizer, Eigen::Matrix* matrix, Eigen::Matrix* rhs, Eigen::Matrix* solution, bool* success) { - CHECK(matrix != nullptr); - CHECK(rhs != nullptr); - CHECK(solution != nullptr); + ABSL_CHECK(matrix != nullptr); + ABSL_CHECK(rhs != nullptr); + ABSL_CHECK(solution != nullptr); *matrix = Eigen::Matrix::Zero(); *rhs = Eigen::Matrix::Zero(); @@ -4054,8 +4062,8 @@ bool MixtureHomographyL2DLTSolve( const MixtureRowWeights& row_weights, float regularizer_lambda, Eigen::MatrixXf* matrix, // least squares matrix Eigen::MatrixXf* solution) { - CHECK(matrix); - CHECK(solution); + ABSL_CHECK(matrix); + ABSL_CHECK(solution); // cv::solve can hang for really bad conditioned systems. const double feature_irls_sum = RegionFlowFeatureIRLSSum(feature_list); @@ -4066,11 +4074,12 @@ bool MixtureHomographyL2DLTSolve( const int num_dof = 8 * num_models; const int num_constraints = num_dof - 8; - CHECK_EQ(matrix->cols(), num_dof); + ABSL_CHECK_EQ(matrix->cols(), num_dof); // 2 Rows (x,y) per feature. - CHECK_EQ(matrix->rows(), 2 * feature_list.feature_size() + num_constraints); - CHECK_EQ(solution->cols(), 1); - CHECK_EQ(solution->rows(), num_dof); + ABSL_CHECK_EQ(matrix->rows(), + 2 * feature_list.feature_size() + num_constraints); + ABSL_CHECK_EQ(solution->cols(), 1); + ABSL_CHECK_EQ(solution->rows(), num_dof); // Compute homography from features. (H * location = prev_location) *matrix = Eigen::MatrixXf::Zero(matrix->rows(), matrix->cols()); @@ -4150,8 +4159,8 @@ bool TransMixtureHomographyL2DLTSolve( const MixtureRowWeights& row_weights, float regularizer_lambda, Eigen::MatrixXf* matrix, // least squares matrix Eigen::MatrixXf* solution) { - CHECK(matrix); - CHECK(solution); + ABSL_CHECK(matrix); + ABSL_CHECK(solution); // cv::solve can hang for really bad conditioned systems. const double feature_irls_sum = RegionFlowFeatureIRLSSum(feature_list); @@ -4162,11 +4171,12 @@ bool TransMixtureHomographyL2DLTSolve( const int num_dof = 6 + 2 * num_models; const int num_constraints = 2 * (num_models - 1); - CHECK_EQ(matrix->cols(), num_dof); + ABSL_CHECK_EQ(matrix->cols(), num_dof); // 2 Rows (x,y) per feature. - CHECK_EQ(matrix->rows(), 2 * feature_list.feature_size() + num_constraints); - CHECK_EQ(solution->cols(), 1); - CHECK_EQ(solution->rows(), num_dof); + ABSL_CHECK_EQ(matrix->rows(), + 2 * feature_list.feature_size() + num_constraints); + ABSL_CHECK_EQ(solution->cols(), 1); + ABSL_CHECK_EQ(solution->rows(), num_dof); // Compute homography from features. (H * location = prev_location) *matrix = Eigen::MatrixXf::Zero(matrix->rows(), matrix->cols()); @@ -4249,8 +4259,8 @@ bool SkewRotMixtureHomographyL2DLTSolve( const MixtureRowWeights& row_weights, float regularizer_lambda, Eigen::MatrixXf* matrix, // least squares matrix Eigen::MatrixXf* solution) { - CHECK(matrix); - CHECK(solution); + ABSL_CHECK(matrix); + ABSL_CHECK(solution); // cv::solve can hang for really bad conditioned systems. const double feature_irls_sum = RegionFlowFeatureIRLSSum(feature_list); @@ -4261,11 +4271,12 @@ bool SkewRotMixtureHomographyL2DLTSolve( const int num_dof = 4 + 4 * num_models; const int num_constraints = 4 * (num_models - 1); - CHECK_EQ(matrix->cols(), num_dof); + ABSL_CHECK_EQ(matrix->cols(), num_dof); // 2 Rows (x,y) per feature. - CHECK_EQ(matrix->rows(), 2 * feature_list.feature_size() + num_constraints); - CHECK_EQ(solution->cols(), 1); - CHECK_EQ(solution->rows(), num_dof); + ABSL_CHECK_EQ(matrix->rows(), + 2 * feature_list.feature_size() + num_constraints); + ABSL_CHECK_EQ(solution->cols(), 1); + ABSL_CHECK_EQ(solution->rows(), num_dof); // Compute homography from features. (H * location = prev_location) *matrix = Eigen::MatrixXf::Zero(matrix->rows(), matrix->cols()); @@ -4349,7 +4360,7 @@ bool SkewRotMixtureHomographyL2DLTSolve( void MotionEstimation::GetHomographyIRLSCenterWeights( const RegionFlowFeatureList& feature_list, std::vector* weights) const { - CHECK(weights != nullptr); + ABSL_CHECK(weights != nullptr); const int num_features = feature_list.feature_size(); weights->clear(); @@ -4382,7 +4393,7 @@ void MotionEstimation::GetHomographyIRLSCenterWeights( weights->push_back(1.0f - weight * 0.5f); break; default: - LOG(INFO) << "Unsupported IRLS weighting."; + ABSL_LOG(INFO) << "Unsupported IRLS weighting."; } } } @@ -4436,7 +4447,7 @@ bool MotionEstimation::IsStableTranslation( void MotionEstimation::CheckTranslationAcceleration( std::vector* camera_motions) const { - CHECK(camera_motions != nullptr); + ABSL_CHECK(camera_motions != nullptr); std::vector magnitudes; for (const auto& motion : *camera_motions) { const float translation_magnitude = @@ -4658,7 +4669,7 @@ bool MotionEstimation::IsStableMixtureHomography( float MotionEstimation::GridCoverage( const RegionFlowFeatureList& feature_list, float min_inlier_score, MotionEstimationThreadStorage* thread_storage) const { - CHECK(thread_storage != nullptr); + ABSL_CHECK(thread_storage != nullptr); // 10x10 grid for coverage estimation. const int grid_size = options_.coverage_grid_size(); @@ -4669,7 +4680,7 @@ float MotionEstimation::GridCoverage( const std::vector& grid_cell_weights = thread_storage->GridCoverageInitializationWeights(); - CHECK_EQ(mask_size, grid_cell_weights.size()); + ABSL_CHECK_EQ(mask_size, grid_cell_weights.size()); const float max_inlier_score = 1.75f * min_inlier_score; const float mid_inlier_score = 0.5 * (min_inlier_score + max_inlier_score); @@ -4694,7 +4705,7 @@ float MotionEstimation::GridCoverage( normalized_domain_.x() / grid_size * overlap_x / num_overlaps; std::vector>& irls_mask = *thread_storage->EmptyGridCoverageIrlsMask(); - CHECK_EQ(mask_size, irls_mask.size()); + ABSL_CHECK_EQ(mask_size, irls_mask.size()); // Bin features. for (const auto& feature : feature_list.feature()) { @@ -4738,7 +4749,7 @@ float MotionEstimation::GridCoverage( const float cell_weight_sum = std::accumulate(grid_cell_weights.begin(), grid_cell_weights.end(), 0.0f); - CHECK_GT(cell_weight_sum, 0); + ABSL_CHECK_GT(cell_weight_sum, 0); return std::inner_product(max_coverage.begin(), max_coverage.end(), grid_cell_weights.begin(), 0.0f) / @@ -4863,7 +4874,7 @@ bool MotionEstimation::EstimateHomographyIRLS( RegionFlowFeatureList* feature_list, CameraMotion* camera_motion) const { if (prior_weights && !prior_weights->HasCorrectDimension( irls_rounds, feature_list->feature_size())) { - LOG(ERROR) << "Prior weights incorrectly initialized, ignoring."; + ABSL_LOG(ERROR) << "Prior weights incorrectly initialized, ignoring."; prior_weights = nullptr; } @@ -4961,13 +4972,13 @@ bool MotionEstimation::EstimateHomographyIRLS( } else { bool success = false; if (options_.use_highest_accuracy_for_normal_equations()) { - CHECK(!use_float); + ABSL_CHECK(!use_float); norm_model = HomographyL2NormalEquationSolve( *feature_list, prev_solution, options_.homography_perspective_regularizer(), &matrix_d, &rhs_d, &solution_d, &success); } else { - CHECK(use_float); + ABSL_CHECK(use_float); norm_model = HomographyL2NormalEquationSolve( *feature_list, prev_solution, options_.homography_perspective_regularizer(), &matrix_f, &rhs_f, @@ -5079,7 +5090,7 @@ bool MotionEstimation::MixtureHomographyFromFeature( MixtureHomography* mix_homography) const { if (prior_weights && !prior_weights->HasCorrectDimension( irls_rounds, feature_list->feature_size())) { - LOG(ERROR) << "Prior weights incorrectly initialized, ignoring."; + ABSL_LOG(ERROR) << "Prior weights incorrectly initialized, ignoring."; prior_weights = nullptr; } @@ -5087,9 +5098,9 @@ bool MotionEstimation::MixtureHomographyFromFeature( // Compute weights if necessary. // Compute scale to index mixture weights from normalization. - CHECK(row_weights_.get() != nullptr); - CHECK_EQ(row_weights_->YScale(), frame_height_ / normalized_domain_.y()); - CHECK_EQ(row_weights_->NumModels(), num_mixtures); + ABSL_CHECK(row_weights_.get() != nullptr); + ABSL_CHECK_EQ(row_weights_->YScale(), frame_height_ / normalized_domain_.y()); + ABSL_CHECK_EQ(row_weights_->NumModels(), num_mixtures); const MotionEstimationOptions::MixtureModelMode mixture_mode = options_.mixture_model_mode(); @@ -5109,7 +5120,7 @@ bool MotionEstimation::MixtureHomographyFromFeature( adjacency_constraints = 4 * (num_mixtures - 1); break; default: - LOG(FATAL) << "Unknown MixtureModelMode specified."; + ABSL_LOG(FATAL) << "Unknown MixtureModelMode specified."; } Eigen::MatrixXf matrix( @@ -5195,7 +5206,7 @@ bool MotionEstimation::MixtureHomographyFromFeature( break; default: - LOG(FATAL) << "Unknown MixtureModelMode specified."; + ABSL_LOG(FATAL) << "Unknown MixtureModelMode specified."; } norm_model = MixtureHomographyAdapter::FromFloatPointer( @@ -5264,7 +5275,7 @@ bool MotionEstimation::MixtureHomographyFromFeature( mix_homography->set_dof(MixtureHomography::SKEW_ROTATION_DOF); break; default: - LOG(FATAL) << "Unknown MixtureModelMode specified."; + ABSL_LOG(FATAL) << "Unknown MixtureModelMode specified."; } return true; } @@ -5363,8 +5374,8 @@ bool MotionEstimation::EstimateMixtureHomographyIRLS( // Cap rolling shutter analysis level to be valid level. if (options_.mixture_rs_analysis_level() >= options_.mixture_regularizer_levels()) { - LOG(WARNING) << "Resetting mixture_rs_analysis_level to " - << options_.mixture_regularizer_levels() - 1; + ABSL_LOG(WARNING) << "Resetting mixture_rs_analysis_level to " + << options_.mixture_regularizer_levels() - 1; } const int rs_analysis_level = @@ -5439,12 +5450,12 @@ bool MotionEstimation::EstimateMixtureHomographyIRLS( void MotionEstimation::DetermineOverlayIndices( bool irls_weights_preinitialized, std::vector* camera_motions, std::vector* feature_lists) const { - CHECK(camera_motions != nullptr); - CHECK(feature_lists != nullptr); + ABSL_CHECK(camera_motions != nullptr); + ABSL_CHECK(feature_lists != nullptr); // Two stage estimation: First translation only, followed by // overlay analysis. const int num_frames = feature_lists->size(); - CHECK_EQ(num_frames, camera_motions->size()); + ABSL_CHECK_EQ(num_frames, camera_motions->size()); std::vector translation_motions(num_frames); const int irls_per_round = options_.irls_rounds(); @@ -5519,9 +5530,9 @@ float MotionEstimation::OverlayAnalysis( const std::vector& translations, std::vector* feature_lists, std::vector* overlay_indices) const { - CHECK(feature_lists != nullptr); - CHECK(overlay_indices != nullptr); - CHECK_EQ(feature_lists->size(), translations.size()); + ABSL_CHECK(feature_lists != nullptr); + ABSL_CHECK(overlay_indices != nullptr); + ABSL_CHECK_EQ(feature_lists->size(), translations.size()); overlay_indices->clear(); const int grid_size = @@ -5609,7 +5620,7 @@ float MotionEstimation::OverlayAnalysis( void MotionEstimation::PostIRLSSmoothing( const std::vector& camera_motions, std::vector* feature_lists) const { - CHECK(feature_lists != nullptr); + ABSL_CHECK(feature_lists != nullptr); std::vector> feature_grids; std::vector> feature_taps_3; @@ -5689,7 +5700,7 @@ void TemporalIRLSPush(const FeatureGrid& curr_grid, float grid_scale, int grid_dim_x, RegionFlowFeatureView* curr_view, RegionFlowFeatureView* prev_view) { - CHECK(curr_view != nullptr); + ABSL_CHECK(curr_view != nullptr); // Spatial filtering of inverse irls weights and the temporally weighted // pushed result from the next frame. for (auto& feature : *curr_view) { @@ -5717,7 +5728,7 @@ void TemporalIRLSPush(const FeatureGrid& curr_grid, } // Only zero if spatial AND feature sigma = 0. - DCHECK_GT(weight_sum, 0); + ABSL_DCHECK_GT(weight_sum, 0); feature->mutable_internal_irls()->set_weight_sum(weight_sum); feature->mutable_internal_irls()->set_value_sum(value_sum); } @@ -5829,7 +5840,7 @@ void TemporalIRLSPull(const FeatureGrid& curr_grid, } } - CHECK_GT(weight_sum, 0) << feature->irls_weight(); + ABSL_CHECK_GT(weight_sum, 0) << feature->irls_weight(); feature->mutable_internal_irls()->set_weight_sum(weight_sum); feature->mutable_internal_irls()->set_value_sum(value_sum); } @@ -5847,7 +5858,7 @@ void TemporalIRLSPull(const FeatureGrid& curr_grid, void MotionEstimation::InitGaussLUT(float sigma, float max_range, std::vector* lut, float* scale) const { - CHECK(lut); + ABSL_CHECK(lut); // Calculate number of bins if scale is non-zero, otherwise use one bin per // integer in the domain [0, max_range]. const int lut_bins = (scale != nullptr) ? (1 << 10) : std::ceil(max_range); diff --git a/mediapipe/util/tracking/motion_models.cc b/mediapipe/util/tracking/motion_models.cc index eb6a8b31..898b7e06 100644 --- a/mediapipe/util/tracking/motion_models.cc +++ b/mediapipe/util/tracking/motion_models.cc @@ -22,6 +22,8 @@ #include "Eigen/Core" #include "Eigen/Dense" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" // Set to true to use catmull rom mixture weights instead of Gaussian weights @@ -43,10 +45,10 @@ AffineModel ModelAdapter::ToAffine( TranslationModel ModelAdapter::FromAffine( const AffineModel& model) { - DCHECK_EQ(model.a(), 1); - DCHECK_EQ(model.b(), 0); - DCHECK_EQ(model.c(), 0); - DCHECK_EQ(model.d(), 1); + ABSL_DCHECK_EQ(model.a(), 1); + ABSL_DCHECK_EQ(model.b(), 0); + ABSL_DCHECK_EQ(model.c(), 0); + ABSL_DCHECK_EQ(model.d(), 1); return TranslationAdapter::FromArgs(model.dx(), model.dy()); } @@ -63,7 +65,7 @@ TranslationModel ModelAdapter::FromHomography( void ModelAdapter::GetJacobianAtPoint(const Vector2_f& pt, float* jacobian) { - DCHECK(jacobian); + ABSL_DCHECK(jacobian); jacobian[0] = 1; jacobian[1] = 0; jacobian[2] = 0; @@ -115,7 +117,7 @@ SimilarityModel ModelAdapter::FromArgs(float dx, float dy, SimilarityModel ModelAdapter::FromFloatPointer( const float* args, bool identity_parametrization) { - DCHECK(args); + ABSL_DCHECK(args); SimilarityModel model; model.set_dx(args[0]); model.set_dy(args[1]); @@ -126,7 +128,7 @@ SimilarityModel ModelAdapter::FromFloatPointer( SimilarityModel ModelAdapter::FromDoublePointer( const double* args, bool identity_parametrization) { - DCHECK(args); + ABSL_DCHECK(args); SimilarityModel model; model.set_dx(args[0]); model.set_dy(args[1]); @@ -151,7 +153,7 @@ SimilarityModel ModelAdapter::Invert( bool success = true; const SimilarityModel result = InvertChecked(model, &success); if (!success) { - LOG(ERROR) << "Model not invertible. Returning identity."; + ABSL_LOG(ERROR) << "Model not invertible. Returning identity."; return SimilarityModel(); } else { return result; @@ -218,7 +220,7 @@ float ModelAdapter::GetParameter(const SimilarityModel& model, case 3: return model.rotation(); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } return 0; @@ -235,7 +237,7 @@ std::string ModelAdapter::ToString( SimilarityModel ModelAdapter::NormalizationTransform( float frame_width, float frame_height) { const float scale = std::hypot(frame_width, frame_height); - DCHECK_NE(scale, 0); + ABSL_DCHECK_NE(scale, 0); return SimilarityAdapter::FromArgs(0, 0, 1.0 / scale, 0); } @@ -262,8 +264,8 @@ AffineModel ModelAdapter::ToAffine( LinearSimilarityModel ModelAdapter::FromAffine( const AffineModel& model) { - DCHECK_EQ(model.a(), model.d()); - DCHECK_EQ(model.b(), -model.c()); + ABSL_DCHECK_EQ(model.a(), model.d()); + ABSL_DCHECK_EQ(model.b(), -model.c()); return LinearSimilarityAdapter::FromArgs(model.dx(), model.dy(), model.a(), -model.b()); @@ -313,7 +315,7 @@ LinearSimilarityModel ModelAdapter::AddIdentity( void ModelAdapter::GetJacobianAtPoint( const Vector2_f& pt, float* jacobian) { - DCHECK(jacobian); + ABSL_DCHECK(jacobian); // First row. jacobian[0] = 1; jacobian[1] = 0; @@ -330,7 +332,7 @@ LinearSimilarityModel ModelAdapter::NormalizationTransform( float frame_width, float frame_height) { const float scale = std::hypot(frame_width, frame_height); - DCHECK_NE(scale, 0); + ABSL_DCHECK_NE(scale, 0); return LinearSimilarityAdapter::FromArgs(0, 0, 1.0 / scale, 0); } @@ -368,7 +370,7 @@ std::string ModelAdapter::ToString(const AffineModel& model) { AffineModel ModelAdapter::NormalizationTransform( float frame_width, float frame_height) { const float scale = std::hypot(frame_width, frame_height); - DCHECK_NE(scale, 0); + ABSL_DCHECK_NE(scale, 0); return AffineAdapter::FromArgs(0, 0, 1.0f / scale, 0, 0, 1.0f / scale); } @@ -379,8 +381,8 @@ Homography ModelAdapter::ToHomography(const AffineModel& model) { } AffineModel ModelAdapter::FromHomography(const Homography& model) { - DCHECK_EQ(model.h_20(), 0); - DCHECK_EQ(model.h_21(), 0); + ABSL_DCHECK_EQ(model.h_20(), 0); + ABSL_DCHECK_EQ(model.h_21(), 0); float params[6] = {model.h_02(), model.h_12(), // dx, dy model.h_00(), model.h_01(), // a, b @@ -411,7 +413,7 @@ AffineModel ModelAdapter::AddIdentity( void ModelAdapter::GetJacobianAtPoint(const Vector2_f& pt, float* jacobian) { - DCHECK(jacobian); + ABSL_DCHECK(jacobian); // First row. jacobian[0] = 1; jacobian[1] = 0; @@ -550,7 +552,7 @@ Homography ModelAdapter::InvertChecked(const Homography& model, Eigen::Matrix3d inv_model_mat = model_mat.inverse(); if (inv_model_mat(2, 2) == 0) { - LOG(ERROR) << "Degenerate homography. See proto."; + ABSL_LOG(ERROR) << "Degenerate homography. See proto."; *success = false; return Homography(); } @@ -582,8 +584,8 @@ std::string ModelAdapter::ToString(const Homography& model) { } AffineModel ModelAdapter::ToAffine(const Homography& model) { - DCHECK_EQ(model.h_20(), 0); - DCHECK_EQ(model.h_21(), 0); + ABSL_DCHECK_EQ(model.h_20(), 0); + ABSL_DCHECK_EQ(model.h_21(), 0); AffineModel affine_model; affine_model.set_a(model.h_00()); affine_model.set_b(model.h_01()); @@ -604,7 +606,7 @@ bool ModelAdapter::IsAffine(const Homography& model) { void ModelAdapter::GetJacobianAtPoint(const Vector2_f& pt, float* jacobian) { - DCHECK(jacobian); + ABSL_DCHECK(jacobian); // First row. jacobian[0] = pt.x(); jacobian[1] = pt.y(); @@ -629,7 +631,7 @@ void ModelAdapter::GetJacobianAtPoint(const Vector2_f& pt, Homography ModelAdapter::NormalizationTransform( float frame_width, float frame_height) { const float scale = std::hypot(frame_width, frame_height); - DCHECK_NE(scale, 0); + ABSL_DCHECK_NE(scale, 0); return HomographyAdapter::FromArgs(1.0f / scale, 0, 0, 0, 1.0f / scale, 0, 0, 0); } @@ -730,7 +732,7 @@ float ModelMethods::NormalizedIntersectionArea(const Model& model_1, const Vector2_f& rect) { const float rect_area = rect.x() * rect.y(); if (rect_area <= 0) { - LOG(WARNING) << "Empty rectangle passed -> empty intersection."; + ABSL_LOG(WARNING) << "Empty rectangle passed -> empty intersection."; return 0.0f; } @@ -756,7 +758,7 @@ float ModelMethods::NormalizedIntersectionArea(const Model& model_1, const float average_area = 0.5f * (model_1_area + model_2_area); if (average_area <= 0) { - LOG(WARNING) << "Degenerative models passed -> empty intersection."; + ABSL_LOG(WARNING) << "Degenerative models passed -> empty intersection."; return 0.0f; } @@ -764,7 +766,7 @@ float ModelMethods::NormalizedIntersectionArea(const Model& model_1, bool success = true; Model diff = ModelDiffChecked(model_2, model_1, &success); if (!success) { - LOG(WARNING) << "Model difference is singular -> empty intersection."; + ABSL_LOG(WARNING) << "Model difference is singular -> empty intersection."; return 0.0f; } @@ -786,7 +788,7 @@ float ModelMethods::NormalizedIntersectionArea(const Model& model_1, // Second, clip transformed rectangle against origin defined by model_2. Model inv_diff = Adapter::InvertChecked(diff, &success); if (!success) { - LOG(WARNING) << "Model difference is singular -> empty intersection."; + ABSL_LOG(WARNING) << "Model difference is singular -> empty intersection."; return 0.0f; } @@ -829,10 +831,11 @@ MixtureRowWeights::MixtureRowWeights(int frame_height, int margin, float sigma, // No margin support for splines. if (margin_ > 0) { - LOG(WARNING) << "No margin support when flag catmull_rom_mixture_weights " - << "is set. Margin is reset to zero, it is recommended " - << "that RowWeightsBoundChecked is used to prevent " - << "segfaults."; + ABSL_LOG(WARNING) + << "No margin support when flag catmull_rom_mixture_weights " + << "is set. Margin is reset to zero, it is recommended " + << "that RowWeightsBoundChecked is used to prevent " + << "segfaults."; margin_ = 0; } @@ -860,7 +863,7 @@ MixtureRowWeights::MixtureRowWeights(int frame_height, int margin, float sigma, weight_ptr[int_pos] += spline_weights[0]; // Double knot. } - CHECK_LT(int_pos, num_models - 1); + ABSL_CHECK_LT(int_pos, num_models - 1); weight_ptr[int_pos + 1] += spline_weights[2]; if (int_pos + 1 < num_models - 1) { weight_ptr[int_pos + 2] += spline_weights[3]; @@ -897,7 +900,7 @@ MixtureRowWeights::MixtureRowWeights(int frame_height, int margin, float sigma, } // Normalize. - DCHECK_GT(weight_sum, 0); + ABSL_DCHECK_GT(weight_sum, 0); const float inv_weight_sum = 1.0f / weight_sum; for (int j = 0; j < num_models; ++j) { weight_ptr[j] *= inv_weight_sum; diff --git a/mediapipe/util/tracking/motion_models.h b/mediapipe/util/tracking/motion_models.h index 567831ad..b0272f97 100644 --- a/mediapipe/util/tracking/motion_models.h +++ b/mediapipe/util/tracking/motion_models.h @@ -21,6 +21,8 @@ #include #include "absl/container/node_hash_map.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/singleton.h" #include "mediapipe/framework/port/vector.h" @@ -50,7 +52,7 @@ class ModelAdapter { static Model InvertChecked(const Model& model, bool* success); // Returns model^(-1), returns identity model if inversion is not possible, - // and warns via LOG(ERROR). It is recommended that InvertChecked is used + // and warns via ABSL_LOG(ERROR). It is recommended that InvertChecked is used // instead. // Note: Default implementation, motion models only need to supply above // function. @@ -704,8 +706,8 @@ bool ModelDiffWithinBounds(const Model& ground_truth, const Model& predicted, ModelAdapter::GetParameter(identity, p)); if (diff_p > bound) { - LOG(WARNING) << "Param diff " << p << " out of bounds: " << diff_p - << " > " << bound << " bound"; + ABSL_LOG(WARNING) << "Param diff " << p << " out of bounds: " << diff_p + << " > " << bound << " bound"; return false; } } @@ -762,8 +764,8 @@ Model UniformModelParameters(const float value) { template Model BlendModels(const Model& a, const Model& b, float weight_b) { Model blended; - DCHECK_GE(weight_b, 0); - DCHECK_LE(weight_b, 1); + ABSL_DCHECK_GE(weight_b, 0); + ABSL_DCHECK_LE(weight_b, 1); const float weight_a = 1 - weight_b; for (int p = 0; p < ModelAdapter::NumParameters(); ++p) { const float pa = ModelAdapter::GetParameter(a, p); @@ -822,8 +824,8 @@ class MixtureRowWeights { const float* RowWeights(float y) const { int bin_y = y * y_scale_ + 0.5; - DCHECK_LT(bin_y, frame_height_ + margin_); - DCHECK_GE(bin_y, -margin_); + ABSL_DCHECK_LT(bin_y, frame_height_ + margin_); + ABSL_DCHECK_GE(bin_y, -margin_); return &weights_[(bin_y + margin_) * num_models_]; } @@ -866,7 +868,7 @@ inline MixtureRowWeights* MixtureRowWeightsFromCameraMotion( template void SmoothModels(const Model& sigma_time_model, const Model* model_sigma, std::vector* models) { - CHECK(models); + ABSL_CHECK(models); const int num_models = models->size(); @@ -966,7 +968,7 @@ inline TranslationModel ModelAdapter::FromArgs(float dx, inline TranslationModel ModelAdapter::FromFloatPointer( const float* args, bool) { - DCHECK(args); + ABSL_DCHECK(args); TranslationModel model; model.set_dx(args[0]); model.set_dy(args[1]); @@ -975,7 +977,7 @@ inline TranslationModel ModelAdapter::FromFloatPointer( inline TranslationModel ModelAdapter::FromDoublePointer( const double* args, bool) { - DCHECK(args); + ABSL_DCHECK(args); TranslationModel model; model.set_dx(args[0]); model.set_dy(args[1]); @@ -992,7 +994,7 @@ inline TranslationModel ModelAdapter::Invert( bool success = true; TranslationModel result = InvertChecked(model, &success); if (!success) { - LOG(ERROR) << "Model not invertible. Returning identity."; + ABSL_LOG(ERROR) << "Model not invertible. Returning identity."; return TranslationModel(); } @@ -1024,7 +1026,7 @@ inline float ModelAdapter::GetParameter( case 1: return model.dy(); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } return 0; } @@ -1037,7 +1039,7 @@ inline void ModelAdapter::SetParameter( case 1: return model->set_dy(value); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } } @@ -1055,7 +1057,7 @@ inline LinearSimilarityModel ModelAdapter::FromArgs( inline LinearSimilarityModel ModelAdapter::FromFloatPointer( const float* args, bool identity_parametrization) { - DCHECK(args); + ABSL_DCHECK(args); LinearSimilarityModel model; const float id_shift = identity_parametrization ? 1.f : 0.f; model.set_dx(args[0]); @@ -1068,7 +1070,7 @@ ModelAdapter::FromFloatPointer( inline LinearSimilarityModel ModelAdapter::FromDoublePointer( const double* args, bool identity_parametrization) { - DCHECK(args); + ABSL_DCHECK(args); LinearSimilarityModel model; const float id_shift = identity_parametrization ? 1.f : 0.f; model.set_dx(args[0]); @@ -1089,7 +1091,7 @@ inline LinearSimilarityModel ModelAdapter::Invert( bool success = true; LinearSimilarityModel result = InvertChecked(model, &success); if (!success) { - LOG(ERROR) << "Model not invertible. Returning identity."; + ABSL_LOG(ERROR) << "Model not invertible. Returning identity."; return LinearSimilarityModel(); } else { return result; @@ -1143,7 +1145,7 @@ inline float ModelAdapter::GetParameter( case 3: return model.b(); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } return 0; @@ -1161,7 +1163,7 @@ inline void ModelAdapter::SetParameter( case 3: return model->set_b(value); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } } @@ -1181,7 +1183,7 @@ inline AffineModel ModelAdapter::FromArgs(float dx, float dy, inline AffineModel ModelAdapter::FromFloatPointer( const float* args, bool identity_parametrization) { - DCHECK(args); + ABSL_DCHECK(args); AffineModel model; const float id_shift = identity_parametrization ? 1.f : 0.f; model.set_dx(args[0]); @@ -1195,7 +1197,7 @@ inline AffineModel ModelAdapter::FromFloatPointer( inline AffineModel ModelAdapter::FromDoublePointer( const double* args, bool identity_parametrization) { - DCHECK(args); + ABSL_DCHECK(args); AffineModel model; const float id_shift = identity_parametrization ? 1.f : 0.f; model.set_dx(args[0]); @@ -1218,7 +1220,7 @@ inline AffineModel ModelAdapter::Invert(const AffineModel& model) { bool success = true; AffineModel result = InvertChecked(model, &success); if (!success) { - LOG(ERROR) << "Model not invertible. Returning identity."; + ABSL_LOG(ERROR) << "Model not invertible. Returning identity."; return AffineModel(); } else { return result; @@ -1279,7 +1281,7 @@ inline float ModelAdapter::GetParameter(const AffineModel& model, case 5: return model.d(); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } return 0; @@ -1301,7 +1303,7 @@ inline void ModelAdapter::SetParameter(int id, float value, case 5: return model->set_d(value); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } } @@ -1324,7 +1326,7 @@ inline Homography ModelAdapter::FromArgs(float h_00, float h_01, inline Homography ModelAdapter::FromFloatPointer( const float* args, bool identity_parametrization) { - DCHECK(args); + ABSL_DCHECK(args); Homography model; const float id_shift = identity_parametrization ? 1.f : 0.f; model.set_h_00(id_shift + args[0]); @@ -1340,7 +1342,7 @@ inline Homography ModelAdapter::FromFloatPointer( inline Homography ModelAdapter::FromDoublePointer( const double* args, bool identity_parametrization) { - DCHECK(args); + ABSL_DCHECK(args); Homography model; const float id_shift = identity_parametrization ? 1.f : 0.f; model.set_h_00(id_shift + args[0]); @@ -1364,8 +1366,8 @@ inline Vector2_f ModelAdapter::TransformPoint( // Enforce z can not assume very small values. constexpr float eps = 1e-12f; if (fabs(z) < eps) { - LOG(ERROR) << "Point mapped to infinity. " - << "Degenerate homography. See proto."; + ABSL_LOG(ERROR) << "Point mapped to infinity. " + << "Degenerate homography. See proto."; z = z >= 0 ? eps : -eps; } return Vector2_f(x / z, y / z); @@ -1386,7 +1388,7 @@ inline Homography ModelAdapter::Invert(const Homography& model) { bool success = true; Homography result = InvertChecked(model, &success); if (!success) { - LOG(ERROR) << "Model not invertible. Returning identity."; + ABSL_LOG(ERROR) << "Model not invertible. Returning identity."; return Homography(); } else { return result; @@ -1398,7 +1400,7 @@ inline Homography ModelAdapter::Compose(const Homography& lhs, Homography result; const float z = lhs.h_20() * rhs.h_02() + lhs.h_21() * rhs.h_12() + 1.0f * 1.0f; - CHECK_NE(z, 0) << "Degenerate homography. See proto."; + ABSL_CHECK_NE(z, 0) << "Degenerate homography. See proto."; const float inv_z = 1.0 / z; result.set_h_00((lhs.h_00() * rhs.h_00() + lhs.h_01() * rhs.h_10() + @@ -1450,7 +1452,7 @@ inline float ModelAdapter::GetParameter(const Homography& model, case 7: return model.h_21(); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } return 0; @@ -1476,7 +1478,7 @@ inline void ModelAdapter::SetParameter(int id, float value, case 7: return model->set_h_21(value); default: - LOG(FATAL) << "Parameter id is out of bounds"; + ABSL_LOG(FATAL) << "Parameter id is out of bounds"; } } @@ -1631,7 +1633,7 @@ MixtureModelAdapterBase::LinearModel( } const double denom = sum_xx - inv_models * sum_x * sum_x; - CHECK_NE(denom, 0); // As num_models > 1. + ABSL_CHECK_NE(denom, 0); // As num_models > 1. const double a = (sum_xy - inv_models * sum_x * sum_y) * denom; const double b = inv_models * (sum_y - a * sum_x); @@ -1688,7 +1690,7 @@ Vector2_f MixtureModelAdapter::TransformPoint( BaseModelAdapter::TransformPoint3(model.model(i), pt3 * weights[i]); } - DCHECK_NE(result.z(), 0) << "Degenerate mapping."; + ABSL_DCHECK_NE(result.z(), 0) << "Degenerate mapping."; return Vector2_f(result.x() / result.z(), result.y() / result.z()); } @@ -1767,7 +1769,7 @@ inline Homography MixtureModelAdapter::ToBaseModel( case MixtureHomography::CONST_DOF: return const_homog; default: - LOG(FATAL) << "Unknown type."; + ABSL_LOG(FATAL) << "Unknown type."; } return HomographyAdapter::FromFloatPointer(params, false); @@ -1815,10 +1817,10 @@ inline Vector2_f MixtureModelAdapter::TransformPoint( case MixtureHomography::CONST_DOF: return HomographyAdapter::TransformPoint(model.model(0), pt); default: - LOG(FATAL) << "Unknown type."; + ABSL_LOG(FATAL) << "Unknown type."; } - DCHECK_NE(result.z(), 0) << "Degenerate mapping."; + ABSL_DCHECK_NE(result.z(), 0) << "Degenerate mapping."; return Vector2_f(result.x() / result.z(), result.y() / result.z()); } diff --git a/mediapipe/util/tracking/motion_models_cv.cc b/mediapipe/util/tracking/motion_models_cv.cc index b9b428ad..e11132b3 100644 --- a/mediapipe/util/tracking/motion_models_cv.cc +++ b/mediapipe/util/tracking/motion_models_cv.cc @@ -14,6 +14,8 @@ #include "mediapipe/util/tracking/motion_models_cv.h" +#include "absl/log/absl_check.h" + namespace mediapipe { void ModelCvConvert::ToCvMat(const TranslationModel& model, @@ -41,7 +43,7 @@ void ModelCvConvert::ToCvMat(const AffineModel& model, void ModelCvConvert::ToCvMat(const Homography& model, cv::Mat* matrix) { - CHECK(matrix != nullptr); + ABSL_CHECK(matrix != nullptr); matrix->create(3, 3, CV_32FC1); matrix->at(0, 0) = model.h_00(); matrix->at(0, 1) = model.h_01(); diff --git a/mediapipe/util/tracking/motion_saliency.cc b/mediapipe/util/tracking/motion_saliency.cc index 5adafca4..44f4ec5e 100644 --- a/mediapipe/util/tracking/motion_saliency.cc +++ b/mediapipe/util/tracking/motion_saliency.cc @@ -24,7 +24,8 @@ #include #include -#include "mediapipe/framework/port/logging.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/util/tracking/camera_motion.h" #include "mediapipe/util/tracking/measure_time.h" #include "mediapipe/util/tracking/region_flow.h" @@ -44,12 +45,12 @@ void MotionSaliency::SaliencyFromFeatures( const RegionFlowFeatureList& feature_list, std::vector* irls_weights, // optional. SalientPointFrame* salient_frame) { - CHECK(salient_frame); - CHECK_EQ(frame_width_, feature_list.frame_width()); - CHECK_EQ(frame_height_, feature_list.frame_height()); + ABSL_CHECK(salient_frame); + ABSL_CHECK_EQ(frame_width_, feature_list.frame_width()); + ABSL_CHECK_EQ(frame_height_, feature_list.frame_height()); if (irls_weights) { - CHECK_EQ(feature_list.feature_size(), irls_weights->size()); + ABSL_CHECK_EQ(feature_list.feature_size(), irls_weights->size()); } if (feature_list.feature_size() < 1) { @@ -105,8 +106,8 @@ void MotionSaliency::SaliencyFromPoints(const std::vector* points, const std::vector* weights, SalientPointFrame* salient_frame) { // TODO: Handle vectors of size zero. - CHECK(salient_frame); - CHECK_EQ(points->size(), weights->size()); + ABSL_CHECK(salient_frame); + ABSL_CHECK_EQ(points->size(), weights->size()); float max_weight = *std::max_element(weights->begin(), weights->end()); @@ -212,7 +213,7 @@ void MotionSaliency::SelectSaliencyInliers( void MotionSaliency::FilterMotionSaliency( std::vector* saliency_point_list) { - CHECK(saliency_point_list != nullptr); + ABSL_CHECK(saliency_point_list != nullptr); const float sigma_time = options_.filtering_sigma_time(); const float sigma_space = options_.filtering_sigma_space(); @@ -329,7 +330,7 @@ void MotionSaliency::FilterMotionSaliency( void MotionSaliency::CollapseMotionSaliency( const SaliencyPointList& input_saliency, const Vector4_f& bounds, SaliencyPointList* output_saliency) { - CHECK(output_saliency); + ABSL_CHECK(output_saliency); output_saliency->clear(); output_saliency->resize(input_saliency.size()); @@ -378,8 +379,8 @@ void DetermineFeatureModes( const std::vector& space_lut, float space_scale, std::vector>* mode_grid, std::vector* mode_ptrs) { - CHECK(mode_grid); - CHECK(mode_ptrs); + ABSL_CHECK(mode_grid); + ABSL_CHECK(mode_ptrs); const int num_features = features.size(); mode_ptrs->reserve(num_features); @@ -417,8 +418,8 @@ void DetermineFeatureModes( center = new_center; } } else { - LOG(WARNING) << "No features found in band_width radius, " - << "should not happen. "; + ABSL_LOG(WARNING) << "No features found in band_width radius, " + << "should not happen. "; break; } } @@ -439,8 +440,8 @@ void DetermineFeatureModes( void MotionSaliency::SalientModeFinding(std::vector* locations, std::vector* modes) { - CHECK(modes); - CHECK(locations); + ABSL_CHECK(modes); + ABSL_CHECK(locations); if (locations->empty()) { return; } @@ -477,7 +478,7 @@ void MotionSaliency::SalientModeFinding(std::vector* locations, nullptr, &grid_dims, &feature_grids); // Just one frame input, expect one grid as output. - CHECK_EQ(1, feature_grids.size()); + ABSL_CHECK_EQ(1, feature_grids.size()); const auto& feature_grid = feature_grids[0]; // Setup Gaussian LUT for smoothing in space, using 2^10 discretization bins. @@ -595,8 +596,8 @@ void MotionSaliency::SalientModeFinding(std::vector* locations, if (angle < 0) { angle += M_PI; } - CHECK_GE(angle, 0); - CHECK_LE(angle, M_PI + 1e-3); + ABSL_CHECK_GE(angle, 0); + ABSL_CHECK_LE(angle, M_PI + 1e-3); } SalientMode irls_mode; @@ -622,7 +623,7 @@ void MotionSaliency::SalientModeFinding(std::vector* locations, // mode finding and scales each point based on frame size. void MotionSaliency::DetermineSalientFrame( std::vector locations, SalientPointFrame* salient_frame) { - CHECK(salient_frame); + ABSL_CHECK(salient_frame); std::vector modes; { @@ -660,12 +661,12 @@ void ForegroundWeightsFromFeatures(const RegionFlowFeatureList& feature_list, float foreground_gamma, const CameraMotion* camera_motion, std::vector* weights) { - CHECK(weights != nullptr); + ABSL_CHECK(weights != nullptr); weights->clear(); constexpr float kEpsilon = 1e-4f; - CHECK_GT(foreground_threshold, 0.0f); + ABSL_CHECK_GT(foreground_threshold, 0.0f); if (camera_motion) { foreground_threshold *= std::max(kEpsilon, InlierCoverage(*camera_motion, false)); @@ -694,7 +695,7 @@ void ForegroundWeightsFromFeatures(const RegionFlowFeatureList& feature_list, std::max(kEpsilon, std::pow(foreground_measure, foreground_gamma))); } } - CHECK_EQ(feature_list.feature_size(), weights->size()); + ABSL_CHECK_EQ(feature_list.feature_size(), weights->size()); } } // namespace mediapipe diff --git a/mediapipe/util/tracking/parallel_invoker.h b/mediapipe/util/tracking/parallel_invoker.h index 82352231..a00b5223 100644 --- a/mediapipe/util/tracking/parallel_invoker.h +++ b/mediapipe/util/tracking/parallel_invoker.h @@ -71,8 +71,9 @@ #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/synchronization/mutex.h" -#include "mediapipe/framework/port/logging.h" #ifdef PARALLEL_INVOKER_ACTIVE #include "mediapipe/framework/port/threadpool.h" @@ -233,13 +234,13 @@ inline void CheckAndSetInvokerOptions() { flags_parallel_invoker_mode != PARALLEL_INVOKER_THREAD_POOL && flags_parallel_invoker_mode != PARALLEL_INVOKER_OPENMP) { #if defined(_OPENMP) - LOG(WARNING) << "Unsupported invoker mode selected on Android. " - << "OpenMP linkage detected, so falling back to OpenMP"; + ABSL_LOG(WARNING) << "Unsupported invoker mode selected on Android. " + << "OpenMP linkage detected, so falling back to OpenMP"; flags_parallel_invoker_mode = PARALLEL_INVOKER_OPENMP; #else // _OPENMP // Fallback mode for active parallel invoker without OpenMP is ThreadPool. - LOG(WARNING) << "Unsupported invoker mode selected on Android. " - << "Falling back to ThreadPool"; + ABSL_LOG(WARNING) << "Unsupported invoker mode selected on Android. " + << "Falling back to ThreadPool"; flags_parallel_invoker_mode = PARALLEL_INVOKER_THREAD_POOL; #endif // _OPENMP } @@ -253,8 +254,8 @@ inline void CheckAndSetInvokerOptions() { flags_parallel_invoker_mode != PARALLEL_INVOKER_GCD && #endif // USE_PARALLEL_INVOKER_GCD flags_parallel_invoker_mode != PARALLEL_INVOKER_THREAD_POOL) { - LOG(WARNING) << "Unsupported invoker mode selected on iOS. " - << "Falling back to ThreadPool mode"; + ABSL_LOG(WARNING) << "Unsupported invoker mode selected on iOS. " + << "Falling back to ThreadPool mode"; flags_parallel_invoker_mode = PARALLEL_INVOKER_THREAD_POOL; } #endif // __APPLE__ || __EMSCRIPTEN__ @@ -267,24 +268,27 @@ inline void CheckAndSetInvokerOptions() { // to ThreadPool if not. if (flags_parallel_invoker_mode == PARALLEL_INVOKER_OPENMP) { #if !defined(_OPENMP) - LOG(ERROR) << "OpenMP invoker mode selected but not compiling with OpenMP " - << "enabled. Falling back to ThreadPool"; + ABSL_LOG(ERROR) + << "OpenMP invoker mode selected but not compiling with OpenMP " + << "enabled. Falling back to ThreadPool"; flags_parallel_invoker_mode = PARALLEL_INVOKER_THREAD_POOL; #endif // _OPENMP } #else // PARALLEL_INVOKER_ACTIVE if (flags_parallel_invoker_mode != PARALLEL_INVOKER_NONE) { - LOG(ERROR) << "Parallel execution requested but PARALLEL_INVOKER_ACTIVE " - << "compile flag is not set. Falling back to single threaded " - << "execution."; + ABSL_LOG(ERROR) + << "Parallel execution requested but PARALLEL_INVOKER_ACTIVE " + << "compile flag is not set. Falling back to single threaded " + << "execution."; flags_parallel_invoker_mode = PARALLEL_INVOKER_NONE; } #endif // PARALLEL_INVOKER_ACTIVE - CHECK_LT(flags_parallel_invoker_mode, PARALLEL_INVOKER_MAX_VALUE) + ABSL_CHECK_LT(flags_parallel_invoker_mode, PARALLEL_INVOKER_MAX_VALUE) + << "Invalid invoker mode specified."; + ABSL_CHECK_GE(flags_parallel_invoker_mode, 0) << "Invalid invoker mode specified."; - CHECK_GE(flags_parallel_invoker_mode, 0) << "Invalid invoker mode specified."; } // Performs parallel iteration from [start to end), scheduling grain_size @@ -301,7 +305,7 @@ void ParallelFor(size_t start, size_t end, size_t grain_size, #if defined(__APPLE__) case PARALLEL_INVOKER_GCD: { int iterations_remain = (end - start + grain_size - 1) / grain_size; - CHECK_GT(iterations_remain, 0); + ABSL_CHECK_GT(iterations_remain, 0); if (iterations_remain == 1) { // Execute invoker serially. invoker(BlockedRange(start, std::min(end, start + grain_size), 1)); @@ -313,7 +317,7 @@ void ParallelFor(size_t start, size_t end, size_t grain_size, dispatch_apply_f(iterations_remain, concurrent_queue, &context, ParallelForGCDTask); #if CHECK_GCD_PARALLEL_WORK_COUNT - CHECK_EQ(iterations_remain, context.count()); + ABSL_CHECK_EQ(iterations_remain, context.count()); #endif } break; @@ -322,7 +326,7 @@ void ParallelFor(size_t start, size_t end, size_t grain_size, case PARALLEL_INVOKER_THREAD_POOL: { int iterations_remain = (end - start + grain_size - 1) / grain_size; - CHECK_GT(iterations_remain, 0); + ABSL_CHECK_GT(iterations_remain, 0); if (iterations_remain == 1) { // Execute invoker serially. invoker(BlockedRange(start, std::min(end, start + grain_size), 1)); @@ -385,7 +389,7 @@ void ParallelFor(size_t start, size_t end, size_t grain_size, } case PARALLEL_INVOKER_MAX_VALUE: { - LOG(FATAL) << "Impossible."; + ABSL_LOG(FATAL) << "Impossible."; break; } } @@ -413,7 +417,7 @@ void ParallelFor2D(size_t start_row, size_t end_row, size_t start_col, case PARALLEL_INVOKER_GCD: { const int iterations_remain = (end_row - start_row + grain_size - 1) / grain_size; - CHECK_GT(iterations_remain, 0); + ABSL_CHECK_GT(iterations_remain, 0); if (iterations_remain == 1) { // Execute invoker serially. invoker(BlockedRange2D(BlockedRange(start_row, end_row, 1), @@ -427,7 +431,7 @@ void ParallelFor2D(size_t start_row, size_t end_row, size_t start_col, dispatch_apply_f(iterations_remain, concurrent_queue, &context, ParallelForGCDTask2D); #if CHECK_GCD_PARALLEL_WORK_COUNT - CHECK_EQ(iterations_remain, context.count()); + ABSL_CHECK_EQ(iterations_remain, context.count()); #endif } break; @@ -436,7 +440,7 @@ void ParallelFor2D(size_t start_row, size_t end_row, size_t start_col, case PARALLEL_INVOKER_THREAD_POOL: { int iterations_remain = end_row - start_row; // Guarded by loop_mutex - CHECK_GT(iterations_remain, 0); + ABSL_CHECK_GT(iterations_remain, 0); if (iterations_remain == 1) { // Execute invoker serially. invoker(BlockedRange2D(BlockedRange(start_row, end_row, 1), @@ -493,7 +497,7 @@ void ParallelFor2D(size_t start_row, size_t end_row, size_t start_col, } case PARALLEL_INVOKER_MAX_VALUE: { - LOG(FATAL) << "Impossible."; + ABSL_LOG(FATAL) << "Impossible."; break; } } diff --git a/mediapipe/util/tracking/push_pull_filtering.h b/mediapipe/util/tracking/push_pull_filtering.h index f9b2c6c3..80c63153 100644 --- a/mediapipe/util/tracking/push_pull_filtering.h +++ b/mediapipe/util/tracking/push_pull_filtering.h @@ -33,6 +33,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/util/tracking/image_util.h" #include "mediapipe/util/tracking/push_pull_filtering.pb.h" @@ -148,7 +150,7 @@ class PushPullFiltering { // Returns domain size of n-th pyramid level (including border depending on // filter_type). cv::Size NthPyramidDomain(int level) { - CHECK_LT(level, PyramidLevels()); + ABSL_CHECK_LT(level, PyramidLevels()); return downsample_pyramid_[level].size(); } @@ -309,7 +311,7 @@ PushPullFiltering::PushPullFiltering( weight_adjuster_(weight_adjuster) { border_ = BorderFromFilterType(filter_type); if (border_ < 0) { - LOG(FATAL) << "Unknown filter requested."; + ABSL_LOG(FATAL) << "Unknown filter requested."; } SetupFilters(); @@ -446,7 +448,7 @@ template void PushPullFiltering::AllocatePyramid( const cv::Size& domain_size, int border, int type, bool allocate_base_level, std::vector* pyramid) { - CHECK(pyramid != nullptr); + ABSL_CHECK(pyramid != nullptr); pyramid->clear(); pyramid->reserve(16); // Do not anticipate videos with dimensions // larger than 2^16. @@ -468,15 +470,15 @@ void PushPullFiltering::AllocatePyramid( template void PushPullFiltering::InitializeImagePyramid( const cv::Mat& input_frame, std::vector* pyramid) { - CHECK(pyramid != nullptr); - CHECK_GT(pyramid->size(), 0); + ABSL_CHECK(pyramid != nullptr); + ABSL_CHECK_GT(pyramid->size(), 0); cv::Mat base_level((*pyramid)[0], cv::Range(border_, (*pyramid)[0].rows - border_), cv::Range(border_, (*pyramid)[0].cols - border_)); - CHECK_EQ(base_level.rows, input_frame.rows); - CHECK_EQ(base_level.cols, input_frame.cols); - CHECK_EQ(base_level.type(), input_frame.type()); + ABSL_CHECK_EQ(base_level.rows, input_frame.rows); + ABSL_CHECK_EQ(base_level.cols, input_frame.cols); + ABSL_CHECK_EQ(base_level.type(), input_frame.type()); input_frame.copyTo(base_level); CopyNecessaryBorder(&(*pyramid)[0]); @@ -507,7 +509,7 @@ void PushPullFiltering::CopyNecessaryBorder( CopyMatBorder(mat); break; default: - LOG(FATAL) << "Unknown filter"; + ABSL_LOG(FATAL) << "Unknown filter"; } } @@ -743,11 +745,11 @@ void PushPullFiltering::PerformPushPull( cv::Point2i origin, int readout_level, const std::vector* data_weights, const cv::Mat* input_frame, cv::Mat* results) { - CHECK_EQ(data_locations.size(), data_values.size()); - CHECK(results != nullptr); + ABSL_CHECK_EQ(data_locations.size(), data_values.size()); + ABSL_CHECK(results != nullptr); if (data_weights) { - CHECK_EQ(data_weights->size(), data_locations.size()); + ABSL_CHECK_EQ(data_weights->size(), data_locations.size()); } origin.x += border_; @@ -760,13 +762,13 @@ void PushPullFiltering::PerformPushPull( mip_map[i] = &downsample_pyramid_[i]; } - CHECK_GE(readout_level, 0); - CHECK_LT(readout_level, PyramidLevels()); + ABSL_CHECK_GE(readout_level, 0); + ABSL_CHECK_LT(readout_level, PyramidLevels()); // CHECK if passed results matrix is compatible w.r.t. type and domain. - CHECK_EQ(downsample_pyramid_[readout_level].cols, results->cols); - CHECK_EQ(downsample_pyramid_[readout_level].rows, results->rows); - CHECK_EQ(downsample_pyramid_[readout_level].type(), results->type()); + ABSL_CHECK_EQ(downsample_pyramid_[readout_level].cols, results->cols); + ABSL_CHECK_EQ(downsample_pyramid_[readout_level].rows, results->rows); + ABSL_CHECK_EQ(downsample_pyramid_[readout_level].type(), results->type()); // Use caller-allocated results Mat. mip_map[readout_level] = results; @@ -806,7 +808,7 @@ void PushPullFiltering::PerformPushPullMat( int readout_level, // Default: 0. const cv::Mat* input_frame, // Optional. cv::Mat* results) { - CHECK(results != nullptr); + ABSL_CHECK(results != nullptr); // Create mip-map view (concat displacements with downsample_pyramid). std::vector mip_map(PyramidLevels()); @@ -815,18 +817,18 @@ void PushPullFiltering::PerformPushPullMat( mip_map[i] = &downsample_pyramid_[i]; } - CHECK_GE(readout_level, 0); - CHECK_LT(readout_level, PyramidLevels()); + ABSL_CHECK_GE(readout_level, 0); + ABSL_CHECK_LT(readout_level, PyramidLevels()); // CHECK if passed mip_map at level[0] is compatible w.r.t. type and domain. - CHECK_EQ(mip_map_level_0.cols, results->cols); - CHECK_EQ(mip_map_level_0.rows, results->rows); - CHECK_EQ(mip_map_level_0.type(), results->type()); + ABSL_CHECK_EQ(mip_map_level_0.cols, results->cols); + ABSL_CHECK_EQ(mip_map_level_0.rows, results->rows); + ABSL_CHECK_EQ(mip_map_level_0.type(), results->type()); // CHECK if passed results matrix is compatible w.r.t. type and domain. - CHECK_EQ(downsample_pyramid_[readout_level].cols, results->cols); - CHECK_EQ(downsample_pyramid_[readout_level].rows, results->rows); - CHECK_EQ(downsample_pyramid_[readout_level].type(), results->type()); + ABSL_CHECK_EQ(downsample_pyramid_[readout_level].cols, results->cols); + ABSL_CHECK_EQ(downsample_pyramid_[readout_level].rows, results->rows); + ABSL_CHECK_EQ(downsample_pyramid_[readout_level].type(), results->type()); // Use caller-allocated results Mat. mip_map[readout_level] = results; @@ -867,7 +869,7 @@ void PushPullFiltering::PerformPushPullImpl( filter_weights = gaussian5_weights_.data(); break; default: - LOG(FATAL) << "Unknown filter requested."; + ABSL_LOG(FATAL) << "Unknown filter requested."; } const std::vector& mip_map = *mip_map_ptr; @@ -884,7 +886,7 @@ void PushPullFiltering::PerformPushPullImpl( } if (use_bilateral_) { - CHECK(input_frame != nullptr); + ABSL_CHECK(input_frame != nullptr); InitializeImagePyramid(*input_frame, &input_frame_pyramid_); } @@ -1049,7 +1051,7 @@ void PushPullFiltering::PullDownSampling( } } - DCHECK_GE(weight_sum, 0); + ABSL_DCHECK_GE(weight_sum, 0); if (weight_sum >= kBilateralEps * kBilateralEps) { const float inv_weight_sum = 1.f / weight_sum; @@ -1131,7 +1133,7 @@ void PushPullFiltering::PushUpSampling( tap_weights, tap_offsets, tap_space_offsets); break; default: - LOG(FATAL) << "Filter unknown"; + ABSL_LOG(FATAL) << "Filter unknown"; } // Local copy for faster access. diff --git a/mediapipe/util/tracking/region_flow.cc b/mediapipe/util/tracking/region_flow.cc index cdd6bcd8..7608b76a 100644 --- a/mediapipe/util/tracking/region_flow.cc +++ b/mediapipe/util/tracking/region_flow.cc @@ -22,6 +22,8 @@ #include "absl/container/node_hash_map.h" #include "absl/container/node_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/util/tracking/measure_time.h" @@ -47,7 +49,7 @@ bool IsPointWithinBounds(const Vector2_f& pt, float bounds, int frame_width, void GetRegionFlowFeatureList(const RegionFlowFrame& region_flow_frame, int distance_from_border, RegionFlowFeatureList* flow_feature_list) { - CHECK(flow_feature_list); + ABSL_CHECK(flow_feature_list); flow_feature_list->clear_feature(); const int frame_width = region_flow_frame.frame_width(); const int frame_height = region_flow_frame.frame_height(); @@ -76,8 +78,8 @@ void GetRegionFlowFeatureList(const RegionFlowFrame& region_flow_frame, float RegionFlowFeatureDistance(const PatchDescriptor& patch_desc_1, const PatchDescriptor& patch_desc_2) { - DCHECK_EQ(patch_desc_1.data_size(), patch_desc_2.data_size()); - DCHECK_GE(patch_desc_1.data_size(), 3); + ABSL_DCHECK_EQ(patch_desc_1.data_size(), patch_desc_2.data_size()); + ABSL_DCHECK_GE(patch_desc_1.data_size(), 3); constexpr int kNumMeans = 3; float sq_distance_sum = 0; @@ -118,7 +120,7 @@ void ClampRegionFlowFeatureIRLSWeights(float lower, float upper, void ComputeRegionFlowFeatureTexturedness( const RegionFlowFeatureList& flow_feature_list, bool use_15percent_as_max, std::vector* texturedness) { - CHECK(texturedness != nullptr); + ABSL_CHECK(texturedness != nullptr); *texturedness = std::vector(flow_feature_list.feature_size(), 1.0f); int texture_idx = 0; @@ -128,11 +130,11 @@ void ComputeRegionFlowFeatureTexturedness( PatchDescriptorColorStdevL1(feature->feature_descriptor()); if (feature_stdev_l1 < 0.0f) { - LOG_IF(WARNING, - []() { - static int k = 0; - return k++ < 2; - }()) + ABSL_LOG_IF(WARNING, + []() { + static int k = 0; + return k++ < 2; + }()) << "Feature descriptor does not contain variance information. Was " << "ComputeRegionFlowFeatureDescriptors called?"; continue; @@ -200,7 +202,7 @@ void CornerFilteredRegionFlowFeatureIRLSWeights( void GetRegionFlowFeatureIRLSWeights( const RegionFlowFeatureList& flow_feature_list, std::vector* irls_weights) { - CHECK(irls_weights != nullptr); + ABSL_CHECK(irls_weights != nullptr); irls_weights->clear(); irls_weights->reserve(flow_feature_list.feature_size()); for (auto feature = flow_feature_list.feature().begin(); @@ -211,8 +213,8 @@ void GetRegionFlowFeatureIRLSWeights( void SetRegionFlowFeatureIRLSWeights(const std::vector& irls_weights, RegionFlowFeatureList* flow_feature_list) { - CHECK(flow_feature_list != nullptr); - CHECK_EQ(irls_weights.size(), flow_feature_list->feature_size()); + ABSL_CHECK(flow_feature_list != nullptr); + ABSL_CHECK_EQ(irls_weights.size(), flow_feature_list->feature_size()); int idx = 0; for (auto feature = flow_feature_list->mutable_feature()->begin(); feature != flow_feature_list->mutable_feature()->end(); @@ -284,7 +286,7 @@ void SortRegionFlowById(RegionFlowFrame* flow_frame) { void InvertRegionFlow(const RegionFlowFrame& region_flow_frame, RegionFlowFrame* inverted_flow_frame) { - CHECK(inverted_flow_frame); + ABSL_CHECK(inverted_flow_frame); inverted_flow_frame->CopyFrom(region_flow_frame); for (auto& region_flow : *inverted_flow_frame->mutable_region_flow()) { region_flow.set_centroid_x(region_flow.centroid_x() + region_flow.flow_x()); @@ -303,7 +305,7 @@ void InvertRegionFlow(const RegionFlowFrame& region_flow_frame, void InvertRegionFlowFeatureList(const RegionFlowFeatureList& feature_list, RegionFlowFeatureList* inverted_feature_list) { - CHECK(inverted_feature_list); + ABSL_CHECK(inverted_feature_list); *inverted_feature_list = feature_list; for (auto& feature : *inverted_feature_list->mutable_feature()) { InvertRegionFlowFeature(&feature); @@ -371,7 +373,7 @@ void ScaleSalientPoint(float scale_x, float scale_y, SalientPoint* sp) { void ScaleSaliencyList(float scale, bool normalize_to_scale, SaliencyPointList* saliency_list) { - CHECK(saliency_list != nullptr); + ABSL_CHECK(saliency_list != nullptr); for (auto& point_frame : *saliency_list) { ScaleSalientPointFrame(scale, normalize_to_scale, &point_frame); } @@ -379,7 +381,7 @@ void ScaleSaliencyList(float scale, bool normalize_to_scale, void ScaleSalientPointFrame(float scale, bool normalize_to_scale, SalientPointFrame* saliency) { - CHECK(saliency != nullptr); + ABSL_CHECK(saliency != nullptr); float saliency_scale = scale; if (normalize_to_scale) { float weight_sum = 0.0f; @@ -399,7 +401,7 @@ void ScaleSalientPointFrame(float scale, bool normalize_to_scale, void ResetSaliencyBounds(float left, float bottom, float right, float top, SaliencyPointList* saliency_list) { - CHECK(saliency_list != nullptr); + ABSL_CHECK(saliency_list != nullptr); for (auto& point_frame : *saliency_list) { for (auto& salient_point : *point_frame.mutable_point()) { salient_point.set_left(left); @@ -412,8 +414,8 @@ void ResetSaliencyBounds(float left, float bottom, float right, float top, bool EllipseFromCovariance(float a, float bc, float d, Vector2_f* axis_magnitude, float* angle) { - CHECK(axis_magnitude != nullptr); - CHECK(angle != nullptr); + ABSL_CHECK(axis_magnitude != nullptr); + ABSL_CHECK(angle != nullptr); // Get trace and determinant const float trace = a + d; @@ -475,7 +477,7 @@ bool EllipseFromCovariance(float a, float bc, float d, void BoundingBoxFromEllipse(const Vector2_f& center, float norm_major_axis, float norm_minor_axis, float angle, std::vector* bounding_box) { - CHECK(bounding_box != nullptr); + ABSL_CHECK(bounding_box != nullptr); float dim_x; float dim_y; if (angle < M_PI * 0.25 || angle > M_PI * 0.75) { @@ -501,8 +503,8 @@ void BoundingBoxFromEllipse(const Vector2_f& center, float norm_major_axis, void CopyToEmptyFeatureList(RegionFlowFeatureList* src, RegionFlowFeatureList* dst) { - CHECK(src != nullptr); - CHECK(dst != nullptr); + ABSL_CHECK(src != nullptr); + ABSL_CHECK(dst != nullptr); // Swap out features for empty list. RegionFlowFeatureList empty_list; @@ -515,7 +517,7 @@ void CopyToEmptyFeatureList(RegionFlowFeatureList* src, src->mutable_feature()->Swap(empty_list.mutable_feature()); // src_features should be empty as in the beginning. - CHECK_EQ(0, empty_list.feature_size()); + ABSL_CHECK_EQ(0, empty_list.feature_size()); } void IntersectRegionFlowFeatureList( @@ -523,10 +525,11 @@ void IntersectRegionFlowFeatureList( std::function to_location_eval, RegionFlowFeatureList* from, RegionFlowFeatureList* result, std::vector* source_indices) { - CHECK(from != nullptr); - CHECK(result != nullptr); - CHECK(from->long_tracks()) << "Intersection only works for long features"; - CHECK(to.long_tracks()) << "Intersection only works for long features"; + ABSL_CHECK(from != nullptr); + ABSL_CHECK(result != nullptr); + ABSL_CHECK(from->long_tracks()) + << "Intersection only works for long features"; + ABSL_CHECK(to.long_tracks()) << "Intersection only works for long features"; // Hash features in to, based on track_id. absl::node_hash_map track_map; @@ -563,9 +566,10 @@ void LongFeatureStream::AddFeatures(const RegionFlowFeatureList& feature_list, bool check_connectivity, bool purge_non_present_features) { if (!feature_list.long_tracks()) { - LOG(ERROR) << "Feature stream should be used only used with long feature " - << "tracks. Ensure POLICY_LONG_FEATURE was used for " - << "RegionFlowComputation."; + ABSL_LOG(ERROR) + << "Feature stream should be used only used with long feature " + << "tracks. Ensure POLICY_LONG_FEATURE was used for " + << "RegionFlowComputation."; return; } @@ -575,8 +579,8 @@ void LongFeatureStream::AddFeatures(const RegionFlowFeatureList& feature_list, } if (std::abs(feature_list.match_frame()) != 1) { - LOG(ERROR) << "Only matching frames one frame from current one are " - << "supported"; + ABSL_LOG(ERROR) << "Only matching frames one frame from current one are " + << "supported"; return; } @@ -584,7 +588,7 @@ void LongFeatureStream::AddFeatures(const RegionFlowFeatureList& feature_list, absl::node_hash_set present_tracks; for (auto feature : feature_list.feature()) { // Copy feature. if (feature.track_id() < 0) { - LOG_IF(WARNING, []() { + ABSL_LOG_IF(WARNING, []() { static int k = 0; return k++ < 2; }()) << "Feature does not have a valid track id assigned. Ignoring."; @@ -593,7 +597,7 @@ void LongFeatureStream::AddFeatures(const RegionFlowFeatureList& feature_list, present_tracks.insert(feature.track_id()); if (check_connectivity) { // A new feature should never have been erased before. - CHECK(old_ids_.find(feature.track_id()) == old_ids_.end()) + ABSL_CHECK(old_ids_.find(feature.track_id()) == old_ids_.end()) << "Feature : " << feature.track_id() << "was already removed."; } @@ -607,10 +611,10 @@ void LongFeatureStream::AddFeatures(const RegionFlowFeatureList& feature_list, if (find_pos != tracks_.end()) { // Track is present, add to it. if (check_connectivity) { - CHECK_LT((FeatureLocation(find_pos->second.back()) - - FeatureMatchLocation(feature)) - .Norm2(), - 1e-4); + ABSL_CHECK_LT((FeatureLocation(find_pos->second.back()) - + FeatureMatchLocation(feature)) + .Norm2(), + 1e-4); } find_pos->second.push_back(feature); } else { @@ -638,7 +642,7 @@ void LongFeatureStream::FlattenTrack( const std::vector& features, std::vector* result, std::vector* irls_weight, std::vector* flow) const { - CHECK(result != nullptr); + ABSL_CHECK(result != nullptr); if (features.empty()) { return; } @@ -700,7 +704,8 @@ std::vector LongFeatureStream::FlattenedTrackById(int id) const { void LongFeatureInfo::AddFeatures(const RegionFlowFeatureList& feature_list) { if (!feature_list.long_tracks()) { - LOG(ERROR) << "Passed feature list was not computed with long tracks. "; + ABSL_LOG(ERROR) + << "Passed feature list was not computed with long tracks. "; return; } @@ -730,7 +735,7 @@ void LongFeatureInfo::AddFeature(const RegionFlowFeature& feature) { void LongFeatureInfo::TrackLengths(const RegionFlowFeatureList& feature_list, std::vector* track_lengths) const { - CHECK(track_lengths); + ABSL_CHECK(track_lengths); const int feature_size = feature_list.feature_size(); track_lengths->resize(feature_size); for (int k = 0; k < feature_size; ++k) { @@ -775,7 +780,7 @@ int LongFeatureInfo::GlobalTrackLength(float percentile) const { void GridTaps(int dim_x, int dim_y, int tap_radius, std::vector>* taps) { - CHECK(taps); + ABSL_CHECK(taps); const int grid_size = dim_x * dim_y; const int diam = 2 * tap_radius + 1; taps->resize(grid_size); diff --git a/mediapipe/util/tracking/region_flow.h b/mediapipe/util/tracking/region_flow.h index 2f9b3422..221e1f05 100644 --- a/mediapipe/util/tracking/region_flow.h +++ b/mediapipe/util/tracking/region_flow.h @@ -24,7 +24,8 @@ #include #include -#include "mediapipe/framework/port/logging.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/vector.h" #include "mediapipe/util/tracking/motion_models.h" #include "mediapipe/util/tracking/region_flow.pb.h" @@ -82,9 +83,9 @@ inline float PatchDescriptorColorStdevL1(const PatchDescriptor& descriptor) { constexpr int kRedIdx = 3; constexpr int kGreenIdx = 6; constexpr int kBlueIdx = 8; - DCHECK_GE(descriptor.data(kRedIdx), 0); - DCHECK_GE(descriptor.data(kGreenIdx), 0); - DCHECK_GE(descriptor.data(kBlueIdx), 0); + ABSL_DCHECK_GE(descriptor.data(kRedIdx), 0); + ABSL_DCHECK_GE(descriptor.data(kGreenIdx), 0); + ABSL_DCHECK_GE(descriptor.data(kBlueIdx), 0); if (descriptor.data_size() > kBlueIdx) { return std::sqrt(descriptor.data(kRedIdx)) + @@ -118,7 +119,7 @@ double RegionFlowFeatureIRLSSum(const RegionFlowFeatureList& feature_list); // Computes per region flow feature texturedness score. Score is within [0, 1], // where 0 means low texture and 1 high texture. Requires for each feature // descriptor to be computed (via ComputeRegionFlowFeatureDescriptors). If -// missing, LOG(WARNING) is issued and value defaults to 1. +// missing, ABSL_LOG(WARNING) is issued and value defaults to 1. // If use_15percent_as_max is set, score is scaled and threshold back to [0, 1] // such that 1 is assumed at 15% of maximum PER channel variance. void ComputeRegionFlowFeatureTexturedness( @@ -234,7 +235,7 @@ template <> inline void RegionFlowFeatureListViaTransform( const MixtureHomography& mix, RegionFlowFeatureList* flow_feature_list, float a, float b, bool set_match, const MixtureRowWeights* row_weights) { - CHECK(row_weights) << "Row weights required for mixtures."; + ABSL_CHECK(row_weights) << "Row weights required for mixtures."; for (auto& feature : *flow_feature_list->mutable_feature()) { const float* weights = row_weights->RowWeights(feature.y()); @@ -275,7 +276,7 @@ std::pair GetFilteredWeightImpl(const Predicate& predicate, template int FilterRegionFlowFeatureList(const Predicate& predicate, float reset_value, RegionFlowFeatureList* flow_feature_list) { - CHECK(flow_feature_list != nullptr); + ABSL_CHECK(flow_feature_list != nullptr); int num_passing_features = 0; for (auto& feature : *flow_feature_list->mutable_feature()) { std::pair filter_result = @@ -296,7 +297,7 @@ int FilterRegionFlowFeatureWeights(const Predicate& predicate, float reset_value, const RegionFlowFeatureList& feature_list, std::vector* result_weights) { - CHECK(result_weights != nullptr); + ABSL_CHECK(result_weights != nullptr); result_weights->clear(); int num_passing_features = 0; @@ -318,8 +319,8 @@ template void SelectFeaturesFromList(const Predicate& predicate, RegionFlowFeatureList* feature_list, RegionFlowFeatureView* feature_view) { - CHECK(feature_list != nullptr); - CHECK(feature_view != nullptr); + ABSL_CHECK(feature_list != nullptr); + ABSL_CHECK(feature_view != nullptr); for (auto& feature : *feature_list->mutable_feature()) { if (predicate(feature)) { feature_view->push_back(&feature); @@ -329,8 +330,8 @@ void SelectFeaturesFromList(const Predicate& predicate, inline void SelectAllFeaturesFromList(RegionFlowFeatureList* feature_list, RegionFlowFeatureView* feature_view) { - CHECK(feature_list != nullptr); - CHECK(feature_view != nullptr); + ABSL_CHECK(feature_list != nullptr); + ABSL_CHECK(feature_view != nullptr); for (auto& feature : *feature_list->mutable_feature()) { feature_view->push_back(&feature); } @@ -342,7 +343,7 @@ inline void SelectAllFeaturesFromList(RegionFlowFeatureList* feature_list, template void SortRegionFlowFeatureView(const Predicate& predicate, RegionFlowFeatureView* feature_view) { - CHECK(feature_view != nullptr); + ABSL_CHECK(feature_view != nullptr); std::sort(feature_view->begin(), feature_view->end(), predicate); } @@ -590,8 +591,8 @@ void BuildFeatureGrid( std::vector>* feature_taps_5, // Optional. Vector2_i* num_grid_bins, // Optional. std::vector>* feature_grids) { - CHECK(feature_grids); - CHECK_GT(grid_resolution, 0.0f); + ABSL_CHECK(feature_grids); + ABSL_CHECK_GT(grid_resolution, 0.0f); const int num_frames = feature_views.size(); const int grid_dim_x = std::ceil(frame_width / grid_resolution); @@ -612,8 +613,8 @@ void BuildFeatureGrid( Vector2_f feature_loc = evaluator(*feature); const int x = feature_loc.x() * grid_scale; const int y = feature_loc.y() * grid_scale; - DCHECK_LT(y, grid_dim_y); - DCHECK_LT(x, grid_dim_x); + ABSL_DCHECK_LT(y, grid_dim_y); + ABSL_DCHECK_LT(x, grid_dim_x); const int grid_loc = y * grid_dim_x + x; curr_grid[grid_loc].push_back(feature); } diff --git a/mediapipe/util/tracking/region_flow_computation.cc b/mediapipe/util/tracking/region_flow_computation.cc index b6704cc6..e9088df2 100644 --- a/mediapipe/util/tracking/region_flow_computation.cc +++ b/mediapipe/util/tracking/region_flow_computation.cc @@ -28,6 +28,8 @@ #include "Eigen/Core" #include "absl/container/flat_hash_map.h" #include "absl/container/node_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" @@ -132,7 +134,7 @@ namespace { void GetPatchDescriptorAtPoint(const cv::Mat& rgb_frame, const Vector2_i& pt, const int radius, cv::Mat* lab_window, PatchDescriptor* descriptor) { - CHECK(descriptor); + ABSL_CHECK(descriptor); descriptor->clear_data(); // Reserve enough data for mean and upper triangular part of @@ -207,19 +209,19 @@ class PatchDescriptorInvoker { ++feature_idx) { RegionFlowFeature* feature = features_->mutable_feature(feature_idx); Vector2_i pt(FeatureIntLocation(*feature)); - DCHECK_GE(pt.x(), radius_); - DCHECK_GE(pt.y(), radius_); - DCHECK_LT(pt.x(), rgb_frame_.cols - radius_); - DCHECK_LT(pt.y(), rgb_frame_.rows - radius_); + ABSL_DCHECK_GE(pt.x(), radius_); + ABSL_DCHECK_GE(pt.y(), radius_); + ABSL_DCHECK_LT(pt.x(), rgb_frame_.cols - radius_); + ABSL_DCHECK_LT(pt.y(), rgb_frame_.rows - radius_); GetPatchDescriptorAtPoint(rgb_frame_, pt, radius_, &lab_window, feature->mutable_feature_descriptor()); if (prev_rgb_frame_) { Vector2_i pt_match(FeatureMatchIntLocation(*feature)); - DCHECK_GE(pt_match.x(), radius_); - DCHECK_GE(pt_match.y(), radius_); - DCHECK_LT(pt_match.x(), rgb_frame_.cols - radius_); - DCHECK_LT(pt_match.y(), rgb_frame_.rows - radius_); + ABSL_DCHECK_GE(pt_match.x(), radius_); + ABSL_DCHECK_GE(pt_match.y(), radius_); + ABSL_DCHECK_LT(pt_match.x(), rgb_frame_.cols - radius_); + ABSL_DCHECK_LT(pt_match.y(), rgb_frame_.rows - radius_); GetPatchDescriptorAtPoint(*prev_rgb_frame_, pt_match, radius_, &lab_window, feature->mutable_feature_match_descriptor()); @@ -247,17 +249,18 @@ void ComputeRegionFlowFeatureDescriptors( int patch_descriptor_radius, RegionFlowFeatureList* flow_feature_list) { const int rows = rgb_frame.rows; const int cols = rgb_frame.cols; - CHECK_EQ(rgb_frame.depth(), CV_8U); - CHECK_EQ(rgb_frame.channels(), 3); + ABSL_CHECK_EQ(rgb_frame.depth(), CV_8U); + ABSL_CHECK_EQ(rgb_frame.channels(), 3); if (prev_rgb_frame) { - CHECK_EQ(prev_rgb_frame->depth(), CV_8U); - CHECK_EQ(prev_rgb_frame->channels(), 3); - CHECK_EQ(prev_rgb_frame->rows, rows); - CHECK_EQ(prev_rgb_frame->cols, cols); + ABSL_CHECK_EQ(prev_rgb_frame->depth(), CV_8U); + ABSL_CHECK_EQ(prev_rgb_frame->channels(), 3); + ABSL_CHECK_EQ(prev_rgb_frame->rows, rows); + ABSL_CHECK_EQ(prev_rgb_frame->cols, cols); } - CHECK_LE(patch_descriptor_radius, flow_feature_list->distance_from_border()); + ABSL_CHECK_LE(patch_descriptor_radius, + flow_feature_list->distance_from_border()); ParallelFor( 0, flow_feature_list->feature_size(), 1, @@ -380,7 +383,7 @@ struct RegionFlowComputation::FrameTrackingData { iwidth = (iwidth + 1) / 2; iheight = (iheight + 1) / 2; } - CHECK_GE(extraction_levels, 1); + ABSL_CHECK_GE(extraction_levels, 1); // Frame is the same as first extraction level. frame = extraction_pyramid[0]; @@ -458,7 +461,7 @@ struct RegionFlowComputation::FrameTrackingData { } void RemoveFeature(int pos) { - DCHECK_LT(pos, features.size()); + ABSL_DCHECK_LT(pos, features.size()); features.erase(features.begin() + pos); feature_source_map.erase(feature_source_map.begin() + pos); corner_responses.erase(corner_responses.begin() + pos); @@ -472,7 +475,7 @@ struct RegionFlowComputation::FrameTrackingData { // Stores grayscale square patch with length patch_size extracted at center in // image frame and stores result in patch. void ExtractPatch(const cv::Point2f& center, int patch_size, cv::Mat* patch) { - CHECK(patch != nullptr); + ABSL_CHECK(patch != nullptr); patch->create(patch_size, patch_size, CV_8UC1); cv::getRectSubPix(frame, cv::Size(patch_size, patch_size), center, *patch); } @@ -490,9 +493,10 @@ struct RegionFlowComputation::LongTrackData { // Advance. ++next_track_id; if (next_track_id < 0) { - LOG(ERROR) << "Exhausted maximum possible ids. RegionFlowComputation " - << "instance lifetime is likely to be too long. Consider " - << "chunking the input."; + ABSL_LOG(ERROR) + << "Exhausted maximum possible ids. RegionFlowComputation " + << "instance lifetime is likely to be too long. Consider " + << "chunking the input."; next_track_id = 0; } @@ -531,13 +535,13 @@ struct RegionFlowComputation::LongTrackData { float MotionMagForId(int id) const { auto id_iter = track_info.find(id); - DCHECK(id_iter != track_info.end()); + ABSL_DCHECK(id_iter != track_info.end()); return id_iter->second.motion_mag; } void UpdateMotion(int id, float motion_mag) { auto id_iter = track_info.find(id); - DCHECK(id_iter != track_info.end()); + ABSL_DCHECK(id_iter != track_info.end()); if (id_iter->second.motion_mag >= 0) { id_iter->second.motion_mag = id_iter->second.motion_mag * 0.5f + 0.5f * motion_mag; @@ -616,8 +620,8 @@ RegionFlowComputation::RegionFlowComputation( } } - CHECK_NE(options.tracking_options().output_flow_direction(), - TrackingOptions::CONSECUTIVELY) + ABSL_CHECK_NE(options.tracking_options().output_flow_direction(), + TrackingOptions::CONSECUTIVELY) << "Output direction must be either set to FORWARD or BACKWARD."; use_downsampling_ = options_.downsample_mode() != RegionFlowComputationOptions::DOWNSAMPLE_NONE; @@ -650,7 +654,7 @@ RegionFlowComputation::RegionFlowComputation( } case RegionFlowComputationOptions::DOWNSAMPLE_BY_FACTOR: case RegionFlowComputationOptions::DOWNSAMPLE_TO_INPUT_SIZE: { - CHECK_GE(options_.downsample_factor(), 1); + ABSL_CHECK_GE(options_.downsample_factor(), 1); downsample_scale_ = options_.downsample_factor(); break; } @@ -683,7 +687,7 @@ RegionFlowComputation::RegionFlowComputation( frame_width_ += frame_width_ % 2; frame_height_ += frame_height_ % 2; - LOG(INFO) << "Using a downsampling scale of " << downsample_scale_; + ABSL_LOG(INFO) << "Using a downsampling scale of " << downsample_scale_; } // Make sure value is equal to local variable, in case someone uses that on @@ -720,31 +724,32 @@ RegionFlowComputation::RegionFlowComputation( switch (options_.tracking_options().tracking_policy()) { case TrackingOptions::POLICY_SINGLE_FRAME: if (options_.tracking_options().multi_frames_to_track() > 1) { - LOG(ERROR) << "TrackingOptions::multi_frames_to_track is > 1, " - << "but tracking_policy is set to POLICY_SINGLE_FRAME. " - << "Consider using POLICY_MULTI_FRAME instead."; + ABSL_LOG(ERROR) << "TrackingOptions::multi_frames_to_track is > 1, " + << "but tracking_policy is set to POLICY_SINGLE_FRAME. " + << "Consider using POLICY_MULTI_FRAME instead."; } frames_to_track_ = 1; break; case TrackingOptions::POLICY_MULTI_FRAME: - CHECK_GT(options_.tracking_options().multi_frames_to_track(), 0); + ABSL_CHECK_GT(options_.tracking_options().multi_frames_to_track(), 0); frames_to_track_ = options_.tracking_options().multi_frames_to_track(); break; case TrackingOptions::POLICY_LONG_TRACKS: if (options_.tracking_options().multi_frames_to_track() > 1) { - LOG(ERROR) << "TrackingOptions::multi_frames_to_track is > 1, " - << "but tracking_policy is set to POLICY_LONG_TRACKS. " - << "Use TrackingOptions::long_tracks_max_frames to set " - << "length of long feature tracks."; + ABSL_LOG(ERROR) << "TrackingOptions::multi_frames_to_track is > 1, " + << "but tracking_policy is set to POLICY_LONG_TRACKS. " + << "Use TrackingOptions::long_tracks_max_frames to set " + << "length of long feature tracks."; } if (options_.tracking_options().internal_tracking_direction() != TrackingOptions::FORWARD) { - LOG(ERROR) << "Long tracks are only supported if tracking direction " - << "is set to FORWARD. Adjusting direction to FORWARD. " - << "This does not affect the expected " - << "output_flow_direction"; + ABSL_LOG(ERROR) + << "Long tracks are only supported if tracking direction " + << "is set to FORWARD. Adjusting direction to FORWARD. " + << "This does not affect the expected " + << "output_flow_direction"; options_.mutable_tracking_options()->set_internal_tracking_direction( TrackingOptions::FORWARD); } @@ -756,7 +761,7 @@ RegionFlowComputation::RegionFlowComputation( break; } - CHECK(!options_.gain_correction() || !IsVerifyLongFeatures()) + ABSL_CHECK(!options_.gain_correction() || !IsVerifyLongFeatures()) << "Gain correction mode with verification of long features is not " << "supported."; @@ -764,8 +769,9 @@ RegionFlowComputation::RegionFlowComputation( use_cv_tracking_ = options_.tracking_options().use_cv_tracking_algorithm(); #if CV_MAJOR_VERSION < 3 if (use_cv_tracking_) { - LOG(WARNING) << "Compiled without OpenCV 3.0 but cv_tracking_algorithm " - << "was requested. Falling back to older algorithm"; + ABSL_LOG(WARNING) + << "Compiled without OpenCV 3.0 but cv_tracking_algorithm " + << "was requested. Falling back to older algorithm"; use_cv_tracking_ = false; } #endif @@ -808,7 +814,7 @@ RegionFlowComputation::RegionFlowComputation( // Compute settings for block based flow. const float block_size = options_.fast_estimation_block_size(); - CHECK_GT(block_size, 0) << "Need positive block size"; + ABSL_CHECK_GT(block_size, 0) << "Need positive block size"; block_width_ = block_size < 1 ? block_size * original_width_ : block_size; block_height_ = block_size < 1 ? block_size * original_height_ : block_size; @@ -869,18 +875,18 @@ RegionFlowComputation::RetrieveRegionFlowFeatureListImpl( int track_index, bool compute_feature_descriptor, bool compute_match_descriptor, const cv::Mat* curr_color_image, const cv::Mat* prev_color_image) { - CHECK_GT(region_flow_results_.size(), track_index); - CHECK(region_flow_results_[track_index].get()); + ABSL_CHECK_GT(region_flow_results_.size(), track_index); + ABSL_CHECK(region_flow_results_[track_index].get()); std::unique_ptr feature_list( std::move(region_flow_results_[track_index])); if (compute_feature_descriptor) { - CHECK(curr_color_image != nullptr); - CHECK_EQ(3, curr_color_image->channels()); + ABSL_CHECK(curr_color_image != nullptr); + ABSL_CHECK_EQ(3, curr_color_image->channels()); if (compute_match_descriptor) { - CHECK(prev_color_image != nullptr); - CHECK_EQ(3, prev_color_image->channels()); + ABSL_CHECK(prev_color_image != nullptr); + ABSL_CHECK_EQ(3, prev_color_image->channels()); } ComputeRegionFlowFeatureDescriptors( @@ -888,8 +894,9 @@ RegionFlowComputation::RetrieveRegionFlowFeatureListImpl( compute_match_descriptor ? prev_color_image : nullptr, options_.patch_descriptor_radius(), feature_list.get()); } else { - CHECK(!compute_match_descriptor) << "Set compute_feature_descriptor also " - << "if setting compute_match_descriptor"; + ABSL_CHECK(!compute_match_descriptor) + << "Set compute_feature_descriptor also " + << "if setting compute_match_descriptor"; } return feature_list; @@ -963,15 +970,15 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, options_.image_format() != RegionFlowComputationOptions::FORMAT_GRAYSCALE) { options_.set_image_format(RegionFlowComputationOptions::FORMAT_GRAYSCALE); - LOG(WARNING) << "#channels = 1, but image_format was not set to " - "FORMAT_GRAYSCALE. Assuming GRAYSCALE input."; + ABSL_LOG(WARNING) << "#channels = 1, but image_format was not set to " + "FORMAT_GRAYSCALE. Assuming GRAYSCALE input."; } // Convert image to grayscale. switch (options_.image_format()) { case RegionFlowComputationOptions::FORMAT_RGB: if (3 != source_ptr->channels()) { - LOG(ERROR) << "Expecting 3 channel input for RGB."; + ABSL_LOG(ERROR) << "Expecting 3 channel input for RGB."; return false; } cv::cvtColor(*source_ptr, dest_frame, cv::COLOR_RGB2GRAY); @@ -979,7 +986,7 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, case RegionFlowComputationOptions::FORMAT_BGR: if (3 != source_ptr->channels()) { - LOG(ERROR) << "Expecting 3 channel input for BGR."; + ABSL_LOG(ERROR) << "Expecting 3 channel input for BGR."; return false; } cv::cvtColor(*source_ptr, dest_frame, cv::COLOR_BGR2GRAY); @@ -987,7 +994,7 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, case RegionFlowComputationOptions::FORMAT_RGBA: if (4 != source_ptr->channels()) { - LOG(ERROR) << "Expecting 4 channel input for RGBA."; + ABSL_LOG(ERROR) << "Expecting 4 channel input for RGBA."; return false; } cv::cvtColor(*source_ptr, dest_frame, cv::COLOR_RGBA2GRAY); @@ -995,7 +1002,7 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, case RegionFlowComputationOptions::FORMAT_BGRA: if (4 != source_ptr->channels()) { - LOG(ERROR) << "Expecting 4 channel input for BGRA."; + ABSL_LOG(ERROR) << "Expecting 4 channel input for BGRA."; return false; } cv::cvtColor(*source_ptr, dest_frame, cv::COLOR_BGRA2GRAY); @@ -1003,10 +1010,10 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, case RegionFlowComputationOptions::FORMAT_GRAYSCALE: if (1 != source_ptr->channels()) { - LOG(ERROR) << "Expecting 1 channel input for GRAYSCALE."; + ABSL_LOG(ERROR) << "Expecting 1 channel input for GRAYSCALE."; return false; } - CHECK_EQ(1, source_ptr->channels()); + ABSL_CHECK_EQ(1, source_ptr->channels()); if (source_ptr != &dest_frame) { source_ptr->copyTo(dest_frame); } @@ -1024,8 +1031,8 @@ bool RegionFlowComputation::InitFrame(const cv::Mat& source, } // Consistency checks; not input governed. - CHECK_EQ(dest_frame.cols, frame_width_); - CHECK_EQ(dest_frame.rows, frame_height_); + ABSL_CHECK_EQ(dest_frame.cols, frame_width_); + ABSL_CHECK_EQ(dest_frame.rows, frame_height_); data->BuildPyramid(pyramid_levels_, options_.tracking_options().tracking_window_size(), @@ -1043,33 +1050,33 @@ bool RegionFlowComputation::AddImageAndTrack( if (options_.downsample_mode() == RegionFlowComputationOptions::DOWNSAMPLE_TO_INPUT_SIZE) { if (frame_width_ != source.cols || frame_height_ != source.rows) { - LOG(ERROR) << "Source input dimensions incompatible with " - << "DOWNSAMPLE_TO_INPUT_SIZE. frame_width_: " << frame_width_ - << ", source.cols: " << source.cols - << ", frame_height_: " << frame_height_ - << ", source.rows: " << source.rows; + ABSL_LOG(ERROR) << "Source input dimensions incompatible with " + << "DOWNSAMPLE_TO_INPUT_SIZE. frame_width_: " + << frame_width_ << ", source.cols: " << source.cols + << ", frame_height_: " << frame_height_ + << ", source.rows: " << source.rows; return false; } if (!source_mask.empty()) { if (frame_width_ != source_mask.cols || frame_height_ != source_mask.rows) { - LOG(ERROR) << "Input mask dimensions incompatible with " - << "DOWNSAMPLE_TO_INPUT_SIZE"; + ABSL_LOG(ERROR) << "Input mask dimensions incompatible with " + << "DOWNSAMPLE_TO_INPUT_SIZE"; return false; } } } else { if (original_width_ != source.cols || original_height_ != source.rows) { - LOG(ERROR) << "Source input dimensions differ from those specified " - << "in the constructor"; + ABSL_LOG(ERROR) << "Source input dimensions differ from those specified " + << "in the constructor"; return false; } if (!source_mask.empty()) { if (original_width_ != source_mask.cols || original_height_ != source_mask.rows) { - LOG(ERROR) << "Input mask dimensions incompatible with those " - << "specified in the constructor"; + ABSL_LOG(ERROR) << "Input mask dimensions incompatible with those " + << "specified in the constructor"; return false; } } @@ -1089,8 +1096,8 @@ bool RegionFlowComputation::AddImageAndTrack( curr_data->Reset(frame_num_, timestamp_usec); if (!IsModelIdentity(initial_transform)) { - CHECK_EQ(1, frames_to_track_) << "Initial transform is not supported " - << "for multi frame tracking"; + ABSL_CHECK_EQ(1, frames_to_track_) << "Initial transform is not supported " + << "for multi frame tracking"; Homography transform = initial_transform; if (downsample_scale_ != 1) { const float scale = 1.0f / downsample_scale_; @@ -1100,7 +1107,7 @@ bool RegionFlowComputation::AddImageAndTrack( } if (!InitFrame(source, source_mask, curr_data)) { - LOG(ERROR) << "Could not init frame."; + ABSL_LOG(ERROR) << "Could not init frame."; return false; } @@ -1204,17 +1211,17 @@ bool RegionFlowComputation::AddImageAndTrack( } cv::Mat RegionFlowComputation::GetGrayscaleFrameFromResults() { - CHECK_GT(data_queue_.size(), 0) << "Empty queue, was AddImage* called?"; + ABSL_CHECK_GT(data_queue_.size(), 0) << "Empty queue, was AddImage* called?"; FrameTrackingData* curr_data = data_queue_.back().get(); - CHECK(curr_data); + ABSL_CHECK(curr_data); return curr_data->frame; } void RegionFlowComputation::GetFeatureTrackInliers( bool skip_estimation, TrackedFeatureList* features, TrackedFeatureView* inliers) const { - CHECK(features != nullptr); - CHECK(inliers != nullptr); + ABSL_CHECK(features != nullptr); + ABSL_CHECK(inliers != nullptr); inliers->clear(); if (skip_estimation) { inliers->reserve(features->size()); @@ -1228,9 +1235,9 @@ void RegionFlowComputation::GetFeatureTrackInliers( float RegionFlowComputation::ComputeVisualConsistency( FrameTrackingData* previous, FrameTrackingData* current) const { - CHECK_EQ(previous->frame_num + 1, current->frame_num); + ABSL_CHECK_EQ(previous->frame_num + 1, current->frame_num); const int total = previous->tiny_image.total(); - CHECK_GT(total, 0) << "Tiny image dimension set to zero."; + ABSL_CHECK_GT(total, 0) << "Tiny image dimension set to zero."; current->tiny_image_diff = FrameDifferenceMedian(previous->tiny_image, current->tiny_image) * (1.0f / total); @@ -1263,10 +1270,10 @@ void RegionFlowComputation::ComputeRegionFlow( } else { const int index1 = data_queue_.size() + from - 1; const int index2 = data_queue_.size() + to - 1; - CHECK_GE(index1, 0); - CHECK_LT(index1, data_queue_.size()); - CHECK_GE(index2, 0); - CHECK_LT(index2, data_queue_.size()); + ABSL_CHECK_GE(index1, 0); + ABSL_CHECK_LT(index1, data_queue_.size()); + ABSL_CHECK_GE(index2, 0); + ABSL_CHECK_LT(index2, data_queue_.size()); data1 = data_queue_[index1].get(); data2 = data_queue_[index2].get(); @@ -1298,7 +1305,7 @@ void RegionFlowComputation::ComputeRegionFlow( bool track_features = true; bool force_feature_extraction_next_frame = false; if (options_.tracking_options().wide_baseline_matching()) { - CHECK(initial_transform == nullptr) + ABSL_CHECK(initial_transform == nullptr) << "Can't use wide baseline matching and initial transform as the " << "same time."; @@ -1611,14 +1618,14 @@ class GridFeatureLocator { // or adds K to the existing mask if add is set to true. template inline void SetMaskNeighborhood(int mask_x, int mask_y, cv::Mat* mask) { - DCHECK_EQ(mask->type(), CV_8U); + ABSL_DCHECK_EQ(mask->type(), CV_8U); const int mask_start_x = max(0, mask_x - N); const int mask_end_x = min(mask->cols - 1, mask_x + N); const int mask_dx = mask_end_x - mask_start_x + 1; const int mask_start_y = max(0, mask_y - N); const int mask_end_y = min(mask->rows - 1, mask_y + N); - DCHECK_LE(mask_start_x, mask_end_x); - DCHECK_LE(mask_start_y, mask_end_y); + ABSL_DCHECK_LE(mask_start_x, mask_end_x); + ABSL_DCHECK_LE(mask_start_y, mask_end_y); if (!add) { for (int i = mask_start_y; i <= mask_end_y; ++i) { @@ -1640,9 +1647,9 @@ inline void SetMaskNeighborhood(int mask_x, int mask_y, cv::Mat* mask) { void RegionFlowComputation::AdaptiveGoodFeaturesToTrack( const std::vector& extraction_pyramid, int max_features, float mask_scale, cv::Mat* mask, FrameTrackingData* data) { - CHECK(data != nullptr); - CHECK(feature_tmp_image_1_.get() != nullptr); - CHECK(feature_tmp_image_2_.get() != nullptr); + ABSL_CHECK(data != nullptr); + ABSL_CHECK(feature_tmp_image_1_.get() != nullptr); + ABSL_CHECK(feature_tmp_image_2_.get() != nullptr); cv::Mat* eig_image = feature_tmp_image_1_.get(); cv::Mat* tmp_image = feature_tmp_image_2_.get(); @@ -1651,7 +1658,7 @@ void RegionFlowComputation::AdaptiveGoodFeaturesToTrack( // Setup grid information. const float block_size = tracking_options.adaptive_features_block_size(); - CHECK_GT(block_size, 0) << "Need positive block size"; + ABSL_CHECK_GT(block_size, 0) << "Need positive block size"; int block_width = block_size < 1 ? block_size * frame_width_ : block_size; int block_height = block_size < 1 ? block_size * frame_height_ : block_size; @@ -1703,8 +1710,8 @@ void RegionFlowComputation::AdaptiveGoodFeaturesToTrack( std::vector fast_keypoints; if (e == 0) { MEASURE_TIME << "Corner extraction"; - CHECK_EQ(rows, frame_height_); - CHECK_EQ(cols, frame_width_); + ABSL_CHECK_EQ(rows, frame_height_); + ABSL_CHECK_EQ(cols, frame_width_); if (use_fast) { fast_detector->detect(image, fast_keypoints); @@ -1716,8 +1723,8 @@ void RegionFlowComputation::AdaptiveGoodFeaturesToTrack( } else { // Compute corner response on a down-scaled image and upsample. step *= 2; - CHECK_EQ(rows, (extraction_pyramid[e - 1].rows + 1) / 2); - CHECK_EQ(cols, (extraction_pyramid[e - 1].cols + 1) / 2); + ABSL_CHECK_EQ(rows, (extraction_pyramid[e - 1].rows + 1) / 2); + ABSL_CHECK_EQ(cols, (extraction_pyramid[e - 1].cols + 1) / 2); if (use_fast) { fast_detector->detect(image, fast_keypoints); @@ -1885,7 +1892,7 @@ void RegionFlowComputation::AdaptiveGoodFeaturesToTrack( AffineModel RegionFlowComputation::AffineModelFromFeatures( TrackedFeatureList* features) const { - CHECK(features != nullptr); + ABSL_CHECK(features != nullptr); // Downscaled domain as output. MotionEstimation motion_estimation(MotionEstimationOptions(), frame_width_, @@ -1908,7 +1915,7 @@ AffineModel RegionFlowComputation::AffineModelFromFeatures( void RegionFlowComputation::ZeroMotionGridFeatures( int frame_width, int frame_height, float frac_grid_step_x, float frac_grid_step_y, RegionFlowFeatureList* result) { - CHECK(result != nullptr); + ABSL_CHECK(result != nullptr); result->Clear(); TrackedFeatureList features; @@ -1931,7 +1938,7 @@ void RegionFlowComputation::ZeroMotionGridFeatures( void RegionFlowComputation::DenseZeroMotionSamples( int frame_width, int frame_height, float frac_diameter, float frac_steps_x, float frac_steps_y, RegionFlowFeatureList* result) { - CHECK(result != nullptr); + ABSL_CHECK(result != nullptr); // Ensure patch fits into frame. const int radius = @@ -1978,7 +1985,7 @@ int RegionFlowComputation::ZeroMotionGridTracks(int frame_width, float frac_grid_step_x, float frac_grid_step_y, TrackedFeatureList* results) { - CHECK(results); + ABSL_CHECK(results); auto& tracked_features = *results; tracked_features.clear(); @@ -2014,9 +2021,9 @@ bool RegionFlowComputation::GainCorrectFrame(const cv::Mat& reference_frame, float reference_mean, float input_mean, cv::Mat* calibrated_frame) const { - CHECK(calibrated_frame); - CHECK_EQ(reference_frame.rows, input_frame.rows); - CHECK_EQ(reference_frame.cols, input_frame.cols); + ABSL_CHECK(calibrated_frame); + ABSL_CHECK_EQ(reference_frame.rows, input_frame.rows); + ABSL_CHECK_EQ(reference_frame.cols, input_frame.cols); // Do not attempt gain correction for tiny images. if (std::min(reference_frame.rows, reference_frame.cols) < 10) { @@ -2098,8 +2105,8 @@ void RegionFlowComputation::WideBaselineMatchFeatures( TrackedFeatureList* results) { #if (defined(__ANDROID__) || defined(__APPLE__) || defined(__EMSCRIPTEN__)) && \ !defined(CV_WRAPPER_3X) - LOG(FATAL) << "Supported on only with OpenCV 3.0. " - << "Use bazel build flag : --define CV_WRAPPER=3X"; + ABSL_LOG(FATAL) << "Supported on only with OpenCV 3.0. " + << "Use bazel build flag : --define CV_WRAPPER=3X"; #else // (defined(__ANDROID__) || defined(__APPLE__) || // defined(__EMSCRIPTEN__)) && !defined(CV_WRAPPER_3X) results->clear(); @@ -2180,12 +2187,12 @@ void RegionFlowComputation::WideBaselineMatchFeatures( void RegionFlowComputation::RemoveAbsentFeatures( const TrackedFeatureList& prev_result, FrameTrackingData* data) { - CHECK(long_track_data_ != nullptr); + ABSL_CHECK(long_track_data_ != nullptr); // Build hash set of track ids. absl::node_hash_set track_ids; for (const auto& feature : prev_result) { - DCHECK_NE(feature.track_id, -1); + ABSL_DCHECK_NE(feature.track_id, -1); track_ids.insert(feature.track_id); } @@ -2218,8 +2225,8 @@ void RegionFlowComputation::ExtractFeatures( const TrackedFeatureList* prev_result, FrameTrackingData* data) { MEASURE_TIME << "ExtractFeatures"; if (!options_.tracking_options().adaptive_good_features_to_track()) { - LOG(FATAL) << "Deprecated! Activate adaptive_good_features_to_track " - << "in TrackingOptions"; + ABSL_LOG(FATAL) << "Deprecated! Activate adaptive_good_features_to_track " + << "in TrackingOptions"; } // Check if features can simply be re-used. @@ -2233,8 +2240,8 @@ void RegionFlowComputation::ExtractFeatures( if (data->last_feature_extraction_time == 0) { // Features already extracted from this frame. - CHECK_EQ(data->corner_responses.size(), data->features.size()); - CHECK_EQ(data->octaves.size(), data->features.size()); + ABSL_CHECK_EQ(data->corner_responses.size(), data->features.size()); + ABSL_CHECK_EQ(data->octaves.size(), data->features.size()); VLOG(1) << "Features already present (extracted from this frame)"; return; } @@ -2242,8 +2249,8 @@ void RegionFlowComputation::ExtractFeatures( // Remove features that lie outside feature extraction mask. RemoveFeaturesOutsideMask(data); - CHECK_EQ(data->corner_responses.size(), data->features.size()); - CHECK_EQ(data->octaves.size(), data->features.size()); + ABSL_CHECK_EQ(data->corner_responses.size(), data->features.size()); + ABSL_CHECK_EQ(data->octaves.size(), data->features.size()); float feature_fraction = 0; if (data->num_original_extracted_and_tracked > 0) { @@ -2309,7 +2316,7 @@ void RegionFlowComputation::ExtractFeatures( data->neighborhoods->reserve(features_to_allocate); } - CHECK_EQ(data->extraction_pyramid.size(), extraction_levels_); + ABSL_CHECK_EQ(data->extraction_pyramid.size(), extraction_levels_); for (int i = 1; i < extraction_levels_; ++i) { // Need factor 2 as OpenCV stores image + gradient pairs when // "with_derivative" is set to true. @@ -2329,7 +2336,7 @@ void RegionFlowComputation::ExtractFeatures( if (prev_result) { // Seed feature mask and results with tracking ids. - CHECK(long_track_data_ != nullptr); + ABSL_CHECK(long_track_data_ != nullptr); const int max_track_length = options_.tracking_options().long_tracks_max_frames(); // Drop a feature with a propability X, such that all qualifying @@ -2357,8 +2364,8 @@ void RegionFlowComputation::ExtractFeatures( // For FORWARD output flow, we need to add flow to obtain the match // position, for BACKWARD output flow, flow is inverted, so that feature // locations already point to locations in the current frame. - CHECK_EQ(options_.tracking_options().internal_tracking_direction(), - TrackingOptions::FORWARD); + ABSL_CHECK_EQ(options_.tracking_options().internal_tracking_direction(), + TrackingOptions::FORWARD); float match_sign = options_.tracking_options().output_flow_direction() == TrackingOptions::FORWARD ? 1.0f @@ -2373,11 +2380,11 @@ void RegionFlowComputation::ExtractFeatures( const int track_id = feature.track_id; if (track_id < 0) { // TODO: Use LOG_FIRST_N here. - LOG_IF(WARNING, - []() { - static int k = 0; - return k++ < 2; - }()) + ABSL_LOG_IF(WARNING, + []() { + static int k = 0; + return k++ < 2; + }()) << "Expecting an assigned track id, " << "skipping feature."; continue; @@ -2386,7 +2393,7 @@ void RegionFlowComputation::ExtractFeatures( // Skip features for which the track would get too long. const int start_frame = long_track_data_->StartFrameForId(track_id); if (start_frame < 0) { - LOG(ERROR) << "Id is not present, skipping feature."; + ABSL_LOG(ERROR) << "Id is not present, skipping feature."; continue; } @@ -2426,9 +2433,9 @@ void RegionFlowComputation::ExtractFeatures( mask_scale, &mask, data); const int num_features = data->features.size(); - CHECK_EQ(num_features, data->octaves.size()); - CHECK_EQ(num_features, data->corner_responses.size()); - CHECK_EQ(num_features, data->track_ids.size()); + ABSL_CHECK_EQ(num_features, data->octaves.size()); + ABSL_CHECK_EQ(num_features, data->corner_responses.size()); + ABSL_CHECK_EQ(num_features, data->track_ids.size()); } // Selects features based on lambda evaluator: bool (int index) @@ -2439,23 +2446,23 @@ int RegionFlowComputation::InplaceFeatureSelection( std::vector*> float_vecs, const Eval& eval) { int num_selected_features = 0; const int num_features = data->features.size(); - DCHECK_EQ(num_features, data->corner_responses.size()); - DCHECK_EQ(num_features, data->octaves.size()); - DCHECK_EQ(num_features, data->track_ids.size()); - DCHECK_EQ(num_features, data->feature_source_map.size()); + ABSL_DCHECK_EQ(num_features, data->corner_responses.size()); + ABSL_DCHECK_EQ(num_features, data->octaves.size()); + ABSL_DCHECK_EQ(num_features, data->track_ids.size()); + ABSL_DCHECK_EQ(num_features, data->feature_source_map.size()); if (data->neighborhoods != nullptr) { - DCHECK_EQ(num_features, data->neighborhoods->size()); + ABSL_DCHECK_EQ(num_features, data->neighborhoods->size()); } for (const auto vec_ptr : int_vecs) { - DCHECK_EQ(num_features, vec_ptr->size()); + ABSL_DCHECK_EQ(num_features, vec_ptr->size()); } for (const auto vec_ptr : float_vecs) { - DCHECK_EQ(num_features, vec_ptr->size()); + ABSL_DCHECK_EQ(num_features, vec_ptr->size()); } for (int i = 0; i < num_features; ++i) { - DCHECK_LE(num_selected_features, i); + ABSL_DCHECK_LE(num_selected_features, i); if (eval(i)) { data->features[num_selected_features] = data->features[i]; data->corner_responses[num_selected_features] = data->corner_responses[i]; @@ -2549,14 +2556,14 @@ void RegionFlowComputation::TrackFeatures(FrameTrackingData* from_data_ptr, octaves2.resize(num_features); data2.source = from_data_ptr; } else { - CHECK_EQ(data2.source, from_data_ptr); - CHECK_EQ(num_features, features2.size()); + ABSL_CHECK_EQ(data2.source, from_data_ptr); + ABSL_CHECK_EQ(num_features, features2.size()); tracking_flags |= cv::OPTFLOW_USE_INITIAL_FLOW; } const int track_win_size = options_.tracking_options().tracking_window_size(); - CHECK_GT(track_win_size, 1) << "Needs to be at least 2 pixels in each " - << "direction"; + ABSL_CHECK_GT(track_win_size, 1) << "Needs to be at least 2 pixels in each " + << "direction"; // Proceed with gain correction only if it succeeds, and set flag accordingly. bool frame1_gain_reference = true; @@ -2611,12 +2618,12 @@ void RegionFlowComputation::TrackFeatures(FrameTrackingData* from_data_ptr, cv_window_size, pyramid_levels_, cv_criteria, tracking_flags); } else { - LOG(ERROR) << "Tracking method unspecified."; + ABSL_LOG(ERROR) << "Tracking method unspecified."; return; } #endif } else { - LOG(ERROR) << "only cv tracking is supported."; + ABSL_LOG(ERROR) << "only cv tracking is supported."; return; } @@ -2640,7 +2647,7 @@ void RegionFlowComputation::TrackFeatures(FrameTrackingData* from_data_ptr, // Init neighborhoods if needed. if (IsVerifyLongFeatures()) { // data1 should be initialized at this point. - CHECK(data1.neighborhoods != nullptr); + ABSL_CHECK(data1.neighborhoods != nullptr); if (data2.neighborhoods == nullptr) { data2.neighborhoods.reset(new std::vector()); data2.neighborhoods->resize(num_valid_features); @@ -2791,7 +2798,7 @@ void RegionFlowComputation::TrackFeatures(FrameTrackingData* from_data_ptr, pyramid_levels_, cv_criteria, tracking_flags); #endif } else { - LOG(ERROR) << "only cv tracking is supported."; + ABSL_LOG(ERROR) << "only cv tracking is supported."; return; } @@ -2946,17 +2953,17 @@ void RegionFlowComputation::InitializeFeatureLocationsFromTransform( void RegionFlowComputation::InitializeFeatureLocationsFromPreviousResult( int from, int to) { - CHECK_NE(from, to) << "Cannot initialize FrameTrackingData from itself."; + ABSL_CHECK_NE(from, to) << "Cannot initialize FrameTrackingData from itself."; const int index1 = data_queue_.size() + from - 1; const int index2 = data_queue_.size() + to - 1; - CHECK_GE(index1, 0); - CHECK_LT(index1, data_queue_.size()); - CHECK_GE(index2, 0); - CHECK_LT(index2, data_queue_.size()); + ABSL_CHECK_GE(index1, 0); + ABSL_CHECK_LT(index1, data_queue_.size()); + ABSL_CHECK_GE(index2, 0); + ABSL_CHECK_LT(index2, data_queue_.size()); const FrameTrackingData& data1 = *data_queue_[index1]; FrameTrackingData* data2 = data_queue_[index2].get(); - CHECK(data1.source != nullptr); + ABSL_CHECK(data1.source != nullptr); if (!data1.features_initialized) { data2->features = data1.source->features; @@ -2965,7 +2972,7 @@ void RegionFlowComputation::InitializeFeatureLocationsFromPreviousResult( } } else { data2->features = data1.features; - CHECK_EQ(data1.features.size(), data1.source->features.size()); + ABSL_CHECK_EQ(data1.features.size(), data1.source->features.size()); } data2->source = data1.source; data2->features_initialized = true; @@ -3138,7 +3145,7 @@ void RegionFlowComputation::ComputeBlockBasedFlow( void RegionFlowComputation::DetermineRegionFlowInliers( const TrackedFeatureMap& region_feature_map, TrackedFeatureView* inliers) const { - CHECK(inliers); + ABSL_CHECK(inliers); inliers->clear(); // Run RANSAC on each region. @@ -3241,7 +3248,7 @@ int RegionFlowComputation::GetMinNumFeatureInliers( total_features += region_features.size(); } - CHECK(!region_feature_map.empty()) + ABSL_CHECK(!region_feature_map.empty()) << "Empty grid passed. Check input dimensions"; const float threshold = @@ -3254,7 +3261,7 @@ int RegionFlowComputation::GetMinNumFeatureInliers( void RegionFlowComputation::RegionFlowFeatureListToRegionFlow( const RegionFlowFeatureList& feature_list, RegionFlowFrame* frame) const { - CHECK(frame != nullptr); + ABSL_CHECK(frame != nullptr); frame->set_num_total_features(feature_list.feature_size()); frame->set_unstable_frame(feature_list.unstable()); diff --git a/mediapipe/util/tracking/region_flow_computation_test.cc b/mediapipe/util/tracking/region_flow_computation_test.cc index 435a8e20..40a1ed54 100644 --- a/mediapipe/util/tracking/region_flow_computation_test.cc +++ b/mediapipe/util/tracking/region_flow_computation_test.cc @@ -22,11 +22,12 @@ #include #include "absl/flags/flag.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/time/clock.h" #include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" @@ -104,8 +105,8 @@ INSTANTIATE_TEST_SUITE_P(FlowDirection, RegionFlowComputationTest, void RegionFlowComputationTest::MakeMovie( int num_frames, RegionFlowComputationOptions::ImageFormat format, std::vector* movie, std::vector* positions) { - CHECK(positions != nullptr); - CHECK(movie != nullptr); + ABSL_CHECK(positions != nullptr); + ABSL_CHECK(movie != nullptr); const int border = 40; int frame_width = original_frame_.cols - 2 * border; @@ -117,7 +118,7 @@ void RegionFlowComputationTest::MakeMovie( int seed = 900913; // google. if (absl::GetFlag(FLAGS_time_seed)) { seed = ToUnixMillis(absl::Now()) % (1 << 16); - LOG(INFO) << "Using time seed: " << seed; + ABSL_LOG(INFO) << "Using time seed: " << seed; } RandomEngine random(seed); @@ -178,7 +179,7 @@ void RegionFlowComputationTest::MakeMovie( void RegionFlowComputationTest::GetResizedFrame(int width, int height, cv::Mat* result) const { - CHECK(result != nullptr); + ABSL_CHECK(result != nullptr); cv::resize(original_frame_, *result, cv::Size(width, height)); } diff --git a/mediapipe/util/tracking/region_flow_visualization.cc b/mediapipe/util/tracking/region_flow_visualization.cc index dc067da7..901dce19 100644 --- a/mediapipe/util/tracking/region_flow_visualization.cc +++ b/mediapipe/util/tracking/region_flow_visualization.cc @@ -19,6 +19,7 @@ #include #include +#include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/util/tracking/measure_time.h" @@ -47,7 +48,7 @@ void VisualizeRegionFlowImpl(const RegionFlowFrame& region_flow_frame, void VisualizeRegionFlow(const RegionFlowFrame& region_flow_frame, cv::Mat* output) { - CHECK(output); + ABSL_CHECK(output); VisualizeRegionFlowImpl(region_flow_frame, output); } @@ -118,7 +119,7 @@ void VisualizeRegionFlowFeatures(const RegionFlowFeatureList& feature_list, const cv::Scalar& outlier, bool irls_visualization, float scale_x, float scale_y, cv::Mat* output) { - CHECK(output); + ABSL_CHECK(output); VisualizeRegionFlowFeaturesImpl(feature_list, color, outlier, irls_visualization, scale_x, scale_y, output); } @@ -138,7 +139,7 @@ void VisualizeLongFeatureStreamImpl(const LongFeatureStream& stream, if (min_track_length > 0 && pts.size() < min_track_length) { continue; } - CHECK_GT(pts.size(), 1); // Should have at least two points per track. + ABSL_CHECK_GT(pts.size(), 1); // Should have at least two points per track. // Tracks are ordered with oldest point first, most recent one last. const int start_k = @@ -186,7 +187,7 @@ void VisualizeLongFeatureStream(const LongFeatureStream& stream, const cv::Scalar& outlier, int min_track_length, int max_points_per_track, float scale_x, float scale_y, cv::Mat* output) { - CHECK(output); + ABSL_CHECK(output); VisualizeLongFeatureStreamImpl(stream, color, outlier, min_track_length, max_points_per_track, scale_x, scale_y, diff --git a/mediapipe/util/tracking/streaming_buffer.cc b/mediapipe/util/tracking/streaming_buffer.cc index 2e5b0ac2..218ca046 100644 --- a/mediapipe/util/tracking/streaming_buffer.cc +++ b/mediapipe/util/tracking/streaming_buffer.cc @@ -14,6 +14,8 @@ #include "mediapipe/util/tracking/streaming_buffer.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_cat.h" namespace mediapipe { @@ -21,9 +23,9 @@ namespace mediapipe { StreamingBuffer::StreamingBuffer( const std::vector& data_configuration, int overlap) : overlap_(overlap) { - CHECK_GE(overlap, 0); + ABSL_CHECK_GE(overlap, 0); for (auto& item : data_configuration) { - CHECK(data_config_.find(item.first) == data_config_.end()) + ABSL_CHECK(data_config_.find(item.first) == data_config_.end()) << "Tag " << item.first << " already exists"; data_config_[item.first] = item.second; // Init deque. @@ -45,7 +47,7 @@ bool StreamingBuffer::HasTags(const std::vector& tags) const { } int StreamingBuffer::BufferSize(const std::string& tag) const { - CHECK(HasTag(tag)); + ABSL_CHECK(HasTag(tag)); return data_.find(tag)->second.size(); } @@ -94,9 +96,9 @@ bool StreamingBuffer::TruncateBuffer(bool flush) { const int buffer_elems_to_clear = std::min(elems_to_clear, buffer.size()); if (buffer_elems_to_clear < elems_to_clear) { - LOG(WARNING) << "For tag " << item.first << " got " - << elems_to_clear - buffer_elems_to_clear - << "fewer elements than buffer can hold."; + ABSL_LOG(WARNING) << "For tag " << item.first << " got " + << elems_to_clear - buffer_elems_to_clear + << "fewer elements than buffer can hold."; is_consistent = false; } buffer.erase(buffer.begin(), buffer.begin() + buffer_elems_to_clear); @@ -108,9 +110,9 @@ bool StreamingBuffer::TruncateBuffer(bool flush) { for (const auto& item : data_) { const auto& buffer = item.second; if (buffer.size() != remaining_elems) { - LOG(WARNING) << "After trunctation, for tag " << item.first << "got " - << buffer.size() << " elements, " - << "expected " << remaining_elems; + ABSL_LOG(WARNING) << "After trunctation, for tag " << item.first << "got " + << buffer.size() << " elements, " + << "expected " << remaining_elems; is_consistent = false; } } @@ -119,7 +121,7 @@ bool StreamingBuffer::TruncateBuffer(bool flush) { } void StreamingBuffer::DiscardDatum(const std::string& tag, int num_frames) { - CHECK(HasTag(tag)); + ABSL_CHECK(HasTag(tag)); auto& queue = data_[tag]; if (queue.empty()) { return; @@ -130,7 +132,7 @@ void StreamingBuffer::DiscardDatum(const std::string& tag, int num_frames) { void StreamingBuffer::DiscardDatumFromEnd(const std::string& tag, int num_frames) { - CHECK(HasTag(tag)); + ABSL_CHECK(HasTag(tag)); auto& queue = data_[tag]; if (queue.empty()) { return; diff --git a/mediapipe/util/tracking/streaming_buffer.h b/mediapipe/util/tracking/streaming_buffer.h index 41aadbbb..ea4a5f27 100644 --- a/mediapipe/util/tracking/streaming_buffer.h +++ b/mediapipe/util/tracking/streaming_buffer.h @@ -23,8 +23,9 @@ #include #include "absl/container/node_hash_map.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/types/any.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/tool/type_util.h" namespace mediapipe { @@ -77,7 +78,7 @@ namespace mediapipe { // // Reached chunk boundary? // if (buffer_size == 100) { // // Check that we buffered one frame for each motion. -// CHECK(streaming_buffer.HaveEqualSize({"frame", "motion"})); +// ABSL_CHECK(streaming_buffer.HaveEqualSize({"frame", "motion"})); // // // Compute saliency. // for (int k = 0; k < 100; ++k) { @@ -296,7 +297,7 @@ class StreamingBuffer { // Terminates recursive template expansion for AddDataImpl. Will never be // called. void AddDataImpl(const std::vector& tags) { - CHECK(tags.empty()); + ABSL_CHECK(tags.empty()); } private: @@ -323,8 +324,8 @@ StreamingBuffer::PointerType StreamingBuffer::CreatePointer(T* t) { template void StreamingBuffer::AddDatum(const std::string& tag, std::unique_ptr pointer) { - CHECK(HasTag(tag)); - CHECK_EQ(data_config_[tag], kTypeId>.hash_code()); + ABSL_CHECK(HasTag(tag)); + ABSL_CHECK_EQ(data_config_[tag], kTypeId>.hash_code()); auto& buffer = data_[tag]; absl::any packet(PointerType(CreatePointer(pointer.release()))); buffer.push_back(packet); @@ -344,7 +345,7 @@ void StreamingBuffer::AddDatumCopy(const std::string& tag, const T& datum) { template void StreamingBuffer::AddData(const std::vector& tags, std::unique_ptr... pointers) { - CHECK_EQ(tags.size(), sizeof...(pointers)) + ABSL_CHECK_EQ(tags.size(), sizeof...(pointers)) << "Number of tags and data pointers is inconsistent"; return AddDataImpl(tags, std::move(pointers)...); } @@ -387,16 +388,16 @@ T& StreamingBuffer::GetDatumRef(const std::string& tag, int frame_index) const { template T* StreamingBuffer::GetMutableDatum(const std::string& tag, int frame_index) const { - CHECK_GE(frame_index, 0); - CHECK(HasTag(tag)); + ABSL_CHECK_GE(frame_index, 0); + ABSL_CHECK(HasTag(tag)); auto& buffer = data_.find(tag)->second; if (frame_index > buffer.size()) { return nullptr; } else { const absl::any& packet = buffer[frame_index]; if (absl::any_cast>(&packet) == nullptr) { - LOG(ERROR) << "Stored item is not of requested type. " - << "Check data configuration."; + ABSL_LOG(ERROR) << "Stored item is not of requested type. " + << "Check data configuration."; return nullptr; } @@ -440,15 +441,15 @@ StreamingBuffer::GetConstReferenceVector(const std::string& tag) const { template bool StreamingBuffer::IsInitialized(const std::string& tag) const { - CHECK(HasTag(tag)); + ABSL_CHECK(HasTag(tag)); const auto& buffer = data_.find(tag)->second; int idx = 0; for (const auto& item : buffer) { const PointerType* pointer = absl::any_cast>(&item); - CHECK(pointer != nullptr); + ABSL_CHECK(pointer != nullptr); if (*pointer == nullptr) { - LOG(ERROR) << "Data for " << tag << " at frame " << idx - << " is not initialized."; + ABSL_LOG(ERROR) << "Data for " << tag << " at frame " << idx + << " is not initialized."; return false; } } @@ -458,13 +459,13 @@ bool StreamingBuffer::IsInitialized(const std::string& tag) const { template std::vector StreamingBuffer::GetMutableDatumVector( const std::string& tag) const { - CHECK(HasTag(tag)); + ABSL_CHECK(HasTag(tag)); auto& buffer = data_.find(tag)->second; std::vector result; for (const auto& packet : buffer) { if (absl::any_cast>(&packet) == nullptr) { - LOG(ERROR) << "Stored item is not of requested type. " - << "Check data configuration."; + ABSL_LOG(ERROR) << "Stored item is not of requested type. " + << "Check data configuration."; result.push_back(nullptr); } else { result.push_back( @@ -477,7 +478,7 @@ std::vector StreamingBuffer::GetMutableDatumVector( template void StreamingBuffer::OutputDatum(bool flush, const std::string& tag, const Functor& functor) { - CHECK(HasTag(tag)); + ABSL_CHECK(HasTag(tag)); const int end_frame = MaxBufferSize() - (flush ? 0 : overlap_); for (int k = 0; k < end_frame; ++k) { functor(k, ReleaseDatum(tag, k)); @@ -487,8 +488,8 @@ void StreamingBuffer::OutputDatum(bool flush, const std::string& tag, template std::unique_ptr StreamingBuffer::ReleaseDatum(const std::string& tag, int frame_index) { - CHECK(HasTag(tag)); - CHECK_GE(frame_index, 0); + ABSL_CHECK(HasTag(tag)); + ABSL_CHECK_GE(frame_index, 0); auto& buffer = data_.find(tag)->second; if (frame_index >= buffer.size()) { @@ -496,8 +497,8 @@ std::unique_ptr StreamingBuffer::ReleaseDatum(const std::string& tag, } else { const absl::any& packet = buffer[frame_index]; if (absl::any_cast>(&packet) == nullptr) { - LOG(ERROR) << "Stored item is not of requested type. " - << "Check data configuration."; + ABSL_LOG(ERROR) << "Stored item is not of requested type. " + << "Check data configuration."; return nullptr; } diff --git a/mediapipe/util/tracking/tone_estimation.cc b/mediapipe/util/tracking/tone_estimation.cc index 587fe96f..2f2c2356 100644 --- a/mediapipe/util/tracking/tone_estimation.cc +++ b/mediapipe/util/tracking/tone_estimation.cc @@ -21,6 +21,8 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/util/tracking/motion_models.pb.h" #include "mediapipe/util/tracking/tone_models.pb.h" @@ -58,7 +60,7 @@ ToneEstimation::ToneEstimation(const ToneEstimationOptions& options, break; } case ToneEstimationOptions::DOWNSAMPLE_BY_FACTOR: { - CHECK_GE(options_.downsample_factor(), 1); + ABSL_CHECK_GE(options_.downsample_factor(), 1); frame_width_ /= options_.downsample_factor(); frame_height_ /= options_.downsample_factor(); downsample_scale_ = options_.downsample_factor(); @@ -80,9 +82,9 @@ void ToneEstimation::EstimateToneChange( const RegionFlowFeatureList& feature_list_input, const cv::Mat& curr_frame_input, const cv::Mat* prev_frame_input, ToneChange* tone_change, cv::Mat* debug_output) { - CHECK_EQ(original_height_, curr_frame_input.rows); - CHECK_EQ(original_width_, curr_frame_input.cols); - CHECK(tone_change != nullptr); + ABSL_CHECK_EQ(original_height_, curr_frame_input.rows); + ABSL_CHECK_EQ(original_width_, curr_frame_input.cols); + ABSL_CHECK(tone_change != nullptr); const cv::Mat& curr_frame = use_downsampling_ ? *resized_input_ : curr_frame_input; @@ -106,8 +108,8 @@ void ToneEstimation::EstimateToneChange( TransformRegionFlowFeatureList(scale_transform, &scaled_feature_list); } - CHECK_EQ(frame_height_, curr_frame.rows); - CHECK_EQ(frame_width_, curr_frame.cols); + ABSL_CHECK_EQ(frame_height_, curr_frame.rows); + ABSL_CHECK_EQ(frame_width_, curr_frame.cols); ClipMask<3> curr_clip; ComputeClipMask<3>(options_.clip_mask_options(), curr_frame, &curr_clip); @@ -212,15 +214,15 @@ void ToneEstimation::IntensityPercentiles(const cv::Mat& frame, void ToneEstimation::EstimateGainBiasModel(int irls_iterations, ColorToneMatches* color_tone_matches, GainBiasModel* gain_bias_model) { - CHECK(color_tone_matches != nullptr); - CHECK(gain_bias_model != nullptr); + ABSL_CHECK(color_tone_matches != nullptr); + ABSL_CHECK(gain_bias_model != nullptr); // Effectively estimate each model independently. float solution_ptr[6] = {1.0f, 0.0f, 1.0f, 0.0f, 1.0f, 0.0f}; const int num_channels = color_tone_matches->size(); - CHECK_GT(num_channels, 0); - CHECK_LE(num_channels, 3); + ABSL_CHECK_GT(num_channels, 0); + ABSL_CHECK_LE(num_channels, 3); // TODO: One IRLS weight per color match. for (int c = 0; c < num_channels; ++c) { @@ -303,8 +305,8 @@ void ToneEstimation::EstimateGainBiasModel(int irls_iterations, const float det = gain_bias_model->gain_c1() * gain_bias_model->gain_c2() * gain_bias_model->gain_c3(); if (fabs(det) < 1e-6f) { - LOG(WARNING) << "Estimated gain bias model is not invertible. " - << "Falling back to identity model."; + ABSL_LOG(WARNING) << "Estimated gain bias model is not invertible. " + << "Falling back to identity model."; gain_bias_model->CopyFrom(GainBiasModel()); } } diff --git a/mediapipe/util/tracking/tone_estimation.h b/mediapipe/util/tracking/tone_estimation.h index 0fa049e2..3d7defd2 100644 --- a/mediapipe/util/tracking/tone_estimation.h +++ b/mediapipe/util/tracking/tone_estimation.h @@ -25,6 +25,7 @@ #include #include +#include "absl/log/absl_check.h" #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" @@ -150,8 +151,8 @@ template void ToneEstimation::ComputeClipMask(const ClipMaskOptions& options, const cv::Mat& frame, ClipMask* clip_mask) { - CHECK(clip_mask != nullptr); - CHECK_EQ(frame.channels(), C); + ABSL_CHECK(clip_mask != nullptr); + ABSL_CHECK_EQ(frame.channels(), C); // Over / Underexposure handling. // Masks pixels affected by clipping. @@ -163,7 +164,7 @@ void ToneEstimation::ComputeClipMask(const ClipMaskOptions& options, std::vector planes; cv::split(frame, planes); - CHECK_EQ(C, planes.size()); + ABSL_CHECK_EQ(C, planes.size()); float min_exposure[C]; float max_exposure[C]; for (int c = 0; c < C; ++c) { @@ -223,9 +224,9 @@ void ToneEstimation::ComputeToneMatches( const ClipMask& curr_clip_mask, // Optional. const ClipMask& prev_clip_mask, // Optional. ColorToneMatches* color_tone_matches, cv::Mat* debug_output) { - CHECK(color_tone_matches != nullptr); - CHECK_EQ(curr_frame.channels(), C); - CHECK_EQ(prev_frame.channels(), C); + ABSL_CHECK(color_tone_matches != nullptr); + ABSL_CHECK_EQ(curr_frame.channels(), C); + ABSL_CHECK_EQ(prev_frame.channels(), C); color_tone_matches->clear(); color_tone_matches->resize(C); diff --git a/mediapipe/util/tracking/tone_models.cc b/mediapipe/util/tracking/tone_models.cc index 9410834b..ecc59d4b 100644 --- a/mediapipe/util/tracking/tone_models.cc +++ b/mediapipe/util/tracking/tone_models.cc @@ -16,6 +16,7 @@ #include +#include "absl/log/absl_check.h" #include "absl/strings/str_format.h" namespace mediapipe { @@ -47,13 +48,13 @@ void ToneModelMethods::MapImage(const Model& model, bool normalized_model, const cv::Mat& input, cv::Mat* output) { - CHECK(output != nullptr); + ABSL_CHECK(output != nullptr); const int out_channels = output->channels(); - CHECK_EQ(input.channels(), 3); - CHECK_LE(out_channels, 3); - CHECK_EQ(input.rows, output->rows); - CHECK_EQ(input.cols, output->cols); + ABSL_CHECK_EQ(input.channels(), 3); + ABSL_CHECK_LE(out_channels, 3); + ABSL_CHECK_EQ(input.rows, output->rows); + ABSL_CHECK_EQ(input.cols, output->cols); float norm_scale = normalized_model diff --git a/mediapipe/util/tracking/tone_models.h b/mediapipe/util/tracking/tone_models.h index 266257e1..bcbf1585 100644 --- a/mediapipe/util/tracking/tone_models.h +++ b/mediapipe/util/tracking/tone_models.h @@ -23,8 +23,9 @@ #include #include +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/integral_types.h" -#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/vector.h" #include "mediapipe/util/tracking/tone_models.pb.h" @@ -246,7 +247,7 @@ typedef MixtureToneAdapter MixtureAffineToneModelAdapter; template GainBiasModel ToneModelAdapter::FromPointer(const T* args, bool identity) { - DCHECK(args); + ABSL_DCHECK(args); GainBiasModel model; const float id_shift = identity ? 1.0f : 0.0f; model.set_gain_c1(args[0] + id_shift); @@ -292,7 +293,7 @@ inline GainBiasModel ToneModelAdapter::InvertChecked( const float det = GainBiasModelAdapter::Determinant(model); if (fabs(det) < 1e-10f) { *success = false; - LOG(ERROR) << "Model not invertible."; + ABSL_LOG(ERROR) << "Model not invertible."; return GainBiasModel(); } @@ -338,7 +339,7 @@ inline float ToneModelAdapter::GetParameter( case 5: return model.bias_c3(); default: - LOG(FATAL) << "Unknown parameter requested."; + ABSL_LOG(FATAL) << "Unknown parameter requested."; } return 0.0f; @@ -346,7 +347,7 @@ inline float ToneModelAdapter::GetParameter( template AffineToneModel ToneModelAdapter::FromPointer(const T* args, bool identity) { - DCHECK(args); + ABSL_DCHECK(args); AffineToneModel model; const float id_shift = identity ? 1.0f : 0.0f; model.set_g_00(args[0] + id_shift); @@ -369,7 +370,7 @@ AffineToneModel ToneModelAdapter::FromPointer(const T* args, template void ToneModelAdapter::ToPointerPad( const AffineToneModel& model, bool pad_square, T* args) { - DCHECK(args); + ABSL_DCHECK(args); args[0] = model.g_00(); args[1] = model.g_01(); args[2] = model.g_02(); @@ -413,7 +414,7 @@ inline AffineToneModel ToneModelAdapter::InvertChecked( cv::Mat inv_model_mat(4, 4, CV_64F, inv_data); if (cv::invert(model_mat, inv_model_mat) < 1e-10) { - LOG(ERROR) << "AffineToneModel not invertible, det is zero."; + ABSL_LOG(ERROR) << "AffineToneModel not invertible, det is zero."; *success = false; return AffineToneModel(); } @@ -467,7 +468,7 @@ inline float ToneModelAdapter::GetParameter( case 11: return model.g_23(); default: - LOG(FATAL) << "Unknown parameter requested."; + ABSL_LOG(FATAL) << "Unknown parameter requested."; } return 0.0f; @@ -592,9 +593,9 @@ template void ToneModelMethods::MapImageIndependent( const Model& model, bool log_domain, bool normalized_model, const cv::Mat& input, cv::Mat* output) { - CHECK(output != nullptr); - CHECK_EQ(input.channels(), C); - CHECK_EQ(output->channels(), C); + ABSL_CHECK(output != nullptr); + ABSL_CHECK_EQ(input.channels(), C); + ABSL_CHECK_EQ(output->channels(), C); // Input LUT which will be mapped to the output LUT by the tone change model. // Needs 3 channels to represent input RGB colors, but since they are assumed diff --git a/mediapipe/util/tracking/tracking.cc b/mediapipe/util/tracking/tracking.cc index 50aaa940..7c9bfa70 100644 --- a/mediapipe/util/tracking/tracking.cc +++ b/mediapipe/util/tracking/tracking.cc @@ -25,6 +25,8 @@ #include "Eigen/Dense" #include "Eigen/SVD" #include "absl/algorithm/container.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/memory/memory.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_calib3d_inc.h" @@ -63,7 +65,7 @@ void StoreInternalState(const std::vector& vectors, const std::vector& inlier_weights, float aspect_ratio, MotionBoxInternalState* internal) { const int num_vectors = vectors.size(); - CHECK_EQ(num_vectors, inlier_weights.size()); + ABSL_CHECK_EQ(num_vectors, inlier_weights.size()); float scale_x = 1.0f; float scale_y = 1.0f; @@ -98,7 +100,7 @@ std::string TrackStatusToString(MotionBoxState::TrackStatus status) { case MotionBoxState::BOX_TRACKED_OUT_OF_BOUND: return "BOX_TRACKED_OUT_OF_BOUND"; } - LOG(FATAL) << "Should not happen."; + ABSL_LOG(FATAL) << "Should not happen."; return "UNKNOWN"; } @@ -158,9 +160,10 @@ bool PointWithinInlierExtent(const Vector2_f pt, const MotionBoxState& state) { bool LinearSimilarityL2Solve( const std::vector& motion_vectors, const std::vector& weights, LinearSimilarityModel* model) { - CHECK(model); + ABSL_CHECK(model); if (motion_vectors.size() < 4) { - LOG(ERROR) << "Requiring at least 4 input vectors for sufficient solve."; + ABSL_LOG(ERROR) + << "Requiring at least 4 input vectors for sufficient solve."; return false; } @@ -171,7 +174,7 @@ bool LinearSimilarityL2Solve( matrix.setTo(0); rhs.setTo(0); - CHECK_EQ(motion_vectors.size(), weights.size()); + ABSL_CHECK_EQ(motion_vectors.size(), weights.size()); for (int k = 0; k < motion_vectors.size(); ++k) { const float x = motion_vectors[k]->pos.x(); const float y = motion_vectors[k]->pos.y(); @@ -234,7 +237,7 @@ bool LinearSimilarityL2Solve( // Taken from MotionEstimation::HomographyL2NormalEquationSolve bool HomographyL2Solve(const std::vector& motion_vectors, const std::vector& weights, Homography* model) { - CHECK(model); + ABSL_CHECK(model); cv::Mat matrix(8, 8, CV_32F); cv::Mat solution(8, 1, CV_32F); @@ -245,7 +248,7 @@ bool HomographyL2Solve(const std::vector& motion_vectors, // Matrix multiplications are hand-coded for speed improvements vs. // opencv's cvGEMM calls. - CHECK_EQ(motion_vectors.size(), weights.size()); + ABSL_CHECK_EQ(motion_vectors.size(), weights.size()); for (int k = 0; k < motion_vectors.size(); ++k) { const float x = motion_vectors[k]->pos.x(); const float y = motion_vectors[k]->pos.y(); @@ -374,10 +377,10 @@ bool HomographyL2Solve(const std::vector& motion_vectors, void TransformQuadInMotionBoxState(const MotionBoxState& curr_pos, const Homography& homography, MotionBoxState* next_pos) { - CHECK(next_pos != nullptr); + ABSL_CHECK(next_pos != nullptr); if (!curr_pos.has_pos_x() || !curr_pos.has_pos_y() || !curr_pos.has_width() || !curr_pos.has_height()) { - LOG(ERROR) << "Previous box does not exist, cannot transform!"; + ABSL_LOG(ERROR) << "Previous box does not exist, cannot transform!"; return; } const int kQuadVerticesSize = 8; @@ -574,11 +577,11 @@ bool IsBoxValid(const MotionBoxState& state) { const float kMaxBoxWidth = 10000.0f; // as relative to normalized [0, 1] space if (state.width() > kMaxBoxWidth) { - LOG(ERROR) << "box width " << state.width() << " too big"; + ABSL_LOG(ERROR) << "box width " << state.width() << " too big"; return false; } if (state.height() > kMaxBoxHeight) { - LOG(ERROR) << "box height " << state.height() << " too big"; + ABSL_LOG(ERROR) << "box height " << state.height() << " too big"; return false; } @@ -646,7 +649,7 @@ std::array MotionBoxCorners(const MotionBoxState& state, bool MotionBoxLines(const MotionBoxState& state, const Vector2_f& scaling, std::array* box_lines) { - CHECK(box_lines); + ABSL_CHECK(box_lines); std::array corners = MotionBoxCorners(state, scaling); for (int k = 0; k < 4; ++k) { const Vector2_f diff = corners[(k + 1) % 4] - corners[k]; @@ -656,7 +659,8 @@ bool MotionBoxLines(const MotionBoxState& state, const Vector2_f& scaling, if (box_lines->at(k).DotProd(Vector3_f(corners[(k + 1) % 4].x(), corners[(k + 1) % 4].y(), 1.0f)) >= 0.02f) { - LOG(ERROR) << "box is abnormal. Line equations don't satisfy constraint"; + ABSL_LOG(ERROR) + << "box is abnormal. Line equations don't satisfy constraint"; return false; } } @@ -665,8 +669,8 @@ bool MotionBoxLines(const MotionBoxState& state, const Vector2_f& scaling, void MotionBoxBoundingBox(const MotionBoxState& state, Vector2_f* top_left, Vector2_f* bottom_right) { - CHECK(top_left); - CHECK(bottom_right); + ABSL_CHECK(top_left); + ABSL_CHECK(bottom_right); std::array corners = MotionBoxCorners(state); @@ -687,7 +691,7 @@ void MotionBoxBoundingBox(const MotionBoxState& state, Vector2_f* top_left, void MotionBoxInlierLocations(const MotionBoxState& state, std::vector* inlier_pos) { - CHECK(inlier_pos); + ABSL_CHECK(inlier_pos); inlier_pos->clear(); for (int k = 0; k < state.inlier_id_match_pos_size(); k += 2) { inlier_pos->push_back( @@ -698,7 +702,7 @@ void MotionBoxInlierLocations(const MotionBoxState& state, void MotionBoxOutlierLocations(const MotionBoxState& state, std::vector* outlier_pos) { - CHECK(outlier_pos); + ABSL_CHECK(outlier_pos); outlier_pos->clear(); for (int k = 0; k < state.outlier_id_match_pos_size(); k += 2) { outlier_pos->push_back( @@ -736,7 +740,7 @@ std::array GetCornersOfRotatedRect(const MotionBoxState& state, } void InitializeQuadInMotionBoxState(MotionBoxState* state) { - CHECK(state != nullptr); + ABSL_CHECK(state != nullptr); // Every quad has 4 vertices. Each vertex has x and y 2 coordinates. So // a total of 8 floating point values. if (state->quad().vertices_size() != 8) { @@ -758,7 +762,7 @@ void InitializeInliersOutliersInMotionBoxState(const TrackingData& tracking, std::array box_lines; if (!MotionBoxLines(*state, Vector2_f(1.0f, 1.0f), &box_lines)) { - LOG(ERROR) << "Error in computing MotionBoxLines."; + ABSL_LOG(ERROR) << "Error in computing MotionBoxLines."; return; } @@ -828,7 +832,7 @@ void InitializePnpHomographyInMotionBoxState( } const int kQuadCornersSize = 4; - CHECK_EQ(state->quad().vertices_size(), kQuadCornersSize * 2); + ABSL_CHECK_EQ(state->quad().vertices_size(), kQuadCornersSize * 2); float scale_x, scale_y; ScaleFromAspect(tracking.frame_aspect(), false, &scale_x, &scale_y); std::vector corners_2d(kQuadCornersSize); @@ -865,7 +869,7 @@ void InitializePnpHomographyInMotionBoxState( constexpr float kEpsilon = 1e-6f; const float denominator = u2_u0 * v3_v1 - v2_v0 * u3_u1; if (std::abs(denominator) < kEpsilon) { - LOG(WARNING) << "Zero denominator. Failed calculating aspect ratio."; + ABSL_LOG(WARNING) << "Zero denominator. Failed calculating aspect ratio."; return; } @@ -882,7 +886,7 @@ void InitializePnpHomographyInMotionBoxState( std::vector corners(kQuadCornersSize); for (int i = 0; i < kQuadCornersSize; ++i) { if (s[0] <= 0) { - LOG(WARNING) << "Negative scale. Failed calculating aspect ratio."; + ABSL_LOG(WARNING) << "Negative scale. Failed calculating aspect ratio."; return; } corners[i] = @@ -894,7 +898,7 @@ void InitializePnpHomographyInMotionBoxState( const float height_norm = height_edge.Norm(); const float width_norm = width_edge.Norm(); if (height_norm < kEpsilon || width_norm < kEpsilon) { - LOG(WARNING) + ABSL_LOG(WARNING) << "abnormal 3d quadrangle. Failed calculating aspect ratio."; return; } @@ -902,7 +906,7 @@ void InitializePnpHomographyInMotionBoxState( constexpr float kMaxCosAngle = 0.258819; // which is cos(75 deg) if (width_edge.DotProd(height_edge) / height_norm / width_norm > kMaxCosAngle) { - LOG(WARNING) + ABSL_LOG(WARNING) << "abnormal 3d quadrangle. Failed calculating aspect ratio."; return; } @@ -910,7 +914,7 @@ void InitializePnpHomographyInMotionBoxState( state->set_aspect_ratio(width_norm / height_norm); } - CHECK_GT(state->aspect_ratio(), 0.0f); + ABSL_CHECK_GT(state->aspect_ratio(), 0.0f); const float half_width = state->aspect_ratio(); const float half_height = 1.0f; @@ -973,7 +977,7 @@ void ScaleStateAspect(float aspect, bool invert, MotionBoxState* state) { MotionVector MotionVector::FromInternalState( const MotionBoxInternalState& internal, int index) { - CHECK_LT(index, internal.pos_x_size()); + ABSL_CHECK_LT(index, internal.pos_x_size()); MotionVector v; v.pos = Vector2_f(internal.pos_x(index), internal.pos_y(index)); v.object = Vector2_f(internal.dx(index), internal.dy(index)); @@ -1003,8 +1007,8 @@ bool MotionBox::TrackStep(int from_frame, const MotionVectorFrame& motion_vectors, bool forward) { if (!TrackableFromFrame(from_frame)) { - LOG(WARNING) << "Tracking requested for initial position that is not " - << "trackable."; + ABSL_LOG(WARNING) << "Tracking requested for initial position that is not " + << "trackable."; return false; } const int queue_pos = from_frame - queue_start_; @@ -1072,7 +1076,7 @@ bool MotionBox::TrackStep(int from_frame, } if (num_track_errors >= options_.max_track_failures()) { - LOG_IF(INFO, print_motion_box_warnings_) + ABSL_LOG_IF(INFO, print_motion_box_warnings_) << "Tracking failed during max track failure " << "verification."; states_[new_pos].set_track_status(MotionBoxState::BOX_UNTRACKED); @@ -1105,7 +1109,7 @@ bool MotionBox::TrackStep(int from_frame, } if (num_track_errors >= options_.max_track_failures()) { - LOG_IF(INFO, print_motion_box_warnings_) + ABSL_LOG_IF(INFO, print_motion_box_warnings_) << "Tracking failed during max track failure " << "verification."; states_[new_pos].set_track_status(MotionBoxState::BOX_UNTRACKED); @@ -1117,7 +1121,7 @@ bool MotionBox::TrackStep(int from_frame, // Signal track success. return true; } else { - LOG_IF(WARNING, print_motion_box_warnings_) + ABSL_LOG_IF(WARNING, print_motion_box_warnings_) << "Tracking error at " << from_frame << " status : " << TrackStatusToString(new_state.track_status()); return false; @@ -1150,9 +1154,9 @@ void ComputeSpatialPrior(bool interpolate, bool use_next_position, std::vector old_confidence(update_pos->spatial_confidence().begin(), update_pos->spatial_confidence().end()); - CHECK_EQ(old_confidence.size(), old_prior.size()); - CHECK(old_confidence.empty() || - grid_size * grid_size == old_confidence.size()) + ABSL_CHECK_EQ(old_confidence.size(), old_prior.size()); + ABSL_CHECK(old_confidence.empty() || + grid_size * grid_size == old_confidence.size()) << "Empty or priors of constant size expected"; update_pos->clear_spatial_prior(); @@ -1192,10 +1196,10 @@ void ComputeSpatialPrior(bool interpolate, bool use_next_position, const int int_x = static_cast(grid_pos.x()); const int int_y = static_cast(grid_pos.y()); - CHECK_GE(grid_pos.x(), 0) << pos.x() << ", " << update_pos->pos_x(); - CHECK_GE(grid_pos.y(), 0); - CHECK_LE(grid_pos.x(), grid_size - 1); - CHECK_LE(grid_pos.y(), grid_size - 1); + ABSL_CHECK_GE(grid_pos.x(), 0) << pos.x() << ", " << update_pos->pos_x(); + ABSL_CHECK_GE(grid_pos.y(), 0); + ABSL_CHECK_LE(grid_pos.x(), grid_size - 1); + ABSL_CHECK_LE(grid_pos.y(), grid_size - 1); const float dx = grid_pos.x() - int_x; const float dy = grid_pos.y() - int_y; @@ -1281,9 +1285,9 @@ void MotionBox::GetStartPosition(const MotionBoxState& curr_pos, float aspect_ratio, float* expand_mag, Vector2_f* top_left, Vector2_f* bottom_right) const { - CHECK(top_left); - CHECK(bottom_right); - CHECK(expand_mag); + ABSL_CHECK(top_left); + ABSL_CHECK(bottom_right); + ABSL_CHECK(expand_mag); MotionBoxBoundingBox(curr_pos, top_left, bottom_right); @@ -1310,8 +1314,8 @@ void MotionBox::GetSpatialGaussWeights(const MotionBoxState& box_state, const Vector2_f& inv_box_domain, float* spatial_gauss_x, float* spatial_gauss_y) const { - CHECK(spatial_gauss_x); - CHECK(spatial_gauss_y); + ABSL_CHECK(spatial_gauss_x); + ABSL_CHECK(spatial_gauss_y); // Space sigma depends on how much the tracked object fills the rectangle. // We get this information from the inlier extent of the previous @@ -1340,7 +1344,7 @@ bool ComputeGridPositions(const Vector2_f& top_left, const Vector2_f& bottom_right, const std::vector& vectors, std::vector* grid_positions) { - CHECK(grid_positions); + ABSL_CHECK(grid_positions); // Slightly larger domain to avoid boundary issues. const Vector2_f inv_grid_domain( @@ -1431,8 +1435,8 @@ MotionBox::DistanceWeightsComputer::DistanceWeightsComputer( tracking_degrees_ = options.tracking_degrees(); const Vector2_f box_domain(current_state.width() * current_state.scale(), current_state.height() * current_state.scale()); - CHECK_GT(box_domain.x(), 0.0f); - CHECK_GT(box_domain.y(), 0.0f); + ABSL_CHECK_GT(box_domain.x(), 0.0f); + ABSL_CHECK_GT(box_domain.y(), 0.0f); inv_box_domain_ = Vector2_f(1.0f / box_domain.x(), 1.0f / box_domain.y()); // Space sigma depends on how much the tracked object fills the rectangle. @@ -1473,8 +1477,8 @@ MotionBox::DistanceWeightsComputer::DistanceWeightsComputer( std::min(kMaxBoxCenterBlendWeight, current_state.prior_weight())); if (tracking_degrees_ == TrackStepOptions::TRACKING_DEGREE_OBJECT_PERSPECTIVE) { - CHECK(initial_state.has_quad()); - CHECK(current_state.has_quad()); + ABSL_CHECK(initial_state.has_quad()); + ABSL_CHECK(current_state.has_quad()); homography_ = ComputeHomographyFromQuad(current_state.quad(), initial_state.quad()); box_center_transformed_ = @@ -1561,10 +1565,10 @@ bool MotionBox::GetVectorsAndWeights( const std::vector& history, std::vector* vectors, std::vector* weights, int* number_of_good_prior, int* number_of_cont_inliers) const { - CHECK(weights); - CHECK(vectors); - CHECK(number_of_good_prior); - CHECK(number_of_cont_inliers); + ABSL_CHECK(weights); + ABSL_CHECK(vectors); + ABSL_CHECK(number_of_good_prior); + ABSL_CHECK(number_of_cont_inliers); const int num_max_vectors = end_idx - start_idx; weights->clear(); @@ -1575,15 +1579,16 @@ bool MotionBox::GetVectorsAndWeights( const Vector2_f box_domain(box_state.width() * box_state.scale(), box_state.height() * box_state.scale()); - CHECK_GT(box_domain.x(), 0.0f); - CHECK_GT(box_domain.y(), 0.0f); + ABSL_CHECK_GT(box_domain.x(), 0.0f); + ABSL_CHECK_GT(box_domain.y(), 0.0f); const Vector2_f inv_box_domain(1.0f / box_domain.x(), 1.0f / box_domain.y()); // The four lines of the rotated and scaled box. std::array box_lines; if (!MotionBoxLines(box_state, Vector2_f(1.0f, 1.0f), &box_lines)) { - LOG(ERROR) << "Error in computing MotionBoxLines. Return 0 good inits and " - "continued inliers"; + ABSL_LOG(ERROR) + << "Error in computing MotionBoxLines. Return 0 good inits and " + "continued inliers"; return false; } @@ -1670,8 +1675,8 @@ bool MotionBox::GetVectorsAndWeights( is_outlier.push_back(is_outlier_flag); } - CHECK_EQ(vectors->size(), is_inlier.size()); - CHECK_EQ(vectors->size(), is_outlier.size()); + ABSL_CHECK_EQ(vectors->size(), is_inlier.size()); + ABSL_CHECK_EQ(vectors->size(), is_outlier.size()); const float prev_motion_mag = MotionBoxVelocity(box_state).Norm(); @@ -1814,7 +1819,7 @@ bool MotionBox::GetVectorsAndWeights( } const int num_vectors = vectors->size(); - CHECK_EQ(num_vectors, weights->size()); + ABSL_CHECK_EQ(num_vectors, weights->size()); const float weight_sum = std::accumulate(weights->begin(), weights->end(), 0.0f); @@ -1912,13 +1917,13 @@ void MotionBox::EstimateObjectMotion( const Vector2_f& irls_scale, std::vector* weights, Vector2_f* object_translation, LinearSimilarityModel* object_similarity, Homography* object_homography) const { - CHECK(object_translation); - CHECK(object_similarity); - CHECK(object_homography); + ABSL_CHECK(object_translation); + ABSL_CHECK(object_similarity); + ABSL_CHECK(object_homography); const int num_vectors = motion_vectors.size(); - CHECK_EQ(num_vectors, prior_weights.size()); - CHECK_EQ(num_vectors, weights->size()); + ABSL_CHECK_EQ(num_vectors, prior_weights.size()); + ABSL_CHECK_EQ(num_vectors, weights->size()); // Create backup of weights if needed. std::vector similarity_weights; @@ -1965,8 +1970,8 @@ void MotionBox::EstimateObjectMotion( if (!ObjectMotionValidator::IsValidSimilarity( *object_similarity, options_.box_similarity_max_scale(), options_.box_similarity_max_rotation())) { - LOG(WARNING) << "Unstable similarity model - falling back to " - << "translation."; + ABSL_LOG(WARNING) << "Unstable similarity model - falling back to " + << "translation."; *object_similarity = LinearSimilarityAdapter::Embed(translation_model); } else { @@ -1985,8 +1990,8 @@ void MotionBox::EstimateObjectMotion( if (!ObjectMotionValidator::IsValidHomography( *object_homography, options_.quad_homography_max_scale(), options_.quad_homography_max_rotation())) { - LOG(WARNING) << "Unstable homography model - falling back to " - << "translation."; + ABSL_LOG(WARNING) << "Unstable homography model - falling back to " + << "translation."; *object_homography = HomographyAdapter::Embed(translation_model); } else { weights->swap(similarity_weights); @@ -2007,8 +2012,8 @@ void MotionBox::EstimateTranslation( const std::vector& motion_vectors, const std::vector& prior_weights, const Vector2_f& irls_scale, std::vector* weights, Vector2_f* translation) const { - CHECK(weights); - CHECK(translation); + ABSL_CHECK(weights); + ABSL_CHECK(translation); const int iterations = options_.irls_iterations(); @@ -2057,8 +2062,8 @@ bool MotionBox::EstimateSimilarity( const std::vector& motion_vectors, const std::vector& prior_weights, const Vector2_f& irls_scale, std::vector* weights, LinearSimilarityModel* lin_sim) const { - CHECK(weights); - CHECK(lin_sim); + ABSL_CHECK(weights); + ABSL_CHECK(lin_sim); const int iterations = options_.irls_iterations(); LinearSimilarityModel object_similarity; @@ -2097,7 +2102,7 @@ bool MotionBox::EstimateHomography( const std::vector& motion_vectors, const std::vector& prior_weights, const Vector2_f& irls_scale, std::vector* weights, Homography* object_homography) const { - CHECK(weights); + ABSL_CHECK(weights); const int iterations = options_.irls_iterations(); Homography homography; @@ -2307,12 +2312,12 @@ void MotionBox::ScoreAndRecordInliers( std::vector* inlier_weights, std::vector* inlier_density, int* continued_inliers, int* swapped_inliers, float* motion_inliers_out, float* kinetic_average_out) const { - CHECK(inlier_weights); - CHECK(inlier_density); - CHECK(continued_inliers); - CHECK(swapped_inliers); - CHECK(motion_inliers_out); - CHECK(kinetic_average_out); + ABSL_CHECK(inlier_weights); + ABSL_CHECK(inlier_density); + ABSL_CHECK(continued_inliers); + ABSL_CHECK(swapped_inliers); + ABSL_CHECK(motion_inliers_out); + ABSL_CHECK(kinetic_average_out); std::unordered_map prev_inliers; MotionBoxInliers(curr_pos, &prev_inliers); @@ -2433,15 +2438,15 @@ void MotionBox::ComputeInlierCenterAndExtent( const std::vector& weights, const std::vector& density, const MotionBoxState& box_state, float* min_inlier_sum, Vector2_f* center, Vector2_f* extent) const { - CHECK(min_inlier_sum); - CHECK(center); - CHECK(extent); + ABSL_CHECK(min_inlier_sum); + ABSL_CHECK(center); + ABSL_CHECK(extent); float weight_sum = 0; float inlier_sum = 0; const int num_vectors = motion_vectors.size(); - CHECK_EQ(num_vectors, weights.size()); - CHECK_EQ(num_vectors, density.size()); + ABSL_CHECK_EQ(num_vectors, weights.size()); + ABSL_CHECK_EQ(num_vectors, density.size()); Vector2_f first_moment(0.0f, 0.0f); Vector2_f second_moment(0.0f, 0.0f); @@ -2498,7 +2503,7 @@ float MotionBox::ScaleEstimate( const std::vector& motion_vectors, const std::vector& weights, float min_sum) const { const int num_vectors = motion_vectors.size(); - CHECK_EQ(num_vectors, weights.size()); + ABSL_CHECK_EQ(num_vectors, weights.size()); float scale_sum = 0; @@ -2652,7 +2657,7 @@ void MotionBox::TrackStepImplDeNormalized( const MotionVectorFrame& motion_frame, const std::vector& history, MotionBoxState* next_pos) const { - CHECK(next_pos); + ABSL_CHECK(next_pos); constexpr float kDefaultPeriodMs = 1000.0f / kTrackingDefaultFps; float temporal_scale = (motion_frame.duration_ms == 0) @@ -2663,7 +2668,7 @@ void MotionBox::TrackStepImplDeNormalized( *next_pos = curr_pos; if (!IsBoxValid(curr_pos)) { - LOG(ERROR) << "curr_pos is not a valid box. Stop tracking!"; + ABSL_LOG(ERROR) << "curr_pos is not a valid box. Stop tracking!"; next_pos->set_track_status(MotionBoxState::BOX_UNTRACKED); return; } @@ -2725,7 +2730,7 @@ void MotionBox::TrackStepImplDeNormalized( (ObjectMotionValidator::IsQuadOutOfFov( next_pos->quad(), Vector2_f(domain_x, domain_y)) || !ObjectMotionValidator::IsValidQuad(next_pos->quad()))) { - LOG(ERROR) << "Quad is out of fov or not convex. Cancel tracking."; + ABSL_LOG(ERROR) << "Quad is out of fov or not convex. Cancel tracking."; next_pos->set_track_status(MotionBoxState::BOX_UNTRACKED); return; } @@ -2763,7 +2768,7 @@ void MotionBox::TrackStepImplDeNormalized( temporal_scale, expand_mag, history, &vectors, &prior_weights, &num_good_inits, &num_cont_inliers); if (!get_vec_weights_status) { - LOG(ERROR) << "error in GetVectorsAndWeights. Terminate tracking."; + ABSL_LOG(ERROR) << "error in GetVectorsAndWeights. Terminate tracking."; next_pos->set_track_status(MotionBoxState::BOX_UNTRACKED); return; } @@ -2783,7 +2788,7 @@ void MotionBox::TrackStepImplDeNormalized( if (next_pos->has_quad() && !ObjectMotionValidator::IsValidQuad(next_pos->quad())) { - LOG(ERROR) << "Quad is not convex. Cancel tracking."; + ABSL_LOG(ERROR) << "Quad is not convex. Cancel tracking."; next_pos->set_track_status(MotionBoxState::BOX_UNTRACKED); return; } @@ -2793,7 +2798,7 @@ void MotionBox::TrackStepImplDeNormalized( VLOG(1) << "Good inits: " << num_good_inits; const int num_vectors = vectors.size(); - CHECK_EQ(num_vectors, prior_weights.size()); + ABSL_CHECK_EQ(num_vectors, prior_weights.size()); Vector2_f object_translation; @@ -2952,8 +2957,8 @@ void MotionBox::TrackStepImplDeNormalized( options_.cancel_tracking_with_occlusion_options() .min_motion_continuity()) { next_pos->set_track_status(MotionBoxState::BOX_UNTRACKED); - LOG(INFO) << "Occlusion detected. continued_inlier_fraction: " - << continued_inlier_fraction << " too low. Stop tracking"; + ABSL_LOG(INFO) << "Occlusion detected. continued_inlier_fraction: " + << continued_inlier_fraction << " too low. Stop tracking"; return; } @@ -2981,7 +2986,7 @@ void MotionBox::TrackStepImplDeNormalized( // Assign full confidence on first frame, otherwise all other stats // are zero and there is no way to compute. next_pos->set_tracking_confidence(1.0f); - LOG(INFO) << "no history. confidence : 1.0"; + ABSL_LOG(INFO) << "no history. confidence : 1.0"; } else { next_pos->set_tracking_confidence(ComputeTrackingConfidence(*next_pos)); VLOG(1) << "confidence: " << next_pos->tracking_confidence(); @@ -3018,9 +3023,9 @@ void MotionBox::TrackStepImplDeNormalized( inlier_ratio < options_.cancel_tracking_with_occlusion_options() .min_inlier_ratio()) { next_pos->set_track_status(MotionBoxState::BOX_UNTRACKED); - LOG(INFO) << "inlier_ratio: " << inlier_ratio - << " too small. Stop tracking. inlier_max: " << inlier_max - << ". length in history: " << history.size(); + ABSL_LOG(INFO) << "inlier_ratio: " << inlier_ratio + << " too small. Stop tracking. inlier_max: " << inlier_max + << ". length in history: " << history.size(); return; } @@ -3052,7 +3057,7 @@ void MotionBox::TrackStepImplDeNormalized( if (next_pos->has_quad() && !ObjectMotionValidator::IsValidQuad(next_pos->quad())) { - LOG(ERROR) << "Quad is not convex. Cancel tracking."; + ABSL_LOG(ERROR) << "Quad is not convex. Cancel tracking."; next_pos->set_track_status(MotionBoxState::BOX_UNTRACKED); return; } @@ -3162,13 +3167,14 @@ void MotionBox::TrackStepImplDeNormalized( void MotionVectorFrameFromTrackingData(const TrackingData& tracking_data, MotionVectorFrame* motion_vector_frame) { - CHECK(motion_vector_frame != nullptr); + ABSL_CHECK(motion_vector_frame != nullptr); const auto& motion_data = tracking_data.motion_data(); float aspect_ratio = tracking_data.frame_aspect(); if (aspect_ratio < 0.1 || aspect_ratio > 10.0f) { - LOG(ERROR) << "Aspect ratio : " << aspect_ratio << " is out of bounds. " - << "Resetting to 1.0."; + ABSL_LOG(ERROR) << "Aspect ratio : " << aspect_ratio + << " is out of bounds. " + << "Resetting to 1.0."; aspect_ratio = 1.0f; } @@ -3245,13 +3251,14 @@ void FeatureAndDescriptorFromTrackingData( const auto& motion_data = tracking_data.motion_data(); float aspect_ratio = tracking_data.frame_aspect(); if (aspect_ratio < 0.1 || aspect_ratio > 10.0f) { - LOG(ERROR) << "Aspect ratio : " << aspect_ratio << " is out of bounds. " - << "Resetting to 1.0."; + ABSL_LOG(ERROR) << "Aspect ratio : " << aspect_ratio + << " is out of bounds. " + << "Resetting to 1.0."; aspect_ratio = 1.0f; } if (motion_data.feature_descriptors_size() == 0) { - LOG(WARNING) << "Feature descriptors not exist"; + ABSL_LOG(WARNING) << "Feature descriptors not exist"; return; } @@ -3288,7 +3295,7 @@ void FeatureAndDescriptorFromTrackingData( void InvertMotionVectorFrame(const MotionVectorFrame& input, MotionVectorFrame* output) { - CHECK(output != nullptr); + ABSL_CHECK(output != nullptr); output->background_model.CopyFrom(ModelInvert(input.background_model)); output->valid_background_model = input.valid_background_model; @@ -3341,13 +3348,13 @@ void GetFeatureIndicesWithinBox(const std::vector& features, const Vector2_f& box_scaling, float max_enlarge_size, int min_num_features, std::vector* inlier_indices) { - CHECK(inlier_indices); + ABSL_CHECK(inlier_indices); inlier_indices->clear(); if (features.empty()) return; std::array box_lines; if (!MotionBoxLines(box_state, box_scaling, &box_lines)) { - LOG(ERROR) << "Error in computing MotionBoxLines."; + ABSL_LOG(ERROR) << "Error in computing MotionBoxLines."; return; } diff --git a/mediapipe/util/tracking/tracking.h b/mediapipe/util/tracking/tracking.h index 4d3343f5..4a12a19e 100644 --- a/mediapipe/util/tracking/tracking.h +++ b/mediapipe/util/tracking/tracking.h @@ -26,6 +26,8 @@ #include #include "absl/container/flat_hash_set.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/port/vector.h" #include "mediapipe/util/tracking/flow_packager.pb.h" #include "mediapipe/util/tracking/motion_models.h" @@ -120,9 +122,9 @@ void MotionBoxBoundingBox(const MotionBoxState& state, Vector2_f* top_left, // existing score. inline void MotionBoxInliers(const MotionBoxState& state, std::unordered_map* inliers) { - CHECK(inliers); + ABSL_CHECK(inliers); const int num_inliers = state.inlier_ids_size(); - DCHECK_EQ(num_inliers, state.inlier_length_size()); + ABSL_DCHECK_EQ(num_inliers, state.inlier_length_size()); for (int k = 0; k < num_inliers; ++k) { (*inliers)[state.inlier_ids(k)] = @@ -314,8 +316,8 @@ class MotionBox { MotionBoxState StateAtFrame(int frame) const { if (frame < queue_start_ || frame >= queue_start_ + static_cast(states_.size())) { - LOG(ERROR) << "Requesting state at unknown frame " << frame - << ". Returning UNTRACKED."; + ABSL_LOG(ERROR) << "Requesting state at unknown frame " << frame + << ". Returning UNTRACKED."; MotionBoxState invalid; invalid.set_track_status(MotionBoxState::BOX_UNTRACKED); return invalid; @@ -560,7 +562,7 @@ class MotionBox { // Filter out abnormal homography. Otherwise the determinant of // projected affine matrix will be negative. if (!IsInverseStable(homography)) { - LOG(WARNING) << "Homography matrix is not stable."; + ABSL_LOG(WARNING) << "Homography matrix is not stable."; return false; } @@ -572,7 +574,7 @@ class MotionBox { // Check if it is a convex quad. static bool IsValidQuad(const MotionBoxState::Quad& quad) { const int kQuadVerticesSize = 8; - CHECK_EQ(quad.vertices_size(), kQuadVerticesSize); + ABSL_CHECK_EQ(quad.vertices_size(), kQuadVerticesSize); for (int a = 0; a < kQuadVerticesSize; a += 2) { int b = (a + 2) % kQuadVerticesSize; int c = (a - 2 + kQuadVerticesSize) % kQuadVerticesSize; @@ -595,7 +597,7 @@ class MotionBox { static bool IsQuadOutOfFov(const MotionBoxState::Quad& quad, const Vector2_f& fov) { const int kQuadVerticesSize = 8; - CHECK_EQ(quad.vertices_size(), kQuadVerticesSize); + ABSL_CHECK_EQ(quad.vertices_size(), kQuadVerticesSize); bool too_far = true; for (int j = 0; j < kQuadVerticesSize; j += 2) { if (quad.vertices(j) < fov.x() && quad.vertices(j) > 0.0f && diff --git a/mediapipe/util/tracking/tracking_visualization_utilities.cc b/mediapipe/util/tracking/tracking_visualization_utilities.cc index 0b586087..5ce45042 100644 --- a/mediapipe/util/tracking/tracking_visualization_utilities.cc +++ b/mediapipe/util/tracking/tracking_visualization_utilities.cc @@ -14,6 +14,8 @@ #include "mediapipe/util/tracking/tracking_visualization_utilities.h" +#include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/util/tracking/box_tracker.h" @@ -24,7 +26,7 @@ namespace mediapipe { void RenderState(const MotionBoxState& box_state, bool print_stats, cv::Mat* frame) { #ifndef NO_RENDERING - CHECK(frame != nullptr); + ABSL_CHECK(frame != nullptr); const int frame_width = frame->cols; const int frame_height = frame->rows; @@ -129,14 +131,14 @@ void RenderState(const MotionBoxState& box_state, bool print_stats, cv::putText(*frame, lock_text, cv::Point(top_left.x(), top_left.y() - 5), cv::FONT_HERSHEY_PLAIN, 0.8, lock_color); #else - LOG(FATAL) << "Code stripped out because of NO_RENDERING"; + ABSL_LOG(FATAL) << "Code stripped out because of NO_RENDERING"; #endif } void RenderInternalState(const MotionBoxInternalState& internal, cv::Mat* frame) { #ifndef NO_RENDERING - CHECK(frame != nullptr); + ABSL_CHECK(frame != nullptr); const int num_vectors = internal.pos_x_size(); @@ -169,14 +171,14 @@ void RenderInternalState(const MotionBoxInternalState& internal, cv::circle(*frame, p1, 2.0, color_scaled, 1); } #else - LOG(FATAL) << "Code stripped out because of NO_RENDERING"; + ABSL_LOG(FATAL) << "Code stripped out because of NO_RENDERING"; #endif } void RenderTrackingData(const TrackingData& data, cv::Mat* mat, bool antialiasing) { #ifndef NO_RENDERING - CHECK(mat != nullptr); + ABSL_CHECK(mat != nullptr); MotionVectorFrame mvf; MotionVectorFrameFromTrackingData(data, &mvf); @@ -199,13 +201,13 @@ void RenderTrackingData(const TrackingData& data, cv::Mat* mat, antialiasing ? cv::LINE_AA : 8); } #else - LOG(FATAL) << "Code stripped out because of NO_RENDERING"; + ABSL_LOG(FATAL) << "Code stripped out because of NO_RENDERING"; #endif } void RenderBox(const TimedBoxProto& box_proto, cv::Mat* mat) { #ifndef NO_RENDERING - CHECK(mat != nullptr); + ABSL_CHECK(mat != nullptr); TimedBox box = TimedBox::FromProto(box_proto); std::array corners = box.Corners(mat->cols, mat->rows); @@ -217,7 +219,7 @@ void RenderBox(const TimedBoxProto& box_proto, cv::Mat* mat) { 4); } #else - LOG(FATAL) << "Code stripped out because of NO_RENDERING"; + ABSL_LOG(FATAL) << "Code stripped out because of NO_RENDERING"; #endif } diff --git a/mediapipe/web/graph_runner/graph_runner.ts b/mediapipe/web/graph_runner/graph_runner.ts index 5d0c87b1..bf033750 100644 --- a/mediapipe/web/graph_runner/graph_runner.ts +++ b/mediapipe/web/graph_runner/graph_runner.ts @@ -66,6 +66,7 @@ export declare interface WasmModule { (parent: string, name: string, canRead: boolean, canWrite: boolean) => void; FS_unlink(path: string): void; + gpuOriginForWebTexturesIsBottomLeft?: boolean; errorListener?: ErrorListener; _bindTextureToCanvas: () => boolean; @@ -349,6 +350,31 @@ export class GraphRunner { this.wasmModule._setAutoRenderToScreen(enabled); } + /** + * Overrides the vertical orientation for input GpuBuffers and the automatic + * render-to-screen code. The default for our OpenGL code on other platforms + * (Android, Linux) is to use a bottom-left origin. But the default for WebGL + * is to use a top-left origin. We use WebGL default normally, and many + * calculators and graphs have platform-specific code to handle the resulting + * orientation flip. However, in order to be able to use a single graph on all + * platforms without alterations, it may be useful to send images into a web + * graph using the OpenGL orientation. Users can call this function with + * `bottomLeftIsOrigin = true` in order to enforce an orientation for all + * GpuBuffer inputs which is consistent with OpenGL on other platforms. + * This call will also vertically flip the automatic render-to-screen code as + * well, so that webcam input (for example) will render properly when passed + * through the graph still. + * NOTE: This will immediately affect GpuBuffer inputs, but must be called + * *before* graph start in order to affect the automatic render-to-screen + * code! + * @param bottomLeftIsOrigin True will flip our input GpuBuffers and auto + * render-to-screen to match the classic OpenGL orientation, while false will + * disable this feature to match the default WebGL orientation. + */ + setGpuBufferVerticalFlip(bottomLeftIsOrigin: boolean): void { + this.wasmModule.gpuOriginForWebTexturesIsBottomLeft = bottomLeftIsOrigin; + } + /** * Bind texture to our internal canvas, and upload image source to GPU. * Returns tuple [width, height] of texture. Intended for internal usage. @@ -374,8 +400,14 @@ export class GraphRunner { 'Failed to obtain WebGL context from the provided canvas. ' + '`getContext()` should only be invoked with `webgl` or `webgl2`.'); } + if (this.wasmModule.gpuOriginForWebTexturesIsBottomLeft) { + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true); + } gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, imageSource); + if (this.wasmModule.gpuOriginForWebTexturesIsBottomLeft) { + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false); + } let width, height; if ((imageSource as HTMLVideoElement).videoWidth) { @@ -1221,23 +1253,24 @@ export async function createMediaPipeLib( assetLoaderScript?: string|null, glCanvas?: HTMLCanvasElement|OffscreenCanvas|null, fileLocator?: FileLocator): Promise { - const scripts = []; // Run wasm-loader script here if (wasmLoaderScript) { - scripts.push(wasmLoaderScript); - } - // Run asset-loader script here - if (assetLoaderScript) { - scripts.push(assetLoaderScript); - } - // Load scripts in parallel, browser will execute them in sequence. - if (scripts.length) { - await Promise.all(scripts.map(runScript)); + await runScript(wasmLoaderScript); } + if (!self.ModuleFactory) { throw new Error('ModuleFactory not set.'); } + // Run asset-loader script here; must be run after wasm-loader script if we + // are re-wrapping the existing MODULARIZE export. + if (assetLoaderScript) { + await runScript(assetLoaderScript); + if (!self.ModuleFactory) { + throw new Error('ModuleFactory not set.'); + } + } + // Until asset scripts work nicely with MODULARIZE, when we are given both // self.Module and a fileLocator, we manually merge them into self.Module and // use that. TODO: Remove this when asset scripts are fixed. diff --git a/mediapipe/web/graph_runner/platform_utils.ts b/mediapipe/web/graph_runner/platform_utils.ts index 7e1decf3..d86e002d 100644 --- a/mediapipe/web/graph_runner/platform_utils.ts +++ b/mediapipe/web/graph_runner/platform_utils.ts @@ -32,5 +32,6 @@ export function isIOS() { // tslint:disable-next-line:deprecation ].includes(navigator.platform) // iPad on iOS 13 detection - || (navigator.userAgent.includes('Mac') && 'ontouchend' in document); + || (navigator.userAgent.includes('Mac') && + (typeof document !== undefined && 'ontouchend' in document)); } diff --git a/platform_mappings b/platform_mappings new file mode 100644 index 00000000..debf1e4b --- /dev/null +++ b/platform_mappings @@ -0,0 +1,61 @@ +# This file allows automatically mapping flags such as '--cpu' to the more +# modern Bazel platforms (https://bazel.build/concepts/platforms). + +# In particular, Bazel platforms lack support for Apple for now if no such +# mapping is put into place. It's inspired from: +# https://github.com/bazelbuild/rules_apple/issues/1764 + +flags: + --cpu=x86 + --crosstool_top=//external:android/crosstool + @mediapipe//mediapipe:android_x86_platform + + --cpu=x86_64 + --crosstool_top=//external:android/crosstool + @mediapipe//mediapipe:android_x86_64_platform + + --cpu=armeabi-v7a + --crosstool_top=//external:android/crosstool + @mediapipe//mediapipe:android_arm_platform + + --cpu=arm64-v8a + --crosstool_top=//external:android/crosstool + @mediapipe//mediapipe:android_arm64_platform + + --cpu=darwin_x86_64 + --apple_platform_type=macos + @@mediapipe//mediapipe:macos_x86_64_platform + + --cpu=darwin_arm64 + --apple_platform_type=macos + @@mediapipe//mediapipe:macos_arm64_platform + + --cpu=ios_i386 + --apple_platform_type=ios + @@mediapipe//mediapipe:ios_i386_platform + + --cpu=ios_x86_64 + --apple_platform_type=ios + @@mediapipe//mediapipe:ios_x86_64_platform + + --cpu=ios_sim_arm64 + --apple_platform_type=ios + @@mediapipe//mediapipe:ios_sim_arm64_platform + + --cpu=ios_armv7 + --apple_platform_type=ios + @@mediapipe//mediapipe:ios_armv7_platform + + --cpu=ios_arm64 + --apple_platform_type=ios + @@mediapipe//mediapipe:ios_arm64_platform + + --cpu=ios_arm64e + --apple_platform_type=ios + @@mediapipe//mediapipe:ios_arm64e_platform + + --cpu=x64_windows + @mediapipe//mediapipe:windows_platform + + --cpu=k8 + @mediapipe//mediapipe:linux_platform diff --git a/setup.py b/setup.py index 4eaa0dcf..d801cd98 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ import os import platform import posixpath import re +import shlex import shutil import subprocess import sys @@ -38,6 +39,15 @@ MP_DIR_INIT_PY = os.path.join(MP_ROOT_PATH, 'mediapipe/__init__.py') MP_THIRD_PARTY_BUILD = os.path.join(MP_ROOT_PATH, 'third_party/BUILD') MP_ROOT_INIT_PY = os.path.join(MP_ROOT_PATH, '__init__.py') +GPU_OPTIONS_DISBALED = ['--define=MEDIAPIPE_DISABLE_GPU=1'] +GPU_OPTIONS_ENBALED = [ + '--copt=-DTFLITE_GPU_EXTRA_GLES_DEPS', + '--copt=-DMEDIAPIPE_OMIT_EGL_WINDOW_BIT', + '--copt=-DMESA_EGL_NO_X11_HEADERS', + '--copt=-DEGL_NO_X11', +] +GPU_OPTIONS = GPU_OPTIONS_DISBALED if MP_DISABLE_GPU else GPU_OPTIONS_ENBALED + def _normalize_path(path): return path.replace('\\', '/') if IS_WINDOWS else path @@ -140,6 +150,16 @@ def _copy_to_build_lib_dir(build_lib, file): shutil.copyfile(os.path.join('bazel-bin/', file), dst) +def _invoke_shell_command(shell_commands): + """Invokes shell command from the list of arguments.""" + print('Invoking:', shlex.join(shell_commands)) + try: + subprocess.run(shell_commands, check=True) + except subprocess.CalledProcessError as e: + print(e) + sys.exit(e.returncode) + + class GeneratePyProtos(build_ext.build_ext): """Generate MediaPipe Python protobuf files by Protocol Compiler.""" @@ -204,9 +224,7 @@ class GeneratePyProtos(build_ext.build_ext): self._protoc, '-I.', '--python_out=' + os.path.abspath(self.build_lib), source ] - print('Invoking: ', protoc_command) - if subprocess.call(protoc_command) != 0: - sys.exit(-1) + _invoke_shell_command(protoc_command) class BuildModules(build_ext.build_ext): @@ -269,9 +287,7 @@ class BuildModules(build_ext.build_ext): 'build', external_file, ] - print('Invoking: ', fetch_model_command) - if subprocess.call(fetch_model_command) != 0: - sys.exit(-1) + _invoke_shell_command(fetch_model_command) _copy_to_build_lib_dir(self.build_lib, external_file) def _generate_binary_graph(self, binary_graph_target): @@ -284,20 +300,12 @@ class BuildModules(build_ext.build_ext): '--copt=-DNDEBUG', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), binary_graph_target, - ] - - if MP_DISABLE_GPU: - bazel_command.append('--define=MEDIAPIPE_DISABLE_GPU=1') - else: - bazel_command.append('--copt=-DMESA_EGL_NO_X11_HEADERS') - bazel_command.append('--copt=-DEGL_NO_X11') + ] + GPU_OPTIONS if not self.link_opencv and not IS_WINDOWS: bazel_command.append('--define=OPENCV=source') - print('Invoking: ', bazel_command) - if subprocess.call(bazel_command) != 0: - sys.exit(-1) + _invoke_shell_command(bazel_command) _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') @@ -318,17 +326,9 @@ class GenerateMetadataSchema(build_ext.build_ext): '--compilation_mode=opt', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), '//mediapipe/tasks/metadata:' + target, - ] + ] + GPU_OPTIONS - if MP_DISABLE_GPU: - bazel_command.append('--define=MEDIAPIPE_DISABLE_GPU=1') - else: - bazel_command.append('--copt=-DMESA_EGL_NO_X11_HEADERS') - bazel_command.append('--copt=-DEGL_NO_X11') - - print('Invoking: ', bazel_command) - if subprocess.call(bazel_command) != 0: - sys.exit(-1) + _invoke_shell_command(bazel_command) _copy_to_build_lib_dir( self.build_lib, 'mediapipe/tasks/metadata/' + target + '_generated.py') @@ -397,10 +397,7 @@ class BuildExtension(build_ext.build_ext): x86_name, arm64_name, ] - - print('Invoking: ', lipo_command) - if subprocess.call(lipo_command) != 0: - sys.exit(-1) + _invoke_shell_command(lipo_command) else: for ext in self.extensions: self._build_binary(ext) @@ -416,22 +413,14 @@ class BuildExtension(build_ext.build_ext): '--copt=-DNDEBUG', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), str(ext.bazel_target + '.so'), - ] - - if MP_DISABLE_GPU: - bazel_command.append('--define=MEDIAPIPE_DISABLE_GPU=1') - else: - bazel_command.append('--copt=-DMESA_EGL_NO_X11_HEADERS') - bazel_command.append('--copt=-DEGL_NO_X11') + ] + GPU_OPTIONS if extra_args: bazel_command += extra_args if not self.link_opencv and not IS_WINDOWS: bazel_command.append('--define=OPENCV=source') - print('Invoking: ', bazel_command) - if subprocess.call(bazel_command) != 0: - sys.exit(-1) + _invoke_shell_command(bazel_command) ext_bazel_bin_path = os.path.join('bazel-bin', ext.relpath, ext.target_name + '.so') ext_dest_path = self.get_ext_fullpath(ext.name) diff --git a/third_party/BUILD b/third_party/BUILD index 470b7ff9..22925208 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -32,9 +32,6 @@ cc_library( "//mediapipe:android_x86_64": [ "@com_github_glog_glog_no_gflags//:glog", ], - "//mediapipe:android_armeabi": [ - "@com_github_glog_glog_no_gflags//:glog", - ], "//mediapipe:android_arm": [ "@com_github_glog_glog_no_gflags//:glog", ], @@ -249,7 +246,6 @@ alias( actual = select({ "//mediapipe:android_x86": "@android_opencv//:libopencv_x86", "//mediapipe:android_x86_64": "@android_opencv//:libopencv_x86_64", - "//mediapipe:android_armeabi": "@android_opencv//:libopencv_armeabi-v7a", "//mediapipe:android_arm": "@android_opencv//:libopencv_armeabi-v7a", "//mediapipe:android_arm64": "@android_opencv//:libopencv_arm64-v8a", "//mediapipe:ios": "@ios_opencv//:opencv", @@ -265,7 +261,6 @@ cc_library( deps = select({ "//mediapipe:android_x86": [], "//mediapipe:android_x86_64": [], - "//mediapipe:android_armeabi": [], "//mediapipe:android_arm": [], "//mediapipe:android_arm64": [], "//mediapipe:ios": [], @@ -378,3 +373,10 @@ java_library( "@maven//:com_google_auto_value_auto_value_annotations", ], ) + +java_import( + name = "any_java_proto", + jars = [ + "@com_google_protobuf//java/core:libcore.jar", + ], +) diff --git a/third_party/com_github_glog_glog.diff b/third_party/com_github_glog_glog.diff new file mode 100644 index 00000000..15447d79 --- /dev/null +++ b/third_party/com_github_glog_glog.diff @@ -0,0 +1,68 @@ +diff --git a/src/logging.cc b/src/logging.cc +index 4028ccc..483e639 100644 +--- a/src/logging.cc ++++ b/src/logging.cc +@@ -1743,6 +1743,23 @@ ostream& LogMessage::stream() { + return data_->stream_; + } + ++namespace { ++#if defined(__ANDROID__) ++int AndroidLogLevel(const int severity) { ++ switch (severity) { ++ case 3: ++ return ANDROID_LOG_FATAL; ++ case 2: ++ return ANDROID_LOG_ERROR; ++ case 1: ++ return ANDROID_LOG_WARN; ++ default: ++ return ANDROID_LOG_INFO; ++ } ++} ++#endif // defined(__ANDROID__) ++} // namespace ++ + // Flush buffered message, called by the destructor, or any other function + // that needs to synchronize the log. + void LogMessage::Flush() { +@@ -1779,6 +1796,12 @@ void LogMessage::Flush() { + } + LogDestination::WaitForSinks(data_); + ++#if defined(__ANDROID__) ++ const int level = AndroidLogLevel((int)data_->severity_); ++ const std::string text = std::string(data_->message_text_); ++ __android_log_write(level, "native", text.substr(0,data_->num_chars_to_log_).c_str()); ++#endif // defined(__ANDROID__) ++ + if (append_newline) { + // Fix the ostrstream back how it was before we screwed with it. + // It's 99.44% certain that we don't need to worry about doing this. + +diff --git a/bazel/glog.bzl b/bazel/glog.bzl +index dacd934..d7b3d78 100644 +--- a/bazel/glog.bzl ++++ b/bazel/glog.bzl +@@ -53,7 +53,6 @@ def glog_library(namespace = "google", with_gflags = 1, **kwargs): + ) + + common_copts = [ +- "-std=c++14", + "-DGLOG_BAZEL_BUILD", + # Inject a C++ namespace. + "-DGOOGLE_NAMESPACE='%s'" % namespace, +@@ -145,7 +144,13 @@ def glog_library(namespace = "google", with_gflags = 1, **kwargs): + ], + }) + ++ c14_opts = ["-std=c++14"] ++ c17_opts = ["-std=c++17"] ++ + final_lib_copts = select({ ++ "@bazel_tools//src/conditions:windows": c17_opts, ++ "//conditions:default": c14_opts, ++ }) + select({ + "@bazel_tools//src/conditions:windows": common_copts + windows_only_copts, + "@bazel_tools//src/conditions:darwin": common_copts + linux_or_darwin_copts + darwin_only_copts, + "@bazel_tools//src/conditions:freebsd": common_copts + linux_or_darwin_copts + freebsd_only_copts, diff --git a/third_party/com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff b/third_party/com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff deleted file mode 100644 index 471cf2aa..00000000 --- a/third_party/com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff +++ /dev/null @@ -1,52 +0,0 @@ -diff --git a/src/logging.cc b/src/logging.cc -index 0b5e6ee..be5a506 100644 ---- a/src/logging.cc -+++ b/src/logging.cc -@@ -67,6 +67,10 @@ - # include "stacktrace.h" - #endif - -+#ifdef __ANDROID__ -+#include -+#endif -+ - using std::string; - using std::vector; - using std::setw; -@@ -1279,6 +1283,23 @@ ostream& LogMessage::stream() { - return data_->stream_; - } - -+namespace { -+#if defined(__ANDROID__) -+int AndroidLogLevel(const int severity) { -+ switch (severity) { -+ case 3: -+ return ANDROID_LOG_FATAL; -+ case 2: -+ return ANDROID_LOG_ERROR; -+ case 1: -+ return ANDROID_LOG_WARN; -+ default: -+ return ANDROID_LOG_INFO; -+ } -+} -+#endif // defined(__ANDROID__) -+} // namespace -+ - // Flush buffered message, called by the destructor, or any other function - // that needs to synchronize the log. - void LogMessage::Flush() { -@@ -1313,6 +1334,12 @@ void LogMessage::Flush() { - } - LogDestination::WaitForSinks(data_); - -+#if defined(__ANDROID__) -+ const int level = AndroidLogLevel((int)data_->severity_); -+ const std::string text = std::string(data_->message_text_); -+ __android_log_write(level, "native", text.substr(0,data_->num_chars_to_log_).c_str()); -+#endif // defined(__ANDROID__) -+ - if (append_newline) { - // Fix the ostrstream back how it was before we screwed with it. - // It's 99.44% certain that we don't need to worry about doing this. diff --git a/third_party/com_github_glog_glog_f2cf2e1bd040fd15016af53598db0cb9b16a6655.diff b/third_party/com_github_glog_glog_f2cf2e1bd040fd15016af53598db0cb9b16a6655.diff deleted file mode 100644 index 560e83ec..00000000 --- a/third_party/com_github_glog_glog_f2cf2e1bd040fd15016af53598db0cb9b16a6655.diff +++ /dev/null @@ -1,45 +0,0 @@ -https://github.com/google/glog/pull/342 - -diff --git a/CONTRIBUTORS b/CONTRIBUTORS -index d63f62d1..aa0dd4a8 100644 ---- a/CONTRIBUTORS -+++ b/CONTRIBUTORS -@@ -26,6 +26,7 @@ Abhishek Dasgupta - Abhishek Parmar - Andrew Schwartzmeyer - Andy Ying -+Bret McKee - Brian Silverman - Fumitoshi Ukai - Guillaume Dumont -diff --git a/src/glog/logging.h.in b/src/glog/logging.h.in -index 9968b96d..f6dccb29 100644 ---- a/src/glog/logging.h.in -+++ b/src/glog/logging.h.in -@@ -649,6 +649,10 @@ void MakeCheckOpValueString(std::ostream* os, const signed char& v); - template <> GOOGLE_GLOG_DLL_DECL - void MakeCheckOpValueString(std::ostream* os, const unsigned char& v); - -+// Provide printable value for nullptr_t -+template <> GOOGLE_GLOG_DLL_DECL -+void MakeCheckOpValueString(std::ostream* os, const std::nullptr_t& v); -+ - // Build the error message string. Specify no inlining for code size. - template - std::string* MakeCheckOpString(const T1& v1, const T2& v2, const char* exprtext) -diff --git a/src/logging.cc b/src/logging.cc -index 0c86cf62..256655e5 100644 ---- a/src/logging.cc -+++ b/src/logging.cc -@@ -2163,6 +2163,11 @@ void MakeCheckOpValueString(std::ostream* os, const unsigned char& v) { - } - } - -+template <> -+void MakeCheckOpValueString(std::ostream* os, const std::nullptr_t& v) { -+ (*os) << "nullptr"; -+} -+ - void InitGoogleLogging(const char* argv0) { - glog_internal_namespace_::InitGoogleLoggingUtilities(argv0); - } diff --git a/third_party/external_files.bzl b/third_party/external_files.bzl index 4b51d9de..86d50460 100644 --- a/third_party/external_files.bzl +++ b/third_party/external_files.bzl @@ -264,8 +264,8 @@ def external_files(): http_file( name = "com_google_mediapipe_dynamic_input_classifier_tflite", - sha256 = "fb34b05e1cd4081f3c2bb882092f617efb19266b3353d51b3790a172cae09784", - urls = ["https://storage.googleapis.com/mediapipe-assets/dynamic_input_classifier.tflite?generation=1680543275416843"], + sha256 = "c5499daf5773cef89ce984df329c6324194a83bea7c7cf83159bf660a58de85c", + urls = ["https://storage.googleapis.com/mediapipe-assets/dynamic_input_classifier.tflite?generation=1693433004555536"], ) http_file( @@ -282,8 +282,8 @@ def external_files(): http_file( name = "com_google_mediapipe_efficientdet_lite0_fp16_no_nms_tflite", - sha256 = "237a58389081333e5cf4154e42b593ce7dd357445536fcaf4ca5bc51c2c50f1c", - urls = ["https://storage.googleapis.com/mediapipe-assets/efficientdet_lite0_fp16_no_nms.tflite?generation=1682632067597216"], + sha256 = "bcda125c96d3767bca894c8cbe7bc458379c9974c9fd8bdc6204e7124a74082a", + urls = ["https://storage.googleapis.com/mediapipe-assets/efficientdet_lite0_fp16_no_nms.tflite?generation=1693433007348701"], ) http_file( @@ -306,26 +306,14 @@ def external_files(): http_file( name = "com_google_mediapipe_expected_left_down_hand_landmarks_prototxt", - sha256 = "ae9cb01035f18b0023fc12256c048666da76b41b327cec09c2d2820054b1295f", - urls = ["https://storage.googleapis.com/mediapipe-assets/expected_left_down_hand_landmarks.prototxt?generation=1661875720230540"], - ) - - http_file( - name = "com_google_mediapipe_expected_left_down_hand_rotated_landmarks_prototxt", - sha256 = "c4dfdcc2e4cd366eb5f8ad227be94049eb593e3a528564611094687912463687", - urls = ["https://storage.googleapis.com/mediapipe-assets/expected_left_down_hand_rotated_landmarks.prototxt?generation=1666629474155924"], + sha256 = "f281b745175aaa7f458def6cf4c89521fb56302dd61a05642b3b4a4f237ffaa3", + urls = ["https://storage.googleapis.com/mediapipe-assets/expected_left_down_hand_landmarks.prototxt?generation=1692121979089068"], ) http_file( name = "com_google_mediapipe_expected_left_up_hand_landmarks_prototxt", - sha256 = "1353ba617c4f048083618587cd23a8a22115f634521c153d4e1bd1ebd4f49dd7", - urls = ["https://storage.googleapis.com/mediapipe-assets/expected_left_up_hand_landmarks.prototxt?generation=1661875726008879"], - ) - - http_file( - name = "com_google_mediapipe_expected_left_up_hand_rotated_landmarks_prototxt", - sha256 = "7fb2d33cf69d2da50952a45bad0c0618f30859e608958fee95948a6e0de63ccb", - urls = ["https://storage.googleapis.com/mediapipe-assets/expected_left_up_hand_rotated_landmarks.prototxt?generation=1666629476401757"], + sha256 = "174cf5f7c3ab547f0affb666ee7be933b0758c60fbfe7b7e93795c5082555592", + urls = ["https://storage.googleapis.com/mediapipe-assets/expected_left_up_hand_landmarks.prototxt?generation=1692121981605963"], ) http_file( @@ -336,14 +324,26 @@ def external_files(): http_file( name = "com_google_mediapipe_expected_right_down_hand_landmarks_prototxt", - sha256 = "f281b745175aaa7f458def6cf4c89521fb56302dd61a05642b3b4a4f237ffaa3", - urls = ["https://storage.googleapis.com/mediapipe-assets/expected_right_down_hand_landmarks.prototxt?generation=1661875730821226"], + sha256 = "ae9cb01035f18b0023fc12256c048666da76b41b327cec09c2d2820054b1295f", + urls = ["https://storage.googleapis.com/mediapipe-assets/expected_right_down_hand_landmarks.prototxt?generation=1692121986324450"], + ) + + http_file( + name = "com_google_mediapipe_expected_right_down_hand_rotated_landmarks_prototxt", + sha256 = "c4dfdcc2e4cd366eb5f8ad227be94049eb593e3a528564611094687912463687", + urls = ["https://storage.googleapis.com/mediapipe-assets/expected_right_down_hand_rotated_landmarks.prototxt?generation=1692121989028161"], ) http_file( name = "com_google_mediapipe_expected_right_up_hand_landmarks_prototxt", - sha256 = "174cf5f7c3ab547f0affb666ee7be933b0758c60fbfe7b7e93795c5082555592", - urls = ["https://storage.googleapis.com/mediapipe-assets/expected_right_up_hand_landmarks.prototxt?generation=1661875733440313"], + sha256 = "1353ba617c4f048083618587cd23a8a22115f634521c153d4e1bd1ebd4f49dd7", + urls = ["https://storage.googleapis.com/mediapipe-assets/expected_right_up_hand_landmarks.prototxt?generation=1692121991596258"], + ) + + http_file( + name = "com_google_mediapipe_expected_right_up_hand_rotated_landmarks_prototxt", + sha256 = "7fb2d33cf69d2da50952a45bad0c0618f30859e608958fee95948a6e0de63ccb", + urls = ["https://storage.googleapis.com/mediapipe-assets/expected_right_up_hand_rotated_landmarks.prototxt?generation=1692121994043161"], ) http_file( @@ -432,8 +432,8 @@ def external_files(): http_file( name = "com_google_mediapipe_face_stylizer_task", - sha256 = "b34f3896cbe860468538cf5a562c0468964f182b8bb07cb527224312969d1625", - urls = ["https://storage.googleapis.com/mediapipe-assets/face_stylizer.task?generation=1682627841126340"], + sha256 = "423f350aab236123818adb7b39e0a14e14708a9a019fb2fe00a015a2561fd0c8", + urls = ["https://storage.googleapis.com/mediapipe-assets/face_stylizer.task?generation=1693433010526766"], ) http_file( @@ -450,8 +450,8 @@ def external_files(): http_file( name = "com_google_mediapipe_fist_landmarks_pbtxt", - sha256 = "76d6489e6163211ce5e9080e51983165bb9b24ff50146cc7487bd629f011c598", - urls = ["https://storage.googleapis.com/mediapipe-assets/fist_landmarks.pbtxt?generation=1666999360561864"], + sha256 = "4b0ad2b00d5f2d140450f9f168af0f7422ecf6b630b7d64a213bcf6f04fb078b", + urls = ["https://storage.googleapis.com/mediapipe-assets/fist_landmarks.pbtxt?generation=1692121997451835"], ) http_file( @@ -636,14 +636,20 @@ def external_files(): http_file( name = "com_google_mediapipe_left_hands_jpg", - sha256 = "4b5134daa4cb60465535239535f9f74c2842aba3aa5fd30bf04ef5678f93d87f", - urls = ["https://storage.googleapis.com/mediapipe-assets/left_hands.jpg?generation=1661875796949017"], + sha256 = "240c082e80128ff1ca8a83ce645e2ba4d8bc30f0967b7991cf5fa375bab489e1", + urls = ["https://storage.googleapis.com/mediapipe-assets/left_hands.jpg?generation=1692122001487742"], ) http_file( name = "com_google_mediapipe_left_hands_rotated_jpg", - sha256 = "8609c6202bca43a99bbf23fa8e687e49fa525e89481152e4c0987f46d60d7931", - urls = ["https://storage.googleapis.com/mediapipe-assets/left_hands_rotated.jpg?generation=1666037068103465"], + sha256 = "b3bdf692f0d54b86c8b67e6d1286dd0078fbe6e9dfcd507b187e3bd8b398c0f9", + urls = ["https://storage.googleapis.com/mediapipe-assets/left_hands_rotated.jpg?generation=1692122004272021"], + ) + + http_file( + name = "com_google_mediapipe_leopard_bg_removal_result_512x512_png", + sha256 = "30be22e89fdd1d7b985294498ec67509b0caa1ca941fe291fa25f43a3873e4dd", + urls = ["https://storage.googleapis.com/mediapipe-assets/leopard_bg_removal_result_512x512.png?generation=1690239134617707"], ) http_file( @@ -658,6 +664,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/leopard.jpg?generation=1685997280368627"], ) + http_file( + name = "com_google_mediapipe_libimagegenerator_gpu_so", + sha256 = "39ed9738297fa051a7f3cc9bdb7189418a9e118aa3cad4e1d577995837fdd58c", + urls = ["https://storage.googleapis.com/mediapipe-assets/libimagegenerator_gpu.so?generation=1693433013917189"], + ) + http_file( name = "com_google_mediapipe_mobilebert_embedding_with_metadata_tflite", sha256 = "fa47142dcc6f446168bc672f2df9605b6da5d0c0d6264e9be62870282365b95c", @@ -712,6 +724,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/mobile_ica_8bit-with-unsupported-metadata-version.tflite?generation=1661875819091013"], ) + http_file( + name = "com_google_mediapipe_mobilenetsweep_dptrigmqn384_unit_384_384_fp16quant_fp32input_opt_tflite", + sha256 = "3c4c7e36b35fc903ecfb51b351b4849b23c57cc18d1416cf6cabaa1522d84760", + urls = ["https://storage.googleapis.com/mediapipe-assets/mobilenetsweep_dptrigmqn384_unit_384_384_fp16quant_fp32input_opt.tflite?generation=1690302146106240"], + ) + http_file( name = "com_google_mediapipe_mobilenet_v1_0_25_192_quantized_1_default_1_tflite", sha256 = "f80999b6324c6f101300c3ee38fbe7e11e74a743b5e0be7350602087fe7430a3", @@ -936,8 +954,8 @@ def external_files(): http_file( name = "com_google_mediapipe_pointing_up_landmarks_pbtxt", - sha256 = "a3cd7f088a9e997dbb8f00d91dbf3faaacbdb262c8f2fde3c07a9d0656488065", - urls = ["https://storage.googleapis.com/mediapipe-assets/pointing_up_landmarks.pbtxt?generation=1665174976408451"], + sha256 = "6bfcd360c0caa82559396d387ac30e1d59efab3b3d96b5512f4f018d0abae7c4", + urls = ["https://storage.googleapis.com/mediapipe-assets/pointing_up_landmarks.pbtxt?generation=1692122010006268"], ) http_file( @@ -948,8 +966,8 @@ def external_files(): http_file( name = "com_google_mediapipe_pointing_up_rotated_landmarks_pbtxt", - sha256 = "5ec37218d8b613436f5c10121dc689bf9ee69af0656a6ccf8c2e3e8b652e2ad6", - urls = ["https://storage.googleapis.com/mediapipe-assets/pointing_up_rotated_landmarks.pbtxt?generation=1666629486774022"], + sha256 = "cc58cbe1ead8c5051e643d2b90b77d00843cab2f1227af3489513d2b02359dd1", + urls = ["https://storage.googleapis.com/mediapipe-assets/pointing_up_rotated_landmarks.pbtxt?generation=1692122012510778"], ) http_file( @@ -1110,14 +1128,14 @@ def external_files(): http_file( name = "com_google_mediapipe_right_hands_jpg", - sha256 = "240c082e80128ff1ca8a83ce645e2ba4d8bc30f0967b7991cf5fa375bab489e1", - urls = ["https://storage.googleapis.com/mediapipe-assets/right_hands.jpg?generation=1661875908672404"], + sha256 = "4b5134daa4cb60465535239535f9f74c2842aba3aa5fd30bf04ef5678f93d87f", + urls = ["https://storage.googleapis.com/mediapipe-assets/right_hands.jpg?generation=1692122016203904"], ) http_file( name = "com_google_mediapipe_right_hands_rotated_jpg", - sha256 = "b3bdf692f0d54b86c8b67e6d1286dd0078fbe6e9dfcd507b187e3bd8b398c0f9", - urls = ["https://storage.googleapis.com/mediapipe-assets/right_hands_rotated.jpg?generation=1666037076873345"], + sha256 = "8609c6202bca43a99bbf23fa8e687e49fa525e89481152e4c0987f46d60d7931", + urls = ["https://storage.googleapis.com/mediapipe-assets/right_hands_rotated.jpg?generation=1692122018668162"], ) http_file( @@ -1308,14 +1326,14 @@ def external_files(): http_file( name = "com_google_mediapipe_thumb_up_landmarks_pbtxt", - sha256 = "b129ae0536be4e25d6cdee74aabe9dedf1bcfe87430a40b68be4079db3a4d926", - urls = ["https://storage.googleapis.com/mediapipe-assets/thumb_up_landmarks.pbtxt?generation=1665174979747784"], + sha256 = "feddaa81e188b9bceae12a96766f71e8ff3b2b316b4a31d64054d8a329e6015e", + urls = ["https://storage.googleapis.com/mediapipe-assets/thumb_up_landmarks.pbtxt?generation=1692122022310696"], ) http_file( name = "com_google_mediapipe_thumb_up_rotated_landmarks_pbtxt", - sha256 = "6645bbd98ea7f90b3e1ba297e16ea5280847fc5bf5400726d98c282f6c597257", - urls = ["https://storage.googleapis.com/mediapipe-assets/thumb_up_rotated_landmarks.pbtxt?generation=1666629489421733"], + sha256 = "f0e90db82890ad2e0304af5e6e88b2e64f3774eec4d43e56b634a296553b7196", + urls = ["https://storage.googleapis.com/mediapipe-assets/thumb_up_rotated_landmarks.pbtxt?generation=1692122024789637"], ) http_file( @@ -1350,8 +1368,8 @@ def external_files(): http_file( name = "com_google_mediapipe_victory_landmarks_pbtxt", - sha256 = "b25ab4f222674489f543afb6454396ecbc1437a7ae6213dbf0553029ae939ab0", - urls = ["https://storage.googleapis.com/mediapipe-assets/victory_landmarks.pbtxt?generation=1666999366036622"], + sha256 = "73fb59741872bc66b79982d4c9765a4128d6308cc5d919100615080c0f4c0c55", + urls = ["https://storage.googleapis.com/mediapipe-assets/victory_landmarks.pbtxt?generation=1692122027459905"], ) http_file( diff --git a/third_party/flatbuffers/workspace.bzl b/third_party/flatbuffers/workspace.bzl index 0edb7a6f..d06e2cbe 100644 --- a/third_party/flatbuffers/workspace.bzl +++ b/third_party/flatbuffers/workspace.bzl @@ -5,11 +5,11 @@ load("//third_party:repo.bzl", "third_party_http_archive") def repo(): third_party_http_archive( name = "flatbuffers", - strip_prefix = "flatbuffers-23.5.8", - sha256 = "55b75dfa5b6f6173e4abf9c35284a10482ba65db886b39db511eba6c244f1e88", + strip_prefix = "flatbuffers-23.5.26", + sha256 = "1cce06b17cddd896b6d73cc047e36a254fb8df4d7ea18a46acf16c4c0cd3f3f3", urls = [ - "https://github.com/google/flatbuffers/archive/v23.5.8.tar.gz", - "https://github.com/google/flatbuffers/archive/v23.5.8.tar.gz", + "https://github.com/google/flatbuffers/archive/v23.5.26.tar.gz", + "https://github.com/google/flatbuffers/archive/v23.5.26.tar.gz", ], build_file = "//third_party/flatbuffers:BUILD.bazel", delete = ["build_defs.bzl", "BUILD.bazel"], diff --git a/third_party/halide.BUILD b/third_party/halide.BUILD index 677fa9f3..5521f6bb 100644 --- a/third_party/halide.BUILD +++ b/third_party/halide.BUILD @@ -42,7 +42,7 @@ cc_library( cc_library( name = "lib_halide_static", srcs = select({ - "@halide//:halide_config_windows_x86_64": [ + "@mediapipe//mediapipe:windows": [ "bin/Release/Halide.dll", "lib/Release/Halide.lib", ], diff --git a/third_party/halide/BUILD.bazel b/third_party/halide/BUILD.bazel index 8b69a250..272d7826 100644 --- a/third_party/halide/BUILD.bazel +++ b/third_party/halide/BUILD.bazel @@ -22,24 +22,29 @@ package( halide_library_runtimes() -# Aliases to platform-specific targets. -[ - alias( - name = target_name, - actual = select( - { - ":halide_config_linux_x86_64": "@linux_halide//:%s" % target_name, - ":halide_config_macos_x86_64": "@macos_x86_64_halide//:%s" % target_name, - ":halide_config_macos_arm64": "@macos_arm_64_halide//:%s" % target_name, - ":halide_config_windows_x86_64": "@windows_halide//:%s" % target_name, - # deliberately no //condition:default clause here - }, - no_match_error = "Compiling Halide code requires that the build host is one of Linux x86-64, Windows x86-64, macOS x86-64, or macOS arm64.", - ), - ) - for target_name in [ - "language", - "runtime", - "gengen", - ] -] +# Alias the 'gengen' target so that it uses the correct Halide release based on host platform. +alias( + name = "gengen", + actual = select( + { + "@mediapipe//mediapipe:macos_x86_64": "@macos_x86_64_halide//:gengen", + "@mediapipe//mediapipe:macos_arm64": "@macos_arm_64_halide//:gengen", + "@mediapipe//mediapipe:windows": "@windows_halide//:gengen", + "@mediapipe//mediapipe:linux": "@linux_halide//:gengen", + # Deliberately no //condition:default clause here. + }, + no_match_error = "Compiling Halide code requires that the build host is one of Linux x86-64, Windows x86-64, macOS x86-64, or macOS arm64.", + ), +) + +# Arbitrarily alias the 'runtime' and 'language' targets to the Linux Halide release. The underlying +# targets are identical for all host platforms. +alias( + name = "runtime", + actual = "@linux_halide//:runtime", +) + +alias( + name = "language", + actual = "@linux_halide//:language", +) diff --git a/third_party/halide/halide.bzl b/third_party/halide/halide.bzl index bbb0a1f9..9f67f875 100644 --- a/third_party/halide/halide.bzl +++ b/third_party/halide/halide.bzl @@ -82,22 +82,22 @@ def halide_runtime_linkopts(): # Map of halide-target-base -> config_settings _HALIDE_TARGET_CONFIG_SETTINGS_MAP = { # Android - "arm-32-android": ["@halide//:halide_config_android_arm"], - "arm-64-android": ["@halide//:halide_config_android_arm64"], - "x86-32-android": ["@halide//:halide_config_android_x86_32"], - "x86-64-android": ["@halide//:halide_config_android_x86_64"], + "arm-32-android": ["@mediapipe//mediapipe:android_arm"], + "arm-64-android": ["@mediapipe//mediapipe:android_arm64"], + "x86-32-android": ["@mediapipe//mediapipe:android_x86"], + "x86-64-android": ["@mediapipe//mediapipe:android_x86_64"], # iOS - "arm-32-ios": ["@halide//:halide_config_ios_arm"], - "arm-64-ios": ["@halide//:halide_config_ios_arm64"], + "arm-32-ios": ["@mediapipe//mediapipe:ios_armv7"], + "arm-64-ios": ["@mediapipe//mediapipe:ios_arm64", "@mediapipe//mediapipe:ios_arm64e"], # OSX (or iOS simulator) - "x86-32-osx": ["@halide//:halide_config_macos_x86_32", "@halide//:halide_config_ios_x86_32"], - "x86-64-osx": ["@halide//:halide_config_macos_x86_64", "@halide//:halide_config_ios_x86_64"], - "arm-64-osx": ["@halide//:halide_config_macos_arm64"], + "x86-32-osx": ["@mediapipe//mediapipe:ios_i386"], + "x86-64-osx": ["@mediapipe//mediapipe:macos_x86_64", "@mediapipe//mediapipe:ios_x86_64"], + "arm-64-osx": ["@mediapipe//mediapipe:macos_arm64"], # Windows - "x86-64-windows": ["@halide//:halide_config_windows_x86_64"], + "x86-64-windows": ["@mediapipe//mediapipe:windows"], # Linux - "x86-64-linux": ["@halide//:halide_config_linux_x86_64"], - # deliberately nothing here using //conditions:default + "x86-64-linux": ["@mediapipe//mediapipe:linux"], + # Deliberately no //condition:default clause here. } _HALIDE_TARGET_MAP_DEFAULT = { @@ -618,19 +618,6 @@ def _standard_library_runtime_names(): return collections.uniq([_halide_library_runtime_target_name(f) for f in _standard_library_runtime_features()]) def halide_library_runtimes(compatible_with = []): - # Note that we don't use all of these combinations - # (and some are invalid), but that's ok. - for cpu in ["arm", "arm64", "x86_32", "x86_64"]: - for os in ["android", "linux", "windows", "ios", "macos"]: - native.config_setting( - name = "halide_config_%s_%s" % (os, cpu), - constraint_values = [ - "@platforms//os:%s" % os, - "@platforms//cpu:%s" % cpu, - ], - visibility = ["//visibility:public"], - ) - unused = [ _define_halide_library_runtime(f, compatible_with = compatible_with) for f in _standard_library_runtime_features() diff --git a/third_party/org_tensorflow_system_python.diff b/third_party/org_tensorflow_system_python.diff new file mode 100644 index 00000000..57ac01c4 --- /dev/null +++ b/third_party/org_tensorflow_system_python.diff @@ -0,0 +1,51 @@ +diff --git a/tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl b/tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl +index a2bdd6a7eed..ec25c23d8d4 100644 +--- a/tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl ++++ b/tensorflow/tools/toolchains/cpus/aarch64/aarch64_compiler_configure.bzl +@@ -2,7 +2,7 @@ + + load("//tensorflow/tools/toolchains:cpus/aarch64/aarch64.bzl", "remote_aarch64_configure") + load("//third_party/remote_config:remote_platform_configure.bzl", "remote_platform_configure") +-load("//third_party/py:python_configure.bzl", "remote_python_configure") ++load("//third_party/py/non_hermetic:python_configure.bzl", "remote_python_configure") + + def ml2014_tf_aarch64_configs(name_container_map, env): + for name, container in name_container_map.items(): +diff --git a/tensorflow/tools/toolchains/remote_config/rbe_config.bzl b/tensorflow/tools/toolchains/remote_config/rbe_config.bzl +index 9f71a414bf7..57f70752323 100644 +--- a/tensorflow/tools/toolchains/remote_config/rbe_config.bzl ++++ b/tensorflow/tools/toolchains/remote_config/rbe_config.bzl +@@ -1,6 +1,6 @@ + """Macro that creates external repositories for remote config.""" + +-load("//third_party/py:python_configure.bzl", "local_python_configure", "remote_python_configure") ++load("//third_party/py/non_hermetic:python_configure.bzl", "local_python_configure", "remote_python_configure") + load("//third_party/gpus:cuda_configure.bzl", "remote_cuda_configure") + load("//third_party/nccl:nccl_configure.bzl", "remote_nccl_configure") + load("//third_party/gpus:rocm_configure.bzl", "remote_rocm_configure") +diff --git a/tensorflow/workspace2.bzl b/tensorflow/workspace2.bzl +index 953e1d1bea6..664608592a5 100644 +--- a/tensorflow/workspace2.bzl ++++ b/tensorflow/workspace2.bzl +@@ -8,7 +8,7 @@ load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") + load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") + load("//third_party/nccl:nccl_configure.bzl", "nccl_configure") + load("//third_party/git:git_configure.bzl", "git_configure") +-load("//third_party/py:python_configure.bzl", "python_configure") ++load("//third_party/py/non_hermetic:python_configure.bzl", "python_configure") + load("//third_party/systemlibs:syslibs_configure.bzl", "syslibs_configure") + load("//tensorflow/tools/toolchains:cpus/aarch64/aarch64_compiler_configure.bzl", "aarch64_compiler_configure") + load("//tensorflow/tools/toolchains:cpus/arm/arm_compiler_configure.bzl", "arm_compiler_configure") +diff --git a/third_party/py/non_hermetic/python_configure.bzl b/third_party/py/non_hermetic/python_configure.bzl +index 300cbfb6c71..09d98505dd9 100644 +--- a/third_party/py/non_hermetic/python_configure.bzl ++++ b/third_party/py/non_hermetic/python_configure.bzl +@@ -206,7 +206,7 @@ def _create_local_python_repository(repository_ctx): + # Resolve all labels before doing any real work. Resolving causes the + # function to be restarted with all previous state being lost. This + # can easily lead to a O(n^2) runtime in the number of labels. +- build_tpl = repository_ctx.path(Label("//third_party/py:BUILD.tpl")) ++ build_tpl = repository_ctx.path(Label("//third_party/py/non_hermetic:BUILD.tpl")) + + python_bin = get_python_bin(repository_ctx) + _check_python_bin(repository_ctx, python_bin) \ No newline at end of file diff --git a/third_party/wasm_files.bzl b/third_party/wasm_files.bzl index 8ef0a71a..1aae204d 100644 --- a/third_party/wasm_files.bzl +++ b/third_party/wasm_files.bzl @@ -12,72 +12,72 @@ def wasm_files(): http_file( name = "com_google_mediapipe_wasm_audio_wasm_internal_js", - sha256 = "0d66a26fa5ca638c54ec3e5bffb50aec74ee0880b108d4b5f7d316e9ae36cc9a", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1685638894464709"], + sha256 = "9e5f88363212ac1ad505a0b9e59e3dd34413064f3b70219ff8b0216d6a53128f", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1690577772170421"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_internal_wasm", - sha256 = "014963d19ef6b1f25720379c3df07a6e08b24894ada4938d45b1256e97739318", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1685638897160853"], + sha256 = "8e4c7e9efcfe0d1107b40626f14070f17a817d2b830205ae642ea645fa882d28", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1690577774642876"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_js", - sha256 = "f03d4826c251783bfc1fb8b82b2d08c00b2e3cb2efcc606305eb210f09fc686b", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1685638899477366"], + sha256 = "9b9d1fbbead06a26461bb664189d46f0c327a1077e67f0aeeb0628d04de13a81", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1690577777075565"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_wasm", - sha256 = "36972cf62138bcb5fde37a1fecce334a86b0261eefc1f1daa17b4b8acdc784b4", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1685638901926088"], + sha256 = "44734a8fdb979eb9359de0c0282565d74cdced5d3a6687be849875e0eb11503c", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1690577779811164"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_internal_js", - sha256 = "5745360da942f3bcb585547e8720cb11f19793e68851b119b8f9ea22b120fd06", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1685638904214551"], + sha256 = "93275ebbae8dd2e9be0394391b722a0de5ac9ed51066093b1ac6ec24bebf5813", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1690577782193422"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_internal_wasm", - sha256 = "b6d8b03fa7fc3e969febfcb63e3db2de900f1f54b82bf2205f02d865fc4790b2", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1685638906864568"], + sha256 = "35e734890cae0c51c1ad91e3589d5777b013bcbac64a5bcbb3a67ce4a5815dd6", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1690577784996034"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_js", - sha256 = "837ca361044441e6202858b4a9d94b3296c8440099b40e6dafb1efcce76a8f63", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1685638909139832"], + sha256 = "4e6cea3ae95ffac595bfc08f0dab4ff452c91434eb71f92c0dd34250a46825a1", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1690577787398460"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_wasm", - sha256 = "507f4089f4a2cf8fe7fb61f48e180f3f86d5e8057fc60ef24c77aae724eb66ba", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1685638911843312"], + sha256 = "43cfab25c1d47822015e434d726a80d84e0bfdb5e685a511ab45d8b5cbe944d3", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1690577790301890"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_internal_js", - sha256 = "82de7a40fdb14833b5ceaeb1ebf219421dbb06ba5e525204737dec196161420d", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1685638914190745"], + sha256 = "6a73602a14484297690e69d716e683341b62a5fde8f5debde78de2651cb69bbe", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1690577792657082"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_internal_wasm", - sha256 = "d06ac49f4c156cf0c24ef62387b13e48b67476e7f04a423889c59ee835c460f2", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1685638917012370"], + sha256 = "3431f70071f3980bf13e638551e9bb333335223e35542ee768db06501f7a26f2", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1690577795814175"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_js", - sha256 = "fff428ef91d8cc936f9c3ec81750f5e7ee3c20bc0c76677eb5d8d4d010d2fac0", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1685638919406810"], + sha256 = "ece9ac1f41b93340b08682514ca291431ff7084c858caf6455e65b0c6c3eb717", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1690577798226032"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_wasm", - sha256 = "f87c51b8744b0ba564ce725fc3659dba5ef90b4615ac34135ca91c6508434fe9", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1685638922016130"], + sha256 = "4d54739714db6b3d0fbdd0608c2824c4ccceaaf279aa4ba160f2eab2663b30f2", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1690577801077668"], )