Compare commits

..
Author SHA1 Message Date
dependabot[bot]andGitHub a0a3357151 Bump protobufjs from 7.1.2 to 7.2.4
Bumps [protobufjs](https://github.com/protobufjs/protobuf.js) from 7.1.2 to 7.2.4.
- [Release notes](https://github.com/protobufjs/protobuf.js/releases)
- [Changelog](https://github.com/protobufjs/protobuf.js/blob/master/CHANGELOG.md)
- [Commits](https://github.com/protobufjs/protobuf.js/compare/protobufjs-v7.1.2...protobufjs-v7.2.4)

---
updated-dependencies:
- dependency-name: protobufjs
  dependency-type: direct:development
...

Signed-off-by: dependabot[bot] <[email protected]>
2023-07-08 00:36:26 +00:00
757 changed files with 8155 additions and 23078 deletions
@@ -40,16 +40,18 @@ body:
label: Programming Language and version (e.g. C++, Python, Java) label: Programming Language and version (e.g. C++, Python, Java)
validations: validations:
required: true required: true
- type: input - type: textarea
id: current_model id: current_model
attributes: attributes:
label: Describe the actual behavior label: Describe the actual behavior
render: shell
validations: validations:
required: true required: true
- type: input - type: textarea
id: expected_model id: expected_model
attributes: attributes:
label: Describe the expected behaviour label: Describe the expected behaviour
render: shell
validations: validations:
required: true required: true
- type: textarea - type: textarea
@@ -41,16 +41,18 @@ body:
label: Task name (e.g. Image classification, Gesture recognition etc.) label: Task name (e.g. Image classification, Gesture recognition etc.)
validations: validations:
required: true required: true
- type: input - type: textarea
id: current_model id: current_model
attributes: attributes:
label: Describe the actual behavior label: Describe the actual behavior
render: shell
validations: validations:
required: true required: true
- type: input - type: textarea
id: expected_model id: expected_model
attributes: attributes:
label: Describe the expected behaviour label: Describe the expected behaviour
render: shell
validations: validations:
required: true required: true
- type: textarea - type: textarea
@@ -31,16 +31,18 @@ body:
label: URL that shows the problem label: URL that shows the problem
validations: validations:
required: false required: false
- type: input - type: textarea
id: current_model id: current_model
attributes: attributes:
label: Describe the actual behavior label: Describe the actual behavior
render: shell
validations: validations:
required: false required: false
- type: input - type: textarea
id: expected_model id: expected_model
attributes: attributes:
label: Describe the expected behaviour label: Describe the expected behaviour
render: shell
validations: validations:
required: false required: false
- type: textarea - type: textarea
@@ -28,33 +28,37 @@ body:
- 'No' - 'No'
validations: validations:
required: false required: false
- type: input - type: textarea
id: behaviour id: behaviour
attributes: attributes:
label: Describe the feature and the current behaviour/state label: Describe the feature and the current behaviour/state
render: shell
validations: validations:
required: true required: true
- type: input - type: textarea
id: api_change id: api_change
attributes: attributes:
label: Will this change the current API? How? label: Will this change the current API? How?
render: shell
validations: validations:
required: false required: false
- type: input - type: textarea
id: benifit id: benifit
attributes: attributes:
label: Who will benefit with this feature? label: Who will benefit with this feature?
validations: validations:
required: false required: false
- type: input - type: textarea
id: use_case id: use_case
attributes: attributes:
label: Please specify the use cases for this feature label: Please specify the use cases for this feature
render: shell
validations: validations:
required: true required: true
- type: input - type: textarea
id: info_other id: info_other
attributes: attributes:
label: Any Other info label: Any Other info
render: shell
validations: validations:
required: false required: false
@@ -87,13 +87,14 @@ body:
placeholder: placeholder:
validations: validations:
required: false required: false
- type: input - type: textarea
id: what-happened id: what-happened
attributes: attributes:
label: Describe the problem label: Describe the problem
description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html) description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html)
placeholder: Tell us what you see! placeholder: Tell us what you see!
value: "A bug happened!" value: "A bug happened!"
render: shell
validations: validations:
required: true required: true
- type: textarea - type: textarea
@@ -80,16 +80,18 @@ body:
label: Xcode & Tulsi version (if issue is related to building for iOS) label: Xcode & Tulsi version (if issue is related to building for iOS)
validations: validations:
required: false required: false
- type: input - type: textarea
id: current_model id: current_model
attributes: attributes:
label: Describe the actual behavior label: Describe the actual behavior
render: shell
validations: validations:
required: true required: true
- type: input - type: textarea
id: expected_model id: expected_model
attributes: attributes:
label: Describe the expected behaviour label: Describe the expected behaviour
render: shell
validations: validations:
required: true required: true
- type: textarea - type: textarea
@@ -48,16 +48,18 @@ body:
placeholder: e.g. C++, Python, Java placeholder: e.g. C++, Python, Java
validations: validations:
required: false required: false
- type: input - type: textarea
id: current_model id: current_model
attributes: attributes:
label: Describe the actual behavior label: Describe the actual behavior
render: shell
validations: validations:
required: false required: false
- type: input - type: textarea
id: expected_model id: expected_model
attributes: attributes:
label: Describe the expected behaviour label: Describe the expected behaviour
render: shell
validations: validations:
required: false required: false
- type: textarea - type: textarea
+15 -13
View File
@@ -73,9 +73,12 @@ http_archive(
http_archive( http_archive(
name = "zlib", name = "zlib",
build_file = "@//third_party:zlib.BUILD", build_file = "@//third_party:zlib.BUILD",
sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30", sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
strip_prefix = "zlib-1.2.13", strip_prefix = "zlib-1.2.11",
url = "http://zlib.net/fossils/zlib-1.2.13.tar.gz", urls = [
"http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
"http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
],
patches = [ patches = [
"@//third_party:zlib.diff", "@//third_party:zlib.diff",
], ],
@@ -154,19 +157,19 @@ http_archive(
# 2020-08-21 # 2020-08-21
http_archive( http_archive(
name = "com_github_glog_glog", name = "com_github_glog_glog",
strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372", strip_prefix = "glog-0.6.0",
sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb", sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6",
urls = [ urls = [
"https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip", "https://github.com/google/glog/archive/v0.6.0.tar.gz",
], ],
) )
http_archive( http_archive(
name = "com_github_glog_glog_no_gflags", name = "com_github_glog_glog_no_gflags",
strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372", strip_prefix = "glog-0.6.0",
sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb", sha256 = "8a83bf982f37bb70825df71a9709fa90ea9f4447fb3c099e1d720a439d88bad6",
build_file = "@//third_party:glog_no_gflags.BUILD", build_file = "@//third_party:glog_no_gflags.BUILD",
urls = [ urls = [
"https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip", "https://github.com/google/glog/archive/v0.6.0.tar.gz",
], ],
patches = [ patches = [
"@//third_party:com_github_glog_glog.diff", "@//third_party:com_github_glog_glog.diff",
@@ -482,10 +485,10 @@ http_archive(
) )
# TensorFlow repo should always go after the other external dependencies. # TensorFlow repo should always go after the other external dependencies.
# TF on 2023-07-26. # TF on 2023-06-13.
_TENSORFLOW_GIT_COMMIT = "e92261fd4cec0b726692081c4d2966b75abf31dd" _TENSORFLOW_GIT_COMMIT = "491681a5620e41bf079a582ac39c585cc86878b9"
# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256 # curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
_TENSORFLOW_SHA256 = "478a229bd4ec70a5b568ac23b5ea013d9fca46a47d6c43e30365a0412b9febf4" _TENSORFLOW_SHA256 = "9f76389af7a2835e68413322c1eaabfadc912f02a76d71dc16be507f9ca3d3ac"
http_archive( http_archive(
name = "org_tensorflow", name = "org_tensorflow",
urls = [ urls = [
@@ -493,7 +496,6 @@ http_archive(
], ],
patches = [ patches = [
"@//third_party:org_tensorflow_compatibility_fixes.diff", "@//third_party:org_tensorflow_compatibility_fixes.diff",
"@//third_party:org_tensorflow_system_python.diff",
# Diff is generated with a script, don't update it manually. # Diff is generated with a script, don't update it manually.
"@//third_party:org_tensorflow_custom_ops.diff", "@//third_party:org_tensorflow_custom_ops.diff",
], ],
+2 -2
View File
@@ -50,7 +50,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
3. The [`hello world`] example uses a simple MediaPipe graph in the 3. The [`hello world`] example uses a simple MediaPipe graph in the
`PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto. `PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
```c++ ```C++
absl::Status PrintHelloWorld() { absl::Status PrintHelloWorld() {
// Configures a simple graph, which concatenates 2 PassThroughCalculators. // Configures a simple graph, which concatenates 2 PassThroughCalculators.
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"( CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
@@ -126,7 +126,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
```c++ ```c++
mediapipe::Packet packet; mediapipe::Packet packet;
while (poller.Next(&packet)) { while (poller.Next(&packet)) {
ABSL_LOG(INFO) << packet.Get<string>(); LOG(INFO) << packet.Get<string>();
} }
``` ```
+1 -1
View File
@@ -138,7 +138,7 @@ Create a `BUILD` file in the `$APPLICATION_PATH` and add the following build
rules: rules:
``` ```
MIN_IOS_VERSION = "12.0" MIN_IOS_VERSION = "11.0"
load( load(
"@build_bazel_rules_apple//apple:ios.bzl", "@build_bazel_rules_apple//apple:ios.bzl",
+104 -150
View File
@@ -14,155 +14,81 @@
licenses(["notice"]) # Apache 2.0 licenses(["notice"]) # Apache 2.0
load("@mediapipe//mediapipe:platforms.bzl", "config_setting_and_platform") # Note: yes, these need to use "//external:android/crosstool", not
# @androidndk//:default_crosstool.
# Generic Android
config_setting( config_setting(
name = "android", name = "android",
constraint_values = [ values = {"crosstool_top": "//external:android/crosstool"},
"@platforms//os:android",
],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
# Android x86 32-bit.
config_setting_and_platform(
name = "android_x86",
constraint_values = [
"@platforms//os:android",
"@platforms//cpu:x86_32",
],
visibility = ["//visibility:public"],
)
# Android x86 64-bit.
config_setting_and_platform(
name = "android_x86_64",
constraint_values = [
"@platforms//os:android",
"@platforms//cpu:x86_64",
],
visibility = ["//visibility:public"],
)
# Android ARMv7.
config_setting_and_platform(
name = "android_arm",
constraint_values = [
"@platforms//os:android",
"@platforms//cpu:armv7",
],
visibility = ["//visibility:public"],
)
# Android ARM64.
config_setting_and_platform(
name = "android_arm64",
constraint_values = [
"@platforms//os:android",
"@platforms//cpu:arm64",
],
visibility = ["//visibility:public"],
)
# Generic MacOS.
config_setting( config_setting(
name = "android_x86",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "x86",
},
visibility = ["//visibility:public"],
)
config_setting(
name = "android_x86_64",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "x86_64",
},
visibility = ["//visibility:public"],
)
config_setting(
name = "android_armeabi",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "armeabi",
},
visibility = ["//visibility:public"],
)
config_setting(
name = "android_arm",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "armeabi-v7a",
},
visibility = ["//visibility:public"],
)
config_setting(
name = "android_arm64",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "arm64-v8a",
},
visibility = ["//visibility:public"],
)
# Note: this cannot just match "apple_platform_type": "macos" because that option
# defaults to "macos" even when building on Linux!
alias(
name = "macos", name = "macos",
constraint_values = [ actual = select({
"@platforms//os:macos", ":macos_i386": ":macos_i386",
], ":macos_x86_64": ":macos_x86_64",
":macos_arm64": ":macos_arm64",
"//conditions:default": ":macos_i386", # Arbitrarily chosen from above.
}),
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
# MacOS x86 64-bit. # Note: this also matches on crosstool_top so that it does not produce ambiguous
config_setting_and_platform( # selectors when used together with "android".
name = "macos_x86_64",
constraint_values = [
"@platforms//os:macos",
"@platforms//cpu:x86_64",
],
visibility = ["//visibility:public"],
)
# MacOS ARM64.
config_setting_and_platform(
name = "macos_arm64",
constraint_values = [
"@platforms//os:macos",
"@platforms//cpu:arm64",
],
visibility = ["//visibility:public"],
)
# Generic iOS.
config_setting( config_setting(
name = "ios", name = "ios",
constraint_values = [ values = {
"@platforms//os:ios", "crosstool_top": "@bazel_tools//tools/cpp:toolchain",
], "apple_platform_type": "ios",
visibility = ["//visibility:public"], },
)
# iOS device ARM32.
config_setting_and_platform(
name = "ios_armv7",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:arm",
],
visibility = ["//visibility:public"],
)
# iOS device ARM64.
config_setting_and_platform(
name = "ios_arm64",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:arm64",
],
visibility = ["//visibility:public"],
)
# iOS device ARM64E.
config_setting_and_platform(
name = "ios_arm64e",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:arm64e",
],
visibility = ["//visibility:public"],
)
# iOS simulator x86 32-bit.
config_setting_and_platform(
name = "ios_i386",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:x86_32",
"@build_bazel_apple_support//constraints:simulator",
],
visibility = ["//visibility:public"],
)
# iOS simulator x86 64-bit.
config_setting_and_platform(
name = "ios_x86_64",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:x86_64",
"@build_bazel_apple_support//constraints:simulator",
],
visibility = ["//visibility:public"],
)
# iOS simulator ARM64.
config_setting_and_platform(
name = "ios_sim_arm64",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:arm64",
"@build_bazel_apple_support//constraints:simulator",
],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
@@ -176,26 +102,54 @@ alias(
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
# Windows 64-bit. config_setting(
config_setting_and_platform( name = "macos_i386",
name = "windows", values = {
constraint_values = [ "apple_platform_type": "macos",
"@platforms//os:windows", "cpu": "darwin",
"@platforms//cpu:x86_64", },
],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
# Linux 64-bit. config_setting(
config_setting_and_platform( name = "macos_x86_64",
name = "linux", values = {
constraint_values = [ "apple_platform_type": "macos",
"@platforms//os:linux", "cpu": "darwin_x86_64",
"@platforms//cpu:x86_64", },
],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
) )
config_setting(
name = "macos_arm64",
values = {
"apple_platform_type": "macos",
"cpu": "darwin_arm64",
},
visibility = ["//visibility:public"],
)
[
config_setting(
name = arch,
values = {"cpu": arch},
visibility = ["//visibility:public"],
)
for arch in [
"ios_i386",
"ios_x86_64",
"ios_armv7",
"ios_arm64",
"ios_arm64e",
"ios_sim_arm64",
]
]
config_setting(
name = "windows",
values = {"cpu": "x64_windows"},
)
exports_files( exports_files(
["provisioning_profile.mobileprovision"], ["provisioning_profile.mobileprovision"],
visibility = ["//visibility:public"], visibility = ["//visibility:public"],
+1 -9
View File
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# Placeholder: load py_proto_library
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"]) licenses(["notice"])
@@ -146,7 +145,6 @@ cc_library(
"//mediapipe/framework/port:logging", "//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util", "//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp/mfcc", "@com_google_audio_tools//audio/dsp/mfcc",
"@eigen_archive//:eigen3", "@eigen_archive//:eigen3",
@@ -165,9 +163,8 @@ cc_library(
"//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:logging",
"//mediapipe/util:time_series_util", "//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp:resampler", "@com_google_audio_tools//audio/dsp:resampler",
"@com_google_audio_tools//audio/dsp:resampler_q", "@com_google_audio_tools//audio/dsp:resampler_q",
@@ -188,7 +185,6 @@ cc_library(
"//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:core_proto",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util", "//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -228,7 +224,6 @@ cc_library(
"//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/util:time_series_util", "//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_audio_tools//audio/dsp:window_functions", "@com_google_audio_tools//audio/dsp:window_functions",
"@eigen_archive//:eigen3", "@eigen_archive//:eigen3",
], ],
@@ -299,7 +294,6 @@ cc_test(
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util", "//mediapipe/util:time_series_test_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_audio_tools//audio/dsp:number_util", "@com_google_audio_tools//audio/dsp:number_util",
"@eigen_archive//:eigen3", "@eigen_archive//:eigen3",
], ],
@@ -333,7 +327,6 @@ cc_binary(
"//mediapipe/framework:packet", "//mediapipe/framework:packet",
"//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/formats:time_series_header_cc_proto",
"@com_google_absl//absl/log:absl_check",
"@com_google_benchmark//:benchmark", "@com_google_benchmark//:benchmark",
], ],
) )
@@ -352,7 +345,6 @@ cc_test(
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util", "//mediapipe/util:time_series_test_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_audio_tools//audio/dsp:window_functions", "@com_google_audio_tools//audio/dsp:window_functions",
"@eigen_archive//:eigen3", "@eigen_archive//:eigen3",
], ],
@@ -23,7 +23,6 @@
#include <vector> #include <vector>
#include "Eigen/Core" #include "Eigen/Core"
#include "absl/log/absl_check.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/strings/substitute.h" #include "absl/strings/substitute.h"
@@ -139,7 +138,7 @@ absl::Status FramewiseTransformCalculatorBase::Process(CalculatorContext* cc) {
TransformFrame(input_frame, &output_frame); TransformFrame(input_frame, &output_frame);
// Copy output from vector<float> to Eigen::Vector. // Copy output from vector<float> to Eigen::Vector.
ABSL_CHECK_EQ(output_frame.size(), num_output_channels_); CHECK_EQ(output_frame.size(), num_output_channels_);
Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0], Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0],
output_frame.size(), 1); output_frame.size(), 1);
output->col(frame) = output_frame_map.cast<float>(); output->col(frame) = output_frame_map.cast<float>();
@@ -16,8 +16,6 @@
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h" #include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "audio/dsp/resampler_q.h" #include "audio/dsp/resampler_q.h"
using audio_dsp::Resampler; using audio_dsp::Resampler;
@@ -47,9 +45,9 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
if (matrix->cols() == 0) { if (matrix->cols() == 0) {
matrix->resize(matrix->rows(), vec.size()); matrix->resize(matrix->rows(), vec.size());
} else { } else {
ABSL_CHECK_EQ(vec.size(), matrix->cols()); CHECK_EQ(vec.size(), matrix->cols());
} }
ABSL_CHECK_LT(channel, matrix->rows()); CHECK_LT(channel, matrix->rows());
matrix->row(channel) = matrix->row(channel) =
Eigen::Map<const Eigen::ArrayXf>(vec.data(), vec.size()); Eigen::Map<const Eigen::ArrayXf>(vec.data(), vec.size());
} }
@@ -79,7 +77,7 @@ absl::Status RationalFactorResampleCalculator::Open(CalculatorContext* cc) {
r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_, r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
resample_options); resample_options);
if (!r) { if (!r) {
ABSL_LOG(ERROR) << "Failed to initialize resampler."; LOG(ERROR) << "Failed to initialize resampler.";
return absl::UnknownError("Failed to initialize resampler."); return absl::UnknownError("Failed to initialize resampler.");
} }
} }
@@ -27,6 +27,7 @@
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/time_series_header.pb.h" #include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/util/time_series_util.h" #include "mediapipe/util/time_series_util.h"
namespace mediapipe { namespace mediapipe {
@@ -22,7 +22,6 @@
#include <vector> #include <vector>
#include "Eigen/Core" #include "Eigen/Core"
#include "absl/log/absl_log.h"
#include "audio/dsp/number_util.h" #include "audio/dsp/number_util.h"
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h" #include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -883,11 +882,11 @@ void BM_ProcessDC(benchmark::State& state) {
const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0); const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
const Matrix& output_matrix = output.packets[0].Get<Matrix>(); const Matrix& output_matrix = output.packets[0].Get<Matrix>();
ABSL_LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x" LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
<< output_matrix.cols(); << output_matrix.cols();
ABSL_LOG(INFO) << "First values=" << output_matrix(0, 0) << ", " LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
<< output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", " << output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
<< output_matrix(3, 0); << output_matrix(3, 0);
} }
BENCHMARK(BM_ProcessDC); BENCHMARK(BM_ProcessDC);
@@ -18,7 +18,6 @@
#include <memory> #include <memory>
#include <string> #include <string>
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h" #include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
@@ -60,7 +59,7 @@ class StabilizedLogCalculator : public CalculatorBase {
output_scale_ = stabilized_log_calculator_options.output_scale(); output_scale_ = stabilized_log_calculator_options.output_scale();
check_nonnegativity_ = check_nonnegativity_ =
stabilized_log_calculator_options.check_nonnegativity(); stabilized_log_calculator_options.check_nonnegativity();
ABSL_CHECK_GE(stabilizer_, 0.0) CHECK_GE(stabilizer_, 0.0)
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_; << "stabilizer must be >= 0.0, received a value of " << stabilizer_;
// If the input packets have a header, propagate the header to the output. // If the input packets have a header, propagate the header to the output.
@@ -18,7 +18,6 @@
#include <vector> #include <vector>
#include "Eigen/Core" #include "Eigen/Core"
#include "absl/log/absl_check.h"
#include "audio/dsp/window_functions.h" #include "audio/dsp/window_functions.h"
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h" #include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -105,7 +104,7 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
// All numbers are in input samples. // All numbers are in input samples.
const int64_t current_output_frame_start = static_cast<int64_t>( const int64_t current_output_frame_start = static_cast<int64_t>(
round(cumulative_output_frames_ * average_frame_step_samples_)); round(cumulative_output_frames_ * average_frame_step_samples_));
ABSL_CHECK_EQ(current_output_frame_start, cumulative_completed_samples_); CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
const int64_t next_output_frame_start = static_cast<int64_t>( const int64_t next_output_frame_start = static_cast<int64_t>(
round((cumulative_output_frames_ + 1) * average_frame_step_samples_)); round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
return next_output_frame_start - current_output_frame_start; return next_output_frame_start - current_output_frame_start;
@@ -17,7 +17,6 @@
#include <random> #include <random>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "benchmark/benchmark.h" #include "benchmark/benchmark.h"
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h" #include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -71,7 +70,7 @@ void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
} }
// Initialize graph. // Initialize graph.
mediapipe::CalculatorGraph graph; mediapipe::CalculatorGraph graph;
ABSL_CHECK_OK(graph.Initialize(config)); CHECK_OK(graph.Initialize(config));
// Prepare input header. // Prepare input header.
auto header = std::make_unique<mediapipe::TimeSeriesHeader>(); auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
header->set_sample_rate(kSampleRate); header->set_sample_rate(kSampleRate);
@@ -79,13 +78,13 @@ void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
state.ResumeTiming(); // Resume benchmark timing. state.ResumeTiming(); // Resume benchmark timing.
ABSL_CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}})); CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
for (auto& packet : input_packets) { for (auto& packet : input_packets) {
ABSL_CHECK_OK(graph.AddPacketToInputStream("input", packet)); CHECK_OK(graph.AddPacketToInputStream("input", packet));
} }
ABSL_CHECK(!graph.HasError()); CHECK(!graph.HasError());
ABSL_CHECK_OK(graph.CloseAllInputStreams()); CHECK_OK(graph.CloseAllInputStreams());
ABSL_CHECK_OK(graph.WaitUntilIdle()); CHECK_OK(graph.WaitUntilIdle());
} }
} }
BENCHMARK(BM_TimeSeriesFramerCalculator); BENCHMARK(BM_TimeSeriesFramerCalculator);
@@ -19,7 +19,6 @@
#include <vector> #include <vector>
#include "Eigen/Core" #include "Eigen/Core"
#include "absl/log/absl_log.h"
#include "audio/dsp/window_functions.h" #include "audio/dsp/window_functions.h"
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h" #include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -187,12 +186,11 @@ class TimeSeriesFramerCalculatorTest
const int num_unique_output_samples = const int num_unique_output_samples =
round((output().packets.size() - 1) * frame_step_samples) + round((output().packets.size() - 1) * frame_step_samples) +
frame_duration_samples; frame_duration_samples;
ABSL_LOG(INFO) << "packets.size()=" << output().packets.size() LOG(INFO) << "packets.size()=" << output().packets.size()
<< " frame_duration_samples=" << frame_duration_samples << " frame_duration_samples=" << frame_duration_samples
<< " frame_step_samples=" << frame_step_samples << " frame_step_samples=" << frame_step_samples
<< " num_input_samples_=" << num_input_samples_ << " num_input_samples_=" << num_input_samples_
<< " num_unique_output_samples=" << " num_unique_output_samples=" << num_unique_output_samples;
<< num_unique_output_samples;
const int num_padding_samples = const int num_padding_samples =
num_unique_output_samples - num_input_samples_; num_unique_output_samples - num_input_samples_;
if (options_.pad_final_packet()) { if (options_.pad_final_packet()) {
+12 -9
View File
@@ -381,6 +381,17 @@ cc_library(
alwayslink = 1, alwayslink = 1,
) )
cc_library(
name = "clip_detection_vector_size_calculator",
srcs = ["clip_detection_vector_size_calculator.cc"],
deps = [
":clip_vector_size_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:detection_cc_proto",
],
alwayslink = 1,
)
cc_test( cc_test(
name = "clip_vector_size_calculator_test", name = "clip_vector_size_calculator_test",
srcs = ["clip_vector_size_calculator_test.cc"], srcs = ["clip_vector_size_calculator_test.cc"],
@@ -582,7 +593,6 @@ cc_library(
"//mediapipe/framework/port:logging", "//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util", "//mediapipe/framework/tool:options_util",
"@com_google_absl//absl/log:absl_check",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -598,7 +608,6 @@ cc_test(
"//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],
) )
@@ -631,7 +640,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -779,11 +787,10 @@ cc_library(
"//mediapipe/framework/deps:random", "//mediapipe/framework/deps:random",
"//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util", "//mediapipe/framework/tool:options_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],
alwayslink = 1, alwayslink = 1,
@@ -839,7 +846,6 @@ cc_test(
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:logging", "//mediapipe/framework/port:logging",
"//mediapipe/framework/tool:validate_type", "//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/log:absl_check",
"@eigen_archive//:eigen3", "@eigen_archive//:eigen3",
], ],
) )
@@ -1027,7 +1033,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node", "//mediapipe/framework/api2:node",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -1066,7 +1071,6 @@ cc_test(
"//mediapipe/framework:calculator_runner", "//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_log",
], ],
) )
@@ -1113,7 +1117,6 @@ cc_library(
"//mediapipe/framework/api2:node", "//mediapipe/framework/api2:node",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -1,4 +1,4 @@
// Copyright 2023 The MediaPipe Authors. // Copyright 2019 The MediaPipe Authors.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@@ -11,15 +11,16 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
//
// Forked from mediapipe/framework/tool/source.proto.
// The forked proto must remain identical to the original proto and should be
// ONLY used by mediapipe open source project.
syntax = "proto3"; #include <vector>
package mediapipe; #include "mediapipe/calculators/core/clip_vector_size_calculator.h"
#include "mediapipe/framework/formats/detection.pb.h"
message Proto3Options { namespace mediapipe {
double test_value = 1;
} typedef ClipVectorSizeCalculator<::mediapipe::Detection>
ClipDetectionVectorSizeCalculator;
REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator);
} // namespace mediapipe
@@ -14,8 +14,6 @@
#include "mediapipe/calculators/core/end_loop_calculator.h" #include "mediapipe/calculators/core/end_loop_calculator.h"
#include <array>
#include <utility>
#include <vector> #include <vector>
#include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/classification.pb.h"
@@ -86,8 +84,4 @@ typedef EndLoopCalculator<std::vector<std::array<float, 16>>>
EndLoopAffineMatrixCalculator; EndLoopAffineMatrixCalculator;
REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator); REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
typedef EndLoopCalculator<std::vector<std::pair<int, int>>>
EndLoopImageSizeCalculator;
REGISTER_CALCULATOR(EndLoopImageSizeCalculator);
} // namespace mediapipe } // namespace mediapipe
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_log.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/gtest.h"
@@ -357,18 +356,18 @@ TEST_F(GateCalculatorTest, AllowWithStateChangeNoDataStreams) {
RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true); RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true);
constexpr int64_t kTimestampValue3 = 45; constexpr int64_t kTimestampValue3 = 45;
RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false); RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false);
ABSL_LOG(INFO) << "a"; LOG(INFO) << "a";
const std::vector<Packet>& output = const std::vector<Packet>& output =
runner()->Outputs().Get("STATE_CHANGE", 0).packets; runner()->Outputs().Get("STATE_CHANGE", 0).packets;
ABSL_LOG(INFO) << "s"; LOG(INFO) << "s";
ASSERT_EQ(2, output.size()); ASSERT_EQ(2, output.size());
ABSL_LOG(INFO) << "d"; LOG(INFO) << "d";
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value()); EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value()); EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
ABSL_LOG(INFO) << "f"; LOG(INFO) << "f";
EXPECT_EQ(true, output[0].Get<bool>()); // Allow. EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow. EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
ABSL_LOG(INFO) << "g"; LOG(INFO) << "g";
} }
TEST_F(GateCalculatorTest, DisallowWithStateChange) { TEST_F(GateCalculatorTest, DisallowWithStateChange) {
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_log.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
@@ -79,7 +78,7 @@ absl::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) { if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) {
cc->Outputs().Index(0).AddPacket(packet); cc->Outputs().Index(0).AddPacket(packet);
} else { } else {
ABSL_LOG_FIRST_N(WARNING, 5) LOG_FIRST_N(WARNING, 5)
<< "Dropping a packet with timestamp " << packet.Timestamp(); << "Dropping a packet with timestamp " << packet.Timestamp();
} }
if (cc->Outputs().NumEntries() >= 2) { if (cc->Outputs().NumEntries() >= 2) {
@@ -16,7 +16,6 @@
#include <vector> #include <vector>
#include "Eigen/Core" #include "Eigen/Core"
#include "absl/log/absl_check.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
@@ -210,7 +209,7 @@ TEST(MatrixMultiplyCalculatorTest, Multiply) {
MatrixFromTextProto(kSamplesText, &samples); MatrixFromTextProto(kSamplesText, &samples);
Matrix expected; Matrix expected;
MatrixFromTextProto(kExpectedText, &expected); MatrixFromTextProto(kExpectedText, &expected);
ABSL_CHECK_EQ(samples.cols(), expected.cols()); CHECK_EQ(samples.cols(), expected.cols());
for (int i = 0; i < samples.cols(); ++i) { for (int i = 0; i < samples.cols(); ++i) {
// Take a column from samples and produce a packet with just that // Take a column from samples and produce a packet with just that
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_log.h"
#include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
@@ -54,7 +53,7 @@ class MergeCalculator : public Node {
static absl::Status UpdateContract(CalculatorContract* cc) { static absl::Status UpdateContract(CalculatorContract* cc) {
RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream"; RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
if (kIn(cc).Count() == 1) { if (kIn(cc).Count() == 1) {
ABSL_LOG(WARNING) LOG(WARNING)
<< "MergeCalculator expects multiple input streams to merge but is " << "MergeCalculator expects multiple input streams to merge but is "
"receiving only one. Make sure the calculator is configured " "receiving only one. Make sure the calculator is configured "
"correctly or consider removing this calculator to reduce " "correctly or consider removing this calculator to reduce "
@@ -73,8 +72,8 @@ class MergeCalculator : public Node {
} }
} }
ABSL_LOG(WARNING) << "Empty input packets at timestamp " LOG(WARNING) << "Empty input packets at timestamp "
<< cc->InputTimestamp().Value(); << cc->InputTimestamp().Value();
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -16,9 +16,6 @@
#include <memory> #include <memory>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
namespace { namespace {
// Reflect an integer against the lower and upper bound of an interval. // Reflect an integer against the lower and upper bound of an interval.
int64_t ReflectBetween(int64_t ts, int64_t ts_min, int64_t ts_max) { int64_t ReflectBetween(int64_t ts, int64_t ts_min, int64_t ts_max) {
@@ -180,7 +177,7 @@ PacketResamplerCalculator::GetSamplingStrategy(
const PacketResamplerCalculatorOptions& options) { const PacketResamplerCalculatorOptions& options) {
if (options.reproducible_sampling()) { if (options.reproducible_sampling()) {
if (!options.jitter_with_reflection()) { if (!options.jitter_with_reflection()) {
ABSL_LOG(WARNING) LOG(WARNING)
<< "reproducible_sampling enabled w/ jitter_with_reflection " << "reproducible_sampling enabled w/ jitter_with_reflection "
"disabled. " "disabled. "
<< "reproducible_sampling always uses jitter with reflection, " << "reproducible_sampling always uses jitter with reflection, "
@@ -203,15 +200,15 @@ PacketResamplerCalculator::GetSamplingStrategy(
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp( Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(
int64_t index) const { int64_t index) const {
ABSL_CHECK_EQ(jitter_, 0.0); CHECK_EQ(jitter_, 0.0);
ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset()); CHECK_NE(first_timestamp_, Timestamp::Unset());
return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_); return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_);
} }
int64_t PacketResamplerCalculator::TimestampToPeriodIndex( int64_t PacketResamplerCalculator::TimestampToPeriodIndex(
Timestamp timestamp) const { Timestamp timestamp) const {
ABSL_CHECK_EQ(jitter_, 0.0); CHECK_EQ(jitter_, 0.0);
ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset()); CHECK_NE(first_timestamp_, Timestamp::Unset());
return MathUtil::SafeRound<int64_t, double>( return MathUtil::SafeRound<int64_t, double>(
(timestamp - first_timestamp_).Seconds() * frame_rate_); (timestamp - first_timestamp_).Seconds() * frame_rate_);
} }
@@ -232,15 +229,13 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
if (resampler_options.output_header() != if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) { PacketResamplerCalculatorOptions::NONE) {
ABSL_LOG(WARNING) LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
<< "VideoHeader::frame_rate holds the target value and not " "the actual value.";
"the actual value.";
} }
if (calculator_->flush_last_packet_) { if (calculator_->flush_last_packet_) {
ABSL_LOG(WARNING) LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
<< "PacketResamplerCalculatorOptions.flush_last_packet is " "ignored, because we are adding jitter.";
"ignored, because we are adding jitter.";
} }
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>(); const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
@@ -259,7 +254,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
} }
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) { absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
if (!packet_reservoir_->IsEmpty()) { if (!packet_reservoir_->IsEmpty()) {
ABSL_LOG(INFO) << "Emitting pack from reservoir."; LOG(INFO) << "Emitting pack from reservoir.";
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample()); calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
} }
return absl::OkStatus(); return absl::OkStatus();
@@ -290,7 +285,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Process(
if (calculator_->frame_time_usec_ < if (calculator_->frame_time_usec_ <
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) { (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
ABSL_LOG_FIRST_N(WARNING, 2) LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling."; << "Adding jitter is not very useful when upsampling.";
} }
@@ -345,8 +340,8 @@ void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
next_output_timestamp_ = Timestamp(ReflectBetween( next_output_timestamp_ = Timestamp(ReflectBetween(
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(), next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
next_output_timestamp_max_.Value())); next_output_timestamp_max_.Value()));
ABSL_CHECK_GE(next_output_timestamp_, next_output_timestamp_min_); CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
ABSL_CHECK_LT(next_output_timestamp_, next_output_timestamp_max_); CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
} }
absl::Status ReproducibleJitterWithReflectionStrategy::Open( absl::Status ReproducibleJitterWithReflectionStrategy::Open(
@@ -357,15 +352,13 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Open(
if (resampler_options.output_header() != if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) { PacketResamplerCalculatorOptions::NONE) {
ABSL_LOG(WARNING) LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
<< "VideoHeader::frame_rate holds the target value and not " "the actual value.";
"the actual value.";
} }
if (calculator_->flush_last_packet_) { if (calculator_->flush_last_packet_) {
ABSL_LOG(WARNING) LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
<< "PacketResamplerCalculatorOptions.flush_last_packet is " "ignored, because we are adding jitter.";
"ignored, because we are adding jitter.";
} }
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>(); const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
@@ -418,7 +411,7 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Process(
// Note, if the stream is upsampling, this could lead to the same packet // Note, if the stream is upsampling, this could lead to the same packet
// being emitted twice. Upsampling and jitter doesn't make much sense // being emitted twice. Upsampling and jitter doesn't make much sense
// but does technically work. // but does technically work.
ABSL_LOG_FIRST_N(WARNING, 2) LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling."; << "Adding jitter is not very useful when upsampling.";
} }
@@ -506,15 +499,13 @@ absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
if (resampler_options.output_header() != if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) { PacketResamplerCalculatorOptions::NONE) {
ABSL_LOG(WARNING) LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
<< "VideoHeader::frame_rate holds the target value and not " "the actual value.";
"the actual value.";
} }
if (calculator_->flush_last_packet_) { if (calculator_->flush_last_packet_) {
ABSL_LOG(WARNING) LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
<< "PacketResamplerCalculatorOptions.flush_last_packet is " "ignored, because we are adding jitter.";
"ignored, because we are adding jitter.";
} }
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>(); const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
@@ -564,7 +555,7 @@ absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
if (calculator_->frame_time_usec_ < if (calculator_->frame_time_usec_ <
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) { (cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
ABSL_LOG_FIRST_N(WARNING, 2) LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling."; << "Adding jitter is not very useful when upsampling.";
} }
@@ -13,6 +13,7 @@
#include "mediapipe/framework/deps/random_base.h" #include "mediapipe/framework/deps/random_base.h"
#include "mediapipe/framework/formats/video_stream_header.h" #include "mediapipe/framework/formats/video_stream_header.h"
#include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/framework/port/status_macros.h"
@@ -17,7 +17,6 @@
#include <cmath> // for ceil #include <cmath> // for ceil
#include <memory> #include <memory>
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h" #include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
#include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -161,8 +160,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
thinner_type_ = options.thinner_type(); thinner_type_ = options.thinner_type();
// This check enables us to assume only two thinner types exist in Process() // This check enables us to assume only two thinner types exist in Process()
ABSL_CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC || CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
thinner_type_ == PacketThinnerCalculatorOptions::SYNC) thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
<< "Unsupported thinner type."; << "Unsupported thinner type.";
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) { if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
@@ -178,8 +177,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
} else { } else {
period_ = TimestampDiff(options.period()); period_ = TimestampDiff(options.period());
} }
ABSL_CHECK_LT(TimestampDiff(0), period_) CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive.";
<< "Specified period must be positive.";
if (options.has_start_time()) { if (options.has_start_time()) {
start_time_ = Timestamp(options.start_time()); start_time_ = Timestamp(options.start_time());
@@ -191,7 +189,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
end_time_ = end_time_ =
options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max(); options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
ABSL_CHECK_LT(start_time_, end_time_) CHECK_LT(start_time_, end_time_)
<< "Invalid PacketThinner: start_time must be earlier than end_time"; << "Invalid PacketThinner: start_time must be earlier than end_time";
sync_output_timestamps_ = options.sync_output_timestamps(); sync_output_timestamps_ = options.sync_output_timestamps();
@@ -234,7 +232,7 @@ absl::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
// Emit any saved packets before quitting. // Emit any saved packets before quitting.
if (!saved_packet_.IsEmpty()) { if (!saved_packet_.IsEmpty()) {
// Only sync thinner should have saved packets. // Only sync thinner should have saved packets.
ABSL_CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_); CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
if (sync_output_timestamps_) { if (sync_output_timestamps_) {
cc->Outputs().Index(0).AddPacket( cc->Outputs().Index(0).AddPacket(
saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp()))); saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
@@ -271,7 +269,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
const Timestamp saved_sync = NearestSyncTimestamp(saved); const Timestamp saved_sync = NearestSyncTimestamp(saved);
const Timestamp now = cc->InputTimestamp(); const Timestamp now = cc->InputTimestamp();
const Timestamp now_sync = NearestSyncTimestamp(now); const Timestamp now_sync = NearestSyncTimestamp(now);
ABSL_CHECK_LE(saved_sync, now_sync); CHECK_LE(saved_sync, now_sync);
if (saved_sync == now_sync) { if (saved_sync == now_sync) {
// Saved Packet is in same interval as current packet. // Saved Packet is in same interval as current packet.
// Replace saved packet with current if it is at least as // Replace saved packet with current if it is at least as
@@ -297,7 +295,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
} }
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const { Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
ABSL_CHECK_NE(start_time_, Timestamp::Unset()) CHECK_NE(start_time_, Timestamp::Unset())
<< "Method only valid for sync thinner calculator."; << "Method only valid for sync thinner calculator.";
// Computation is done using int64 arithmetic. No easy way to avoid // Computation is done using int64 arithmetic. No easy way to avoid
@@ -305,12 +303,12 @@ Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
const int64_t now64 = now.Value(); const int64_t now64 = now.Value();
const int64_t start64 = start_time_.Value(); const int64_t start64 = start_time_.Value();
const int64_t period64 = period_.Value(); const int64_t period64 = period_.Value();
ABSL_CHECK_LE(0, period64); CHECK_LE(0, period64);
// Round now64 to its closest interval (units of period64). // Round now64 to its closest interval (units of period64).
int64_t sync64 = int64_t sync64 =
(now64 - start64 + period64 / 2) / period64 * period64 + start64; (now64 - start64 + period64 / 2) / period64 * period64 + start64;
ABSL_CHECK_LE(abs(now64 - sync64), period64 / 2) CHECK_LE(abs(now64 - sync64), period64 / 2)
<< "start64: " << start64 << "; now64: " << now64 << "start64: " << start64 << "; now64: " << now64
<< "; sync64: " << sync64; << "; sync64: " << sync64;
@@ -16,7 +16,6 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h" #include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -71,7 +70,7 @@ class SimpleRunner : public CalculatorRunner {
} }
double GetFrameRate() const { double GetFrameRate() const {
ABSL_CHECK(!Outputs().Index(0).header.IsEmpty()); CHECK(!Outputs().Index(0).header.IsEmpty());
return Outputs().Index(0).header.Get<VideoHeader>().frame_rate; return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
} }
}; };
@@ -14,7 +14,6 @@
#include <deque> #include <deque>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h" #include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
#include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -102,7 +101,7 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp())); kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
packet_cache_.pop_front(); packet_cache_.pop_front();
} else if (emit_empty_packets_before_first_packet_) { } else if (emit_empty_packets_before_first_packet_) {
ABSL_LOG(FATAL) << "Not supported yet"; LOG(FATAL) << "Not supported yet";
} }
// Store current packet for later output. // Store current packet for later output.
packet_cache_.push_back(kIn(cc).packet()); packet_cache_.push_back(kIn(cc).packet());
+2 -44
View File
@@ -97,7 +97,6 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:source_location", "//mediapipe/framework/port:source_location",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -126,7 +125,6 @@ cc_library(
"//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -153,11 +151,11 @@ cc_library(
"//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector", "//mediapipe/framework/port:vector",
"@com_google_absl//absl/log:absl_log",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [ "//conditions:default": [
@@ -204,7 +202,6 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector", "//mediapipe/framework/port:vector",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
@@ -264,12 +261,9 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/gpu:scale_mode_cc_proto", "//mediapipe/gpu:scale_mode_cc_proto",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [ "//conditions:default": [
"//mediapipe/gpu:gl_base_hdr",
"//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_quad_renderer", "//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gl_simple_shaders",
@@ -279,36 +273,6 @@ cc_library(
alwayslink = 1, alwayslink = 1,
) )
cc_test(
name = "image_transformation_calculator_test",
srcs = ["image_transformation_calculator_test.cc"],
data = ["//mediapipe/calculators/image/testdata:test_images"],
tags = [
"desktop_only_test",
],
deps = [
":image_transformation_calculator",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:gtest",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
"//third_party:opencv",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest_main",
],
)
cc_library( cc_library(
name = "image_cropping_calculator", name = "image_cropping_calculator",
srcs = ["image_cropping_calculator.cc"], srcs = ["image_cropping_calculator.cc"],
@@ -336,7 +300,6 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [ "//conditions:default": [
@@ -433,7 +396,6 @@ cc_library(
"//mediapipe/framework/port:logging", "//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],
) )
@@ -458,8 +420,6 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/util:image_frame_util", "//mediapipe/util:image_frame_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@libyuv", "@libyuv",
], ],
@@ -665,9 +625,9 @@ cc_library(
"//mediapipe/framework/formats:image", "//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector", "//mediapipe/framework/port:vector",
"@com_google_absl//absl/log:absl_log",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [ "//conditions:default": [
@@ -705,7 +665,6 @@ cc_test(
"//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_log",
], ],
) )
@@ -728,7 +687,6 @@ cc_library(
"//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/gpu:shader_util", "//mediapipe/gpu:shader_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor", "@com_google_absl//absl/status:statusor",
@@ -20,7 +20,6 @@
#include "Eigen/Core" #include "Eigen/Core"
#include "Eigen/Geometry" #include "Eigen/Geometry"
#include "Eigen/LU" #include "Eigen/LU"
#include "absl/log/absl_log.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/status/status.h" #include "absl/status/status.h"
#include "absl/status/statusor.h" #include "absl/status/statusor.h"
@@ -54,10 +53,6 @@ bool IsMatrixVerticalFlipNeeded(GpuOrigin::Mode gpu_origin) {
#endif // __APPLE__ #endif // __APPLE__
case GpuOrigin::TOP_LEFT: case GpuOrigin::TOP_LEFT:
return false; return false;
default:
ABSL_LOG(ERROR) << "Incorrect GpuOrigin: "
<< static_cast<int>(gpu_origin);
return true;
} }
} }
@@ -389,8 +384,6 @@ class GlTextureWarpAffineRunner
glActiveTexture(GL_TEXTURE0); glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, 0); glBindTexture(GL_TEXTURE_2D, 0);
glFlush();
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -15,7 +15,6 @@
#include <memory> #include <memory>
#include <string> #include <string>
#include "absl/log/absl_check.h"
#include "absl/strings/str_replace.h" #include "absl/strings/str_replace.h"
#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h" #include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -113,7 +112,7 @@ class BilateralFilterCalculator : public CalculatorBase {
REGISTER_CALCULATOR(BilateralFilterCalculator); REGISTER_CALCULATOR(BilateralFilterCalculator);
absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) { absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK_GE(cc->Inputs().NumEntries(), 1); CHECK_GE(cc->Inputs().NumEntries(), 1);
if (cc->Inputs().HasTag(kInputFrameTag) && if (cc->Inputs().HasTag(kInputFrameTag) &&
cc->Inputs().HasTag(kInputFrameTagGpu)) { cc->Inputs().HasTag(kInputFrameTagGpu)) {
@@ -184,8 +183,8 @@ absl::Status BilateralFilterCalculator::Open(CalculatorContext* cc) {
sigma_color_ = options_.sigma_color(); sigma_color_ = options_.sigma_color();
sigma_space_ = options_.sigma_space(); sigma_space_ = options_.sigma_space();
ABSL_CHECK_GE(sigma_color_, 0.0); CHECK_GE(sigma_color_, 0.0);
ABSL_CHECK_GE(sigma_space_, 0.0); CHECK_GE(sigma_space_, 0.0);
if (!use_gpu_) sigma_color_ *= 255.0; if (!use_gpu_) sigma_color_ *= 255.0;
if (use_gpu_) { if (use_gpu_) {
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_check.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/image_frame_opencv.h"
@@ -26,8 +25,8 @@
namespace mediapipe { namespace mediapipe {
namespace { namespace {
void SetColorChannel(int channel, uint8 value, cv::Mat* mat) { void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
ABSL_CHECK(mat->depth() == CV_8U); CHECK(mat->depth() == CV_8U);
ABSL_CHECK(channel < mat->channels()); CHECK(channel < mat->channels());
const int step = mat->channels(); const int step = mat->channels();
for (int r = 0; r < mat->rows; ++r) { for (int r = 0; r < mat->rows; ++r) {
uint8* row_ptr = mat->ptr<uint8>(r); uint8* row_ptr = mat->ptr<uint8>(r);
@@ -16,7 +16,6 @@
#include <cmath> #include <cmath>
#include "absl/log/absl_log.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/rect.pb.h" #include "mediapipe/framework/formats/rect.pb.h"
@@ -203,9 +202,8 @@ absl::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
switch (options.border_mode()) { switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO: case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
ABSL_LOG(WARNING) LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
<< "BORDER_ZERO mode is not supported by GPU " << "implementation and will fall back into BORDER_REPLICATE";
<< "implementation and will fall back into BORDER_REPLICATE";
break; break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE: case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
break; break;
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/status/status.h"
#include "mediapipe/calculators/image/image_transformation_calculator.pb.h" #include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
#include "mediapipe/calculators/image/rotation_mode.pb.h" #include "mediapipe/calculators/image/rotation_mode.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -28,7 +27,6 @@
#include "mediapipe/gpu/scale_mode.pb.h" #include "mediapipe/gpu/scale_mode.pb.h"
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_quad_renderer.h" #include "mediapipe/gpu/gl_quad_renderer.h"
#include "mediapipe/gpu/gl_simple_shaders.h" #include "mediapipe/gpu/gl_simple_shaders.h"
@@ -62,42 +60,42 @@ constexpr char kVideoPrestreamTag[] = "VIDEO_PRESTREAM";
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) { int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
switch (rotation) { switch (rotation) {
case mediapipe::RotationMode::UNKNOWN: case mediapipe::RotationMode_Mode_UNKNOWN:
case mediapipe::RotationMode::ROTATION_0: case mediapipe::RotationMode_Mode_ROTATION_0:
return 0; return 0;
case mediapipe::RotationMode::ROTATION_90: case mediapipe::RotationMode_Mode_ROTATION_90:
return 90; return 90;
case mediapipe::RotationMode::ROTATION_180: case mediapipe::RotationMode_Mode_ROTATION_180:
return 180; return 180;
case mediapipe::RotationMode::ROTATION_270: case mediapipe::RotationMode_Mode_ROTATION_270:
return 270; return 270;
} }
} }
mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) { mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) {
switch (degrees) { switch (degrees) {
case 0: case 0:
return mediapipe::RotationMode::ROTATION_0; return mediapipe::RotationMode_Mode_ROTATION_0;
case 90: case 90:
return mediapipe::RotationMode::ROTATION_90; return mediapipe::RotationMode_Mode_ROTATION_90;
case 180: case 180:
return mediapipe::RotationMode::ROTATION_180; return mediapipe::RotationMode_Mode_ROTATION_180;
case 270: case 270:
return mediapipe::RotationMode::ROTATION_270; return mediapipe::RotationMode_Mode_ROTATION_270;
default: default:
return mediapipe::RotationMode::UNKNOWN; return mediapipe::RotationMode_Mode_UNKNOWN;
} }
} }
mediapipe::ScaleMode_Mode ParseScaleMode( mediapipe::ScaleMode_Mode ParseScaleMode(
mediapipe::ScaleMode_Mode scale_mode, mediapipe::ScaleMode_Mode scale_mode,
mediapipe::ScaleMode_Mode default_mode) { mediapipe::ScaleMode_Mode default_mode) {
switch (scale_mode) { switch (scale_mode) {
case mediapipe::ScaleMode::DEFAULT: case mediapipe::ScaleMode_Mode_DEFAULT:
return default_mode; return default_mode;
case mediapipe::ScaleMode::STRETCH: case mediapipe::ScaleMode_Mode_STRETCH:
return scale_mode; return scale_mode;
case mediapipe::ScaleMode::FIT: case mediapipe::ScaleMode_Mode_FIT:
return scale_mode; return scale_mode;
case mediapipe::ScaleMode::FILL_AND_CROP: case mediapipe::ScaleMode_Mode_FILL_AND_CROP:
return scale_mode; return scale_mode;
default: default:
return default_mode; return default_mode;
@@ -210,8 +208,6 @@ class ImageTransformationCalculator : public CalculatorBase {
bool use_gpu_ = false; bool use_gpu_ = false;
cv::Scalar padding_color_; cv::Scalar padding_color_;
ImageTransformationCalculatorOptions::InterpolationMode interpolation_mode_;
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
GlCalculatorHelper gpu_helper_; GlCalculatorHelper gpu_helper_;
std::unique_ptr<QuadRenderer> rgb_renderer_; std::unique_ptr<QuadRenderer> rgb_renderer_;
@@ -347,11 +343,6 @@ absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
options_.padding_color().green(), options_.padding_color().green(),
options_.padding_color().blue()); options_.padding_color().blue());
interpolation_mode_ = options_.interpolation_mode();
if (options_.interpolation_mode() ==
ImageTransformationCalculatorOptions::DEFAULT) {
interpolation_mode_ = ImageTransformationCalculatorOptions::LINEAR;
}
if (use_gpu_) { if (use_gpu_) {
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
// Let the helper access the GL context information. // Let the helper access the GL context information.
@@ -466,48 +457,26 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
ComputeOutputDimensions(input_width, input_height, &output_width, ComputeOutputDimensions(input_width, input_height, &output_width,
&output_height); &output_height);
int opencv_interpolation_mode = cv::INTER_LINEAR;
if (output_width_ > 0 && output_height_ > 0) { if (output_width_ > 0 && output_height_ > 0) {
cv::Mat scaled_mat; cv::Mat scaled_mat;
if (scale_mode_ == mediapipe::ScaleMode::STRETCH) { if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) { int scale_flag =
// Use INTER_AREA for downscaling if interpolation mode is set to input_mat.cols > output_width_ && input_mat.rows > output_height_
// LINEAR. ? cv::INTER_AREA
if (input_mat.cols > output_width_ && input_mat.rows > output_height_) { : cv::INTER_LINEAR;
opencv_interpolation_mode = cv::INTER_AREA;
} else {
opencv_interpolation_mode = cv::INTER_LINEAR;
}
} else {
opencv_interpolation_mode = cv::INTER_NEAREST;
}
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_), cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
0, 0, opencv_interpolation_mode); 0, 0, scale_flag);
} else { } else {
const float scale = const float scale =
std::min(static_cast<float>(output_width_) / input_width, std::min(static_cast<float>(output_width_) / input_width,
static_cast<float>(output_height_) / input_height); static_cast<float>(output_height_) / input_height);
const int target_width = std::round(input_width * scale); const int target_width = std::round(input_width * scale);
const int target_height = std::round(input_height * scale); const int target_height = std::round(input_height * scale);
int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR;
if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) { if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
// Use INTER_AREA for downscaling if interpolation mode is set to
// LINEAR.
if (scale < 1.0f) {
opencv_interpolation_mode = cv::INTER_AREA;
} else {
opencv_interpolation_mode = cv::INTER_LINEAR;
}
} else {
opencv_interpolation_mode = cv::INTER_NEAREST;
}
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
cv::Mat intermediate_mat; cv::Mat intermediate_mat;
cv::resize(input_mat, intermediate_mat, cv::resize(input_mat, intermediate_mat,
cv::Size(target_width, target_height), 0, 0, cv::Size(target_width, target_height), 0, 0, scale_flag);
opencv_interpolation_mode);
const int top = (output_height_ - target_height) / 2; const int top = (output_height_ - target_height) / 2;
const int bottom = output_height_ - target_height - top; const int bottom = output_height_ - target_height - top;
const int left = (output_width_ - target_width) / 2; const int left = (output_width_ - target_width) / 2;
@@ -519,7 +488,7 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
padding_color_); padding_color_);
} else { } else {
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height), cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
0, 0, opencv_interpolation_mode); 0, 0, scale_flag);
output_width = target_width; output_width = target_width;
output_height = target_height; output_height = target_height;
} }
@@ -545,17 +514,17 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size); cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size);
} else { } else {
switch (rotation_) { switch (rotation_) {
case mediapipe::RotationMode::UNKNOWN: case mediapipe::RotationMode_Mode_UNKNOWN:
case mediapipe::RotationMode::ROTATION_0: case mediapipe::RotationMode_Mode_ROTATION_0:
rotated_mat = input_mat; rotated_mat = input_mat;
break; break;
case mediapipe::RotationMode::ROTATION_90: case mediapipe::RotationMode_Mode_ROTATION_90:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE); cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE);
break; break;
case mediapipe::RotationMode::ROTATION_180: case mediapipe::RotationMode_Mode_ROTATION_180:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_180); cv::rotate(input_mat, rotated_mat, cv::ROTATE_180);
break; break;
case mediapipe::RotationMode::ROTATION_270: case mediapipe::RotationMode_Mode_ROTATION_270:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE); cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE);
break; break;
} }
@@ -592,7 +561,7 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
ComputeOutputDimensions(input_width, input_height, &output_width, ComputeOutputDimensions(input_width, input_height, &output_width,
&output_height); &output_height);
if (scale_mode_ == mediapipe::ScaleMode::FILL_AND_CROP) { if (scale_mode_ == mediapipe::ScaleMode_Mode_FILL_AND_CROP) {
const float scale = const float scale =
std::min(static_cast<float>(output_width_) / input_width, std::min(static_cast<float>(output_width_) / input_width,
static_cast<float>(output_height_) / input_height); static_cast<float>(output_height_) / input_height);
@@ -659,12 +628,6 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
glActiveTexture(GL_TEXTURE1); glActiveTexture(GL_TEXTURE1);
glBindTexture(src1.target(), src1.name()); glBindTexture(src1.target(), src1.name());
if (interpolation_mode_ == ImageTransformationCalculatorOptions::NEAREST) {
// TODO: revert texture params.
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
}
MP_RETURN_IF_ERROR(renderer->GlRender( MP_RETURN_IF_ERROR(renderer->GlRender(
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode, src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
rotation, flip_horizontally_, flip_vertically_, rotation, flip_horizontally_, flip_vertically_,
@@ -689,8 +652,8 @@ void ImageTransformationCalculator::ComputeOutputDimensions(
if (output_width_ > 0 && output_height_ > 0) { if (output_width_ > 0 && output_height_ > 0) {
*output_width = output_width_; *output_width = output_width_;
*output_height = output_height_; *output_height = output_height_;
} else if (rotation_ == mediapipe::RotationMode::ROTATION_90 || } else if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
rotation_ == mediapipe::RotationMode::ROTATION_270) { rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
*output_width = input_height; *output_width = input_height;
*output_height = input_width; *output_height = input_width;
} else { } else {
@@ -703,9 +666,9 @@ void ImageTransformationCalculator::ComputeOutputLetterboxPadding(
int input_width, int input_height, int output_width, int output_height, int input_width, int input_height, int output_width, int output_height,
std::array<float, 4>* padding) { std::array<float, 4>* padding) {
padding->fill(0.f); padding->fill(0.f);
if (scale_mode_ == mediapipe::ScaleMode::FIT) { if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
if (rotation_ == mediapipe::RotationMode::ROTATION_90 || if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
rotation_ == mediapipe::RotationMode::ROTATION_270) { rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
std::swap(input_width, input_height); std::swap(input_width, input_height);
} }
const float input_aspect_ratio = const float input_aspect_ratio =
@@ -54,15 +54,4 @@ message ImageTransformationCalculatorOptions {
// The color for the padding. This option is only used when the scale mode is // The color for the padding. This option is only used when the scale mode is
// FIT. Default is black. This is for CPU only. // FIT. Default is black. This is for CPU only.
optional Color padding_color = 8; optional Color padding_color = 8;
// Interpolation method to use. Note that on CPU when LINEAR is specified,
// INTER_LINEAR is used for upscaling and INTER_AREA is used for downscaling.
enum InterpolationMode {
DEFAULT = 0;
LINEAR = 1;
NEAREST = 2;
}
// Mode DEFAULT will use LINEAR interpolation.
optional InterpolationMode interpolation_mode = 9;
} }
@@ -1,315 +0,0 @@
#include <string>
#include <utility>
#include <vector>
#include "absl/container/flat_hash_set.h"
#include "absl/flags/flag.h"
#include "absl/strings/substitute.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "testing/base/public/gmock.h"
#include "testing/base/public/googletest.h"
#include "third_party/OpenCV/core.hpp" // IWYU pragma: keep
#include "third_party/OpenCV/core/mat.hpp"
namespace mediapipe {
namespace {
absl::flat_hash_set<int> computeUniqueValues(const cv::Mat& mat) {
// Compute the unique values in cv::Mat
absl::flat_hash_set<int> unique_values;
for (int i = 0; i < mat.rows; i++) {
for (int j = 0; j < mat.cols; j++) {
unique_values.insert(mat.at<unsigned char>(i, j));
}
}
return unique_values;
}
TEST(ImageTransformationCalculatorTest, NearestNeighborResizing) {
cv::Mat input_mat;
cv::cvtColor(cv::imread(file::JoinPath("./",
"/mediapipe/calculators/"
"image/testdata/binary_mask.png")),
input_mat, cv::COLOR_BGR2GRAY);
Packet input_image_packet = MakePacket<ImageFrame>(
ImageFormat::GRAY8, input_mat.size().width, input_mat.size().height);
input_mat.copyTo(formats::MatView(&(input_image_packet.Get<ImageFrame>())));
std::vector<std::pair<int, int>> output_dims{
{256, 333}, {512, 512}, {1024, 1024}};
for (auto& output_dim : output_dims) {
Packet input_output_dim_packet =
MakePacket<std::pair<int, int>>(output_dim);
std::vector<std::string> scale_modes{"FIT", "STRETCH"};
for (const auto& scale_mode : scale_modes) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
absl::Substitute(R"(
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:input_image"
input_stream: "OUTPUT_DIMENSIONS:image_size"
output_stream: "IMAGE:output_image"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: $0
interpolation_mode: NEAREST
}
})",
scale_mode));
CalculatorRunner runner(node_config);
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_image_packet.At(Timestamp(0)));
runner.MutableInputs()
->Tag("OUTPUT_DIMENSIONS")
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
ASSERT_EQ(outputs.NumEntries(), 1);
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
ASSERT_EQ(packets.size(), 1);
const auto& result = packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, result.Width());
ASSERT_EQ(output_dim.second, result.Height());
auto unique_input_values = computeUniqueValues(input_mat);
auto unique_output_values =
computeUniqueValues(formats::MatView(&result));
EXPECT_THAT(unique_input_values,
::testing::ContainerEq(unique_output_values));
}
}
}
TEST(ImageTransformationCalculatorTest,
NearestNeighborResizingWorksForFloatInput) {
cv::Mat input_mat;
cv::cvtColor(cv::imread(file::JoinPath("./",
"/mediapipe/calculators/"
"image/testdata/binary_mask.png")),
input_mat, cv::COLOR_BGR2GRAY);
Packet input_image_packet = MakePacket<ImageFrame>(
ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
cv::Mat packet_mat_view =
formats::MatView(&(input_image_packet.Get<ImageFrame>()));
input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
std::vector<std::pair<int, int>> output_dims{
{256, 333}, {512, 512}, {1024, 1024}};
for (auto& output_dim : output_dims) {
Packet input_output_dim_packet =
MakePacket<std::pair<int, int>>(output_dim);
std::vector<std::string> scale_modes{"FIT", "STRETCH"};
for (const auto& scale_mode : scale_modes) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
absl::Substitute(R"(
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:input_image"
input_stream: "OUTPUT_DIMENSIONS:image_size"
output_stream: "IMAGE:output_image"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: $0
interpolation_mode: NEAREST
}
})",
scale_mode));
CalculatorRunner runner(node_config);
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_image_packet.At(Timestamp(0)));
runner.MutableInputs()
->Tag("OUTPUT_DIMENSIONS")
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
ASSERT_EQ(outputs.NumEntries(), 1);
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
ASSERT_EQ(packets.size(), 1);
const auto& result = packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, result.Width());
ASSERT_EQ(output_dim.second, result.Height());
auto unique_input_values = computeUniqueValues(packet_mat_view);
auto unique_output_values =
computeUniqueValues(formats::MatView(&result));
EXPECT_THAT(unique_input_values,
::testing::ContainerEq(unique_output_values));
}
}
}
TEST(ImageTransformationCalculatorTest, NearestNeighborResizingGpu) {
cv::Mat input_mat;
cv::cvtColor(cv::imread(file::JoinPath("./",
"/mediapipe/calculators/"
"image/testdata/binary_mask.png")),
input_mat, cv::COLOR_BGR2RGBA);
std::vector<std::pair<int, int>> output_dims{
{256, 333}, {512, 512}, {1024, 1024}};
for (auto& output_dim : output_dims) {
std::vector<std::string> scale_modes{"FIT"}; //, "STRETCH"};
for (const auto& scale_mode : scale_modes) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(R"(
input_stream: "input_image"
input_stream: "image_size"
output_stream: "output_image"
node {
calculator: "ImageFrameToGpuBufferCalculator"
input_stream: "input_image"
output_stream: "input_image_gpu"
}
node {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:input_image_gpu"
input_stream: "OUTPUT_DIMENSIONS:image_size"
output_stream: "IMAGE_GPU:output_image_gpu"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: $0
interpolation_mode: NEAREST
}
}
}
node {
calculator: "GpuBufferToImageFrameCalculator"
input_stream: "output_image_gpu"
output_stream: "output_image"
})",
scale_mode));
ImageFrame input_image(ImageFormat::SRGBA, input_mat.size().width,
input_mat.size().height);
input_mat.copyTo(formats::MatView(&input_image));
std::vector<Packet> output_image_packets;
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
CalculatorGraph graph(graph_config);
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image",
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"image_size",
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, output_image.Width());
ASSERT_EQ(output_dim.second, output_image.Height());
auto unique_input_values = computeUniqueValues(input_mat);
auto unique_output_values =
computeUniqueValues(formats::MatView(&output_image));
EXPECT_THAT(unique_input_values,
::testing::ContainerEq(unique_output_values));
}
}
}
TEST(ImageTransformationCalculatorTest,
NearestNeighborResizingWorksForFloatTexture) {
cv::Mat input_mat;
cv::cvtColor(cv::imread(file::JoinPath("./",
"/mediapipe/calculators/"
"image/testdata/binary_mask.png")),
input_mat, cv::COLOR_BGR2GRAY);
Packet input_image_packet = MakePacket<ImageFrame>(
ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
cv::Mat packet_mat_view =
formats::MatView(&(input_image_packet.Get<ImageFrame>()));
input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
std::vector<std::pair<int, int>> output_dims{
{256, 333}, {512, 512}, {1024, 1024}};
for (auto& output_dim : output_dims) {
std::vector<std::string> scale_modes{"FIT"}; //, "STRETCH"};
for (const auto& scale_mode : scale_modes) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(R"(
input_stream: "input_image"
input_stream: "image_size"
output_stream: "output_image"
node {
calculator: "ImageFrameToGpuBufferCalculator"
input_stream: "input_image"
output_stream: "input_image_gpu"
}
node {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:input_image_gpu"
input_stream: "OUTPUT_DIMENSIONS:image_size"
output_stream: "IMAGE_GPU:output_image_gpu"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: $0
interpolation_mode: NEAREST
}
}
}
node {
calculator: "GpuBufferToImageFrameCalculator"
input_stream: "output_image_gpu"
output_stream: "output_image"
})",
scale_mode));
std::vector<Packet> output_image_packets;
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
CalculatorGraph graph(graph_config);
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", input_image_packet.At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"image_size",
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, output_image.Width());
ASSERT_EQ(output_dim.second, output_image.Height());
auto unique_input_values = computeUniqueValues(packet_mat_view);
auto unique_output_values =
computeUniqueValues(formats::MatView(&output_image));
EXPECT_THAT(unique_input_values,
::testing::ContainerEq(unique_output_values));
}
}
}
} // namespace
} // namespace mediapipe
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" #include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/image_frame_opencv.h"
@@ -62,7 +61,7 @@ absl::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) {
absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) { absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) {
const ImageFrame& image_frame = cc->Inputs().Index(0).Get<ImageFrame>(); const ImageFrame& image_frame = cc->Inputs().Index(0).Get<ImageFrame>();
ABSL_CHECK_EQ(1, image_frame.ByteDepth()); CHECK_EQ(1, image_frame.ByteDepth());
std::unique_ptr<OpenCvImageEncoderCalculatorResults> encoded_result = std::unique_ptr<OpenCvImageEncoderCalculatorResults> encoded_result =
absl::make_unique<OpenCvImageEncoderCalculatorResults>(); absl::make_unique<OpenCvImageEncoderCalculatorResults>();
@@ -18,8 +18,6 @@
#include <memory> #include <memory>
#include <string> #include <string>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/substitute.h" #include "absl/strings/substitute.h"
#include "libyuv/scale.h" #include "libyuv/scale.h"
@@ -295,7 +293,7 @@ absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) {
header->width = output_width_; header->width = output_width_;
header->height = output_height_; header->height = output_height_;
header->format = output_format_; header->format = output_format_;
ABSL_LOG(INFO) << "OUTPUTTING HEADER on stream"; LOG(INFO) << "OUTPUTTING HEADER on stream";
cc->Outputs() cc->Outputs()
.Tag("VIDEO_HEADER") .Tag("VIDEO_HEADER")
.Add(header.release(), Timestamp::PreStream()); .Add(header.release(), Timestamp::PreStream());
@@ -395,11 +393,10 @@ absl::Status ScaleImageCalculator::Open(CalculatorContext* cc) {
.SetHeader(Adopt(output_header.release())); .SetHeader(Adopt(output_header.release()));
has_header_ = true; has_header_ = true;
} else { } else {
ABSL_LOG(WARNING) LOG(WARNING) << "Stream had a VideoHeader which didn't have sufficient "
<< "Stream had a VideoHeader which didn't have sufficient " "information. "
"information. " "Dropping VideoHeader and trying to deduce needed "
"Dropping VideoHeader and trying to deduce needed " "information.";
"information.";
input_width_ = 0; input_width_ = 0;
input_height_ = 0; input_height_ = 0;
if (!options_.has_input_format()) { if (!options_.has_input_format()) {
@@ -510,7 +507,7 @@ absl::Status ScaleImageCalculator::ValidateImageFrame(
absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc, absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc,
const YUVImage& yuv_image) { const YUVImage& yuv_image) {
ABSL_CHECK_EQ(input_format_, ImageFormat::YCBCR420P); CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
if (!has_header_) { if (!has_header_) {
if (input_width_ != yuv_image.width() || if (input_width_ != yuv_image.width() ||
input_height_ != yuv_image.height()) { input_height_ != yuv_image.height()) {
@@ -18,7 +18,6 @@
#include <string> #include <string>
#include "absl/log/absl_check.h"
#include "absl/strings/str_split.h" #include "absl/strings/str_split.h"
#include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
@@ -41,10 +40,10 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
const std::string& max_aspect_ratio, // const std::string& max_aspect_ratio, //
int* crop_width, int* crop_height, // int* crop_width, int* crop_height, //
int* col_start, int* row_start) { int* col_start, int* row_start) {
ABSL_CHECK(crop_width); CHECK(crop_width);
ABSL_CHECK(crop_height); CHECK(crop_height);
ABSL_CHECK(col_start); CHECK(col_start);
ABSL_CHECK(row_start); CHECK(row_start);
double min_aspect_ratio_q = 0.0; double min_aspect_ratio_q = 0.0;
double max_aspect_ratio_q = 0.0; double max_aspect_ratio_q = 0.0;
@@ -84,8 +83,8 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
} }
} }
ABSL_CHECK_LE(*crop_width, input_width); CHECK_LE(*crop_width, input_width);
ABSL_CHECK_LE(*crop_height, input_height); CHECK_LE(*crop_height, input_height);
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -97,8 +96,8 @@ absl::Status FindOutputDimensions(int input_width, //
bool preserve_aspect_ratio, // bool preserve_aspect_ratio, //
int scale_to_multiple_of, // int scale_to_multiple_of, //
int* output_width, int* output_height) { int* output_width, int* output_height) {
ABSL_CHECK(output_width); CHECK(output_width);
ABSL_CHECK(output_height); CHECK(output_height);
if (target_max_area > 0 && input_width * input_height > target_max_area) { if (target_max_area > 0 && input_width * input_height > target_max_area) {
preserve_aspect_ratio = true; preserve_aspect_ratio = true;
@@ -15,13 +15,13 @@
#include <algorithm> #include <algorithm>
#include <memory> #include <memory>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h" #include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/vector.h" #include "mediapipe/framework/port/vector.h"
@@ -110,7 +110,7 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator);
absl::Status SegmentationSmoothingCalculator::GetContract( absl::Status SegmentationSmoothingCalculator::GetContract(
CalculatorContract* cc) { CalculatorContract* cc) {
RET_CHECK_GE(cc->Inputs().NumEntries(), 1); CHECK_GE(cc->Inputs().NumEntries(), 1);
cc->Inputs().Tag(kCurrentMaskTag).Set<Image>(); cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
cc->Inputs().Tag(kPreviousMaskTag).Set<Image>(); cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();
@@ -273,7 +273,7 @@ absl::Status SegmentationSmoothingCalculator::RenderGpu(CalculatorContext* cc) {
const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>(); const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>();
if (previous_frame.format() != current_frame.format()) { if (previous_frame.format() != current_frame.format()) {
ABSL_LOG(ERROR) << "Warning: mixing input format types. "; LOG(ERROR) << "Warning: mixing input format types. ";
} }
auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame); auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame);
@@ -14,7 +14,6 @@
#include <memory> #include <memory>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h" #include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"
@@ -170,7 +169,7 @@ void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) {
} }
} }
} else { } else {
ABSL_LOG(ERROR) << "invalid ratio"; LOG(ERROR) << "invalid ratio";
} }
} }
@@ -14,13 +14,13 @@
#include <memory> #include <memory>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/set_alpha_calculator.pb.h" #include "mediapipe/calculators/image/set_alpha_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
@@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase {
REGISTER_CALCULATOR(SetAlphaCalculator); REGISTER_CALCULATOR(SetAlphaCalculator);
absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) { absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK_GE(cc->Inputs().NumEntries(), 1); CHECK_GE(cc->Inputs().NumEntries(), 1);
bool use_gpu = false; bool use_gpu = false;
@@ -268,7 +268,7 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>(); const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
const cv::Mat input_mat = formats::MatView(&input_frame); const cv::Mat input_mat = formats::MatView(&input_frame);
if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) { if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) {
ABSL_LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported"; LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
} }
// Setup destination image // Setup destination image
@@ -328,7 +328,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>(); cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 || if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 ||
input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) { input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) {
ABSL_LOG(ERROR) << "Only RGB or RGBA input image supported"; LOG(ERROR) << "Only RGB or RGBA input image supported";
} }
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame); auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
-1
View File
@@ -18,7 +18,6 @@ licenses(["notice"])
filegroup( filegroup(
name = "test_images", name = "test_images",
srcs = [ srcs = [
"binary_mask.png",
"dino.jpg", "dino.jpg",
"dino_quality_50.jpg", "dino_quality_50.jpg",
"dino_quality_80.jpg", "dino_quality_80.jpg",
Binary file not shown.

Before

Width:  |  Height:  |  Size: 771 B

-2
View File
@@ -31,14 +31,12 @@ mediapipe_proto_library(
cc_library( cc_library(
name = "callback_packet_calculator", name = "callback_packet_calculator",
srcs = ["callback_packet_calculator.cc"], srcs = ["callback_packet_calculator.cc"],
hdrs = ["callback_packet_calculator.h"],
visibility = ["//mediapipe/framework:__subpackages__"], visibility = ["//mediapipe/framework:__subpackages__"],
deps = [ deps = [
":callback_packet_calculator_cc_proto", ":callback_packet_calculator_cc_proto",
"//mediapipe/framework:calculator_base", "//mediapipe/framework:calculator_base",
"//mediapipe/framework:calculator_registry", "//mediapipe/framework:calculator_registry",
"//mediapipe/framework:output_side_packet", "//mediapipe/framework:output_side_packet",
"@com_google_absl//absl/status",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -11,12 +11,10 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "mediapipe/calculators/internal/callback_packet_calculator.h"
#include <functional> #include <functional>
#include <string> #include <string>
#include "absl/status/status.h"
#include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT #include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT
#include "mediapipe/framework/calculator_base.h" #include "mediapipe/framework/calculator_base.h"
#include "mediapipe/framework/calculator_registry.h" #include "mediapipe/framework/calculator_registry.h"
@@ -41,55 +39,64 @@ void DumpPostStreamPacket(Packet* post_stream_packet, const Packet& packet) {
*post_stream_packet = packet; *post_stream_packet = packet;
} }
} }
} // namespace } // namespace
absl::Status CallbackPacketCalculator::GetContract(CalculatorContract* cc) { // Creates a callback which takes a packet and stores it either in a
const auto& options = cc->Options<CallbackPacketCalculatorOptions>(); // vector of packets or stores only the packet at PostStream timestamp.
switch (options.type()) { // The kind of callback is controlled by an option. The callback is
case CallbackPacketCalculatorOptions::VECTOR_PACKET: // a std::function and is directly usable by CallbackCalculator.
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET: // Since the options for the packet generator include a serialized pointer
cc->OutputSidePackets() // value, the resulting callback is only valid on the original machine
.Index(0) // while that pointer is still alive.
.Set<std::function<void(const Packet&)>>(); class CallbackPacketCalculator : public CalculatorBase {
break; public:
default: static absl::Status GetContract(CalculatorContract* cc) {
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
<< "Invalid type of callback to produce."; switch (options.type()) {
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
cc->OutputSidePackets()
.Index(0)
.Set<std::function<void(const Packet&)>>();
break;
default:
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Invalid type of callback to produce.";
}
return absl::OkStatus();
} }
return absl::OkStatus();
}
absl::Status CallbackPacketCalculator::Open(CalculatorContext* cc) { absl::Status Open(CalculatorContext* cc) override {
const auto& options = cc->Options<CallbackPacketCalculatorOptions>(); const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
void* ptr; void* ptr;
if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) { if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Stored pointer value in options is invalid.";
}
switch (options.type()) {
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
cc->OutputSidePackets().Index(0).Set(
MakePacket<std::function<void(const Packet&)>>(std::bind(
&DumpToVector, reinterpret_cast<std::vector<Packet>*>(ptr),
std::placeholders::_1)));
break;
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
cc->OutputSidePackets().Index(0).Set(
MakePacket<std::function<void(const Packet&)>>(
std::bind(&DumpPostStreamPacket, reinterpret_cast<Packet*>(ptr),
std::placeholders::_1)));
break;
default:
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Invalid type to dump into."; << "Stored pointer value in options is invalid.";
}
switch (options.type()) {
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
cc->OutputSidePackets().Index(0).Set(
MakePacket<std::function<void(const Packet&)>>(std::bind(
&DumpToVector, reinterpret_cast<std::vector<Packet>*>(ptr),
std::placeholders::_1)));
break;
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
cc->OutputSidePackets().Index(0).Set(
MakePacket<std::function<void(const Packet&)>>(
std::bind(&DumpPostStreamPacket, reinterpret_cast<Packet*>(ptr),
std::placeholders::_1)));
break;
default:
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Invalid type to dump into.";
}
return absl::OkStatus();
} }
return absl::OkStatus();
}
absl::Status CallbackPacketCalculator::Process(CalculatorContext* cc) { absl::Status Process(CalculatorContext* cc) override {
return absl::OkStatus(); return absl::OkStatus();
} }
};
REGISTER_CALCULATOR(CallbackPacketCalculator); REGISTER_CALCULATOR(CallbackPacketCalculator);
@@ -1,39 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
#include "absl/status/status.h"
#include "mediapipe/framework/calculator_base.h"
namespace mediapipe {
// Creates a callback which takes a packet and stores it either in a
// vector of packets or stores only the packet at PostStream timestamp.
// The kind of callback is controlled by an option. The callback is
// a std::function and is directly usable by CallbackCalculator.
// Since the options for the packet generator include a serialized pointer
// value, the resulting callback is only valid on the original machine
// while that pointer is still alive.
class CallbackPacketCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
-26
View File
@@ -87,7 +87,6 @@ cc_library(
"//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/util:time_series_util", "//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor", "@com_google_absl//absl/status:statusor",
@@ -182,7 +181,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node", "//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
], ],
alwayslink = 1, alwayslink = 1,
@@ -200,7 +198,6 @@ cc_test(
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_check",
"@org_tensorflow//tensorflow/lite/c:common", "@org_tensorflow//tensorflow/lite/c:common",
], ],
) )
@@ -448,7 +445,6 @@ cc_library(
"//mediapipe/framework/deps:file_path", "//mediapipe/framework/deps:file_path",
"//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/util/tflite:tflite_gpu_runner", "//mediapipe/util/tflite:tflite_gpu_runner",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor", "@com_google_absl//absl/status:statusor",
@@ -478,7 +474,6 @@ cc_library(
"//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer",
"//mediapipe/objc:mediapipe_framework_ios", "//mediapipe/objc:mediapipe_framework_ios",
"//mediapipe/util/tflite:config", "//mediapipe/util/tflite:config",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate", "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
@@ -625,7 +620,6 @@ mediapipe_proto_library(
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
"//mediapipe/gpu:gpu_origin_proto",
], ],
) )
@@ -655,18 +649,7 @@ cc_library(
"//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util", "//mediapipe/util:resource_util",
"@com_google_absl//absl/log",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": ["tensor_converter_calculator_gpu_deps"], "//conditions:default": ["tensor_converter_calculator_gpu_deps"],
@@ -716,11 +699,9 @@ cc_test(
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:validate_type", "//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],
) )
@@ -756,8 +737,6 @@ cc_library(
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"//mediapipe/framework/formats/object_detection:anchor_cc_proto", "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span", "@com_google_absl//absl/types:span",
] + selects.with_or({ ] + selects.with_or({
@@ -814,7 +793,6 @@ cc_library(
"//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor", "//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/log:absl_check",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -1007,8 +985,6 @@ cc_library(
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor", "//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/gpu:gpu_origin_cc_proto",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [":image_to_tensor_calculator_gpu_deps"], "//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
@@ -1101,7 +1077,6 @@ cc_test(
"//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:parse_text_proto",
"//mediapipe/util:image_test_utils", "//mediapipe/util:image_test_utils",
"@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
@@ -1229,7 +1204,6 @@ cc_library(
"//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util", "//mediapipe/gpu:shader_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],
}), }),
@@ -20,7 +20,6 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/status/status.h" #include "absl/status/status.h"
#include "absl/status/statusor.h" #include "absl/status/statusor.h"
@@ -349,7 +348,7 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
return absl::InvalidArgumentError( return absl::InvalidArgumentError(
"The audio data should be stored in column-major."); "The audio data should be stored in column-major.");
} }
ABSL_CHECK(channels_match || mono_output); CHECK(channels_match || mono_output);
const Matrix& input = channels_match ? input_frame const Matrix& input = channels_match ? input_frame
// Mono mixdown. // Mono mixdown.
: input_frame.colwise().mean(); : input_frame.colwise().mean();
@@ -458,7 +457,7 @@ absl::Status AudioToTensorCalculator::SetupStreamingResampler(
} }
void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) { void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) {
ABSL_CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`. CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`.
if (num_samples == 0) { if (num_samples == 0) {
return; return;
} }
@@ -18,7 +18,6 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h" #include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h" #include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -66,7 +65,7 @@ template <typename T>
Tensor MakeTensor(std::initializer_list<int> shape, Tensor MakeTensor(std::initializer_list<int> shape,
std::initializer_list<T> values) { std::initializer_list<T> values) {
Tensor tensor(TensorElementType<T>::value, shape); Tensor tensor(TensorElementType<T>::value, shape);
ABSL_CHECK_EQ(values.size(), tensor.shape().num_elements()) CHECK_EQ(values.size(), tensor.shape().num_elements())
<< "The size of `values` is incompatible with `shape`"; << "The size of `values` is incompatible with `shape`";
absl::c_copy(values, tensor.GetCpuWriteView().buffer<T>()); absl::c_copy(values, tensor.GetCpuWriteView().buffer<T>());
return tensor; return tensor;
@@ -16,7 +16,6 @@
#include <memory> #include <memory>
#include <vector> #include <vector>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h" #include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" #include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h" #include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
@@ -285,9 +284,9 @@ class ImageToTensorCalculator : public Node {
cc, GetBorderMode(options_.border_mode()), cc, GetBorderMode(options_.border_mode()),
GetOutputTensorType(/*uses_gpu=*/false, params_))); GetOutputTensorType(/*uses_gpu=*/false, params_)));
#else #else
ABSL_LOG(FATAL) << "Cannot create image to tensor CPU converter since " LOG(FATAL) << "Cannot create image to tensor CPU converter since "
"MEDIAPIPE_DISABLE_OPENCV is defined and " "MEDIAPIPE_DISABLE_OPENCV is defined and "
"MEDIAPIPE_ENABLE_HALIDE is not defined."; "MEDIAPIPE_ENABLE_HALIDE is not defined.";
#endif // !MEDIAPIPE_DISABLE_HALIDE #endif // !MEDIAPIPE_DISABLE_HALIDE
} }
} }
@@ -18,7 +18,6 @@
#include <vector> #include <vector>
#include "absl/flags/flag.h" #include "absl/flags/flag.h"
#include "absl/log/absl_check.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/strings/substitute.h" #include "absl/strings/substitute.h"
@@ -206,7 +205,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
} else if (image_channels == 1) { } else if (image_channels == 1) {
return ImageFormat::GRAY8; return ImageFormat::GRAY8;
} }
ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels; CHECK(false) << "Unsupported input image channles: " << image_channels;
} }
Packet MakeImageFramePacket(cv::Mat input) { Packet MakeImageFramePacket(cv::Mat input) {
@@ -22,7 +22,6 @@
#include <memory> #include <memory>
#include <vector> #include <vector>
#include "absl/log/absl_log.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h" #include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h" #include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
@@ -260,7 +259,7 @@ class GlProcessor : public ImageToTensorConverter {
// error. So in that case, we'll grab the transpose of our original matrix // error. So in that case, we'll grab the transpose of our original matrix
// and send that instead. // and send that instead.
const auto gl_context = mediapipe::GlContext::GetCurrent(); const auto gl_context = mediapipe::GlContext::GetCurrent();
ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread."; LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) { if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) {
GetTransposedRotatedSubRectToRectTransformMatrix( GetTransposedRotatedSubRectToRectTransformMatrix(
sub_rect, texture.width(), texture.height(), flip_horizontaly, sub_rect, texture.width(), texture.height(), flip_horizontaly,
@@ -88,20 +88,6 @@ message InferenceCalculatorOptions {
// serialized model is invalid or missing. // serialized model is invalid or missing.
optional string serialized_model_dir = 7; optional string serialized_model_dir = 7;
enum CacheWritingBehavior {
// Do not write any caches.
NO_WRITE = 0;
// Try to write caches, log on failure.
TRY_WRITE = 1;
// Write caches or return an error if write fails.
WRITE_OR_ERROR = 2;
}
// Specifies how GPU caches are written to disk.
optional CacheWritingBehavior cache_writing_behavior = 10
[default = WRITE_OR_ERROR];
// Unique token identifying the model. Used in conjunction with // Unique token identifying the model. Used in conjunction with
// "serialized_model_dir". It is the caller's responsibility to ensure // "serialized_model_dir". It is the caller's responsibility to ensure
// there is no clash of the tokens. // there is no clash of the tokens.
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include <cstdint>
#include <cstring> #include <cstring>
#include <memory> #include <memory>
#include <string> #include <string>
@@ -27,7 +26,6 @@
#include "mediapipe/util/tflite/tflite_gpu_runner.h" #include "mediapipe/util/tflite/tflite_gpu_runner.h"
#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS) #if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
#include "absl/log/absl_log.h"
#include "mediapipe/framework/deps/file_path.h" #include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/util/android/file/base/file.h" #include "mediapipe/util/android/file/base/file.h"
#include "mediapipe/util/android/file/base/filesystem.h" #include "mediapipe/util/android/file/base/filesystem.h"
@@ -70,21 +68,13 @@ class InferenceCalculatorGlAdvancedImpl
const mediapipe::InferenceCalculatorOptions::Delegate::Gpu& const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
gpu_delegate_options); gpu_delegate_options);
absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const; absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
// Writes caches to disk based on |cache_writing_behavior_|.
absl::Status SaveGpuCachesBasedOnBehavior(
tflite::gpu::TFLiteGPURunner* gpu_runner) const;
bool UseSerializedModel() const { return use_serialized_model_; }
private:
// Writes caches to disk, returns error on failure.
absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const; absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
private:
bool use_kernel_caching_ = false; bool use_kernel_caching_ = false;
std::string cached_kernel_filename_; std::string cached_kernel_filename_;
bool use_serialized_model_ = false; bool use_serialized_model_ = false;
std::string serialized_model_path_; std::string serialized_model_path_;
mediapipe::InferenceCalculatorOptions::Delegate::Gpu::CacheWritingBehavior
cache_writing_behavior_;
}; };
// Helper class that wraps everything related to GPU inference acceleration. // Helper class that wraps everything related to GPU inference acceleration.
@@ -160,6 +150,8 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
} }
absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() { absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
MP_RETURN_IF_ERROR(
on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get()));
return gpu_helper_.RunInGlContext([this]() -> absl::Status { return gpu_helper_.RunInGlContext([this]() -> absl::Status {
tflite_gpu_runner_.reset(); tflite_gpu_runner_.reset();
return absl::OkStatus(); return absl::OkStatus();
@@ -234,15 +226,9 @@ InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
tflite_gpu_runner_->GetOutputShapes()[i].c}; tflite_gpu_runner_->GetOutputShapes()[i].c};
} }
if (on_disk_cache_helper_.UseSerializedModel()) {
tflite_gpu_runner_->ForceOpenCLInitFromSerializedModel();
}
MP_RETURN_IF_ERROR( MP_RETURN_IF_ERROR(
on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get())); on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
MP_RETURN_IF_ERROR(tflite_gpu_runner_->Build()); return tflite_gpu_runner_->Build();
return on_disk_cache_helper_.SaveGpuCachesBasedOnBehavior(
tflite_gpu_runner_.get());
} }
#if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS) #if defined(MEDIAPIPE_ANDROID) || defined(MEDIAPIPE_CHROMIUMOS)
@@ -271,36 +257,9 @@ absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
mediapipe::file::JoinPath(gpu_delegate_options.serialized_model_dir(), mediapipe::file::JoinPath(gpu_delegate_options.serialized_model_dir(),
gpu_delegate_options.model_token()); gpu_delegate_options.model_token());
} }
cache_writing_behavior_ = gpu_delegate_options.has_cache_writing_behavior()
? gpu_delegate_options.cache_writing_behavior()
: mediapipe::InferenceCalculatorOptions::
Delegate::Gpu::WRITE_OR_ERROR;
return absl::OkStatus(); return absl::OkStatus();
} }
absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::
SaveGpuCachesBasedOnBehavior(
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
switch (cache_writing_behavior_) {
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::NO_WRITE:
return absl::OkStatus();
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::TRY_WRITE: {
auto status = SaveGpuCaches(gpu_runner);
if (!status.ok()) {
ABSL_LOG_FIRST_N(WARNING, 1) << "Failed to save gpu caches: " << status;
}
return absl::OkStatus();
}
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::WRITE_OR_ERROR:
return SaveGpuCaches(gpu_runner);
default:
ABSL_LOG_FIRST_N(ERROR, 1)
<< "Unknown cache writing behavior: "
<< static_cast<uint32_t>(cache_writing_behavior_);
return absl::InvalidArgumentError("Unknown cache writing behavior.");
}
}
absl::Status absl::Status
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches( InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
tflite::gpu::TFLiteGPURunner* gpu_runner) const { tflite::gpu::TFLiteGPURunner* gpu_runner) const {
@@ -355,12 +314,6 @@ absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
return absl::OkStatus(); return absl::OkStatus();
} }
absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::
SaveGpuCachesBasedOnBehavior(
tflite::gpu::TFLiteGPURunner* gpu_runner) const {
return absl::OkStatus();
}
absl::Status absl::Status
InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches( InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
tflite::gpu::TFLiteGPURunner* gpu_runner) const { tflite::gpu::TFLiteGPURunner* gpu_runner) const {
@@ -21,7 +21,6 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "absl/log/absl_log.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "mediapipe/calculators/tensor/inference_calculator.h" #include "mediapipe/calculators/tensor/inference_calculator.h"
@@ -75,7 +74,7 @@ tflite::gpu::BHWC BhwcFromTensorShape(const Tensor::Shape& shape) {
break; break;
default: default:
// Handles 0 and >4. // Handles 0 and >4.
ABSL_LOG(FATAL) LOG(FATAL)
<< "Dimensions size must be in range [1,4] for GPU inference, but " << "Dimensions size must be in range [1,4] for GPU inference, but "
<< shape.dims.size() << " is provided"; << shape.dims.size() << " is provided";
} }
@@ -16,7 +16,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "absl/log/absl_check.h" #include "absl/log/check.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/str_replace.h" #include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
@@ -12,15 +12,9 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include <cstdint>
#include <string> #include <string>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h" #include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
@@ -28,8 +22,7 @@
#include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port.h" #include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/util/resource_util.h"
#include "mediapipe/gpu/gpu_origin.pb.h"
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gpu_buffer.h" #include "mediapipe/gpu/gpu_buffer.h"
@@ -50,50 +43,12 @@
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
namespace { namespace {
constexpr int kWorkgroupSize = 8; // Block size for GPU shader. constexpr int kWorkgroupSize = 8; // Block size for GPU shader.
// Commonly used to compute the number of blocks to launch in a kernel. // Commonly used to compute the number of blocks to launch in a kernel.
int NumGroups(const int size, const int group_size) { // NOLINT int NumGroups(const int size, const int group_size) { // NOLINT
return (size + group_size - 1) / group_size; return (size + group_size - 1) / group_size;
} }
absl::StatusOr<bool> ShouldFlipVertically(
const mediapipe::TensorConverterCalculatorOptions& options, bool use_gpu) {
if (options.has_flip_vertically() && options.has_gpu_origin()) {
return absl::FailedPreconditionError(absl::StrFormat(
"Cannot specify both flip_vertically and gpu_origin options"));
}
if (!options.has_gpu_origin()) {
// Fall back to flip_vertically.
return options.flip_vertically();
}
// Warn if gpu_origin is specified with a CPU input image.
// Those are always TOP_LEFT, so no flipping is necessary.
if (!use_gpu) {
ABSL_LOG(WARNING)
<< "Ignoring gpu_origin option since IMAGE_GPU input is not specified";
return false;
}
switch (options.gpu_origin()) {
case mediapipe::GpuOrigin::TOP_LEFT:
return false;
case mediapipe::GpuOrigin::DEFAULT:
case mediapipe::GpuOrigin::CONVENTIONAL:
// TOP_LEFT on Metal, BOTTOM_LEFT on OpenGL.
#ifdef __APPLE__
return false;
#else
return true;
#endif
default:
return absl::InvalidArgumentError(
absl::StrFormat("Unhandled GPU origin %i", options.gpu_origin()));
}
}
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor> typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
RowMajorMatrixXf; RowMajorMatrixXf;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor> typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
@@ -103,7 +58,6 @@ constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kGpuBufferTag[] = "IMAGE_GPU"; constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kTensorsTag[] = "TENSORS"; constexpr char kTensorsTag[] = "TENSORS";
constexpr char kMatrixTag[] = "MATRIX"; constexpr char kMatrixTag[] = "MATRIX";
} // namespace } // namespace
namespace mediapipe { namespace mediapipe {
@@ -155,7 +109,7 @@ class TensorConverterCalculator : public CalculatorBase {
private: private:
absl::Status InitGpu(CalculatorContext* cc); absl::Status InitGpu(CalculatorContext* cc);
absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu); absl::Status LoadOptions(CalculatorContext* cc);
template <class T> template <class T>
absl::Status NormalizeImage(const ImageFrame& image_frame, absl::Status NormalizeImage(const ImageFrame& image_frame,
bool flip_vertically, float* tensor_ptr); bool flip_vertically, float* tensor_ptr);
@@ -191,8 +145,7 @@ absl::Status TensorConverterCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK(static_cast<int>(cc->Inputs().HasTag(kImageFrameTag)) + RET_CHECK(static_cast<int>(cc->Inputs().HasTag(kImageFrameTag)) +
static_cast<int>(cc->Inputs().HasTag(kGpuBufferTag)) + static_cast<int>(cc->Inputs().HasTag(kGpuBufferTag)) +
static_cast<int>(cc->Inputs().HasTag(kMatrixTag)) == static_cast<int>(cc->Inputs().HasTag(kMatrixTag)) ==
1) 1);
<< "Only one input tag of {IMAGE, IMAGE_GPU, MATRIX} may be specified";
if (cc->Inputs().HasTag(kImageFrameTag)) { if (cc->Inputs().HasTag(kImageFrameTag)) {
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>(); cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
@@ -220,6 +173,8 @@ absl::Status TensorConverterCalculator::GetContract(CalculatorContract* cc) {
absl::Status TensorConverterCalculator::Open(CalculatorContext* cc) { absl::Status TensorConverterCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0)); cc->SetOffset(TimestampDiff(0));
MP_RETURN_IF_ERROR(LoadOptions(cc));
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
if (cc->Inputs().HasTag(kGpuBufferTag)) { if (cc->Inputs().HasTag(kGpuBufferTag)) {
use_gpu_ = true; use_gpu_ = true;
@@ -232,8 +187,6 @@ absl::Status TensorConverterCalculator::Open(CalculatorContext* cc) {
} }
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(LoadOptions(cc, use_gpu_));
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -425,34 +378,23 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
// Get input image sizes. // Get input image sizes.
const auto& input = const auto& input =
cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>(); cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
mediapipe::GpuBufferFormat format = input.format(); mediapipe::ImageFormat::Format format =
mediapipe::ImageFormatForGpuBufferFormat(input.format());
const bool include_alpha = (max_num_channels_ == 4); const bool include_alpha = (max_num_channels_ == 4);
const bool single_channel = (max_num_channels_ == 1); const bool single_channel = (max_num_channels_ == 1);
if (!(format == mediapipe::ImageFormat::GRAY8 ||
RET_CHECK(format == mediapipe::GpuBufferFormat::kBGRA32 || format == mediapipe::ImageFormat::SRGB ||
format == mediapipe::GpuBufferFormat::kRGB24 || format == mediapipe::ImageFormat::SRGBA))
format == mediapipe::GpuBufferFormat::kRGBA32 || RET_CHECK_FAIL() << "Unsupported GPU input format.";
format == mediapipe::GpuBufferFormat::kRGBAFloat128 || if (include_alpha && (format != mediapipe::ImageFormat::SRGBA))
format == mediapipe::GpuBufferFormat::kRGBAHalf64 || RET_CHECK_FAIL() << "Num input channels is less than desired output.";
format == mediapipe::GpuBufferFormat::kGrayFloat32 ||
format == mediapipe::GpuBufferFormat::kGrayHalf16 ||
format == mediapipe::GpuBufferFormat::kOneComponent8)
<< "Unsupported GPU input format: " << static_cast<uint32_t>(format);
if (include_alpha) {
RET_CHECK(format == mediapipe::GpuBufferFormat::kBGRA32 ||
format == mediapipe::GpuBufferFormat::kRGBA32 ||
format == mediapipe::GpuBufferFormat::kRGBAFloat128 ||
format == mediapipe::GpuBufferFormat::kRGBAHalf64)
<< "Num input channels is less than desired output, input format: "
<< static_cast<uint32_t>(format);
}
#if MEDIAPIPE_METAL_ENABLED #if MEDIAPIPE_METAL_ENABLED
id<MTLDevice> device = gpu_helper_.mtlDevice; id<MTLDevice> device = gpu_helper_.mtlDevice;
// Shader to convert GL Texture to Metal Buffer, // Shader to convert GL Texture to Metal Buffer,
// with normalization to either: [0,1] or [-1,1]. // with normalization to either: [0,1] or [-1,1].
const std::string shader_source = absl::Substitute( const std::string shader_source = absl::Substitute(
R"glsl( R"(
#include <metal_stdlib> #include <metal_stdlib>
using namespace metal; using namespace metal;
@@ -471,7 +413,7 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
$3 // g & b channels $3 // g & b channels
$4 // alpha channel $4 // alpha channel
} }
)glsl", )",
/*$0=*/ /*$0=*/
output_range_.has_value() output_range_.has_value()
? absl::Substitute("pixel = pixel * half($0) + half($1);", ? absl::Substitute("pixel = pixel * half($0) + half($1);",
@@ -481,8 +423,8 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
/*$1=*/max_num_channels_, /*$1=*/max_num_channels_,
/*$2=*/flip_vertically_ ? "(in_tex.get_height() - 1 - gid.y)" : "gid.y", /*$2=*/flip_vertically_ ? "(in_tex.get_height() - 1 - gid.y)" : "gid.y",
/*$3=*/ /*$3=*/
single_channel ? "" : R"glsl(out_buf[linear_index + 1] = pixel.y; single_channel ? "" : R"(out_buf[linear_index + 1] = pixel.y;
out_buf[linear_index + 2] = pixel.z;)glsl", out_buf[linear_index + 2] = pixel.z;)",
/*$4=*/include_alpha ? "out_buf[linear_index + 3] = pixel.w;" : ""); /*$4=*/include_alpha ? "out_buf[linear_index + 3] = pixel.w;" : "");
NSString* library_source = NSString* library_source =
@@ -500,17 +442,17 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
RET_CHECK(to_buffer_program_ != nil) << "Couldn't create pipeline state " << RET_CHECK(to_buffer_program_ != nil) << "Couldn't create pipeline state " <<
[[error localizedDescription] UTF8String]; [[error localizedDescription] UTF8String];
#elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
MP_RETURN_IF_ERROR( MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, &include_alpha,
gpu_helper_.RunInGlContext([this, &include_alpha,
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
&input, &input,
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
&single_channel]() -> absl::Status { &single_channel]()
-> absl::Status {
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
// Shader to convert GL Texture to Shader Storage Buffer Object (SSBO), // Shader to convert GL Texture to Shader Storage Buffer Object (SSBO),
// with normalization to either: [0,1] or [-1,1]. // with normalization to either: [0,1] or [-1,1].
const std::string shader_source = absl::Substitute( const std::string shader_source = absl::Substitute(
R"glsl( #version 310 es R"( #version 310 es
layout(local_size_x = $0, local_size_y = $0) in; layout(local_size_x = $0, local_size_y = $0) in;
layout(binding = 0) uniform sampler2D input_texture; layout(binding = 0) uniform sampler2D input_texture;
layout(std430, binding = 1) buffer Output {float elements[];} output_data; layout(std430, binding = 1) buffer Output {float elements[];} output_data;
@@ -524,40 +466,38 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
output_data.elements[linear_index + 0] = pixel.x; // r channel output_data.elements[linear_index + 0] = pixel.x; // r channel
$5 // g & b channels $5 // g & b channels
$6 // alpha channel $6 // alpha channel
})glsl", })",
/*$0=*/kWorkgroupSize, /*$1=*/input.width(), /*$2=*/input.height(), /*$0=*/kWorkgroupSize, /*$1=*/input.width(), /*$2=*/input.height(),
/*$3=*/ /*$3=*/
output_range_.has_value() output_range_.has_value()
? absl::Substitute( ? absl::Substitute("pixel = pixel * float($0) + float($1);",
"pixel = pixel * float($0) + float($1);", (output_range_->second - output_range_->first),
(output_range_->second - output_range_->first), output_range_->first)
output_range_->first) : "",
: "", /*$4=*/flip_vertically_ ? "(width_height.y - 1 - gid.y)" : "gid.y",
/*$4=*/flip_vertically_ ? "(width_height.y - 1 - gid.y)" : "gid.y", /*$5=*/
/*$5=*/ single_channel ? ""
single_channel : R"(output_data.elements[linear_index + 1] = pixel.y;
? "" output_data.elements[linear_index + 2] = pixel.z;)",
: R"glsl(output_data.elements[linear_index + 1] = pixel.y; /*$6=*/
output_data.elements[linear_index + 2] = pixel.z;)glsl", include_alpha ? "output_data.elements[linear_index + 3] = pixel.w;"
/*$6=*/ : "",
include_alpha ? "output_data.elements[linear_index + 3] = pixel.w;" /*$7=*/max_num_channels_);
: "", GLuint shader = glCreateShader(GL_COMPUTE_SHADER);
/*$7=*/max_num_channels_); const GLchar* sources[] = {shader_source.c_str()};
GLuint shader = glCreateShader(GL_COMPUTE_SHADER); glShaderSource(shader, 1, sources, NULL);
const GLchar* sources[] = {shader_source.c_str()}; glCompileShader(shader);
glShaderSource(shader, 1, sources, NULL); GLint compiled = GL_FALSE;
glCompileShader(shader); glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled);
GLint compiled = GL_FALSE; RET_CHECK(compiled == GL_TRUE);
glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); to_buffer_program_ = glCreateProgram();
RET_CHECK(compiled == GL_TRUE); glAttachShader(to_buffer_program_, shader);
to_buffer_program_ = glCreateProgram(); glDeleteShader(shader);
glAttachShader(to_buffer_program_, shader); glLinkProgram(to_buffer_program_);
glDeleteShader(shader);
glLinkProgram(to_buffer_program_);
#else #else
// OpenGL ES 3.0 fragment shader Texture2d -> Texture2d conversion. // OpenGL ES 3.0 fragment shader Texture2d -> Texture2d conversion.
const std::string shader_source = absl::Substitute( const std::string shader_source = absl::Substitute(
R"glsl( R"(
#if __VERSION__ < 130 #if __VERSION__ < 130
#define in varying #define in varying
#endif // __VERSION__ < 130 #endif // __VERSION__ < 130
@@ -583,51 +523,49 @@ absl::Status TensorConverterCalculator::InitGpu(CalculatorContext* cc) {
fragColor.r = pixel.r; // r channel fragColor.r = pixel.r; // r channel
$3 // g & b channels $3 // g & b channels
$4 // alpha channel $4 // alpha channel
})glsl", })",
/*$0=*/single_channel ? "vec1" : "vec4", /*$0=*/single_channel ? "vec1" : "vec4",
/*$1=*/ /*$1=*/
flip_vertically_ flip_vertically_
? "vec2(sample_coordinate.x, 1.0 - sample_coordinate.y);" ? "vec2(sample_coordinate.x, 1.0 - sample_coordinate.y);"
: "sample_coordinate;", : "sample_coordinate;",
/*$2=*/output_range_.has_value() /*$2=*/output_range_.has_value()
? absl::Substitute( ? absl::Substitute("pixel = pixel * float($0) + float($1);",
"pixel = pixel * float($0) + float($1);", (output_range_->second - output_range_->first),
(output_range_->second - output_range_->first), output_range_->first)
output_range_->first) : "",
: "", /*$3=*/single_channel ? "" : R"(fragColor.g = pixel.g;
/*$3=*/single_channel ? "" : R"glsl(fragColor.g = pixel.g; fragColor.b = pixel.b;)",
fragColor.b = pixel.b;)glsl", /*$4=*/
/*$4=*/ include_alpha ? "fragColor.a = pixel.a;"
include_alpha ? "fragColor.a = pixel.a;" : (single_channel ? "" : "fragColor.a = 1.0;"));
: (single_channel ? "" : "fragColor.a = 1.0;"));
const GLint attr_location[NUM_ATTRIBUTES] = { const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX, ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION, ATTRIB_TEXTURE_POSITION,
}; };
const GLchar* attr_name[NUM_ATTRIBUTES] = { const GLchar* attr_name[NUM_ATTRIBUTES] = {
"position", "position",
"texture_coordinate", "texture_coordinate",
}; };
// shader program and params // shader program and params
mediapipe::GlhCreateProgram( mediapipe::GlhCreateProgram(
mediapipe::kBasicVertexShader, shader_source.c_str(), mediapipe::kBasicVertexShader, shader_source.c_str(), NUM_ATTRIBUTES,
NUM_ATTRIBUTES, &attr_name[0], attr_location, &to_tex2d_program_); &attr_name[0], attr_location, &to_tex2d_program_);
RET_CHECK(to_tex2d_program_) << "Problem initializing the program."; RET_CHECK(to_tex2d_program_) << "Problem initializing the program.";
glUseProgram(to_tex2d_program_); glUseProgram(to_tex2d_program_);
glUniform1i(glGetUniformLocation(to_tex2d_program_, "frame"), 1); glUniform1i(glGetUniformLocation(to_tex2d_program_, "frame"), 1);
glGenFramebuffers(1, &framebuffer_); glGenFramebuffers(1, &framebuffer_);
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
return absl::OkStatus(); return absl::OkStatus();
})); }));
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30 #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
return absl::OkStatus(); return absl::OkStatus();
} }
absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc, absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc) {
bool use_gpu) {
// Get calculator options specified in the graph. // Get calculator options specified in the graph.
const auto& options = const auto& options =
cc->Options<::mediapipe::TensorConverterCalculatorOptions>(); cc->Options<::mediapipe::TensorConverterCalculatorOptions>();
@@ -644,7 +582,7 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc,
if (options.has_output_tensor_float_range()) { if (options.has_output_tensor_float_range()) {
output_range_.emplace(options.output_tensor_float_range().min(), output_range_.emplace(options.output_tensor_float_range().min(),
options.output_tensor_float_range().max()); options.output_tensor_float_range().max());
ABSL_CHECK_GT(output_range_->second, output_range_->first); CHECK_GT(output_range_->second, output_range_->first);
} }
// Custom div and sub values. // Custom div and sub values.
@@ -655,16 +593,16 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc,
} }
// Get y-flip mode. // Get y-flip mode.
ASSIGN_OR_RETURN(flip_vertically_, ShouldFlipVertically(options, use_gpu)); flip_vertically_ = options.flip_vertically();
// Get row_major_matrix mode. // Get row_major_matrix mode.
row_major_matrix_ = options.row_major_matrix(); row_major_matrix_ = options.row_major_matrix();
// Get desired way to handle input channels. // Get desired way to handle input channels.
max_num_channels_ = options.max_num_channels(); max_num_channels_ = options.max_num_channels();
ABSL_CHECK_GE(max_num_channels_, 1); CHECK_GE(max_num_channels_, 1);
ABSL_CHECK_LE(max_num_channels_, 4); CHECK_LE(max_num_channels_, 4);
ABSL_CHECK_NE(max_num_channels_, 2); CHECK_NE(max_num_channels_, 2);
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -3,7 +3,6 @@ syntax = "proto2";
package mediapipe; package mediapipe;
import "mediapipe/framework/calculator.proto"; import "mediapipe/framework/calculator.proto";
import "mediapipe/gpu/gpu_origin.proto";
// Full Example: // Full Example:
// //
@@ -44,16 +43,8 @@ message TensorConverterCalculatorOptions {
// with a coordinate system where the origin is at the bottom-left corner // with a coordinate system where the origin is at the bottom-left corner
// (e.g., in OpenGL) whereas the ML model expects an image with a top-left // (e.g., in OpenGL) whereas the ML model expects an image with a top-left
// origin. // origin.
// Prefer gpu_origin over this field when using GPU input images.
optional bool flip_vertically = 2 [default = false]; optional bool flip_vertically = 2 [default = false];
// Determines when the input GPU image should be flipped vertically.
// See GpuOrigin.Mode for more information.
// Affects only IMAGE_GPU inputs.
// If unset, falls back to flip_vertically for backwards compatibility.
// Cannot set both gpu_origin and flip_vertically.
optional GpuOrigin.Mode gpu_origin = 10;
// Controls how many channels of the input image get passed through to the // Controls how many channels of the input image get passed through to the
// tensor. Valid values are 1,3,4 only. Ignored for iOS GPU. // tensor. Valid values are 1,3,4 only. Ignored for iOS GPU.
optional int32 max_num_channels = 3 [default = 3]; optional int32 max_num_channels = 3 [default = 3];
@@ -12,15 +12,10 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include <cmath>
#include <cstdint>
#include <memory>
#include <random> #include <random>
#include <utility>
#include <vector> #include <vector>
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/strings/substitute.h" #include "absl/strings/substitute.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"
@@ -29,10 +24,8 @@
#include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h" // NOLINT #include "mediapipe/framework/port/status_matchers.h" // NOLINT
#include "mediapipe/framework/tool/validate_type.h" #include "mediapipe/framework/tool/validate_type.h"
@@ -47,7 +40,7 @@ constexpr char kTransposeOptionsString[] =
} // namespace } // namespace
using RandomEngine = std::mt19937_64; using RandomEngine = std::mt19937_64;
using ::testing::HasSubstr; using testing::Eq;
const uint32_t kSeed = 1234; const uint32_t kSeed = 1234;
const int kNumSizes = 8; const int kNumSizes = 8;
const int sizes[kNumSizes][2] = {{1, 1}, {12, 1}, {1, 9}, {2, 2}, const int sizes[kNumSizes][2] = {{1, 1}, {12, 1}, {1, 9}, {2, 2},
@@ -60,7 +53,7 @@ class TensorConverterCalculatorTest : public ::testing::Test {
bool row_major_matrix = false) { bool row_major_matrix = false) {
RandomEngine random(kSeed); RandomEngine random(kSeed);
std::uniform_real_distribution<> uniform_dist(0, 1.0); std::uniform_real_distribution<> uniform_dist(0, 1.0);
auto matrix = std::make_unique<Matrix>(); auto matrix = ::absl::make_unique<Matrix>();
matrix->resize(num_rows, num_columns); matrix->resize(num_rows, num_columns);
if (row_major_matrix) { if (row_major_matrix) {
for (int y = 0; y < num_rows; ++y) { for (int y = 0; y < num_rows; ++y) {
@@ -108,7 +101,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
tool::AddVectorSink("tensor", &graph_config, &output_packets); tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph. // Run the graph.
graph_ = std::make_unique<CalculatorGraph>(); graph_ = absl::make_unique<CalculatorGraph>();
MP_ASSERT_OK(graph_->Initialize(graph_config)); MP_ASSERT_OK(graph_->Initialize(graph_config));
MP_ASSERT_OK(graph_->StartRun({})); MP_ASSERT_OK(graph_->StartRun({}));
@@ -117,12 +110,12 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
// Wait until the calculator done processing. // Wait until the calculator done processing.
MP_ASSERT_OK(graph_->WaitUntilIdle()); MP_ASSERT_OK(graph_->WaitUntilIdle());
ASSERT_EQ(output_packets.size(), 1); EXPECT_EQ(1, output_packets.size());
// Get and process results. // Get and process results.
const std::vector<Tensor>& tensor_vec = const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>(); output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1); EXPECT_EQ(1, tensor_vec.size());
const Tensor* tensor = &tensor_vec[0]; const Tensor* tensor = &tensor_vec[0];
EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type());
@@ -134,7 +127,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixColMajor) {
auto tensor_buffer = view.buffer<float>(); auto tensor_buffer = view.buffer<float>();
for (int i = 0; i < num_rows * num_columns; ++i) { for (int i = 0; i < num_rows * num_columns; ++i) {
const float expected = uniform_dist(random); const float expected = uniform_dist(random);
EXPECT_FLOAT_EQ(tensor_buffer[i], expected) << "at i = " << i; EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i;
} }
// Fully close graph at end, otherwise calculator+tensors are destroyed // Fully close graph at end, otherwise calculator+tensors are destroyed
@@ -170,7 +163,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
tool::AddVectorSink("tensor", &graph_config, &output_packets); tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph. // Run the graph.
graph_ = std::make_unique<CalculatorGraph>(); graph_ = absl::make_unique<CalculatorGraph>();
MP_ASSERT_OK(graph_->Initialize(graph_config)); MP_ASSERT_OK(graph_->Initialize(graph_config));
MP_ASSERT_OK(graph_->StartRun({})); MP_ASSERT_OK(graph_->StartRun({}));
@@ -179,12 +172,12 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
// Wait until the calculator done processing. // Wait until the calculator done processing.
MP_ASSERT_OK(graph_->WaitUntilIdle()); MP_ASSERT_OK(graph_->WaitUntilIdle());
ASSERT_EQ(output_packets.size(), 1); EXPECT_EQ(1, output_packets.size());
// Get and process results. // Get and process results.
const std::vector<Tensor>& tensor_vec = const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>(); output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1); EXPECT_EQ(1, tensor_vec.size());
const Tensor* tensor = &tensor_vec[0]; const Tensor* tensor = &tensor_vec[0];
EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type());
@@ -196,7 +189,7 @@ TEST_F(TensorConverterCalculatorTest, RandomMatrixRowMajor) {
auto tensor_buffer = view.buffer<float>(); auto tensor_buffer = view.buffer<float>();
for (int i = 0; i < num_rows * num_columns; ++i) { for (int i = 0; i < num_rows * num_columns; ++i) {
const float expected = uniform_dist(random); const float expected = uniform_dist(random);
EXPECT_EQ(tensor_buffer[i], expected) << "at i = " << i; EXPECT_EQ(expected, tensor_buffer[i]) << "at i = " << i;
} }
// Fully close graph at end, otherwise calculator+tensors are destroyed // Fully close graph at end, otherwise calculator+tensors are destroyed
@@ -234,7 +227,7 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
// Run the graph. // Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config)); MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({})); MP_ASSERT_OK(graph.StartRun({}));
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1); auto input_image = absl::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
cv::Mat mat = mediapipe::formats::MatView(input_image.get()); cv::Mat mat = mediapipe::formats::MatView(input_image.get());
mat.at<uint8_t>(0, 0) = 200; mat.at<uint8_t>(0, 0) = 200;
MP_ASSERT_OK(graph.AddPacketToInputStream( MP_ASSERT_OK(graph.AddPacketToInputStream(
@@ -246,12 +239,12 @@ TEST_F(TensorConverterCalculatorTest, CustomDivAndSub) {
// Get and process results. // Get and process results.
const std::vector<Tensor>& tensor_vec = const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>(); output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1); EXPECT_EQ(1, tensor_vec.size());
const Tensor* tensor = &tensor_vec[0]; const Tensor* tensor = &tensor_vec[0];
EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type()); EXPECT_EQ(Tensor::ElementType::kFloat32, tensor->element_type());
auto view = tensor->GetCpuReadView(); auto view = tensor->GetCpuReadView();
EXPECT_FLOAT_EQ(*view.buffer<float>(), 67.0f); EXPECT_FLOAT_EQ(67.0f, *view.buffer<float>());
// Fully close graph at end, otherwise calculator+tensors are destroyed // Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone(). // after calling WaitUntilDone().
@@ -266,29 +259,32 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
for (std::pair<float, float> range : range_values) { for (std::pair<float, float> range : range_values) {
CalculatorGraph graph; CalculatorGraph graph;
CalculatorGraphConfig graph_config = CalculatorGraphConfig graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(absl::Substitute( mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb( absl::Substitute(R"(
input_stream: "input_image" input_stream: "input_image"
node { node {
calculator: "TensorConverterCalculator" calculator: "TensorConverterCalculator"
input_stream: "IMAGE:input_image" input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor" output_stream: "TENSORS:tensor"
options { options {
[mediapipe.TensorConverterCalculatorOptions.ext] { [mediapipe.TensorConverterCalculatorOptions.ext] {
output_tensor_float_range { min: $0 max: $1 } output_tensor_float_range {
} min: $0
} max: $1
} }
)pb", }
/*$0=*/range.first, }
/*$1=*/range.second)); }
)",
/*$0=*/range.first,
/*$1=*/range.second));
std::vector<Packet> output_packets; std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets); tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph. // Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config)); MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({})); MP_ASSERT_OK(graph.StartRun({}));
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1); auto input_image = absl::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
cv::Mat mat = mediapipe::formats::MatView(input_image.get()); cv::Mat mat = mediapipe::formats::MatView(input_image.get());
mat.at<uint8_t>(0, 0) = 200; mat.at<uint8_t>(0, 0) = 200;
MP_ASSERT_OK(graph.AddPacketToInputStream( MP_ASSERT_OK(graph.AddPacketToInputStream(
@@ -296,23 +292,26 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
// Wait until the calculator finishes processing. // Wait until the calculator finishes processing.
MP_ASSERT_OK(graph.WaitUntilIdle()); MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_EQ(output_packets.size(), 1); EXPECT_THAT(output_packets.size(), Eq(1));
// Get and process results. // Get and process results.
const std::vector<Tensor>& tensor_vec = const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>(); output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1); EXPECT_THAT(tensor_vec.size(), Eq(1));
const Tensor* tensor = &tensor_vec[0]; const Tensor* tensor = &tensor_vec[0];
// Calculate the expected normalized value: // Calculate the expected normalized value:
float expected_value = float normalized_value =
range.first + (200 * (range.second - range.first)) / 255.0; range.first + (200 * (range.second - range.first)) / 255.0;
EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32); EXPECT_THAT(tensor->element_type(), Eq(Tensor::ElementType::kFloat32));
auto view = tensor->GetCpuReadView(); auto view = tensor->GetCpuReadView();
float actual_value = *view.buffer<float>(); float dataf = *view.buffer<float>();
EXPECT_FLOAT_EQ(actual_value, expected_value); EXPECT_THAT(
normalized_value,
testing::FloatNear(dataf, 2.0f * std::abs(dataf) *
std::numeric_limits<float>::epsilon()));
// Fully close graph at end, otherwise calculator+tensors are destroyed // Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone(). // after calling WaitUntilDone().
@@ -321,153 +320,4 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
} }
} }
TEST_F(TensorConverterCalculatorTest, FlipVertically) {
CalculatorGraph graph;
CalculatorGraphConfig graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
input_stream: "input_image"
node {
calculator: "TensorConverterCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.TensorConverterCalculatorOptions.ext] {
flip_vertically: true
output_tensor_float_range { min: 0 max: 255 }
}
}
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 2);
cv::Mat mat = mediapipe::formats::MatView(input_image.get());
constexpr uint8_t kY0Value = 100;
constexpr uint8_t kY1Value = 200;
mat.at<uint8_t>(0, 0) = kY0Value;
mat.at<uint8_t>(1, 0) = kY1Value; // Note: y, x!
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
// Wait until the calculator finishes processing.
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_EQ(output_packets.size(), 1);
// Get and process results.
const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1);
const Tensor* tensor = &tensor_vec[0];
EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32);
const float* dataf = tensor->GetCpuReadView().buffer<float>();
EXPECT_EQ(static_cast<int>(roundf(dataf[0])), kY1Value); // Y0, Y1 flipped!
EXPECT_EQ(static_cast<int>(roundf(dataf[1])), kY0Value);
// Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone().
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
MP_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(TensorConverterCalculatorTest,
CannotSpecifyBothFlipVerticallyAndGpuOrigin) {
CalculatorGraph graph;
CalculatorGraphConfig graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
input_stream: "input_image"
node {
calculator: "TensorConverterCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.TensorConverterCalculatorOptions.ext] {
flip_vertically: true
gpu_origin: TOP_LEFT
output_tensor_float_range { min: 0 max: 255 }
}
}
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
// Processing should fail as we specified both flip_vertically and gpu_origin.
absl::Status status = graph.WaitUntilIdle();
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.message(), HasSubstr("flip_vertically and gpu_origin"));
EXPECT_EQ(output_packets.size(), 0);
// Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone().
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
EXPECT_FALSE(graph.WaitUntilDone().ok());
}
TEST_F(TensorConverterCalculatorTest, GpuOriginIsIgnoredWithCpuImage) {
CalculatorGraph graph;
CalculatorGraphConfig graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"pb(
input_stream: "input_image"
node {
calculator: "TensorConverterCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.TensorConverterCalculatorOptions.ext] {
gpu_origin: CONVENTIONAL
output_tensor_float_range { min: 0 max: 255 }
}
}
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 2);
cv::Mat mat = mediapipe::formats::MatView(input_image.get());
constexpr uint8_t kY0Value = 100;
constexpr uint8_t kY1Value = 200;
mat.at<uint8_t>(0, 0) = kY0Value;
mat.at<uint8_t>(1, 0) = kY1Value; // Note: y, x!
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
// Wait until the calculator finishes processing.
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_EQ(output_packets.size(), 1);
// Get and process results.
const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1);
const Tensor* tensor = &tensor_vec[0];
EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32);
const float* dataf = tensor->GetCpuReadView().buffer<float>();
EXPECT_EQ(static_cast<int>(roundf(dataf[0])), kY0Value); // Not flipped!
EXPECT_EQ(static_cast<int>(roundf(dataf[1])), kY1Value);
// Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone().
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
MP_ASSERT_OK(graph.WaitUntilDone());
}
} // namespace mediapipe } // namespace mediapipe
@@ -15,7 +15,6 @@
#include <unordered_map> #include <unordered_map>
#include <vector> #include <vector>
#include "absl/log/absl_log.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/types/span.h" #include "absl/types/span.h"
#include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h" #include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h"
@@ -84,7 +83,7 @@ void ConvertRawValuesToAnchors(const float* raw_anchors, int num_boxes,
void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors, void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
int num_boxes, float* raw_anchors) { int num_boxes, float* raw_anchors) {
ABSL_CHECK_EQ(anchors.size(), num_boxes); CHECK_EQ(anchors.size(), num_boxes);
int box = 0; int box = 0;
for (const auto& anchor : anchors) { for (const auto& anchor : anchors) {
raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center(); raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center();
@@ -330,7 +329,7 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
} else if (status.code() == absl::StatusCode::kFailedPrecondition) { } else if (status.code() == absl::StatusCode::kFailedPrecondition) {
// For initialization error because of hardware limitation, fallback to // For initialization error because of hardware limitation, fallback to
// CPU processing. // CPU processing.
ABSL_LOG(WARNING) << status.message(); LOG(WARNING) << status.message();
} else { } else {
// For other error, let the error propagates. // For other error, let the error propagates.
return status; return status;
@@ -669,7 +668,7 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU(
output_detections)); output_detections));
#else #else
ABSL_LOG(ERROR) << "GPU input on non-Android not supported yet."; LOG(ERROR) << "GPU input on non-Android not supported yet.";
#endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) #endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -704,18 +703,18 @@ absl::Status TensorsToDetectionsCalculator::LoadOptions(CalculatorContext* cc) {
num_boxes_ = options_.num_boxes(); num_boxes_ = options_.num_boxes();
num_coords_ = options_.num_coords(); num_coords_ = options_.num_coords();
box_output_format_ = GetBoxFormat(options_); box_output_format_ = GetBoxFormat(options_);
ABSL_CHECK_NE(options_.max_results(), 0) CHECK_NE(options_.max_results(), 0)
<< "The maximum number of the top-scored detection results must be " << "The maximum number of the top-scored detection results must be "
"non-zero."; "non-zero.";
max_results_ = options_.max_results(); max_results_ = options_.max_results();
// Currently only support 2D when num_values_per_keypoint equals to 2. // Currently only support 2D when num_values_per_keypoint equals to 2.
ABSL_CHECK_EQ(options_.num_values_per_keypoint(), 2); CHECK_EQ(options_.num_values_per_keypoint(), 2);
// Check if the output size is equal to the requested boxes and keypoints. // Check if the output size is equal to the requested boxes and keypoints.
ABSL_CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() +
kNumCoordsPerBox, kNumCoordsPerBox,
num_coords_); num_coords_);
if (kSideInIgnoreClasses(cc).IsConnected()) { if (kSideInIgnoreClasses(cc).IsConnected()) {
RET_CHECK(!kSideInIgnoreClasses(cc).IsEmpty()); RET_CHECK(!kSideInIgnoreClasses(cc).IsEmpty());
@@ -1155,12 +1154,11 @@ void main() {
} }
// TODO support better filtering. // TODO support better filtering.
if (class_index_set_.is_allowlist) { if (class_index_set_.is_allowlist) {
ABSL_CHECK_EQ(class_index_set_.values.size(), CHECK_EQ(class_index_set_.values.size(),
IsClassIndexAllowed(0) ? num_classes_ : num_classes_ - 1) IsClassIndexAllowed(0) ? num_classes_ : num_classes_ - 1)
<< "Only all classes >= class 0 or >= class 1"; << "Only all classes >= class 0 or >= class 1";
} else { } else {
ABSL_CHECK_EQ(class_index_set_.values.size(), CHECK_EQ(class_index_set_.values.size(), IsClassIndexAllowed(0) ? 0 : 1)
IsClassIndexAllowed(0) ? 0 : 1)
<< "Only ignore class 0 is allowed"; << "Only ignore class 0 is allowed";
} }
@@ -1381,12 +1379,11 @@ kernel void scoreKernel(
// TODO support better filtering. // TODO support better filtering.
if (class_index_set_.is_allowlist) { if (class_index_set_.is_allowlist) {
ABSL_CHECK_EQ(class_index_set_.values.size(), CHECK_EQ(class_index_set_.values.size(),
IsClassIndexAllowed(0) ? num_classes_ : num_classes_ - 1) IsClassIndexAllowed(0) ? num_classes_ : num_classes_ - 1)
<< "Only all classes >= class 0 or >= class 1"; << "Only all classes >= class 0 or >= class 1";
} else { } else {
ABSL_CHECK_EQ(class_index_set_.values.size(), CHECK_EQ(class_index_set_.values.size(), IsClassIndexAllowed(0) ? 0 : 1)
IsClassIndexAllowed(0) ? 0 : 1)
<< "Only ignore class 0 is allowed"; << "Only ignore class 0 is allowed";
} }
@@ -142,7 +142,7 @@ absl::Status TensorsToLandmarksCalculator::Process(CalculatorContext* cc) {
RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32); RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
int num_values = input_tensors[0].shape().num_elements(); int num_values = input_tensors[0].shape().num_elements();
const int num_dimensions = num_values / num_landmarks_; const int num_dimensions = num_values / num_landmarks_;
ABSL_CHECK_GT(num_dimensions, 0); CHECK_GT(num_dimensions, 0);
auto view = input_tensors[0].GetCpuReadView(); auto view = input_tensors[0].GetCpuReadView();
auto raw_landmarks = view.buffer<float>(); auto raw_landmarks = view.buffer<float>();
+11 -37
View File
@@ -13,7 +13,6 @@
# limitations under the License. # limitations under the License.
# #
# Placeholder: load py_proto_library
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
licenses(["notice"]) licenses(["notice"])
@@ -315,7 +314,6 @@ cc_library(
"//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
] + select({ ] + select({
"//conditions:default": [ "//conditions:default": [
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
@@ -368,18 +366,18 @@ cc_library(
name = "pack_media_sequence_calculator", name = "pack_media_sequence_calculator",
srcs = ["pack_media_sequence_calculator.cc"], srcs = ["pack_media_sequence_calculator.cc"],
deps = [ deps = [
":pack_media_sequence_calculator_cc_proto",
"//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto", "//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto",
"//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location", "//mediapipe/framework/formats:location",
"//mediapipe/framework/formats:location_opencv", "//mediapipe/framework/formats:location_opencv",
"//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util/sequence:media_sequence", "//mediapipe/util/sequence:media_sequence",
"//mediapipe/util/sequence:media_sequence_util", "//mediapipe/util/sequence:media_sequence_util",
"@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:protos_all_cc",
], ],
@@ -408,13 +406,8 @@ cc_library(
alwayslink = 1, alwayslink = 1,
) )
# This dependency removed the following 3 targets because they failed Boq conformance test: # This dependency removed tensorflow_jellyfish_deps and xprofilez_with_server because they failed
# # Boq conformance test. Weigh your use case to see if this will work for you.
# tensorflow_jellyfish_deps
# jfprof_lib
# xprofilez_with_server
#
# If you need them plz consider tensorflow_inference_calculator_no_envelope_loader.
cc_library( cc_library(
name = "tensorflow_inference_calculator_for_boq", name = "tensorflow_inference_calculator_for_boq",
srcs = ["tensorflow_inference_calculator.cc"], srcs = ["tensorflow_inference_calculator.cc"],
@@ -431,7 +424,7 @@ cc_library(
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/tool:status_util", "//mediapipe/framework/tool:status_util",
"@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/log:check",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_absl//absl/synchronization", "@com_google_absl//absl/synchronization",
@@ -490,10 +483,10 @@ cc_library(
"//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto", "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/deps:clock", "//mediapipe/framework/deps:clock",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/tool:status_util", "//mediapipe/framework/tool:status_util",
"@com_google_absl//absl/log:absl_log",
"@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:protos_all_cc",
] + select({ ] + select({
"//conditions:default": [ "//conditions:default": [
@@ -521,10 +514,10 @@ cc_library(
":tensorflow_session_from_frozen_graph_generator_cc_proto", ":tensorflow_session_from_frozen_graph_generator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/deps:clock", "//mediapipe/framework/deps:clock",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/tool:status_util", "//mediapipe/framework/tool:status_util",
"@com_google_absl//absl/log:absl_log",
"@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:protos_all_cc",
] + select({ ] + select({
"//conditions:default": [ "//conditions:default": [
@@ -557,7 +550,6 @@ cc_library(
"//mediapipe/framework/deps:file_path", "//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/cc/saved_model:constants", "@org_tensorflow//tensorflow/cc/saved_model:constants",
"@org_tensorflow//tensorflow/cc/saved_model:loader_lite", "@org_tensorflow//tensorflow/cc/saved_model:loader_lite",
@@ -635,7 +627,6 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/tool:status_util", "//mediapipe/framework/tool:status_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/cc/saved_model:constants", "@org_tensorflow//tensorflow/cc/saved_model:constants",
@@ -657,7 +648,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
], ],
alwayslink = 1, alwayslink = 1,
@@ -672,7 +662,6 @@ cc_library(
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
], ],
alwayslink = 1, alwayslink = 1,
@@ -688,7 +677,6 @@ cc_library(
"//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
] + select({ ] + select({
"//conditions:default": [ "//conditions:default": [
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
@@ -723,7 +711,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core/platform:bfloat16",
] + select({ ] + select({
"//conditions:default": [ "//conditions:default": [
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
@@ -786,7 +773,6 @@ cc_library(
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/util:audio_decoder_cc_proto", "//mediapipe/util:audio_decoder_cc_proto",
"//mediapipe/util/sequence:media_sequence", "//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:protos_all_cc",
], ],
@@ -801,8 +787,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
], ],
alwayslink = 1, alwayslink = 1,
@@ -816,7 +800,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
], ],
alwayslink = 1, alwayslink = 1,
@@ -830,7 +813,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
], ],
alwayslink = 1, alwayslink = 1,
@@ -844,8 +826,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet", "//mediapipe/framework:packet",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:protos_all_cc",
], ],
alwayslink = 1, alwayslink = 1,
@@ -940,24 +920,22 @@ cc_test(
srcs = ["pack_media_sequence_calculator_test.cc"], srcs = ["pack_media_sequence_calculator_test.cc"],
deps = [ deps = [
":pack_media_sequence_calculator", ":pack_media_sequence_calculator",
":pack_media_sequence_calculator_cc_proto",
"//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto", "//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto",
"//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner", "//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
"//mediapipe/framework:timestamp", "//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:location", "//mediapipe/framework/formats:location",
"//mediapipe/framework/formats:location_opencv", "//mediapipe/framework/formats:location_opencv",
"//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/util/sequence:media_sequence", "//mediapipe/util/sequence:media_sequence",
"//mediapipe/util/sequence:media_sequence_util", "@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_googletest//:gtest_main",
"@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:protos_all_cc",
], ],
) )
@@ -1140,7 +1118,6 @@ cc_test(
"//mediapipe/util:packet_test_util", "//mediapipe/util:packet_test_util",
"@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core/platform:bfloat16",
], ],
) )
@@ -1186,7 +1163,6 @@ cc_test(
"//mediapipe/framework/port:rectangle", "//mediapipe/framework/port:rectangle",
"//mediapipe/util:audio_decoder_cc_proto", "//mediapipe/util:audio_decoder_cc_proto",
"//mediapipe/util/sequence:media_sequence", "//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:protos_all_cc",
@@ -1268,8 +1244,6 @@ cc_test(
"//mediapipe/framework/tool:sink", "//mediapipe/framework/tool:sink",
"//mediapipe/framework/tool:validate_type", "//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
] + select({ ] + select({
"//conditions:default": [ "//conditions:default": [
"@org_tensorflow//tensorflow/core:direct_session", "@org_tensorflow//tensorflow/core:direct_session",
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_options.pb.h" #include "mediapipe/calculators/tensorflow/matrix_to_tensor_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
@@ -29,7 +28,7 @@ namespace mediapipe {
namespace { namespace {
absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
TimeSeriesHeader* header) { TimeSeriesHeader* header) {
ABSL_CHECK(header); CHECK(header);
if (header_packet.IsEmpty()) { if (header_packet.IsEmpty()) {
return absl::UnknownError("No header found."); return absl::UnknownError("No header found.");
} }
@@ -12,23 +12,21 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include <cstdint>
#include <optional>
#include <string> #include <string>
#include <vector> #include <vector>
#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_map.h"
#include "absl/status/status.h"
#include "absl/strings/match.h" #include "absl/strings/match.h"
#include "absl/strings/strip.h"
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" #include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
#include "mediapipe/calculators/tensorflow/pack_media_sequence_calculator.pb.h" #include "mediapipe/calculators/tensorflow/pack_media_sequence_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/location.h" #include "mediapipe/framework/formats/location.h"
#include "mediapipe/framework/formats/location_opencv.h" #include "mediapipe/framework/formats/location_opencv.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/util/sequence/media_sequence.h" #include "mediapipe/util/sequence/media_sequence.h"
#include "mediapipe/util/sequence/media_sequence_util.h" #include "mediapipe/util/sequence/media_sequence_util.h"
#include "tensorflow/core/example/example.pb.h" #include "tensorflow/core/example/example.pb.h"
@@ -38,11 +36,7 @@ namespace mediapipe {
const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE"; const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
const char kImageTag[] = "IMAGE"; const char kImageTag[] = "IMAGE";
const char kImageLabelPrefixTag[] = "IMAGE_LABEL_";
const char kClipLabelPrefixTag[] = "CLIP_LABEL_";
const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_"; const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_";
const char kIntsContextFeaturePrefixTag[] = "INTS_CONTEXT_FEATURE_";
const char kBytesContextFeaturePrefixTag[] = "BYTES_CONTEXT_FEATURE_";
const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_"; const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_";
const char kIntFeaturePrefixTag[] = "INT_FEATURE_"; const char kIntFeaturePrefixTag[] = "INT_FEATURE_";
const char kBytesFeaturePrefixTag[] = "BYTES_FEATURE_"; const char kBytesFeaturePrefixTag[] = "BYTES_FEATURE_";
@@ -50,7 +44,6 @@ const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
const char kBBoxTag[] = "BBOX"; const char kBBoxTag[] = "BBOX";
const char kKeypointsTag[] = "KEYPOINTS"; const char kKeypointsTag[] = "KEYPOINTS";
const char kSegmentationMaskTag[] = "CLASS_SEGMENTATION"; const char kSegmentationMaskTag[] = "CLASS_SEGMENTATION";
const char kClipMediaIdTag[] = "CLIP_MEDIA_ID";
namespace tf = ::tensorflow; namespace tf = ::tensorflow;
namespace mpms = mediapipe::mediasequence; namespace mpms = mediapipe::mediasequence;
@@ -62,23 +55,16 @@ namespace mpms = mediapipe::mediasequence;
// context features can be supplied verbatim in the calculator's options. The // context features can be supplied verbatim in the calculator's options. The
// SequenceExample will conform to the description in media_sequence.h. // SequenceExample will conform to the description in media_sequence.h.
// //
// The supported input stream tags are: // The supported input stream tags are "IMAGE", which stores the encoded
// * "IMAGE", which stores the encoded images from the // images from the OpenCVImageEncoderCalculator, "FORWARD_FLOW_ENCODED", which
// OpenCVImageEncoderCalculator, // stores the encoded optical flow from the same calculator, "BBOX" which stores
// * "IMAGE_LABEL", which stores whole image labels from Detection, // bounding boxes from vector<Detections>, and streams with the
// * "FORWARD_FLOW_ENCODED", which stores the encoded optical flow from the same // "FLOAT_FEATURE_${NAME}" pattern, which stores the values from vector<float>'s
// calculator, // associated with the name ${NAME}. "KEYPOINTS" stores a map of 2D keypoints
// * "BBOX" which stores bounding boxes from vector<Detections>, // from flat_hash_map<string, vector<pair<float, float>>>. "IMAGE_${NAME}",
// * streams with the "FLOAT_FEATURE_${NAME}" pattern, which stores the values // "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store prefixed versions of
// from vector<float>'s associated with the name ${NAME}, // each stream, which allows for multiple image streams to be included. However,
// * "KEYPOINTS" stores a map of 2D keypoints from flat_hash_map<string, // the default names are suppored by more tools.
// vector<pair<float, float>>>,
// * "CLIP_MEDIA_ID", which stores the clip's media ID as a string.
// * "CLIP_LABEL_${NAME}" which stores sparse feature labels, ID and scores in
// mediapipe::Detection.
// "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store
// prefixed versions of each stream, which allows for multiple image streams to
// be included. However, the default names are suppored by more tools.
// //
// Example config: // Example config:
// node { // node {
@@ -114,9 +100,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
static absl::Status GetContract(CalculatorContract* cc) { static absl::Status GetContract(CalculatorContract* cc) {
RET_CHECK(cc->InputSidePackets().HasTag(kSequenceExampleTag)); RET_CHECK(cc->InputSidePackets().HasTag(kSequenceExampleTag));
cc->InputSidePackets().Tag(kSequenceExampleTag).Set<tf::SequenceExample>(); cc->InputSidePackets().Tag(kSequenceExampleTag).Set<tf::SequenceExample>();
if (cc->InputSidePackets().HasTag(kClipMediaIdTag)) {
cc->InputSidePackets().Tag(kClipMediaIdTag).Set<std::string>();
}
if (cc->Inputs().HasTag(kForwardFlowEncodedTag)) { if (cc->Inputs().HasTag(kForwardFlowEncodedTag)) {
cc->Inputs() cc->Inputs()
@@ -129,10 +112,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
for (const auto& tag : cc->Inputs().GetTags()) { for (const auto& tag : cc->Inputs().GetTags()) {
if (absl::StartsWith(tag, kImageTag)) { if (absl::StartsWith(tag, kImageTag)) {
if (absl::StartsWith(tag, kImageLabelPrefixTag)) {
cc->Inputs().Tag(tag).Set<Detection>();
continue;
}
std::string key = ""; std::string key = "";
if (tag != kImageTag) { if (tag != kImageTag) {
int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1;
@@ -171,18 +150,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
} }
cc->Inputs().Tag(tag).Set<std::vector<Detection>>(); cc->Inputs().Tag(tag).Set<std::vector<Detection>>();
} }
if (absl::StartsWith(tag, kClipLabelPrefixTag)) {
cc->Inputs().Tag(tag).Set<Detection>();
}
if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag)) { if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag)) {
cc->Inputs().Tag(tag).Set<std::vector<float>>(); cc->Inputs().Tag(tag).Set<std::vector<float>>();
} }
if (absl::StartsWith(tag, kIntsContextFeaturePrefixTag)) {
cc->Inputs().Tag(tag).Set<std::vector<int64_t>>();
}
if (absl::StartsWith(tag, kBytesContextFeaturePrefixTag)) {
cc->Inputs().Tag(tag).Set<std::vector<std::string>>();
}
if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) { if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
cc->Inputs().Tag(tag).Set<std::vector<float>>(); cc->Inputs().Tag(tag).Set<std::vector<float>>();
} }
@@ -194,8 +164,8 @@ class PackMediaSequenceCalculator : public CalculatorBase {
} }
} }
RET_CHECK(cc->Outputs().HasTag(kSequenceExampleTag) || CHECK(cc->Outputs().HasTag(kSequenceExampleTag) ||
cc->OutputSidePackets().HasTag(kSequenceExampleTag)) cc->OutputSidePackets().HasTag(kSequenceExampleTag))
<< "Neither the output stream nor the output side packet is set to " << "Neither the output stream nor the output side packet is set to "
"output the sequence example."; "output the sequence example.";
if (cc->Outputs().HasTag(kSequenceExampleTag)) { if (cc->Outputs().HasTag(kSequenceExampleTag)) {
@@ -214,11 +184,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
cc->InputSidePackets() cc->InputSidePackets()
.Tag(kSequenceExampleTag) .Tag(kSequenceExampleTag)
.Get<tf::SequenceExample>()); .Get<tf::SequenceExample>());
if (cc->InputSidePackets().HasTag(kClipMediaIdTag) &&
!cc->InputSidePackets().Tag(kClipMediaIdTag).IsEmpty()) {
clip_media_id_ =
cc->InputSidePackets().Tag(kClipMediaIdTag).Get<std::string>();
}
const auto& context_features = const auto& context_features =
cc->Options<PackMediaSequenceCalculatorOptions>().context_feature_map(); cc->Options<PackMediaSequenceCalculatorOptions>().context_feature_map();
@@ -232,19 +197,8 @@ class PackMediaSequenceCalculator : public CalculatorBase {
replace_keypoints_ = false; replace_keypoints_ = false;
if (cc->Options<PackMediaSequenceCalculatorOptions>() if (cc->Options<PackMediaSequenceCalculatorOptions>()
.replace_data_instead_of_append()) { .replace_data_instead_of_append()) {
// Clear the existing values under the same key.
for (const auto& tag : cc->Inputs().GetTags()) { for (const auto& tag : cc->Inputs().GetTags()) {
if (absl::StartsWith(tag, kImageTag)) { if (absl::StartsWith(tag, kImageTag)) {
if (absl::StartsWith(tag, kImageLabelPrefixTag)) {
std::string key =
std::string(absl::StripPrefix(tag, kImageLabelPrefixTag));
mpms::ClearImageLabelString(key, sequence_.get());
mpms::ClearImageLabelConfidence(key, sequence_.get());
if (!key.empty() || mpms::HasImageEncoded(*sequence_)) {
mpms::ClearImageTimestamp(key, sequence_.get());
}
continue;
}
std::string key = ""; std::string key = "";
if (tag != kImageTag) { if (tag != kImageTag) {
int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1;
@@ -273,41 +227,12 @@ class PackMediaSequenceCalculator : public CalculatorBase {
mpms::ClearBBoxNumRegions(key, sequence_.get()); mpms::ClearBBoxNumRegions(key, sequence_.get());
mpms::ClearBBoxLabelString(key, sequence_.get()); mpms::ClearBBoxLabelString(key, sequence_.get());
mpms::ClearBBoxLabelIndex(key, sequence_.get()); mpms::ClearBBoxLabelIndex(key, sequence_.get());
mpms::ClearBBoxLabelConfidence(key, sequence_.get());
mpms::ClearBBoxClassString(key, sequence_.get()); mpms::ClearBBoxClassString(key, sequence_.get());
mpms::ClearBBoxClassIndex(key, sequence_.get()); mpms::ClearBBoxClassIndex(key, sequence_.get());
mpms::ClearBBoxTrackString(key, sequence_.get()); mpms::ClearBBoxTrackString(key, sequence_.get());
mpms::ClearBBoxTrackIndex(key, sequence_.get()); mpms::ClearBBoxTrackIndex(key, sequence_.get());
mpms::ClearUnmodifiedBBoxTimestamp(key, sequence_.get()); mpms::ClearUnmodifiedBBoxTimestamp(key, sequence_.get());
} }
if (absl::StartsWith(tag, kClipLabelPrefixTag)) {
const std::string& key = tag.substr(
sizeof(kClipLabelPrefixTag) / sizeof(*kClipLabelPrefixTag) - 1);
mpms::ClearClipLabelIndex(key, sequence_.get());
mpms::ClearClipLabelString(key, sequence_.get());
mpms::ClearClipLabelConfidence(key, sequence_.get());
}
if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag)) {
const std::string& key =
tag.substr(sizeof(kFloatContextFeaturePrefixTag) /
sizeof(*kFloatContextFeaturePrefixTag) -
1);
mpms::ClearContextFeatureFloats(key, sequence_.get());
}
if (absl::StartsWith(tag, kIntsContextFeaturePrefixTag)) {
const std::string& key =
tag.substr(sizeof(kIntsContextFeaturePrefixTag) /
sizeof(*kIntsContextFeaturePrefixTag) -
1);
mpms::ClearContextFeatureInts(key, sequence_.get());
}
if (absl::StartsWith(tag, kBytesContextFeaturePrefixTag)) {
const std::string& key =
tag.substr(sizeof(kBytesContextFeaturePrefixTag) /
sizeof(*kBytesContextFeaturePrefixTag) -
1);
mpms::ClearContextFeatureBytes(key, sequence_.get());
}
if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) { if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
std::string key = tag.substr(sizeof(kFloatFeaturePrefixTag) / std::string key = tag.substr(sizeof(kFloatFeaturePrefixTag) /
sizeof(*kFloatFeaturePrefixTag) - sizeof(*kFloatFeaturePrefixTag) -
@@ -418,34 +343,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
if (absl::StartsWith(tag, kImageTag) && if (absl::StartsWith(tag, kImageTag) &&
!cc->Inputs().Tag(tag).IsEmpty()) { !cc->Inputs().Tag(tag).IsEmpty()) {
std::string key = ""; std::string key = "";
if (absl::StartsWith(tag, kImageLabelPrefixTag)) {
std::string key =
std::string(absl::StripPrefix(tag, kImageLabelPrefixTag));
const auto& detection = cc->Inputs().Tag(tag).Get<Detection>();
if (detection.label().empty()) continue;
RET_CHECK(detection.label_size() == detection.score_size())
<< "Wrong image label data format: " << detection.label_size()
<< " vs " << detection.score_size();
if (!detection.label_id().empty()) {
RET_CHECK(detection.label_id_size() == detection.label_size())
<< "Wrong image label ID format: " << detection.label_id_size()
<< " vs " << detection.label_size();
}
std::vector<std::string> labels(detection.label().begin(),
detection.label().end());
std::vector<float> confidences(detection.score().begin(),
detection.score().end());
std::vector<int32_t> ids(detection.label_id().begin(),
detection.label_id().end());
if (!key.empty() || mpms::HasImageEncoded(*sequence_)) {
mpms::AddImageTimestamp(key, cc->InputTimestamp().Value(),
sequence_.get());
}
mpms::AddImageLabelString(key, labels, sequence_.get());
mpms::AddImageLabelConfidence(key, confidences, sequence_.get());
if (!ids.empty()) mpms::AddImageLabelIndex(key, ids, sequence_.get());
continue;
}
if (tag != kImageTag) { if (tag != kImageTag) {
int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1; int tag_length = sizeof(kImageTag) / sizeof(*kImageTag) - 1;
if (tag[tag_length] == '_') { if (tag[tag_length] == '_') {
@@ -496,7 +393,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
mpms::ClearBBoxNumRegions(prefix, sequence_.get()); mpms::ClearBBoxNumRegions(prefix, sequence_.get());
mpms::ClearBBoxLabelString(prefix, sequence_.get()); mpms::ClearBBoxLabelString(prefix, sequence_.get());
mpms::ClearBBoxLabelIndex(prefix, sequence_.get()); mpms::ClearBBoxLabelIndex(prefix, sequence_.get());
mpms::ClearBBoxLabelConfidence(prefix, sequence_.get());
mpms::ClearBBoxClassString(prefix, sequence_.get()); mpms::ClearBBoxClassString(prefix, sequence_.get());
mpms::ClearBBoxClassIndex(prefix, sequence_.get()); mpms::ClearBBoxClassIndex(prefix, sequence_.get());
mpms::ClearBBoxTrackString(prefix, sequence_.get()); mpms::ClearBBoxTrackString(prefix, sequence_.get());
@@ -509,33 +405,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
} }
replace_keypoints_ = false; replace_keypoints_ = false;
} }
if (absl::StartsWith(tag, kClipLabelPrefixTag) &&
!cc->Inputs().Tag(tag).IsEmpty()) {
const std::string& key = tag.substr(
sizeof(kClipLabelPrefixTag) / sizeof(*kClipLabelPrefixTag) - 1);
const Detection& detection = cc->Inputs().Tag(tag).Get<Detection>();
if (detection.label().size() != detection.score().size()) {
return absl::InvalidArgumentError(
"Different size of detection.label and detection.score");
}
// Allow empty label_ids, but if label_ids is not empty, it should have
// the same size as the label and score fields.
if (!detection.label_id().empty()) {
if (detection.label_id().size() != detection.label().size()) {
return absl::InvalidArgumentError(
"Different size of detection.label_id and detection.label");
}
}
for (int i = 0; i < detection.label().size(); ++i) {
if (!detection.label_id().empty()) {
mpms::AddClipLabelIndex(key, detection.label_id(i),
sequence_.get());
}
mpms::AddClipLabelString(key, detection.label(i), sequence_.get());
mpms::AddClipLabelConfidence(key, detection.score(i),
sequence_.get());
}
}
if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag) && if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag) &&
!cc->Inputs().Tag(tag).IsEmpty()) { !cc->Inputs().Tag(tag).IsEmpty()) {
std::string key = std::string key =
@@ -543,36 +412,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
sizeof(*kFloatContextFeaturePrefixTag) - sizeof(*kFloatContextFeaturePrefixTag) -
1); 1);
RET_CHECK_EQ(cc->InputTimestamp(), Timestamp::PostStream()); RET_CHECK_EQ(cc->InputTimestamp(), Timestamp::PostStream());
for (const auto& value : mpms::SetContextFeatureFloats(
cc->Inputs().Tag(tag).Get<std::vector<float>>()) { key, cc->Inputs().Tag(tag).Get<std::vector<float>>(),
mpms::AddContextFeatureFloats(key, value, sequence_.get()); sequence_.get());
}
}
if (absl::StartsWith(tag, kIntsContextFeaturePrefixTag) &&
!cc->Inputs().Tag(tag).IsEmpty()) {
const std::string& key =
tag.substr(sizeof(kIntsContextFeaturePrefixTag) /
sizeof(*kIntsContextFeaturePrefixTag) -
1);
// To ensure only one packet is provided for this tag.
RET_CHECK_EQ(cc->InputTimestamp(), Timestamp::PostStream());
for (const auto& value :
cc->Inputs().Tag(tag).Get<std::vector<int64_t>>()) {
mpms::AddContextFeatureInts(key, value, sequence_.get());
}
}
if (absl::StartsWith(tag, kBytesContextFeaturePrefixTag) &&
!cc->Inputs().Tag(tag).IsEmpty()) {
const std::string& key =
tag.substr(sizeof(kBytesContextFeaturePrefixTag) /
sizeof(*kBytesContextFeaturePrefixTag) -
1);
// To ensure only one packet is provided for this tag.
RET_CHECK_EQ(cc->InputTimestamp(), Timestamp::PostStream());
for (const auto& value :
cc->Inputs().Tag(tag).Get<std::vector<std::string>>()) {
mpms::AddContextFeatureBytes(key, value, sequence_.get());
}
} }
if (absl::StartsWith(tag, kFloatFeaturePrefixTag) && if (absl::StartsWith(tag, kFloatFeaturePrefixTag) &&
!cc->Inputs().Tag(tag).IsEmpty()) { !cc->Inputs().Tag(tag).IsEmpty()) {
@@ -618,7 +460,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
} }
std::vector<Location> predicted_locations; std::vector<Location> predicted_locations;
std::vector<std::string> predicted_class_strings; std::vector<std::string> predicted_class_strings;
std::vector<float> predicted_class_confidences;
std::vector<int> predicted_label_ids; std::vector<int> predicted_label_ids;
for (auto& detection : for (auto& detection :
cc->Inputs().Tag(tag).Get<std::vector<Detection>>()) { cc->Inputs().Tag(tag).Get<std::vector<Detection>>()) {
@@ -647,9 +488,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
if (detection.label_id_size() > 0) { if (detection.label_id_size() > 0) {
predicted_label_ids.push_back(detection.label_id(0)); predicted_label_ids.push_back(detection.label_id(0));
} }
if (detection.score_size() > 0) {
predicted_class_confidences.push_back(detection.score(0));
}
} }
} }
if (!predicted_locations.empty()) { if (!predicted_locations.empty()) {
@@ -663,10 +501,6 @@ class PackMediaSequenceCalculator : public CalculatorBase {
if (!predicted_label_ids.empty()) { if (!predicted_label_ids.empty()) {
mpms::AddBBoxLabelIndex(key, predicted_label_ids, sequence_.get()); mpms::AddBBoxLabelIndex(key, predicted_label_ids, sequence_.get());
} }
if (!predicted_class_confidences.empty()) {
mpms::AddBBoxLabelConfidence(key, predicted_class_confidences,
sequence_.get());
}
} }
} }
} }
@@ -714,14 +548,10 @@ class PackMediaSequenceCalculator : public CalculatorBase {
} }
} }
} }
if (clip_media_id_.has_value()) {
mpms::SetClipMediaId(*clip_media_id_, sequence_.get());
}
return absl::OkStatus(); return absl::OkStatus();
} }
std::unique_ptr<tf::SequenceExample> sequence_; std::unique_ptr<tf::SequenceExample> sequence_;
std::optional<std::string> clip_media_id_ = std::nullopt;
std::map<std::string, bool> features_present_; std::map<std::string, bool> features_present_;
bool replace_keypoints_; bool replace_keypoints_;
}; };
@@ -12,32 +12,28 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include <cstdint> #include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "absl/log/absl_check.h" #include "absl/container/flat_hash_map.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/status/status.h" #include "absl/strings/numbers.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h" #include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
#include "mediapipe/calculators/tensorflow/pack_media_sequence_calculator.pb.h" #include "mediapipe/calculators/tensorflow/pack_media_sequence_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/location.h" #include "mediapipe/framework/formats/location.h"
#include "mediapipe/framework/formats/location_opencv.h" #include "mediapipe/framework/formats/location_opencv.h"
#include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" #include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
#include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/timestamp.h" #include "mediapipe/framework/timestamp.h"
#include "mediapipe/util/sequence/media_sequence.h" #include "mediapipe/util/sequence/media_sequence.h"
#include "mediapipe/util/sequence/media_sequence_util.h"
#include "tensorflow/core/example/example.pb.h" #include "tensorflow/core/example/example.pb.h"
#include "tensorflow/core/example/feature.pb.h" #include "tensorflow/core/example/feature.pb.h"
#include "testing/base/public/gmock.h"
#include "testing/base/public/gunit.h"
namespace mediapipe { namespace mediapipe {
namespace { namespace {
@@ -59,22 +55,13 @@ constexpr char kBytesFeatureTestTag[] = "BYTES_FEATURE_TEST";
constexpr char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED"; constexpr char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
constexpr char kFloatContextFeatureOtherTag[] = "FLOAT_CONTEXT_FEATURE_OTHER"; constexpr char kFloatContextFeatureOtherTag[] = "FLOAT_CONTEXT_FEATURE_OTHER";
constexpr char kFloatContextFeatureTestTag[] = "FLOAT_CONTEXT_FEATURE_TEST"; constexpr char kFloatContextFeatureTestTag[] = "FLOAT_CONTEXT_FEATURE_TEST";
constexpr char kIntsContextFeatureTestTag[] = "INTS_CONTEXT_FEATURE_TEST";
constexpr char kIntsContextFeatureOtherTag[] = "INTS_CONTEXT_FEATURE_OTHER";
constexpr char kBytesContextFeatureTestTag[] = "BYTES_CONTEXT_FEATURE_TEST";
constexpr char kBytesContextFeatureOtherTag[] = "BYTES_CONTEXT_FEATURE_OTHER";
constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER"; constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER";
constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST"; constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST";
constexpr char kIntFeatureOtherTag[] = "INT_FEATURE_OTHER"; constexpr char kIntFeatureOtherTag[] = "INT_FEATURE_OTHER";
constexpr char kIntFeatureTestTag[] = "INT_FEATURE_TEST"; constexpr char kIntFeatureTestTag[] = "INT_FEATURE_TEST";
constexpr char kImageLabelTestTag[] = "IMAGE_LABEL_TEST";
constexpr char kImageLabelOtherTag[] = "IMAGE_LABEL_OTHER";
constexpr char kImagePrefixTag[] = "IMAGE_PREFIX"; constexpr char kImagePrefixTag[] = "IMAGE_PREFIX";
constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE"; constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
constexpr char kImageTag[] = "IMAGE"; constexpr char kImageTag[] = "IMAGE";
constexpr char kClipMediaIdTag[] = "CLIP_MEDIA_ID";
constexpr char kClipLabelTestTag[] = "CLIP_LABEL_TEST";
constexpr char kClipLabelOtherTag[] = "CLIP_LABEL_OTHER";
class PackMediaSequenceCalculatorTest : public ::testing::Test { class PackMediaSequenceCalculatorTest : public ::testing::Test {
protected: protected:
@@ -82,14 +69,10 @@ class PackMediaSequenceCalculatorTest : public ::testing::Test {
const tf::Features& features, const tf::Features& features,
const bool output_only_if_all_present, const bool output_only_if_all_present,
const bool replace_instead_of_append, const bool replace_instead_of_append,
const bool output_as_zero_timestamp = false, const bool output_as_zero_timestamp = false) {
const std::vector<std::string>& input_side_packets = {
"SEQUENCE_EXAMPLE:input_sequence"}) {
CalculatorGraphConfig::Node config; CalculatorGraphConfig::Node config;
config.set_calculator("PackMediaSequenceCalculator"); config.set_calculator("PackMediaSequenceCalculator");
for (const std::string& side_packet : input_side_packets) { config.add_input_side_packet("SEQUENCE_EXAMPLE:input_sequence");
config.add_input_side_packet(side_packet);
}
config.add_output_stream("SEQUENCE_EXAMPLE:output_sequence"); config.add_output_stream("SEQUENCE_EXAMPLE:output_sequence");
for (const std::string& stream : input_streams) { for (const std::string& stream : input_streams) {
config.add_input_stream(stream); config.add_input_stream(stream);
@@ -113,8 +96,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImages) {
mpms::SetClipMediaId(test_video_id, input_sequence.get()); mpms::SetClipMediaId(test_video_id, input_sequence.get());
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
OpenCvImageEncoderCalculatorResults encoded_image; OpenCvImageEncoderCalculatorResults encoded_image;
encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_encoded_image(bytes.data(), bytes.size());
encoded_image.set_width(2); encoded_image.set_width(2);
@@ -157,8 +139,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoPrefixedImages) {
mpms::SetClipMediaId(test_video_id, input_sequence.get()); mpms::SetClipMediaId(test_video_id, input_sequence.get());
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
OpenCvImageEncoderCalculatorResults encoded_image; OpenCvImageEncoderCalculatorResults encoded_image;
encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_encoded_image(bytes.data(), bytes.size());
encoded_image.set_width(2); encoded_image.set_width(2);
@@ -331,76 +312,6 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBytesLists) {
} }
} }
TEST_F(PackMediaSequenceCalculatorTest, PacksTwoImageLabels) {
SetUpCalculator(
{"IMAGE_LABEL_TEST:test_labels", "IMAGE_LABEL_OTHER:test_labels2"}, {},
false, true);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
int num_timesteps = 2;
for (int i = 0; i < num_timesteps; ++i) {
Detection detection1;
detection1.add_label(absl::StrCat("foo", 2 << i));
detection1.add_label_id(i);
detection1.add_score(0.1 * i);
detection1.add_label(absl::StrCat("foo", 2 << i));
detection1.add_label_id(i);
detection1.add_score(0.1 * i);
auto label_ptr1 = ::absl::make_unique<Detection>(detection1);
runner_->MutableInputs()
->Tag(kImageLabelTestTag)
.packets.push_back(Adopt(label_ptr1.release()).At(Timestamp(i)));
Detection detection2;
detection2.add_label(absl::StrCat("bar", 2 << i));
detection2.add_score(0.2 * i);
detection2.add_label(absl::StrCat("bar", 2 << i));
detection2.add_score(0.2 * i);
auto label_ptr2 = ::absl::make_unique<Detection>(detection2);
runner_->MutableInputs()
->Tag(kImageLabelOtherTag)
.packets.push_back(Adopt(label_ptr2.release()).At(Timestamp(i)));
}
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_EQ(num_timesteps,
mpms::GetImageTimestampSize("TEST", output_sequence));
ASSERT_EQ(num_timesteps,
mpms::GetImageLabelStringSize("TEST", output_sequence));
ASSERT_EQ(num_timesteps,
mpms::GetImageLabelConfidenceSize("TEST", output_sequence));
ASSERT_EQ(num_timesteps,
mpms::GetImageTimestampSize("OTHER", output_sequence));
ASSERT_EQ(num_timesteps,
mpms::GetImageLabelStringSize("OTHER", output_sequence));
ASSERT_EQ(num_timesteps,
mpms::GetImageLabelConfidenceSize("OTHER", output_sequence));
for (int i = 0; i < num_timesteps; ++i) {
ASSERT_EQ(i, mpms::GetImageTimestampAt("TEST", output_sequence, i));
ASSERT_THAT(mpms::GetImageLabelStringAt("TEST", output_sequence, i),
::testing::ElementsAreArray(
std::vector<std::string>(2, absl::StrCat("foo", 2 << i))));
ASSERT_THAT(mpms::GetImageLabelIndexAt("TEST", output_sequence, i),
::testing::ElementsAreArray(std::vector<int32_t>(2, i)));
ASSERT_THAT(mpms::GetImageLabelConfidenceAt("TEST", output_sequence, i),
::testing::ElementsAreArray(std::vector<float>(2, 0.1 * i)));
ASSERT_EQ(i, mpms::GetImageTimestampAt("OTHER", output_sequence, i));
ASSERT_THAT(mpms::GetImageLabelStringAt("OTHER", output_sequence, i),
::testing::ElementsAreArray(
std::vector<std::string>(2, absl::StrCat("bar", 2 << i))));
ASSERT_THAT(mpms::GetImageLabelConfidenceAt("OTHER", output_sequence, i),
::testing::ElementsAreArray(std::vector<float>(2, 0.2 * i)));
}
}
TEST_F(PackMediaSequenceCalculatorTest, OutputAsZeroTimestamp) { TEST_F(PackMediaSequenceCalculatorTest, OutputAsZeroTimestamp) {
SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true, true); SetUpCalculator({"FLOAT_FEATURE_TEST:test"}, {}, false, true, true);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>(); auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
@@ -456,315 +367,6 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) {
testing::ElementsAre(4, 4)); testing::ElementsAre(4, 4));
} }
TEST_F(PackMediaSequenceCalculatorTest, ReplaceTwoContextFloatLists) {
SetUpCalculator(
/*input_streams=*/{"FLOAT_CONTEXT_FEATURE_TEST:test",
"FLOAT_CONTEXT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false, /*replace_instead_of_append=*/true);
auto input_sequence = std::make_unique<tf::SequenceExample>();
mpms::SetContextFeatureFloats("TEST", {2, 3}, input_sequence.get());
mpms::SetContextFeatureFloats("OTHER", {2, 4}, input_sequence.get());
const std::vector<float> vf_1 = {5, 6};
runner_->MutableInputs()
->Tag(kFloatContextFeatureTestTag)
.packets.push_back(
MakePacket<std::vector<float>>(vf_1).At(Timestamp::PostStream()));
const std::vector<float> vf_2 = {7, 8};
runner_->MutableInputs()
->Tag(kFloatContextFeatureOtherTag)
.packets.push_back(
MakePacket<std::vector<float>>(vf_2).At(Timestamp::PostStream()));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetContextFeatureFloats("TEST", output_sequence),
testing::ElementsAre(5, 6));
ASSERT_THAT(mpms::GetContextFeatureFloats("OTHER", output_sequence),
testing::ElementsAre(7, 8));
}
TEST_F(PackMediaSequenceCalculatorTest, AppendTwoContextFloatLists) {
SetUpCalculator(
/*input_streams=*/{"FLOAT_CONTEXT_FEATURE_TEST:test",
"FLOAT_CONTEXT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/false);
auto input_sequence = std::make_unique<tf::SequenceExample>();
mpms::SetContextFeatureFloats("TEST", {2, 3}, input_sequence.get());
mpms::SetContextFeatureFloats("OTHER", {2, 4}, input_sequence.get());
const std::vector<float> vf_1 = {5, 6};
runner_->MutableInputs()
->Tag(kFloatContextFeatureTestTag)
.packets.push_back(
MakePacket<std::vector<float>>(vf_1).At(Timestamp::PostStream()));
const std::vector<float> vf_2 = {7, 8};
runner_->MutableInputs()
->Tag(kFloatContextFeatureOtherTag)
.packets.push_back(
MakePacket<std::vector<float>>(vf_2).At(Timestamp::PostStream()));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
EXPECT_THAT(mpms::GetContextFeatureFloats("TEST", output_sequence),
testing::ElementsAre(2, 3, 5, 6));
EXPECT_THAT(mpms::GetContextFeatureFloats("OTHER", output_sequence),
testing::ElementsAre(2, 4, 7, 8));
}
TEST_F(PackMediaSequenceCalculatorTest, PackTwoContextIntLists) {
SetUpCalculator(
/*input_streams=*/{"INTS_CONTEXT_FEATURE_TEST:test",
"INTS_CONTEXT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false, /*replace_instead_of_append=*/true);
auto input_sequence = absl::make_unique<tf::SequenceExample>();
const std::vector<int64_t> vi_1 = {2, 3};
runner_->MutableInputs()
->Tag(kIntsContextFeatureTestTag)
.packets.push_back(
MakePacket<std::vector<int64_t>>(vi_1).At(Timestamp::PostStream()));
const std::vector<int64_t> vi_2 = {2, 4};
runner_->MutableInputs()
->Tag(kIntsContextFeatureOtherTag)
.packets.push_back(
MakePacket<std::vector<int64_t>>(vi_2).At(Timestamp::PostStream()));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetContextFeatureInts("TEST", output_sequence),
testing::ElementsAre(2, 3));
ASSERT_THAT(mpms::GetContextFeatureInts("OTHER", output_sequence),
testing::ElementsAre(2, 4));
}
TEST_F(PackMediaSequenceCalculatorTest, ReplaceTwoContextIntLists) {
SetUpCalculator(
/*input_streams=*/{"INTS_CONTEXT_FEATURE_TEST:test",
"INTS_CONTEXT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false, /*replace_instead_of_append=*/true);
auto input_sequence = absl::make_unique<tf::SequenceExample>();
mpms::SetContextFeatureInts("TEST", {2, 3}, input_sequence.get());
mpms::SetContextFeatureInts("OTHER", {2, 4}, input_sequence.get());
const std::vector<int64_t> vi_1 = {5, 6};
runner_->MutableInputs()
->Tag(kIntsContextFeatureTestTag)
.packets.push_back(
MakePacket<std::vector<int64_t>>(vi_1).At(Timestamp::PostStream()));
const std::vector<int64_t> vi_2 = {7, 8};
runner_->MutableInputs()
->Tag(kIntsContextFeatureOtherTag)
.packets.push_back(
MakePacket<std::vector<int64_t>>(vi_2).At(Timestamp::PostStream()));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetContextFeatureInts("TEST", output_sequence),
testing::ElementsAre(5, 6));
ASSERT_THAT(mpms::GetContextFeatureInts("OTHER", output_sequence),
testing::ElementsAre(7, 8));
}
TEST_F(PackMediaSequenceCalculatorTest, AppendTwoContextIntLists) {
SetUpCalculator(
/*input_streams=*/{"INTS_CONTEXT_FEATURE_TEST:test",
"INTS_CONTEXT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/false);
auto input_sequence = absl::make_unique<tf::SequenceExample>();
mpms::SetContextFeatureInts("TEST", {2, 3}, input_sequence.get());
mpms::SetContextFeatureInts("OTHER", {2, 4}, input_sequence.get());
const std::vector<int64_t> vi_1 = {5, 6};
runner_->MutableInputs()
->Tag(kIntsContextFeatureTestTag)
.packets.push_back(
MakePacket<std::vector<int64_t>>(vi_1).At(Timestamp::PostStream()));
const std::vector<int64_t> vi_2 = {7, 8};
runner_->MutableInputs()
->Tag(kIntsContextFeatureOtherTag)
.packets.push_back(
MakePacket<std::vector<int64_t>>(vi_2).At(Timestamp::PostStream()));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetContextFeatureInts("TEST", output_sequence),
testing::ElementsAre(2, 3, 5, 6));
ASSERT_THAT(mpms::GetContextFeatureInts("OTHER", output_sequence),
testing::ElementsAre(2, 4, 7, 8));
}
TEST_F(PackMediaSequenceCalculatorTest, PackTwoContextByteLists) {
SetUpCalculator(
/*input_streams=*/{"BYTES_CONTEXT_FEATURE_TEST:test",
"BYTES_CONTEXT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false, /*replace_instead_of_append=*/true);
auto input_sequence = absl::make_unique<tf::SequenceExample>();
const std::vector<std::string> vb_1 = {"value_1", "value_2"};
runner_->MutableInputs()
->Tag(kBytesContextFeatureTestTag)
.packets.push_back(MakePacket<std::vector<std::string>>(vb_1).At(
Timestamp::PostStream()));
const std::vector<std::string> vb_2 = {"value_3", "value_4"};
runner_->MutableInputs()
->Tag(kBytesContextFeatureOtherTag)
.packets.push_back(MakePacket<std::vector<std::string>>(vb_2).At(
Timestamp::PostStream()));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetContextFeatureBytes("TEST", output_sequence),
testing::ElementsAre("value_1", "value_2"));
ASSERT_THAT(mpms::GetContextFeatureBytes("OTHER", output_sequence),
testing::ElementsAre("value_3", "value_4"));
}
TEST_F(PackMediaSequenceCalculatorTest, ReplaceTwoContextByteLists) {
SetUpCalculator(
/*input_streams=*/{"BYTES_CONTEXT_FEATURE_TEST:test",
"BYTES_CONTEXT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false, /*replace_instead_of_append=*/true);
auto input_sequence = absl::make_unique<tf::SequenceExample>();
mpms::SetContextFeatureBytes("TEST", {"existing_value_1", "existing_value_2"},
input_sequence.get());
mpms::SetContextFeatureBytes(
"OTHER", {"existing_value_3", "existing_value_4"}, input_sequence.get());
const std::vector<std::string> vb_1 = {"value_1", "value_2"};
runner_->MutableInputs()
->Tag(kBytesContextFeatureTestTag)
.packets.push_back(MakePacket<std::vector<std::string>>(vb_1).At(
Timestamp::PostStream()));
const std::vector<std::string> vb_2 = {"value_3", "value_4"};
runner_->MutableInputs()
->Tag(kBytesContextFeatureOtherTag)
.packets.push_back(MakePacket<std::vector<std::string>>(vb_2).At(
Timestamp::PostStream()));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetContextFeatureBytes("TEST", output_sequence),
testing::ElementsAre("value_1", "value_2"));
ASSERT_THAT(mpms::GetContextFeatureBytes("OTHER", output_sequence),
testing::ElementsAre("value_3", "value_4"));
}
TEST_F(PackMediaSequenceCalculatorTest, AppendTwoContextByteLists) {
SetUpCalculator(
/*input_streams=*/{"BYTES_CONTEXT_FEATURE_TEST:test",
"BYTES_CONTEXT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/false);
auto input_sequence = absl::make_unique<tf::SequenceExample>();
mpms::SetContextFeatureBytes("TEST", {"existing_value_1", "existing_value_2"},
input_sequence.get());
mpms::SetContextFeatureBytes(
"OTHER", {"existing_value_3", "existing_value_4"}, input_sequence.get());
const std::vector<std::string> vb_1 = {"value_1", "value_2"};
runner_->MutableInputs()
->Tag(kBytesContextFeatureTestTag)
.packets.push_back(MakePacket<std::vector<std::string>>(vb_1).At(
Timestamp::PostStream()));
const std::vector<std::string> vb_2 = {"value_3", "value_4"};
runner_->MutableInputs()
->Tag(kBytesContextFeatureOtherTag)
.packets.push_back(MakePacket<std::vector<std::string>>(vb_2).At(
Timestamp::PostStream()));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetContextFeatureBytes("TEST", output_sequence),
testing::ElementsAre("existing_value_1", "existing_value_2",
"value_1", "value_2"));
ASSERT_THAT(mpms::GetContextFeatureBytes("OTHER", output_sequence),
testing::ElementsAre("existing_value_3", "existing_value_4",
"value_3", "value_4"));
}
TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) { TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
tf::Features context; tf::Features context;
(*context.mutable_feature())["TEST"].mutable_bytes_list()->add_value("YES"); (*context.mutable_feature())["TEST"].mutable_bytes_list()->add_value("YES");
@@ -776,8 +378,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksAdditionalContext) {
Adopt(input_sequence.release()); Adopt(input_sequence.release());
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
OpenCvImageEncoderCalculatorResults encoded_image; OpenCvImageEncoderCalculatorResults encoded_image;
encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_encoded_image(bytes.data(), bytes.size());
auto image_ptr = auto image_ptr =
@@ -809,8 +410,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoForwardFlowEncodeds) {
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
std::string test_flow_string(bytes.begin(), bytes.end()); std::string test_flow_string(bytes.begin(), bytes.end());
OpenCvImageEncoderCalculatorResults encoded_flow; OpenCvImageEncoderCalculatorResults encoded_flow;
encoded_flow.set_encoded_image(test_flow_string); encoded_flow.set_encoded_image(test_flow_string);
@@ -926,10 +526,6 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBBoxDetections) {
auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i); auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i);
ASSERT_EQ(0, class_indices[0]); ASSERT_EQ(0, class_indices[0]);
ASSERT_EQ(1, class_indices[1]); ASSERT_EQ(1, class_indices[1]);
auto class_scores =
mpms::GetPredictedBBoxLabelConfidenceAt(output_sequence, i);
ASSERT_FLOAT_EQ(0.5, class_scores[0]);
ASSERT_FLOAT_EQ(0.75, class_scores[1]);
} }
} }
@@ -1022,8 +618,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
} }
cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(height, width, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
OpenCvImageEncoderCalculatorResults encoded_image; OpenCvImageEncoderCalculatorResults encoded_image;
encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_encoded_image(bytes.data(), bytes.size());
encoded_image.set_width(width); encoded_image.set_width(width);
@@ -1072,10 +667,6 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i); auto class_indices = mpms::GetPredictedBBoxLabelIndexAt(output_sequence, i);
ASSERT_EQ(0, class_indices[0]); ASSERT_EQ(0, class_indices[0]);
ASSERT_EQ(1, class_indices[1]); ASSERT_EQ(1, class_indices[1]);
auto class_scores =
mpms::GetPredictedBBoxLabelConfidenceAt(output_sequence, i);
ASSERT_FLOAT_EQ(0.5, class_scores[0]);
ASSERT_FLOAT_EQ(0.75, class_scores[1]);
} }
} }
@@ -1166,365 +757,6 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoMaskDetections) {
testing::ElementsAreArray(::std::vector<std::string>({"mask"}))); testing::ElementsAreArray(::std::vector<std::string>({"mask"})));
} }
TEST_F(PackMediaSequenceCalculatorTest, PackTwoClipLabels) {
SetUpCalculator(
/*input_streams=*/{"CLIP_LABEL_TEST:test", "CLIP_LABEL_OTHER:test2"},
/*features=*/{}, /*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/true);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
Detection detection_1;
detection_1.add_label("label_1");
detection_1.add_label("label_2");
detection_1.add_label_id(1);
detection_1.add_label_id(2);
detection_1.add_score(0.1);
detection_1.add_score(0.2);
runner_->MutableInputs()
->Tag(kClipLabelTestTag)
.packets.push_back(MakePacket<Detection>(detection_1).At(Timestamp(1)));
// No label ID for detection_2.
Detection detection_2;
detection_2.add_label("label_3");
detection_2.add_label("label_4");
detection_2.add_score(0.3);
detection_2.add_score(0.4);
runner_->MutableInputs()
->Tag(kClipLabelOtherTag)
.packets.push_back(MakePacket<Detection>(detection_2).At(Timestamp(2)));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetClipLabelString("TEST", output_sequence),
testing::ElementsAre("label_1", "label_2"));
ASSERT_THAT(mpms::GetClipLabelIndex("TEST", output_sequence),
testing::ElementsAre(1, 2));
ASSERT_THAT(mpms::GetClipLabelConfidence("TEST", output_sequence),
testing::ElementsAre(0.1, 0.2));
ASSERT_THAT(mpms::GetClipLabelString("OTHER", output_sequence),
testing::ElementsAre("label_3", "label_4"));
ASSERT_FALSE(mpms::HasClipLabelIndex("OTHER", output_sequence));
ASSERT_THAT(mpms::GetClipLabelConfidence("OTHER", output_sequence),
testing::ElementsAre(0.3, 0.4));
}
TEST_F(PackMediaSequenceCalculatorTest,
PackTwoClipLabels_DifferentLabelScoreSize) {
SetUpCalculator(
/*input_streams=*/{"CLIP_LABEL_TEST:test", "CLIP_LABEL_OTHER:test2"},
/*features=*/{}, /*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/true);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
// 2 labels and 1 score in detection_1.
Detection detection_1;
detection_1.add_label("label_1");
detection_1.add_label("label_2");
detection_1.add_score(0.1);
runner_->MutableInputs()
->Tag(kClipLabelTestTag)
.packets.push_back(MakePacket<Detection>(detection_1).At(Timestamp(1)));
Detection detection_2;
detection_2.add_label("label_3");
detection_2.add_label("label_4");
detection_2.add_score(0.3);
detection_2.add_score(0.4);
runner_->MutableInputs()
->Tag(kClipLabelOtherTag)
.packets.push_back(MakePacket<Detection>(detection_2).At(Timestamp(2)));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
ASSERT_THAT(
runner_->Run(),
testing::status::StatusIs(
absl::StatusCode::kInvalidArgument,
testing::HasSubstr(
"Different size of detection.label and detection.score")));
}
TEST_F(PackMediaSequenceCalculatorTest,
PackTwoClipLabels_DifferentLabelIdSize) {
SetUpCalculator(
/*input_streams=*/{"CLIP_LABEL_TEST:test", "CLIP_LABEL_OTHER:test2"},
/*features=*/{}, /*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/true);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
// 2 labels and 1 label_id in detection_1.
Detection detection_1;
detection_1.add_label("label_1");
detection_1.add_label("label_2");
detection_1.add_label_id(1);
detection_1.add_score(0.1);
detection_1.add_score(0.2);
runner_->MutableInputs()
->Tag(kClipLabelTestTag)
.packets.push_back(MakePacket<Detection>(detection_1).At(Timestamp(1)));
Detection detection_2;
detection_2.add_label("label_3");
detection_2.add_label("label_4");
detection_2.add_score(0.3);
detection_2.add_score(0.4);
runner_->MutableInputs()
->Tag(kClipLabelOtherTag)
.packets.push_back(MakePacket<Detection>(detection_2).At(Timestamp(2)));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
ASSERT_THAT(
runner_->Run(),
testing::status::StatusIs(
absl::StatusCode::kInvalidArgument,
testing::HasSubstr(
"Different size of detection.label_id and detection.label")));
}
TEST_F(PackMediaSequenceCalculatorTest, ReplaceTwoClipLabels) {
// Replace existing clip/label/string and clip/label/confidence values for
// the prefixes.
SetUpCalculator(
/*input_streams=*/{"CLIP_LABEL_TEST:test", "CLIP_LABEL_OTHER:test2"},
/*features=*/{}, /*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/true);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
mpms::SetClipLabelString("TEST", {"old_label_1", "old_label_2"},
input_sequence.get());
mpms::SetClipLabelConfidence("TEST", {0.1, 0.2}, input_sequence.get());
mpms::SetClipLabelString("OTHER", {"old_label_3", "old_label_4"},
input_sequence.get());
mpms::SetClipLabelConfidence("OTHER", {0.3, 0.4}, input_sequence.get());
Detection detection_1;
detection_1.add_label("label_1");
detection_1.add_label("label_2");
detection_1.add_label_id(1);
detection_1.add_label_id(2);
detection_1.add_score(0.9);
detection_1.add_score(0.8);
runner_->MutableInputs()
->Tag(kClipLabelTestTag)
.packets.push_back(MakePacket<Detection>(detection_1).At(Timestamp(1)));
Detection detection_2;
detection_2.add_label("label_3");
detection_2.add_label("label_4");
detection_2.add_label_id(3);
detection_2.add_label_id(4);
detection_2.add_score(0.7);
detection_2.add_score(0.6);
runner_->MutableInputs()
->Tag(kClipLabelOtherTag)
.packets.push_back(MakePacket<Detection>(detection_2).At(Timestamp(2)));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(mpms::GetClipLabelString("TEST", output_sequence),
testing::ElementsAre("label_1", "label_2"));
ASSERT_THAT(mpms::GetClipLabelIndex("TEST", output_sequence),
testing::ElementsAre(1, 2));
ASSERT_THAT(mpms::GetClipLabelConfidence("TEST", output_sequence),
testing::ElementsAre(0.9, 0.8));
ASSERT_THAT(mpms::GetClipLabelString("OTHER", output_sequence),
testing::ElementsAre("label_3", "label_4"));
ASSERT_THAT(mpms::GetClipLabelIndex("OTHER", output_sequence),
testing::ElementsAre(3, 4));
ASSERT_THAT(mpms::GetClipLabelConfidence("OTHER", output_sequence),
testing::ElementsAre(0.7, 0.6));
}
TEST_F(PackMediaSequenceCalculatorTest, AppendTwoClipLabels) {
// Append to the existing clip/label/string and clip/label/confidence values
// for the prefixes.
SetUpCalculator(
/*input_streams=*/{"CLIP_LABEL_TEST:test", "CLIP_LABEL_OTHER:test2"},
/*features=*/{}, /*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/false);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
mpms::SetClipLabelString("TEST", {"old_label_1", "old_label_2"},
input_sequence.get());
mpms::SetClipLabelIndex("TEST", {1, 2}, input_sequence.get());
mpms::SetClipLabelConfidence("TEST", {0.1, 0.2}, input_sequence.get());
mpms::SetClipLabelString("OTHER", {"old_label_3", "old_label_4"},
input_sequence.get());
mpms::SetClipLabelIndex("OTHER", {3, 4}, input_sequence.get());
mpms::SetClipLabelConfidence("OTHER", {0.3, 0.4}, input_sequence.get());
Detection detection_1;
detection_1.add_label("label_1");
detection_1.add_label("label_2");
detection_1.add_label_id(9);
detection_1.add_label_id(8);
detection_1.add_score(0.9);
detection_1.add_score(0.8);
runner_->MutableInputs()
->Tag(kClipLabelTestTag)
.packets.push_back(MakePacket<Detection>(detection_1).At(Timestamp(1)));
Detection detection_2;
detection_2.add_label("label_3");
detection_2.add_label("label_4");
detection_2.add_label_id(7);
detection_2.add_label_id(6);
detection_2.add_score(0.7);
detection_2.add_score(0.6);
runner_->MutableInputs()
->Tag(kClipLabelOtherTag)
.packets.push_back(MakePacket<Detection>(detection_2).At(Timestamp(2)));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_THAT(
mpms::GetClipLabelString("TEST", output_sequence),
testing::ElementsAre("old_label_1", "old_label_2", "label_1", "label_2"));
ASSERT_THAT(mpms::GetClipLabelIndex("TEST", output_sequence),
testing::ElementsAre(1, 2, 9, 8));
ASSERT_THAT(mpms::GetClipLabelConfidence("TEST", output_sequence),
testing::ElementsAre(0.1, 0.2, 0.9, 0.8));
ASSERT_THAT(
mpms::GetClipLabelString("OTHER", output_sequence),
testing::ElementsAre("old_label_3", "old_label_4", "label_3", "label_4"));
ASSERT_THAT(mpms::GetClipLabelIndex("OTHER", output_sequence),
testing::ElementsAre(3, 4, 7, 6));
ASSERT_THAT(mpms::GetClipLabelConfidence("OTHER", output_sequence),
testing::ElementsAre(0.3, 0.4, 0.7, 0.6));
}
TEST_F(PackMediaSequenceCalculatorTest,
DifferentClipLabelScoreAndConfidenceSize) {
SetUpCalculator(
/*input_streams=*/{"CLIP_LABEL_TEST:test", "CLIP_LABEL_OTHER:test2"},
/*features=*/{}, /*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/true);
auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
Detection detection_1;
// 2 labels and 1 score.
detection_1.add_label("label_1");
detection_1.add_label("label_2");
detection_1.add_score(0.1);
runner_->MutableInputs()
->Tag(kClipLabelTestTag)
.packets.push_back(MakePacket<Detection>(detection_1).At(Timestamp(1)));
Detection detection_2;
detection_2.add_label("label_3");
detection_2.add_label("label_4");
detection_2.add_score(0.3);
detection_2.add_score(0.4);
runner_->MutableInputs()
->Tag(kClipLabelOtherTag)
.packets.push_back(MakePacket<Detection>(detection_2).At(Timestamp(2)));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
ASSERT_THAT(runner_->Run(),
testing::status::StatusIs(absl::StatusCode::kInvalidArgument));
}
TEST_F(PackMediaSequenceCalculatorTest, AddClipMediaId) {
SetUpCalculator(
/*input_streams=*/{"FLOAT_FEATURE_TEST:test",
"FLOAT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/true,
/*output_as_zero_timestamp=*/false, /*input_side_packets=*/
{"SEQUENCE_EXAMPLE:input_sequence", "CLIP_MEDIA_ID:video_id"});
auto input_sequence = absl::make_unique<tf::SequenceExample>();
const std::string test_video_id = "test_video_id";
int num_timesteps = 2;
for (int i = 0; i < num_timesteps; ++i) {
auto vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
runner_->MutableInputs()
->Tag(kFloatFeatureTestTag)
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
runner_->MutableInputs()
->Tag(kFloatFeatureOtherTag)
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
}
runner_->MutableSidePackets()->Tag(kClipMediaIdTag) =
MakePacket<std::string>(test_video_id);
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence));
}
TEST_F(PackMediaSequenceCalculatorTest, ReplaceClipMediaId) {
SetUpCalculator(
/*input_streams=*/{"FLOAT_FEATURE_TEST:test",
"FLOAT_FEATURE_OTHER:test2"},
/*features=*/{},
/*output_only_if_all_present=*/false,
/*replace_instead_of_append=*/true,
/*output_as_zero_timestamp=*/false, /*input_side_packets=*/
{"SEQUENCE_EXAMPLE:input_sequence", "CLIP_MEDIA_ID:video_id"});
auto input_sequence = absl::make_unique<tf::SequenceExample>();
const std::string existing_video_id = "existing_video_id";
mpms::SetClipMediaId(existing_video_id, input_sequence.get());
const std::string test_video_id = "test_video_id";
int num_timesteps = 2;
for (int i = 0; i < num_timesteps; ++i) {
auto vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
runner_->MutableInputs()
->Tag(kFloatFeatureTestTag)
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
vf_ptr = ::absl::make_unique<std::vector<float>>(2, 2 << i);
runner_->MutableInputs()
->Tag(kFloatFeatureOtherTag)
.packets.push_back(Adopt(vf_ptr.release()).At(Timestamp(i)));
}
runner_->MutableSidePackets()->Tag(kClipMediaIdTag) =
MakePacket<std::string>(test_video_id).At(Timestamp(0));
runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
Adopt(input_sequence.release());
MP_ASSERT_OK(runner_->Run());
const std::vector<Packet>& output_packets =
runner_->Outputs().Tag(kSequenceExampleTag).packets;
ASSERT_EQ(1, output_packets.size());
const tf::SequenceExample& output_sequence =
output_packets[0].Get<tf::SequenceExample>();
ASSERT_EQ(test_video_id, mpms::GetClipMediaId(output_sequence));
}
TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) { TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) {
SetUpCalculator( SetUpCalculator(
{"FORWARD_FLOW_ENCODED:flow", "FLOAT_FEATURE_I3D_FLOW:feature"}, {}, {"FORWARD_FLOW_ENCODED:flow", "FLOAT_FEATURE_I3D_FLOW:feature"}, {},
@@ -1535,8 +767,7 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamOK) {
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
std::string test_flow_string(bytes.begin(), bytes.end()); std::string test_flow_string(bytes.begin(), bytes.end());
OpenCvImageEncoderCalculatorResults encoded_flow; OpenCvImageEncoderCalculatorResults encoded_flow;
encoded_flow.set_encoded_image(test_flow_string); encoded_flow.set_encoded_image(test_flow_string);
@@ -1582,8 +813,7 @@ TEST_F(PackMediaSequenceCalculatorTest, MissingStreamNotOK) {
mpms::SetClipMediaId(test_video_id, input_sequence.get()); mpms::SetClipMediaId(test_video_id, input_sequence.get());
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
std::string test_flow_string(bytes.begin(), bytes.end()); std::string test_flow_string(bytes.begin(), bytes.end());
OpenCvImageEncoderCalculatorResults encoded_flow; OpenCvImageEncoderCalculatorResults encoded_flow;
encoded_flow.set_encoded_image(test_flow_string); encoded_flow.set_encoded_image(test_flow_string);
@@ -1740,8 +970,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestReconcilingAnnotations) {
auto input_sequence = ::absl::make_unique<tf::SequenceExample>(); auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
OpenCvImageEncoderCalculatorResults encoded_image; OpenCvImageEncoderCalculatorResults encoded_image;
encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_encoded_image(bytes.data(), bytes.size());
encoded_image.set_width(2); encoded_image.set_width(2);
@@ -1792,8 +1021,7 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
auto input_sequence = ::absl::make_unique<tf::SequenceExample>(); auto input_sequence = ::absl::make_unique<tf::SequenceExample>();
cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255)); cv::Mat image(2, 3, CV_8UC3, cv::Scalar(0, 0, 255));
std::vector<uchar> bytes; std::vector<uchar> bytes;
ASSERT_TRUE( ASSERT_TRUE(cv::imencode(".jpg", image, bytes, {80}));
cv::imencode(".jpg", image, bytes, {cv::IMWRITE_HDR_COMPRESSION, 1}));
OpenCvImageEncoderCalculatorResults encoded_image; OpenCvImageEncoderCalculatorResults encoded_image;
encoded_image.set_encoded_image(bytes.data(), bytes.size()); encoded_image.set_encoded_image(bytes.data(), bytes.size());
int height = 2; int height = 2;
@@ -1829,7 +1057,6 @@ TEST_F(PackMediaSequenceCalculatorTest, TestOverwritingAndReconciling) {
mpms::AddBBoxNumRegions(-1, input_sequence.get()); mpms::AddBBoxNumRegions(-1, input_sequence.get());
mpms::AddBBoxLabelString({"anything"}, input_sequence.get()); mpms::AddBBoxLabelString({"anything"}, input_sequence.get());
mpms::AddBBoxLabelIndex({-1}, input_sequence.get()); mpms::AddBBoxLabelIndex({-1}, input_sequence.get());
mpms::AddBBoxLabelConfidence({-1}, input_sequence.get());
mpms::AddBBoxClassString({"anything"}, input_sequence.get()); mpms::AddBBoxClassString({"anything"}, input_sequence.get());
mpms::AddBBoxClassIndex({-1}, input_sequence.get()); mpms::AddBBoxClassIndex({-1}, input_sequence.get());
mpms::AddBBoxTrackString({"anything"}, input_sequence.get()); mpms::AddBBoxTrackString({"anything"}, input_sequence.get());
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.pb.h" #include "mediapipe/calculators/tensorflow/tensor_squeeze_dimensions_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
@@ -100,11 +99,10 @@ class TensorSqueezeDimensionsCalculator : public CalculatorBase {
} }
} }
if (remove_dims_.empty()) { if (remove_dims_.empty()) {
ABSL_LOG(ERROR) LOG(ERROR) << "TensorSqueezeDimensionsCalculator is squeezing input with "
<< "TensorSqueezeDimensionsCalculator is squeezing input with " "no single-dimensions. Calculator will be a no-op.";
"no single-dimensions. Calculator will be a no-op."; LOG(ERROR) << "Input to TensorSqueezeDimensionsCalculator has shape "
ABSL_LOG(ERROR) << "Input to TensorSqueezeDimensionsCalculator has shape " << tensor_shape.DebugString();
<< tensor_shape.DebugString();
} }
} }
}; };
@@ -14,7 +14,6 @@
#include <iostream> #include <iostream>
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h" #include "mediapipe/calculators/tensorflow/tensor_to_image_frame_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
@@ -100,7 +99,7 @@ absl::Status TensorToImageFrameCalculator::Process(CalculatorContext* cc) {
const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get<tf::Tensor>(); const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get<tf::Tensor>();
int32_t depth = 1; int32_t depth = 1;
if (input_tensor.dims() != 2) { // Depth is 1 for 2D tensors. if (input_tensor.dims() != 2) { // Depth is 1 for 2D tensors.
ABSL_CHECK(3 == input_tensor.dims()) CHECK(3 == input_tensor.dims())
<< "Only 2 or 3-D Tensors can be converted to frames. Instead got: " << "Only 2 or 3-D Tensors can be converted to frames. Instead got: "
<< input_tensor.dims(); << input_tensor.dims();
depth = input_tensor.dim_size(2); depth = input_tensor.dim_size(2);
@@ -15,7 +15,6 @@
// Calculator converts from one-dimensional Tensor of DT_FLOAT to Matrix // Calculator converts from one-dimensional Tensor of DT_FLOAT to Matrix
// OR from (batched) two-dimensional Tensor of DT_FLOAT to Matrix. // OR from (batched) two-dimensional Tensor of DT_FLOAT to Matrix.
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.pb.h" #include "mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/matrix.h"
@@ -37,7 +36,7 @@ constexpr char kReference[] = "REFERENCE";
absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
TimeSeriesHeader* header) { TimeSeriesHeader* header) {
ABSL_CHECK(header); CHECK(header);
if (header_packet.IsEmpty()) { if (header_packet.IsEmpty()) {
return absl::UnknownError("No header found."); return absl::UnknownError("No header found.");
} }
@@ -192,7 +191,7 @@ absl::Status TensorToMatrixCalculator::Process(CalculatorContext* cc) {
<< "Tensor stream packet does not contain a Tensor."; << "Tensor stream packet does not contain a Tensor.";
const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get<tf::Tensor>(); const tf::Tensor& input_tensor = cc->Inputs().Tag(kTensor).Get<tf::Tensor>();
ABSL_CHECK(1 == input_tensor.dims() || 2 == input_tensor.dims()) CHECK(1 == input_tensor.dims() || 2 == input_tensor.dims())
<< "Only 1-D or 2-D Tensors can be converted to matrices."; << "Only 1-D or 2-D Tensors can be converted to matrices.";
const int32_t length = input_tensor.dim_size(input_tensor.dims() - 1); const int32_t length = input_tensor.dim_size(input_tensor.dims() - 1);
const int32_t width = const int32_t width =
@@ -15,16 +15,12 @@
// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float> // Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float>
// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>. // OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>.
#include <memory>
#include <vector>
#include "mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.pb.h" #include "mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.h"
#include "tensorflow/core/platform/bfloat16.h"
namespace mediapipe { namespace mediapipe {
@@ -80,31 +76,21 @@ absl::Status TensorToVectorFloatCalculator::Open(CalculatorContext* cc) {
absl::Status TensorToVectorFloatCalculator::Process(CalculatorContext* cc) { absl::Status TensorToVectorFloatCalculator::Process(CalculatorContext* cc) {
const tf::Tensor& input_tensor = const tf::Tensor& input_tensor =
cc->Inputs().Index(0).Value().Get<tf::Tensor>(); cc->Inputs().Index(0).Value().Get<tf::Tensor>();
RET_CHECK(tf::DT_FLOAT == input_tensor.dtype() || RET_CHECK(tf::DT_FLOAT == input_tensor.dtype())
tf::DT_BFLOAT16 == input_tensor.dtype()) << "expected DT_FLOAT input but got "
<< "expected DT_FLOAT or DT_BFLOAT_16 input but got "
<< tensorflow::DataTypeString(input_tensor.dtype()); << tensorflow::DataTypeString(input_tensor.dtype());
if (options_.tensor_is_2d()) { if (options_.tensor_is_2d()) {
RET_CHECK(2 == input_tensor.dims()) RET_CHECK(2 == input_tensor.dims())
<< "Expected 2-dimensional Tensor, but the tensor shape is: " << "Expected 2-dimensional Tensor, but the tensor shape is: "
<< input_tensor.shape().DebugString(); << input_tensor.shape().DebugString();
auto output = std::make_unique<std::vector<std::vector<float>>>( auto output = absl::make_unique<std::vector<std::vector<float>>>(
input_tensor.dim_size(0), std::vector<float>(input_tensor.dim_size(1))); input_tensor.dim_size(0), std::vector<float>(input_tensor.dim_size(1)));
for (int i = 0; i < input_tensor.dim_size(0); ++i) { for (int i = 0; i < input_tensor.dim_size(0); ++i) {
auto& instance_output = output->at(i); auto& instance_output = output->at(i);
if (tf::DT_BFLOAT16 == input_tensor.dtype()) { const auto& slice = input_tensor.Slice(i, i + 1).unaligned_flat<float>();
const auto& slice = for (int j = 0; j < input_tensor.dim_size(1); ++j) {
input_tensor.Slice(i, i + 1).unaligned_flat<tf::bfloat16>(); instance_output.at(j) = slice(j);
for (int j = 0; j < input_tensor.dim_size(1); ++j) {
instance_output.at(j) = static_cast<float>(slice(j));
}
} else {
const auto& slice =
input_tensor.Slice(i, i + 1).unaligned_flat<float>();
for (int j = 0; j < input_tensor.dim_size(1); ++j) {
instance_output.at(j) = slice(j);
}
} }
} }
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
@@ -115,17 +101,10 @@ absl::Status TensorToVectorFloatCalculator::Process(CalculatorContext* cc) {
<< "tensor shape is: " << input_tensor.shape().DebugString(); << "tensor shape is: " << input_tensor.shape().DebugString();
} }
auto output = auto output =
std::make_unique<std::vector<float>>(input_tensor.NumElements()); absl::make_unique<std::vector<float>>(input_tensor.NumElements());
if (tf::DT_BFLOAT16 == input_tensor.dtype()) { const auto& tensor_values = input_tensor.unaligned_flat<float>();
const auto& tensor_values = input_tensor.unaligned_flat<tf::bfloat16>(); for (int i = 0; i < input_tensor.NumElements(); ++i) {
for (int i = 0; i < input_tensor.NumElements(); ++i) { output->at(i) = tensor_values(i);
output->at(i) = static_cast<float>(tensor_values(i));
}
} else {
const auto& tensor_values = input_tensor.unaligned_flat<float>();
for (int i = 0; i < input_tensor.NumElements(); ++i) {
output->at(i) = tensor_values(i);
}
} }
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
} }
@@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include <memory>
#include "mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.pb.h" #include "mediapipe/calculators/tensorflow/tensor_to_vector_float_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"
@@ -21,7 +19,6 @@
#include "mediapipe/util/packet_test_util.h" #include "mediapipe/util/packet_test_util.h"
#include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/platform/bfloat16.h"
namespace mediapipe { namespace mediapipe {
@@ -75,62 +72,6 @@ TEST_F(TensorToVectorFloatCalculatorTest, ConvertsToVectorFloat) {
} }
} }
TEST_F(TensorToVectorFloatCalculatorTest, CheckBFloat16Type) {
SetUpRunner(false, false);
const tf::TensorShape tensor_shape(std::vector<tf::int64>{5});
auto tensor = std::make_unique<tf::Tensor>(tf::DT_BFLOAT16, tensor_shape);
auto tensor_vec = tensor->vec<tf::bfloat16>();
for (int i = 0; i < 5; ++i) {
tensor_vec(i) = static_cast<tf::bfloat16>(1 << i);
}
const int64_t time = 1234;
runner_->MutableInputs()->Index(0).packets.push_back(
Adopt(tensor.release()).At(Timestamp(time)));
EXPECT_TRUE(runner_->Run().ok());
const std::vector<Packet>& output_packets =
runner_->Outputs().Index(0).packets;
EXPECT_EQ(1, output_packets.size());
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
const std::vector<float>& output_vector =
output_packets[0].Get<std::vector<float>>();
EXPECT_EQ(5, output_vector.size());
for (int i = 0; i < 5; ++i) {
const float expected = static_cast<float>(1 << i);
EXPECT_EQ(expected, output_vector[i]);
}
}
TEST_F(TensorToVectorFloatCalculatorTest, CheckBFloat16TypeAllDim) {
SetUpRunner(false, true);
const tf::TensorShape tensor_shape(std::vector<tf::int64>{2, 2, 2});
auto tensor = std::make_unique<tf::Tensor>(tf::DT_BFLOAT16, tensor_shape);
auto slice = tensor->flat<tf::bfloat16>();
for (int i = 0; i < 2 * 2 * 2; ++i) {
// 2^i can be represented exactly in floating point numbers if 'i' is small.
slice(i) = static_cast<tf::bfloat16>(1 << i);
}
const int64_t time = 1234;
runner_->MutableInputs()->Index(0).packets.push_back(
Adopt(tensor.release()).At(Timestamp(time)));
EXPECT_TRUE(runner_->Run().ok());
const std::vector<Packet>& output_packets =
runner_->Outputs().Index(0).packets;
EXPECT_EQ(1, output_packets.size());
EXPECT_EQ(time, output_packets[0].Timestamp().Value());
const std::vector<float>& output_vector =
output_packets[0].Get<std::vector<float>>();
EXPECT_EQ(2 * 2 * 2, output_vector.size());
for (int i = 0; i < 2 * 2 * 2; ++i) {
const float expected = static_cast<float>(1 << i);
EXPECT_EQ(expected, output_vector[i]);
}
}
TEST_F(TensorToVectorFloatCalculatorTest, ConvertsBatchedToVectorVectorFloat) { TEST_F(TensorToVectorFloatCalculatorTest, ConvertsBatchedToVectorVectorFloat) {
SetUpRunner(true, false); SetUpRunner(true, false);
const tf::TensorShape tensor_shape(std::vector<tf::int64>{1, 5}); const tf::TensorShape tensor_shape(std::vector<tf::int64>{1, 5});
@@ -20,7 +20,6 @@
#include <vector> #include <vector>
#include "absl/base/thread_annotations.h" #include "absl/base/thread_annotations.h"
#include "absl/log/absl_check.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/strings/str_split.h" #include "absl/strings/str_split.h"
#include "absl/synchronization/mutex.h" #include "absl/synchronization/mutex.h"
@@ -516,7 +515,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
tf::Tensor concated; tf::Tensor concated;
const tf::Status concat_status = const tf::Status concat_status =
tf::tensor::Concat(keyed_tensors.second, &concated); tf::tensor::Concat(keyed_tensors.second, &concated);
ABSL_CHECK(concat_status.ok()) << concat_status.ToString(); CHECK(concat_status.ok()) << concat_status.ToString();
input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first], input_tensors.emplace_back(tag_to_tensor_map_[keyed_tensors.first],
concated); concated);
} }
@@ -598,7 +597,7 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
std::vector<tf::Tensor> split_tensors; std::vector<tf::Tensor> split_tensors;
const tf::Status split_status = const tf::Status split_status =
tf::tensor::Split(outputs[i], split_vector, &split_tensors); tf::tensor::Split(outputs[i], split_vector, &split_tensors);
ABSL_CHECK(split_status.ok()) << split_status.ToString(); CHECK(split_status.ok()) << split_status.ToString();
// Loop over timestamps so that we don't copy the padding. // Loop over timestamps so that we don't copy the padding.
for (int j = 0; j < inference_state->batch_timestamps_.size(); ++j) { for (int j = 0; j < inference_state->batch_timestamps_.size(); ++j) {
tf::Tensor output_tensor(split_tensors[j]); tf::Tensor output_tensor(split_tensors[j]);
@@ -17,8 +17,6 @@
#include <vector> #include <vector>
#include "absl/flags/flag.h" #include "absl/flags/flag.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h" #include "mediapipe/calculators/tensorflow/tensorflow_inference_calculator.pb.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -120,7 +118,7 @@ class TensorflowInferenceCalculatorTest : public ::testing::Test {
// Create tensor from Vector and add as a Packet to the provided tag as input. // Create tensor from Vector and add as a Packet to the provided tag as input.
void AddVectorToInputsAsPacket(const std::vector<Packet>& packets, void AddVectorToInputsAsPacket(const std::vector<Packet>& packets,
const std::string& tag) { const std::string& tag) {
ABSL_CHECK(!packets.empty()) CHECK(!packets.empty())
<< "Please specify at least some data in the packet"; << "Please specify at least some data in the packet";
auto packets_ptr = absl::make_unique<std::vector<Packet>>(packets); auto packets_ptr = absl::make_unique<std::vector<Packet>>(packets);
runner_->MutableInputs()->Tag(tag).packets.push_back( runner_->MutableInputs()->Tag(tag).packets.push_back(
@@ -588,12 +586,12 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStates) {
runner_->Outputs().Tag(kMultipliedTag).packets; runner_->Outputs().Tag(kMultipliedTag).packets;
ASSERT_EQ(2, output_packets_mult.size()); ASSERT_EQ(2, output_packets_mult.size());
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>(); const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
ABSL_LOG(INFO) << "timestamp: " << 0; LOG(INFO) << "timestamp: " << 0;
auto expected_tensor = tf::test::AsTensor<int32_t>({3, 8, 15}); auto expected_tensor = tf::test::AsTensor<int32_t>({3, 8, 15});
tf::test::ExpectTensorEqual<int32_t>(tensor_mult, expected_tensor); tf::test::ExpectTensorEqual<int32_t>(tensor_mult, expected_tensor);
const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get<tf::Tensor>(); const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get<tf::Tensor>();
auto expected_tensor1 = tf::test::AsTensor<int32_t>({9, 32, 75}); auto expected_tensor1 = tf::test::AsTensor<int32_t>({9, 32, 75});
ABSL_LOG(INFO) << "timestamp: " << 1; LOG(INFO) << "timestamp: " << 1;
tf::test::ExpectTensorEqual<int32_t>(tensor_mult1, expected_tensor1); tf::test::ExpectTensorEqual<int32_t>(tensor_mult1, expected_tensor1);
EXPECT_EQ(2, runner_ EXPECT_EQ(2, runner_
@@ -629,12 +627,12 @@ TEST_F(TensorflowInferenceCalculatorTest, TestRecurrentStateOverride) {
runner_->Outputs().Tag(kMultipliedTag).packets; runner_->Outputs().Tag(kMultipliedTag).packets;
ASSERT_EQ(2, output_packets_mult.size()); ASSERT_EQ(2, output_packets_mult.size());
const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>(); const tf::Tensor& tensor_mult = output_packets_mult[0].Get<tf::Tensor>();
ABSL_LOG(INFO) << "timestamp: " << 0; LOG(INFO) << "timestamp: " << 0;
auto expected_tensor = tf::test::AsTensor<int32_t>({3, 4, 5}); auto expected_tensor = tf::test::AsTensor<int32_t>({3, 4, 5});
tf::test::ExpectTensorEqual<int32_t>(tensor_mult, expected_tensor); tf::test::ExpectTensorEqual<int32_t>(tensor_mult, expected_tensor);
const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get<tf::Tensor>(); const tf::Tensor& tensor_mult1 = output_packets_mult[1].Get<tf::Tensor>();
auto expected_tensor1 = tf::test::AsTensor<int32_t>({3, 4, 5}); auto expected_tensor1 = tf::test::AsTensor<int32_t>({3, 4, 5});
ABSL_LOG(INFO) << "timestamp: " << 1; LOG(INFO) << "timestamp: " << 1;
tf::test::ExpectTensorEqual<int32_t>(tensor_mult1, expected_tensor1); tf::test::ExpectTensorEqual<int32_t>(tensor_mult1, expected_tensor1);
EXPECT_EQ(2, runner_ EXPECT_EQ(2, runner_
@@ -23,12 +23,12 @@
#include <string> #include <string>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/deps/clock.h" #include "mediapipe/framework/deps/clock.h"
#include "mediapipe/framework/deps/monotonic_clock.h" #include "mediapipe/framework/deps/monotonic_clock.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/tool/status_util.h" #include "mediapipe/framework/tool/status_util.h"
@@ -156,8 +156,8 @@ class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
cc->OutputSidePackets().Tag(kSessionTag).Set(Adopt(session.release())); cc->OutputSidePackets().Tag(kSessionTag).Set(Adopt(session.release()));
const uint64_t end_time = absl::ToUnixMicros(clock->TimeNow()); const uint64_t end_time = absl::ToUnixMicros(clock->TimeNow());
ABSL_LOG(INFO) << "Loaded frozen model in: " << end_time - start_time LOG(INFO) << "Loaded frozen model in: " << end_time - start_time
<< " microseconds."; << " microseconds.";
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -24,13 +24,13 @@
#include <string> #include <string>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_generator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/deps/clock.h" #include "mediapipe/framework/deps/clock.h"
#include "mediapipe/framework/deps/monotonic_clock.h" #include "mediapipe/framework/deps/monotonic_clock.h"
#include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/tool/status_util.h" #include "mediapipe/framework/tool/status_util.h"
@@ -155,8 +155,8 @@ class TensorFlowSessionFromFrozenGraphGenerator : public PacketGenerator {
output_side_packets->Tag(kSessionTag) = Adopt(session.release()); output_side_packets->Tag(kSessionTag) = Adopt(session.release());
const uint64_t end_time = absl::ToUnixMicros(clock->TimeNow()); const uint64_t end_time = absl::ToUnixMicros(clock->TimeNow());
ABSL_LOG(INFO) << "Loaded frozen model in: " << end_time - start_time LOG(INFO) << "Loaded frozen model in: " << end_time - start_time
<< " microseconds."; << " microseconds.";
return absl::OkStatus(); return absl::OkStatus();
} }
}; };
@@ -17,7 +17,6 @@
#if !defined(__ANDROID__) #if !defined(__ANDROID__)
#include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/file_helpers.h"
#endif #endif
#include "absl/log/absl_log.h"
#include "absl/strings/str_replace.h" #include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
@@ -70,7 +69,7 @@ const std::string MaybeConvertSignatureToTag(
[](unsigned char c) { return std::toupper(c); }); [](unsigned char c) { return std::toupper(c); });
output = absl::StrReplaceAll( output = absl::StrReplaceAll(
output, {{"/", "_"}, {"-", "_"}, {".", "_"}, {":", "_"}}); output, {{"/", "_"}, {"-", "_"}, {".", "_"}, {":", "_"}});
ABSL_LOG(INFO) << "Renamed TAG from: " << name << " to " << output; LOG(INFO) << "Renamed TAG from: " << name << " to " << output;
return output; return output;
} else { } else {
return name; return name;
@@ -19,7 +19,6 @@
#if !defined(__ANDROID__) #if !defined(__ANDROID__)
#include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/file_helpers.h"
#endif #endif
#include "absl/log/absl_log.h"
#include "absl/strings/str_replace.h" #include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
@@ -76,7 +75,7 @@ const std::string MaybeConvertSignatureToTag(
[](unsigned char c) { return std::toupper(c); }); [](unsigned char c) { return std::toupper(c); });
output = absl::StrReplaceAll( output = absl::StrReplaceAll(
output, {{"/", "_"}, {"-", "_"}, {".", "_"}, {":", "_"}}); output, {{"/", "_"}, {"-", "_"}, {".", "_"}, {":", "_"}});
ABSL_LOG(INFO) << "Renamed TAG from: " << name << " to " << output; LOG(INFO) << "Renamed TAG from: " << name << " to " << output;
return output; return output;
} else { } else {
return name; return name;
@@ -13,7 +13,6 @@
// limitations under the License. // limitations under the License.
#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_map.h"
#include "absl/log/absl_log.h"
#include "absl/strings/match.h" #include "absl/strings/match.h"
#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h" #include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
#include "mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.pb.h" #include "mediapipe/calculators/tensorflow/unpack_media_sequence_calculator.pb.h"
@@ -202,8 +201,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
first_timestamp_seen_ = Timestamp::OneOverPostStream().Value(); first_timestamp_seen_ = Timestamp::OneOverPostStream().Value();
for (const auto& map_kv : sequence_->feature_lists().feature_list()) { for (const auto& map_kv : sequence_->feature_lists().feature_list()) {
if (absl::StrContains(map_kv.first, "/timestamp")) { if (absl::StrContains(map_kv.first, "/timestamp")) {
ABSL_LOG(INFO) << "Found feature timestamps: " << map_kv.first LOG(INFO) << "Found feature timestamps: " << map_kv.first
<< " with size: " << map_kv.second.feature_size(); << " with size: " << map_kv.second.feature_size();
int64_t recent_timestamp = Timestamp::PreStream().Value(); int64_t recent_timestamp = Timestamp::PreStream().Value();
for (int i = 0; i < map_kv.second.feature_size(); ++i) { for (int i = 0; i < map_kv.second.feature_size(); ++i) {
int64_t next_timestamp = int64_t next_timestamp =
@@ -310,8 +309,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
audio_decoder_options->set_end_time( audio_decoder_options->set_end_time(
end_time + options.extra_padding_from_media_decoder()); end_time + options.extra_padding_from_media_decoder());
} }
ABSL_LOG(INFO) << "Created AudioDecoderOptions:\n" LOG(INFO) << "Created AudioDecoderOptions:\n"
<< audio_decoder_options->DebugString(); << audio_decoder_options->DebugString();
cc->OutputSidePackets() cc->OutputSidePackets()
.Tag(kAudioDecoderOptions) .Tag(kAudioDecoderOptions)
.Set(Adopt(audio_decoder_options.release())); .Set(Adopt(audio_decoder_options.release()));
@@ -332,8 +331,8 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
->set_end_time(Timestamp::FromSeconds(end_time).Value()); ->set_end_time(Timestamp::FromSeconds(end_time).Value());
} }
ABSL_LOG(INFO) << "Created PacketResamplerOptions:\n" LOG(INFO) << "Created PacketResamplerOptions:\n"
<< resampler_options->DebugString(); << resampler_options->DebugString();
cc->OutputSidePackets() cc->OutputSidePackets()
.Tag(kPacketResamplerOptions) .Tag(kPacketResamplerOptions)
.Set(Adopt(resampler_options.release())); .Set(Adopt(resampler_options.release()));
@@ -352,8 +351,7 @@ class UnpackMediaSequenceCalculator : public CalculatorBase {
absl::Status Process(CalculatorContext* cc) override { absl::Status Process(CalculatorContext* cc) override {
if (timestamps_.empty()) { if (timestamps_.empty()) {
// This occurs when we only have metadata to unpack. // This occurs when we only have metadata to unpack.
ABSL_LOG(INFO) LOG(INFO) << "only unpacking metadata because there are no timestamps.";
<< "only unpacking metadata because there are no timestamps.";
return tool::StatusStop(); return tool::StatusStop();
} }
// In Process(), we loop through timestamps on a reference stream and emit // In Process(), we loop through timestamps on a reference stream and emit
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_log.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/strings/numbers.h" #include "absl/strings/numbers.h"
#include "mediapipe/calculators/core/packet_resampler_calculator.pb.h" #include "mediapipe/calculators/core/packet_resampler_calculator.pb.h"
@@ -82,7 +81,7 @@ class UnpackMediaSequenceCalculatorTest : public ::testing::Test {
if (options != nullptr) { if (options != nullptr) {
*config.mutable_options() = *options; *config.mutable_options() = *options;
} }
ABSL_LOG(INFO) << config.DebugString(); LOG(INFO) << config.DebugString();
runner_ = absl::make_unique<CalculatorRunner>(config); runner_ = absl::make_unique<CalculatorRunner>(config);
} }
@@ -14,8 +14,6 @@
#include <iterator> #include <iterator>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.pb.h" #include "mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/packet.h" #include "mediapipe/framework/packet.h"
@@ -48,7 +46,7 @@ std::string GetQuantizedFeature(
.Get(index) .Get(index)
.bytes_list() .bytes_list()
.value(); .value();
ABSL_CHECK_EQ(1, bytes_list.size()); CHECK_EQ(1, bytes_list.size());
return bytes_list.Get(0); return bytes_list.Get(0);
} }
} // namespace } // namespace
@@ -151,9 +149,8 @@ class UnpackYt8mSequenceExampleCalculator : public CalculatorBase {
.Set(MakePacket<int>(segment_size)); .Set(MakePacket<int>(segment_size));
} }
} }
ABSL_LOG(INFO) << "Reading the sequence example that contains yt8m id: " LOG(INFO) << "Reading the sequence example that contains yt8m id: "
<< yt8m_id << yt8m_id << ". Feature list length: " << feature_list_length_;
<< ". Feature list length: " << feature_list_length_;
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -14,7 +14,6 @@
// //
// Converts vector<float> (or vector<vector<float>>) to 1D (or 2D) tf::Tensor. // Converts vector<float> (or vector<vector<float>>) to 1D (or 2D) tf::Tensor.
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_options.pb.h" #include "mediapipe/calculators/tensorflow/vector_float_to_tensor_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
@@ -69,7 +68,7 @@ absl::Status VectorFloatToTensorCalculator::GetContract(
// Output vector<float>. // Output vector<float>.
); );
} else { } else {
ABSL_LOG(FATAL) << "input size not supported"; LOG(FATAL) << "input size not supported";
} }
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
<< "Only one output stream is supported."; << "Only one output stream is supported.";
@@ -126,7 +125,7 @@ absl::Status VectorFloatToTensorCalculator::Process(CalculatorContext* cc) {
} }
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
} else { } else {
ABSL_LOG(FATAL) << "input size not supported"; LOG(FATAL) << "input size not supported";
} }
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -15,8 +15,6 @@
// Converts a single int or vector<int> or vector<vector<int>> to 1D (or 2D) // Converts a single int or vector<int> or vector<vector<int>> to 1D (or 2D)
// tf::Tensor. // tf::Tensor.
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.pb.h" #include "mediapipe/calculators/tensorflow/vector_int_to_tensor_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
@@ -88,7 +86,7 @@ absl::Status VectorIntToTensorCalculator::GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kVectorInt).Set<std::vector<int>>(); cc->Inputs().Tag(kVectorInt).Set<std::vector<int>>();
} }
} else { } else {
ABSL_LOG(FATAL) << "input size not supported"; LOG(FATAL) << "input size not supported";
} }
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
<< "Only one output stream is supported."; << "Only one output stream is supported.";
@@ -115,11 +113,11 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) {
.Get<std::vector<std::vector<int>>>(); .Get<std::vector<std::vector<int>>>();
const int32_t rows = input.size(); const int32_t rows = input.size();
ABSL_CHECK_GE(rows, 1); CHECK_GE(rows, 1);
const int32_t cols = input[0].size(); const int32_t cols = input[0].size();
ABSL_CHECK_GE(cols, 1); CHECK_GE(cols, 1);
for (int i = 1; i < rows; ++i) { for (int i = 1; i < rows; ++i) {
ABSL_CHECK_EQ(input[i].size(), cols); CHECK_EQ(input[i].size(), cols);
} }
if (options_.transpose()) { if (options_.transpose()) {
tensor_shape = tf::TensorShape({cols, rows}); tensor_shape = tf::TensorShape({cols, rows});
@@ -142,7 +140,7 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) {
AssignMatrixValue<int>(c, r, input[r][c], output.get()); AssignMatrixValue<int>(c, r, input[r][c], output.get());
break; break;
default: default:
ABSL_LOG(FATAL) << "tensor data type is not supported."; LOG(FATAL) << "tensor data type is not supported.";
} }
} }
} }
@@ -160,7 +158,7 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) {
AssignMatrixValue<int>(r, c, input[r][c], output.get()); AssignMatrixValue<int>(r, c, input[r][c], output.get());
break; break;
default: default:
ABSL_LOG(FATAL) << "tensor data type is not supported."; LOG(FATAL) << "tensor data type is not supported.";
} }
} }
} }
@@ -173,7 +171,7 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) {
} else { } else {
input = cc->Inputs().Tag(kVectorInt).Value().Get<std::vector<int>>(); input = cc->Inputs().Tag(kVectorInt).Value().Get<std::vector<int>>();
} }
ABSL_CHECK_GE(input.size(), 1); CHECK_GE(input.size(), 1);
const int32_t length = input.size(); const int32_t length = input.size();
tensor_shape = tf::TensorShape({length}); tensor_shape = tf::TensorShape({length});
auto output = ::absl::make_unique<tf::Tensor>(options_.tensor_data_type(), auto output = ::absl::make_unique<tf::Tensor>(options_.tensor_data_type(),
@@ -190,12 +188,12 @@ absl::Status VectorIntToTensorCalculator::Process(CalculatorContext* cc) {
output->tensor<int, 1>()(i) = input.at(i); output->tensor<int, 1>()(i) = input.at(i);
break; break;
default: default:
ABSL_LOG(FATAL) << "tensor data type is not supported."; LOG(FATAL) << "tensor data type is not supported.";
} }
} }
cc->Outputs().Tag(kTensorOut).Add(output.release(), cc->InputTimestamp()); cc->Outputs().Tag(kTensorOut).Add(output.release(), cc->InputTimestamp());
} else { } else {
ABSL_LOG(FATAL) << "input size not supported"; LOG(FATAL) << "input size not supported";
} }
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -15,7 +15,6 @@
// Converts vector<std::string> (or vector<vector<std::string>>) to 1D (or 2D) // Converts vector<std::string> (or vector<vector<std::string>>) to 1D (or 2D)
// tf::Tensor. // tf::Tensor.
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_options.pb.h" #include "mediapipe/calculators/tensorflow/vector_string_to_tensor_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
@@ -70,7 +69,7 @@ absl::Status VectorStringToTensorCalculator::GetContract(
// Input vector<std::string>. // Input vector<std::string>.
); );
} else { } else {
ABSL_LOG(FATAL) << "input size not supported"; LOG(FATAL) << "input size not supported";
} }
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1) RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
<< "Only one output stream is supported."; << "Only one output stream is supported.";
@@ -130,7 +129,7 @@ absl::Status VectorStringToTensorCalculator::Process(CalculatorContext* cc) {
} }
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
} else { } else {
ABSL_LOG(FATAL) << "input size not supported"; LOG(FATAL) << "input size not supported";
} }
return absl::OkStatus(); return absl::OkStatus();
} }
-10
View File
@@ -103,8 +103,6 @@ cc_library(
"//mediapipe/framework/formats/object_detection:anchor_cc_proto", "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -198,13 +196,10 @@ cc_library(
deps = [ deps = [
":tflite_inference_calculator_cc_proto", ":tflite_inference_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
"//mediapipe/util/tflite:config", "//mediapipe/util/tflite:config",
"//mediapipe/util/tflite:tflite_model_loader", "//mediapipe/util/tflite:tflite_model_loader",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate", "@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
@@ -280,7 +275,6 @@ cc_library(
"//mediapipe/framework/stream_handler:fixed_size_input_stream_handler", "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
"//mediapipe/util:resource_util", "//mediapipe/util:resource_util",
"//mediapipe/util/tflite:config", "//mediapipe/util/tflite:config",
"@com_google_absl//absl/log:absl_check",
"@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops", "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
] + selects.with_or({ ] + selects.with_or({
@@ -398,8 +392,6 @@ cc_library(
"//mediapipe/framework/formats/object_detection:anchor_cc_proto", "//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/util/tflite:config", "//mediapipe/util/tflite:config",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span", "@com_google_absl//absl/types:span",
"@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite:framework",
@@ -436,7 +428,6 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/util:resource_util", "//mediapipe/util:resource_util",
"@com_google_absl//absl/container:node_hash_map", "@com_google_absl//absl/container:node_hash_map",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span", "@com_google_absl//absl/types:span",
"@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite:framework",
@@ -465,7 +456,6 @@ cc_library(
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/log:absl_check",
"@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite:framework",
], ],
alwayslink = 1, alwayslink = 1,
@@ -16,8 +16,6 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tflite/ssd_anchors_calculator.pb.h" #include "mediapipe/calculators/tflite/ssd_anchors_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/object_detection/anchor.pb.h" #include "mediapipe/framework/formats/object_detection/anchor.pb.h"
@@ -274,13 +272,13 @@ absl::Status SsdAnchorsCalculator::GenerateAnchors(
if (options.feature_map_height_size()) { if (options.feature_map_height_size()) {
if (options.strides_size()) { if (options.strides_size()) {
ABSL_LOG(ERROR) << "Found feature map shapes. Strides will be ignored."; LOG(ERROR) << "Found feature map shapes. Strides will be ignored.";
} }
ABSL_CHECK_EQ(options.feature_map_height_size(), kNumLayers); CHECK_EQ(options.feature_map_height_size(), kNumLayers);
ABSL_CHECK_EQ(options.feature_map_height_size(), CHECK_EQ(options.feature_map_height_size(),
options.feature_map_width_size()); options.feature_map_width_size());
} else { } else {
ABSL_CHECK_EQ(options.strides_size(), kNumLayers); CHECK_EQ(options.strides_size(), kNumLayers);
} }
if (options.multiscale_anchor_generation()) { if (options.multiscale_anchor_generation()) {
@@ -15,7 +15,6 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/tflite/tflite_converter_calculator.pb.h" #include "mediapipe/calculators/tflite/tflite_converter_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
@@ -644,7 +643,7 @@ absl::Status TfLiteConverterCalculator::LoadOptions(CalculatorContext* cc) {
if (options.has_output_tensor_float_range()) { if (options.has_output_tensor_float_range()) {
output_range_.emplace(options.output_tensor_float_range().min(), output_range_.emplace(options.output_tensor_float_range().min(),
options.output_tensor_float_range().max()); options.output_tensor_float_range().max());
ABSL_CHECK_GT(output_range_->second, output_range_->first); CHECK_GT(output_range_->second, output_range_->first);
} }
// Custom div and sub values. // Custom div and sub values.
@@ -662,9 +661,9 @@ absl::Status TfLiteConverterCalculator::LoadOptions(CalculatorContext* cc) {
// Get desired way to handle input channels. // Get desired way to handle input channels.
max_num_channels_ = options.max_num_channels(); max_num_channels_ = options.max_num_channels();
ABSL_CHECK_GE(max_num_channels_, 1); CHECK_GE(max_num_channels_, 1);
ABSL_CHECK_LE(max_num_channels_, 4); CHECK_LE(max_num_channels_, 4);
ABSL_CHECK_NE(max_num_channels_, 2); CHECK_NE(max_num_channels_, 2);
#if defined(MEDIAPIPE_IOS) #if defined(MEDIAPIPE_IOS)
if (cc->Inputs().HasTag(kGpuBufferTag)) if (cc->Inputs().HasTag(kGpuBufferTag))
// Currently on iOS, tflite gpu input tensor must be 4 channels, // Currently on iOS, tflite gpu input tensor must be 4 channels,
@@ -17,12 +17,9 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h" #include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/util/tflite/config.h" #include "mediapipe/util/tflite/config.h"
@@ -112,8 +109,8 @@ std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
edgetpu::EdgeTpuContext* edgetpu_context) { edgetpu::EdgeTpuContext* edgetpu_context) {
resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp()); resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp());
std::unique_ptr<tflite::Interpreter> interpreter; std::unique_ptr<tflite::Interpreter> interpreter;
ABSL_CHECK_EQ(tflite::InterpreterBuilder(model, *resolver)(&interpreter), CHECK_EQ(tflite::InterpreterBuilder(model, *resolver)(&interpreter),
kTfLiteOk); kTfLiteOk);
interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context); interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context);
return interpreter; return interpreter;
} }
@@ -409,12 +406,11 @@ absl::Status TfLiteInferenceCalculator::Open(CalculatorContext* cc) {
} }
if (use_advanced_gpu_api_ && !gpu_input_) { if (use_advanced_gpu_api_ && !gpu_input_) {
ABSL_LOG(WARNING) LOG(WARNING) << "Cannot use advanced GPU APIs, input must be GPU buffers."
<< "Cannot use advanced GPU APIs, input must be GPU buffers." "Falling back to the default TFLite API.";
"Falling back to the default TFLite API.";
use_advanced_gpu_api_ = false; use_advanced_gpu_api_ = false;
} }
ABSL_CHECK(!use_advanced_gpu_api_ || gpu_inference_); CHECK(!use_advanced_gpu_api_ || gpu_inference_);
MP_RETURN_IF_ERROR(LoadModel(cc)); MP_RETURN_IF_ERROR(LoadModel(cc));
@@ -806,10 +802,9 @@ absl::Status TfLiteInferenceCalculator::InitTFLiteGPURunner(
const int tensor_idx = interpreter_->inputs()[i]; const int tensor_idx = interpreter_->inputs()[i];
interpreter_->SetTensorParametersReadWrite(tensor_idx, kTfLiteFloat32, "", interpreter_->SetTensorParametersReadWrite(tensor_idx, kTfLiteFloat32, "",
shape, quant); shape, quant);
ABSL_CHECK(interpreter_->ResizeInputTensor(tensor_idx, shape) == CHECK(interpreter_->ResizeInputTensor(tensor_idx, shape) == kTfLiteOk);
kTfLiteOk);
} }
ABSL_CHECK(interpreter_->AllocateTensors() == kTfLiteOk); CHECK(interpreter_->AllocateTensors() == kTfLiteOk);
} }
// Create and bind OpenGL buffers for outputs. // Create and bind OpenGL buffers for outputs.
@@ -1058,7 +1053,7 @@ absl::Status TfLiteInferenceCalculator::LoadDelegate(CalculatorContext* cc) {
gpu_data_in_[i]->shape.w * gpu_data_in_[i]->shape.c; gpu_data_in_[i]->shape.w * gpu_data_in_[i]->shape.c;
// Input to model can be RGBA only. // Input to model can be RGBA only.
if (tensor->dims->data[3] != 4) { if (tensor->dims->data[3] != 4) {
ABSL_LOG(WARNING) << "Please ensure input GPU tensor is 4 channels."; LOG(WARNING) << "Please ensure input GPU tensor is 4 channels.";
} }
const std::string shader_source = const std::string shader_source =
absl::Substitute(R"(#include <metal_stdlib> absl::Substitute(R"(#include <metal_stdlib>
@@ -17,7 +17,6 @@
#include <vector> #include <vector>
#include "absl/container/node_hash_map.h" #include "absl/container/node_hash_map.h"
#include "absl/log/absl_check.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/types/span.h" #include "absl/types/span.h"
#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h" #include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h"
@@ -173,7 +172,7 @@ absl::Status TfLiteTensorsToClassificationCalculator::Process(
// Note that partial_sort will raise error when top_k_ > // Note that partial_sort will raise error when top_k_ >
// classification_list->classification_size(). // classification_list->classification_size().
ABSL_CHECK_GE(classification_list->classification_size(), top_k_); CHECK_GE(classification_list->classification_size(), top_k_);
auto raw_classification_list = classification_list->mutable_classification(); auto raw_classification_list = classification_list->mutable_classification();
if (top_k_ > 0 && classification_list->classification_size() >= top_k_) { if (top_k_ > 0 && classification_list->classification_size() >= top_k_) {
std::partial_sort(raw_classification_list->begin(), std::partial_sort(raw_classification_list->begin(),
@@ -15,8 +15,6 @@
#include <unordered_map> #include <unordered_map>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/types/span.h" #include "absl/types/span.h"
#include "mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.pb.h" #include "mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.pb.h"
@@ -95,7 +93,7 @@ void ConvertRawValuesToAnchors(const float* raw_anchors, int num_boxes,
void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors, void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
int num_boxes, float* raw_anchors) { int num_boxes, float* raw_anchors) {
ABSL_CHECK_EQ(anchors.size(), num_boxes); CHECK_EQ(anchors.size(), num_boxes);
int box = 0; int box = 0;
for (const auto& anchor : anchors) { for (const auto& anchor : anchors) {
raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center(); raw_anchors[box * kNumCoordsPerBox + 0] = anchor.y_center();
@@ -290,14 +288,14 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessCPU(
const TfLiteTensor* raw_score_tensor = &input_tensors[1]; const TfLiteTensor* raw_score_tensor = &input_tensors[1];
// TODO: Add flexible input tensor size handling. // TODO: Add flexible input tensor size handling.
ABSL_CHECK_EQ(raw_box_tensor->dims->size, 3); CHECK_EQ(raw_box_tensor->dims->size, 3);
ABSL_CHECK_EQ(raw_box_tensor->dims->data[0], 1); CHECK_EQ(raw_box_tensor->dims->data[0], 1);
ABSL_CHECK_EQ(raw_box_tensor->dims->data[1], num_boxes_); CHECK_EQ(raw_box_tensor->dims->data[1], num_boxes_);
ABSL_CHECK_EQ(raw_box_tensor->dims->data[2], num_coords_); CHECK_EQ(raw_box_tensor->dims->data[2], num_coords_);
ABSL_CHECK_EQ(raw_score_tensor->dims->size, 3); CHECK_EQ(raw_score_tensor->dims->size, 3);
ABSL_CHECK_EQ(raw_score_tensor->dims->data[0], 1); CHECK_EQ(raw_score_tensor->dims->data[0], 1);
ABSL_CHECK_EQ(raw_score_tensor->dims->data[1], num_boxes_); CHECK_EQ(raw_score_tensor->dims->data[1], num_boxes_);
ABSL_CHECK_EQ(raw_score_tensor->dims->data[2], num_classes_); CHECK_EQ(raw_score_tensor->dims->data[2], num_classes_);
const float* raw_boxes = raw_box_tensor->data.f; const float* raw_boxes = raw_box_tensor->data.f;
const float* raw_scores = raw_score_tensor->data.f; const float* raw_scores = raw_score_tensor->data.f;
@@ -305,13 +303,13 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessCPU(
if (!anchors_init_) { if (!anchors_init_) {
if (input_tensors.size() == kNumInputTensorsWithAnchors) { if (input_tensors.size() == kNumInputTensorsWithAnchors) {
const TfLiteTensor* anchor_tensor = &input_tensors[2]; const TfLiteTensor* anchor_tensor = &input_tensors[2];
ABSL_CHECK_EQ(anchor_tensor->dims->size, 2); CHECK_EQ(anchor_tensor->dims->size, 2);
ABSL_CHECK_EQ(anchor_tensor->dims->data[0], num_boxes_); CHECK_EQ(anchor_tensor->dims->data[0], num_boxes_);
ABSL_CHECK_EQ(anchor_tensor->dims->data[1], kNumCoordsPerBox); CHECK_EQ(anchor_tensor->dims->data[1], kNumCoordsPerBox);
const float* raw_anchors = anchor_tensor->data.f; const float* raw_anchors = anchor_tensor->data.f;
ConvertRawValuesToAnchors(raw_anchors, num_boxes_, &anchors_); ConvertRawValuesToAnchors(raw_anchors, num_boxes_, &anchors_);
} else if (side_packet_anchors_) { } else if (side_packet_anchors_) {
ABSL_CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty());
anchors_ = anchors_ =
cc->InputSidePackets().Tag("ANCHORS").Get<std::vector<Anchor>>(); cc->InputSidePackets().Tag("ANCHORS").Get<std::vector<Anchor>>();
} else { } else {
@@ -411,7 +409,7 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU(
CopyBuffer(input_tensors[1], gpu_data_->raw_scores_buffer)); CopyBuffer(input_tensors[1], gpu_data_->raw_scores_buffer));
if (!anchors_init_) { if (!anchors_init_) {
if (side_packet_anchors_) { if (side_packet_anchors_) {
ABSL_CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty());
const auto& anchors = const auto& anchors =
cc->InputSidePackets().Tag("ANCHORS").Get<std::vector<Anchor>>(); cc->InputSidePackets().Tag("ANCHORS").Get<std::vector<Anchor>>();
std::vector<float> raw_anchors(num_boxes_ * kNumCoordsPerBox); std::vector<float> raw_anchors(num_boxes_ * kNumCoordsPerBox);
@@ -419,7 +417,7 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU(
MP_RETURN_IF_ERROR(gpu_data_->raw_anchors_buffer.Write<float>( MP_RETURN_IF_ERROR(gpu_data_->raw_anchors_buffer.Write<float>(
absl::MakeSpan(raw_anchors))); absl::MakeSpan(raw_anchors)));
} else { } else {
ABSL_CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors); CHECK_EQ(input_tensors.size(), kNumInputTensorsWithAnchors);
MP_RETURN_IF_ERROR( MP_RETURN_IF_ERROR(
CopyBuffer(input_tensors[2], gpu_data_->raw_anchors_buffer)); CopyBuffer(input_tensors[2], gpu_data_->raw_anchors_buffer));
} }
@@ -479,7 +477,7 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU(
commandBuffer:[gpu_helper_ commandBuffer]]; commandBuffer:[gpu_helper_ commandBuffer]];
if (!anchors_init_) { if (!anchors_init_) {
if (side_packet_anchors_) { if (side_packet_anchors_) {
ABSL_CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty()); CHECK(!cc->InputSidePackets().Tag("ANCHORS").IsEmpty());
const auto& anchors = const auto& anchors =
cc->InputSidePackets().Tag("ANCHORS").Get<std::vector<Anchor>>(); cc->InputSidePackets().Tag("ANCHORS").Get<std::vector<Anchor>>();
std::vector<float> raw_anchors(num_boxes_ * kNumCoordsPerBox); std::vector<float> raw_anchors(num_boxes_ * kNumCoordsPerBox);
@@ -543,7 +541,7 @@ absl::Status TfLiteTensorsToDetectionsCalculator::ProcessGPU(
output_detections)); output_detections));
#else #else
ABSL_LOG(ERROR) << "GPU input on non-Android not supported yet."; LOG(ERROR) << "GPU input on non-Android not supported yet.";
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE #endif // MEDIAPIPE_TFLITE_GL_INFERENCE
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -569,12 +567,12 @@ absl::Status TfLiteTensorsToDetectionsCalculator::LoadOptions(
num_coords_ = options_.num_coords(); num_coords_ = options_.num_coords();
// Currently only support 2D when num_values_per_keypoint equals to 2. // Currently only support 2D when num_values_per_keypoint equals to 2.
ABSL_CHECK_EQ(options_.num_values_per_keypoint(), 2); CHECK_EQ(options_.num_values_per_keypoint(), 2);
// Check if the output size is equal to the requested boxes and keypoints. // Check if the output size is equal to the requested boxes and keypoints.
ABSL_CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() + CHECK_EQ(options_.num_keypoints() * options_.num_values_per_keypoint() +
kNumCoordsPerBox, kNumCoordsPerBox,
num_coords_); num_coords_);
for (int i = 0; i < options_.ignore_classes_size(); ++i) { for (int i = 0; i < options_.ignore_classes_size(); ++i) {
ignore_classes_.insert(options_.ignore_classes(i)); ignore_classes_.insert(options_.ignore_classes(i));
@@ -899,11 +897,10 @@ void main() {
int max_wg_size; // typically <= 1024 int max_wg_size; // typically <= 1024
glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1, glGetIntegeri_v(GL_MAX_COMPUTE_WORK_GROUP_SIZE, 1,
&max_wg_size); // y-dim &max_wg_size); // y-dim
ABSL_CHECK_LT(num_classes_, max_wg_size) CHECK_LT(num_classes_, max_wg_size)
<< "# classes must be < " << max_wg_size; << "# classes must be < " << max_wg_size;
// TODO support better filtering. // TODO support better filtering.
ABSL_CHECK_LE(ignore_classes_.size(), 1) CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed";
<< "Only ignore class 0 is allowed";
// Shader program // Shader program
GlShader score_shader; GlShader score_shader;
@@ -1118,7 +1115,7 @@ kernel void scoreKernel(
ignore_classes_.size() ? 1 : 0); ignore_classes_.size() ? 1 : 0);
// TODO support better filtering. // TODO support better filtering.
ABSL_CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed"; CHECK_LE(ignore_classes_.size(), 1) << "Only ignore class 0 is allowed";
{ {
// Shader program // Shader program
@@ -1150,8 +1147,7 @@ kernel void scoreKernel(
options:MTLResourceStorageModeShared]; options:MTLResourceStorageModeShared];
// # filter classes supported is hardware dependent. // # filter classes supported is hardware dependent.
int max_wg_size = gpu_data_->score_program.maxTotalThreadsPerThreadgroup; int max_wg_size = gpu_data_->score_program.maxTotalThreadsPerThreadgroup;
ABSL_CHECK_LT(num_classes_, max_wg_size) CHECK_LT(num_classes_, max_wg_size) << "# classes must be <" << max_wg_size;
<< "# classes must be <" << max_wg_size;
} }
#endif // MEDIAPIPE_TFLITE_GL_INFERENCE #endif // MEDIAPIPE_TFLITE_GL_INFERENCE
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.pb.h" #include "mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/landmark.pb.h"
@@ -200,7 +199,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Process(
num_values *= raw_tensor->dims->data[i]; num_values *= raw_tensor->dims->data[i];
} }
const int num_dimensions = num_values / num_landmarks_; const int num_dimensions = num_values / num_landmarks_;
ABSL_CHECK_GT(num_dimensions, 0); CHECK_GT(num_dimensions, 0);
const float* raw_landmarks = raw_tensor->data.f; const float* raw_landmarks = raw_tensor->data.f;
+3 -13
View File
@@ -183,9 +183,9 @@ cc_library(
"//mediapipe/framework:calculator_options_cc_proto", "//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework:timestamp", "//mediapipe/framework:timestamp",
"//mediapipe/framework/deps:clock", "//mediapipe/framework/deps:clock",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_absl//absl/time", "@com_google_absl//absl/time",
], ],
@@ -248,12 +248,11 @@ cc_library(
":annotation_overlay_calculator_cc_proto", ":annotation_overlay_calculator_cc_proto",
"//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_options_cc_proto", "//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_format_cc_proto", "//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:image_opencv",
"//mediapipe/framework/formats:video_stream_header", "//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
@@ -261,7 +260,6 @@ cc_library(
"//mediapipe/util:annotation_renderer", "//mediapipe/util:annotation_renderer",
"//mediapipe/util:color_cc_proto", "//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto", "//mediapipe/util:render_data_cc_proto",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
] + select({ ] + select({
"//mediapipe/gpu:disable_gpu": [], "//mediapipe/gpu:disable_gpu": [],
@@ -269,7 +267,6 @@ cc_library(
"//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:shader_util", "//mediapipe/gpu:shader_util",
], ],
}), }),
@@ -377,10 +374,9 @@ cc_library(
"//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:location", "//mediapipe/framework/formats:location",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:rectangle", "//mediapipe/framework/port:rectangle",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -679,7 +675,6 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/util:color_cc_proto", "//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto", "//mediapipe/util:render_data_cc_proto",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],
@@ -736,7 +731,6 @@ cc_library(
"//mediapipe/framework/port:statusor", "//mediapipe/framework/port:statusor",
"//mediapipe/util:color_cc_proto", "//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto", "//mediapipe/util:render_data_cc_proto",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],
alwayslink = 1, alwayslink = 1,
@@ -752,7 +746,6 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/util:color_cc_proto", "//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto", "//mediapipe/util:render_data_cc_proto",
"@com_google_absl//absl/log:absl_check",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -1156,7 +1149,6 @@ cc_library(
"//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
], ],
alwayslink = 1, alwayslink = 1,
) )
@@ -1217,7 +1209,6 @@ cc_library(
"//mediapipe/framework/port:rectangle", "//mediapipe/framework/port:rectangle",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/util:rectangle_util", "//mediapipe/util:rectangle_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
], ],
alwayslink = 1, alwayslink = 1,
@@ -1489,7 +1480,6 @@ cc_library(
"//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:core_proto",
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
], ],
alwayslink = 1, alwayslink = 1,
@@ -14,17 +14,15 @@
#include <memory> #include <memory>
#include "absl/log/absl_log.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "mediapipe/calculators/util/annotation_overlay_calculator.pb.h" #include "mediapipe/calculators/util/annotation_overlay_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/image_opencv.h"
#include "mediapipe/framework/formats/video_stream_header.h" #include "mediapipe/framework/formats/video_stream_header.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
@@ -37,7 +35,6 @@
#include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_simple_shaders.h" #include "mediapipe/gpu/gl_simple_shaders.h"
#include "mediapipe/gpu/gpu_buffer.h" #include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/shader_util.h" #include "mediapipe/gpu/shader_util.h"
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
@@ -48,7 +45,6 @@ namespace {
constexpr char kVectorTag[] = "VECTOR"; constexpr char kVectorTag[] = "VECTOR";
constexpr char kGpuBufferTag[] = "IMAGE_GPU"; constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kImageFrameTag[] = "IMAGE"; constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kImageTag[] = "UIMAGE"; // Universal Image
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
@@ -61,16 +57,13 @@ size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT
constexpr uchar kAnnotationBackgroundColor = 2; // Grayscale value. constexpr uchar kAnnotationBackgroundColor = 2; // Grayscale value.
// Future Image type. // Future Image type.
inline bool HasImageTag(mediapipe::CalculatorContext* cc) { inline bool HasImageTag(mediapipe::CalculatorContext* cc) { return false; }
return cc->Inputs().HasTag(kImageTag);
}
} // namespace } // namespace
// A calculator for rendering data on images. // A calculator for rendering data on images.
// //
// Inputs: // Inputs:
// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer), // 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer),
// or UIMAGE (an Image).
// containing the input image. // containing the input image.
// If output is CPU, and input isn't provided, the renderer creates a // If output is CPU, and input isn't provided, the renderer creates a
// blank canvas with the width, height and color provided in the options. // blank canvas with the width, height and color provided in the options.
@@ -83,7 +76,6 @@ inline bool HasImageTag(mediapipe::CalculatorContext* cc) {
// //
// Output: // Output:
// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer), // 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer),
// or UIMAGE (an Image).
// Note: Output types should match their corresponding input stream type. // Note: Output types should match their corresponding input stream type.
// //
// For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The // For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The
@@ -143,9 +135,6 @@ class AnnotationOverlayCalculator : public CalculatorBase {
absl::Status CreateRenderTargetCpu(CalculatorContext* cc, absl::Status CreateRenderTargetCpu(CalculatorContext* cc,
std::unique_ptr<cv::Mat>& image_mat, std::unique_ptr<cv::Mat>& image_mat,
ImageFormat::Format* target_format); ImageFormat::Format* target_format);
absl::Status CreateRenderTargetCpuImage(CalculatorContext* cc,
std::unique_ptr<cv::Mat>& image_mat,
ImageFormat::Format* target_format);
template <typename Type, const char* Tag> template <typename Type, const char* Tag>
absl::Status CreateRenderTargetGpu(CalculatorContext* cc, absl::Status CreateRenderTargetGpu(CalculatorContext* cc,
std::unique_ptr<cv::Mat>& image_mat); std::unique_ptr<cv::Mat>& image_mat);
@@ -183,38 +172,30 @@ class AnnotationOverlayCalculator : public CalculatorBase {
REGISTER_CALCULATOR(AnnotationOverlayCalculator); REGISTER_CALCULATOR(AnnotationOverlayCalculator);
absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) { absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) {
RET_CHECK_GE(cc->Inputs().NumEntries(), 1); CHECK_GE(cc->Inputs().NumEntries(), 1);
bool use_gpu = false; bool use_gpu = false;
RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) + if (cc->Inputs().HasTag(kImageFrameTag) &&
cc->Inputs().HasTag(kGpuBufferTag) + cc->Inputs().HasTag(kGpuBufferTag)) {
cc->Inputs().HasTag(kImageTag) <= return absl::InternalError("Cannot have multiple input images.");
1); }
RET_CHECK(cc->Outputs().HasTag(kImageFrameTag) + if (cc->Inputs().HasTag(kGpuBufferTag) !=
cc->Outputs().HasTag(kGpuBufferTag) + cc->Outputs().HasTag(kGpuBufferTag)) {
cc->Outputs().HasTag(kImageTag) == return absl::InternalError("GPU output must have GPU input.");
1); }
// Input image to render onto copy of. Should be same type as output. // Input image to render onto copy of. Should be same type as output.
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
if (cc->Inputs().HasTag(kGpuBufferTag)) { if (cc->Inputs().HasTag(kGpuBufferTag)) {
cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>(); cc->Inputs().Tag(kGpuBufferTag).Set<mediapipe::GpuBuffer>();
RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag)); CHECK(cc->Outputs().HasTag(kGpuBufferTag));
use_gpu = true; use_gpu = true;
} }
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
if (cc->Inputs().HasTag(kImageFrameTag)) { if (cc->Inputs().HasTag(kImageFrameTag)) {
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>(); cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
RET_CHECK(cc->Outputs().HasTag(kImageFrameTag)); CHECK(cc->Outputs().HasTag(kImageFrameTag));
}
if (cc->Inputs().HasTag(kImageTag)) {
cc->Inputs().Tag(kImageTag).Set<mediapipe::Image>();
RET_CHECK(cc->Outputs().HasTag(kImageTag));
#if !MEDIAPIPE_DISABLE_GPU
use_gpu = true; // Prepare GPU resources because images can come in on GPU.
#endif
} }
// Data streams to render. // Data streams to render.
@@ -239,9 +220,6 @@ absl::Status AnnotationOverlayCalculator::GetContract(CalculatorContract* cc) {
if (cc->Outputs().HasTag(kImageFrameTag)) { if (cc->Outputs().HasTag(kImageFrameTag)) {
cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>(); cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
} }
if (cc->Outputs().HasTag(kImageTag)) {
cc->Outputs().Tag(kImageTag).Set<mediapipe::Image>();
}
if (use_gpu) { if (use_gpu) {
#if !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU
@@ -274,14 +252,9 @@ absl::Status AnnotationOverlayCalculator::Open(CalculatorContext* cc) {
renderer_ = absl::make_unique<AnnotationRenderer>(); renderer_ = absl::make_unique<AnnotationRenderer>();
renderer_->SetFlipTextVertically(options_.flip_text_vertically()); renderer_->SetFlipTextVertically(options_.flip_text_vertically());
if (use_gpu_) renderer_->SetScaleFactor(options_.gpu_scale_factor()); if (use_gpu_) renderer_->SetScaleFactor(options_.gpu_scale_factor());
if (renderer_->GetScaleFactor() < 1.0 && HasImageTag(cc))
ABSL_LOG(WARNING)
<< "Annotation scale factor only supports GPU backed Image.";
// Set the output header based on the input header (if present). // Set the output header based on the input header (if present).
const char* tag = HasImageTag(cc) ? kImageTag const char* tag = use_gpu_ ? kGpuBufferTag : kImageFrameTag;
: use_gpu_ ? kGpuBufferTag
: kImageFrameTag;
if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) { if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) {
const auto& input_header = const auto& input_header =
cc->Inputs().Tag(tag).Header().Get<VideoHeader>(); cc->Inputs().Tag(tag).Header().Get<VideoHeader>();
@@ -307,12 +280,6 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) {
cc->Inputs().Tag(kImageFrameTag).IsEmpty()) { cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
return absl::OkStatus(); return absl::OkStatus();
} }
if (cc->Inputs().HasTag(kImageTag) && cc->Inputs().Tag(kImageTag).IsEmpty()) {
return absl::OkStatus();
}
if (HasImageTag(cc)) {
use_gpu_ = cc->Inputs().Tag(kImageTag).Get<mediapipe::Image>().UsesGpu();
}
// Initialize render target, drawn with OpenCV. // Initialize render target, drawn with OpenCV.
std::unique_ptr<cv::Mat> image_mat; std::unique_ptr<cv::Mat> image_mat;
@@ -322,17 +289,10 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) {
if (!gpu_initialized_) { if (!gpu_initialized_) {
MP_RETURN_IF_ERROR( MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
if (HasImageTag(cc)) {
return GlSetup<mediapipe::Image, kImageTag>(cc);
}
return GlSetup<mediapipe::GpuBuffer, kGpuBufferTag>(cc); return GlSetup<mediapipe::GpuBuffer, kGpuBufferTag>(cc);
})); }));
gpu_initialized_ = true; gpu_initialized_ = true;
} }
if (HasImageTag(cc)) {
MP_RETURN_IF_ERROR(
(CreateRenderTargetGpu<mediapipe::Image, kImageTag>(cc, image_mat)));
}
if (cc->Inputs().HasTag(kGpuBufferTag)) { if (cc->Inputs().HasTag(kGpuBufferTag)) {
MP_RETURN_IF_ERROR( MP_RETURN_IF_ERROR(
(CreateRenderTargetGpu<mediapipe::GpuBuffer, kGpuBufferTag>( (CreateRenderTargetGpu<mediapipe::GpuBuffer, kGpuBufferTag>(
@@ -340,10 +300,6 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) {
} }
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
} else { } else {
if (cc->Outputs().HasTag(kImageTag)) {
MP_RETURN_IF_ERROR(
CreateRenderTargetCpuImage(cc, image_mat, &target_format));
}
if (cc->Outputs().HasTag(kImageFrameTag)) { if (cc->Outputs().HasTag(kImageFrameTag)) {
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
} }
@@ -383,9 +339,6 @@ absl::Status AnnotationOverlayCalculator::Process(CalculatorContext* cc) {
uchar* image_mat_ptr = image_mat->data; uchar* image_mat_ptr = image_mat->data;
MP_RETURN_IF_ERROR( MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc, image_mat_ptr]() -> absl::Status { gpu_helper_.RunInGlContext([this, cc, image_mat_ptr]() -> absl::Status {
if (HasImageTag(cc)) {
return RenderToGpu<mediapipe::Image, kImageTag>(cc, image_mat_ptr);
}
return RenderToGpu<mediapipe::GpuBuffer, kGpuBufferTag>( return RenderToGpu<mediapipe::GpuBuffer, kGpuBufferTag>(
cc, image_mat_ptr); cc, image_mat_ptr);
})); }));
@@ -428,10 +381,6 @@ absl::Status AnnotationOverlayCalculator::RenderToCpu(
ImageFrame::kDefaultAlignmentBoundary); ImageFrame::kDefaultAlignmentBoundary);
#endif // !MEDIAPIPE_DISABLE_GPU #endif // !MEDIAPIPE_DISABLE_GPU
if (HasImageTag(cc)) {
auto out = std::make_unique<mediapipe::Image>(std::move(output_frame));
cc->Outputs().Tag(kImageTag).Add(out.release(), cc->InputTimestamp());
}
if (cc->Outputs().HasTag(kImageFrameTag)) { if (cc->Outputs().HasTag(kImageFrameTag)) {
cc->Outputs() cc->Outputs()
.Tag(kImageFrameTag) .Tag(kImageFrameTag)
@@ -450,8 +399,7 @@ absl::Status AnnotationOverlayCalculator::RenderToGpu(CalculatorContext* cc,
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame); auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
auto output_texture = gpu_helper_.CreateDestinationTexture( auto output_texture = gpu_helper_.CreateDestinationTexture(
input_texture.width(), input_texture.height(), width_, height_, mediapipe::GpuBufferFormat::kBGRA32);
mediapipe::GpuBufferFormat::kBGRA32);
// Upload render target to GPU. // Upload render target to GPU.
{ {
@@ -480,7 +428,7 @@ absl::Status AnnotationOverlayCalculator::RenderToGpu(CalculatorContext* cc,
} }
// Send out blended image as GPU packet. // Send out blended image as GPU packet.
auto output_frame = output_texture.template GetFrame<Type>(); auto output_frame = output_texture.GetFrame<Type>();
cc->Outputs().Tag(Tag).Add(output_frame.release(), cc->InputTimestamp()); cc->Outputs().Tag(Tag).Add(output_frame.release(), cc->InputTimestamp());
// Cleanup // Cleanup
@@ -539,54 +487,6 @@ absl::Status AnnotationOverlayCalculator::CreateRenderTargetCpu(
return absl::OkStatus(); return absl::OkStatus();
} }
absl::Status AnnotationOverlayCalculator::CreateRenderTargetCpuImage(
CalculatorContext* cc, std::unique_ptr<cv::Mat>& image_mat,
ImageFormat::Format* target_format) {
if (image_frame_available_) {
const auto& input_frame =
cc->Inputs().Tag(kImageTag).Get<mediapipe::Image>();
int target_mat_type;
switch (input_frame.image_format()) {
case ImageFormat::SRGBA:
*target_format = ImageFormat::SRGBA;
target_mat_type = CV_8UC4;
break;
case ImageFormat::SRGB:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
case ImageFormat::GRAY8:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
default:
return absl::UnknownError("Unexpected image frame format.");
break;
}
image_mat = absl::make_unique<cv::Mat>(
input_frame.height(), input_frame.width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
if (input_frame.image_format() == ImageFormat::GRAY8) {
cv::Mat rgb_mat;
cv::cvtColor(*input_mat, rgb_mat, cv::COLOR_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
} else {
input_mat->copyTo(*image_mat);
}
} else {
image_mat = absl::make_unique<cv::Mat>(
options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3,
cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(),
options_.canvas_color().b()));
*target_format = ImageFormat::SRGB;
}
return absl::OkStatus();
}
template <typename Type, const char* Tag> template <typename Type, const char* Tag>
absl::Status AnnotationOverlayCalculator::CreateRenderTargetGpu( absl::Status AnnotationOverlayCalculator::CreateRenderTargetGpu(
CalculatorContext* cc, std::unique_ptr<cv::Mat>& image_mat) { CalculatorContext* cc, std::unique_ptr<cv::Mat>& image_mat) {
@@ -18,7 +18,6 @@
#include <memory> #include <memory>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "mediapipe/calculators/util/association_calculator.pb.h" #include "mediapipe/calculators/util/association_calculator.pb.h"
#include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_context.h"
@@ -73,7 +72,7 @@ class AssociationCalculator : public CalculatorBase {
prev_input_stream_id_ = cc->Inputs().GetId("PREV", 0); prev_input_stream_id_ = cc->Inputs().GetId("PREV", 0);
} }
options_ = cc->Options<::mediapipe::AssociationCalculatorOptions>(); options_ = cc->Options<::mediapipe::AssociationCalculatorOptions>();
ABSL_CHECK_GE(options_.min_similarity_threshold(), 0); CHECK_GE(options_.min_similarity_threshold(), 0);
return absl::OkStatus(); return absl::OkStatus();
} }
@@ -19,7 +19,6 @@
#include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/proto_ns.h" #include "mediapipe/framework/port/proto_ns.h"
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/util/label_map.pb.h" #include "mediapipe/util/label_map.pb.h"
#include "mediapipe/util/resource_util.h" #include "mediapipe/util/resource_util.h"
@@ -86,8 +85,7 @@ absl::Status DetectionLabelIdToTextCalculator::Open(CalculatorContext* cc) {
ASSIGN_OR_RETURN(string_path, ASSIGN_OR_RETURN(string_path,
PathToResourceAsFile(options.label_map_path())); PathToResourceAsFile(options.label_map_path()));
std::string label_map_string; std::string label_map_string;
MP_RETURN_IF_ERROR( MP_RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
mediapipe::GetResourceContents(string_path, &label_map_string));
std::istringstream stream(label_map_string); std::istringstream stream(label_map_string);
std::string line; std::string line;
@@ -12,7 +12,6 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "absl/log/absl_check.h"
#include "absl/memory/memory.h" #include "absl/memory/memory.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "absl/strings/str_join.h" #include "absl/strings/str_join.h"
@@ -234,13 +233,13 @@ void DetectionsToRenderDataCalculator::AddLabels(
const Detection& detection, const Detection& detection,
const DetectionsToRenderDataCalculatorOptions& options, const DetectionsToRenderDataCalculatorOptions& options,
float text_line_height, RenderData* render_data) { float text_line_height, RenderData* render_data) {
ABSL_CHECK(detection.label().empty() || detection.label_id().empty() || CHECK(detection.label().empty() || detection.label_id().empty() ||
detection.label_size() == detection.label_id_size()) detection.label_size() == detection.label_id_size())
<< "String or integer labels should be of same size. Or only one of them " << "String or integer labels should be of same size. Or only one of them "
"is present."; "is present.";
const auto num_labels = const auto num_labels =
std::max(detection.label_size(), detection.label_id_size()); std::max(detection.label_size(), detection.label_id_size());
ABSL_CHECK_EQ(detection.score_size(), num_labels) CHECK_EQ(detection.score_size(), num_labels)
<< "Number of scores and labels should match for detection."; << "Number of scores and labels should match for detection.";
// Extracts all "label(_id),score" for the detection. // Extracts all "label(_id),score" for the detection.
@@ -362,9 +361,9 @@ void DetectionsToRenderDataCalculator::AddDetectionToRenderData(
const Detection& detection, const Detection& detection,
const DetectionsToRenderDataCalculatorOptions& options, const DetectionsToRenderDataCalculatorOptions& options,
RenderData* render_data) { RenderData* render_data) {
ABSL_CHECK(detection.location_data().format() == LocationData::BOUNDING_BOX || CHECK(detection.location_data().format() == LocationData::BOUNDING_BOX ||
detection.location_data().format() == detection.location_data().format() ==
LocationData::RELATIVE_BOUNDING_BOX) LocationData::RELATIVE_BOUNDING_BOX)
<< "Only Detection with formats of BOUNDING_BOX or RELATIVE_BOUNDING_BOX " << "Only Detection with formats of BOUNDING_BOX or RELATIVE_BOUNDING_BOX "
"are supported."; "are supported.";
double text_line_height; double text_line_height;
@@ -19,7 +19,6 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "absl/log/absl_check.h"
#include "absl/strings/str_cat.h" #include "absl/strings/str_cat.h"
#include "mediapipe/calculators/util/labels_to_render_data_calculator.pb.h" #include "mediapipe/calculators/util/labels_to_render_data_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
@@ -115,8 +114,7 @@ absl::Status LabelsToRenderDataCalculator::Process(CalculatorContext* cc) {
video_height_ = video_header.height; video_height_ = video_header.height;
return absl::OkStatus(); return absl::OkStatus();
} else { } else {
ABSL_CHECK_EQ(options_.location(), CHECK_EQ(options_.location(), LabelsToRenderDataCalculatorOptions::TOP_LEFT)
LabelsToRenderDataCalculatorOptions::TOP_LEFT)
<< "Only TOP_LEFT is supported without VIDEO_PRESTREAM."; << "Only TOP_LEFT is supported without VIDEO_PRESTREAM.";
} }
@@ -146,7 +144,7 @@ absl::Status LabelsToRenderDataCalculator::Process(CalculatorContext* cc) {
if (cc->Inputs().HasTag(kScoresTag)) { if (cc->Inputs().HasTag(kScoresTag)) {
std::vector<float> score_vector = std::vector<float> score_vector =
cc->Inputs().Tag(kScoresTag).Get<std::vector<float>>(); cc->Inputs().Tag(kScoresTag).Get<std::vector<float>>();
ABSL_CHECK_EQ(label_vector.size(), score_vector.size()); CHECK_EQ(label_vector.size(), score_vector.size());
scores.resize(label_vector.size()); scores.resize(label_vector.size());
for (int i = 0; i < label_vector.size(); ++i) { for (int i = 0; i < label_vector.size(); ++i) {
scores[i] = score_vector[i]; scores[i] = score_vector[i];

Some files were not shown because too many files have changed in this diff Show More