talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 73d686c40057684f8bfaca285368bf1813f9fc26
This commit is contained in:
MediaPipe Team
2022-03-21 12:12:39 -07:00
committed by jqtang
parent e6c19885c6
commit cc6a2f7af6
266 changed files with 3658 additions and 1681 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/gpu/BUILD
+87 -14
View File
@@ -12,10 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("@bazel_skylib//lib:selects.bzl", "selects")
load("//mediapipe/gpu:metal.bzl", "metal_library")
load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
load("//mediapipe/framework:more_selects.bzl", "more_selects")
licenses(["notice"])
@@ -168,6 +170,8 @@ cc_library(
":gl_base",
":gl_thread_collector",
":gpu_buffer_format",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"//mediapipe/framework:executor",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
@@ -185,6 +189,11 @@ cc_library(
"//mediapipe:apple": [
"//mediapipe/objc:CFHolder",
],
}) + select({
"//conditions:default": [
],
"//mediapipe:ios": [],
"//mediapipe:macos": [],
}),
)
@@ -199,6 +208,7 @@ cc_library(
":gl_texture_view",
":gpu_buffer_format",
":gpu_buffer_storage",
":gpu_buffer_storage_image_frame",
# TODO: remove this dependency. Some other teams' tests
# depend on having an indirect image_frame dependency, need to be
# fixed first.
@@ -215,6 +225,29 @@ cc_library(
deps = [
":gl_base",
":gl_context",
":gpu_buffer_storage",
],
)
# Workaround for "Multiple matches are not allowed unless one is unambiguously more specialized".
more_selects.config_setting_negation(
name = "not_disable_gpu",
negate = ":disable_gpu",
)
selects.config_setting_group(
name = "platform_ios_with_gpu",
match_all = [
":not_disable_gpu",
"//mediapipe:ios",
],
)
selects.config_setting_group(
name = "platform_macos_with_gpu",
match_all = [
":not_disable_gpu",
"//mediapipe:macos",
],
)
@@ -224,25 +257,28 @@ cc_library(
hdrs = ["gpu_buffer.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_context",
":gpu_buffer_format",
":gpu_buffer_storage",
":gl_texture_view",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:logging",
":gpu_buffer_storage_image_frame",
] + select({
"//conditions:default": [
":gl_texture_view",
":gl_texture_buffer",
],
"//mediapipe:ios": [
":platform_ios_with_gpu": [
":gl_texture_view",
":gpu_buffer_storage_cv_pixel_buffer",
"//mediapipe/objc:util",
"//mediapipe/objc:CFHolder",
],
"//mediapipe:macos": [
":platform_macos_with_gpu": [
"//mediapipe/objc:CFHolder",
":gl_texture_view",
":gl_texture_buffer",
],
":disable_gpu": [],
}),
)
@@ -252,24 +288,28 @@ cc_library(
hdrs = ["gpu_buffer_format.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
"//mediapipe/framework/deps:no_destructor",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/container:flat_hash_map",
],
"//mediapipe/framework/deps:no_destructor",
"//mediapipe/framework/port:logging",
] + select({
"//conditions:default": [
":gl_base",
],
"//mediapipe/gpu:disable_gpu": [],
}),
)
cc_library(
name = "gpu_buffer_storage",
srcs = ["gpu_buffer_storage.cc"],
hdrs = ["gpu_buffer_storage.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gpu_buffer_format",
"//mediapipe/framework/deps:no_destructor",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/tool:type_util",
"@com_google_absl//absl/container:flat_hash_map",
],
)
@@ -284,11 +324,36 @@ cc_library(
":gl_context",
":gl_texture_view",
":gpu_buffer_storage",
":gpu_buffer_storage_image_frame",
":image_frame_view",
"//mediapipe/objc:CFHolder",
"//mediapipe/objc:util",
],
)
cc_library(
name = "gpu_buffer_storage_image_frame",
hdrs = ["gpu_buffer_storage_image_frame.h"],
visibility = ["//visibility:public"],
deps = [
":gpu_buffer_format",
":gpu_buffer_storage",
":image_frame_view",
"//mediapipe/framework/formats:image_frame",
],
)
cc_library(
name = "image_frame_view",
hdrs = ["image_frame_view.h"],
visibility = ["//visibility:public"],
deps = [
":gpu_buffer_format",
":gpu_buffer_storage",
"//mediapipe/framework/formats:image_frame",
],
)
mediapipe_proto_library(
name = "gpu_origin_proto",
srcs = ["gpu_origin.proto"],
@@ -326,6 +391,7 @@ objc_library(
"-Wno-shorten-64-to-32",
"-std=c++17",
],
features = ["-layering_check"],
sdk_frameworks = [
"CoreVideo",
"Metal",
@@ -418,9 +484,6 @@ alias(
cc_library(
name = "gpu_shared_data_internal_stub",
hdrs = [
"gpu_shared_data_internal.h",
],
visibility = ["//visibility:private"],
deps = [
":graph_support",
@@ -587,12 +650,14 @@ cc_library(
deps = [
":gl_base",
":gl_context",
":gl_texture_buffer_pool",
":gpu_buffer",
":gpu_buffer_format",
":gpu_buffer_multi_pool",
":gpu_shared_data_internal",
":gpu_service",
":graph_support",
":image_frame_view",
":shader_util",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_cc_proto",
@@ -644,6 +709,7 @@ objc_library(
"-Wno-shorten-64-to-32",
"-std=c++17",
],
features = ["-layering_check"],
sdk_frameworks = [
"CoreVideo",
"Metal",
@@ -735,6 +801,7 @@ cc_library(
visibility = ["//visibility:public"],
deps = [
":gl_calculator_helper",
":gpu_buffer_storage_image_frame",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:status",
@@ -861,6 +928,7 @@ objc_library(
name = "metal_copy_calculator",
srcs = ["MetalCopyCalculator.mm"],
copts = ["-std=c++17"],
features = ["-layering_check"],
sdk_frameworks = [
"CoreVideo",
"Metal",
@@ -879,6 +947,7 @@ objc_library(
name = "metal_rgb_weight_calculator",
srcs = ["MetalRgbWeightCalculator.mm"],
copts = ["-std=c++17"],
features = ["-layering_check"],
sdk_frameworks = [
"CoreVideo",
"Metal",
@@ -896,6 +965,7 @@ objc_library(
name = "metal_sobel_calculator",
srcs = ["MetalSobelCalculator.mm"],
copts = ["-std=c++17"],
features = ["-layering_check"],
sdk_frameworks = [
"CoreVideo",
"Metal",
@@ -913,6 +983,7 @@ objc_library(
name = "metal_sobel_compute_calculator",
srcs = ["MetalSobelComputeCalculator.mm"],
copts = ["-std=c++17"],
features = ["-layering_check"],
sdk_frameworks = [
"CoreVideo",
"Metal",
@@ -930,6 +1001,7 @@ objc_library(
name = "mps_sobel_calculator",
srcs = ["MPSSobelCalculator.mm"],
copts = ["-std=c++17"],
features = ["-layering_check"],
sdk_frameworks = [
"CoreVideo",
"Metal",
@@ -986,6 +1058,7 @@ objc_library(
data = [
"//mediapipe/objc:testdata/googlelogo_color_272x92dp.png",
],
features = ["-layering_check"],
deps = [
":MPPGraphGPUData",
":gl_scaler_calculator",
mediapipe/gpu/MPPGraphGPUData.mm
+2 -2
View File
@@ -40,8 +40,8 @@ typedef CVOpenGLTextureCacheRef CVTextureCacheType;
typedef CVOpenGLESTextureCacheRef CVTextureCacheType;
#endif // TARGET_OS_OSX
- (instancetype)initWithContext:(mediapipe::GlContext*)context
multiPool:(mediapipe::GpuBufferMultiPool*)pool {
- (instancetype)initWithContext:(mediapipe::GlContext *)context
multiPool:(mediapipe::GpuBufferMultiPool *)pool {
self = [super init];
if (self) {
_gpuBufferPool = pool;
mediapipe/gpu/MPPMetalHelper.mm
+2 -3
View File
@@ -122,8 +122,7 @@ class MetalHelperLegacySupport {
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer
plane:(size_t)plane {
CVPixelBufferRef pixel_buffer = gpuBuffer.GetCVPixelBufferRef();
CVPixelBufferRef pixel_buffer = mediapipe::GetCVPixelBufferRef(gpuBuffer);
OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
MTLPixelFormat metalPixelFormat = MTLPixelFormatInvalid;
@@ -170,7 +169,7 @@ class MetalHelperLegacySupport {
CVMetalTextureRef texture;
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(
NULL, _gpuShared.mtlTextureCache, gpuBuffer.GetCVPixelBufferRef(), NULL,
NULL, _gpuShared.mtlTextureCache, mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL,
metalPixelFormat, width, height, plane, &texture);
CHECK_EQ(err, kCVReturnSuccess);
return texture;
mediapipe/gpu/gl_calculator_helper.cc
+16 -2
View File
@@ -35,9 +35,13 @@ GlCalculatorHelper::~GlCalculatorHelper() {}
absl::Status GlCalculatorHelper::Open(CalculatorContext* cc) {
CHECK(cc);
auto gpu_service = cc->Service(kGpuService);
RET_CHECK(gpu_service.IsAvailable())
<< "GPU service not available. Did you forget to call "
"GlCalculatorHelper::UpdateContract?";
// TODO return error from impl_ (needs two-stage init)
impl_ = absl::make_unique<GlCalculatorHelperImpl>(
cc, &cc->Service(kGpuService).GetObject());
impl_ =
absl::make_unique<GlCalculatorHelperImpl>(cc, &gpu_service.GetObject());
return absl::OkStatus();
}
@@ -114,6 +118,16 @@ GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& pixel_buffer,
return impl_->CreateSourceTexture(pixel_buffer, plane);
}
GpuBuffer GlCalculatorHelper::GpuBufferWithImageFrame(
std::shared_ptr<ImageFrame> image_frame) {
return impl_->GpuBufferWithImageFrame(std::move(image_frame));
}
GpuBuffer GlCalculatorHelper::GpuBufferCopyingImageFrame(
const ImageFrame& image_frame) {
return impl_->GpuBufferCopyingImageFrame(image_frame);
}
void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer,
int* width, int* height) {
CHECK(width);
mediapipe/gpu/gl_calculator_helper.h
+23 -9
View File
@@ -15,6 +15,8 @@
#ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
#define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
@@ -128,8 +130,19 @@ class GlCalculatorHelper {
// Convenience function for converting an ImageFrame to GpuBuffer and then
// accessing it as a texture.
// This is deprecated because: 1) it encourages the use of GlTexture as a
// long-lived object; 2) it requires copying the ImageFrame's contents,
// which may not always be necessary.
ABSL_DEPRECATED("Use `GpuBufferWithImageFrame`.")
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
// Creates a GpuBuffer sharing ownership of image_frame. The contents of
// image_frame should not be modified after calling this.
GpuBuffer GpuBufferWithImageFrame(std::shared_ptr<ImageFrame> image_frame);
// Creates a GpuBuffer copying the contents of image_frame.
GpuBuffer GpuBufferCopyingImageFrame(const ImageFrame& image_frame);
// Extracts GpuBuffer dimensions without creating a texture.
ABSL_DEPRECATED("Use width and height methods on GpuBuffer instead")
void GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, int* width,
@@ -170,13 +183,13 @@ class GlCalculatorHelper {
// memory.
class GlTexture {
public:
GlTexture() {}
~GlTexture() { Release(); }
GlTexture() : view_(std::make_shared<GlTextureView>()) {}
~GlTexture() = default;
int width() const { return view_.width(); }
int height() const { return view_.height(); }
GLenum target() const { return view_.target(); }
GLuint name() const { return view_.name(); }
int width() const { return view_->width(); }
int height() const { return view_->height(); }
GLenum target() const { return view_->target(); }
GLuint name() const { return view_->name(); }
// Returns a buffer that can be sent to another calculator.
// & manages sync token
@@ -185,12 +198,13 @@ class GlTexture {
std::unique_ptr<T> GetFrame() const;
// Releases texture memory & manages sync token
void Release() { view_.Release(); }
void Release() { view_ = std::make_shared<GlTextureView>(); }
private:
explicit GlTexture(GlTextureView view) : view_(std::move(view)) {}
explicit GlTexture(GlTextureView view)
: view_(std::make_shared<GlTextureView>(std::move(view))) {}
friend class GlCalculatorHelperImpl;
GlTextureView view_;
std::shared_ptr<GlTextureView> view_;
};
// Returns the entry with the given tag if the collection uses tags, with the
mediapipe/gpu/gl_calculator_helper_impl.h
+3
View File
@@ -50,6 +50,9 @@ class GlCalculatorHelperImpl {
GlTexture CreateDestinationTexture(int output_width, int output_height,
GpuBufferFormat format);
GpuBuffer GpuBufferWithImageFrame(std::shared_ptr<ImageFrame> image_frame);
GpuBuffer GpuBufferCopyingImageFrame(const ImageFrame& image_frame);
GLuint framebuffer() const { return framebuffer_; }
void BindFramebuffer(const GlTexture& dst);
mediapipe/gpu/gl_calculator_helper_impl_common.cc
+30 -4
View File
@@ -14,10 +14,12 @@
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#include "mediapipe/gpu/image_frame_view.h"
namespace mediapipe {
@@ -117,18 +119,42 @@ GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const ImageFrame& image_frame) {
auto gpu_buffer = GpuBuffer::CopyingImageFrame(image_frame);
auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(0));
}
GpuBuffer GlCalculatorHelperImpl::GpuBufferWithImageFrame(
std::shared_ptr<ImageFrame> image_frame) {
return GpuBuffer(
std::make_shared<GpuBufferStorageImageFrame>(std::move(image_frame)));
}
GpuBuffer GlCalculatorHelperImpl::GpuBufferCopyingImageFrame(
const ImageFrame& image_frame) {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
// Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
// deals with absl::Status in MediaPipe OSS.
CHECK_OK(maybe_buffer.status());
return GpuBuffer(std::move(maybe_buffer).value());
#else
return GpuBuffer(GlTextureBuffer::Create(image_frame));
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
}
template <>
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
return view_.gpu_buffer().AsImageFrame();
view_->DoneWriting();
std::shared_ptr<const ImageFrame> view =
view_->gpu_buffer().GetReadView<ImageFrame>();
auto copy = absl::make_unique<ImageFrame>();
copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary);
return copy;
}
template <>
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
auto gpu_buffer = view_.gpu_buffer();
auto gpu_buffer = view_->gpu_buffer();
#ifdef __EMSCRIPTEN__
// When WebGL is used, the GL context may be spontaneously lost which can
// cause GpuBuffer allocations to fail. In that case, return a dummy buffer
@@ -137,7 +163,7 @@ std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
return std::make_unique<GpuBuffer>();
}
#endif // __EMSCRIPTEN__
view_.DoneWriting();
view_->DoneWriting();
return absl::make_unique<GpuBuffer>(gpu_buffer);
}
mediapipe/gpu/gl_context.cc
+25 -5
View File
@@ -304,7 +304,7 @@ absl::Status GlContext::FinishInitialization(bool create_thread) {
glGetIntegerv(GL_MINOR_VERSION, &gl_minor_version_);
} else {
// GL_MAJOR_VERSION is not supported on GL versions below 3. We have to
// parse the version std::string.
// parse the version string.
if (!ParseGlVersion(version_string, &gl_major_version_,
&gl_minor_version_)) {
LOG(WARNING) << "invalid GL_VERSION format: '" << version_string
@@ -344,7 +344,8 @@ absl::Status GlContext::FinishInitialization(bool create_thread) {
#if GL_ES_VERSION_2_0 // This actually means "is GLES available".
// No linear float filtering by default, check extensions.
can_linear_filter_float_textures_ =
HasGlExtension("OES_texture_float_linear");
HasGlExtension("OES_texture_float_linear") ||
HasGlExtension("GL_OES_texture_float_linear");
#else
// Desktop GL should always allow linear filtering.
can_linear_filter_float_textures_ = true;
@@ -548,7 +549,11 @@ class GlFenceSyncPoint : public GlSyncPoint {
: GlSyncPoint(gl_context) {
gl_context_->Run([this] {
sync_ = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
// Defer the flush for WebGL until the glClientWaitSync call as it's a
// costly IPC call in Chrome's WebGL implementation.
#ifndef __EMSCRIPTEN__
glFlush();
#endif
});
}
@@ -565,8 +570,17 @@ class GlFenceSyncPoint : public GlSyncPoint {
void Wait() override {
if (!sync_) return;
gl_context_->Run([this] {
GLenum result =
glClientWaitSync(sync_, 0, std::numeric_limits<uint64_t>::max());
GLuint flags = 0;
uint64_t timeout = std::numeric_limits<uint64_t>::max();
#ifdef __EMSCRIPTEN__
// Setting GL_SYNC_FLUSH_COMMANDS_BIT ensures flush happens before we wait
// on the fence. This is necessary since we defer the flush on WebGL.
flags = GL_SYNC_FLUSH_COMMANDS_BIT;
// WebGL only supports small implementation dependent timeout values. In
// particular, Chrome only supports a timeout of 0.
timeout = 0;
#endif
GLenum result = glClientWaitSync(sync_, flags, timeout);
if (result == GL_ALREADY_SIGNALED || result == GL_CONDITION_SATISFIED) {
glDeleteSync(sync_);
sync_ = nullptr;
@@ -592,7 +606,13 @@ class GlFenceSyncPoint : public GlSyncPoint {
bool ready = false;
// TODO: we should not block on the original context if possible.
gl_context_->Run([this, &ready] {
GLenum result = glClientWaitSync(sync_, 0, 0);
GLuint flags = 0;
#ifdef __EMSCRIPTEN__
// Setting GL_SYNC_FLUSH_COMMANDS_BIT ensures flush happens before we wait
// on the fence. This is necessary since we defer the flush on WebGL.
flags = GL_SYNC_FLUSH_COMMANDS_BIT;
#endif
GLenum result = glClientWaitSync(sync_, flags, 0);
if (result == GL_ALREADY_SIGNALED || result == GL_CONDITION_SATISFIED) {
glDeleteSync(sync_);
sync_ = nullptr;
mediapipe/gpu/gl_context.h
+7
View File
@@ -228,6 +228,12 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
CVOpenGLTextureCacheRef cv_texture_cache() const { return *texture_cache_; }
#endif // HAS_EGL
// Returns whatever the current platform's native context handle is.
// Prefer the explicit *_context methods above, unless you're going to use
// this in a context that you are sure will work with whatever definition of
// PlatformGlContext is in use.
PlatformGlContext native_context() const { return context_; }
// Check if the context is current on this thread. Mainly for test purposes.
bool IsCurrent() const;
@@ -432,4 +438,5 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
int plane);
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_CONTEXT_H_
mediapipe/gpu/gl_context_egl.cc
+28 -12
View File
@@ -15,6 +15,8 @@
#include <utility>
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
@@ -30,6 +32,8 @@
namespace mediapipe {
namespace {
static pthread_key_t egl_release_thread_key;
static pthread_once_t egl_release_key_once = PTHREAD_ONCE_INIT;
@@ -67,6 +71,29 @@ static void EnsureEglThreadRelease() {
reinterpret_cast<void*>(0xDEADBEEF));
}
static absl::StatusOr<EGLDisplay> GetInitializedDefaultEglDisplay() {
EGLDisplay display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
RET_CHECK(display != EGL_NO_DISPLAY)
<< "eglGetDisplay() returned error " << std::showbase << std::hex
<< eglGetError();
EGLint major = 0;
EGLint minor = 0;
EGLBoolean egl_initialized = eglInitialize(display, &major, &minor);
RET_CHECK(egl_initialized) << "Unable to initialize EGL";
LOG(INFO) << "Successfully initialized EGL. Major : " << major
<< " Minor: " << minor;
return display;
}
static absl::StatusOr<EGLDisplay> GetInitializedEglDisplay() {
auto status_or_display = GetInitializedDefaultEglDisplay();
return status_or_display;
}
} // namespace
GlContext::StatusOrGlContext GlContext::Create(std::nullptr_t nullp,
bool create_thread) {
return Create(EGL_NO_CONTEXT, create_thread);
@@ -149,18 +176,7 @@ absl::Status GlContext::CreateContextInternal(EGLContext share_context,
}
absl::Status GlContext::CreateContext(EGLContext share_context) {
EGLint major = 0;
EGLint minor = 0;
display_ = eglGetDisplay(EGL_DEFAULT_DISPLAY);
RET_CHECK(display_ != EGL_NO_DISPLAY)
<< "eglGetDisplay() returned error " << std::showbase << std::hex
<< eglGetError();
EGLBoolean success = eglInitialize(display_, &major, &minor);
RET_CHECK(success) << "Unable to initialize EGL";
LOG(INFO) << "Successfully initialized EGL. Major : " << major
<< " Minor: " << minor;
ASSIGN_OR_RETURN(display_, GetInitializedEglDisplay());
auto status = CreateContextInternal(share_context, 3);
if (!status.ok()) {
mediapipe/gpu/gl_quad_renderer.h
+1 -1
View File
@@ -81,7 +81,7 @@ class QuadRenderer {
GLuint program_ = 0;
GLint scale_unif_ = -1;
std::vector<GLint> frame_unifs_;
GLuint vao_; // vertex array object
GLuint vao_ = 0; // vertex array object
GLuint vbo_[2] = {0, 0}; // for vertex buffer storage
};
mediapipe/gpu/gl_simple_shaders.cc
+7 -7
View File
@@ -16,7 +16,7 @@
namespace mediapipe {
// This macro converts everything between its parentheses to a std::string.
// This macro converts everything between its parentheses to a string.
// Using this instead of R"()" preserves C-like syntax coloring in most
// editors, which is desirable for shaders.
#if !defined(_STRINGIFY)
@@ -28,15 +28,15 @@ namespace mediapipe {
// for a type. The macro strips out the precision declaration on desktop GL,
// where it's not supported.
//
// Note: this does not use a raw std::string because some compilers don't handle
// raw strings inside macros correctly. It uses a macro because we want to be
// able to concatenate strings by juxtaposition. We want to concatenate strings
// by juxtaposition so we can export const char* static data containing the
// Note: this does not use a raw string because some compilers don't handle raw
// strings inside macros correctly. It uses a macro because we want to be able
// to concatenate strings by juxtaposition. We want to concatenate strings by
// juxtaposition so we can export const char* static data containing the
// pre-expanded strings.
//
// TODO: this was written before we could rely on C++11 support.
// Consider replacing it with constexpr std::string concatenation, or replacing
// the static variables with functions.
// Consider replacing it with constexpr string concatenation, or replacing the
// static variables with functions.
#define PRECISION_COMPAT \
GLES_VERSION_COMPAT \
"#ifdef GL_ES \n" \
mediapipe/gpu/gl_texture_buffer.cc
+50 -45
View File
@@ -16,6 +16,7 @@
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_texture_view.h"
#include "mediapipe/gpu/gpu_buffer_storage_image_frame.h"
namespace mediapipe {
@@ -228,14 +229,14 @@ void GlTextureBuffer::WaitForConsumersOnGpu() {
}
GlTextureView GlTextureBuffer::GetReadView(
mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) const {
internal::types<GlTextureView>, std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
CHECK_EQ(plane, 0);
// Insert wait call to sync with the producer.
WaitOnGpu();
GlTextureView::DetachFn detach = [this](mediapipe::GlTextureView& texture) {
GlTextureView::DetachFn detach = [this](GlTextureView& texture) {
// Inform the GlTextureBuffer that we have finished accessing its
// contents, and create a consumer sync point.
DidRead(texture.gl_context()->CreateSyncToken());
@@ -246,8 +247,8 @@ GlTextureView GlTextureBuffer::GetReadView(
}
GlTextureView GlTextureBuffer::GetWriteView(
mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) {
internal::types<GlTextureView>, std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
CHECK_EQ(plane, 0);
@@ -256,9 +257,7 @@ GlTextureView GlTextureBuffer::GetWriteView(
Reuse(); // TODO: the producer wait should probably be part of Reuse in the
// case when there are no consumers.
GlTextureView::DoneWritingFn done_writing =
[this](const mediapipe::GlTextureView& texture) {
ViewDoneWriting(texture);
};
[this](const GlTextureView& texture) { ViewDoneWriting(texture); };
return GlTextureView(gl_context.get(), target(), name(), width(), height(),
std::move(gpu_buffer), plane, nullptr,
std::move(done_writing));
@@ -311,46 +310,52 @@ static void ReadTexture(const GlTextureView& view, GpuBufferFormat format,
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
format, view.plane(), view.gl_context()->GetGlVersion());
GLint current_fbo;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &current_fbo);
CHECK_NE(current_fbo, 0);
GLint previous_fbo;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &previous_fbo);
GLint color_attachment_name;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment_name);
if (color_attachment_name != view.name()) {
// Save the viewport. Note that we assume that the color attachment is a
// GL_TEXTURE_2D texture.
GLint viewport[4];
glGetIntegerv(GL_VIEWPORT, viewport);
// Set the data from GLTextureView object.
glViewport(0, 0, view.width(), view.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, view.target(),
view.name(), 0);
glReadPixels(0, 0, view.width(), view.height(), info.gl_format,
info.gl_type, output);
// Restore from the saved viewport and color attachment name.
glViewport(viewport[0], viewport[1], viewport[2], viewport[3]);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
color_attachment_name, 0);
} else {
glReadPixels(0, 0, view.width(), view.height(), info.gl_format,
info.gl_type, output);
}
// We use a temp fbo to avoid depending on the app having an existing one.
// TODO: keep a utility fbo around in the context?
GLuint fbo = 0;
glGenFramebuffers(1, &fbo);
glBindFramebuffer(GL_FRAMEBUFFER, fbo);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, view.target(),
view.name(), 0);
glReadPixels(0, 0, view.width(), view.height(), info.gl_format, info.gl_type,
output);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, 0,
0);
// TODO: just set the binding to 0 to avoid the get call?
glBindFramebuffer(GL_FRAMEBUFFER, previous_fbo);
glDeleteFramebuffers(1, &fbo);
}
std::unique_ptr<ImageFrame> GlTextureBuffer::AsImageFrame() const {
ImageFormat::Format image_format = ImageFormatForGpuBufferFormat(format());
auto output = absl::make_unique<ImageFrame>(
image_format, width(), height(), ImageFrame::kGlDefaultAlignmentBoundary);
auto view =
GetReadView(mediapipe::internal::types<GlTextureView>{}, nullptr, 0);
ReadTexture(view, format(), output->MutablePixelData(),
output->PixelDataSize());
return output;
static std::shared_ptr<GpuBufferStorageImageFrame> ConvertToImageFrame(
std::shared_ptr<GlTextureBuffer> buf) {
ImageFormat::Format image_format =
ImageFormatForGpuBufferFormat(buf->format());
auto output =
absl::make_unique<ImageFrame>(image_format, buf->width(), buf->height(),
ImageFrame::kGlDefaultAlignmentBoundary);
buf->GetProducerContext()->Run([buf, &output] {
auto view = buf->GetReadView(internal::types<GlTextureView>{}, nullptr, 0);
ReadTexture(view, buf->format(), output->MutablePixelData(),
output->PixelDataSize());
});
return std::make_shared<GpuBufferStorageImageFrame>(std::move(output));
}
static std::shared_ptr<GlTextureBuffer> ConvertFromImageFrame(
std::shared_ptr<GpuBufferStorageImageFrame> frame) {
return GlTextureBuffer::Create(*frame->image_frame());
}
static auto kConverterRegistration =
internal::GpuBufferStorageRegistry::Get()
.RegisterConverter<GlTextureBuffer, GpuBufferStorageImageFrame>(
ConvertToImageFrame);
static auto kConverterRegistration2 =
internal::GpuBufferStorageRegistry::Get()
.RegisterConverter<GpuBufferStorageImageFrame, GlTextureBuffer>(
ConvertFromImageFrame);
} // namespace mediapipe
mediapipe/gpu/gl_texture_buffer.h
+5 -5
View File
@@ -24,6 +24,7 @@
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gl_texture_view.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
@@ -33,8 +34,8 @@ class GlCalculatorHelperImpl;
// Implements a GPU memory buffer as an OpenGL texture. For internal use.
class GlTextureBuffer
: public mediapipe::internal::GpuBufferStorageImpl<
GlTextureBuffer, mediapipe::internal::ViewProvider<GlTextureView>> {
: public internal::GpuBufferStorageImpl<
GlTextureBuffer, internal::ViewProvider<GlTextureView>> {
public:
// This is called when the texture buffer is deleted. It is passed a sync
// token created at that time on the GlContext. If the GlTextureBuffer has
@@ -88,13 +89,12 @@ class GlTextureBuffer
int height() const { return height_; }
GpuBufferFormat format() const { return format_; }
GlTextureView GetReadView(mediapipe::internal::types<GlTextureView>,
GlTextureView GetReadView(internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override;
GlTextureView GetWriteView(mediapipe::internal::types<GlTextureView>,
GlTextureView GetWriteView(internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override;
std::unique_ptr<ImageFrame> AsImageFrame() const override;
// If this texture is going to be used outside of the context that produced
// it, this method should be called to ensure that its updated contents are
mediapipe/gpu/gl_texture_view.cc
+8
View File
@@ -3,6 +3,7 @@
namespace mediapipe {
void GlTextureView::Release() {
DoneWriting();
if (detach_) detach_(*this);
detach_ = nullptr;
gl_context_ = nullptr;
@@ -13,4 +14,11 @@ void GlTextureView::Release() {
height_ = 0;
}
void GlTextureView::DoneWriting() const {
if (done_writing_) {
done_writing_(*this);
done_writing_ = nullptr;
}
}
} // namespace mediapipe
mediapipe/gpu/gl_texture_view.h
+44 -5
View File
@@ -20,6 +20,7 @@
#include <utility>
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
namespace mediapipe {
@@ -31,7 +32,23 @@ class GlTextureView {
public:
GlTextureView() {}
~GlTextureView() { Release(); }
// TODO: make this class move-only.
GlTextureView(const GlTextureView&) = delete;
GlTextureView(GlTextureView&& other) { *this = std::move(other); }
GlTextureView& operator=(const GlTextureView&) = delete;
GlTextureView& operator=(GlTextureView&& other) {
DoneWriting();
if (detach_) detach_(*this);
gl_context_ = other.gl_context_;
target_ = other.target_;
name_ = other.name_;
width_ = other.width_;
height_ = other.height_;
gpu_buffer_ = std::move(other.gpu_buffer_);
plane_ = other.plane_;
detach_ = std::exchange(other.detach_, nullptr);
done_writing_ = std::exchange(other.done_writing_, nullptr);
return *this;
}
GlContext* gl_context() const { return gl_context_; }
int width() const { return width_; }
@@ -63,11 +80,11 @@ class GlTextureView {
// TODO: remove this friend declaration.
friend class GlTexture;
void Release();
// TODO: make this non-const.
void DoneWriting() const {
if (done_writing_) done_writing_(*this);
}
void DoneWriting() const;
GlContext* gl_context_ = nullptr;
GLenum target_ = GL_TEXTURE_2D;
@@ -78,9 +95,31 @@ class GlTextureView {
std::shared_ptr<GpuBuffer> gpu_buffer_; // using shared_ptr temporarily
int plane_ = 0;
DetachFn detach_;
DoneWritingFn done_writing_;
mutable DoneWritingFn done_writing_;
};
namespace internal {
template <>
class ViewProvider<GlTextureView> {
public:
virtual ~ViewProvider() = default;
// Note that the view type is encoded in an argument to allow overloading,
// so a storage class can implement GetRead/WriteView for multiple view types.
// We cannot use a template function because it cannot be virtual; we want to
// have a virtual function here to enforce that different storages supporting
// the same view implement the same signature.
// Note that we allow different views to have custom signatures, providing
// additional view-specific arguments that may be needed.
virtual GlTextureView GetReadView(types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const = 0;
virtual GlTextureView GetWriteView(types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) = 0;
};
} // namespace internal
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_TEXTURE_VIEW_H_
mediapipe/gpu/gpu_buffer.cc
+56 -15
View File
@@ -1,6 +1,8 @@
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gl_context.h"
#include <memory>
#include "mediapipe/framework/port/logging.h"
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#include "mediapipe/objc/util.h"
@@ -8,22 +10,61 @@
namespace mediapipe {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
internal::GpuBufferStorage& GpuBuffer::GetStorageForView(
TypeRef view_provider_type, bool for_writing) const {
const std::shared_ptr<internal::GpuBufferStorage>* chosen_storage = nullptr;
GpuBuffer GpuBuffer::CopyingImageFrame(const ImageFrame& image_frame) {
auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
// Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
// deals with absl::Status in MediaPipe OSS.
CHECK_OK(maybe_buffer.status());
return GpuBuffer(std::move(maybe_buffer).value());
// First see if any current storage supports the view.
for (const auto& s : storages_) {
if (s->can_down_cast_to(view_provider_type)) {
chosen_storage = &s;
break;
}
}
// Then try to convert existing storages to one that does.
// TODO: choose best conversion.
if (!chosen_storage) {
for (const auto& s : storages_) {
auto converter = internal::GpuBufferStorageRegistry::Get()
.StorageConverterForViewProvider(view_provider_type,
s->storage_type());
if (converter) {
storages_.push_back(converter(s));
chosen_storage = &storages_.back();
}
}
}
if (for_writing) {
if (!chosen_storage) {
// Allocate a new storage supporting the requested view.
auto factory = internal::GpuBufferStorageRegistry::Get()
.StorageFactoryForViewProvider(view_provider_type);
if (factory) {
storages_ = {factory(width(), height(), format())};
chosen_storage = &storages_.back();
}
} else {
// Discard all other storages.
storages_ = {*chosen_storage};
chosen_storage = &storages_.back();
}
}
CHECK(chosen_storage) << "no view provider found";
DCHECK((*chosen_storage)->can_down_cast_to(view_provider_type));
return **chosen_storage;
}
#if !MEDIAPIPE_DISABLE_GPU
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CVPixelBufferRef GetCVPixelBufferRef(const GpuBuffer& buffer) {
auto p = buffer.internal_storage<GpuBufferStorageCvPixelBuffer>();
if (p) return **p;
return nullptr;
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GpuBuffer GpuBuffer::CopyingImageFrame(const ImageFrame& image_frame) {
return GpuBuffer(GlTextureBuffer::Create(image_frame));
}
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#endif // !MEDIAPIPE_DISABLE_GPU
} // namespace mediapipe
mediapipe/gpu/gpu_buffer.h
+75 -73
View File
@@ -19,11 +19,11 @@
#include <utility>
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_texture_view.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
#if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_texture_view.h"
// Note: these headers are needed for the legacy storage APIs. Do not add more
// storage-specific headers here. See WebGpuTextureBuffer/View for an example
// of adding a new storage and view.
@@ -39,19 +39,26 @@
#else
#include "mediapipe/gpu/gl_texture_buffer.h"
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#endif // MEDIAPIPE_DISABLE_GPU
namespace mediapipe {
class GlContext;
// This class wraps a platform-specific buffer of GPU data.
// An instance of GpuBuffer acts as an opaque reference to the underlying
// data object.
class GpuBuffer {
public:
using Format = GpuBufferFormat;
// Default constructor creates invalid object.
GpuBuffer() = default;
// Creates an empty buffer of a given size and format. It will be allocated
// when a view is requested.
GpuBuffer(int width, int height, Format format)
: GpuBuffer(std::make_shared<PlaceholderGpuBufferStorage>(width, height,
format)) {}
// Copy and move constructors and assignment operators are supported.
GpuBuffer(const GpuBuffer& other) = default;
GpuBuffer(GpuBuffer&& other) = default;
@@ -63,30 +70,17 @@ class GpuBuffer {
// are not portable. Applications and calculators should normally obtain
// GpuBuffers in a portable way from the framework, e.g. using
// GpuBufferMultiPool.
explicit GpuBuffer(
std::shared_ptr<mediapipe::internal::GpuBufferStorage> storage)
: storage_(std::move(storage)) {}
// Note: these constructors and accessors for specific storage types exist
// for backwards compatibility reasons. Do not add new ones.
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
explicit GpuBuffer(CFHolder<CVPixelBufferRef> pixel_buffer)
: storage_(std::make_shared<GpuBufferStorageCvPixelBuffer>(
std::move(pixel_buffer))) {}
explicit GpuBuffer(CVPixelBufferRef pixel_buffer)
: storage_(
std::make_shared<GpuBufferStorageCvPixelBuffer>(pixel_buffer)) {}
CVPixelBufferRef GetCVPixelBufferRef() const {
auto p = storage_->down_cast<GpuBufferStorageCvPixelBuffer>();
if (p) return **p;
return nullptr;
explicit GpuBuffer(std::shared_ptr<internal::GpuBufferStorage> storage) {
storages_.push_back(std::move(storage));
}
#else
GlTextureBufferSharedPtr GetGlTextureBufferSharedPtr() const {
return internal_storage<GlTextureBuffer>();
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
// This is used to support backward-compatible construction of GpuBuffer from
// some platform-specific types without having to make those types visible in
// this header.
template <class T, class = std::void_t<decltype(internal::AsGpuBufferStorage(
std::declval<T>()))>>
explicit GpuBuffer(T&& storage_convertible)
: GpuBuffer(internal::AsGpuBufferStorage(storage_convertible)) {}
int width() const { return current_storage().width(); }
int height() const { return current_storage().height(); }
@@ -108,35 +102,19 @@ class GpuBuffer {
// Gets a read view of the specified type. The arguments depend on the
// specific view type; see the corresponding ViewProvider.
template <class View, class... Args>
auto GetReadView(Args... args) const {
return current_storage()
.down_cast<mediapipe::internal::ViewProvider<View>>()
->GetReadView(mediapipe::internal::types<View>{},
std::make_shared<GpuBuffer>(*this),
std::forward<Args>(args)...);
decltype(auto) GetReadView(Args... args) const {
return GetViewProvider<View>(false)->GetReadView(
internal::types<View>{}, std::make_shared<GpuBuffer>(*this),
std::forward<Args>(args)...);
}
// Gets a write view of the specified type. The arguments depend on the
// specific view type; see the corresponding ViewProvider.
template <class View, class... Args>
auto GetWriteView(Args... args) {
return current_storage()
.down_cast<mediapipe::internal::ViewProvider<View>>()
->GetWriteView(mediapipe::internal::types<View>{},
std::make_shared<GpuBuffer>(*this),
std::forward<Args>(args)...);
}
// Make a GpuBuffer copying the data from an ImageFrame.
static GpuBuffer CopyingImageFrame(const ImageFrame& image_frame);
// Make an ImageFrame, possibly sharing the same data. The data is shared if
// the GpuBuffer's storage supports memory sharing; otherwise, it is copied.
// In order to work correctly across platforms, callers should always treat
// the returned ImageFrame as if it shares memory with the GpuBuffer, i.e.
// treat it as immutable if the GpuBuffer must not be modified.
std::unique_ptr<ImageFrame> AsImageFrame() const {
return current_storage().AsImageFrame();
decltype(auto) GetWriteView(Args... args) {
return GetViewProvider<View>(true)->GetWriteView(
internal::types<View>{}, std::make_shared<GpuBuffer>(*this),
std::forward<Args>(args)...);
}
// Attempts to access an underlying storage object of the specified type.
@@ -144,55 +122,79 @@ class GpuBuffer {
// using views.
template <class T>
std::shared_ptr<T> internal_storage() const {
if (storage_->down_cast<T>()) return std::static_pointer_cast<T>(storage_);
for (const auto& s : storages_)
if (s->down_cast<T>()) return std::static_pointer_cast<T>(s);
return nullptr;
}
private:
using TypeRef = internal::TypeRef;
class PlaceholderGpuBufferStorage
: public mediapipe::internal::GpuBufferStorageImpl<
PlaceholderGpuBufferStorage> {
: public internal::GpuBufferStorageImpl<PlaceholderGpuBufferStorage> {
public:
int width() const override { return 0; }
int height() const override { return 0; }
virtual GpuBufferFormat format() const override {
return GpuBufferFormat::kUnknown;
}
std::unique_ptr<ImageFrame> AsImageFrame() const override {
return nullptr;
}
PlaceholderGpuBufferStorage(int width, int height, Format format)
: width_(width), height_(height), format_(format) {}
int width() const override { return width_; }
int height() const override { return height_; }
GpuBufferFormat format() const override { return format_; }
private:
int width_ = 0;
int height_ = 0;
GpuBufferFormat format_ = GpuBufferFormat::kUnknown;
};
std::shared_ptr<mediapipe::internal::GpuBufferStorage>& no_storage() const {
internal::GpuBufferStorage& GetStorageForView(TypeRef view_provider_type,
bool for_writing) const;
template <class View>
internal::ViewProvider<View>* GetViewProvider(bool for_writing) const {
using VP = internal::ViewProvider<View>;
return GetStorageForView(TypeRef::Get<VP>(), for_writing)
.template down_cast<VP>();
}
std::shared_ptr<internal::GpuBufferStorage>& no_storage() const {
static auto placeholder =
std::static_pointer_cast<mediapipe::internal::GpuBufferStorage>(
std::make_shared<PlaceholderGpuBufferStorage>());
std::static_pointer_cast<internal::GpuBufferStorage>(
std::make_shared<PlaceholderGpuBufferStorage>(
0, 0, GpuBufferFormat::kUnknown));
return placeholder;
}
const mediapipe::internal::GpuBufferStorage& current_storage() const {
return *storage_;
const internal::GpuBufferStorage& current_storage() const {
return storages_.empty() ? *no_storage() : *storages_[0];
}
mediapipe::internal::GpuBufferStorage& current_storage() { return *storage_; }
internal::GpuBufferStorage& current_storage() {
return storages_.empty() ? *no_storage() : *storages_[0];
}
std::shared_ptr<mediapipe::internal::GpuBufferStorage> storage_ =
no_storage();
// This is mutable because view methods that do not change the contents may
// still need to allocate new storages.
mutable std::vector<std::shared_ptr<internal::GpuBufferStorage>> storages_;
};
inline bool GpuBuffer::operator==(std::nullptr_t other) const {
return storage_ == no_storage();
return storages_.empty();
}
inline bool GpuBuffer::operator==(const GpuBuffer& other) const {
return storage_ == other.storage_;
return storages_ == other.storages_;
}
inline GpuBuffer& GpuBuffer::operator=(std::nullptr_t other) {
storage_ = no_storage();
storages_.clear();
return *this;
}
// Note: these constructors and accessors for specific storage types exist
// for backwards compatibility reasons. Do not add new ones.
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CVPixelBufferRef GetCVPixelBufferRef(const GpuBuffer& buffer);
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_H_
mediapipe/gpu/gpu_buffer_format.cc
+4
View File
@@ -28,6 +28,7 @@ namespace mediapipe {
#define GL_HALF_FLOAT 0x140B
#endif // GL_HALF_FLOAT
#if !MEDIAPIPE_DISABLE_GPU
#ifdef GL_ES_VERSION_2_0
static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) {
switch (info->gl_internal_format) {
@@ -184,6 +185,7 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
CHECK_LT(plane, planes.size()) << "invalid plane number";
return planes[plane];
}
#endif // MEDIAPIPE_DISABLE_GPU
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
switch (format) {
@@ -202,6 +204,8 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
return ImageFormat::SRGB;
case GpuBufferFormat::kTwoComponentFloat32:
return ImageFormat::VEC32F2;
case GpuBufferFormat::kRGBA32:
// TODO: this likely maps to ImageFormat::SRGBA
case GpuBufferFormat::kGrayHalf16:
case GpuBufferFormat::kOneComponent8Red:
case GpuBufferFormat::kTwoComponent8:
mediapipe/gpu/gpu_buffer_format.h
+11
View File
@@ -23,7 +23,9 @@
#endif // defined(__APPLE__)
#include "mediapipe/framework/formats/image_format.pb.h"
#if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_base.h"
#endif // !MEDIAPIPE_DISABLE_GPU
// The behavior of multi-char constants is implementation-defined, so out of an
// excess of caution we define them in this portable way.
@@ -32,9 +34,12 @@
namespace mediapipe {
using mediapipe::ImageFormat;
enum class GpuBufferFormat : uint32_t {
kUnknown = 0,
kBGRA32 = MEDIAPIPE_FOURCC('B', 'G', 'R', 'A'),
kRGBA32 = MEDIAPIPE_FOURCC('R', 'G', 'B', 'A'),
kGrayFloat32 = MEDIAPIPE_FOURCC('L', '0', '0', 'f'),
kGrayHalf16 = MEDIAPIPE_FOURCC('L', '0', '0', 'h'),
kOneComponent8 = MEDIAPIPE_FOURCC('L', '0', '0', '8'),
@@ -49,6 +54,7 @@ enum class GpuBufferFormat : uint32_t {
kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'),
};
#if !MEDIAPIPE_DISABLE_GPU
// TODO: make this more generally applicable.
enum class GlVersion {
kGL = 1,
@@ -68,6 +74,7 @@ struct GlTextureInfo {
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
int plane,
GlVersion gl_version);
#endif // !MEDIAPIPE_DISABLE_GPU
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format);
GpuBufferFormat GpuBufferFormatForImageFormat(ImageFormat::Format format);
@@ -78,6 +85,8 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) {
switch (format) {
case GpuBufferFormat::kBGRA32:
return kCVPixelFormatType_32BGRA;
case GpuBufferFormat::kRGBA32:
return kCVPixelFormatType_32RGBA;
case GpuBufferFormat::kGrayHalf16:
return kCVPixelFormatType_OneComponent16Half;
case GpuBufferFormat::kGrayFloat32:
@@ -112,6 +121,8 @@ inline GpuBufferFormat GpuBufferFormatForCVPixelFormat(OSType format) {
switch (format) {
case kCVPixelFormatType_32BGRA:
return GpuBufferFormat::kBGRA32;
case kCVPixelFormatType_32RGBA:
return GpuBufferFormat::kRGBA32;
case kCVPixelFormatType_DepthFloat32:
return GpuBufferFormat::kGrayFloat32;
case kCVPixelFormatType_OneComponent16Half:
mediapipe/gpu/gpu_buffer_storage.cc
+47
View File
@@ -0,0 +1,47 @@
#include "mediapipe/gpu/gpu_buffer_storage.h"
namespace mediapipe {
namespace internal {
using StorageFactory = GpuBufferStorageRegistry::StorageFactory;
using StorageConverter = GpuBufferStorageRegistry::StorageConverter;
using RegistryToken = GpuBufferStorageRegistry::RegistryToken;
StorageFactory GpuBufferStorageRegistry::StorageFactoryForViewProvider(
TypeRef view_provider_type) {
auto it = factory_for_view_provider_.find(view_provider_type);
if (it == factory_for_view_provider_.end()) return nullptr;
return it->second;
}
StorageConverter GpuBufferStorageRegistry::StorageConverterForViewProvider(
TypeRef view_provider_type, TypeRef existing_storage_type) {
auto it = converter_for_view_provider_and_existing_storage_.find(
{view_provider_type, existing_storage_type});
if (it == converter_for_view_provider_and_existing_storage_.end())
return nullptr;
return it->second;
}
RegistryToken GpuBufferStorageRegistry::Register(
StorageFactory factory, std::vector<TypeRef> provider_hashes) {
// TODO: choose between multiple factories for same provider type.
for (const auto p : provider_hashes) {
factory_for_view_provider_[p] = factory;
}
return {};
}
RegistryToken GpuBufferStorageRegistry::Register(
StorageConverter converter, std::vector<TypeRef> provider_hashes,
TypeRef source_storage) {
// TODO: choose between multiple converters for same provider type.
for (const auto p : provider_hashes) {
converter_for_view_provider_and_existing_storage_[{p, source_storage}] =
converter;
}
return {};
}
} // namespace internal
} // namespace mediapipe
mediapipe/gpu/gpu_buffer_storage.h
+152 -38
View File
@@ -1,15 +1,17 @@
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_H_
#include "mediapipe/framework/formats/image_frame.h"
#include <functional>
#include <memory>
#include <sstream>
#include "absl/container/flat_hash_map.h"
#include "mediapipe/framework/deps/no_destructor.h"
#include "mediapipe/framework/tool/type_util.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
namespace mediapipe {
class GlTextureView;
class GpuBuffer;
} // namespace mediapipe
namespace mediapipe {
namespace internal {
template <class... T>
@@ -18,69 +20,181 @@ struct types {};
template <class V>
class ViewProvider;
// Note: this specialization temporarily lives here for backwards compatibility
// reasons. New specializations should be put in the same file as their view.
template <>
class ViewProvider<GlTextureView> {
// An identifier for a type. We have often used size_t holding a hash for this
// purpose in MediaPipe, but a non-primitive type makes the code more readable.
// Ideally we should clean up the various ways this is handled throughout the
// framework and consolidate the utilities in type_util. When that is done, this
// type can be replaced.
class TypeRef {
public:
virtual ~ViewProvider() = default;
// Note that the view type is encoded in an argument to allow overloading,
// so a storage class can implement GetRead/WriteView for multiple view types.
// We cannot use a template function because it cannot be virtual; we want to
// have a virtual function here to enforce that different storages supporting
// the same view implement the same signature.
// Note that we allow different views to have custom signatures, providing
// additional view-specific arguments that may be needed.
virtual GlTextureView GetReadView(types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const = 0;
virtual GlTextureView GetWriteView(types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) = 0;
template <class T>
static TypeRef Get() {
return TypeRef{tool::GetTypeHash<T>()};
}
bool operator==(const TypeRef& other) const { return hash_ == other.hash_; }
template <typename H>
friend H AbslHashValue(H h, const TypeRef& r) {
return H::combine(std::move(h), r.hash_);
}
private:
explicit TypeRef(size_t hash) : hash_(hash) {}
size_t hash_;
};
// Interface for a backing storage for GpuBuffer.
class GpuBufferStorage {
public:
virtual ~GpuBufferStorage() = default;
virtual int width() const = 0;
virtual int height() const = 0;
virtual GpuBufferFormat format() const = 0;
virtual std::unique_ptr<ImageFrame> AsImageFrame() const = 0;
// We can't use dynamic_cast since we want to support building without RTTI.
// The public methods delegate to the type-erased private virtual method.
template <class T>
T* down_cast() {
return static_cast<T*>(
const_cast<void*>(down_cast(tool::GetTypeHash<T>())));
return static_cast<T*>(const_cast<void*>(down_cast(TypeRef::Get<T>())));
}
template <class T>
const T* down_cast() const {
return static_cast<const T*>(down_cast(tool::GetTypeHash<T>()));
return static_cast<const T*>(down_cast(TypeRef::Get<T>()));
}
bool can_down_cast_to(TypeRef to) const { return down_cast(to) != nullptr; }
virtual TypeRef storage_type() const = 0;
private:
virtual const void* down_cast(size_t type_hash) const = 0;
virtual size_t storage_type_hash() const = 0;
virtual const void* down_cast(TypeRef to) const = 0;
};
// Used to disambiguate between overloads by manually specifying their priority.
// Higher Ns will be picked first. The caller should pass overload_priority<M>
// where M is >= the largest N used in overloads (e.g. 10).
template <int N>
struct overload_priority : public overload_priority<N - 1> {};
template <>
struct overload_priority<0> {};
// Manages the available GpuBufferStorage implementations. The list of available
// implementations is built at runtime using a registration mechanism, so that
// it can be determined by the program's dependencies.
class GpuBufferStorageRegistry {
public:
struct RegistryToken {};
using StorageFactory = std::function<std::shared_ptr<GpuBufferStorage>(
int, int, GpuBufferFormat)>;
using StorageConverter = std::function<std::shared_ptr<GpuBufferStorage>(
std::shared_ptr<GpuBufferStorage>)>;
static GpuBufferStorageRegistry& Get() {
static NoDestructor<GpuBufferStorageRegistry> registry;
return *registry;
}
template <class Storage>
RegistryToken Register() {
return Register(
[](int width, int height,
GpuBufferFormat format) -> std::shared_ptr<Storage> {
return CreateStorage<Storage>(overload_priority<10>{}, width, height,
format);
},
Storage::GetProviderTypes());
}
template <class StorageFrom, class StorageTo, class F>
RegistryToken RegisterConverter(F&& converter) {
return Register(
[converter](std::shared_ptr<GpuBufferStorage> source)
-> std::shared_ptr<GpuBufferStorage> {
return converter(std::static_pointer_cast<StorageFrom>(source));
},
StorageTo::GetProviderTypes(), TypeRef::Get<StorageFrom>());
}
// Returns a factory function for a storage that implements
// view_provider_type.
StorageFactory StorageFactoryForViewProvider(TypeRef view_provider_type);
// Returns a conversion function that, given a storage of
// existing_storage_type, converts its contents to a new storage that
// implements view_provider_type.
StorageConverter StorageConverterForViewProvider(
TypeRef view_provider_type, TypeRef existing_storage_type);
private:
template <class Storage, class... Args>
static auto CreateStorage(overload_priority<1>, Args... args)
-> decltype(Storage::Create(args...)) {
return Storage::Create(args...);
}
template <class Storage, class... Args>
static auto CreateStorage(overload_priority<0>, Args... args) {
return std::make_shared<Storage>(args...);
}
RegistryToken Register(StorageFactory factory,
std::vector<TypeRef> provider_hashes);
RegistryToken Register(StorageConverter converter,
std::vector<TypeRef> provider_hashes,
TypeRef source_storage);
absl::flat_hash_map<TypeRef, StorageFactory> factory_for_view_provider_;
absl::flat_hash_map<std::pair<TypeRef, TypeRef>, StorageConverter>
converter_for_view_provider_and_existing_storage_;
};
// Defining a member of this type causes P to be ODR-used, which forces its
// instantiation if it's a static member of a template.
template <auto* P>
struct ForceStaticInstantiation {
#ifdef _MSC_VER
// Just having it as the template argument does not count as a use for
// MSVC.
static constexpr bool Use() { return P != nullptr; }
char force_static[Use()];
#endif // _MSC_VER
};
// T: storage type
// U...: ViewProvider<SomeView>
template <class T, class... U>
class GpuBufferStorageImpl : public GpuBufferStorage, public U... {
private:
virtual const void* down_cast(size_t type_hash) const override {
return down_cast_impl(type_hash, types<T, U...>{});
public:
static const std::vector<TypeRef>& GetProviderTypes() {
static std::vector<TypeRef> kHashes{TypeRef::Get<U>()...};
return kHashes;
}
size_t storage_type_hash() const override { return tool::GetTypeHash<T>(); }
const void* down_cast_impl(size_t type_hash, types<>) const {
return nullptr;
private:
virtual const void* down_cast(TypeRef to) const override {
return down_cast_impl(to, types<T, U...>{});
}
TypeRef storage_type() const override { return TypeRef::Get<T>(); }
const void* down_cast_impl(TypeRef to, types<>) const { return nullptr; }
template <class V, class... W>
const void* down_cast_impl(size_t type_hash, types<V, W...>) const {
if (type_hash == tool::GetTypeHash<V>()) return static_cast<const V*>(this);
return down_cast_impl(type_hash, types<W...>{});
const void* down_cast_impl(TypeRef to, types<V, W...>) const {
if (to == TypeRef::Get<V>()) return static_cast<const V*>(this);
return down_cast_impl(to, types<W...>{});
}
inline static auto registration =
GpuBufferStorageRegistry::Get().Register<T>();
using RequireStatics = ForceStaticInstantiation<&registration>;
};
// This function can be overridden to enable construction of a GpuBuffer from
// platform-specific types without having to expose that type in the GpuBuffer
// definition. It is only needed for backward compatibility reasons; do not add
// overrides for new types.
std::shared_ptr<internal::GpuBufferStorage> AsGpuBufferStorage();
} // namespace internal
} // namespace mediapipe
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc
+66 -21
View File
@@ -1,6 +1,9 @@
#include "mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.h"
#include <memory>
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gpu_buffer_storage_image_frame.h"
#include "mediapipe/objc/util.h"
namespace mediapipe {
@@ -11,9 +14,20 @@ typedef CVOpenGLTextureRef CVTextureType;
typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
GlTextureView GpuBufferStorageCvPixelBuffer::GetReadView(
mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) const {
GpuBufferStorageCvPixelBuffer::GpuBufferStorageCvPixelBuffer(
int width, int height, GpuBufferFormat format) {
OSType cv_format = CVPixelFormatForGpuBufferFormat(format);
CHECK_NE(cv_format, -1) << "unsupported pixel format";
CVPixelBufferRef buffer;
CVReturn err =
CreateCVPixelBufferWithoutPool(width, height, cv_format, &buffer);
CHECK(!err) << "Error creating pixel buffer: " << err;
adopt(buffer);
}
GlTextureView GpuBufferStorageCvPixelBuffer::GetTexture(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane,
GlTextureView::DoneWritingFn done_writing) const {
CVReturn err;
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
@@ -29,8 +43,8 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetReadView(
return GlTextureView(
gl_context.get(), CVOpenGLTextureGetTarget(*cv_texture),
CVOpenGLTextureGetName(*cv_texture), width(), height(), *this, plane,
[cv_texture](
mediapipe::GlTextureView&) { /* only retains cv_texture */ });
[cv_texture](mediapipe::GlTextureView&) { /* only retains cv_texture */ },
done_writing);
#else
const GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
format(), plane, gl_context->GetGlVersion());
@@ -49,23 +63,31 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetReadView(
CVOpenGLESTextureGetName(*cv_texture), width(), height(),
std::move(gpu_buffer), plane,
[cv_texture](mediapipe::GlTextureView&) { /* only retains cv_texture */ },
// TODO: make GetGlTextureView for write view non-const, remove cast
// Note: we have to copy *this here because this storage is currently
// stored in GpuBuffer by value, and so the this pointer becomes invalid
// if the GpuBuffer is moved/copied. TODO: fix this.
[me = *this](const mediapipe::GlTextureView& view) {
const_cast<GpuBufferStorageCvPixelBuffer*>(&me)->ViewDoneWriting(view);
});
done_writing);
#endif // TARGET_OS_OSX
}
GlTextureView GpuBufferStorageCvPixelBuffer::GetReadView(
internal::types<GlTextureView>, std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const {
return GetTexture(std::move(gpu_buffer), plane, nullptr);
}
GlTextureView GpuBufferStorageCvPixelBuffer::GetWriteView(
mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) {
// For this storage there is currently no difference between read and write
// views, so we delegate to the read method.
return GetReadView(mediapipe::internal::types<GlTextureView>{},
std::move(gpu_buffer), plane);
internal::types<GlTextureView>, std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) {
return GetTexture(
std::move(gpu_buffer), plane,
[this](const mediapipe::GlTextureView& view) { ViewDoneWriting(view); });
}
std::shared_ptr<const ImageFrame> GpuBufferStorageCvPixelBuffer::GetReadView(
internal::types<ImageFrame>, std::shared_ptr<GpuBuffer> gpu_buffer) const {
return CreateImageFrameForCVPixelBuffer(**this);
}
std::shared_ptr<ImageFrame> GpuBufferStorageCvPixelBuffer::GetWriteView(
internal::types<ImageFrame>, std::shared_ptr<GpuBuffer> gpu_buffer) {
return CreateImageFrameForCVPixelBuffer(**this);
}
void GpuBufferStorageCvPixelBuffer::ViewDoneWriting(const GlTextureView& view) {
@@ -111,9 +133,32 @@ void GpuBufferStorageCvPixelBuffer::ViewDoneWriting(const GlTextureView& view) {
#endif
}
std::unique_ptr<ImageFrame> GpuBufferStorageCvPixelBuffer::AsImageFrame()
const {
return CreateImageFrameForCVPixelBuffer(**this);
static std::shared_ptr<GpuBufferStorageCvPixelBuffer> ConvertFromImageFrame(
std::shared_ptr<GpuBufferStorageImageFrame> frame) {
auto status_or_buffer =
CreateCVPixelBufferForImageFrame(frame->image_frame());
CHECK(status_or_buffer.ok());
return std::make_shared<GpuBufferStorageCvPixelBuffer>(
std::move(status_or_buffer).value());
}
static auto kConverterFromImageFrameRegistration =
internal::GpuBufferStorageRegistry::Get()
.RegisterConverter<GpuBufferStorageImageFrame,
GpuBufferStorageCvPixelBuffer>(
ConvertFromImageFrame);
namespace internal {
std::shared_ptr<internal::GpuBufferStorage> AsGpuBufferStorage(
CFHolder<CVPixelBufferRef> pixel_buffer) {
return std::make_shared<GpuBufferStorageCvPixelBuffer>(
std::move(pixel_buffer));
}
std::shared_ptr<internal::GpuBufferStorage> AsGpuBufferStorage(
CVPixelBufferRef pixel_buffer) {
return std::make_shared<GpuBufferStorageCvPixelBuffer>(pixel_buffer);
}
} // namespace internal
} // namespace mediapipe
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.h
+25 -6
View File
@@ -5,6 +5,7 @@
#include "mediapipe/gpu/gl_texture_view.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
#include "mediapipe/gpu/image_frame_view.h"
#include "mediapipe/objc/CFHolder.h"
namespace mediapipe {
@@ -12,12 +13,13 @@ namespace mediapipe {
class GlContext;
class GpuBufferStorageCvPixelBuffer
: public mediapipe::internal::GpuBufferStorageImpl<
GpuBufferStorageCvPixelBuffer,
mediapipe::internal::ViewProvider<GlTextureView>>,
: public internal::GpuBufferStorageImpl<
GpuBufferStorageCvPixelBuffer, internal::ViewProvider<GlTextureView>,
internal::ViewProvider<ImageFrame>>,
public CFHolder<CVPixelBufferRef> {
public:
using CFHolder<CVPixelBufferRef>::CFHolder;
GpuBufferStorageCvPixelBuffer(int width, int height, GpuBufferFormat format);
GpuBufferStorageCvPixelBuffer(const CFHolder<CVPixelBufferRef>& other)
: CFHolder(other) {}
GpuBufferStorageCvPixelBuffer(CFHolder<CVPixelBufferRef>&& other)
@@ -30,18 +32,35 @@ class GpuBufferStorageCvPixelBuffer
return GpuBufferFormatForCVPixelFormat(
CVPixelBufferGetPixelFormatType(**this));
}
GlTextureView GetReadView(mediapipe::internal::types<GlTextureView>,
GlTextureView GetReadView(internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override;
GlTextureView GetWriteView(mediapipe::internal::types<GlTextureView>,
GlTextureView GetWriteView(internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override;
std::unique_ptr<ImageFrame> AsImageFrame() const override;
std::shared_ptr<const ImageFrame> GetReadView(
internal::types<ImageFrame>,
std::shared_ptr<GpuBuffer> gpu_buffer) const override;
std::shared_ptr<ImageFrame> GetWriteView(
internal::types<ImageFrame>,
std::shared_ptr<GpuBuffer> gpu_buffer) override;
private:
GlTextureView GetTexture(std::shared_ptr<GpuBuffer> gpu_buffer, int plane,
GlTextureView::DoneWritingFn done_writing) const;
void ViewDoneWriting(const GlTextureView& view);
};
namespace internal {
// These functions enable backward-compatible construction of a GpuBuffer from
// CVPixelBufferRef without having to expose that type in the main GpuBuffer
// header.
std::shared_ptr<internal::GpuBufferStorage> AsGpuBufferStorage(
CFHolder<CVPixelBufferRef> pixel_buffer);
std::shared_ptr<internal::GpuBufferStorage> AsGpuBufferStorage(
CVPixelBufferRef pixel_buffer);
} // namespace internal
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_CV_PIXEL_BUFFER_H_
mediapipe/gpu/gpu_buffer_storage_image_frame.h
+48
View File
@@ -0,0 +1,48 @@
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_IMAGE_FRAME_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_IMAGE_FRAME_H_
#include <memory>
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
#include "mediapipe/gpu/image_frame_view.h"
namespace mediapipe {
// Implements support for ImageFrame as a backing storage of GpuBuffer.
class GpuBufferStorageImageFrame
: public internal::GpuBufferStorageImpl<
GpuBufferStorageImageFrame, internal::ViewProvider<ImageFrame>> {
public:
explicit GpuBufferStorageImageFrame(std::shared_ptr<ImageFrame> image_frame)
: image_frame_(image_frame) {}
GpuBufferStorageImageFrame(int width, int height, GpuBufferFormat format) {
image_frame_ = std::make_shared<ImageFrame>(
ImageFormatForGpuBufferFormat(format), width, height);
}
int width() const override { return image_frame_->Width(); }
int height() const override { return image_frame_->Height(); }
GpuBufferFormat format() const override {
return GpuBufferFormatForImageFormat(image_frame_->Format());
}
std::shared_ptr<const ImageFrame> image_frame() const { return image_frame_; }
std::shared_ptr<ImageFrame> image_frame() { return image_frame_; }
std::shared_ptr<const ImageFrame> GetReadView(
internal::types<ImageFrame>,
std::shared_ptr<GpuBuffer> gpu_buffer) const override {
return image_frame_;
}
std::shared_ptr<ImageFrame> GetWriteView(
internal::types<ImageFrame>,
std::shared_ptr<GpuBuffer> gpu_buffer) override {
return image_frame_;
}
private:
std::shared_ptr<ImageFrame> image_frame_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_IMAGE_FRAME_H_
mediapipe/gpu/gpu_buffer_test.cc
+215
View File
@@ -14,13 +14,89 @@
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/tool/test_util.h"
#include "mediapipe/gpu/gpu_buffer_storage_image_frame.h"
#include "mediapipe/gpu/gpu_test_base.h"
#include "stb_image.h"
#include "stb_image_write.h"
namespace mediapipe {
namespace {
// Write an ImageFrame as PNG to the test undeclared outputs directory.
// The image's name will contain the given prefix and a timestamp.
// Returns the path to the output if successful.
std::string SavePngImage(const mediapipe::ImageFrame& image,
absl::string_view prefix) {
std::string output_dir = mediapipe::GetTestOutputsDir();
std::string now_string = absl::FormatTime(absl::Now());
std::string out_file_path =
absl::StrCat(output_dir, "/", prefix, "_", now_string, ".png");
EXPECT_TRUE(stbi_write_png(out_file_path.c_str(), image.Width(),
image.Height(), image.NumberOfChannels(),
image.PixelData(), image.WidthStep()))
<< " path: " << out_file_path;
return out_file_path;
}
void FillImageFrameRGBA(ImageFrame& image, uint8 r, uint8 g, uint8 b, uint8 a) {
auto* data = image.MutablePixelData();
for (int y = 0; y < image.Height(); ++y) {
auto* row = data + image.WidthStep() * y;
for (int x = 0; x < image.Width(); ++x) {
auto* pixel = row + x * image.NumberOfChannels();
pixel[0] = r;
pixel[1] = g;
pixel[2] = b;
pixel[3] = a;
}
}
}
// Assumes a framebuffer is already set up
void CopyGlTexture(const GlTextureView& src, GlTextureView& dst) {
glViewport(0, 0, src.width(), src.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, src.target(),
src.name(), 0);
glActiveTexture(GL_TEXTURE0);
glBindTexture(dst.target(), dst.name());
glCopyTexSubImage2D(dst.target(), 0, 0, 0, 0, 0, dst.width(), dst.height());
glBindTexture(dst.target(), 0);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, src.target(), 0,
0);
}
void FillGlTextureRgba(GlTextureView& view, float r, float g, float b,
float a) {
glViewport(0, 0, view.width(), view.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, view.target(),
view.name(), 0);
glClearColor(r, g, b, a);
glClear(GL_COLOR_BUFFER_BIT);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, view.target(), 0,
0);
}
class TempGlFramebuffer {
public:
TempGlFramebuffer() {
glGenFramebuffers(1, &framebuffer_);
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
}
~TempGlFramebuffer() {
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glDeleteFramebuffers(1, &framebuffer_);
}
private:
GLuint framebuffer_;
};
class GpuBufferTest : public GpuTestBase {};
TEST_F(GpuBufferTest, BasicTest) {
@@ -46,5 +122,144 @@ TEST_F(GpuBufferTest, BasicTest) {
});
}
TEST_F(GpuBufferTest, GlTextureView) {
GpuBuffer buffer(300, 200, GpuBufferFormat::kBGRA32);
EXPECT_EQ(buffer.width(), 300);
EXPECT_EQ(buffer.height(), 200);
EXPECT_TRUE(buffer);
EXPECT_FALSE(buffer == nullptr);
RunInGlContext([&buffer] {
TempGlFramebuffer fb;
auto view = buffer.GetWriteView<GlTextureView>(0);
FillGlTextureRgba(view, 1.0, 0.0, 0.0, 1.0);
glFlush();
});
std::shared_ptr<const ImageFrame> view = buffer.GetReadView<ImageFrame>();
EXPECT_EQ(view->Width(), 300);
EXPECT_EQ(view->Height(), 200);
ImageFrame red(ImageFormat::SRGBA, 300, 200);
FillImageFrameRGBA(red, 255, 0, 0, 255);
EXPECT_TRUE(mediapipe::CompareImageFrames(*view, red, 0.0, 0.0));
SavePngImage(red, "gltv_red_gold");
SavePngImage(*view, "gltv_red_view");
}
TEST_F(GpuBufferTest, ImageFrame) {
GpuBuffer buffer(300, 200, GpuBufferFormat::kBGRA32);
EXPECT_EQ(buffer.width(), 300);
EXPECT_EQ(buffer.height(), 200);
EXPECT_TRUE(buffer);
EXPECT_FALSE(buffer == nullptr);
{
std::shared_ptr<ImageFrame> view = buffer.GetWriteView<ImageFrame>();
EXPECT_EQ(view->Width(), 300);
EXPECT_EQ(view->Height(), 200);
FillImageFrameRGBA(*view, 255, 0, 0, 255);
}
GpuBuffer buffer2(300, 200, GpuBufferFormat::kBGRA32);
RunInGlContext([&buffer, &buffer2] {
TempGlFramebuffer fb;
auto src = buffer.GetReadView<GlTextureView>(0);
auto dst = buffer2.GetWriteView<GlTextureView>(0);
CopyGlTexture(src, dst);
glFlush();
});
{
std::shared_ptr<const ImageFrame> view = buffer2.GetReadView<ImageFrame>();
EXPECT_EQ(view->Width(), 300);
EXPECT_EQ(view->Height(), 200);
ImageFrame red(ImageFormat::SRGBA, 300, 200);
FillImageFrameRGBA(red, 255, 0, 0, 255);
EXPECT_TRUE(mediapipe::CompareImageFrames(*view, red, 0.0, 0.0));
SavePngImage(red, "if_red_gold");
SavePngImage(*view, "if_red_view");
}
}
TEST_F(GpuBufferTest, Overwrite) {
GpuBuffer buffer(300, 200, GpuBufferFormat::kBGRA32);
EXPECT_EQ(buffer.width(), 300);
EXPECT_EQ(buffer.height(), 200);
EXPECT_TRUE(buffer);
EXPECT_FALSE(buffer == nullptr);
{
std::shared_ptr<ImageFrame> view = buffer.GetWriteView<ImageFrame>();
EXPECT_EQ(view->Width(), 300);
EXPECT_EQ(view->Height(), 200);
FillImageFrameRGBA(*view, 255, 0, 0, 255);
}
GpuBuffer red_copy(300, 200, GpuBufferFormat::kBGRA32);
RunInGlContext([&buffer, &red_copy] {
TempGlFramebuffer fb;
auto src = buffer.GetReadView<GlTextureView>(0);
auto dst = red_copy.GetWriteView<GlTextureView>(0);
CopyGlTexture(src, dst);
glFlush();
});
{
std::shared_ptr<const ImageFrame> view = red_copy.GetReadView<ImageFrame>();
ImageFrame red(ImageFormat::SRGBA, 300, 200);
FillImageFrameRGBA(red, 255, 0, 0, 255);
EXPECT_TRUE(mediapipe::CompareImageFrames(*view, red, 0.0, 0.0));
SavePngImage(red, "ow_red_gold");
SavePngImage(*view, "ow_red_view");
}
{
std::shared_ptr<ImageFrame> view = buffer.GetWriteView<ImageFrame>();
EXPECT_EQ(view->Width(), 300);
EXPECT_EQ(view->Height(), 200);
FillImageFrameRGBA(*view, 0, 255, 0, 255);
}
GpuBuffer green_copy(300, 200, GpuBufferFormat::kBGRA32);
RunInGlContext([&buffer, &green_copy] {
TempGlFramebuffer fb;
auto src = buffer.GetReadView<GlTextureView>(0);
auto dst = green_copy.GetWriteView<GlTextureView>(0);
CopyGlTexture(src, dst);
glFlush();
});
RunInGlContext([&buffer] {
TempGlFramebuffer fb;
auto view = buffer.GetWriteView<GlTextureView>(0);
FillGlTextureRgba(view, 0.0, 0.0, 1.0, 1.0);
glFlush();
});
{
std::shared_ptr<const ImageFrame> view =
green_copy.GetReadView<ImageFrame>();
ImageFrame green(ImageFormat::SRGBA, 300, 200);
FillImageFrameRGBA(green, 0, 255, 0, 255);
EXPECT_TRUE(mediapipe::CompareImageFrames(*view, green, 0.0, 0.0));
SavePngImage(green, "ow_green_gold");
SavePngImage(*view, "ow_green_view");
}
{
std::shared_ptr<const ImageFrame> view = buffer.GetReadView<ImageFrame>();
ImageFrame blue(ImageFormat::SRGBA, 300, 200);
FillImageFrameRGBA(blue, 0, 0, 255, 255);
EXPECT_TRUE(mediapipe::CompareImageFrames(*view, blue, 0.0, 0.0));
SavePngImage(blue, "ow_blue_gold");
SavePngImage(*view, "ow_blue_view");
}
}
} // anonymous namespace
} // namespace mediapipe
mediapipe/gpu/gpu_buffer_to_image_frame_calculator.cc
+1 -1
View File
@@ -76,7 +76,7 @@ absl::Status GpuBufferToImageFrameCalculator::Process(CalculatorContext* cc) {
const auto& input = cc->Inputs().Index(0).Get<GpuBuffer>();
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
std::unique_ptr<ImageFrame> frame =
CreateImageFrameForCVPixelBuffer(input.GetCVPixelBufferRef());
CreateImageFrameForCVPixelBuffer(GetCVPixelBufferRef(input));
cc->Outputs().Index(0).Add(frame.release(), cc->InputTimestamp());
#else
helper_.RunInGlContext([this, &input, &cc]() {
mediapipe/gpu/gpu_service.cc
+1 -1
View File
@@ -16,6 +16,6 @@
namespace mediapipe {
const GraphService<::mediapipe::GpuResources> kGpuService("kGpuService");
const GraphService<GpuResources> kGpuService("kGpuService");
} // namespace mediapipe
mediapipe/gpu/gpu_service.h
+2 -5
View File
@@ -18,12 +18,9 @@
#include "mediapipe/framework/graph_service.h"
namespace mediapipe {
class GpuResources;
} // namespace mediapipe
namespace mediapipe {
extern const GraphService<::mediapipe::GpuResources> kGpuService;
extern const GraphService<GpuResources> kGpuService;
} // namespace mediapipe
mediapipe/gpu/gpu_shared_data_internal.cc
+2 -1
View File
@@ -117,7 +117,8 @@ absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) {
(node_type == "GpuBufferToImageFrameCalculator") ||
(node_type == "GlSurfaceSinkCalculator");
const auto& options = node->GetCalculatorState().Options<GlContextOptions>();
const auto& options =
node->GetCalculatorState().Options<mediapipe::GlContextOptions>();
if (options.has_gl_context_name() && !options.gl_context_name().empty()) {
context_key = absl::StrCat("user:", options.gl_context_name());
} else if (gets_own_context) {
mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc
+14 -22
View File
@@ -16,10 +16,7 @@
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#ifdef __APPLE__
#include "mediapipe/objc/util.h"
#endif
#include "mediapipe/gpu/gpu_buffer_storage_image_frame.h"
namespace mediapipe {
@@ -34,9 +31,7 @@ class ImageFrameToGpuBufferCalculator : public CalculatorBase {
absl::Status Process(CalculatorContext* cc) override;
private:
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlCalculatorHelper helper_;
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
};
REGISTER_CALCULATOR(ImageFrameToGpuBufferCalculator);
@@ -56,28 +51,25 @@ absl::Status ImageFrameToGpuBufferCalculator::Open(CalculatorContext* cc) {
// Inform the framework that we always output at the same timestamp
// as we receive a packet at.
cc->SetOffset(TimestampDiff(0));
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
MP_RETURN_IF_ERROR(helper_.Open(cc));
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return absl::OkStatus();
}
absl::Status ImageFrameToGpuBufferCalculator::Process(CalculatorContext* cc) {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CFHolder<CVPixelBufferRef> buffer;
MP_RETURN_IF_ERROR(CreateCVPixelBufferForImageFramePacket(
cc->Inputs().Index(0).Value(), &buffer));
cc->Outputs().Index(0).Add(new GpuBuffer(buffer), cc->InputTimestamp());
#else
const auto& input = cc->Inputs().Index(0).Get<ImageFrame>();
helper_.RunInGlContext([this, &input, &cc]() {
auto src = helper_.CreateSourceTexture(input);
auto output = src.GetFrame<GpuBuffer>();
glFlush();
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
src.Release();
auto image_frame = std::const_pointer_cast<ImageFrame>(
mediapipe::SharedPtrWithPacket<ImageFrame>(
cc->Inputs().Index(0).Value()));
auto gpu_buffer = MakePacket<GpuBuffer>(
std::make_shared<mediapipe::GpuBufferStorageImageFrame>(
std::move(image_frame)))
.At(cc->InputTimestamp());
// Request GPU access to ensure the data is available to the GPU.
// TODO: have a better way to do this, or defer until later.
helper_.RunInGlContext([&gpu_buffer] {
auto view = gpu_buffer.Get<GpuBuffer>().GetReadView<GlTextureView>(0);
});
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
cc->Outputs().Index(0).AddPacket(std::move(gpu_buffer));
return absl::OkStatus();
}
mediapipe/gpu/image_frame_view.h
+23
View File
@@ -0,0 +1,23 @@
#ifndef MEDIAPIPE_GPU_IMAGE_FRAME_VIEW_H_
#define MEDIAPIPE_GPU_IMAGE_FRAME_VIEW_H_
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
namespace mediapipe {
namespace internal {
template <>
class ViewProvider<ImageFrame> {
public:
virtual ~ViewProvider() = default;
virtual std::shared_ptr<const ImageFrame> GetReadView(
types<ImageFrame>, std::shared_ptr<GpuBuffer> gpu_buffer) const = 0;
virtual std::shared_ptr<ImageFrame> GetWriteView(
types<ImageFrame>, std::shared_ptr<GpuBuffer> gpu_buffer) = 0;
};
} // namespace internal
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_IMAGE_FRAME_VIEW_H_
mediapipe/gpu/metal.bzl
-8
View File
@@ -134,16 +134,8 @@ def _metal_library_impl(ctx):
),
)
# This circumlocution is needed because new_objc_provider rejects
# an empty depset, with the error:
# "Value for key header must be a set of File, instead found set of unknown."
# It also rejects an explicit "None".
additional_params = {}
if ctx.files.hdrs:
additional_params["header"] = depset([f for f in ctx.files.hdrs])
objc_provider = apple_common.new_objc_provider(
providers = [x[apple_common.Objc] for x in ctx.attr.deps if apple_common.Objc in x],
**additional_params
)
cc_infos = [dep[CcInfo] for dep in ctx.attr.deps if CcInfo in dep]