talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 373e3ac1e5839befd95bf7d73ceff3c5f1171969
This commit is contained in:
MediaPipe Team
2021-10-06 14:27:49 -07:00
committed by jqtang
parent 137e1cc763
commit 33d683c671
153 changed files with 7871 additions and 1349 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/gpu/BUILD
+39 -62
View File
@@ -207,16 +207,20 @@ cc_library(
cc_library(
name = "gpu_buffer",
srcs = ["gpu_buffer.cc"],
hdrs = ["gpu_buffer.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_context",
":gpu_buffer_format",
"//mediapipe/framework/formats:image_frame",
] + select({
"//conditions:default": [
":gl_texture_buffer",
],
"//mediapipe:ios": [
"//mediapipe/objc:util",
"//mediapipe/objc:CFHolder",
],
"//mediapipe:macos": [
@@ -478,6 +482,7 @@ cc_library(
"//mediapipe:ios": [
":pixel_buffer_pool_util",
"//mediapipe/objc:CFHolder",
"//mediapipe/objc:util",
],
"//mediapipe:macos": [
":pixel_buffer_pool_util",
@@ -498,55 +503,40 @@ cc_library(
],
)
HELPER_ANDROID_SRCS = [
"gl_calculator_helper_impl_android.cc",
"gl_calculator_helper_impl_common.cc",
]
HELPER_ANDROID_HDRS = [
"egl_surface_holder.h",
]
HELPER_COMMON_SRCS = [
"gl_calculator_helper.cc",
]
HELPER_COMMON_HDRS = [
"gl_calculator_helper.h",
"gl_calculator_helper_impl.h",
]
HELPER_IOS_SRCS = [
"gl_calculator_helper_impl_ios.mm",
"gl_calculator_helper_impl_common.cc",
]
HELPER_IOS_FRAMEWORKS = [
"AVFoundation",
"CoreVideo",
"CoreGraphics",
"CoreMedia",
"GLKit",
"QuartzCore",
] + select({
"//conditions:default": [
"OpenGLES",
cc_library(
name = "egl_surface_holder",
hdrs = ["egl_surface_holder.h"],
deps = [
":gl_base",
"@com_google_absl//absl/synchronization",
],
"//mediapipe:macos": [
"OpenGL",
"AppKit",
],
})
)
cc_library(
name = "gl_calculator_helper",
srcs = select({
"//conditions:default": HELPER_COMMON_SRCS + HELPER_ANDROID_SRCS,
"//mediapipe:apple": [],
}),
hdrs = HELPER_COMMON_HDRS + select({
"//conditions:default": HELPER_ANDROID_HDRS,
"//mediapipe:apple": [],
srcs = [
"gl_calculator_helper.cc",
"gl_calculator_helper_impl_common.cc",
],
hdrs = [
"gl_calculator_helper.h",
"gl_calculator_helper_impl.h",
],
linkopts = select({
"//conditions:default": [],
"//mediapipe:apple": [
"-framework AVFoundation",
"-framework CoreVideo",
"-framework CoreGraphics",
"-framework CoreMedia",
"-framework GLKit",
"-framework QuartzCore",
],
}) + select({
"//conditions:default": [],
"//mediapipe:macos": [
"-framework AppKit",
],
}),
visibility = ["//visibility:public"],
deps = [
@@ -582,34 +572,20 @@ cc_library(
] + select({
"//conditions:default": [
],
"//mediapipe:apple": [
":gl_calculator_helper_ios",
"//mediapipe/objc:util",
"//mediapipe/objc:CFHolder",
],
"//mediapipe:apple": [],
}),
)
# TODO: remove
objc_library(
name = "gl_calculator_helper_ios",
srcs = HELPER_COMMON_SRCS + HELPER_IOS_SRCS,
hdrs = HELPER_COMMON_HDRS,
copts = [
"-Wno-shorten-64-to-32",
"-std=c++17",
],
sdk_frameworks = HELPER_IOS_FRAMEWORKS,
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_context",
":gpu_buffer",
":gpu_buffer_multi_pool",
":gpu_service",
":gpu_shared_data_internal",
":shader_util",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image",
":gl_calculator_helper",
"//mediapipe/objc:mediapipe_framework_ios",
"//mediapipe/objc:util",
],
@@ -769,6 +745,7 @@ cc_library(
srcs = ["gl_surface_sink_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":egl_surface_holder",
":gl_calculator_helper",
":gl_quad_renderer",
":gpu_buffer",
mediapipe/gpu/gl_calculator_helper.cc
-7
View File
@@ -24,15 +24,8 @@
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_service.h"
#ifdef __APPLE__
#include "mediapipe/objc/util.h"
#endif
namespace mediapipe {
GlTexture::GlTexture(GLuint name, int width, int height)
: name_(name), width_(width), height_(height), target_(GL_TEXTURE_2D) {}
// The constructor and destructor need to be defined here so that
// std::unique_ptr can see the full definition of GlCalculatorHelperImpl.
// In the header, it is an incomplete type.
mediapipe/gpu/gl_calculator_helper.h
+7 -33
View File
@@ -31,8 +31,6 @@
#ifdef __APPLE__
#include <CoreVideo/CoreVideo.h>
#include "mediapipe/objc/CFHolder.h"
#endif // __APPLE__
namespace mediapipe {
@@ -42,14 +40,6 @@ class GlTexture;
class GpuResources;
struct GpuSharedData;
#ifdef __APPLE__
#if TARGET_OS_OSX
typedef CVOpenGLTextureRef CVTextureType;
#else
typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
#endif // __APPLE__
using ImageFrameSharedPtr = std::shared_ptr<ImageFrame>;
// TODO: remove this and Process below, or make Process available
@@ -174,14 +164,12 @@ class GlCalculatorHelper {
class GlTexture {
public:
GlTexture() {}
GlTexture(GLuint name, int width, int height);
~GlTexture() { Release(); }
int width() const { return width_; }
int height() const { return height_; }
GLenum target() const { return target_; }
GLuint name() const { return name_; }
int width() const { return view_.width(); }
int height() const { return view_.height(); }
GLenum target() const { return view_.target(); }
GLuint name() const { return view_.name(); }
// Returns a buffer that can be sent to another calculator.
// & manages sync token
@@ -190,26 +178,12 @@ class GlTexture {
std::unique_ptr<T> GetFrame() const;
// Releases texture memory & manages sync token
void Release();
void Release() { view_.Release(); }
private:
explicit GlTexture(GlTextureView view) : view_(std::move(view)) {}
friend class GlCalculatorHelperImpl;
GlCalculatorHelperImpl* helper_impl_ = nullptr;
GLuint name_ = 0;
int width_ = 0;
int height_ = 0;
GLenum target_ = GL_TEXTURE_2D;
#ifdef MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
// For CVPixelBufferRef-based rendering
CFHolder<CVTextureType> cv_texture_;
#else
// Keeps track of whether this texture mapping is for read access, so that
// we can create a consumer sync point when releasing it.
bool for_reading_ = false;
#endif
GpuBuffer gpu_buffer_;
int plane_ = 0;
GlTextureView view_;
};
// Returns the entry with the given tag if the collection uses tags, with the
mediapipe/gpu/gl_calculator_helper_impl.h
+5 -14
View File
@@ -58,19 +58,14 @@ class GlCalculatorHelperImpl {
GlContext& GetGlContext() const;
// For internal use.
void ReadTexture(const GlTexture& texture, void* output, size_t size);
static void ReadTexture(const GlTextureView& view, void* output, size_t size);
private:
// Makes a GpuBuffer accessible as a texture in the GL context.
GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, int plane);
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTexture MapGlTextureBuffer(const GlTextureBufferSharedPtr& texture_buffer);
GlTextureBufferSharedPtr MakeGlTextureBuffer(const ImageFrame& image_frame);
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
// Sets default texture filtering parameters.
void SetStandardTextureParams(GLenum target, GLint internal_format);
GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, int plane,
bool for_reading);
void AttachGlTexture(GlTexture& texture, const GpuBuffer& gpu_buffer,
int plane, bool for_reading);
// Create the framebuffer for rendering.
void CreateFramebuffer();
@@ -80,10 +75,6 @@ class GlCalculatorHelperImpl {
GLuint framebuffer_ = 0;
GpuResources& gpu_resources_;
// Necessary to compute for a given GlContext in order to properly enforce the
// SetStandardTextureParams.
bool can_linear_filter_float_textures_;
};
} // namespace mediapipe
mediapipe/gpu/gl_calculator_helper_impl_android.cc
-102
View File
@@ -1,102 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
namespace mediapipe {
// TODO: move this method to GlCalculatorHelper, then we can
// access its framebuffer instead of requiring that one is already set.
template <>
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
auto output =
absl::make_unique<ImageFrame>(ImageFormat::SRGBA, width_, height_,
ImageFrame::kGlDefaultAlignmentBoundary);
CHECK(helper_impl_);
helper_impl_->ReadTexture(*this, output->MutablePixelData(),
output->PixelDataSize());
return output;
}
template <>
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
#ifdef __EMSCRIPTEN__
// When WebGL is used, the GL context may be spontaneously lost which can
// cause GpuBuffer allocations to fail. In that case, return a dummy buffer
// to allow processing of the current frame complete.
if (!gpu_buffer_) {
return std::make_unique<GpuBuffer>();
}
#endif // __EMSCRIPTEN__
CHECK(gpu_buffer_);
// Inform the GlTextureBuffer that we have produced new content, and create
// a producer sync point.
gpu_buffer_.GetGlTextureBufferSharedPtr()->Updated(
helper_impl_->GetGlContext().CreateSyncToken());
#ifdef __ANDROID__
// On (some?) Android devices, the texture may need to be explicitly
// detached from the current framebuffer.
// TODO: is this necessary even with the unbind in BindFramebuffer?
// It is not clear if this affected other contexts too, but let's keep it
// while in doubt.
GLint type = GL_NONE;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE,
&type);
if (type == GL_TEXTURE) {
GLint color_attachment = 0;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment);
if (color_attachment == name_) {
glBindFramebuffer(GL_FRAMEBUFFER, 0);
}
}
// Some Android drivers log a GL_INVALID_ENUM error after the first
// glGetFramebufferAttachmentParameteriv call if there is no bound object,
// even though it should be ok to ask for the type and get back GL_NONE.
// Let's just ignore any pending errors here.
GLenum error;
while ((error = glGetError()) != GL_NO_ERROR) {
}
#endif // __ANDROID__
return absl::make_unique<GpuBuffer>(gpu_buffer_);
}
void GlTexture::Release() {
if (for_reading_ && gpu_buffer_) {
// Inform the GlTextureBuffer that we have finished accessing its contents,
// and create a consumer sync point.
gpu_buffer_.GetGlTextureBufferSharedPtr()->DidRead(
helper_impl_->GetGlContext().CreateSyncToken());
}
helper_impl_ = nullptr;
for_reading_ = false;
gpu_buffer_ = nullptr;
plane_ = 0;
name_ = 0;
width_ = 0;
height_ = 0;
}
} // namespace mediapipe
mediapipe/gpu/gl_calculator_helper_impl_common.cc
+38 -123
View File
@@ -25,17 +25,6 @@ GlCalculatorHelperImpl::GlCalculatorHelperImpl(CalculatorContext* cc,
GpuResources* gpu_resources)
: gpu_resources_(*gpu_resources) {
gl_context_ = gpu_resources_.gl_context(cc);
// GL_ES_VERSION_2_0 and up (at least through ES 3.2) may contain the extension.
// Checking against one also checks against higher ES versions. So this checks
// against GLES >= 2.0.
#if GL_ES_VERSION_2_0
// No linear float filtering by default, check extensions.
can_linear_filter_float_textures_ =
gl_context_->HasGlExtension("OES_texture_float_linear");
#else
// Any float32 texture we create should automatically have linear filtering.
can_linear_filter_float_textures_ = true;
#endif // GL_ES_VERSION_2_0
}
GlCalculatorHelperImpl::~GlCalculatorHelperImpl() {
@@ -101,98 +90,59 @@ void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) {
#endif
}
void GlCalculatorHelperImpl::SetStandardTextureParams(GLenum target,
GLint internal_format) {
// Default to using linear filter everywhere. For float32 textures, fall back
// to GL_NEAREST if linear filtering unsupported.
GLint filter;
switch (internal_format) {
case GL_R32F:
case GL_RG32F:
case GL_RGBA32F:
// 32F (unlike 16f) textures do not always support texture filtering
// (According to OpenGL ES specification [TEXTURE IMAGE SPECIFICATION])
filter = can_linear_filter_float_textures_ ? GL_LINEAR : GL_NEAREST;
break;
default:
filter = GL_LINEAR;
}
glTexParameteri(target, GL_TEXTURE_MIN_FILTER, filter);
glTexParameteri(target, GL_TEXTURE_MAG_FILTER, filter);
glTexParameteri(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
int plane, bool for_reading) {
GlTextureView view = gpu_buffer.GetGlTextureView(plane, for_reading);
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const ImageFrame& image_frame) {
GlTexture texture = MapGlTextureBuffer(MakeGlTextureBuffer(image_frame));
texture.for_reading_ = true;
return texture;
if (gpu_buffer.format() != GpuBufferFormat::kUnknown) {
// TODO: do the params need to be reset here??
glBindTexture(view.target(), view.name());
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
gpu_buffer.format(), view.plane(), GetGlVersion());
gl_context_->SetStandardTextureParams(view.target(),
info.gl_internal_format);
glBindTexture(view.target(), 0);
}
return GlTexture(std::move(view));
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer) {
GlTexture texture = MapGpuBuffer(gpu_buffer, 0);
texture.for_reading_ = true;
return texture;
return MapGpuBuffer(gpu_buffer, 0, true);
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer, int plane) {
GlTexture texture = MapGpuBuffer(gpu_buffer, plane);
texture.for_reading_ = true;
return texture;
return MapGpuBuffer(gpu_buffer, plane, true);
}
GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
int plane) {
CHECK_EQ(plane, 0);
return MapGlTextureBuffer(gpu_buffer.GetGlTextureBufferSharedPtr());
}
GlTexture GlCalculatorHelperImpl::MapGlTextureBuffer(
const GlTextureBufferSharedPtr& texture_buffer) {
// Insert wait call to sync with the producer.
texture_buffer->WaitOnGpu();
GlTexture texture;
texture.helper_impl_ = this;
texture.gpu_buffer_ = GpuBuffer(texture_buffer);
texture.plane_ = 0;
texture.width_ = texture_buffer->width_;
texture.height_ = texture_buffer->height_;
texture.target_ = texture_buffer->target_;
texture.name_ = texture_buffer->name_;
if (texture_buffer->format() != GpuBufferFormat::kUnknown) {
// TODO: do the params need to be reset here??
glBindTexture(texture.target(), texture.name());
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
texture_buffer->format(), texture.plane_, GetGlVersion());
SetStandardTextureParams(texture.target(), info.gl_internal_format);
glBindTexture(texture.target(), 0);
}
return texture;
}
GlTextureBufferSharedPtr GlCalculatorHelperImpl::MakeGlTextureBuffer(
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const ImageFrame& image_frame) {
CHECK(gl_context_->IsCurrent());
GlTexture texture =
MapGpuBuffer(GpuBuffer::CopyingImageFrame(image_frame), 0, true);
return texture;
}
auto buffer = GlTextureBuffer::Create(image_frame);
template <>
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
return view_.gpu_buffer().AsImageFrame();
}
if (buffer->format_ != GpuBufferFormat::kUnknown) {
glBindTexture(GL_TEXTURE_2D, buffer->name_);
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
buffer->format_, /*plane=*/0, GetGlVersion());
SetStandardTextureParams(buffer->target_, info.gl_internal_format);
glBindTexture(GL_TEXTURE_2D, 0);
template <>
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
auto gpu_buffer = view_.gpu_buffer();
#ifdef __EMSCRIPTEN__
// When WebGL is used, the GL context may be spontaneously lost which can
// cause GpuBuffer allocations to fail. In that case, return a dummy buffer
// to allow processing of the current frame complete.
if (!gpu_buffer) {
return std::make_unique<GpuBuffer>();
}
return buffer;
#endif // __EMSCRIPTEN__
view_.DoneWriting();
return absl::make_unique<GpuBuffer>(gpu_buffer);
}
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
int width, int height, GpuBufferFormat format) {
@@ -202,44 +152,9 @@ GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
GpuBuffer buffer =
gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format);
GlTexture texture = MapGpuBuffer(buffer, 0);
GlTexture texture = MapGpuBuffer(buffer, 0, false);
return texture;
}
void GlCalculatorHelperImpl::ReadTexture(const GlTexture& texture, void* output,
size_t size) {
CHECK_GE(size, texture.width_ * texture.height_ * 4);
GLint current_fbo;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &current_fbo);
CHECK_NE(current_fbo, 0);
GLint color_attachment_name;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment_name);
if (color_attachment_name != texture.name_) {
// Save the viewport. Note that we assume that the color attachment is a
// GL_TEXTURE_2D texture.
GLint viewport[4];
glGetIntegerv(GL_VIEWPORT, viewport);
// Set the data from GLTexture object.
glViewport(0, 0, texture.width_, texture.height_);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
texture.target_, texture.name_, 0);
glReadPixels(0, 0, texture.width_, texture.height_, GL_RGBA,
GL_UNSIGNED_BYTE, output);
// Restore from the saved viewport and color attachment name.
glViewport(viewport[0], viewport[1], viewport[2], viewport[3]);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
color_attachment_name, 0);
} else {
glReadPixels(0, 0, texture.width_, texture.height_, GL_RGBA,
GL_UNSIGNED_BYTE, output);
}
}
} // namespace mediapipe
mediapipe/gpu/gl_calculator_helper_impl_ios.mm
-197
View File
@@ -1,197 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#if TARGET_OS_OSX
#import <AppKit/NSOpenGL.h>
#else
#import <OpenGLES/EAGL.h>
#endif // TARGET_OS_OSX
#import <AVFoundation/AVFoundation.h>
#include "absl/memory/memory.h"
#include "mediapipe/gpu/gpu_buffer_multi_pool.h"
#include "mediapipe/gpu/pixel_buffer_pool_util.h"
#include "mediapipe/objc/util.h"
namespace mediapipe {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const mediapipe::ImageFrame& image_frame) {
GlTexture texture;
texture.helper_impl_ = this;
texture.width_ = image_frame.Width();
texture.height_ = image_frame.Height();
auto format = GpuBufferFormatForImageFormat(image_frame.Format());
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(format, 0, GetGlVersion());
glGenTextures(1, &texture.name_);
glBindTexture(GL_TEXTURE_2D, texture.name_);
glTexImage2D(GL_TEXTURE_2D, 0, info.gl_internal_format, texture.width_,
texture.height_, 0, info.gl_format, info.gl_type,
image_frame.PixelData());
SetStandardTextureParams(GL_TEXTURE_2D, info.gl_internal_format);
return texture;
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer) {
return MapGpuBuffer(gpu_buffer, 0);
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer, int plane) {
return MapGpuBuffer(gpu_buffer, plane);
}
GlTexture GlCalculatorHelperImpl::MapGpuBuffer(
const GpuBuffer& gpu_buffer, int plane) {
CVReturn err;
GlTexture texture;
texture.helper_impl_ = this;
texture.gpu_buffer_ = gpu_buffer;
texture.plane_ = plane;
const GlTextureInfo info =
GlTextureInfoForGpuBufferFormat(gpu_buffer.format(), plane, GetGlVersion());
// When scale is not 1, we still give the nominal size of the image.
texture.width_ = gpu_buffer.width();
texture.height_ = gpu_buffer.height();
#if TARGET_OS_OSX
CVOpenGLTextureRef cv_texture_temp;
err = CVOpenGLTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context_->cv_texture_cache(), gpu_buffer.GetCVPixelBufferRef(), NULL,
&cv_texture_temp);
NSCAssert(cv_texture_temp && !err,
@"Error at CVOpenGLTextureCacheCreateTextureFromImage %d", err);
texture.cv_texture_.adopt(cv_texture_temp);
texture.target_ = CVOpenGLTextureGetTarget(*texture.cv_texture_);
texture.name_ = CVOpenGLTextureGetName(*texture.cv_texture_);
#else
CVOpenGLESTextureRef cv_texture_temp;
err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context_->cv_texture_cache(), gpu_buffer.GetCVPixelBufferRef(), NULL,
GL_TEXTURE_2D, info.gl_internal_format, texture.width_ / info.downscale,
texture.height_ / info.downscale, info.gl_format, info.gl_type, plane,
&cv_texture_temp);
NSCAssert(cv_texture_temp && !err,
@"Error at CVOpenGLESTextureCacheCreateTextureFromImage %d", err);
texture.cv_texture_.adopt(cv_texture_temp);
texture.target_ = CVOpenGLESTextureGetTarget(*texture.cv_texture_);
texture.name_ = CVOpenGLESTextureGetName(*texture.cv_texture_);
#endif // TARGET_OS_OSX
glBindTexture(texture.target(), texture.name());
SetStandardTextureParams(texture.target(), info.gl_internal_format);
return texture;
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
template<>
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
if (gpu_buffer_.GetCVPixelBufferRef()) {
return CreateImageFrameForCVPixelBuffer(gpu_buffer_.GetCVPixelBufferRef());
}
ImageFormat::Format image_format =
ImageFormatForGpuBufferFormat(gpu_buffer_.format());
CHECK(helper_impl_);
GlTextureInfo info =
GlTextureInfoForGpuBufferFormat(gpu_buffer_.format(), plane_, helper_impl_->GetGlVersion());
auto output = absl::make_unique<ImageFrame>(
image_format, width_, height_);
glReadPixels(0, 0, width_, height_, info.gl_format, info.gl_type,
output->MutablePixelData());
return output;
#else
CHECK(gpu_buffer_.format() == GpuBufferFormat::kBGRA32);
auto output =
absl::make_unique<ImageFrame>(ImageFormat::SRGBA, width_, height_,
ImageFrame::kGlDefaultAlignmentBoundary);
CHECK(helper_impl_);
helper_impl_->ReadTexture(*this, output->MutablePixelData(), output->PixelDataSize());
return output;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
}
template<>
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
NSCAssert(gpu_buffer_, @"gpu_buffer_ must be valid");
#if TARGET_IPHONE_SIMULATOR
CVPixelBufferRef pixel_buffer = gpu_buffer_.GetCVPixelBufferRef();
CVReturn err = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
NSCAssert(err == kCVReturnSuccess, @"CVPixelBufferLockBaseAddress failed: %d", err);
OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
size_t bytes_per_row = CVPixelBufferGetBytesPerRow(pixel_buffer);
uint8_t* pixel_ptr = static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixel_buffer));
if (pixel_format == kCVPixelFormatType_32BGRA) {
// TODO: restore previous framebuffer? Move this to helper so we can
// use BindFramebuffer?
glViewport(0, 0, width_, height_);
glFramebufferTexture2D(
GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, target_, name_, 0);
size_t contiguous_bytes_per_row = width_ * 4;
if (bytes_per_row == contiguous_bytes_per_row) {
glReadPixels(0, 0, width_, height_, GL_BGRA, GL_UNSIGNED_BYTE, pixel_ptr);
} else {
std::vector<uint8_t> contiguous_buffer(contiguous_bytes_per_row * height_);
uint8_t* temp_ptr = contiguous_buffer.data();
glReadPixels(0, 0, width_, height_, GL_BGRA, GL_UNSIGNED_BYTE, temp_ptr);
for (int i = 0; i < height_; ++i) {
memcpy(pixel_ptr, temp_ptr, contiguous_bytes_per_row);
temp_ptr += contiguous_bytes_per_row;
pixel_ptr += bytes_per_row;
}
}
} else {
uint32_t format_big = CFSwapInt32HostToBig(pixel_format);
NSLog(@"unsupported pixel format: %.4s", (char*)&format_big);
}
err = CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
NSCAssert(err == kCVReturnSuccess, @"CVPixelBufferUnlockBaseAddress failed: %d", err);
#endif
return absl::make_unique<GpuBuffer>(gpu_buffer_);
}
void GlTexture::Release() {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
if (*cv_texture_) {
cv_texture_.reset(NULL);
} else if (name_) {
// This is only needed because of the glGenTextures in
// CreateSourceTexture(ImageFrame)... change.
glDeleteTextures(1, &name_);
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
helper_impl_ = nullptr;
gpu_buffer_ = nullptr;
plane_ = 0;
name_ = 0;
width_ = 0;
height_ = 0;
}
} // namespace mediapipe
mediapipe/gpu/gl_context.cc
+39 -4
View File
@@ -222,6 +222,9 @@ bool GlContext::HasGlExtension(absl::string_view extension) const {
// to work with GL_EXTENSIONS for newer GL versions, so we must maintain both
// variations of this function.
absl::Status GlContext::GetGlExtensions() {
// RET_CHECK logs by default, but here we just want to check the precondition;
// we'll fall back to the alternative implementation for older versions.
RET_CHECK(gl_major_version_ >= 3).SetNoLogging();
gl_extensions_.clear();
// glGetStringi only introduced in GL 3.0+; so we exit out this function if
// we don't have that function defined, regardless of version number reported.
@@ -330,13 +333,24 @@ absl::Status GlContext::FinishInitialization(bool create_thread) {
LOG(INFO) << "GL version: " << gl_major_version_ << "." << gl_minor_version_
<< " (" << glGetString(GL_VERSION) << ")";
if (gl_major_version_ >= 3) {
{
auto status = GetGlExtensions();
if (status.ok()) {
return absl::OkStatus();
if (!status.ok()) {
status = GetGlExtensionsCompat();
}
MP_RETURN_IF_ERROR(status);
}
return GetGlExtensionsCompat();
#if GL_ES_VERSION_2_0 // This actually means "is GLES available".
// No linear float filtering by default, check extensions.
can_linear_filter_float_textures_ =
HasGlExtension("OES_texture_float_linear");
#else
// Desktop GL should always allow linear filtering.
can_linear_filter_float_textures_ = true;
#endif // GL_ES_VERSION_2_0
return absl::OkStatus();
});
}
@@ -841,4 +855,25 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
return GlTextureInfoForGpuBufferFormat(format, plane, ctx->GetGlVersion());
}
void GlContext::SetStandardTextureParams(GLenum target, GLint internal_format) {
// Default to using linear filter everywhere. For float32 textures, fall back
// to GL_NEAREST if linear filtering unsupported.
GLint filter;
switch (internal_format) {
case GL_R32F:
case GL_RG32F:
case GL_RGBA32F:
// 32F (unlike 16f) textures do not always support texture filtering
// (According to OpenGL ES specification [TEXTURE IMAGE SPECIFICATION])
filter = can_linear_filter_float_textures_ ? GL_LINEAR : GL_NEAREST;
break;
default:
filter = GL_LINEAR;
}
glTexParameteri(target, GL_TEXTURE_MIN_FILTER, filter);
glTexParameteri(target, GL_TEXTURE_MAG_FILTER, filter);
glTexParameteri(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
} // namespace mediapipe
mediapipe/gpu/gl_context.h
+11 -4
View File
@@ -276,6 +276,9 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
}).IgnoreError();
}
// Sets default texture filtering parameters.
void SetStandardTextureParams(GLenum target, GLint internal_format);
// These are used for testing specific SyncToken implementations. Do not use
// outside of tests.
enum class SyncTokenTypeForTest {
@@ -342,11 +345,11 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
// This wraps a thread_local.
static std::weak_ptr<GlContext>& CurrentContext();
static absl::Status SwitchContext(ContextBinding* old_context,
static absl::Status SwitchContext(ContextBinding* saved_context,
const ContextBinding& new_context);
absl::Status EnterContext(ContextBinding* previous_context);
absl::Status ExitContext(const ContextBinding* previous_context);
absl::Status EnterContext(ContextBinding* saved_context);
absl::Status ExitContext(const ContextBinding* saved_context);
void DestroyContext();
bool HasContext() const;
@@ -383,7 +386,7 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
static void GetCurrentContextBinding(ContextBinding* binding);
// Makes the context described by new_context current on this thread.
static absl::Status SetCurrentContextBinding(
const ContextBinding& new_context);
const ContextBinding& new_binding);
// If not null, a dedicated thread used to execute tasks on this context.
// Used on Android due to expensive context switching on some configurations.
@@ -396,6 +399,10 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
// so we should be fine storing the extension pieces as string_view's.
std::set<absl::string_view> gl_extensions_;
// Used by SetStandardTextureParams. Do we want several of these bools, or a
// better mechanism?
bool can_linear_filter_float_textures_;
// Number of glFinish calls completed on the GL thread.
// Changes should be guarded by mutex_. However, we use simple atomic
// loads for efficiency on the fast path.
mediapipe/gpu/gl_context_egl.cc
+5 -6
View File
@@ -85,7 +85,7 @@ GlContext::StatusOrGlContext GlContext::Create(EGLContext share_context,
return std::move(context);
}
absl::Status GlContext::CreateContextInternal(EGLContext external_context,
absl::Status GlContext::CreateContextInternal(EGLContext share_context,
int gl_version) {
CHECK(gl_version == 2 || gl_version == 3);
@@ -131,8 +131,7 @@ absl::Status GlContext::CreateContextInternal(EGLContext external_context,
// clang-format on
};
context_ =
eglCreateContext(display_, config_, external_context, context_attr);
context_ = eglCreateContext(display_, config_, share_context, context_attr);
int error = eglGetError();
RET_CHECK(context_ != EGL_NO_CONTEXT)
<< "Could not create GLES " << gl_version << " context; "
@@ -149,7 +148,7 @@ absl::Status GlContext::CreateContextInternal(EGLContext external_context,
return absl::OkStatus();
}
absl::Status GlContext::CreateContext(EGLContext external_context) {
absl::Status GlContext::CreateContext(EGLContext share_context) {
EGLint major = 0;
EGLint minor = 0;
@@ -163,11 +162,11 @@ absl::Status GlContext::CreateContext(EGLContext external_context) {
LOG(INFO) << "Successfully initialized EGL. Major : " << major
<< " Minor: " << minor;
auto status = CreateContextInternal(external_context, 3);
auto status = CreateContextInternal(share_context, 3);
if (!status.ok()) {
LOG(WARNING) << "Creating a context with OpenGL ES 3 failed: " << status;
LOG(WARNING) << "Fall back on OpenGL ES 2.";
status = CreateContextInternal(external_context, 2);
status = CreateContextInternal(share_context, 2);
}
MP_RETURN_IF_ERROR(status);
mediapipe/gpu/gl_context_internal.h
+1 -1
View File
@@ -36,7 +36,7 @@ class GlContext::DedicatedThread {
DedicatedThread& operator=(DedicatedThread) = delete;
absl::Status Run(GlStatusFunction gl_func);
void RunWithoutWaiting(GlVoidFunction gl_fund);
void RunWithoutWaiting(GlVoidFunction gl_func);
bool IsCurrentThread();
mediapipe/gpu/gl_ios_test.mm
+10 -12
View File
@@ -175,18 +175,16 @@
mediapipe::GlCalculatorHelper helper;
helper.InitializeForTest(&gpuData);
std::vector<std::pair<int, int>> sizes{
{200, 300},
{200, 299},
{196, 300},
{194, 300},
{193, 300},
};
for (const auto& width_height : sizes) {
mediapipe::GlTexture texture =
helper.CreateDestinationTexture(width_height.first, width_height.second);
XCTAssertNotEqual(texture.name(), 0);
}
helper.RunInGlContext([&helper] {
std::vector<std::pair<int, int>> sizes{
{200, 300}, {200, 299}, {196, 300}, {194, 300}, {193, 300},
};
for (const auto& width_height : sizes) {
mediapipe::GlTexture texture =
helper.CreateDestinationTexture(width_height.first, width_height.second);
XCTAssertNotEqual(texture.name(), 0);
}
});
}
- (void)testSimpleConversionFromFormat:(OSType)cvPixelFormat {
mediapipe/gpu/gpu_buffer.cc
+260
View File
@@ -0,0 +1,260 @@
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gl_context.h"
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#include "mediapipe/objc/util.h"
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
namespace mediapipe {
void GlTextureView::Release() {
if (detach_) detach_(*this);
detach_ = nullptr;
gl_context_ = nullptr;
gpu_buffer_ = nullptr;
plane_ = 0;
name_ = 0;
width_ = 0;
height_ = 0;
}
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#if TARGET_OS_OSX
typedef CVOpenGLTextureRef CVTextureType;
#else
typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
GlTextureView GpuBuffer::GetGlTextureView(int plane, bool for_reading) const {
CVReturn err;
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
#if TARGET_OS_OSX
CVTextureType cv_texture_temp;
err = CVOpenGLTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context->cv_texture_cache(),
GetCVPixelBufferRef(), NULL, &cv_texture_temp);
CHECK(cv_texture_temp && !err)
<< "CVOpenGLTextureCacheCreateTextureFromImage failed: " << err;
CFHolder<CVTextureType> cv_texture;
cv_texture.adopt(cv_texture_temp);
return GlTextureView(
gl_context.get(), CVOpenGLTextureGetTarget(*cv_texture),
CVOpenGLTextureGetName(*cv_texture), width(), height(), *this, plane,
[cv_texture](
mediapipe::GlTextureView&) { /* only retains cv_texture */ });
#else
const GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
format(), plane, gl_context->GetGlVersion());
CVTextureType cv_texture_temp;
err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context->cv_texture_cache(),
GetCVPixelBufferRef(), NULL, GL_TEXTURE_2D, info.gl_internal_format,
width() / info.downscale, height() / info.downscale, info.gl_format,
info.gl_type, plane, &cv_texture_temp);
CHECK(cv_texture_temp && !err)
<< "CVOpenGLESTextureCacheCreateTextureFromImage failed: " << err;
CFHolder<CVTextureType> cv_texture;
cv_texture.adopt(cv_texture_temp);
return GlTextureView(
gl_context.get(), CVOpenGLESTextureGetTarget(*cv_texture),
CVOpenGLESTextureGetName(*cv_texture), width(), height(), *this, plane,
[cv_texture](
mediapipe::GlTextureView&) { /* only retains cv_texture */ });
#endif // TARGET_OS_OSX
}
GpuBuffer GpuBuffer::CopyingImageFrame(const ImageFrame& image_frame) {
auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
// Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
// deals with absl::Status in MediaPipe OSS.
CHECK_OK(maybe_buffer.status());
return GpuBuffer(std::move(maybe_buffer).value());
}
std::unique_ptr<ImageFrame> GpuBuffer::AsImageFrame() const {
CHECK(GetCVPixelBufferRef());
return CreateImageFrameForCVPixelBuffer(GetCVPixelBufferRef());
}
void GlTextureView::DoneWriting() const {
CHECK(gpu_buffer_);
#if TARGET_IPHONE_SIMULATOR
CVPixelBufferRef pixel_buffer = gpu_buffer_.GetCVPixelBufferRef();
CVReturn err = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
CHECK(err == kCVReturnSuccess)
<< "CVPixelBufferLockBaseAddress failed: " << err;
OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
size_t bytes_per_row = CVPixelBufferGetBytesPerRow(pixel_buffer);
uint8_t* pixel_ptr =
static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixel_buffer));
if (pixel_format == kCVPixelFormatType_32BGRA) {
// TODO: restore previous framebuffer? Move this to helper so we
// can use BindFramebuffer?
glViewport(0, 0, width(), height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, target(),
name(), 0);
size_t contiguous_bytes_per_row = width() * 4;
if (bytes_per_row == contiguous_bytes_per_row) {
glReadPixels(0, 0, width(), height(), GL_BGRA, GL_UNSIGNED_BYTE,
pixel_ptr);
} else {
std::vector<uint8_t> contiguous_buffer(contiguous_bytes_per_row *
height());
uint8_t* temp_ptr = contiguous_buffer.data();
glReadPixels(0, 0, width(), height(), GL_BGRA, GL_UNSIGNED_BYTE,
temp_ptr);
for (int i = 0; i < height(); ++i) {
memcpy(pixel_ptr, temp_ptr, contiguous_bytes_per_row);
temp_ptr += contiguous_bytes_per_row;
pixel_ptr += bytes_per_row;
}
}
} else {
LOG(ERROR) << "unsupported pixel format: " << pixel_format;
}
err = CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
CHECK(err == kCVReturnSuccess)
<< "CVPixelBufferUnlockBaseAddress failed: " << err;
#endif
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTextureView GpuBuffer::GetGlTextureView(int plane, bool for_reading) const {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
const GlTextureBufferSharedPtr& texture_buffer =
GetGlTextureBufferSharedPtr();
// Insert wait call to sync with the producer.
texture_buffer->WaitOnGpu();
CHECK_EQ(plane, 0);
GlTextureView::DetachFn detach;
if (for_reading) {
detach = [](mediapipe::GlTextureView& texture) {
// Inform the GlTextureBuffer that we have finished accessing its
// contents, and create a consumer sync point.
texture.gpu_buffer().GetGlTextureBufferSharedPtr()->DidRead(
texture.gl_context()->CreateSyncToken());
};
}
return GlTextureView(gl_context.get(), texture_buffer->target(),
texture_buffer->name(), width(), height(), *this, plane,
std::move(detach));
}
GpuBuffer GpuBuffer::CopyingImageFrame(const ImageFrame& image_frame) {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
auto buffer = GlTextureBuffer::Create(image_frame);
// TODO: does this need to set the texture params? We set them again when the
// texture is actually acccessed via GlTexture[View]. Or should they always be
// set on creation?
if (buffer->format() != GpuBufferFormat::kUnknown) {
glBindTexture(GL_TEXTURE_2D, buffer->name());
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
buffer->format(), /*plane=*/0, gl_context->GetGlVersion());
gl_context->SetStandardTextureParams(buffer->target(),
info.gl_internal_format);
glBindTexture(GL_TEXTURE_2D, 0);
}
return GpuBuffer(std::move(buffer));
}
static void ReadTexture(const GlTextureView& view, void* output, size_t size) {
// TODO: check buffer size? We could use glReadnPixels where available
// (OpenGL ES 3.2, i.e. nowhere). Note that, to fully check that the read
// won't overflow the buffer with glReadPixels, we'd also need to check or
// reset several glPixelStore parameters (e.g. what if someone had the
// ill-advised idea of setting GL_PACK_SKIP_PIXELS?).
CHECK(view.gl_context());
GlTextureInfo info =
GlTextureInfoForGpuBufferFormat(view.gpu_buffer().format(), view.plane(),
view.gl_context()->GetGlVersion());
GLint current_fbo;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &current_fbo);
CHECK_NE(current_fbo, 0);
GLint color_attachment_name;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment_name);
if (color_attachment_name != view.name()) {
// Save the viewport. Note that we assume that the color attachment is a
// GL_TEXTURE_2D texture.
GLint viewport[4];
glGetIntegerv(GL_VIEWPORT, viewport);
// Set the data from GLTextureView object.
glViewport(0, 0, view.width(), view.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, view.target(),
view.name(), 0);
glReadPixels(0, 0, view.width(), view.height(), info.gl_format,
info.gl_type, output);
// Restore from the saved viewport and color attachment name.
glViewport(viewport[0], viewport[1], viewport[2], viewport[3]);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
color_attachment_name, 0);
} else {
glReadPixels(0, 0, view.width(), view.height(), info.gl_format,
info.gl_type, output);
}
}
std::unique_ptr<ImageFrame> GpuBuffer::AsImageFrame() const {
ImageFormat::Format image_format = ImageFormatForGpuBufferFormat(format());
auto output = absl::make_unique<ImageFrame>(
image_format, width(), height(), ImageFrame::kGlDefaultAlignmentBoundary);
auto view = GetGlTextureView(0, true);
ReadTexture(view, output->MutablePixelData(), output->PixelDataSize());
return output;
}
void GlTextureView::DoneWriting() const {
CHECK(gpu_buffer_);
// Inform the GlTextureBuffer that we have produced new content, and create
// a producer sync point.
gpu_buffer_.GetGlTextureBufferSharedPtr()->Updated(
gl_context()->CreateSyncToken());
#ifdef __ANDROID__
// On (some?) Android devices, the texture may need to be explicitly
// detached from the current framebuffer.
// TODO: is this necessary even with the unbind in BindFramebuffer?
// It is not clear if this affected other contexts too, but let's keep it
// while in doubt.
GLint type = GL_NONE;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE,
&type);
if (type == GL_TEXTURE) {
GLint color_attachment = 0;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment);
if (color_attachment == name()) {
glBindFramebuffer(GL_FRAMEBUFFER, 0);
}
}
// Some Android drivers log a GL_INVALID_ENUM error after the first
// glGetFramebufferAttachmentParameteriv call if there is no bound object,
// even though it should be ok to ask for the type and get back GL_NONE.
// Let's just ignore any pending errors here.
GLenum error;
while ((error = glGetError()) != GL_NO_ERROR) {
}
#endif // __ANDROID__
}
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
} // namespace mediapipe
mediapipe/gpu/gpu_buffer.h
+62
View File
@@ -17,6 +17,7 @@
#include <utility>
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
@@ -32,6 +33,9 @@
namespace mediapipe {
class GlContext;
class GlTextureView;
// This class wraps a platform-specific buffer of GPU data.
// An instance of GpuBuffer acts as an opaque reference to the underlying
// data object.
@@ -84,6 +88,19 @@ class GpuBuffer {
// Allow assignment from nullptr.
GpuBuffer& operator=(std::nullptr_t other);
// TODO: split into read and write, remove const from write.
GlTextureView GetGlTextureView(int plane, bool for_reading) const;
// Make a GpuBuffer copying the data from an ImageFrame.
static GpuBuffer CopyingImageFrame(const ImageFrame& image_frame);
// Make an ImageFrame, possibly sharing the same data. The data is shared if
// the GpuBuffer's storage supports memory sharing; otherwise, it is copied.
// In order to work correctly across platforms, callers should always treat
// the returned ImageFrame as if it shares memory with the GpuBuffer, i.e.
// treat it as immutable if the GpuBuffer must not be modified.
std::unique_ptr<ImageFrame> AsImageFrame() const;
private:
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CFHolder<CVPixelBufferRef> pixel_buffer_;
@@ -92,6 +109,51 @@ class GpuBuffer {
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
};
class GlTextureView {
public:
GlTextureView() {}
~GlTextureView() { Release(); }
// TODO: make this class move-only.
GlContext* gl_context() const { return gl_context_; }
int width() const { return width_; }
int height() const { return height_; }
GLenum target() const { return target_; }
GLuint name() const { return name_; }
const GpuBuffer& gpu_buffer() const { return gpu_buffer_; }
int plane() const { return plane_; }
private:
friend class GpuBuffer;
using DetachFn = std::function<void(GlTextureView&)>;
GlTextureView(GlContext* context, GLenum target, GLuint name, int width,
int height, GpuBuffer gpu_buffer, int plane, DetachFn detach)
: gl_context_(context),
target_(target),
name_(name),
width_(width),
height_(height),
gpu_buffer_(std::move(gpu_buffer)),
plane_(plane),
detach_(std::move(detach)) {}
// TODO: remove this friend declaration.
friend class GlTexture;
void Release();
// TODO: make this non-const.
void DoneWriting() const;
GlContext* gl_context_ = nullptr;
GLenum target_ = GL_TEXTURE_2D;
GLuint name_ = 0;
// Note: when scale is not 1, we still give the nominal size of the image.
int width_ = 0;
int height_ = 0;
GpuBuffer gpu_buffer_;
int plane_ = 0;
DetachFn detach_;
};
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
inline int GpuBuffer::width() const {
mediapipe/gpu/gpu_buffer_multi_pool.cc
+3 -2
View File
@@ -21,10 +21,11 @@
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#ifdef __APPLE__
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#include "CoreFoundation/CFBase.h"
#include "mediapipe/objc/CFHolder.h"
#endif // __APPLE__
#include "mediapipe/objc/util.h"
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
namespace mediapipe {
mediapipe/gpu/pixel_buffer_pool_util.h
-5
View File
@@ -63,11 +63,6 @@ CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(
int allocationThreshold);
// Create a CVPixelBuffer without using a pool.
CVReturn CreateCVPixelBufferWithoutPool(int width, int height,
OSType pixelFormat,
CVPixelBufferRef* outBuffer);
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_PIXEL_BUFFER_POOL_UTIL_H_
mediapipe/gpu/pixel_buffer_pool_util.mm
-29
View File
@@ -121,33 +121,4 @@ CVReturn CreateCVPixelBufferWithPool(
return err;
}
#if TARGET_IPHONE_SIMULATOR
static void FreeRefConReleaseCallback(void* refCon, const void* baseAddress) {
free(refCon);
}
#endif
CVReturn CreateCVPixelBufferWithoutPool(
int width, int height, OSType pixelFormat, CVPixelBufferRef* outBuffer) {
#if TARGET_IPHONE_SIMULATOR
// On the simulator, syncing the texture with the pixelbuffer does not work,
// and we have to use glReadPixels. Since GL_UNPACK_ROW_LENGTH is not
// available in OpenGL ES 2, we should create the buffer so the pixels are
// contiguous.
//
// TODO: verify if we can use kIOSurfaceBytesPerRow to force
// CoreVideo to give us contiguous data.
size_t bytes_per_row = width * 4;
void* data = malloc(bytes_per_row * height);
return CVPixelBufferCreateWithBytes(
kCFAllocatorDefault, width, height, pixelFormat, data, bytes_per_row,
FreeRefConReleaseCallback, data, GetCVPixelBufferAttributesForGlCompatibility(),
outBuffer);
#else
return CVPixelBufferCreate(
kCFAllocatorDefault, width, height, pixelFormat,
GetCVPixelBufferAttributesForGlCompatibility(), outBuffer);
#endif
}
} // namespace mediapipe