talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: bbbbcb4f5174dea33525729ede47c770069157cd
This commit is contained in:
MediaPipe Team
2021-10-18 17:00:29 -04:00
committed by chuoling
parent 33d683c671
commit 1faeaae7e5
75 changed files with 1944 additions and 560 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/gpu/BUILD
+45
View File
@@ -196,7 +196,9 @@ cc_library(
deps = [
":gl_base",
":gl_context",
":gl_texture_view",
":gpu_buffer_format",
":gpu_buffer_storage",
# TODO: remove this dependency. Some other teams' tests
# depend on having an indirect image_frame dependency, need to be
# fixed first.
@@ -205,6 +207,17 @@ cc_library(
],
)
cc_library(
name = "gl_texture_view",
srcs = ["gl_texture_view.cc"],
hdrs = ["gl_texture_view.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_context",
],
)
cc_library(
name = "gpu_buffer",
srcs = ["gpu_buffer.cc"],
@@ -214,12 +227,15 @@ cc_library(
":gl_base",
":gl_context",
":gpu_buffer_format",
":gpu_buffer_storage",
":gl_texture_view",
"//mediapipe/framework/formats:image_frame",
] + select({
"//conditions:default": [
":gl_texture_buffer",
],
"//mediapipe:ios": [
":gpu_buffer_storage_cv_pixel_buffer",
"//mediapipe/objc:util",
"//mediapipe/objc:CFHolder",
],
@@ -244,6 +260,35 @@ cc_library(
],
)
cc_library(
name = "gpu_buffer_storage",
hdrs = ["gpu_buffer_storage.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gpu_buffer_format",
"//mediapipe/framework/deps:no_destructor",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/container:flat_hash_map",
],
)
cc_library(
name = "gpu_buffer_storage_cv_pixel_buffer",
srcs = ["gpu_buffer_storage_cv_pixel_buffer.cc"],
hdrs = ["gpu_buffer_storage_cv_pixel_buffer.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_context",
":gl_texture_view",
":gpu_buffer_storage",
"//mediapipe/objc:CFHolder",
"//mediapipe/objc:util",
],
)
mediapipe_proto_library(
name = "gpu_origin_proto",
srcs = ["gpu_origin.proto"],
mediapipe/gpu/gl_calculator_helper.cc
-2
View File
@@ -109,12 +109,10 @@ GlTexture GlCalculatorHelper::CreateSourceTexture(
return impl_->CreateSourceTexture(image_frame);
}
#ifdef __APPLE__
GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& pixel_buffer,
int plane) {
return impl_->CreateSourceTexture(pixel_buffer, plane);
}
#endif
void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer,
int* width, int* height) {
mediapipe/gpu/gl_calculator_helper.h
+17 -10
View File
@@ -29,10 +29,6 @@
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/graph_support.h"
#ifdef __APPLE__
#include <CoreVideo/CoreVideo.h>
#endif // __APPLE__
namespace mediapipe {
class GlCalculatorHelperImpl;
@@ -111,24 +107,35 @@ class GlCalculatorHelper {
// where it is supported (iOS, for now) they take advantage of memory sharing
// between the CPU and GPU, avoiding memory copies.
// Creates a texture representing an input frame, and manages sync token.
// Gives access to an input frame as an OpenGL texture for reading (sampling).
//
// IMPORTANT: the returned GlTexture should be treated as a short-term view
// into the frame (typically for the duration of a Process call). Do not store
// it as a member in your calculator. If you need to keep a frame around,
// store the GpuBuffer instead, and call CreateSourceTexture again on each
// Process call.
//
// TODO: rename this; the use of "Create" makes this sound more expensive than
// it is.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
GlTexture CreateSourceTexture(const mediapipe::Image& image);
#ifdef __APPLE__
// Creates a texture from a plane of a planar buffer.
// Gives read access to a plane of a planar buffer.
// The plane index is zero-based. The number of planes depends on the
// internal format of the buffer.
// Note: multi-plane support is not available on all platforms.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer, int plane);
#endif
// Convenience function for converting an ImageFrame to GpuBuffer and then
// accessing it as a texture.
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
// Extracts GpuBuffer dimensions without creating a texture.
ABSL_DEPRECATED("Use width and height methods on GpuBuffer instead")
void GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, int* width,
int* height);
// Creates a texture representing an output frame, and manages sync token.
// Gives access to an OpenGL texture for writing (rendering) a new frame.
// TODO: This should either return errors or a status.
GlTexture CreateDestinationTexture(
int output_width, int output_height,
mediapipe/gpu/gl_calculator_helper_impl.h
+1 -4
View File
@@ -62,10 +62,7 @@ class GlCalculatorHelperImpl {
private:
// Makes a GpuBuffer accessible as a texture in the GL context.
GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, int plane,
bool for_reading);
void AttachGlTexture(GlTexture& texture, const GpuBuffer& gpu_buffer,
int plane, bool for_reading);
GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, GlTextureView view);
// Create the framebuffer for rendering.
void CreateFramebuffer();
mediapipe/gpu/gl_calculator_helper_impl_common.cc
+7 -12
View File
@@ -91,9 +91,7 @@ void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) {
}
GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
int plane, bool for_reading) {
GlTextureView view = gpu_buffer.GetGlTextureView(plane, for_reading);
GlTextureView view) {
if (gpu_buffer.format() != GpuBufferFormat::kUnknown) {
// TODO: do the params need to be reset here??
glBindTexture(view.target(), view.name());
@@ -109,19 +107,18 @@ GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer) {
return MapGpuBuffer(gpu_buffer, 0, true);
return CreateSourceTexture(gpu_buffer, 0);
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer, int plane) {
return MapGpuBuffer(gpu_buffer, plane, true);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetGlTextureReadView(plane));
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const ImageFrame& image_frame) {
GlTexture texture =
MapGpuBuffer(GpuBuffer::CopyingImageFrame(image_frame), 0, true);
return texture;
auto gpu_buffer = GpuBuffer::CopyingImageFrame(image_frame);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetGlTextureReadView(0));
}
template <>
@@ -150,11 +147,9 @@ GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
CreateFramebuffer();
}
GpuBuffer buffer =
GpuBuffer gpu_buffer =
gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format);
GlTexture texture = MapGpuBuffer(buffer, 0, false);
return texture;
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetGlTextureWriteView(0));
}
} // namespace mediapipe
mediapipe/gpu/gl_texture_buffer.cc
+138 -3
View File
@@ -14,6 +14,9 @@
#include "mediapipe/gpu/gl_texture_buffer.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_texture_view.h"
namespace mediapipe {
std::unique_ptr<GlTextureBuffer> GlTextureBuffer::Wrap(
@@ -122,6 +125,15 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) {
if (alignment != 4 && data) glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
// TODO: does this need to set the texture params? We set them again when the
// texture is actually acccessed via GlTexture[View]. Or should they always be
// set on creation?
if (format_ != GpuBufferFormat::kUnknown) {
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
format_, /*plane=*/0, context->GetGlVersion());
context->SetStandardTextureParams(target_, info.gl_internal_format);
}
glBindTexture(target_, 0);
// Use the deletion callback to delete the texture on the context
@@ -167,7 +179,7 @@ void GlTextureBuffer::Updated(std::shared_ptr<GlSyncPoint> prod_token) {
producer_context_ = producer_sync_->GetContext();
}
void GlTextureBuffer::DidRead(std::shared_ptr<GlSyncPoint> cons_token) {
void GlTextureBuffer::DidRead(std::shared_ptr<GlSyncPoint> cons_token) const {
absl::MutexLock lock(&consumer_sync_mutex_);
consumer_multi_sync_->Add(std::move(cons_token));
}
@@ -181,7 +193,7 @@ GlTextureBuffer::~GlTextureBuffer() {
}
}
void GlTextureBuffer::WaitUntilComplete() {
void GlTextureBuffer::WaitUntilComplete() const {
// Buffers created by the application (using the constructor that wraps an
// existing texture) have no sync token and are assumed to be already
// complete.
@@ -190,7 +202,7 @@ void GlTextureBuffer::WaitUntilComplete() {
}
}
void GlTextureBuffer::WaitOnGpu() {
void GlTextureBuffer::WaitOnGpu() const {
// Buffers created by the application (using the constructor that wraps an
// existing texture) have no sync token and are assumed to be already
// complete.
@@ -212,4 +224,127 @@ void GlTextureBuffer::WaitForConsumersOnGpu() {
// precisely, on only one GL context.
}
GlTextureView GlTextureBuffer::GetGlTextureReadView(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) const {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
CHECK_EQ(plane, 0);
// Insert wait call to sync with the producer.
WaitOnGpu();
GlTextureView::DetachFn detach = [this](mediapipe::GlTextureView& texture) {
// Inform the GlTextureBuffer that we have finished accessing its
// contents, and create a consumer sync point.
DidRead(texture.gl_context()->CreateSyncToken());
};
return GlTextureView(gl_context.get(), target(), name(), width(), height(),
std::move(gpu_buffer), plane, std::move(detach),
nullptr);
}
GlTextureView GlTextureBuffer::GetGlTextureWriteView(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
CHECK_EQ(plane, 0);
// Insert wait call to sync with the producer.
WaitOnGpu();
Reuse(); // TODO: the producer wait should probably be part of Reuse in the
// case when there are no consumers.
GlTextureView::DoneWritingFn done_writing =
[this](const mediapipe::GlTextureView& texture) {
ViewDoneWriting(texture);
};
return GlTextureView(gl_context.get(), target(), name(), width(), height(),
std::move(gpu_buffer), plane, nullptr,
std::move(done_writing));
}
void GlTextureBuffer::ViewDoneWriting(const GlTextureView& view) {
// Inform the GlTextureBuffer that we have produced new content, and create
// a producer sync point.
Updated(view.gl_context()->CreateSyncToken());
#ifdef __ANDROID__
// On (some?) Android devices, the texture may need to be explicitly
// detached from the current framebuffer.
// TODO: is this necessary even with the unbind in BindFramebuffer?
// It is not clear if this affected other contexts too, but let's keep it
// while in doubt.
GLint type = GL_NONE;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE,
&type);
if (type == GL_TEXTURE) {
GLint color_attachment = 0;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment);
if (color_attachment == name()) {
glBindFramebuffer(GL_FRAMEBUFFER, 0);
}
}
// Some Android drivers log a GL_INVALID_ENUM error after the first
// glGetFramebufferAttachmentParameteriv call if there is no bound object,
// even though it should be ok to ask for the type and get back GL_NONE.
// Let's just ignore any pending errors here.
GLenum error;
while ((error = glGetError()) != GL_NO_ERROR) {
}
#endif // __ANDROID__
}
static void ReadTexture(const GlTextureView& view, GpuBufferFormat format,
void* output, size_t size) {
// TODO: check buffer size? We could use glReadnPixels where available
// (OpenGL ES 3.2, i.e. nowhere). Note that, to fully check that the read
// won't overflow the buffer with glReadPixels, we'd also need to check or
// reset several glPixelStore parameters (e.g. what if someone had the
// ill-advised idea of setting GL_PACK_SKIP_PIXELS?).
CHECK(view.gl_context());
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
format, view.plane(), view.gl_context()->GetGlVersion());
GLint current_fbo;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &current_fbo);
CHECK_NE(current_fbo, 0);
GLint color_attachment_name;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment_name);
if (color_attachment_name != view.name()) {
// Save the viewport. Note that we assume that the color attachment is a
// GL_TEXTURE_2D texture.
GLint viewport[4];
glGetIntegerv(GL_VIEWPORT, viewport);
// Set the data from GLTextureView object.
glViewport(0, 0, view.width(), view.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, view.target(),
view.name(), 0);
glReadPixels(0, 0, view.width(), view.height(), info.gl_format,
info.gl_type, output);
// Restore from the saved viewport and color attachment name.
glViewport(viewport[0], viewport[1], viewport[2], viewport[3]);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
color_attachment_name, 0);
} else {
glReadPixels(0, 0, view.width(), view.height(), info.gl_format,
info.gl_type, output);
}
}
std::unique_ptr<ImageFrame> GlTextureBuffer::AsImageFrame() const {
ImageFormat::Format image_format = ImageFormatForGpuBufferFormat(format());
auto output = absl::make_unique<ImageFrame>(
image_format, width(), height(), ImageFrame::kGlDefaultAlignmentBoundary);
auto view = GetGlTextureReadView(nullptr, 0);
ReadTexture(view, format(), output->MutablePixelData(),
output->PixelDataSize());
return output;
}
} // namespace mediapipe
mediapipe/gpu/gl_texture_buffer.h
+16 -7
View File
@@ -25,13 +25,14 @@
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
namespace mediapipe {
class GlCalculatorHelperImpl;
// Implements a GPU memory buffer as an OpenGL texture. For internal use.
class GlTextureBuffer {
class GlTextureBuffer : public mediapipe::internal::GpuBufferStorage {
public:
// This is called when the texture buffer is deleted. It is passed a sync
// token created at that time on the GlContext. If the GlTextureBuffer has
@@ -85,6 +86,13 @@ class GlTextureBuffer {
int height() const { return height_; }
GpuBufferFormat format() const { return format_; }
GlTextureView GetGlTextureReadView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override;
GlTextureView GetGlTextureWriteView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override;
void ViewDoneWriting(const GlTextureView& view) override;
std::unique_ptr<ImageFrame> AsImageFrame() const override;
// If this texture is going to be used outside of the context that produced
// it, this method should be called to ensure that its updated contents are
// available. When this method returns, all changed made before the call to
@@ -94,13 +102,13 @@ class GlTextureBuffer {
// NOTE: This blocks the current CPU thread and makes the changes visible
// to the CPU. If you want to access the data via OpenGL, use WaitOnGpu
// instead.
void WaitUntilComplete();
void WaitUntilComplete() const;
// Call this method to synchronize the current GL context with the texture's
// producer. This will not block the current CPU thread, but will ensure that
// subsequent GL commands see the texture in its complete status, with all
// rendering done on the GPU by the generating context.
void WaitOnGpu();
void WaitOnGpu() const;
// Informs the buffer that its contents are going to be overwritten.
// This invalidates the current sync token.
@@ -114,7 +122,7 @@ class GlTextureBuffer {
void Updated(std::shared_ptr<GlSyncPoint> prod_token);
// Informs the buffer that a consumer has finished reading from it.
void DidRead(std::shared_ptr<GlSyncPoint> cons_token);
void DidRead(std::shared_ptr<GlSyncPoint> cons_token) const;
// Waits for all pending consumers to finish accessing the current content
// of the texture. This (preferably the OnGpu version) should be called
@@ -143,10 +151,11 @@ class GlTextureBuffer {
const GLenum target_ = GL_TEXTURE_2D;
// Token tracking changes to this texture. Used by WaitUntilComplete.
std::shared_ptr<GlSyncPoint> producer_sync_;
absl::Mutex consumer_sync_mutex_;
mutable absl::Mutex consumer_sync_mutex_;
// Tokens tracking the point when consumers finished using this texture.
std::unique_ptr<GlMultiSyncPoint> consumer_multi_sync_ ABSL_GUARDED_BY(
consumer_sync_mutex_) = absl::make_unique<GlMultiSyncPoint>();
mutable std::unique_ptr<GlMultiSyncPoint> consumer_multi_sync_
ABSL_GUARDED_BY(consumer_sync_mutex_) =
absl::make_unique<GlMultiSyncPoint>();
DeletionCallback deletion_callback_;
std::shared_ptr<GlContext> producer_context_;
};
mediapipe/gpu/gl_texture_view.cc
+16
View File
@@ -0,0 +1,16 @@
#include "mediapipe/gpu/gl_texture_view.h"
namespace mediapipe {
void GlTextureView::Release() {
if (detach_) detach_(*this);
detach_ = nullptr;
gl_context_ = nullptr;
gpu_buffer_ = nullptr;
plane_ = 0;
name_ = 0;
width_ = 0;
height_ = 0;
}
} // namespace mediapipe
mediapipe/gpu/gl_texture_view.h
+86
View File
@@ -0,0 +1,86 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_TEXTURE_VIEW_H_
#define MEDIAPIPE_GPU_GL_TEXTURE_VIEW_H_
#include <functional>
#include <memory>
#include <utility>
#include "mediapipe/gpu/gl_base.h"
namespace mediapipe {
class GlContext;
class GlTextureViewManager;
class GpuBuffer;
class GlTextureView {
public:
GlTextureView() {}
~GlTextureView() { Release(); }
// TODO: make this class move-only.
GlContext* gl_context() const { return gl_context_; }
int width() const { return width_; }
int height() const { return height_; }
GLenum target() const { return target_; }
GLuint name() const { return name_; }
const GpuBuffer& gpu_buffer() const { return *gpu_buffer_; }
int plane() const { return plane_; }
using DetachFn = std::function<void(GlTextureView&)>;
using DoneWritingFn = std::function<void(const GlTextureView&)>;
private:
friend class GpuBuffer;
friend class GlTextureBuffer;
friend class GpuBufferStorageCvPixelBuffer;
GlTextureView(GlContext* context, GLenum target, GLuint name, int width,
int height, std::shared_ptr<GpuBuffer> gpu_buffer, int plane,
DetachFn detach, DoneWritingFn done_writing)
: gl_context_(context),
target_(target),
name_(name),
width_(width),
height_(height),
gpu_buffer_(std::move(gpu_buffer)),
plane_(plane),
detach_(std::move(detach)),
done_writing_(std::move(done_writing)) {}
// TODO: remove this friend declaration.
friend class GlTexture;
void Release();
// TODO: make this non-const.
void DoneWriting() const {
if (done_writing_) done_writing_(*this);
}
GlContext* gl_context_ = nullptr;
GLenum target_ = GL_TEXTURE_2D;
GLuint name_ = 0;
// Note: when scale is not 1, we still give the nominal size of the image.
int width_ = 0;
int height_ = 0;
std::shared_ptr<GpuBuffer> gpu_buffer_; // using shared_ptr temporarily
int plane_ = 0;
DetachFn detach_;
DoneWritingFn done_writing_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_TEXTURE_VIEW_H_
mediapipe/gpu/gpu_buffer.cc
+1 -232
View File
@@ -8,62 +8,7 @@
namespace mediapipe {
void GlTextureView::Release() {
if (detach_) detach_(*this);
detach_ = nullptr;
gl_context_ = nullptr;
gpu_buffer_ = nullptr;
plane_ = 0;
name_ = 0;
width_ = 0;
height_ = 0;
}
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#if TARGET_OS_OSX
typedef CVOpenGLTextureRef CVTextureType;
#else
typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
GlTextureView GpuBuffer::GetGlTextureView(int plane, bool for_reading) const {
CVReturn err;
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
#if TARGET_OS_OSX
CVTextureType cv_texture_temp;
err = CVOpenGLTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context->cv_texture_cache(),
GetCVPixelBufferRef(), NULL, &cv_texture_temp);
CHECK(cv_texture_temp && !err)
<< "CVOpenGLTextureCacheCreateTextureFromImage failed: " << err;
CFHolder<CVTextureType> cv_texture;
cv_texture.adopt(cv_texture_temp);
return GlTextureView(
gl_context.get(), CVOpenGLTextureGetTarget(*cv_texture),
CVOpenGLTextureGetName(*cv_texture), width(), height(), *this, plane,
[cv_texture](
mediapipe::GlTextureView&) { /* only retains cv_texture */ });
#else
const GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
format(), plane, gl_context->GetGlVersion());
CVTextureType cv_texture_temp;
err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context->cv_texture_cache(),
GetCVPixelBufferRef(), NULL, GL_TEXTURE_2D, info.gl_internal_format,
width() / info.downscale, height() / info.downscale, info.gl_format,
info.gl_type, plane, &cv_texture_temp);
CHECK(cv_texture_temp && !err)
<< "CVOpenGLESTextureCacheCreateTextureFromImage failed: " << err;
CFHolder<CVTextureType> cv_texture;
cv_texture.adopt(cv_texture_temp);
return GlTextureView(
gl_context.get(), CVOpenGLESTextureGetTarget(*cv_texture),
CVOpenGLESTextureGetName(*cv_texture), width(), height(), *this, plane,
[cv_texture](
mediapipe::GlTextureView&) { /* only retains cv_texture */ });
#endif // TARGET_OS_OSX
}
GpuBuffer GpuBuffer::CopyingImageFrame(const ImageFrame& image_frame) {
auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
@@ -72,187 +17,11 @@ GpuBuffer GpuBuffer::CopyingImageFrame(const ImageFrame& image_frame) {
CHECK_OK(maybe_buffer.status());
return GpuBuffer(std::move(maybe_buffer).value());
}
std::unique_ptr<ImageFrame> GpuBuffer::AsImageFrame() const {
CHECK(GetCVPixelBufferRef());
return CreateImageFrameForCVPixelBuffer(GetCVPixelBufferRef());
}
void GlTextureView::DoneWriting() const {
CHECK(gpu_buffer_);
#if TARGET_IPHONE_SIMULATOR
CVPixelBufferRef pixel_buffer = gpu_buffer_.GetCVPixelBufferRef();
CVReturn err = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
CHECK(err == kCVReturnSuccess)
<< "CVPixelBufferLockBaseAddress failed: " << err;
OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
size_t bytes_per_row = CVPixelBufferGetBytesPerRow(pixel_buffer);
uint8_t* pixel_ptr =
static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixel_buffer));
if (pixel_format == kCVPixelFormatType_32BGRA) {
// TODO: restore previous framebuffer? Move this to helper so we
// can use BindFramebuffer?
glViewport(0, 0, width(), height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, target(),
name(), 0);
size_t contiguous_bytes_per_row = width() * 4;
if (bytes_per_row == contiguous_bytes_per_row) {
glReadPixels(0, 0, width(), height(), GL_BGRA, GL_UNSIGNED_BYTE,
pixel_ptr);
} else {
std::vector<uint8_t> contiguous_buffer(contiguous_bytes_per_row *
height());
uint8_t* temp_ptr = contiguous_buffer.data();
glReadPixels(0, 0, width(), height(), GL_BGRA, GL_UNSIGNED_BYTE,
temp_ptr);
for (int i = 0; i < height(); ++i) {
memcpy(pixel_ptr, temp_ptr, contiguous_bytes_per_row);
temp_ptr += contiguous_bytes_per_row;
pixel_ptr += bytes_per_row;
}
}
} else {
LOG(ERROR) << "unsupported pixel format: " << pixel_format;
}
err = CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
CHECK(err == kCVReturnSuccess)
<< "CVPixelBufferUnlockBaseAddress failed: " << err;
#endif
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTextureView GpuBuffer::GetGlTextureView(int plane, bool for_reading) const {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
const GlTextureBufferSharedPtr& texture_buffer =
GetGlTextureBufferSharedPtr();
// Insert wait call to sync with the producer.
texture_buffer->WaitOnGpu();
CHECK_EQ(plane, 0);
GlTextureView::DetachFn detach;
if (for_reading) {
detach = [](mediapipe::GlTextureView& texture) {
// Inform the GlTextureBuffer that we have finished accessing its
// contents, and create a consumer sync point.
texture.gpu_buffer().GetGlTextureBufferSharedPtr()->DidRead(
texture.gl_context()->CreateSyncToken());
};
}
return GlTextureView(gl_context.get(), texture_buffer->target(),
texture_buffer->name(), width(), height(), *this, plane,
std::move(detach));
}
GpuBuffer GpuBuffer::CopyingImageFrame(const ImageFrame& image_frame) {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
auto buffer = GlTextureBuffer::Create(image_frame);
// TODO: does this need to set the texture params? We set them again when the
// texture is actually acccessed via GlTexture[View]. Or should they always be
// set on creation?
if (buffer->format() != GpuBufferFormat::kUnknown) {
glBindTexture(GL_TEXTURE_2D, buffer->name());
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
buffer->format(), /*plane=*/0, gl_context->GetGlVersion());
gl_context->SetStandardTextureParams(buffer->target(),
info.gl_internal_format);
glBindTexture(GL_TEXTURE_2D, 0);
}
return GpuBuffer(std::move(buffer));
}
static void ReadTexture(const GlTextureView& view, void* output, size_t size) {
// TODO: check buffer size? We could use glReadnPixels where available
// (OpenGL ES 3.2, i.e. nowhere). Note that, to fully check that the read
// won't overflow the buffer with glReadPixels, we'd also need to check or
// reset several glPixelStore parameters (e.g. what if someone had the
// ill-advised idea of setting GL_PACK_SKIP_PIXELS?).
CHECK(view.gl_context());
GlTextureInfo info =
GlTextureInfoForGpuBufferFormat(view.gpu_buffer().format(), view.plane(),
view.gl_context()->GetGlVersion());
GLint current_fbo;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &current_fbo);
CHECK_NE(current_fbo, 0);
GLint color_attachment_name;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment_name);
if (color_attachment_name != view.name()) {
// Save the viewport. Note that we assume that the color attachment is a
// GL_TEXTURE_2D texture.
GLint viewport[4];
glGetIntegerv(GL_VIEWPORT, viewport);
// Set the data from GLTextureView object.
glViewport(0, 0, view.width(), view.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, view.target(),
view.name(), 0);
glReadPixels(0, 0, view.width(), view.height(), info.gl_format,
info.gl_type, output);
// Restore from the saved viewport and color attachment name.
glViewport(viewport[0], viewport[1], viewport[2], viewport[3]);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
color_attachment_name, 0);
} else {
glReadPixels(0, 0, view.width(), view.height(), info.gl_format,
info.gl_type, output);
}
}
std::unique_ptr<ImageFrame> GpuBuffer::AsImageFrame() const {
ImageFormat::Format image_format = ImageFormatForGpuBufferFormat(format());
auto output = absl::make_unique<ImageFrame>(
image_format, width(), height(), ImageFrame::kGlDefaultAlignmentBoundary);
auto view = GetGlTextureView(0, true);
ReadTexture(view, output->MutablePixelData(), output->PixelDataSize());
return output;
}
void GlTextureView::DoneWriting() const {
CHECK(gpu_buffer_);
// Inform the GlTextureBuffer that we have produced new content, and create
// a producer sync point.
gpu_buffer_.GetGlTextureBufferSharedPtr()->Updated(
gl_context()->CreateSyncToken());
#ifdef __ANDROID__
// On (some?) Android devices, the texture may need to be explicitly
// detached from the current framebuffer.
// TODO: is this necessary even with the unbind in BindFramebuffer?
// It is not clear if this affected other contexts too, but let's keep it
// while in doubt.
GLint type = GL_NONE;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE,
&type);
if (type == GL_TEXTURE) {
GLint color_attachment = 0;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment);
if (color_attachment == name()) {
glBindFramebuffer(GL_FRAMEBUFFER, 0);
}
}
// Some Android drivers log a GL_INVALID_ENUM error after the first
// glGetFramebufferAttachmentParameteriv call if there is no bound object,
// even though it should be ok to ask for the type and get back GL_NONE.
// Let's just ignore any pending errors here.
GLenum error;
while ((error = glGetError()) != GL_NO_ERROR) {
}
#endif // __ANDROID__
return GpuBuffer(GlTextureBuffer::Create(image_frame));
}
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
mediapipe/gpu/gpu_buffer.h
+76 -94
View File
@@ -19,7 +19,9 @@
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_texture_view.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
#if defined(__APPLE__)
#include <CoreVideo/CoreVideo.h>
@@ -27,6 +29,10 @@
#include "mediapipe/objc/CFHolder.h"
#endif // defined(__APPLE__)
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#include "mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.h"
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#include "mediapipe/gpu/gl_texture_buffer.h"
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
@@ -34,7 +40,6 @@
namespace mediapipe {
class GlContext;
class GlTextureView;
// This class wraps a platform-specific buffer of GPU data.
// An instance of GpuBuffer acts as an opaque reference to the underlying
@@ -71,9 +76,9 @@ class GpuBuffer {
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
int width() const;
int height() const;
GpuBufferFormat format() const;
int width() const { return current_storage().width(); }
int height() const { return current_storage().height(); }
GpuBufferFormat format() const { return current_storage().format(); }
// Converts to true iff valid.
explicit operator bool() const { return operator!=(nullptr); }
@@ -88,8 +93,15 @@ class GpuBuffer {
// Allow assignment from nullptr.
GpuBuffer& operator=(std::nullptr_t other);
// TODO: split into read and write, remove const from write.
GlTextureView GetGlTextureView(int plane, bool for_reading) const;
GlTextureView GetGlTextureReadView(int plane) const {
return current_storage().GetGlTextureReadView(
std::make_shared<GpuBuffer>(*this), plane);
}
GlTextureView GetGlTextureWriteView(int plane) {
return current_storage().GetGlTextureWriteView(
std::make_shared<GpuBuffer>(*this), plane);
}
// Make a GpuBuffer copying the data from an ImageFrame.
static GpuBuffer CopyingImageFrame(const ImageFrame& image_frame);
@@ -99,114 +111,84 @@ class GpuBuffer {
// In order to work correctly across platforms, callers should always treat
// the returned ImageFrame as if it shares memory with the GpuBuffer, i.e.
// treat it as immutable if the GpuBuffer must not be modified.
std::unique_ptr<ImageFrame> AsImageFrame() const;
std::unique_ptr<ImageFrame> AsImageFrame() const {
return current_storage().AsImageFrame();
}
private:
class PlaceholderGpuBufferStorage
: public mediapipe::internal::GpuBufferStorage {
public:
int width() const override { return 0; }
int height() const override { return 0; }
virtual GpuBufferFormat format() const override {
return GpuBufferFormat::kUnknown;
}
GlTextureView GetGlTextureReadView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override {
return {};
}
GlTextureView GetGlTextureWriteView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override {
return {};
}
void ViewDoneWriting(const GlTextureView& view) override{};
std::unique_ptr<ImageFrame> AsImageFrame() const override {
return nullptr;
}
};
mediapipe::internal::GpuBufferStorage& no_storage() const {
static PlaceholderGpuBufferStorage placeholder;
return placeholder;
}
const mediapipe::internal::GpuBufferStorage& current_storage() const {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CFHolder<CVPixelBufferRef> pixel_buffer_;
if (pixel_buffer_ != nullptr) return pixel_buffer_;
#else
if (texture_buffer_) return *texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return no_storage();
}
mediapipe::internal::GpuBufferStorage& current_storage() {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
if (pixel_buffer_ != nullptr) return pixel_buffer_;
#else
if (texture_buffer_) return *texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return no_storage();
}
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GpuBufferStorageCvPixelBuffer pixel_buffer_;
#else
GlTextureBufferSharedPtr texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
};
class GlTextureView {
public:
GlTextureView() {}
~GlTextureView() { Release(); }
// TODO: make this class move-only.
GlContext* gl_context() const { return gl_context_; }
int width() const { return width_; }
int height() const { return height_; }
GLenum target() const { return target_; }
GLuint name() const { return name_; }
const GpuBuffer& gpu_buffer() const { return gpu_buffer_; }
int plane() const { return plane_; }
private:
friend class GpuBuffer;
using DetachFn = std::function<void(GlTextureView&)>;
GlTextureView(GlContext* context, GLenum target, GLuint name, int width,
int height, GpuBuffer gpu_buffer, int plane, DetachFn detach)
: gl_context_(context),
target_(target),
name_(name),
width_(width),
height_(height),
gpu_buffer_(std::move(gpu_buffer)),
plane_(plane),
detach_(std::move(detach)) {}
// TODO: remove this friend declaration.
friend class GlTexture;
void Release();
// TODO: make this non-const.
void DoneWriting() const;
GlContext* gl_context_ = nullptr;
GLenum target_ = GL_TEXTURE_2D;
GLuint name_ = 0;
// Note: when scale is not 1, we still give the nominal size of the image.
int width_ = 0;
int height_ = 0;
GpuBuffer gpu_buffer_;
int plane_ = 0;
DetachFn detach_;
};
inline bool GpuBuffer::operator==(std::nullptr_t other) const {
return &current_storage() == &no_storage();
}
inline bool GpuBuffer::operator==(const GpuBuffer& other) const {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
inline int GpuBuffer::width() const {
return static_cast<int>(CVPixelBufferGetWidth(*pixel_buffer_));
}
inline int GpuBuffer::height() const {
return static_cast<int>(CVPixelBufferGetHeight(*pixel_buffer_));
}
inline GpuBufferFormat GpuBuffer::format() const {
return GpuBufferFormatForCVPixelFormat(
CVPixelBufferGetPixelFormatType(*pixel_buffer_));
}
inline bool GpuBuffer::operator==(std::nullptr_t other) const {
return pixel_buffer_ == other;
}
inline bool GpuBuffer::operator==(const GpuBuffer& other) const {
return pixel_buffer_ == other.pixel_buffer_;
}
inline GpuBuffer& GpuBuffer::operator=(std::nullptr_t other) {
pixel_buffer_.reset(other);
return *this;
}
#else
inline int GpuBuffer::width() const { return texture_buffer_->width(); }
inline int GpuBuffer::height() const { return texture_buffer_->height(); }
inline GpuBufferFormat GpuBuffer::format() const {
return texture_buffer_->format();
}
inline bool GpuBuffer::operator==(std::nullptr_t other) const {
return texture_buffer_ == other;
}
inline bool GpuBuffer::operator==(const GpuBuffer& other) const {
return texture_buffer_ == other.texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
}
inline GpuBuffer& GpuBuffer::operator=(std::nullptr_t other) {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
pixel_buffer_.reset(other);
#else
texture_buffer_ = other;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return *this;
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_H_
mediapipe/gpu/gpu_buffer_storage.h
+41
View File
@@ -0,0 +1,41 @@
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_H_
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
namespace mediapipe {
class GlTextureView;
class GpuBuffer;
} // namespace mediapipe
namespace mediapipe {
namespace internal {
using mediapipe::GlTextureView;
using mediapipe::GpuBuffer;
using mediapipe::GpuBufferFormat;
class GlTextureViewManager {
public:
virtual ~GlTextureViewManager() = default;
virtual GlTextureView GetGlTextureReadView(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) const = 0;
virtual GlTextureView GetGlTextureWriteView(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) = 0;
virtual void ViewDoneWriting(const GlTextureView& view) = 0;
};
class GpuBufferStorage : public GlTextureViewManager {
public:
virtual ~GpuBufferStorage() = default;
virtual int width() const = 0;
virtual int height() const = 0;
virtual GpuBufferFormat format() const = 0;
virtual std::unique_ptr<ImageFrame> AsImageFrame() const = 0;
};
} // namespace internal
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_H_
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc
+116
View File
@@ -0,0 +1,116 @@
#include "mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/objc/util.h"
namespace mediapipe {
#if TARGET_OS_OSX
typedef CVOpenGLTextureRef CVTextureType;
#else
typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
GlTextureView GpuBufferStorageCvPixelBuffer::GetGlTextureReadView(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) const {
CVReturn err;
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
#if TARGET_OS_OSX
CVTextureType cv_texture_temp;
err = CVOpenGLTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context->cv_texture_cache(), **this, NULL,
&cv_texture_temp);
CHECK(cv_texture_temp && !err)
<< "CVOpenGLTextureCacheCreateTextureFromImage failed: " << err;
CFHolder<CVTextureType> cv_texture;
cv_texture.adopt(cv_texture_temp);
return GlTextureView(
gl_context.get(), CVOpenGLTextureGetTarget(*cv_texture),
CVOpenGLTextureGetName(*cv_texture), width(), height(), *this, plane,
[cv_texture](
mediapipe::GlTextureView&) { /* only retains cv_texture */ });
#else
const GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
format(), plane, gl_context->GetGlVersion());
CVTextureType cv_texture_temp;
err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context->cv_texture_cache(), **this, NULL,
GL_TEXTURE_2D, info.gl_internal_format, width() / info.downscale,
height() / info.downscale, info.gl_format, info.gl_type, plane,
&cv_texture_temp);
CHECK(cv_texture_temp && !err)
<< "CVOpenGLESTextureCacheCreateTextureFromImage failed: " << err;
CFHolder<CVTextureType> cv_texture;
cv_texture.adopt(cv_texture_temp);
return GlTextureView(
gl_context.get(), CVOpenGLESTextureGetTarget(*cv_texture),
CVOpenGLESTextureGetName(*cv_texture), width(), height(),
std::move(gpu_buffer), plane,
[cv_texture](mediapipe::GlTextureView&) { /* only retains cv_texture */ },
// TODO: make GetGlTextureView for write view non-const, remove cast
// Note: we have to copy *this here because this storage is currently
// stored in GpuBuffer by value, and so the this pointer becomes invalid
// if the GpuBuffer is moved/copied. TODO: fix this.
[me = *this](const mediapipe::GlTextureView& view) {
const_cast<GpuBufferStorageCvPixelBuffer*>(&me)->ViewDoneWriting(view);
});
#endif // TARGET_OS_OSX
}
GlTextureView GpuBufferStorageCvPixelBuffer::GetGlTextureWriteView(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) {
// For this storage there is currently no difference between read and write
// views, so we delegate to the read method.
return GetGlTextureReadView(std::move(gpu_buffer), plane);
}
void GpuBufferStorageCvPixelBuffer::ViewDoneWriting(const GlTextureView& view) {
#if TARGET_IPHONE_SIMULATOR
CVPixelBufferRef pixel_buffer = **this;
CHECK(pixel_buffer);
CVReturn err = CVPixelBufferLockBaseAddress(pixel_buffer, 0);
CHECK(err == kCVReturnSuccess)
<< "CVPixelBufferLockBaseAddress failed: " << err;
OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
size_t bytes_per_row = CVPixelBufferGetBytesPerRow(pixel_buffer);
uint8_t* pixel_ptr =
static_cast<uint8_t*>(CVPixelBufferGetBaseAddress(pixel_buffer));
if (pixel_format == kCVPixelFormatType_32BGRA) {
// TODO: restore previous framebuffer? Move this to helper so we
// can use BindFramebuffer?
glViewport(0, 0, view.width(), view.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, view.target(),
view.name(), 0);
size_t contiguous_bytes_per_row = view.width() * 4;
if (bytes_per_row == contiguous_bytes_per_row) {
glReadPixels(0, 0, view.width(), view.height(), GL_BGRA, GL_UNSIGNED_BYTE,
pixel_ptr);
} else {
std::vector<uint8_t> contiguous_buffer(contiguous_bytes_per_row *
view.height());
uint8_t* temp_ptr = contiguous_buffer.data();
glReadPixels(0, 0, view.width(), view.height(), GL_BGRA, GL_UNSIGNED_BYTE,
temp_ptr);
for (int i = 0; i < view.height(); ++i) {
memcpy(pixel_ptr, temp_ptr, contiguous_bytes_per_row);
temp_ptr += contiguous_bytes_per_row;
pixel_ptr += bytes_per_row;
}
}
} else {
LOG(ERROR) << "unsupported pixel format: " << pixel_format;
}
err = CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
CHECK(err == kCVReturnSuccess)
<< "CVPixelBufferUnlockBaseAddress failed: " << err;
#endif
}
std::unique_ptr<ImageFrame> GpuBufferStorageCvPixelBuffer::AsImageFrame()
const {
return CreateImageFrameForCVPixelBuffer(**this);
}
} // namespace mediapipe
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.h
+41
View File
@@ -0,0 +1,41 @@
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_CV_PIXEL_BUFFER_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_CV_PIXEL_BUFFER_H_
#include <CoreVideo/CoreVideo.h>
#include "mediapipe/gpu/gl_texture_view.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
#include "mediapipe/objc/CFHolder.h"
namespace mediapipe {
class GlContext;
class GpuBufferStorageCvPixelBuffer
: public mediapipe::internal::GpuBufferStorage,
public CFHolder<CVPixelBufferRef> {
public:
using CFHolder<CVPixelBufferRef>::CFHolder;
GpuBufferStorageCvPixelBuffer(const CFHolder<CVPixelBufferRef>& other)
: CFHolder(other) {}
GpuBufferStorageCvPixelBuffer(CFHolder<CVPixelBufferRef>&& other)
: CFHolder(std::move(other)) {}
int width() const { return static_cast<int>(CVPixelBufferGetWidth(**this)); }
int height() const {
return static_cast<int>(CVPixelBufferGetHeight(**this));
}
virtual GpuBufferFormat format() const {
return GpuBufferFormatForCVPixelFormat(
CVPixelBufferGetPixelFormatType(**this));
}
GlTextureView GetGlTextureReadView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override;
GlTextureView GetGlTextureWriteView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override;
std::unique_ptr<ImageFrame> AsImageFrame() const override;
void ViewDoneWriting(const GlTextureView& view) override;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_STORAGE_CV_PIXEL_BUFFER_H_