diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 196de307..b0c1c22b 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -749,11 +749,9 @@ cc_library( name = "gl_calculator_helper", srcs = [ "gl_calculator_helper.cc", - "gl_calculator_helper_impl_common.cc", ], hdrs = [ "gl_calculator_helper.h", - "gl_calculator_helper_impl.h", ], linkopts = select({ "//conditions:default": [], diff --git a/mediapipe/gpu/gl_calculator_helper.cc b/mediapipe/gpu/gl_calculator_helper.cc index ba142397..7d317e0f 100644 --- a/mediapipe/gpu/gl_calculator_helper.cc +++ b/mediapipe/gpu/gl_calculator_helper.cc @@ -20,18 +20,32 @@ #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/framework/port/status.h" -#include "mediapipe/gpu/gl_calculator_helper_impl.h" #include "mediapipe/gpu/gpu_buffer.h" #include "mediapipe/gpu/gpu_service.h" namespace mediapipe { -// The constructor and destructor need to be defined here so that -// std::unique_ptr can see the full definition of GlCalculatorHelperImpl. -// In the header, it is an incomplete type. GlCalculatorHelper::GlCalculatorHelper() {} -GlCalculatorHelper::~GlCalculatorHelper() {} +GlCalculatorHelper::~GlCalculatorHelper() { + if (!Initialized()) return; + RunInGlContext( + [this] { + if (framebuffer_) { + glDeleteFramebuffers(1, &framebuffer_); + framebuffer_ = 0; + } + return absl::OkStatus(); + }, + /*calculator_context=*/nullptr) + .IgnoreError(); +} + +void GlCalculatorHelper::InitializeInternal(CalculatorContext* cc, + GpuResources* gpu_resources) { + gpu_resources_ = gpu_resources; + gl_context_ = gpu_resources_->gl_context(cc); +} absl::Status GlCalculatorHelper::Open(CalculatorContext* cc) { CHECK(cc); @@ -39,19 +53,16 @@ absl::Status GlCalculatorHelper::Open(CalculatorContext* cc) { RET_CHECK(gpu_service.IsAvailable()) << "GPU service not available. Did you forget to call " "GlCalculatorHelper::UpdateContract?"; - // TODO return error from impl_ (needs two-stage init) - impl_ = - absl::make_unique(cc, &gpu_service.GetObject()); + InitializeInternal(cc, &gpu_service.GetObject()); return absl::OkStatus(); } void GlCalculatorHelper::InitializeForTest(GpuSharedData* gpu_shared) { - impl_ = absl::make_unique( - nullptr, gpu_shared->gpu_resources.get()); + InitializeInternal(nullptr, gpu_shared->gpu_resources.get()); } void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) { - impl_ = absl::make_unique(nullptr, gpu_resources); + InitializeInternal(nullptr, gpu_resources); } // static @@ -88,44 +99,109 @@ absl::Status GlCalculatorHelper::SetupInputSidePackets( return absl::OkStatus(); } +absl::Status GlCalculatorHelper::RunInGlContext( + std::function gl_func, + CalculatorContext* calculator_context) { + if (calculator_context) { + return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(), + calculator_context->InputTimestamp()); + } else { + return gl_context_->Run(std::move(gl_func)); + } +} + absl::Status GlCalculatorHelper::RunInGlContext( std::function gl_func) { - if (!impl_) return absl::InternalError("helper not initialized"); + if (!Initialized()) return absl::InternalError("helper not initialized"); // TODO: Remove LegacyCalculatorSupport from MediaPipe OSS. auto calculator_context = LegacyCalculatorSupport::Scoped::current(); - return impl_->RunInGlContext(gl_func, calculator_context); + return RunInGlContext(gl_func, calculator_context); } -GLuint GlCalculatorHelper::framebuffer() const { return impl_->framebuffer(); } +GLuint GlCalculatorHelper::framebuffer() const { return framebuffer_; } + +void GlCalculatorHelper::CreateFramebuffer() { + // Our framebuffer will have a color attachment but no depth attachment, + // so it's important that the depth test be off. It is disabled by default, + // but we wanted to be explicit. + // TODO: move this to glBindFramebuffer? + glDisable(GL_DEPTH_TEST); + glGenFramebuffers(1, &framebuffer_); +} void GlCalculatorHelper::BindFramebuffer(const GlTexture& dst) { - return impl_->BindFramebuffer(dst); +#ifdef __ANDROID__ + // On (some?) Android devices, attaching a new texture to the frame buffer + // does not seem to detach the old one. As a result, using that texture + // for texturing can produce incorrect output. See b/32091368 for details. + // To fix this, we have to call either glBindFramebuffer with a FBO id of 0 + // or glFramebufferTexture2D with a texture ID of 0. + glBindFramebuffer(GL_FRAMEBUFFER, 0); +#endif + if (!framebuffer_) { + CreateFramebuffer(); + } + glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); + glViewport(0, 0, dst.width(), dst.height()); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(), + dst.name(), 0); + +#ifndef NDEBUG + GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER); + if (status != GL_FRAMEBUFFER_COMPLETE) { + VLOG(2) << "incomplete framebuffer: " << status; + } +#endif } -GlTexture GlCalculatorHelper::CreateSourceTexture( - const GpuBuffer& pixel_buffer) { - return impl_->CreateSourceTexture(pixel_buffer); +GlTexture GlCalculatorHelper::MapGpuBuffer(const GpuBuffer& gpu_buffer, + GlTextureView view) { + if (gpu_buffer.format() != GpuBufferFormat::kUnknown) { + // TODO: do the params need to be reset here?? + glBindTexture(view.target(), view.name()); + GlTextureInfo info = GlTextureInfoForGpuBufferFormat( + gpu_buffer.format(), view.plane(), GetGlVersion()); + gl_context_->SetStandardTextureParams(view.target(), + info.gl_internal_format); + glBindTexture(view.target(), 0); + } + + return GlTexture(std::move(view), gpu_buffer); +} + +GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& gpu_buffer) { + return CreateSourceTexture(gpu_buffer, 0); +} + +GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& gpu_buffer, + int plane) { + return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView(plane)); } GlTexture GlCalculatorHelper::CreateSourceTexture( const ImageFrame& image_frame) { - return impl_->CreateSourceTexture(image_frame); -} - -GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& pixel_buffer, - int plane) { - return impl_->CreateSourceTexture(pixel_buffer, plane); + auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame); + return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView(0)); } GpuBuffer GlCalculatorHelper::GpuBufferWithImageFrame( std::shared_ptr image_frame) { - return impl_->GpuBufferWithImageFrame(std::move(image_frame)); + return GpuBuffer( + std::make_shared(std::move(image_frame))); } GpuBuffer GlCalculatorHelper::GpuBufferCopyingImageFrame( const ImageFrame& image_frame) { - return impl_->GpuBufferCopyingImageFrame(image_frame); +#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER + auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame); + // Converts absl::StatusOr to absl::Status since CHECK_OK() currently only + // deals with absl::Status in MediaPipe OSS. + CHECK_OK(maybe_buffer.status()); + return GpuBuffer(std::move(maybe_buffer).value()); +#else + return GpuBuffer(GlTextureBuffer::Create(image_frame)); +#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER } void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, @@ -136,23 +212,36 @@ void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, *height = pixel_buffer.height(); } -GlTexture GlCalculatorHelper::CreateDestinationTexture(int output_width, - int output_height, +GlTexture GlCalculatorHelper::CreateDestinationTexture(int width, int height, GpuBufferFormat format) { - return impl_->CreateDestinationTexture(output_width, output_height, format); -} + if (!framebuffer_) { + CreateFramebuffer(); + } -GlContext& GlCalculatorHelper::GetGlContext() const { - return impl_->GetGlContext(); -} - -GlVersion GlCalculatorHelper::GetGlVersion() const { - return impl_->GetGlVersion(); + GpuBuffer gpu_buffer = + gpu_resources_->gpu_buffer_pool().GetBuffer(width, height, format); + return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView(0)); } GlTexture GlCalculatorHelper::CreateSourceTexture( const mediapipe::Image& image) { - return impl_->CreateSourceTexture(image.GetGpuBuffer()); + return CreateSourceTexture(image.GetGpuBuffer()); +} + +template <> +std::unique_ptr GlTexture::GetFrame() const { + view_->DoneWriting(); + std::shared_ptr view = + gpu_buffer_.GetReadView(); + auto copy = absl::make_unique(); + copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary); + return copy; +} + +template <> +std::unique_ptr GlTexture::GetFrame() const { + view_->DoneWriting(); + return absl::make_unique(gpu_buffer_); } template <> diff --git a/mediapipe/gpu/gl_calculator_helper.h b/mediapipe/gpu/gl_calculator_helper.h index 0a0cc16c..727be782 100644 --- a/mediapipe/gpu/gl_calculator_helper.h +++ b/mediapipe/gpu/gl_calculator_helper.h @@ -33,7 +33,6 @@ namespace mediapipe { -class GlCalculatorHelperImpl; class GlTexture; class GpuResources; struct GpuSharedData; @@ -161,15 +160,30 @@ class GlCalculatorHelper { // TODO: do we need an unbind method too? void BindFramebuffer(const GlTexture& dst); - GlContext& GetGlContext() const; + GlContext& GetGlContext() const { return *gl_context_; } - GlVersion GetGlVersion() const; + GlVersion GetGlVersion() const { return gl_context_->GetGlVersion(); } // Check if the calculator helper has been previously initialized. - bool Initialized() { return impl_ != nullptr; } + bool Initialized() { return gpu_resources_ != nullptr; } private: - std::unique_ptr impl_; + void InitializeInternal(CalculatorContext* cc, GpuResources* gpu_resources); + + absl::Status RunInGlContext(std::function gl_func, + CalculatorContext* calculator_context); + + // Makes a GpuBuffer accessible as a texture in the GL context. + GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, GlTextureView view); + + // Create the framebuffer for rendering. + void CreateFramebuffer(); + + std::shared_ptr gl_context_; + + GLuint framebuffer_ = 0; + + GpuResources* gpu_resources_ = nullptr; }; // Represents an OpenGL texture, and is a 'view' into the memory pool. @@ -204,7 +218,7 @@ class GlTexture { explicit GlTexture(GlTextureView view, GpuBuffer gpu_buffer) : gpu_buffer_(std::move(gpu_buffer)), view_(std::make_shared(std::move(view))) {} - friend class GlCalculatorHelperImpl; + friend class GlCalculatorHelper; // We store the GpuBuffer to support GetFrame, and to ensure that the storage // outlives the view. GpuBuffer gpu_buffer_; diff --git a/mediapipe/gpu/gl_calculator_helper_impl.h b/mediapipe/gpu/gl_calculator_helper_impl.h deleted file mode 100644 index 72b3265f..00000000 --- a/mediapipe/gpu/gl_calculator_helper_impl.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_ -#define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_ - -#include "mediapipe/gpu/gl_calculator_helper.h" -#include "mediapipe/gpu/gpu_shared_data_internal.h" - -#ifdef __OBJC__ -#import -#import -#endif // __OBJC__ - -#ifdef __ANDROID__ -#include "mediapipe/gpu/gl_texture_buffer_pool.h" -#endif - -namespace mediapipe { - -// This class implements the GlCalculatorHelper for iOS and Android. -// See GlCalculatorHelper for details on these methods. -class GlCalculatorHelperImpl { - public: - explicit GlCalculatorHelperImpl(CalculatorContext* cc, - GpuResources* gpu_resources); - ~GlCalculatorHelperImpl(); - - absl::Status RunInGlContext(std::function gl_func, - CalculatorContext* calculator_context); - - GlTexture CreateSourceTexture(const ImageFrame& image_frame); - GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer); - - // Note: multi-plane support is currently only available on iOS. - GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer, int plane); - - // Creates a framebuffer and returns the texture that it is bound to. - GlTexture CreateDestinationTexture(int output_width, int output_height, - GpuBufferFormat format); - - GpuBuffer GpuBufferWithImageFrame(std::shared_ptr image_frame); - GpuBuffer GpuBufferCopyingImageFrame(const ImageFrame& image_frame); - - GLuint framebuffer() const { return framebuffer_; } - void BindFramebuffer(const GlTexture& dst); - - GlVersion GetGlVersion() const { return gl_context_->GetGlVersion(); } - - GlContext& GetGlContext() const; - - // For internal use. - static void ReadTexture(const GlTextureView& view, void* output, size_t size); - - private: - // Makes a GpuBuffer accessible as a texture in the GL context. - GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, GlTextureView view); - - // Create the framebuffer for rendering. - void CreateFramebuffer(); - - std::shared_ptr gl_context_; - - GLuint framebuffer_ = 0; - - GpuResources& gpu_resources_; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_ diff --git a/mediapipe/gpu/gl_calculator_helper_impl_common.cc b/mediapipe/gpu/gl_calculator_helper_impl_common.cc deleted file mode 100644 index 6311d890..00000000 --- a/mediapipe/gpu/gl_calculator_helper_impl_common.cc +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include - -#include "absl/memory/memory.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/gpu/gl_calculator_helper_impl.h" -#include "mediapipe/gpu/gpu_buffer_format.h" -#include "mediapipe/gpu/gpu_shared_data_internal.h" -#include "mediapipe/gpu/image_frame_view.h" - -namespace mediapipe { - -GlCalculatorHelperImpl::GlCalculatorHelperImpl(CalculatorContext* cc, - GpuResources* gpu_resources) - : gpu_resources_(*gpu_resources) { - gl_context_ = gpu_resources_.gl_context(cc); -} - -GlCalculatorHelperImpl::~GlCalculatorHelperImpl() { - RunInGlContext( - [this] { - if (framebuffer_) { - glDeleteFramebuffers(1, &framebuffer_); - framebuffer_ = 0; - } - return absl::OkStatus(); - }, - /*calculator_context=*/nullptr) - .IgnoreError(); -} - -GlContext& GlCalculatorHelperImpl::GetGlContext() const { return *gl_context_; } - -absl::Status GlCalculatorHelperImpl::RunInGlContext( - std::function gl_func, - CalculatorContext* calculator_context) { - if (calculator_context) { - return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(), - calculator_context->InputTimestamp()); - } else { - return gl_context_->Run(std::move(gl_func)); - } -} - -void GlCalculatorHelperImpl::CreateFramebuffer() { - // Our framebuffer will have a color attachment but no depth attachment, - // so it's important that the depth test be off. It is disabled by default, - // but we wanted to be explicit. - // TODO: move this to glBindFramebuffer? - glDisable(GL_DEPTH_TEST); - glGenFramebuffers(1, &framebuffer_); -} - -void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) { -#ifdef __ANDROID__ - // On (some?) Android devices, attaching a new texture to the frame buffer - // does not seem to detach the old one. As a result, using that texture - // for texturing can produce incorrect output. See b/32091368 for details. - // To fix this, we have to call either glBindFramebuffer with a FBO id of 0 - // or glFramebufferTexture2D with a texture ID of 0. - glBindFramebuffer(GL_FRAMEBUFFER, 0); -#endif - if (!framebuffer_) { - CreateFramebuffer(); - } - glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_); - glViewport(0, 0, dst.width(), dst.height()); - glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(), - dst.name(), 0); - -#ifndef NDEBUG - GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER); - if (status != GL_FRAMEBUFFER_COMPLETE) { - VLOG(2) << "incomplete framebuffer: " << status; - } -#endif -} - -GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer, - GlTextureView view) { - if (gpu_buffer.format() != GpuBufferFormat::kUnknown) { - // TODO: do the params need to be reset here?? - glBindTexture(view.target(), view.name()); - GlTextureInfo info = GlTextureInfoForGpuBufferFormat( - gpu_buffer.format(), view.plane(), GetGlVersion()); - gl_context_->SetStandardTextureParams(view.target(), - info.gl_internal_format); - glBindTexture(view.target(), 0); - } - - return GlTexture(std::move(view), gpu_buffer); -} - -GlTexture GlCalculatorHelperImpl::CreateSourceTexture( - const GpuBuffer& gpu_buffer) { - return CreateSourceTexture(gpu_buffer, 0); -} - -GlTexture GlCalculatorHelperImpl::CreateSourceTexture( - const GpuBuffer& gpu_buffer, int plane) { - return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView(plane)); -} - -GlTexture GlCalculatorHelperImpl::CreateSourceTexture( - const ImageFrame& image_frame) { - auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame); - return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView(0)); -} - -GpuBuffer GlCalculatorHelperImpl::GpuBufferWithImageFrame( - std::shared_ptr image_frame) { - return GpuBuffer( - std::make_shared(std::move(image_frame))); -} - -GpuBuffer GlCalculatorHelperImpl::GpuBufferCopyingImageFrame( - const ImageFrame& image_frame) { -#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER - auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame); - // Converts absl::StatusOr to absl::Status since CHECK_OK() currently only - // deals with absl::Status in MediaPipe OSS. - CHECK_OK(maybe_buffer.status()); - return GpuBuffer(std::move(maybe_buffer).value()); -#else - return GpuBuffer(GlTextureBuffer::Create(image_frame)); -#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER -} - -template <> -std::unique_ptr GlTexture::GetFrame() const { - view_->DoneWriting(); - std::shared_ptr view = - gpu_buffer_.GetReadView(); - auto copy = absl::make_unique(); - copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary); - return copy; -} - -template <> -std::unique_ptr GlTexture::GetFrame() const { - view_->DoneWriting(); - return absl::make_unique(gpu_buffer_); -} - -GlTexture GlCalculatorHelperImpl::CreateDestinationTexture( - int width, int height, GpuBufferFormat format) { - if (!framebuffer_) { - CreateFramebuffer(); - } - - GpuBuffer gpu_buffer = - gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format); - return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView(0)); -} - -} // namespace mediapipe