talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
PiperOrigin-RevId: 253489161
This commit is contained in:
MediaPipe Team
2019-06-16 16:06:57 -07:00
committed by jqtang
commit d68f5e4169
844 changed files with 134997 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/gpu/BUILD
+495
View File
@@ -0,0 +1,495 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
# Disabling GPU support is sometimes useful on desktop Linux because SwiftShader can
# interfere with desktop GL. b/73494271
config_setting(
name = "disable_gpu",
define_values = {
"MEDIAPIPE_DISABLE_GPU": "1",
},
visibility = ["//visibility:public"],
)
cc_library(
name = "gpu_service",
srcs = ["gpu_service.cc"],
hdrs = ["gpu_service.h"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:graph_service"],
)
cc_library(
name = "graph_support",
hdrs = ["graph_support.h"],
visibility = ["//visibility:public"],
deps = [":gpu_service"],
)
cc_library(
name = "gl_base",
features = ["-layering_check"],
linkopts = select({
"//conditions:default": [],
"//mediapipe:android": [
"-lGLESv2",
"-lEGL",
# Note: on Android, libGLESv3.so is normally a symlink to
# libGLESv2.so, so we don't need to link to it. In fact, we
# do not _want_ to link to it, or we would be unable to load
# on API level < 18, where the symlink is missing entirely.
# Note: if we ever find a strange version of Android where the
# GLESv3 library is not a symlink, we will have to load it at
# runtime. Weak GLESv3 symbols will still be resolved if we
# load it early enough.
],
}),
textual_hdrs = ["gl_base.h"],
visibility = ["//visibility:public"],
deps = [":gl_base_hdr"] + select({
"//mediapipe:android": [],
"//conditions:default": [
],
}),
)
cc_library(
name = "gl_base_hdr",
hdrs = ["gl_base.h"],
features = ["-layering_check"],
# Note: need the frameworks on Apple platforms to get the headers.
visibility = ["//visibility:public"],
deps = select({
"//mediapipe:android": [],
"//conditions:default": [
],
}),
)
cc_library(
name = "gl_thread_collector",
hdrs = ["gl_thread_collector.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
],
)
cc_library(
name = "gl_context",
srcs = [
"gl_context.cc",
"gl_context_internal.h",
] + select({
"//conditions:default": [
"gl_context_egl.cc",
],
}),
hdrs = ["gl_context.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_thread_collector",
"//mediapipe/framework:executor",
"//mediapipe/framework:mediapipe_profiling",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/framework/port:threadpool",
"@com_google_absl//absl/base:dynamic_annotations",
"@com_google_absl//absl/debugging:leak_check",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
],
)
cc_library(
name = "gl_texture_buffer",
srcs = ["gl_texture_buffer.cc"],
hdrs = ["gl_texture_buffer.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_context",
":gpu_buffer_format",
# TODO: remove this dependency. Some other teams' tests
# depend on having an indirect image_frame dependency, need to be
# fixed first.
"//mediapipe/framework/formats:image_frame",
"@com_google_absl//absl/memory",
],
)
cc_library(
name = "gpu_buffer",
hdrs = ["gpu_buffer.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gpu_buffer_format",
] + select({
"//conditions:default": [
":gl_texture_buffer",
],
}),
)
cc_library(
name = "gpu_buffer_format",
srcs = ["gpu_buffer_format.cc"],
hdrs = ["gpu_buffer_format.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
"//mediapipe/framework/deps:no_destructor",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/container:flat_hash_map",
],
)
proto_library(
name = "gl_context_options_proto",
srcs = ["gl_context_options.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "gl_context_options_cc_proto",
srcs = ["gl_context_options.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":gl_context_options_proto"],
)
# This is a hack needed to work around some issues with strict hdrs_check.
# See e.g. b/67524270.
cc_library(
name = "gpu_shared_data_header",
textual_hdrs = [
"gpu_shared_data_internal.h",
],
visibility = ["//visibility:private"],
deps = [
":gl_base",
":gl_context",
],
)
cc_library(
name = "gpu_shared_data_internal",
srcs = select({
"//conditions:default": [
"gpu_shared_data_internal.cc",
],
# iOS uses an Objective-C++ version of this, built in MediaPipeGraphGPUData.
":disable_gpu": [],
}),
hdrs = [
"gpu_shared_data_internal.h",
],
defines = select({
"//conditions:default": [],
":disable_gpu": ["MEDIAPIPE_DISABLE_GPU"],
}),
visibility = ["//visibility:public"],
deps = [
"//mediapipe/gpu:gl_context_options_cc_proto",
":graph_support",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:executor",
"//mediapipe/framework:calculator_node",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/deps:no_destructor",
] + select({
"//conditions:default": [
":gl_base",
":gl_context",
":gpu_buffer_multi_pool",
":gpu_shared_data_header",
],
":disable_gpu": [],
}) + select({
"//conditions:default": [],
":disable_gpu": [],
}),
)
cc_library(
name = "gpu_buffer_multi_pool",
srcs = ["gpu_buffer_multi_pool.cc"] + select({
"//conditions:default": [
"gl_texture_buffer_pool.cc",
],
}),
hdrs = ["gpu_buffer_multi_pool.h"] + select({
"//conditions:default": [
"gl_texture_buffer_pool.h",
],
}),
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gpu_buffer",
":gpu_shared_data_header",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_node",
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
] + select({
"//conditions:default": [
":gl_texture_buffer",
],
}),
)
cc_library(
name = "shader_util",
srcs = ["shader_util.cc"],
hdrs = ["shader_util.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
"//mediapipe/framework/port:logging",
],
)
HELPER_ANDROID_SRCS = [
"gl_calculator_helper_impl_android.cc",
"gl_calculator_helper_impl_common.cc",
]
HELPER_ANDROID_HDRS = [
"egl_surface_holder.h",
]
HELPER_COMMON_SRCS = [
"gl_calculator_helper.cc",
]
HELPER_COMMON_HDRS = [
"gl_calculator_helper.h",
"gl_calculator_helper_impl.h",
]
cc_library(
name = "gl_calculator_helper",
srcs = select({
"//conditions:default": HELPER_COMMON_SRCS + HELPER_ANDROID_SRCS,
}),
hdrs = HELPER_COMMON_HDRS + select({
"//conditions:default": HELPER_ANDROID_HDRS,
}),
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_context",
":gpu_buffer",
":gpu_buffer_multi_pool",
":gpu_shared_data_internal",
":gpu_service",
":graph_support",
":shader_util",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_node",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:demangle",
"//mediapipe/framework:legacy_calculator_support",
"//mediapipe/framework:packet",
"//mediapipe/framework:packet_set",
"//mediapipe/framework:packet_type",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
"//mediapipe/framework/deps:registration",
"//mediapipe/framework/port:map_util",
] + select({
"//conditions:default": [
],
}),
)
proto_library(
name = "scale_mode_proto",
srcs = ["scale_mode.proto"],
visibility = ["//visibility:public"],
)
mediapipe_cc_proto_library(
name = "scale_mode_cc_proto",
srcs = ["scale_mode.proto"],
visibility = ["//visibility:public"],
deps = [":scale_mode_proto"],
)
cc_library(
name = "gl_quad_renderer",
srcs = ["gl_quad_renderer.cc"],
hdrs = ["gl_quad_renderer.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_simple_shaders",
":shader_util",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:scale_mode_cc_proto",
],
)
cc_library(
name = "gl_simple_shaders",
srcs = ["gl_simple_shaders.cc"],
hdrs = ["gl_simple_shaders.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
],
)
### General calculator superclasses
cc_library(
name = "gl_simple_calculator",
srcs = ["gl_simple_calculator.cc"],
hdrs = ["gl_simple_calculator.h"],
visibility = ["//visibility:public"],
deps = [
":gl_calculator_helper",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:status",
],
)
### Converters
cc_library(
name = "gpu_buffer_to_image_frame_calculator",
srcs = ["gpu_buffer_to_image_frame_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":gl_calculator_helper",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
cc_library(
name = "image_frame_to_gpu_buffer_calculator",
srcs = ["image_frame_to_gpu_buffer_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":gl_calculator_helper",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
proto_library(
name = "gl_scaler_calculator_proto",
srcs = ["gl_scaler_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_proto",
"//mediapipe/gpu:scale_mode_proto",
],
)
mediapipe_cc_proto_library(
name = "gl_scaler_calculator_cc_proto",
srcs = ["gl_scaler_calculator.proto"],
cc_deps = [
":scale_mode_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":gl_scaler_calculator_proto"],
)
cc_library(
name = "gl_scaler_calculator",
srcs = ["gl_scaler_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":gl_calculator_helper",
":gl_quad_renderer",
":gl_simple_shaders",
":shader_util",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gl_scaler_calculator_cc_proto",
],
alwayslink = 1,
)
cc_library(
name = "gl_surface_sink_calculator",
srcs = ["gl_surface_sink_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":gl_calculator_helper",
":gl_quad_renderer",
":shader_util",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gl_surface_sink_calculator_cc_proto",
"@com_google_absl//absl/synchronization",
],
alwayslink = 1,
)
proto_library(
name = "gl_surface_sink_calculator_proto",
srcs = ["gl_surface_sink_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_proto",
"//mediapipe/gpu:scale_mode_proto",
],
)
mediapipe_cc_proto_library(
name = "gl_surface_sink_calculator_cc_proto",
srcs = ["gl_surface_sink_calculator.proto"],
cc_deps = [
":scale_mode_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":gl_surface_sink_calculator_proto"],
)
mediapipe/gpu/egl_surface_holder.h
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_EGL_SURFACE_HOLDER_H_
#define MEDIAPIPE_GPU_EGL_SURFACE_HOLDER_H_
#include "absl/synchronization/mutex.h"
#include "mediapipe/gpu/gl_base.h"
namespace mediapipe {
// This is used to pass an EGLSurface to a GlSurfaceSinkCalculator.
struct EglSurfaceHolder {
// Access to the surface needs to be protected by a mutex to ensure that the
// application does not destroy the surface while MediaPipe is using it.
// NOTE: Code that needs to grab the GlContext mutex should always do so
// before grabbing this one. For example, do not call GlContext::Run or
// GlCalculatorHelper::RunInGlContext while holding this mutex, but instead
// grab this inside the callable passed to them.
absl::Mutex mutex;
EGLSurface surface GUARDED_BY(mutex) = EGL_NO_SURFACE;
// True if MediaPipe created the surface and is responsible for destroying it.
bool owned GUARDED_BY(mutex) = false;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_EGL_SURFACE_HOLDER_H_
mediapipe/gpu/gl_base.h
+94
View File
@@ -0,0 +1,94 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This header includes platform-specific headers for OpenGL.
#ifndef MEDIAPIPE_GPU_GL_BASE_H_
#define MEDIAPIPE_GPU_GL_BASE_H_
#if defined(__EMSCRIPTEN__)
#include <emscripten/html5.h>
#endif // defined(__EMSCRIPTEN__)
#if defined(__APPLE__)
#include <TargetConditionals.h>
#if TARGET_OS_OSX
#define HAS_NSGL 1
#include <OpenGL/OpenGL.h>
#if CGL_VERSION_1_3
#include <OpenGL/gl3.h>
#include <OpenGL/gl3ext.h>
#else
#include <OpenGL/gl.h>
#include <OpenGL/glext.h>
#endif // CGL_VERSION_1_3
#else
#define HAS_EAGL 1
#include <OpenGLES/ES2/gl.h>
#include <OpenGLES/ES2/glext.h>
#include <OpenGLES/ES3/gl.h>
#include <OpenGLES/ES3/glext.h>
#endif // TARGET_OS_OSX
#else
#define HAS_EGL 1
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#ifdef __ANDROID__
// Weak-link all GL APIs included from this point on.
// TODO: Annotate these with availability attributes for the
// appropriate versions of Android, by including gl{3,31,31}.h and resetting
// GL_APICALL for each.
#undef GL_APICALL
#define GL_APICALL __attribute__((weak_import)) KHRONOS_APICALL
#endif // __ANDROID__
#include <GLES3/gl32.h>
// When using the Linux EGL headers, we may end up pulling a
// "#define Status int" from Xlib.h, which interferes with util::Status.
#undef Status
// More crud from X
#undef None
#undef Bool
#undef Success
#endif // defined(__APPLE__)
namespace mediapipe {
// Doing this as an inline function allows us to avoid unwanted "pointer will
// never be null" errors on certain platforms and compilers.
template <typename T>
inline bool SymbolAvailable(T* symbol) {
return symbol != nullptr;
}
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_BASE_H_
mediapipe/gpu/gl_calculator_helper.cc
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "absl/memory/memory.h"
#include "mediapipe/framework/legacy_calculator_support.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_service.h"
namespace mediapipe {
GlTexture::GlTexture(GLuint name, int width, int height)
: name_(name), width_(width), height_(height), target_(GL_TEXTURE_2D) {}
// The constructor and destructor need to be defined here so that
// std::unique_ptr can see the full definition of GlCalculatorHelperImpl.
// In the header, it is an incomplete type.
GlCalculatorHelper::GlCalculatorHelper() {}
GlCalculatorHelper::~GlCalculatorHelper() {}
::mediapipe::Status GlCalculatorHelper::Open(CalculatorContext* cc) {
CHECK(cc);
// TODO return error from impl_ (needs two-stage init)
impl_ = absl::make_unique<GlCalculatorHelperImpl>(
cc, &cc->Service(kGpuService).GetObject());
return ::mediapipe::OkStatus();
}
void GlCalculatorHelper::InitializeForTest(GpuSharedData* gpu_shared) {
impl_ = absl::make_unique<GlCalculatorHelperImpl>(
nullptr, gpu_shared->gpu_resources.get());
}
void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
impl_ = absl::make_unique<GlCalculatorHelperImpl>(nullptr, gpu_resources);
}
// static
::mediapipe::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc) {
cc->UseService(kGpuService);
// Allow the legacy side packet to be provided, too, for backwards
// compatibility with existing graphs. It will just be ignored.
auto& input_side_packets = cc->InputSidePackets();
auto id = input_side_packets.GetId(kGpuSharedTagName, 0);
if (id.IsValid()) {
input_side_packets.Get(id).Set<GpuSharedData*>();
}
return ::mediapipe::OkStatus();
}
// static
::mediapipe::Status GlCalculatorHelper::SetupInputSidePackets(
PacketTypeSet* input_side_packets) {
auto cc = LegacyCalculatorSupport::Scoped<CalculatorContract>::current();
if (cc) {
CHECK_EQ(input_side_packets, &cc->InputSidePackets());
return UpdateContract(cc);
}
// TODO: remove when we can.
LOG(WARNING)
<< "CalculatorContract not available. If you're calling this "
"from a GetContract method, call GlCalculatorHelper::UpdateContract "
"instead.";
auto id = input_side_packets->GetId(kGpuSharedTagName, 0);
RET_CHECK(id.IsValid()) << "A " << mediapipe::kGpuSharedTagName
<< " input side packet is required here.";
input_side_packets->Get(id).Set<GpuSharedData*>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status GlCalculatorHelper::RunInGlContext(
std::function<::mediapipe::Status(void)> gl_func) {
if (!impl_) return ::mediapipe::InternalError("helper not initialized");
// TODO: Remove LegacyCalculatorSupport from MediaPipe OSS.
auto calculator_context =
LegacyCalculatorSupport::Scoped<CalculatorContext>::current();
return impl_->RunInGlContext(gl_func, calculator_context);
}
GLuint GlCalculatorHelper::framebuffer() const { return impl_->framebuffer(); }
void GlCalculatorHelper::BindFramebuffer(const GlTexture& dst) {
return impl_->BindFramebuffer(dst);
}
GlTexture GlCalculatorHelper::CreateSourceTexture(
const GpuBuffer& pixel_buffer) {
return impl_->CreateSourceTexture(pixel_buffer);
}
GlTexture GlCalculatorHelper::CreateSourceTexture(
const ImageFrame& image_frame) {
return impl_->CreateSourceTexture(image_frame);
}
#ifdef __APPLE__
GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& pixel_buffer,
int plane) {
return impl_->CreateSourceTexture(pixel_buffer, plane);
}
#endif
void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer,
int* width, int* height) {
CHECK(width);
CHECK(height);
*width = pixel_buffer.width();
*height = pixel_buffer.height();
}
GlTexture GlCalculatorHelper::CreateDestinationTexture(int output_width,
int output_height,
GpuBufferFormat format) {
return impl_->CreateDestinationTexture(output_width, output_height, format);
}
GlContext& GlCalculatorHelper::GetGlContext() const {
return impl_->GetGlContext();
}
} // namespace mediapipe
mediapipe/gpu/gl_calculator_helper.h
+236
View File
@@ -0,0 +1,236 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
#define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/packet_set.h"
#include "mediapipe/framework/packet_type.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/graph_support.h"
#ifdef __APPLE__
#include <CoreVideo/CoreVideo.h>
#include "mediapipe/framework/ios/CFHolder.h"
#endif // __APPLE__
namespace mediapipe {
class GlCalculatorHelperImpl;
class GlTexture;
class GpuResources;
class GpuSharedData;
#ifdef __APPLE__
#if TARGET_OS_OSX
typedef CVOpenGLTextureRef CVTextureType;
#else
typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
#endif // __APPLE__
// TODO: remove this and Process below, or make Process available
// on Android.
typedef std::function<void(const GlTexture& src, const GlTexture& dst)>
RenderFunction;
// Helper class that manages OpenGL contexts and operations.
// Calculators that implement an image filter, taking one input stream of
// frames and producing one output stream of frame, should subclass
// GlSimpleCalculatorBase instead of using GlCalculatorHelper directly.
// Direct use of this class is recommended for calculators that do not fit
// that mold (e.g. calculators that combine two video streams).
class GlCalculatorHelper {
public:
GlCalculatorHelper();
~GlCalculatorHelper();
// Call Open from the Open method of a calculator to initialize the helper.
::mediapipe::Status Open(CalculatorContext* cc);
// Can be used to initialize the helper outside of a calculator. Useful for
// testing.
void InitializeForTest(GpuResources* gpu_resources);
void InitializeForTest(GpuSharedData* gpu_shared);
// This method can be called from GetContract to set up the needed GPU
// resources.
static ::mediapipe::Status UpdateContract(CalculatorContract* cc);
// This method can be called from FillExpectations to set the correct types
// for the shared GL input side packet(s).
static ::mediapipe::Status SetupInputSidePackets(
PacketTypeSet* input_side_packets);
// Execute the provided function within the helper's GL context. On some
// platforms, this may be run on a different thread; however, this method
// will still wait for the function to finish executing before returning.
// The status result from the function is passed on to the caller.
::mediapipe::Status RunInGlContext(
std::function<::mediapipe::Status(void)> gl_func);
// Convenience version of RunInGlContext for arguments with a void result
// type. As with the ::mediapipe::Status version, this also waits for the
// function to finish executing before returning.
//
// Implementation note: we cannot use a std::function<void(void)> argument
// here, because that would break passing in a lambda that returns a status;
// e.g.:
// RunInGlContext([]() -> ::mediapipe::Status { ... });
//
// The reason is that std::function<void(...)> allows the implicit conversion
// of a callable with any result type, as long as the argument types match.
// As a result, the above lambda would be implicitly convertible to both
// std::function<::mediapipe::Status(void)> and std::function<void(void)>, and
// the invocation would be ambiguous.
//
// Therefore, instead of using std::function<void(void)>, we use a template
// that only accepts arguments with a void result type.
template <typename T, typename = typename std::enable_if<std::is_void<
typename std::result_of<T()>::type>::value>::type>
void RunInGlContext(T f) {
RunInGlContext([f] {
f();
return ::mediapipe::OkStatus();
}).IgnoreError();
}
// Use CreateSourceTexture and CreateDestinationTexture to set up textures
// for input and output frames. They are not just a convenience; on platforms
// where it is supported (iOS, for now) they take advantage of memory sharing
// between the CPU and GPU, avoiding memory copies.
// Creates a texture representing an input frame.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
#ifdef __APPLE__
// Creates a texture from a plane of a planar buffer.
// The plane index is zero-based. The number of planes depends on the
// internal format of the buffer.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer, int plane);
#endif
// Extracts GpuBuffer dimensions without creating a texture.
ABSL_DEPRECATED("Use width and height methods on GpuBuffer instead")
void GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, int* width,
int* height);
// Creates a texture representing an output frame.
// TODO: This should either return errors or a status.
GlTexture CreateDestinationTexture(
int output_width, int output_height,
GpuBufferFormat format = GpuBufferFormat::kBGRA32);
// The OpenGL name of the output framebuffer.
GLuint framebuffer() const;
// Binds the rendering framebuffer to a destination texture.
// TODO: do we need an unbind method too?
void BindFramebuffer(const GlTexture& dst);
GlContext& GetGlContext() const;
private:
std::unique_ptr<GlCalculatorHelperImpl> impl_;
};
// Represents an OpenGL texture.
class GlTexture {
public:
GlTexture() {}
GlTexture(GLuint name, int width, int height);
~GlTexture() { Release(); }
int width() const { return width_; }
int height() const { return height_; }
GLenum target() const { return target_; }
GLuint name() const { return name_; }
// Returns a buffer that can be sent to another calculator.
// Can be used with GpuBuffer or ImageFrame.
template <typename T>
std::unique_ptr<T> GetFrame() const;
// Releases texture memory
void Release();
private:
friend class GlCalculatorHelperImpl;
GlCalculatorHelperImpl* helper_impl_ = nullptr;
GLuint name_ = 0;
int width_ = 0;
int height_ = 0;
GLenum target_ = GL_TEXTURE_2D;
#ifdef MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
// For CVPixelBufferRef-based rendering
CFHolder<CVTextureType> cv_texture_;
#else
// Keeps track of whether this texture mapping is for read access, so that
// we can create a consumer sync point when releasing it.
bool for_reading_ = false;
#endif
GpuBuffer gpu_buffer_;
int plane_ = 0;
};
// Returns the entry with the given tag if the collection uses tags, with the
// given index otherwise. Can be used with PacketTypeSet*, PacketSet,
// OutputStreamSet, InputStreamSet, etc.
// It would be possible to have a single version of this if we could use
// non-const references. Unfortunately, they are not allowed by the style guide.
// The const-reference version cannot work with PacketTypeSet because the Set
// method is (naturally) non-const. We could add a const_cast, but I figure
// it is better to keep const-safety and accept having two versions of the
// same thing.
template <typename T>
auto TagOrIndex(const T& collection, const std::string& tag, int index)
-> decltype(collection.Tag(tag)) {
return collection.UsesTags() ? collection.Tag(tag) : collection.Index(index);
}
template <typename T>
auto TagOrIndex(T* collection, const std::string& tag, int index)
-> decltype(collection->Tag(tag)) {
return collection->UsesTags() ? collection->Tag(tag)
: collection->Index(index);
}
template <typename T>
bool HasTagOrIndex(const T& collection, const std::string& tag, int index) {
return collection.UsesTags() ? collection.HasTag(tag)
: index < collection.NumEntries();
}
template <typename T>
bool HasTagOrIndex(T* collection, const std::string& tag, int index) {
return collection->UsesTags() ? collection->HasTag(tag)
: index < collection->NumEntries();
}
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
mediapipe/gpu/gl_calculator_helper_impl.h
+90
View File
@@ -0,0 +1,90 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_
#define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#ifdef __OBJC__
#import <AVFoundation/AVFoundation.h>
#import <Foundation/Foundation.h>
#endif // __OBJC__
#ifdef __ANDROID__
#include "mediapipe/gpu/gl_texture_buffer_pool.h"
#endif
namespace mediapipe {
// This class implements the GlCalculatorHelper for iOS and Android.
// See GlCalculatorHelper for details on these methods.
class GlCalculatorHelperImpl {
public:
explicit GlCalculatorHelperImpl(CalculatorContext* cc,
GpuResources* gpu_resources);
~GlCalculatorHelperImpl();
::mediapipe::Status RunInGlContext(
std::function<::mediapipe::Status(void)> gl_func,
CalculatorContext* calculator_context);
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
// Note: multi-plane support is currently only available on iOS.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer, int plane);
// Creates a framebuffer and returns the texture that it is bound to.
GlTexture CreateDestinationTexture(int output_width, int output_height,
GpuBufferFormat format);
GLuint framebuffer() const { return framebuffer_; }
void BindFramebuffer(const GlTexture& dst);
#ifdef __APPLE__
GlVersion GetGlVersion();
#endif
GlContext& GetGlContext() const;
// For internal use.
void ReadTexture(const GlTexture& texture, void* output, size_t size);
private:
// Makes a GpuBuffer accessible as a texture in the GL context.
GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, int plane);
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTexture MapGlTextureBuffer(const GlTextureBufferSharedPtr& texture_buffer);
GlTextureBufferSharedPtr MakeGlTextureBuffer(const ImageFrame& image_frame);
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
// Sets default texture filtering parameters.
void SetStandardTextureParams(GLenum target);
// Create the framebuffer for rendering.
void CreateFramebuffer();
std::shared_ptr<GlContext> gl_context_;
GLuint framebuffer_ = 0;
GpuResources& gpu_resources_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_
mediapipe/gpu/gl_calculator_helper_impl_android.cc
+93
View File
@@ -0,0 +1,93 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
namespace mediapipe {
// TODO: move this method to GlCalculatorHelper, then we can
// access its framebuffer instead of requiring that one is already set.
template <>
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
auto output =
absl::make_unique<ImageFrame>(ImageFormat::SRGBA, width_, height_,
ImageFrame::kGlDefaultAlignmentBoundary);
CHECK(helper_impl_);
helper_impl_->ReadTexture(*this, output->MutablePixelData(),
output->PixelDataSize());
return output;
}
template <>
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
CHECK(gpu_buffer_);
// Inform the GlTextureBuffer that we have produced new content, and create
// a producer sync point.
gpu_buffer_.GetGlTextureBufferSharedPtr()->Updated(
helper_impl_->GetGlContext().CreateSyncToken());
#ifdef __ANDROID__
// On (some?) Android devices, the texture may need to be explicitly
// detached from the current framebuffer.
// TODO: is this necessary even with the unbind in BindFramebuffer?
// It is not clear if this affected other contexts too, but let's keep it
// while in doubt.
GLint type = GL_NONE;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE,
&type);
if (type == GL_TEXTURE) {
GLint color_attachment = 0;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment);
if (color_attachment == name_) {
glBindFramebuffer(GL_FRAMEBUFFER, 0);
}
}
// Some Android drivers log a GL_INVALID_ENUM error after the first
// glGetFramebufferAttachmentParameteriv call if there is no bound object,
// even though it should be ok to ask for the type and get back GL_NONE.
// Let's just ignore any pending errors here.
GLenum error;
while ((error = glGetError()) != GL_NO_ERROR) {
}
#endif // __ANDROID__
return absl::make_unique<GpuBuffer>(gpu_buffer_);
}
void GlTexture::Release() {
if (for_reading_ && gpu_buffer_) {
// Inform the GlTextureBuffer that we have finished accessing its contents,
// and create a consumer sync point.
gpu_buffer_.GetGlTextureBufferSharedPtr()->DidRead(
helper_impl_->GetGlContext().CreateSyncToken());
}
helper_impl_ = nullptr;
for_reading_ = false;
gpu_buffer_ = nullptr;
plane_ = 0;
name_ = 0;
width_ = 0;
height_ = 0;
}
} // namespace mediapipe
mediapipe/gpu/gl_calculator_helper_impl_common.cc
+208
View File
@@ -0,0 +1,208 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
namespace mediapipe {
GlCalculatorHelperImpl::GlCalculatorHelperImpl(CalculatorContext* cc,
GpuResources* gpu_resources)
: gpu_resources_(*gpu_resources) {
gl_context_ = gpu_resources_.gl_context(cc);
}
GlCalculatorHelperImpl::~GlCalculatorHelperImpl() {
RunInGlContext(
[this] {
if (framebuffer_) {
glDeleteFramebuffers(1, &framebuffer_);
framebuffer_ = 0;
}
return ::mediapipe::OkStatus();
},
/*calculator_context=*/nullptr)
.IgnoreError();
}
GlContext& GlCalculatorHelperImpl::GetGlContext() const { return *gl_context_; }
::mediapipe::Status GlCalculatorHelperImpl::RunInGlContext(
std::function<::mediapipe::Status(void)> gl_func,
CalculatorContext* calculator_context) {
if (calculator_context) {
return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(),
calculator_context->InputTimestamp());
} else {
return gl_context_->Run(std::move(gl_func));
}
}
void GlCalculatorHelperImpl::CreateFramebuffer() {
// Our framebuffer will have a color attachment but no depth attachment,
// so it's important that the depth test be off. It is disabled by default,
// but we wanted to be explicit.
// TODO: move this to glBindFramebuffer?
glDisable(GL_DEPTH_TEST);
glGenFramebuffers(1, &framebuffer_);
}
void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) {
#ifdef __ANDROID__
// On (some?) Android devices, attaching a new texture to the frame buffer
// does not seem to detach the old one. As a result, using that texture
// for texturing can produce incorrect output. See b/32091368 for details.
// To fix this, we have to call either glBindFramebuffer with a FBO id of 0
// or glFramebufferTexture2D with a texture ID of 0.
glBindFramebuffer(GL_FRAMEBUFFER, 0);
#endif
if (!framebuffer_) {
CreateFramebuffer();
}
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
glViewport(0, 0, dst.width(), dst.height());
glActiveTexture(GL_TEXTURE0);
glBindTexture(dst.target(), dst.name());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(),
dst.name(), 0);
#ifndef NDEBUG
GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (status != GL_FRAMEBUFFER_COMPLETE) {
VLOG(2) << "incomplete framebuffer: " << status;
}
#endif
}
void GlCalculatorHelperImpl::SetStandardTextureParams(GLenum target) {
glTexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const ImageFrame& image_frame) {
GlTexture texture = MapGlTextureBuffer(MakeGlTextureBuffer(image_frame));
texture.for_reading_ = true;
return texture;
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer) {
GlTexture texture = MapGpuBuffer(gpu_buffer, 0);
texture.for_reading_ = true;
return texture;
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer, int plane) {
GlTexture texture = MapGpuBuffer(gpu_buffer, plane);
texture.for_reading_ = true;
return texture;
}
GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
int plane) {
CHECK_EQ(plane, 0);
return MapGlTextureBuffer(gpu_buffer.GetGlTextureBufferSharedPtr());
}
GlTexture GlCalculatorHelperImpl::MapGlTextureBuffer(
const GlTextureBufferSharedPtr& texture_buffer) {
// Insert wait call to sync with the producer.
texture_buffer->WaitOnGpu();
GlTexture texture;
texture.helper_impl_ = this;
texture.gpu_buffer_ = GpuBuffer(texture_buffer);
texture.plane_ = 0;
texture.width_ = texture_buffer->width_;
texture.height_ = texture_buffer->height_;
texture.target_ = texture_buffer->target_;
texture.name_ = texture_buffer->name_;
// TODO: do the params need to be reset here??
glBindTexture(texture.target(), texture.name());
SetStandardTextureParams(texture.target());
glBindTexture(texture.target(), 0);
return texture;
}
GlTextureBufferSharedPtr GlCalculatorHelperImpl::MakeGlTextureBuffer(
const ImageFrame& image_frame) {
CHECK(gl_context_->IsCurrent());
auto buffer = GlTextureBuffer::Create(
image_frame.Width(), image_frame.Height(),
GpuBufferFormatForImageFormat(image_frame.Format()),
image_frame.PixelData());
glBindTexture(GL_TEXTURE_2D, buffer->name_);
SetStandardTextureParams(buffer->target_);
glBindTexture(GL_TEXTURE_2D, 0);
return buffer;
}
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
int width, int height, GpuBufferFormat format) {
if (!framebuffer_) {
CreateFramebuffer();
}
GpuBuffer buffer =
gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format);
GlTexture texture = MapGpuBuffer(buffer, 0);
return texture;
}
void GlCalculatorHelperImpl::ReadTexture(const GlTexture& texture, void* output,
size_t size) {
CHECK_GE(size, texture.width_ * texture.height_ * 4);
GLint current_fbo;
glGetIntegerv(GL_FRAMEBUFFER_BINDING, &current_fbo);
CHECK_NE(current_fbo, 0);
GLint color_attachment_name;
glGetFramebufferAttachmentParameteriv(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME,
&color_attachment_name);
if (color_attachment_name != texture.name_) {
// Save the viewport. Note that we assume that the color attachment is a
// GL_TEXTURE_2D texture.
GLint viewport[4];
glGetIntegerv(GL_VIEWPORT, viewport);
// Set the data from GLTexture object.
glViewport(0, 0, texture.width_, texture.height_);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
texture.target_, texture.name_, 0);
glReadPixels(0, 0, texture.width_, texture.height_, GL_RGBA,
GL_UNSIGNED_BYTE, output);
// Restore from the saved viewport and color attachment name.
glViewport(viewport[0], viewport[1], viewport[2], viewport[3]);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D,
color_attachment_name, 0);
} else {
glReadPixels(0, 0, texture.width_, texture.height_, GL_RGBA,
GL_UNSIGNED_BYTE, output);
}
}
} // namespace mediapipe
mediapipe/gpu/gl_context.cc
+655
View File
@@ -0,0 +1,655 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_context.h"
#include <sys/types.h>
#include <cmath>
#include <memory>
#include <string>
#include <utility>
#include "absl/base/dynamic_annotations.h"
#include "absl/memory/memory.h"
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_builder.h"
#include "mediapipe/gpu/gl_context_internal.h"
#ifndef __EMSCRIPTEN__
#include "absl/debugging/leak_check.h"
#include "mediapipe/gpu/gl_thread_collector.h"
#endif
#ifndef GL_MAJOR_VERSION
#define GL_MAJOR_VERSION 0x821B
#endif
#ifndef GL_MINOR_VERSION
#define GL_MINOR_VERSION 0x821C
#endif
namespace mediapipe {
static void SetThreadName(const char* name) {
#if defined(__GLIBC_PREREQ)
#define LINUX_STYLE_SETNAME_NP __GLIBC_PREREQ(2, 12)
#elif defined(__BIONIC__)
#define LINUX_STYLE_SETNAME_NP 1
#endif // __GLIBC_PREREQ
#if LINUX_STYLE_SETNAME_NP
char thread_name[16]; // Linux requires names (with nul) fit in 16 chars
strncpy(thread_name, name, sizeof(thread_name));
thread_name[sizeof(thread_name) - 1] = '\0';
int res = pthread_setname_np(pthread_self(), thread_name);
if (res != 0) {
LOG_FIRST_N(INFO, 1) << "Can't set pthread names: name: \"" << name
<< "\"; error: " << res;
}
#elif __APPLE__
pthread_setname_np(name);
#endif
ANNOTATE_THREAD_NAME(name);
}
GlContext::DedicatedThread::DedicatedThread() {
CHECK_EQ(pthread_create(&gl_thread_id_, nullptr, ThreadBody, this), 0);
}
GlContext::DedicatedThread::~DedicatedThread() {
if (IsCurrentThread()) {
CHECK(self_destruct_);
CHECK_EQ(pthread_detach(gl_thread_id_), 0);
} else {
// Give an invalid job to signal termination.
PutJob({});
CHECK_EQ(pthread_join(gl_thread_id_, nullptr), 0);
}
}
void GlContext::DedicatedThread::SelfDestruct() {
self_destruct_ = true;
// Give an invalid job to signal termination.
PutJob({});
}
GlContext::DedicatedThread::Job GlContext::DedicatedThread::GetJob() {
absl::MutexLock lock(&mutex_);
while (jobs_.empty()) {
has_jobs_cv_.Wait(&mutex_);
}
Job job = std::move(jobs_.front());
jobs_.pop_front();
return job;
}
void GlContext::DedicatedThread::PutJob(Job job) {
absl::MutexLock lock(&mutex_);
jobs_.push_back(std::move(job));
has_jobs_cv_.SignalAll();
}
void* GlContext::DedicatedThread::ThreadBody(void* instance) {
DedicatedThread* thread = static_cast<DedicatedThread*>(instance);
thread->ThreadBody();
return nullptr;
}
#ifdef __APPLE__
#define AUTORELEASEPOOL @autoreleasepool
#else
#define AUTORELEASEPOOL
#endif // __APPLE__
void GlContext::DedicatedThread::ThreadBody() {
SetThreadName("mediapipe_gl_runner");
#ifndef __EMSCRIPTEN__
GlThreadCollector::ThreadStarting();
#endif
// The dedicated GL thread is not meant to be used on Apple platforms, but
// in case it is, the use of an autorelease pool here will reap each task's
// temporary allocations.
while (true) AUTORELEASEPOOL {
Job job = GetJob();
// Lack of a job means termination. Or vice versa.
if (!job) {
break;
}
job();
}
if (self_destruct_) {
delete this;
}
#ifndef __EMSCRIPTEN__
GlThreadCollector::ThreadEnding();
#endif
}
::mediapipe::Status GlContext::DedicatedThread::Run(GlStatusFunction gl_func) {
// Neither ENDO_SCOPE nor ENDO_TASK seem to work here.
if (IsCurrentThread()) {
return gl_func();
}
bool done = false; // Guarded by mutex_ after initialization.
::mediapipe::Status status;
PutJob([this, gl_func, &done, &status]() {
status = gl_func();
absl::MutexLock lock(&mutex_);
done = true;
gl_job_done_cv_.SignalAll();
});
absl::MutexLock lock(&mutex_);
while (!done) {
gl_job_done_cv_.Wait(&mutex_);
}
return status;
}
void GlContext::DedicatedThread::RunWithoutWaiting(GlVoidFunction gl_func) {
// Note: this is invoked by GlContextExecutor. To avoid starvation of
// non-calculator tasks in the presence of GL source calculators, calculator
// tasks must always be scheduled as new tasks, or another solution needs to
// be set up to avoid starvation. See b/78522434.
CHECK(gl_func);
PutJob(std::move(gl_func));
}
bool GlContext::DedicatedThread::IsCurrentThread() {
return pthread_equal(gl_thread_id_, pthread_self());
}
bool GlContext::ParseGlVersion(absl::string_view version_string, GLint* major,
GLint* minor) {
size_t pos = version_string.find('.');
if (pos == absl::string_view::npos || pos < 1) {
return false;
}
// GL_VERSION is supposed to start with the version number; see, e.g.,
// https://www.khronos.org/registry/OpenGL-Refpages/es3/html/glGetString.xhtml
// https://www.khronos.org/opengl/wiki/OpenGL_Context#OpenGL_version_number
// However, in rare cases one will encounter non-conforming configurations
// that have some prefix before the number. To deal with that, we walk
// backwards from the dot.
size_t start = pos - 1;
while (start > 0 && isdigit(version_string[start - 1])) --start;
if (!absl::SimpleAtoi(version_string.substr(start, (pos - start)), major)) {
return false;
}
auto rest = version_string.substr(pos + 1);
pos = rest.find(' ');
size_t pos2 = rest.find('.');
if (pos == absl::string_view::npos ||
(pos2 != absl::string_view::npos && pos2 < pos)) {
pos = pos2;
}
if (!absl::SimpleAtoi(rest.substr(0, pos), minor)) {
return false;
}
return true;
}
::mediapipe::Status GlContext::FinishInitialization(bool create_thread) {
if (create_thread) {
thread_ = absl::make_unique<GlContext::DedicatedThread>();
RETURN_IF_ERROR(thread_->Run([this] { return EnterContext(nullptr); }));
}
return Run([this]() -> ::mediapipe::Status {
absl::string_view version_string(
reinterpret_cast<const char*>(glGetString(GL_VERSION)));
// Let's try getting the numeric version if possible.
glGetIntegerv(GL_MAJOR_VERSION, &gl_major_version_);
GLenum err = glGetError();
if (err == GL_NO_ERROR) {
glGetIntegerv(GL_MINOR_VERSION, &gl_minor_version_);
} else {
// GL_MAJOR_VERSION is not supported on GL versions below 3. We have to
// parse the version std::string.
if (!ParseGlVersion(version_string, &gl_major_version_,
&gl_minor_version_)) {
LOG(WARNING) << "invalid GL_VERSION format: '" << version_string
<< "'; assuming 2.0";
gl_major_version_ = 2;
gl_minor_version_ = 0;
}
}
LOG(INFO) << "GL version: " << gl_major_version_ << "." << gl_minor_version_
<< " (" << glGetString(GL_VERSION) << ")";
return ::mediapipe::OkStatus();
});
}
GlContext::GlContext() {}
GlContext::~GlContext() {
// Note: on Apple platforms, this object contains Objective-C objects.
// The destructor will release them, but ARC must be on.
#ifdef __OBJC__
#if !__has_feature(objc_arc)
#error This file must be built with ARC.
#endif
#endif // __OBJC__
if (thread_) {
auto status = thread_->Run([this] {
if (profiling_helper_) {
profiling_helper_->LogAllTimestamps();
}
return ExitContext(nullptr);
});
if (!status.ok()) {
LOG(ERROR) << "Failed to deactivate context on thread: " << status;
}
if (thread_->IsCurrentThread()) {
thread_.release()->SelfDestruct();
}
}
DestroyContext();
}
void GlContext::SetProfilingContext(
std::shared_ptr<mediapipe::ProfilingContext> profiling_context) {
// Create the GlProfilingHelper if it is uninitialized.
if (!profiling_helper_ && profiling_context) {
profiling_helper_ = profiling_context->CreateGlProfilingHelper();
}
}
::mediapipe::Status GlContext::Run(GlStatusFunction gl_func, int node_id,
Timestamp input_timestamp) {
::mediapipe::Status status;
if (thread_) {
bool had_gl_errors = false;
status = thread_->Run(
[this, gl_func, node_id, &input_timestamp, &had_gl_errors] {
if (profiling_helper_) {
profiling_helper_->MarkTimestamp(node_id, input_timestamp,
/*is_finish=*/false);
}
auto status = gl_func();
if (profiling_helper_) {
profiling_helper_->MarkTimestamp(node_id, input_timestamp,
/*is_finish=*/true);
}
had_gl_errors = CheckForGlErrors();
return status;
});
LogUncheckedGlErrors(had_gl_errors);
} else {
ContextBinding saved_context;
RETURN_IF_ERROR(EnterContext(&saved_context));
if (profiling_helper_) {
profiling_helper_->MarkTimestamp(node_id, input_timestamp,
/*is_finish=*/false);
}
status = gl_func();
if (profiling_helper_) {
profiling_helper_->MarkTimestamp(node_id, input_timestamp,
/*is_finish=*/true);
}
LogUncheckedGlErrors(CheckForGlErrors());
RETURN_IF_ERROR(ExitContext(&saved_context));
}
return status;
}
void GlContext::RunWithoutWaiting(GlVoidFunction gl_func) {
if (thread_) {
// Add ref to keep the context alive while the task is executing.
auto context = shared_from_this();
thread_->RunWithoutWaiting([this, context, gl_func] {
gl_func();
LogUncheckedGlErrors(CheckForGlErrors());
});
} else {
// TODO: queue up task instead.
ContextBinding saved_context;
auto status = EnterContext(&saved_context);
if (!status.ok()) {
LOG(ERROR) << "Failed to enter context: " << status;
return;
}
gl_func();
LogUncheckedGlErrors(CheckForGlErrors());
status = ExitContext(&saved_context);
if (!status.ok()) {
LOG(ERROR) << "Failed to exit context: " << status;
}
}
}
std::weak_ptr<GlContext>& GlContext::CurrentContext() {
// Workaround for b/67878799.
#ifndef __EMSCRIPTEN__
absl::LeakCheckDisabler disable_leak_check;
#endif
ABSL_CONST_INIT thread_local std::weak_ptr<GlContext> current_context;
return current_context;
}
::mediapipe::Status GlContext::SwitchContext(ContextBinding* saved_context,
const ContextBinding& new_context)
NO_THREAD_SAFETY_ANALYSIS {
std::shared_ptr<GlContext> old_context_obj = CurrentContext().lock();
std::shared_ptr<GlContext> new_context_obj =
new_context.context_object.lock();
if (saved_context) {
saved_context->context_object = old_context_obj;
GetCurrentContextBinding(saved_context);
}
// Check that the context object is consistent with the native context.
if (old_context_obj && saved_context) {
DCHECK(old_context_obj->context_ == saved_context->context);
}
if (new_context_obj) {
DCHECK(new_context_obj->context_ == new_context.context);
}
if (new_context_obj && (old_context_obj == new_context_obj)) {
return ::mediapipe::OkStatus();
}
if (old_context_obj) {
// 1. Even if we cannot restore the new context, we want to get out of the
// old one (we may be deliberately trying to exit it).
// 2. We need to unset the old context before we unlock the old mutex,
// Therefore, we first unset the old one before setting the new one.
RETURN_IF_ERROR(SetCurrentContextBinding({}));
old_context_obj->context_use_mutex_.Unlock();
CurrentContext().reset();
}
if (new_context_obj) {
new_context_obj->context_use_mutex_.Lock();
auto status = SetCurrentContextBinding(new_context);
if (status.ok()) {
CurrentContext() = new_context_obj;
} else {
new_context_obj->context_use_mutex_.Unlock();
}
return status;
} else {
return SetCurrentContextBinding(new_context);
}
}
::mediapipe::Status GlContext::EnterContext(ContextBinding* saved_context) {
DCHECK(HasContext());
return SwitchContext(saved_context, ThisContextBinding());
}
::mediapipe::Status GlContext::ExitContext(
const ContextBinding* saved_context) {
ContextBinding no_context;
if (!saved_context) {
saved_context = &no_context;
}
return SwitchContext(nullptr, *saved_context);
}
std::shared_ptr<GlContext> GlContext::GetCurrent() {
return CurrentContext().lock();
}
void GlContext::GlFinishCalled() {
absl::MutexLock lock(&mutex_);
++gl_finish_count_;
wait_for_gl_finish_cv_.SignalAll();
}
class GlFinishSyncPoint : public GlSyncPoint {
public:
explicit GlFinishSyncPoint(const std::shared_ptr<GlContext>& gl_context)
: GlSyncPoint(gl_context),
gl_finish_count_(gl_context_->gl_finish_count()) {}
void Wait() override {
gl_context_->WaitForGlFinishCountPast(gl_finish_count_);
}
bool IsReady() override {
return gl_context_->gl_finish_count() > gl_finish_count_;
}
private:
// Number of glFinish calls done before the creation of this token.
int64_t gl_finish_count_ = -1;
};
class GlFenceSyncPoint : public GlSyncPoint {
public:
explicit GlFenceSyncPoint(const std::shared_ptr<GlContext>& gl_context)
: GlSyncPoint(gl_context) {
gl_context_->Run([this] {
sync_ = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
glFlush();
});
}
~GlFenceSyncPoint() {
if (sync_) {
GLsync sync = sync_;
gl_context_->RunWithoutWaiting([sync] { glDeleteSync(sync); });
}
}
GlFenceSyncPoint(const GlFenceSyncPoint&) = delete;
GlFenceSyncPoint& operator=(const GlFenceSyncPoint&) = delete;
void Wait() override {
if (!sync_) return;
gl_context_->Run([this] {
GLenum result =
glClientWaitSync(sync_, 0, std::numeric_limits<uint64_t>::max());
if (result == GL_ALREADY_SIGNALED || result == GL_CONDITION_SATISFIED) {
glDeleteSync(sync_);
sync_ = nullptr;
}
// TODO: do something if the wait fails?
});
}
void WaitOnGpu() override {
if (!sync_) return;
// TODO: do not wait if we are already on the same context?
glWaitSync(sync_, 0, GL_TIMEOUT_IGNORED);
}
bool IsReady() override {
if (!sync_) return true;
bool ready = false;
// TODO: we should not block on the original context if possible.
gl_context_->Run([this, &ready] {
GLenum result = glClientWaitSync(sync_, 0, 0);
if (result == GL_ALREADY_SIGNALED || result == GL_CONDITION_SATISFIED) {
glDeleteSync(sync_);
sync_ = nullptr;
ready = true;
}
});
return ready;
}
private:
GLsync sync_;
};
void GlMultiSyncPoint::Add(std::shared_ptr<GlSyncPoint> new_sync) {
for (auto& sync : syncs_) {
if (&sync->GetContext() == &new_sync->GetContext()) {
sync = std::move(new_sync);
return;
}
}
syncs_.emplace_back(std::move(new_sync));
}
void GlMultiSyncPoint::Wait() {
for (auto& sync : syncs_) {
sync->Wait();
}
// At this point all the syncs have been reached, so clear them out.
syncs_.clear();
}
void GlMultiSyncPoint::WaitOnGpu() {
for (auto& sync : syncs_) {
sync->WaitOnGpu();
}
// TODO: when do we clear out these syncs?
}
bool GlMultiSyncPoint::IsReady() {
syncs_.erase(
std::remove_if(syncs_.begin(), syncs_.end(),
std::bind(&GlSyncPoint::IsReady, std::placeholders::_1)),
syncs_.end());
return syncs_.empty();
}
// Set this to 1 to disable syncing. This can be used to verify that a test
// correctly detects sync issues.
#define MEDIAPIPE_DISABLE_GL_SYNC_FOR_DEBUG 0
#if MEDIAPIPE_DISABLE_GL_SYNC_FOR_DEBUG
class GlNopSyncPoint : public GlSyncPoint {
public:
explicit GlNopSyncPoint(const std::shared_ptr<GlContext>& gl_context)
: GlSyncPoint(gl_context) {}
void Wait() override {}
bool IsReady() override { return true; }
};
#endif
std::shared_ptr<GlSyncPoint> GlContext::CreateSyncToken() {
std::shared_ptr<GlSyncPoint> token;
#if MEDIAPIPE_DISABLE_GL_SYNC_FOR_DEBUG
token.reset(new GlNopSyncPoint(shared_from_this()));
#else
if (SymbolAvailable(&glWaitSync)) {
token.reset(new GlFenceSyncPoint(shared_from_this()));
} else {
token.reset(new GlFinishSyncPoint(shared_from_this()));
}
#endif
return token;
}
std::shared_ptr<GlSyncPoint> GlContext::TestOnly_CreateSpecificSyncToken(
SyncTokenTypeForTest type) {
std::shared_ptr<GlSyncPoint> token;
switch (type) {
case SyncTokenTypeForTest::kGlFinish:
token.reset(new GlFinishSyncPoint(shared_from_this()));
return token;
}
return nullptr;
}
void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) {
if (gl_finish_count_ > count_to_pass) return;
auto finish_task = [this, count_to_pass]() {
// When a GlFinishSyncToken is created it takes the current finish count
// from the GlContext, and we must wait for gl_finish_count_ to pass it.
// Therefore, we need to do at most one more glFinish call. This DCHECK
// is used for documentation and sanity-checking purposes.
DCHECK(gl_finish_count_ >= count_to_pass);
if (gl_finish_count_ == count_to_pass) {
glFinish();
GlFinishCalled();
}
};
if (IsCurrent()) {
// If we are already on the current context, we cannot call
// RunWithoutWaiting, since that task will not run until this function
// returns. Instead, call it directly.
finish_task();
return;
}
// We do not schedule this action using Run because we don't necessarily
// want to wait for it to complete. If another job calls GlFinishCalled
// sooner, we are done.
RunWithoutWaiting(std::move(finish_task));
absl::MutexLock lock(&mutex_);
while (gl_finish_count_ <= count_to_pass) {
wait_for_gl_finish_cv_.Wait(&mutex_);
}
}
void GlContext::WaitSyncToken(const std::shared_ptr<GlSyncPoint>& token) {
CHECK(token);
token->Wait();
}
bool GlContext::SyncTokenIsReady(const std::shared_ptr<GlSyncPoint>& token) {
CHECK(token);
return token->IsReady();
}
bool GlContext::CheckForGlErrors() {
#if UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING
LOG_FIRST_N(WARNING, 1) << "MediaPipe OpenGL error checking is disabled";
return false;
#endif
if (!HasContext()) return false;
GLenum error;
bool had_error = false;
while ((error = glGetError()) != GL_NO_ERROR) {
had_error = true;
switch (error) {
case GL_INVALID_ENUM:
LOG(INFO) << "Found unchecked GL error: GL_INVALID_ENUM";
break;
case GL_INVALID_VALUE:
LOG(INFO) << "Found unchecked GL error: GL_INVALID_VALUE";
break;
case GL_INVALID_OPERATION:
LOG(INFO) << "Found unchecked GL error: GL_INVALID_OPERATION";
break;
case GL_INVALID_FRAMEBUFFER_OPERATION:
LOG(INFO)
<< "Found unchecked GL error: GL_INVALID_FRAMEBUFFER_OPERATION";
break;
case GL_OUT_OF_MEMORY:
LOG(INFO) << "Found unchecked GL error: GL_OUT_OF_MEMORY";
break;
default:
LOG(INFO) << "Found unchecked GL error: UNKNOWN ERROR";
break;
}
}
return had_error;
}
void GlContext::LogUncheckedGlErrors(bool had_gl_errors) {
if (had_gl_errors) {
// TODO: ideally we would print a backtrace here, or at least
// the name of the current calculator, to make it easier to find the
// culprit. In practice, getting a backtrace from Android without crashing
// is nearly impossible, so screw it. Just change this to LOG(FATAL) when
// you want to debug.
LOG(WARNING) << "Ignoring unchecked GL error.";
}
}
} // namespace mediapipe
mediapipe/gpu/gl_context.h
+388
View File
@@ -0,0 +1,388 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_CONTEXT_H_
#define MEDIAPIPE_GPU_GL_CONTEXT_H_
#include <pthread.h>
#include <atomic>
#include <functional>
#include <memory>
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/executor.h"
#include "mediapipe/framework/mediapipe_profiling.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/framework/port/threadpool.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/gpu/gl_base.h"
#ifdef __APPLE__
#include <CoreVideo/CoreVideo.h>
#include "mediapipe/framework/ios/CFHolder.h"
#if TARGET_OS_OSX
#ifdef __OBJC__
@class NSOpenGLContext;
@class NSOpenGLPixelFormat;
#else
struct NSOpenGLContext;
struct NSOpenGLPixelFormat;
#endif // __OBJC___
#else
#ifdef __OBJC__
@class EAGLSharegroup;
@class EAGLContext;
#else
struct EAGLSharegroup;
struct EAGLContext;
#endif // __OBJC___
#endif // TARGET_OS_OSX
#else
#endif // __APPLE__
namespace mediapipe {
typedef std::function<void()> GlVoidFunction;
typedef std::function<::mediapipe::Status()> GlStatusFunction;
class GlContext;
// Generic interface for synchronizing access to a shared resource from a
// different context. This is an abstract class to keep users from
// depending on its contents. The implementation may differ depending on
// the capabilities of the GL context.
class GlSyncPoint {
public:
explicit GlSyncPoint(const std::shared_ptr<GlContext>& gl_context)
: gl_context_(gl_context) {}
virtual ~GlSyncPoint() {}
// Waits until the GPU has executed all commands up to the sync point.
// This blocks the CPU, and ensures the commands are complete from the
// point of view of all threads and contexts.
virtual void Wait() = 0;
// Ensures that the following commands on the current OpenGL context will
// not be executed until the sync point has been reached.
// This does not block the CPU, and only affects the current OpenGL context.
virtual void WaitOnGpu() { Wait(); }
// Returns whether the sync point has been reached. Does not block.
virtual bool IsReady() = 0;
const GlContext& GetContext() { return *gl_context_; }
protected:
std::shared_ptr<GlContext> gl_context_;
};
// Combines sync points for multiple contexts.
class GlMultiSyncPoint : public GlSyncPoint {
public:
GlMultiSyncPoint() : GlSyncPoint(nullptr) {}
// Adds a new sync to the multisync.
// If we already have a sync from the same context, overwrite it.
// Commands on the same context are serialized, and we only care about
// when the last one is done.
void Add(std::shared_ptr<GlSyncPoint> new_sync);
void Wait() override;
void WaitOnGpu() override;
bool IsReady() override;
private:
std::vector<std::shared_ptr<GlSyncPoint>> syncs_;
};
// TODO: remove.
typedef std::shared_ptr<GlSyncPoint> GlSyncToken;
#if defined(__EMSCRIPTEN__)
typedef EMSCRIPTEN_WEBGL_CONTEXT_HANDLE PlatformGlContext;
constexpr PlatformGlContext kPlatformGlContextNone = 0;
#elif HAS_EGL
typedef EGLContext PlatformGlContext;
constexpr PlatformGlContext kPlatformGlContextNone = EGL_NO_CONTEXT;
#elif HAS_EAGL
typedef EAGLContext* PlatformGlContext;
constexpr PlatformGlContext kPlatformGlContextNone = nil;
#elif HAS_NSGL
typedef NSOpenGLContext* PlatformGlContext;
constexpr PlatformGlContext kPlatformGlContextNone = nil;
#endif // defined(__EMSCRIPTEN__)
// This class provides a common API for creating and managing GL contexts.
//
// It handles the following responsibilities:
// - Providing a cross-platform interface over platform-specific APIs like EGL
// and EAGL.
// - Managing the interaction between threads and GL contexts.
// - Managing synchronization between different GL contexts.
//
// See go/mediapipe-gl-context for details.
class GlContext : public std::enable_shared_from_this<GlContext> {
public:
using StatusOrGlContext = ::mediapipe::StatusOr<std::shared_ptr<GlContext>>;
// Creates a GlContext.
//
// The first argument (which can be a GlContext, or a platform-specific type)
// indicates a context with which to share resources (e.g. textures).
// Resources will be shared amongst all contexts linked in this way. You can
// pass null if sharing is not desired.
//
// If create_thread is true, the context will create a thread and run all
// OpenGL tasks on it.
static StatusOrGlContext Create(std::nullptr_t nullp, bool create_thread);
static StatusOrGlContext Create(const GlContext& share_context,
bool create_thread);
static StatusOrGlContext Create(PlatformGlContext share_context,
bool create_thread);
#if HAS_EAGL
static StatusOrGlContext Create(EAGLSharegroup* sharegroup,
bool create_thread);
#endif // HAS_EAGL
// Returns the GlContext that is current on this thread. May return nullptr.
static std::shared_ptr<GlContext> GetCurrent();
GlContext(const GlContext&) = delete;
GlContext& operator=(const GlContext&) = delete;
~GlContext();
// Initializes this GlContext with the graph tracing and profiling interface.
// Also initializes the GlProfilingHelper object for this GlContext if the
// GlProfilingHelper is uninitialized. This ensures that the GlProfilingHelper
// is unique to and only initialized once per GlContext object.
void SetProfilingContext(
std::shared_ptr<mediapipe::ProfilingContext> profiling_context);
// Executes a function in the GL context. Waits for the
// function's execution to be complete before returning to the caller.
::mediapipe::Status Run(GlStatusFunction gl_func, int node_id = -1,
Timestamp input_timestamp = Timestamp::Unset());
// Like Run, but does not wait.
void RunWithoutWaiting(GlVoidFunction gl_func);
// Returns a synchronization token.
// This should not be called outside of the GlContext thread.
std::shared_ptr<GlSyncPoint> CreateSyncToken();
// If another part of the framework calls glFinish, it should call this
// method to let the context know that it has done so. The context can use
// that information to avoid inserting additional glFinish calls in some
// cases.
void GlFinishCalled();
// Ensures that the changes to shared resources covered by the token are
// visible in the current context.
// This should only be called outside a job.
void WaitSyncToken(const std::shared_ptr<GlSyncPoint>& token);
// Checks whether the token's sync point has been reached. Returns true
// iff WaitSyncToken would not have to wait.
// This is thread-safe.
bool SyncTokenIsReady(const std::shared_ptr<GlSyncPoint>& token);
#if defined(__EMSCRIPTEN__)
// Returns the EMSCRIPTEN_WEBGL_CONTEXT_HANDLE for our context.
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE webgl_context() const { return context_; }
EmscriptenWebGLContextAttributes webgl_attributes() const { return attrs_; }
#elif HAS_EGL
// Returns the EGLDisplay used by our context.
EGLDisplay egl_display() const { return display_; }
// Returns the EGLConfig used to create our context.
EGLConfig egl_config() const { return config_; }
// Returns our EGLContext.
EGLContext egl_context() const { return context_; }
#elif HAS_EAGL
EAGLContext* eagl_context() const { return context_; }
CVOpenGLESTextureCacheRef cv_texture_cache() const { return *texture_cache_; }
#elif HAS_NSGL
NSOpenGLContext* nsgl_context() const { return context_; }
NSOpenGLPixelFormat* nsgl_pixel_format() const { return pixel_format_; }
CVOpenGLTextureCacheRef cv_texture_cache() const { return *texture_cache_; }
#endif // HAS_EGL
// Check if the context is current on this thread. Mainly for test purposes.
bool IsCurrent() const;
GLint gl_major_version() const { return gl_major_version_; }
GLint gl_minor_version() const { return gl_minor_version_; }
static bool ParseGlVersion(absl::string_view version_string, GLint* major,
GLint* minor);
int64_t gl_finish_count() { return gl_finish_count_; }
// Used by GlFinishSyncPoint. The count_to_pass cannot exceed the current
// gl_finish_count_ (but it can be equal).
void WaitForGlFinishCountPast(int64_t count_to_pass);
// Convenience version of Run for arguments with a void result type.
// Waits for the function to finish executing before returning.
//
// Implementation note: we cannot use a std::function<void(void)> argument
// here, because that would break passing in a lambda that returns a status;
// e.g.:
// RunInGlContext([]() -> ::mediapipe::Status { ... });
//
// The reason is that std::function<void(...)> allows the implicit conversion
// of a callable with any result type, as long as the argument types match.
// As a result, the above lambda would be implicitly convertible to both
// std::function<::mediapipe::Status(void)> and std::function<void(void)>, and
// the invocation would be ambiguous.
//
// Therefore, instead of using std::function<void(void)>, we use a template
// that only accepts arguments with a void result type.
template <typename T, typename = typename std::enable_if<std::is_void<
typename std::result_of<T()>::type>::value>::type>
void Run(T f) {
Run([f] {
f();
return ::mediapipe::OkStatus();
}).IgnoreError();
}
// These are used for testing specific SyncToken implementations. Do not use
// outside of tests.
enum class SyncTokenTypeForTest {
kGlFinish,
};
std::shared_ptr<GlSyncPoint> TestOnly_CreateSpecificSyncToken(
SyncTokenTypeForTest type);
private:
GlContext();
#if defined(__EMSCRIPTEN__)
::mediapipe::Status CreateContext(
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE share_context);
::mediapipe::Status CreateContextInternal(
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE share_context, int webgl_version);
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE context_ = 0;
EmscriptenWebGLContextAttributes attrs_;
#elif HAS_EGL
::mediapipe::Status CreateContext(EGLContext share_context);
::mediapipe::Status CreateContextInternal(EGLContext share_context,
int gl_version);
EGLDisplay display_ = EGL_NO_DISPLAY;
EGLConfig config_;
EGLSurface surface_ = EGL_NO_SURFACE;
EGLContext context_ = EGL_NO_CONTEXT;
#elif HAS_EAGL
::mediapipe::Status CreateContext(EAGLSharegroup* sharegroup);
EAGLContext* context_;
CFHolder<CVOpenGLESTextureCacheRef> texture_cache_;
#elif HAS_NSGL
::mediapipe::Status CreateContext(NSOpenGLContext* share_context);
NSOpenGLContext* context_;
NSOpenGLPixelFormat* pixel_format_;
CFHolder<CVOpenGLTextureCacheRef> texture_cache_;
#endif // defined(__EMSCRIPTEN__)
class DedicatedThread;
// A context binding represents the minimal set of information needed to make
// a context current on a thread. Its contents depend on the platform.
struct ContextBinding {
// The context_object is null if this binding refers to a context not
// managed by GlContext.
std::weak_ptr<GlContext> context_object;
#if defined(__EMSCRIPTEN__)
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE context = 0;
#elif HAS_EGL
EGLDisplay display = EGL_NO_DISPLAY;
EGLSurface draw_surface = EGL_NO_SURFACE;
EGLSurface read_surface = EGL_NO_SURFACE;
EGLContext context = EGL_NO_CONTEXT;
#elif HAS_EAGL
EAGLContext* context = nullptr;
#elif HAS_NSGL
NSOpenGLContext* context = nullptr;
#endif // HAS_EGL
};
::mediapipe::Status FinishInitialization(bool create_thread);
// This wraps a thread_local.
static std::weak_ptr<GlContext>& CurrentContext();
static ::mediapipe::Status SwitchContext(ContextBinding* old_context,
const ContextBinding& new_context);
::mediapipe::Status EnterContext(ContextBinding* previous_context);
::mediapipe::Status ExitContext(const ContextBinding* previous_context);
void DestroyContext();
bool HasContext() const;
bool CheckForGlErrors();
void LogUncheckedGlErrors(bool had_gl_errors);
// The following ContextBinding functions have platform-specific
// implementations.
// A binding that can be used to make this GlContext current.
ContextBinding ThisContextBinding();
// Fills in a ContextBinding with platform-specific information about which
// context is current on this thread.
static void GetCurrentContextBinding(ContextBinding* binding);
// Makes the context described by new_context current on this thread.
static ::mediapipe::Status SetCurrentContextBinding(
const ContextBinding& new_context);
// If not null, a dedicated thread used to execute tasks on this context.
// Used on Android due to expensive context switching on some configurations.
std::unique_ptr<DedicatedThread> thread_;
GLint gl_major_version_ = 0;
GLint gl_minor_version_ = 0;
// Number of glFinish calls completed on the GL thread.
// Changes should be guarded by mutex_. However, we use simple atomic
// loads for efficiency on the fast path.
std::atomic<int64_t> gl_finish_count_ = ATOMIC_VAR_INIT(0);
// This mutex is held by a thread while this GL context is current on that
// thread. Since it may be held for extended periods of time, it should not
// be used for other pieces of status.
absl::Mutex context_use_mutex_;
// This mutex is used to guard a few different members and condition
// variables. It should only be held for a short time.
absl::Mutex mutex_;
absl::CondVar wait_for_gl_finish_cv_ GUARDED_BY(mutex_);
std::unique_ptr<mediapipe::GlProfilingHelper> profiling_helper_ = nullptr;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_CONTEXT_H_
mediapipe/gpu/gl_context_eagl.cc
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <utility>
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_builder.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gl_context_internal.h"
#if HAS_EAGL
#if !__has_feature(objc_arc)
#error This file must be built with ARC.
#endif
namespace mediapipe {
GlContext::StatusOrGlContext GlContext::Create(std::nullptr_t nullp,
bool create_thread) {
return Create(static_cast<EAGLSharegroup*>(nil), create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(const GlContext& share_context,
bool create_thread) {
return Create(share_context.context_.sharegroup, create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(EAGLContext* share_context,
bool create_thread) {
return Create(share_context.sharegroup, create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(EAGLSharegroup* sharegroup,
bool create_thread) {
std::shared_ptr<GlContext> context(new GlContext());
RETURN_IF_ERROR(context->CreateContext(sharegroup));
RETURN_IF_ERROR(context->FinishInitialization(create_thread));
return std::move(context);
}
::mediapipe::Status GlContext::CreateContext(EAGLSharegroup* sharegroup) {
context_ = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES3
sharegroup:sharegroup];
if (context_) {
gl_major_version_ = 3;
} else {
context_ = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2
sharegroup:sharegroup];
gl_major_version_ = 2;
}
RET_CHECK(context_) << "Could not create an EAGLContext";
CVOpenGLESTextureCacheRef cache;
CVReturn err = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL,
context_, NULL, &cache);
RET_CHECK_EQ(err, kCVReturnSuccess)
<< "Error at CVOpenGLESTextureCacheCreate";
texture_cache_.adopt(cache);
return ::mediapipe::OkStatus();
}
void GlContext::DestroyContext() {}
GlContext::ContextBinding GlContext::ThisContextBinding() {
GlContext::ContextBinding result;
result.context_object = shared_from_this();
result.context = context_;
return result;
}
void GlContext::GetCurrentContextBinding(GlContext::ContextBinding* binding) {
binding->context = [EAGLContext currentContext];
}
::mediapipe::Status GlContext::SetCurrentContextBinding(
const ContextBinding& new_binding) {
BOOL success = [EAGLContext setCurrentContext:new_binding.context];
RET_CHECK(success) << "Cannot set OpenGL context";
return ::mediapipe::OkStatus();
}
bool GlContext::HasContext() const { return context_ != nil; }
bool GlContext::IsCurrent() const {
return HasContext() && ([EAGLContext currentContext] == context_);
}
} // namespace mediapipe
#endif // HAS_EAGL
mediapipe/gpu/gl_context_egl.cc
+250
View File
@@ -0,0 +1,250 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <utility>
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_builder.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gl_context_internal.h"
#ifndef EGL_OPENGL_ES3_BIT_KHR
#define EGL_OPENGL_ES3_BIT_KHR 0x00000040
#endif
#if HAS_EGL
namespace mediapipe {
static pthread_key_t egl_release_thread_key;
static pthread_once_t egl_release_key_once = PTHREAD_ONCE_INIT;
static void EglThreadExitCallback(void* key_value) {
eglMakeCurrent(EGL_NO_DISPLAY, EGL_NO_SURFACE, EGL_NO_SURFACE,
EGL_NO_CONTEXT);
eglReleaseThread();
}
// If a key has a destructor callback, and a thread has a non-NULL value for
// that key, then the destructor is called when the thread exits.
static void MakeEglReleaseThreadKey() {
int err = pthread_key_create(&egl_release_thread_key, EglThreadExitCallback);
if (err) {
LOG(ERROR) << "cannot create pthread key: " << err;
}
}
// This function can be called any number of times. For any thread on which it
// was called at least once, the EglThreadExitCallback will be called (once)
// when the thread exits.
static void EnsureEglThreadRelease() {
pthread_once(&egl_release_key_once, MakeEglReleaseThreadKey);
pthread_setspecific(egl_release_thread_key,
reinterpret_cast<void*>(0xDEADBEEF));
}
GlContext::StatusOrGlContext GlContext::Create(std::nullptr_t nullp,
bool create_thread) {
return Create(EGL_NO_CONTEXT, create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(const GlContext& share_context,
bool create_thread) {
return Create(share_context.context_, create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(EGLContext share_context,
bool create_thread) {
std::shared_ptr<GlContext> context(new GlContext());
RETURN_IF_ERROR(context->CreateContext(share_context));
RETURN_IF_ERROR(context->FinishInitialization(create_thread));
return std::move(context);
}
::mediapipe::Status GlContext::CreateContextInternal(
EGLContext external_context, int gl_version) {
CHECK(gl_version == 2 || gl_version == 3);
const EGLint config_attr[] = {
// clang-format off
EGL_RENDERABLE_TYPE, gl_version == 3 ? EGL_OPENGL_ES3_BIT_KHR
: EGL_OPENGL_ES2_BIT,
// Allow rendering to pixel buffers or directly to windows.
EGL_SURFACE_TYPE, EGL_PBUFFER_BIT | EGL_WINDOW_BIT,
EGL_RED_SIZE, 8,
EGL_GREEN_SIZE, 8,
EGL_BLUE_SIZE, 8,
EGL_ALPHA_SIZE, 8, // if you need the alpha channel
EGL_DEPTH_SIZE, 16, // if you need the depth buffer
EGL_NONE
// clang-format on
};
// TODO: improve config selection.
EGLint num_configs;
EGLBoolean success =
eglChooseConfig(display_, config_attr, &config_, 1, &num_configs);
if (!success) {
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "eglChooseConfig() returned error " << eglGetError();
}
const EGLint context_attr[] = {
// clang-format off
EGL_CONTEXT_CLIENT_VERSION, gl_version,
EGL_NONE
// clang-format on
};
context_ =
eglCreateContext(display_, config_, external_context, context_attr);
int error = eglGetError();
RET_CHECK(context_ != EGL_NO_CONTEXT)
<< "Could not create GLES " << gl_version << " context; "
<< "eglCreateContext() returned error " << error
<< (error == EGL_BAD_CONTEXT
? ": external context uses a different version of OpenGL"
: "");
// We can't always rely on GL_MAJOR_VERSION and GL_MINOR_VERSION, since
// GLES 2 does not have them, so let's set the major version here at least.
gl_major_version_ = gl_version;
return ::mediapipe::OkStatus();
}
::mediapipe::Status GlContext::CreateContext(EGLContext external_context) {
EGLint major = 0;
EGLint minor = 0;
display_ = eglGetDisplay(EGL_DEFAULT_DISPLAY);
RET_CHECK(display_ != EGL_NO_DISPLAY)
<< "eglGetDisplay() returned error " << eglGetError();
EGLBoolean success = eglInitialize(display_, &major, &minor);
RET_CHECK(success) << "Unable to initialize EGL";
LOG(INFO) << "Successfully initialized EGL. Major : " << major
<< " Minor: " << minor;
auto status = CreateContextInternal(external_context, 3);
if (!status.ok()) {
LOG(WARNING) << "Creating a context with OpenGL ES 3 failed: " << status;
LOG(WARNING) << "Fall back on OpenGL ES 2.";
status = CreateContextInternal(external_context, 2);
}
RETURN_IF_ERROR(status);
EGLint pbuffer_attr[] = {EGL_WIDTH, 1, EGL_HEIGHT, 1, EGL_NONE};
surface_ = eglCreatePbufferSurface(display_, config_, pbuffer_attr);
RET_CHECK(surface_ != EGL_NO_SURFACE)
<< "eglCreatePbufferSurface() returned error " << eglGetError();
return ::mediapipe::OkStatus();
}
void GlContext::DestroyContext() {
if (thread_) {
// Delete thread-local storage.
// TODO: in theory our EglThreadExitCallback should suffice for
// this; however, heapcheck still reports a leak without this call here
// when using SwiftShader.
// Perhaps heapcheck misses the thread destructors?
thread_
->Run([] {
eglReleaseThread();
return ::mediapipe::OkStatus();
})
.IgnoreError();
}
// Destroy the context and surface.
if (IsCurrent()) {
if (!eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE,
EGL_NO_CONTEXT)) {
LOG(ERROR) << "eglMakeCurrent() returned error " << eglGetError();
}
}
if (surface_ != EGL_NO_SURFACE) {
if (!eglDestroySurface(display_, surface_)) {
LOG(ERROR) << "eglDestroySurface() returned error " << eglGetError();
}
}
if (context_ != EGL_NO_CONTEXT) {
if (!eglDestroyContext(display_, context_)) {
LOG(ERROR) << "eglDestroyContext() returned error " << eglGetError();
}
context_ = EGL_NO_CONTEXT;
}
// Under standard EGL, eglTerminate will terminate the display connection
// for the entire process, no matter how many times eglInitialize has been
// called. So we do not want to terminate it here, in case someone else is
// using it.
// However, Android implements non-standard reference-counted semantics for
// eglInitialize/eglTerminate, so we should call it on that platform.
#ifdef __ANDROID__
// TODO: this is removed for now since it caused issues on
// YouTube. But in theory we _should_ be calling it. Needs more
// investigation.
// eglTerminate(display_);
#endif // __ANDROID__
}
GlContext::ContextBinding GlContext::ThisContextBinding() {
GlContext::ContextBinding result;
result.context_object = shared_from_this();
result.display = display_;
result.draw_surface = surface_;
result.read_surface = surface_;
result.context = context_;
return result;
}
void GlContext::GetCurrentContextBinding(GlContext::ContextBinding* binding) {
binding->display = eglGetCurrentDisplay();
binding->draw_surface = eglGetCurrentSurface(EGL_DRAW);
binding->read_surface = eglGetCurrentSurface(EGL_READ);
binding->context = eglGetCurrentContext();
}
::mediapipe::Status GlContext::SetCurrentContextBinding(
const ContextBinding& new_binding) {
EnsureEglThreadRelease();
EGLDisplay display = new_binding.display;
if (display == EGL_NO_DISPLAY) {
display = eglGetCurrentDisplay();
}
if (display == EGL_NO_DISPLAY) {
display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
}
EGLBoolean success =
eglMakeCurrent(display, new_binding.draw_surface,
new_binding.read_surface, new_binding.context);
RET_CHECK(success) << "eglMakeCurrent() returned error " << eglGetError();
return ::mediapipe::OkStatus();
}
bool GlContext::HasContext() const { return context_ != EGL_NO_CONTEXT; }
bool GlContext::IsCurrent() const {
return HasContext() && (eglGetCurrentContext() == context_);
}
} // namespace mediapipe
#endif // HAS_EGL
mediapipe/gpu/gl_context_internal.h
+66
View File
@@ -0,0 +1,66 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_CONTEXT_INTERNAL_H_
#define MEDIAPIPE_GPU_GL_CONTEXT_INTERNAL_H_
#include "absl/synchronization/mutex.h"
#ifdef __APPLE__
#if TARGET_OS_OSX
#import <AppKit/NSOpenGL.h>
#else
#import <OpenGLES/EAGL.h>
#endif // TARGET_OS_OSX
#endif // __APPLE__
#include "mediapipe/gpu/gl_context.h"
namespace mediapipe {
class GlContext::DedicatedThread {
public:
DedicatedThread();
~DedicatedThread();
DedicatedThread(const DedicatedThread&) = delete;
DedicatedThread& operator=(DedicatedThread) = delete;
::mediapipe::Status Run(GlStatusFunction gl_func);
void RunWithoutWaiting(GlVoidFunction gl_fund);
bool IsCurrentThread();
void SelfDestruct();
private:
static void* ThreadBody(void* instance);
void ThreadBody();
using Job = std::function<void(void)>;
Job GetJob();
void PutJob(Job job);
absl::Mutex mutex_;
// Used to wait for a job's completion.
absl::CondVar gl_job_done_cv_ GUARDED_BY(mutex_);
pthread_t gl_thread_id_;
std::deque<Job> jobs_ GUARDED_BY(mutex_);
absl::CondVar has_jobs_cv_ GUARDED_BY(mutex_);
bool self_destruct_ = false;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_CONTEXT_INTERNAL_H_
mediapipe/gpu/gl_context_nsgl.cc
+136
View File
@@ -0,0 +1,136 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <utility>
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_builder.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gl_context_internal.h"
#if HAS_NSGL
namespace mediapipe {
GlContext::StatusOrGlContext GlContext::Create(std::nullptr_t nullp,
bool create_thread) {
return Create(static_cast<NSOpenGLContext*>(nil), create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(const GlContext& share_context,
bool create_thread) {
return Create(share_context.context_, create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(NSOpenGLContext* share_context,
bool create_thread) {
std::shared_ptr<GlContext> context(new GlContext());
RETURN_IF_ERROR(context->CreateContext(share_context));
RETURN_IF_ERROR(context->FinishInitialization(create_thread));
return std::move(context);
}
::mediapipe::Status GlContext::CreateContext(NSOpenGLContext* share_context) {
// TODO: choose a better list?
NSOpenGLPixelFormatAttribute attrs[] = {NSOpenGLPFAAccelerated,
NSOpenGLPFAColorSize,
24,
NSOpenGLPFAAlphaSize,
8,
NSOpenGLPFADepthSize,
16,
0};
pixel_format_ = [[NSOpenGLPixelFormat alloc] initWithAttributes:attrs];
if (!pixel_format_) {
// On several Forge machines, the default config fails. For now let's do
// this.
LOG(WARNING)
<< "failed to create pixel format; trying without acceleration";
NSOpenGLPixelFormatAttribute attrs_no_accel[] = {NSOpenGLPFAColorSize,
24,
NSOpenGLPFAAlphaSize,
8,
NSOpenGLPFADepthSize,
16,
0};
pixel_format_ =
[[NSOpenGLPixelFormat alloc] initWithAttributes:attrs_no_accel];
}
if (!pixel_format_)
return ::mediapipe::InternalError(
"Could not create an NSOpenGLPixelFormat");
context_ = [[NSOpenGLContext alloc] initWithFormat:pixel_format_
shareContext:share_context];
// Try to query pixel format from shared context.
if (!context_) {
LOG(WARNING) << "Requested context not created, using queried context.";
CGLContextObj cgl_ctx =
static_cast<CGLContextObj>([share_context CGLContextObj]);
CGLPixelFormatObj cgl_fmt =
static_cast<CGLPixelFormatObj>(CGLGetPixelFormat(cgl_ctx));
pixel_format_ =
[[NSOpenGLPixelFormat alloc] initWithCGLPixelFormatObj:cgl_fmt];
context_ = [[NSOpenGLContext alloc] initWithFormat:pixel_format_
shareContext:share_context];
}
RET_CHECK(context_) << "Could not create an NSOpenGLContext";
CVOpenGLTextureCacheRef cache;
CVReturn err = CVOpenGLTextureCacheCreate(
kCFAllocatorDefault, NULL, context_.CGLContextObj,
pixel_format_.CGLPixelFormatObj, NULL, &cache);
RET_CHECK_EQ(err, kCVReturnSuccess) << "Error at CVOpenGLTextureCacheCreate";
texture_cache_.adopt(cache);
return ::mediapipe::OkStatus();
}
void GlContext::DestroyContext() {}
GlContext::ContextBinding GlContext::ThisContextBinding() {
GlContext::ContextBinding result;
result.context_object = shared_from_this();
result.context = context_;
return result;
}
void GlContext::GetCurrentContextBinding(GlContext::ContextBinding* binding) {
binding->context = [NSOpenGLContext currentContext];
}
::mediapipe::Status GlContext::SetCurrentContextBinding(
const ContextBinding& new_binding) {
if (new_binding.context) {
[new_binding.context makeCurrentContext];
} else {
[NSOpenGLContext clearCurrentContext];
}
return ::mediapipe::OkStatus();
}
bool GlContext::HasContext() const { return context_ != nil; }
bool GlContext::IsCurrent() const {
return HasContext() && ([NSOpenGLContext currentContext] == context_);
}
} // namespace mediapipe
#endif // HAS_NSGL
mediapipe/gpu/gl_context_options.proto
+27
View File
@@ -0,0 +1,27 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message GlContextOptions {
extend CalculatorOptions {
optional GlContextOptions ext = 222332034;
}
optional string gl_context_name = 1;
}
mediapipe/gpu/gl_context_webgl.cc
+158
View File
@@ -0,0 +1,158 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <utility>
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_builder.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gl_context_internal.h"
#if defined(__EMSCRIPTEN__)
namespace mediapipe {
// TODO: Handle webGL "context lost" and "context restored" events.
GlContext::StatusOrGlContext GlContext::Create(std::nullptr_t nullp,
bool create_thread) {
return Create(0, create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(const GlContext& share_context,
bool create_thread) {
return Create(share_context.context_, create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE share_context, bool create_thread) {
std::shared_ptr<GlContext> context(new GlContext());
RETURN_IF_ERROR(context->CreateContext(share_context));
RETURN_IF_ERROR(context->FinishInitialization(create_thread));
return std::move(context);
}
::mediapipe::Status GlContext::CreateContextInternal(
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE external_context, int webgl_version) {
CHECK(webgl_version == 1 || webgl_version == 2);
EmscriptenWebGLContextAttributes attrs;
attrs.explicitSwapControl = 0;
attrs.depth = 1;
attrs.stencil = 0;
attrs.antialias = 0;
attrs.majorVersion = webgl_version;
attrs.minorVersion = 0;
attrs.premultipliedAlpha = 0;
// New one to try out... TODO: see if actually necessary for
// pushing resulting texture through MediaPipe pipeline.
attrs.preserveDrawingBuffer = 0;
// We use id of "0" for now to target our webassembly Module.canvas
// specifically for all GL contexts.
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE context_handle =
emscripten_webgl_create_context(0 /* id */, &attrs);
// Check for failure
if (context_handle <= 0) {
LOG(INFO) << "Couldn't create webGL " << webgl_version << " context.";
return ::mediapipe::UnknownErrorBuilder(MEDIAPIPE_LOC)
<< "emscripten_webgl_create_context() returned error "
<< context_handle;
}
context_ = context_handle;
attrs_ = attrs;
// We can't always rely on GL_MAJOR_VERSION and GL_MINOR_VERSION, since
// GLES 2 does not have them, so let's set the major version here at least.
// WebGL 1.0 maps to GLES 2.0 and WebGL 2.0 maps to GLES 3.0, so we add 1.
gl_major_version_ = webgl_version + 1;
return ::mediapipe::OkStatus();
}
::mediapipe::Status GlContext::CreateContext(
EMSCRIPTEN_WEBGL_CONTEXT_HANDLE external_context) {
// TODO: If we're given a non-0 external_context, could try to use
// that directly, since we're assuming a single-threaded single-context
// environment anyways now.
auto status = CreateContextInternal(external_context, 2);
if (!status.ok()) {
LOG(WARNING) << "Creating a context with WebGL 2 failed: " << status;
LOG(WARNING) << "Fall back on WebGL 1.";
status = CreateContextInternal(external_context, 1);
}
RETURN_IF_ERROR(status);
LOG(INFO) << "Successfully created a WebGL Context with major version "
<< gl_major_version_ << " and context " << context_;
return ::mediapipe::OkStatus();
}
void GlContext::DestroyContext() {
if (thread_) {
// For now, we force web MediaPipe to be single-threaded, so error here.
LOG(ERROR) << "thread_ should not exist in DestroyContext() on web.";
}
// Destroy the context and surface.
if (context_ != 0) {
EMSCRIPTEN_RESULT res = emscripten_webgl_destroy_context(context_);
if (res != EMSCRIPTEN_RESULT_SUCCESS) {
LOG(ERROR) << "emscripten_webgl_destroy_context() returned error " << res;
}
context_ = 0;
}
}
GlContext::ContextBinding GlContext::ThisContextBinding() {
GlContext::ContextBinding result;
result.context_object = shared_from_this();
result.context = context_;
return result;
}
void GlContext::GetCurrentContextBinding(GlContext::ContextBinding* binding) {
binding->context = emscripten_webgl_get_current_context();
}
::mediapipe::Status GlContext::SetCurrentContextBinding(
const ContextBinding& new_binding) {
if (new_binding.context == 0) {
// Calling emscripten_webgl_make_context_current(0) is resulting in an error
// so don't remove context for now, only replace! In the future, can
// perhaps create a separate "do-nothing" context for this.
return ::mediapipe::OkStatus();
}
// TODO: See if setting the same context to current multiple times
// comes with a performance cost, and fix if so.
EMSCRIPTEN_RESULT res =
emscripten_webgl_make_context_current(new_binding.context);
RET_CHECK(res == EMSCRIPTEN_RESULT_SUCCESS)
<< "emscripten_webgl_make_context_current() returned error " << res;
return ::mediapipe::OkStatus();
}
bool GlContext::HasContext() const { return context_ != 0; }
bool GlContext::IsCurrent() const {
return HasContext() && (emscripten_webgl_get_current_context() == context_);
}
} // namespace mediapipe
#endif // defined(__EMSCRIPTEN__)
mediapipe/gpu/gl_quad_renderer.cc
+177
View File
@@ -0,0 +1,177 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_quad_renderer.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/gpu/gl_simple_shaders.h"
#include "mediapipe/gpu/shader_util.h"
namespace mediapipe {
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
// static
FrameScaleMode FrameScaleModeFromProto(ScaleMode_Mode proto_scale_mode,
FrameScaleMode default_mode) {
switch (proto_scale_mode) {
case ScaleMode_Mode_DEFAULT:
return default_mode;
case ScaleMode_Mode_STRETCH:
return FrameScaleMode::kStretch;
case ScaleMode_Mode_FIT:
return FrameScaleMode::kFit;
case ScaleMode_Mode_FILL_AND_CROP:
return FrameScaleMode::kFillAndCrop;
default:
return default_mode;
}
}
FrameRotation FrameRotationFromDegrees(int degrees_ccw) {
switch (degrees_ccw) {
case 0:
return FrameRotation::kNone;
case 90:
return FrameRotation::k90;
case 180:
return FrameRotation::k180;
case 270:
return FrameRotation::k270;
default:
return FrameRotation::kNone;
}
}
::mediapipe::Status QuadRenderer::GlSetup() {
return GlSetup(kBasicTexturedFragmentShader, {"video_frame"});
}
::mediapipe::Status QuadRenderer::GlSetup(
const GLchar* custom_frag_shader,
const std::vector<const GLchar*>& custom_frame_uniforms) {
// Load vertex and fragment shaders
const GLint attr_location[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION,
};
const GLchar* attr_name[NUM_ATTRIBUTES] = {
"position",
"texture_coordinate",
};
GlhCreateProgram(kScaledVertexShader, custom_frag_shader, NUM_ATTRIBUTES,
&attr_name[0], attr_location, &program_);
RET_CHECK(program_) << "Problem initializing the program.";
frame_unifs_.resize(custom_frame_uniforms.size());
for (int i = 0; i < custom_frame_uniforms.size(); ++i) {
frame_unifs_[i] = glGetUniformLocation(program_, custom_frame_uniforms[i]);
RET_CHECK(frame_unifs_[i] != -1)
<< "could not find uniform '" << custom_frame_uniforms[i] << "'";
}
scale_unif_ = glGetUniformLocation(program_, "scale");
RET_CHECK(scale_unif_ != -1) << "could not find uniform 'scale'";
return ::mediapipe::OkStatus();
}
void QuadRenderer::GlTeardown() {
if (program_) {
glDeleteProgram(program_);
program_ = 0;
}
}
::mediapipe::Status QuadRenderer::GlRender(
float frame_width, float frame_height, float view_width, float view_height,
FrameScaleMode scale_mode, FrameRotation rotation, bool flip_horizontal,
bool flip_vertical, bool flip_texture) {
RET_CHECK(program_) << "Must setup the program before rendering.";
glUseProgram(program_);
for (int i = 0; i < frame_unifs_.size(); ++i) {
glUniform1i(frame_unifs_[i], i + 1);
}
// Determine scale parameter.
if (rotation == FrameRotation::k90 || rotation == FrameRotation::k270) {
std::swap(frame_width, frame_height);
}
GLfloat scale_width = frame_width / view_width;
GLfloat scale_height = frame_height / view_height;
GLfloat scale_adjust;
switch (scale_mode) {
case FrameScaleMode::kStretch:
scale_width = scale_height = 1.0;
break;
case FrameScaleMode::kFillAndCrop:
// Make the smallest dimension touch the edge.
scale_adjust = std::min(scale_width, scale_height);
scale_width /= scale_adjust;
scale_height /= scale_adjust;
break;
case FrameScaleMode::kFit:
// Make the largest dimension touch the edge.
scale_adjust = std::max(scale_width, scale_height);
scale_width /= scale_adjust;
scale_height /= scale_adjust;
break;
}
const int h_flip_factor = flip_horizontal ? -1 : 1;
const int v_flip_factor = flip_vertical ? -1 : 1;
GLfloat scale[] = {scale_width * h_flip_factor, scale_height * v_flip_factor,
1.0, 1.0};
glUniform4fv(scale_unif_, 1, scale);
// Choose vertices for rotation.
const GLfloat* vertices; // quad used to render the texture.
switch (rotation) {
case FrameRotation::kNone:
vertices = kBasicSquareVertices;
break;
case FrameRotation::k90:
vertices = kBasicSquareVertices90;
break;
case FrameRotation::k180:
vertices = kBasicSquareVertices180;
break;
case FrameRotation::k270:
vertices = kBasicSquareVertices270;
break;
}
// Draw.
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, vertices);
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(
ATTRIB_TEXTURE_POSITION, 2, GL_FLOAT, 0, 0,
flip_texture ? kBasicTextureVerticesFlipY : kBasicTextureVertices);
glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
return ::mediapipe::OkStatus();
}
::mediapipe::Status FrameRotationFromInt(FrameRotation* rotation,
int degrees_ccw) {
RET_CHECK(degrees_ccw % 90 == 0) << "rotation must be a multiple of 90; "
<< degrees_ccw << " was provided";
*rotation = FrameRotationFromDegrees(degrees_ccw % 360);
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/gpu/gl_quad_renderer.h
+96
View File
@@ -0,0 +1,96 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_QUAD_RENDERER_H_
#define MEDIAPIPE_GPU_GL_QUAD_RENDERER_H_
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/scale_mode.pb.h"
namespace mediapipe {
// Valid rotation values. Counterclockwise.
enum class FrameRotation { kNone, k90, k180, k270 };
enum class FrameScaleMode {
// Stretch the frame to the exact provided output dimensions.
kStretch = 0,
// Scale the frame up to fit the drawing area, preserving aspect ratio; may
// letterbox.
kFit,
// Scale the frame up to fill the drawing area, preserving aspect ratio; may
// crop.
kFillAndCrop,
};
// Converts scale_mode.proto enum value typically used in calculator options
// to FrameScaleMode value.
FrameScaleMode FrameScaleModeFromProto(ScaleMode_Mode proto_scale_mode,
FrameScaleMode default_mode);
// This is a utility class containing some common code to render a texture on
// a quadrilateral with aspect ratio correction, (quarter-circle) rotation,
// mirroring and flipping. It is used in various places where rendering is
// done.
class QuadRenderer {
public:
QuadRenderer() {}
// Creates the rendering program. Must be called within the GL context that
// will be used for rendering.
::mediapipe::Status GlSetup();
// Creates the rendering program. Must be called within the GL context that
// will be used for rendering.
// This version allows you to customize the fragment shader.
::mediapipe::Status GlSetup(
const GLchar* custom_frag_shader,
const std::vector<const GLchar*>& custom_frame_uniforms);
// Renders the texture bound to texture unit 1 onto the current viewport.
// Note: mirroring and flipping are handled differently, by design.
// - flip_texture is meant to be used when the texture image's rows are stored
// top-to-bottom. The OpenGL custom is to store them bottom-to-top, but this
// is the opposite of the way most other graphics APIs and formats represent
// images, so having flipped textures is quite common.
// Because this is a property of the input texture, flipping is applied
// BEFORE rotation.
// - flip_horizontal is meant to be used to flip the output image
// horizontally. This is especially useful for the front-facing camera on
// smartphones. This flipping is applied AFTER rotation, because that is
// what's needed for the front-camera use case.
// - flip_vertical is meant to be used to flip the output image vertically.
// This flipping is applied AFTER rotation.
::mediapipe::Status GlRender(float frame_width, float frame_height,
float view_width, float view_height,
FrameScaleMode scale_mode,
FrameRotation rotation, bool flip_horizontal,
bool flip_vertical, bool flip_texture);
// Deletes the rendering program. Must be called withn the GL context where
// it was created.
void GlTeardown();
private:
GLuint program_ = 0;
GLint scale_unif_ = -1;
std::vector<GLint> frame_unifs_;
};
::mediapipe::Status FrameRotationFromInt(FrameRotation* rotation,
int degrees_ccw);
// Input degrees must be one of: [0, 90, 180, 270].
FrameRotation FrameRotationFromDegrees(int degrees_ccw);
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_QUAD_RENDERER_H_
mediapipe/gpu/gl_scaler_calculator.cc
+337
View File
@@ -0,0 +1,337 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_quad_renderer.h"
#include "mediapipe/gpu/gl_scaler_calculator.pb.h"
#include "mediapipe/gpu/gl_simple_shaders.h"
#include "mediapipe/gpu/shader_util.h"
#ifdef __ANDROID__
// The size of Java arrays is dynamic, which makes it difficult to
// generate the right packet type with a fixed size. Therefore, we
// are using unsized arrays on Android.
typedef int DimensionsPacketType[];
#else
typedef int DimensionsPacketType[2];
#endif
namespace mediapipe {
// Scales, rotates, horizontal or vertical flips the image.
// See GlSimpleCalculatorBase for inputs, outputs and input side packets.
// Additional input streams:
// ROTATION: the counterclockwise rotation angle in degrees. This allows
// user to specify different rotation angles for different frames. If this
// stream is provided, it will override the ROTATION input side packet.
// Additional output streams:
// TOP_BOTTOM_PADDING: If use FIT scale mode, this stream outputs the padding
// size of the input image in normalized value [0, 1] for top and bottom
// sides with equal padding. E.g. Using FIT scale mode, if the input images
// size is 10x10 and the required output size is 20x40, then the top and
// bottom side of the image will both having padding of 10 pixels. So the
// value of output stream is 10 / 40 = 0.25.
// LEFT_RIGHT_PADDING: If use FIT scale mode, this stream outputs the padding
// size of the input image in normalized value [0, 1] for left and right side.
// E.g. Using FIT scale mode, if the input images size is 10x10 and the
// required output size is 6x5, then the left and right side of the image will
// both having padding of 1 pixels. So the value of output stream is 1 / 5 =
// 0.2.
// Additional input side packets:
// OUTPUT_DIMENSIONS: the output width and height in pixels.
// ROTATION: the counterclockwise rotation angle in degrees.
// These can also be specified as options.
// To enable horizontal or vertical flip, specify them in options.
// The flipping is applied after rotation.
class GlScalerCalculator : public CalculatorBase {
public:
GlScalerCalculator() {}
~GlScalerCalculator();
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status GlSetup();
::mediapipe::Status GlRender(const GlTexture& src, const GlTexture& dst);
void GetOutputDimensions(int src_width, int src_height, int* dst_width,
int* dst_height);
void GetOutputPadding(int src_width, int src_height, int dst_width,
int dst_height, float* top_bottom_padding,
float* left_right_padding);
GpuBufferFormat GetOutputFormat() { return GpuBufferFormat::kBGRA32; }
private:
GlCalculatorHelper helper_;
int dst_width_ = 0;
int dst_height_ = 0;
FrameRotation rotation_;
std::unique_ptr<QuadRenderer> rgb_renderer_;
std::unique_ptr<QuadRenderer> yuv_renderer_;
#ifdef __ANDROID__
std::unique_ptr<QuadRenderer> ext_rgb_renderer_;
#endif
bool vertical_flip_output_;
bool horizontal_flip_output_;
FrameScaleMode scale_mode_ = FrameScaleMode::kStretch;
};
REGISTER_CALCULATOR(GlScalerCalculator);
// static
::mediapipe::Status GlScalerCalculator::GetContract(CalculatorContract* cc) {
TagOrIndex(&cc->Inputs(), "VIDEO", 0).Set<GpuBuffer>();
TagOrIndex(&cc->Outputs(), "VIDEO", 0).Set<GpuBuffer>();
if (cc->Inputs().HasTag("ROTATION")) {
cc->Inputs().Tag("ROTATION").Set<int>();
}
RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
if (HasTagOrIndex(&cc->InputSidePackets(), "OUTPUT_DIMENSIONS", 1)) {
TagOrIndex(&cc->InputSidePackets(), "OUTPUT_DIMENSIONS", 1)
.Set<DimensionsPacketType>();
}
if (cc->InputSidePackets().HasTag("ROTATION")) {
// Counterclockwise rotation.
cc->InputSidePackets().Tag("ROTATION").Set<int>();
}
if (cc->Outputs().HasTag("TOP_BOTTOM_PADDING") &&
cc->Outputs().HasTag("LEFT_RIGHT_PADDING")) {
cc->Outputs().Tag("TOP_BOTTOM_PADDING").Set<float>();
cc->Outputs().Tag("LEFT_RIGHT_PADDING").Set<float>();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status GlScalerCalculator::Open(CalculatorContext* cc) {
// Inform the framework that we always output at the same timestamp
// as we receive a packet at.
cc->SetOffset(mediapipe::TimestampDiff(0));
// Let the helper access the GL context information.
RETURN_IF_ERROR(helper_.Open(cc));
int rotation_ccw = 0;
const auto& options = cc->Options<GlScalerCalculatorOptions>();
if (options.has_output_width()) {
dst_width_ = options.output_width();
}
if (options.has_output_height()) {
dst_height_ = options.output_height();
}
if (options.has_rotation()) {
rotation_ccw = options.rotation();
}
if (options.has_flip_vertical()) {
vertical_flip_output_ = options.flip_vertical();
} else {
vertical_flip_output_ = false;
}
if (options.has_flip_horizontal()) {
horizontal_flip_output_ = options.flip_horizontal();
} else {
horizontal_flip_output_ = false;
}
if (options.has_scale_mode()) {
scale_mode_ =
FrameScaleModeFromProto(options.scale_mode(), FrameScaleMode::kStretch);
}
if (HasTagOrIndex(cc->InputSidePackets(), "OUTPUT_DIMENSIONS", 1)) {
const auto& dimensions =
TagOrIndex(cc->InputSidePackets(), "OUTPUT_DIMENSIONS", 1)
.Get<DimensionsPacketType>();
dst_width_ = dimensions[0];
dst_height_ = dimensions[1];
}
if (cc->InputSidePackets().HasTag("ROTATION")) {
rotation_ccw = cc->InputSidePackets().Tag("ROTATION").Get<int>();
}
RETURN_IF_ERROR(FrameRotationFromInt(&rotation_, rotation_ccw));
return ::mediapipe::OkStatus();
}
::mediapipe::Status GlScalerCalculator::Process(CalculatorContext* cc) {
return helper_.RunInGlContext([this, cc]() -> ::mediapipe::Status {
const auto& input = TagOrIndex(cc->Inputs(), "VIDEO", 0).Get<GpuBuffer>();
QuadRenderer* renderer = nullptr;
GlTexture src1;
GlTexture src2;
#ifdef __APPLE__
if (input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange ||
input.format() == GpuBufferFormat::kBiPlanar420YpCbCr8FullRange) {
if (!yuv_renderer_) {
yuv_renderer_ = absl::make_unique<QuadRenderer>();
RETURN_IF_ERROR(yuv_renderer_->GlSetup(
kYUV2TexToRGBFragmentShader, {"video_frame_y", "video_frame_uv"}));
}
renderer = yuv_renderer_.get();
src1 = helper_.CreateSourceTexture(input, 0);
src2 = helper_.CreateSourceTexture(input, 1);
} else // NOLINT(readability/braces)
#endif // __APPLE__
{
src1 = helper_.CreateSourceTexture(input);
#ifdef __ANDROID__
if (src1.target() == GL_TEXTURE_EXTERNAL_OES) {
if (!ext_rgb_renderer_) {
ext_rgb_renderer_ = absl::make_unique<QuadRenderer>();
RETURN_IF_ERROR(ext_rgb_renderer_->GlSetup(
kBasicTexturedFragmentShaderOES, {"video_frame"}));
}
renderer = ext_rgb_renderer_.get();
} else // NOLINT(readability/braces)
#endif // __ANDROID__
{
if (!rgb_renderer_) {
rgb_renderer_ = absl::make_unique<QuadRenderer>();
RETURN_IF_ERROR(rgb_renderer_->GlSetup());
}
renderer = rgb_renderer_.get();
}
}
RET_CHECK(renderer) << "Unsupported input texture type";
// Override input side packet if ROTATION input packet is provided.
if (cc->Inputs().HasTag("ROTATION")) {
int rotation_ccw = cc->Inputs().Tag("ROTATION").Get<int>();
RETURN_IF_ERROR(FrameRotationFromInt(&rotation_, rotation_ccw));
}
int dst_width;
int dst_height;
GetOutputDimensions(src1.width(), src1.height(), &dst_width, &dst_height);
if (cc->Outputs().HasTag("TOP_BOTTOM_PADDING") &&
cc->Outputs().HasTag("LEFT_RIGHT_PADDING")) {
float top_bottom_padding;
float left_right_padding;
GetOutputPadding(src1.width(), src1.height(), dst_width, dst_height,
&top_bottom_padding, &left_right_padding);
cc->Outputs()
.Tag("TOP_BOTTOM_PADDING")
.AddPacket(
MakePacket<float>(top_bottom_padding).At(cc->InputTimestamp()));
cc->Outputs()
.Tag("LEFT_RIGHT_PADDING")
.AddPacket(
MakePacket<float>(left_right_padding).At(cc->InputTimestamp()));
}
auto dst = helper_.CreateDestinationTexture(dst_width, dst_height,
GetOutputFormat());
helper_.BindFramebuffer(dst);
glActiveTexture(GL_TEXTURE1);
glBindTexture(src1.target(), src1.name());
if (src2.name()) {
glActiveTexture(GL_TEXTURE2);
glBindTexture(src2.target(), src2.name());
}
RETURN_IF_ERROR(renderer->GlRender(
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode_,
rotation_, horizontal_flip_output_, vertical_flip_output_,
/*flip_texture*/ false));
glActiveTexture(GL_TEXTURE1);
glBindTexture(src1.target(), 0);
if (src2.name()) {
glActiveTexture(GL_TEXTURE2);
glBindTexture(src2.target(), 0);
}
glFlush();
auto output = dst.GetFrame<GpuBuffer>();
TagOrIndex(&cc->Outputs(), "VIDEO", 0)
.Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
});
}
void GlScalerCalculator::GetOutputDimensions(int src_width, int src_height,
int* dst_width, int* dst_height) {
if (dst_width_ > 0 && dst_height_ > 0) {
*dst_width = dst_width_;
*dst_height = dst_height_;
} else if (rotation_ == FrameRotation::k90 ||
rotation_ == FrameRotation::k270) {
*dst_width = src_height;
*dst_height = src_width;
} else {
*dst_width = src_width;
*dst_height = src_height;
}
}
void GlScalerCalculator::GetOutputPadding(int src_width, int src_height,
int dst_width, int dst_height,
float* top_bottom_padding,
float* left_right_padding) {
*top_bottom_padding = 0.0f;
*left_right_padding = 0.0f;
if (rotation_ == FrameRotation::k90 || rotation_ == FrameRotation::k270) {
const int tmp = src_width;
src_width = src_height;
src_height = tmp;
}
if (scale_mode_ == FrameScaleMode::kFit) {
const float src_scale = 1.0f * src_width / src_height;
const float dst_scale = 1.0f * dst_width / dst_height;
if (src_scale - dst_scale > 1e-5) {
// Total padding on top and bottom sides.
*top_bottom_padding =
1.0f - 1.0f * dst_width / src_width * src_height / dst_height;
// Get padding on each side.
*top_bottom_padding /= 2.0f;
} else if (dst_scale - src_scale > 1e-5) {
// Total padding on left and right sides.
*left_right_padding =
1.0f - 1.0f / dst_width * src_width / src_height * dst_height;
// Get padding on each side.
*left_right_padding /= 2.0f;
}
}
}
GlScalerCalculator::~GlScalerCalculator() {
// TODO: use move capture when we have C++14 or better.
QuadRenderer* rgb_renderer = rgb_renderer_.release();
QuadRenderer* yuv_renderer = yuv_renderer_.release();
if (rgb_renderer || yuv_renderer) {
helper_.RunInGlContext([rgb_renderer, yuv_renderer] {
if (rgb_renderer) {
rgb_renderer->GlTeardown();
delete rgb_renderer;
}
if (yuv_renderer) {
yuv_renderer->GlTeardown();
delete yuv_renderer;
}
});
}
}
} // namespace mediapipe
mediapipe/gpu/gl_scaler_calculator.proto
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
import "mediapipe/gpu/scale_mode.proto";
message GlScalerCalculatorOptions {
extend CalculatorOptions {
optional GlScalerCalculatorOptions ext = 166373014;
}
// Output dimensions.
optional int32 output_width = 1;
optional int32 output_height = 2;
// Counterclockwise rotation in degrees. Must be a multiple of 90.
optional int32 rotation = 3;
// Flip the output texture vertically. This is applied after rotation.
optional bool flip_vertical = 4;
// Flip the output texture horizontally. This is applied after rotation.
optional bool flip_horizontal = 5;
optional ScaleMode.Mode scale_mode = 6;
}
mediapipe/gpu/gl_simple_calculator.cc
+81
View File
@@ -0,0 +1,81 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_simple_calculator.h"
namespace mediapipe {
// static
::mediapipe::Status GlSimpleCalculator::GetContract(CalculatorContract* cc) {
TagOrIndex(&cc->Inputs(), "VIDEO", 0).Set<GpuBuffer>();
TagOrIndex(&cc->Outputs(), "VIDEO", 0).Set<GpuBuffer>();
// Currently we pass GL context information and other stuff as external
// inputs, which are handled by the helper.
return GlCalculatorHelper::UpdateContract(cc);
}
::mediapipe::Status GlSimpleCalculator::Open(CalculatorContext* cc) {
// Inform the framework that we always output at the same timestamp
// as we receive a packet at.
cc->SetOffset(mediapipe::TimestampDiff(0));
// Let the helper access the GL context information.
return helper_.Open(cc);
}
::mediapipe::Status GlSimpleCalculator::Process(CalculatorContext* cc) {
return RunInGlContext([this, cc]() -> ::mediapipe::Status {
const auto& input = TagOrIndex(cc->Inputs(), "VIDEO", 0).Get<GpuBuffer>();
if (!initialized_) {
RETURN_IF_ERROR(GlSetup());
initialized_ = true;
}
auto src = helper_.CreateSourceTexture(input);
int dst_width;
int dst_height;
GetOutputDimensions(src.width(), src.height(), &dst_width, &dst_height);
auto dst = helper_.CreateDestinationTexture(dst_width, dst_height,
GetOutputFormat());
helper_.BindFramebuffer(dst);
glActiveTexture(GL_TEXTURE1);
glBindTexture(src.target(), src.name());
RETURN_IF_ERROR(GlBind());
// Run core program.
RETURN_IF_ERROR(GlRender(src, dst));
glBindTexture(src.target(), 0);
glFlush();
auto output = dst.GetFrame<GpuBuffer>();
src.Release();
dst.Release();
TagOrIndex(&cc->Outputs(), "VIDEO", 0)
.Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
});
}
::mediapipe::Status GlSimpleCalculator::Close(CalculatorContext* cc) {
return RunInGlContext(
[this]() -> ::mediapipe::Status { return GlTeardown(); });
}
} // namespace mediapipe
mediapipe/gpu/gl_simple_calculator.h
+110
View File
@@ -0,0 +1,110 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_SIMPLE_CALCULATOR_H_
#define MEDIAPIPE_GPU_GL_SIMPLE_CALCULATOR_H_
#include <utility> // for declval
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
namespace mediapipe {
// This class saves some boilerplate for the common case of processing one
// input stream of frames and outputting it as one output stream of frames,
// processed using OpenGL.
//
// If you use tags, both input and output streams should be tagged as VIDEO.
// Otherwise, this will use the first input and output.
//
// Subclasses should define at least:
// - GlSetup(), which is called once (on the first frame) and should set up any
// GL objects the calculator will reuse throughout its life.
// - GlRender(), which is called for each frame.
// - A destructor, to destroy the objects created in GlSetup.
// Note that when GlSetup and GlRender are called, the GL context has already
// been set, but in the destructor it has not. The destructor should have a
// local variable set to ContextAutoSetter() to make sure it is doing the
// destruction in the right GL context.
//
// Additionally, you can define a GlBind() method, which will be called to
// enable shader programs, bind any additional textures you may need, etc.
// If your calculator shares a context with other calculators, GlBind() will be
// called before each GlRender(); if it has its own context, it will be called
// only once.
class GlSimpleCalculator : public CalculatorBase {
public:
GlSimpleCalculator() : initialized_(false) {}
GlSimpleCalculator(const GlSimpleCalculator&) = delete;
GlSimpleCalculator& operator=(const GlSimpleCalculator&) = delete;
~GlSimpleCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
// This method is called once on the first frame. Use it to setup any objects
// that will be reused throughout the calculator's life.
virtual ::mediapipe::Status GlSetup() = 0;
// You can use this optional method to do any pre-rendering setup that needs
// to be redone after the context has been used by another calculator.
// If your context is not shared, it will only be called once.
virtual ::mediapipe::Status GlBind() { return ::mediapipe::OkStatus(); }
// Do your rendering here. The source and destination textures have already
// been created and bound for you.
// - src: source texture (contains input frame); already bound to GL_TEXTURE1.
// - dst: destination texture (write output frame here); already bound to
// GL_TEXTURE0 and attached to the framebuffer.
virtual ::mediapipe::Status GlRender(const GlTexture& src,
const GlTexture& dst) = 0;
// The method is called to delete all the programs.
virtual ::mediapipe::Status GlTeardown() = 0;
// You can override this method to compute the size of the destination
// texture. By default, it will take the same size as the source texture.
virtual void GetOutputDimensions(int src_width, int src_height,
int* dst_width, int* dst_height) {
*dst_width = src_width;
*dst_height = src_height;
}
// Override this to output a different type of buffer.
virtual GpuBufferFormat GetOutputFormat() { return GpuBufferFormat::kBGRA32; }
protected:
// Forward invocations of RunInGlContext to the helper.
// The decltype part just says that this method returns whatever type the
// helper method invocation returns. In C++14 we could remove it and use
// return type deduction, i.e.:
// template <typename F> auto RunInGlContext(F&& f) { ... }
template <typename F>
auto RunInGlContext(F&& f)
-> decltype(std::declval<GlCalculatorHelper>().RunInGlContext(f)) {
return helper_.RunInGlContext(std::forward<F>(f));
}
GlCalculatorHelper helper_;
bool initialized_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_SIMPLE_CALCULATOR_H_
mediapipe/gpu/gl_simple_shaders.cc
+134
View File
@@ -0,0 +1,134 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_simple_shaders.h"
namespace mediapipe {
// This macro converts everything between its parentheses to a std::string.
// Using this instead of R"()" preserves C-like syntax coloring in most
// editors, which is desirable for shaders.
#if !defined(_STRINGIFY)
#define __STRINGIFY(_x) #_x
#define _STRINGIFY(_x) __STRINGIFY(_x)
#endif
#define PRECISION_COMPAT \
GLES_VERSION_COMPAT \
"#ifdef GL_ES \n" \
"#define DEFAULT_PRECISION(p, t) precision p t; \n" \
"#else \n" \
"#define DEFAULT_PRECISION(p, t) \n" \
"#define lowp \n" \
"#define mediump \n" \
"#define highp \n" \
"#endif // defined(GL_ES) \n"
#define VERTEX_PREAMBLE \
PRECISION_COMPAT \
"#if __VERSION__ < 130\n" \
"#define in attribute\n" \
"#define out varying\n" \
"#endif // __VERSION__ < 130\n"
#define FRAGMENT_PREAMBLE \
PRECISION_COMPAT \
"#if __VERSION__ < 130\n" \
"#define in varying\n" \
"#endif // __VERSION__ < 130\n"
const GLchar* const kMediaPipeVertexShaderPreamble = VERTEX_PREAMBLE;
const GLchar* const kMediaPipeFragmentShaderPreamble = FRAGMENT_PREAMBLE;
const GLchar* const kBasicVertexShader = VERTEX_PREAMBLE _STRINGIFY(
// vertex position in clip space (-1..1)
in vec4 position;
// texture coordinate for each vertex in normalized texture space (0..1)
in mediump vec4 texture_coordinate;
// texture coordinate for fragment shader (will be interpolated)
out mediump vec2 sample_coordinate;
void main() {
gl_Position = position;
sample_coordinate = texture_coordinate.xy;
});
const GLchar* const kScaledVertexShader = VERTEX_PREAMBLE _STRINGIFY(
in vec4 position; in mediump vec4 texture_coordinate;
out mediump vec2 sample_coordinate; uniform vec4 scale;
void main() {
gl_Position = position * scale;
sample_coordinate = texture_coordinate.xy;
});
const GLchar* const kBasicTexturedFragmentShader = FRAGMENT_PREAMBLE _STRINGIFY(
DEFAULT_PRECISION(mediump, float)
in mediump vec2 sample_coordinate; // texture coordinate (0..1)
uniform sampler2D video_frame;
void main() { gl_FragColor = texture2D(video_frame, sample_coordinate); });
const GLchar* const kBasicTexturedFragmentShaderOES = FRAGMENT_PREAMBLE
"#extension GL_OES_EGL_image_external : require\n" _STRINGIFY(
DEFAULT_PRECISION(mediump, float)
in mediump vec2 sample_coordinate; // texture coordinate (0..1)
uniform samplerExternalOES video_frame;
void main() {
gl_FragColor = texture2D(video_frame, sample_coordinate);
});
const GLchar* const kFlatColorFragmentShader = FRAGMENT_PREAMBLE _STRINGIFY(
DEFAULT_PRECISION(mediump, float)
uniform vec3 color; // r,g,b color components
void main() { gl_FragColor = vec4(color.r, color.g, color.b, 1.0); });
const GLchar* const kRgbWeightFragmentShader = FRAGMENT_PREAMBLE _STRINGIFY(
DEFAULT_PRECISION(mediump, float)
in mediump vec2 sample_coordinate; // texture coordinate (0..1)
uniform sampler2D video_frame; uniform vec3 weights; // r,g,b weights
void main() {
vec4 color = texture2D(video_frame, sample_coordinate);
gl_FragColor.bgra = vec4(weights.z * color.b, weights.y * color.g,
weights.x * color.r, color.a);
});
const GLchar* const kYUV2TexToRGBFragmentShader = FRAGMENT_PREAMBLE _STRINGIFY(
DEFAULT_PRECISION(mediump, float)
in highp vec2 sample_coordinate;
uniform sampler2D video_frame_y; uniform sampler2D video_frame_uv;
void main() {
mediump vec3 yuv;
lowp vec3 rgb;
yuv.r = texture2D(video_frame_y, sample_coordinate).r;
// Subtract (0.5, 0.5) because conversion is done assuming UV color
// midpoint of (128, 128).
yuv.gb = texture2D(video_frame_uv, sample_coordinate).rg - vec2(0.5, 0.5);
// Using BT.709 which is the standard for HDTV.
rgb = mat3(1, 1, 1, 0, -0.18732, 1.8556, 1.57481, -0.46813, 0) * yuv;
gl_FragColor = vec4(rgb, 1);
});
} // namespace mediapipe
mediapipe/gpu/gl_simple_shaders.h
+128
View File
@@ -0,0 +1,128 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_SIMPLE_SHADERS_H_
#define MEDIAPIPE_GPU_GL_SIMPLE_SHADERS_H_
#include "mediapipe/gpu/gl_base.h"
namespace mediapipe {
// Leaves vertex and texture coordinates as they are.
// Input attributes:
// vec4 position - vertex position in clip space (-1..1)
// vec4 texture_coordinate - texture coordinate for each vertex in
// normalized texture space (0..1)
// Output varying:
// vec2 sample_coordinate - texture coordinate for shader
extern const GLchar* const kBasicVertexShader;
// Leaves vertex and texture coordinates as they are.
// Input attributes:
// vec4 position - vertex position
// vec4 texture_coordinate - texture coordinate
// Input uniform:
// vec4 scale - scale factor for vertices
// Output varying:
// vec2 sample_coordinate - texture coordinate for shader
extern const GLchar* const kScaledVertexShader;
// Outputs the texture as it is.
// Input varying:
// vec2 sample_coordinate - texture coordinate
// Input uniform:
// sampler2d video_frame - texture
extern const GLchar* const kBasicTexturedFragmentShader;
// Same as kBasicTexturedFragmentShader
// except using OES textures.
extern const GLchar* const kBasicTexturedFragmentShaderOES;
// Paints the fragment with a flat color.
// Input uniform:
// vec3 color - the RGB color.
extern const GLchar* const kFlatColorFragmentShader;
// Multiplies each R, G, B value for a weight.
// Input varying:
// vec2 sample_coordinate - texture coordinate
// Input uniform:
// sampler2d video_frame - texture
// vec3 weights - r,g,b weights
extern const GLchar* const kRgbWeightFragmentShader;
// Converts a YUV input into RGB.
// Input uniform:
// sampler2D video_frame_y - Y texture
// sampler2D video_frame_uv - UV texture
extern const GLchar* const kYUV2TexToRGBFragmentShader;
// A square covering the full clip space.
static const GLfloat kBasicSquareVertices[] = {
-1.0f, -1.0f, // bottom left
1.0f, -1.0f, // bottom right
-1.0f, 1.0f, // top left
1.0f, 1.0f, // top right
};
// Temporary macros to copy vertices.
#define V(source, n) source[2 * (n)], source[2 * (n) + 1]
#define V4(source, a, b, c, d) \
V(source, a), V(source, b), V(source, c), V(source, d)
// A square covering the full clip space, rotated 90 degrees counterclockwise.
static const GLfloat kBasicSquareVertices90[] = {
V4(kBasicSquareVertices, 2, 0, 3, 1)};
// A square covering the full clip space, rotated 180 degrees counterclockwise.
static const GLfloat kBasicSquareVertices180[] = {
V4(kBasicSquareVertices, 3, 2, 1, 0)};
// A square covering the full clip space, rotated 270 degrees counterclockwise.
static const GLfloat kBasicSquareVertices270[] = {
V4(kBasicSquareVertices, 1, 3, 0, 2)};
// Places a texture on kBasicSquareVertices with normal alignment.
static const GLfloat kBasicTextureVertices[] = {
0.0f, 0.0f, // bottom left
1.0f, 0.0f, // bottom right
0.0f, 1.0f, // top left
1.0f, 1.0f, // top right
};
// Places a texture on kBasicSquareVertices, flipped vertically.
static const GLfloat kBasicTextureVerticesFlipY[] = {
V4(kBasicTextureVertices, 2, 3, 0, 1)};
#undef V4
#undef V
// Used in shaders to differentiate desktop OpenGL vs OpenGL ES.
// The newer spec requires this to be the first line of a shader.
// Desktop OpenGL 3.3+ = #version 330
#ifndef GL_ES_VERSION_2_0
#define GLES_VERSION_COMPAT "#version 330 \n"
#else
#define GLES_VERSION_COMPAT "\n"
#endif // GL_ES_VERSION_2_0
// Used in vertex shaders to differentiate different versions of OpenGL.
extern const GLchar* const kMediaPipeVertexShaderPreamble;
// Used in fragment shaders to differentiate different versions of OpenGL.
extern const GLchar* const kMediaPipeFragmentShaderPreamble;
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_SIMPLE_SHADERS_H_
mediapipe/gpu/gl_surface_sink_calculator.cc
+157
View File
@@ -0,0 +1,157 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/gpu/egl_surface_holder.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_quad_renderer.h"
#include "mediapipe/gpu/gl_surface_sink_calculator.pb.h"
#include "mediapipe/gpu/shader_util.h"
namespace mediapipe {
enum { kAttribVertex, kAttribTexturePosition, kNumberOfAttributes };
// Receives GpuBuffers and renders them to an EGL surface.
// Can be used to render to an Android SurfaceTexture.
//
// Inputs:
// VIDEO or index 0: GpuBuffers to be rendered.
// Side inputs:
// SURFACE: unique_ptr to an EglSurfaceHolder to draw to.
// GPU_SHARED: shared GPU resources.
//
// See GlSurfaceSinkCalculatorOptions for options.
class GlSurfaceSinkCalculator : public CalculatorBase {
public:
GlSurfaceSinkCalculator() : initialized_(false) {}
~GlSurfaceSinkCalculator() override;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
GlCalculatorHelper helper_;
EglSurfaceHolder* surface_holder_;
bool initialized_;
std::unique_ptr<QuadRenderer> renderer_;
FrameScaleMode scale_mode_ = FrameScaleMode::kFillAndCrop;
};
REGISTER_CALCULATOR(GlSurfaceSinkCalculator);
// static
::mediapipe::Status GlSurfaceSinkCalculator::GetContract(
CalculatorContract* cc) {
TagOrIndex(&(cc->Inputs()), "VIDEO", 0).Set<GpuBuffer>();
cc->InputSidePackets()
.Tag("SURFACE")
.Set<std::unique_ptr<EglSurfaceHolder>>();
// Currently we pass GL context information and other stuff as external
// inputs, which are handled by the helper.
return GlCalculatorHelper::UpdateContract(cc);
}
::mediapipe::Status GlSurfaceSinkCalculator::Open(CalculatorContext* cc) {
surface_holder_ = cc->InputSidePackets()
.Tag("SURFACE")
.Get<std::unique_ptr<EglSurfaceHolder>>()
.get();
scale_mode_ = FrameScaleModeFromProto(
cc->Options<GlSurfaceSinkCalculatorOptions>().frame_scale_mode(),
FrameScaleMode::kFillAndCrop);
// Let the helper access the GL context information.
return helper_.Open(cc);
}
::mediapipe::Status GlSurfaceSinkCalculator::Process(CalculatorContext* cc) {
return helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
absl::MutexLock lock(&surface_holder_->mutex);
EGLSurface surface = surface_holder_->surface;
if (surface == EGL_NO_SURFACE) {
LOG_EVERY_N(INFO, 300) << "GlSurfaceSinkCalculator: no surface";
return ::mediapipe::OkStatus();
}
const auto& input = TagOrIndex(cc->Inputs(), "VIDEO", 0).Get<GpuBuffer>();
if (!initialized_) {
renderer_ = absl::make_unique<QuadRenderer>();
RETURN_IF_ERROR(renderer_->GlSetup());
initialized_ = true;
}
auto src = helper_.CreateSourceTexture(input);
EGLSurface old_surface = eglGetCurrentSurface(EGL_DRAW);
EGLDisplay display = eglGetCurrentDisplay();
EGLContext context = eglGetCurrentContext();
// Note that eglMakeCurrent can be very slow on Android if you use it to
// change the current context, but it is fast if you only change the
// current surface.
EGLBoolean success = eglMakeCurrent(display, surface, surface, context);
RET_CHECK(success) << "failed to make surface current";
EGLint dst_width;
success = eglQuerySurface(display, surface, EGL_WIDTH, &dst_width);
RET_CHECK(success) << "failed to query surface width";
EGLint dst_height;
success = eglQuerySurface(display, surface, EGL_HEIGHT, &dst_height);
RET_CHECK(success) << "failed to query surface height";
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glViewport(0, 0, dst_width, dst_height);
glActiveTexture(GL_TEXTURE1);
glBindTexture(src.target(), src.name());
RETURN_IF_ERROR(
renderer_->GlRender(src.width(), src.height(), dst_width, dst_height,
scale_mode_, FrameRotation::kNone,
/*flip_horizontal=*/false, /*flip_vertical=*/false,
/*flip_texture=*/false));
glBindTexture(src.target(), 0);
success = eglSwapBuffers(display, surface);
RET_CHECK(success) << "failed to swap buffers";
success = eglMakeCurrent(display, old_surface, old_surface, context);
RET_CHECK(success) << "failed to restore old surface";
src.Release();
return ::mediapipe::OkStatus();
});
}
GlSurfaceSinkCalculator::~GlSurfaceSinkCalculator() {
if (renderer_) {
// TODO: use move capture when we have C++14 or better.
QuadRenderer* renderer = renderer_.release();
helper_.RunInGlContext([renderer] {
renderer->GlTeardown();
delete renderer;
});
}
}
} // namespace mediapipe
mediapipe/gpu/gl_surface_sink_calculator.proto
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
import "mediapipe/gpu/scale_mode.proto";
message GlSurfaceSinkCalculatorOptions {
extend CalculatorOptions {
optional GlSurfaceSinkCalculatorOptions ext = 243334538;
}
// Output frame scale mode. Default is FILL_AND_CROP.
optional ScaleMode.Mode frame_scale_mode = 1;
}
mediapipe/gpu/gl_texture_buffer.cc
+136
View File
@@ -0,0 +1,136 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_texture_buffer.h"
namespace mediapipe {
std::unique_ptr<GlTextureBuffer> GlTextureBuffer::Wrap(
GLenum target, GLuint name, int width, int height, GpuBufferFormat format,
DeletionCallback deletion_callback) {
return absl::make_unique<GlTextureBuffer>(target, name, width, height, format,
deletion_callback);
}
std::unique_ptr<GlTextureBuffer> GlTextureBuffer::Create(int width, int height,
GpuBufferFormat format,
const void* data) {
auto buf = absl::make_unique<GlTextureBuffer>(GL_TEXTURE_2D, 0, width, height,
format, nullptr);
if (!buf->CreateInternal(data)) {
return nullptr;
}
return buf;
}
GlTextureBuffer::GlTextureBuffer(GLenum target, GLuint name, int width,
int height, GpuBufferFormat format,
DeletionCallback deletion_callback)
: name_(name),
width_(width),
height_(height),
format_(format),
target_(target),
deletion_callback_(deletion_callback) {}
bool GlTextureBuffer::CreateInternal(const void* data) {
auto context = GlContext::GetCurrent();
if (!context) return false;
glGenTextures(1, &name_);
if (!name_) return false;
glBindTexture(target_, name_);
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(format_, 0);
// See b/70294573 for details about this.
if (info.gl_internal_format == GL_RGBA16F &&
SymbolAvailable(&glTexStorage2D)) {
CHECK(data == nullptr) << "unimplemented";
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
} else {
glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_,
height_, 0 /* border */, info.gl_format, info.gl_type, data);
}
glBindTexture(target_, 0);
// Use the deletion callback to delete the texture on the context
// that created it.
CHECK(!deletion_callback_);
deletion_callback_ = [this,
context](std::shared_ptr<GlSyncPoint> sync_token) {
CHECK_NE(name_, 0);
GLuint name_to_delete = name_;
context->RunWithoutWaiting([name_to_delete, sync_token]() {
// TODO: maybe we do not actually have to wait for the
// consumer sync here. Check docs.
sync_token->WaitOnGpu();
if (glIsTexture(name_to_delete)) glDeleteTextures(1, &name_to_delete);
});
};
return true;
}
void GlTextureBuffer::Reuse() {
WaitForConsumersOnGpu();
// TODO: should we just do this inside WaitForConsumersOnGpu?
// if we do that, WaitForConsumersOnGpu can be called only once.
consumer_multi_sync_ = absl::make_unique<GlMultiSyncPoint>();
// Reset the token.
producer_sync_ = nullptr;
}
void GlTextureBuffer::Updated(std::shared_ptr<GlSyncPoint> prod_token) {
CHECK(!producer_sync_)
<< "Updated existing texture which had not been marked for reuse!";
producer_sync_ = std::move(prod_token);
}
void GlTextureBuffer::DidRead(std::shared_ptr<GlSyncPoint> cons_token) {
consumer_multi_sync_->Add(std::move(cons_token));
}
GlTextureBuffer::~GlTextureBuffer() {
if (deletion_callback_) {
deletion_callback_(std::move(consumer_multi_sync_));
}
}
void GlTextureBuffer::WaitUntilComplete() {
// Buffers created by the application (using the constructor that wraps an
// existing texture) have no sync token and are assumed to be already
// complete.
if (producer_sync_) {
producer_sync_->Wait();
}
}
void GlTextureBuffer::WaitOnGpu() {
// Buffers created by the application (using the constructor that wraps an
// existing texture) have no sync token and are assumed to be already
// complete.
if (producer_sync_) {
producer_sync_->WaitOnGpu();
}
}
void GlTextureBuffer::WaitForConsumers() { consumer_multi_sync_->Wait(); }
void GlTextureBuffer::WaitForConsumersOnGpu() {
consumer_multi_sync_->WaitOnGpu();
}
} // namespace mediapipe
mediapipe/gpu/gl_texture_buffer.h
+140
View File
@@ -0,0 +1,140 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Consider this file an implementation detail. None of this is part of the
// public API.
#ifndef MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
#define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
#include <atomic>
#include "absl/memory/memory.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
namespace mediapipe {
class GlCalculatorHelperImpl;
// Implements a GPU memory buffer as an OpenGL texture. For internal use.
class GlTextureBuffer {
public:
// This is called when the texture buffer is deleted. It is passed a sync
// token created at that time on the GlContext. If the GlTextureBuffer has
// been created from a texture not owned by MediaPipe, the sync token can be
// used to wait until a point when it is certain that MediaPipe's GPU tasks
// are done reading from the texture. This is improtant if the code outside
// of MediaPipe is going to reuse the texture.
using DeletionCallback =
std::function<void(std::shared_ptr<GlSyncPoint> sync_token)>;
// Wraps an existing texture, but does not take ownership of it.
// deletion_callback is invoked when the GlTextureBuffer is released, so
// the caller knows that the texture is no longer in use.
// The commands producing the texture are assumed to be completed at the
// time of this call. If not, call Updated on the result.
static std::unique_ptr<GlTextureBuffer> Wrap(
GLenum target, GLuint name, int width, int height, GpuBufferFormat format,
DeletionCallback deletion_callback);
// Creates a texture of dimensions width x height and allocates space for it.
// If data is provided, it is uploaded to the texture; otherwise, it can be
// provided later via glTexSubImage2D.
static std::unique_ptr<GlTextureBuffer> Create(int width, int height,
GpuBufferFormat format,
const void* data = nullptr);
// Wraps an existing texture, but does not take ownership of it.
// deletion_callback is invoked when the GlTextureBuffer is released, so
// the caller knows that the texture is no longer in use.
// The commands producing the texture are assumed to be completed at the
// time of this call. If not, call Updated on the result.
GlTextureBuffer(GLenum target, GLuint name, int width, int height,
GpuBufferFormat format, DeletionCallback deletion_callback);
~GlTextureBuffer();
// Included to support nativeGetGpuBuffer* in Java.
// TODO: turn into a single call?
GLuint name() const { return name_; }
GLenum target() const { return target_; }
int width() const { return width_; }
int height() const { return height_; }
GpuBufferFormat format() const { return format_; }
// If this texture is going to be used outside of the context that produced
// it, this method should be called to ensure that its updated contents are
// available. When this method returns, all changed made before the call to
// Updated have become visible.
// This is necessary because texture changes are not synchronized across
// contexts in a sharegroup.
// NOTE: This blocks the current CPU thread and makes the changes visible
// to the CPU. If you want to access the data via OpenGL, use WaitOnGpu
// instead.
void WaitUntilComplete();
// Call this method to synchronize the current GL context with the texture's
// producer. This will not block the current CPU thread, but will ensure that
// subsequent GL commands see the texture in its complete status, with all
// rendering done on the GPU by the generating context.
void WaitOnGpu();
// Informs the buffer that its contents are going to be overwritten.
// This invalidates the current sync token.
// NOTE: this must be called on the context that will become the new
// producer.
void Reuse();
// Informs the buffer that its contents have been updated.
// The provided sync token marks the point when the producer has finished
// writing the new contents.
void Updated(std::shared_ptr<GlSyncPoint> prod_token);
// Informs the buffer that a consumer has finished reading from it.
void DidRead(std::shared_ptr<GlSyncPoint> cons_token);
// Waits for all pending consumers to finish accessing the current content
// of the texture. This (preferably the OnGpu version) should be called
// before overwriting the texture's contents.
void WaitForConsumers();
void WaitForConsumersOnGpu();
private:
// Creates a texture of dimensions width x height and allocates space for it.
// If data is provided, it is uploaded to the texture; otherwise, it can be
// provided later via glTexSubImage2D.
// Returns true on success.
bool CreateInternal(const void* data = nullptr);
friend class GlCalculatorHelperImpl;
GLuint name_ = 0;
int width_ = 0;
int height_ = 0;
GpuBufferFormat format_ = GpuBufferFormat::kUnknown;
GLenum target_ = GL_TEXTURE_2D;
// Token tracking changes to this texture. Used by WaitUntilComplete.
std::shared_ptr<GlSyncPoint> producer_sync_;
// Tokens tracking the point when consumers finished using this texture.
std::unique_ptr<GlMultiSyncPoint> consumer_multi_sync_ =
absl::make_unique<GlMultiSyncPoint>();
DeletionCallback deletion_callback_;
};
using GlTextureBufferSharedPtr = std::shared_ptr<GlTextureBuffer>;
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
mediapipe/gpu/gl_texture_buffer_pool.cc
+76
View File
@@ -0,0 +1,76 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gl_texture_buffer_pool.h"
#include "absl/synchronization/mutex.h"
namespace mediapipe {
GlTextureBufferPool::GlTextureBufferPool(int width, int height,
GpuBufferFormat format, int keep_count)
: width_(width),
height_(height),
format_(format),
keep_count_(keep_count) {}
GlTextureBufferSharedPtr GlTextureBufferPool::GetBuffer() {
absl::MutexLock lock(&mutex_);
std::unique_ptr<GlTextureBuffer> buffer;
if (available_.empty()) {
buffer = GlTextureBuffer::Create(width_, height_, format_);
if (!buffer) return nullptr;
} else {
buffer = std::move(available_.back());
available_.pop_back();
buffer->Reuse();
}
++in_use_count_;
// Return a shared_ptr with a custom deleter that adds the buffer back
// to our available list.
std::weak_ptr<GlTextureBufferPool> weak_pool(shared_from_this());
return std::shared_ptr<GlTextureBuffer>(buffer.release(),
[weak_pool](GlTextureBuffer* buf) {
auto pool = weak_pool.lock();
if (pool) {
pool->Return(buf);
} else {
delete buf;
}
});
}
std::pair<int, int> GlTextureBufferPool::GetInUseAndAvailableCounts() {
absl::MutexLock lock(&mutex_);
return {in_use_count_, available_.size()};
}
void GlTextureBufferPool::Return(GlTextureBuffer* buf) {
absl::MutexLock lock(&mutex_);
--in_use_count_;
available_.emplace_back(buf);
TrimAvailable();
}
void GlTextureBufferPool::TrimAvailable() {
int keep = std::max(keep_count_ - in_use_count_, 0);
if (available_.size() > keep) {
available_.resize(keep);
}
}
} // namespace mediapipe
mediapipe/gpu/gl_texture_buffer_pool.h
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Consider this file an implementation detail. None of this is part of the
// public API.
#ifndef MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_POOL_H_
#define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_POOL_H_
#include <utility>
#include <vector>
#include "absl/synchronization/mutex.h"
#include "mediapipe/gpu/gl_texture_buffer.h"
namespace mediapipe {
class GlTextureBufferPool
: public std::enable_shared_from_this<GlTextureBufferPool> {
public:
// Creates a pool. This pool will manage buffers of the specified dimensions,
// and will keep keep_count buffers around for reuse.
// We enforce creation as a shared_ptr so that we can use a weak reference in
// the buffers' deleters.
static std::shared_ptr<GlTextureBufferPool> Create(int width, int height,
GpuBufferFormat format,
int keep_count) {
return std::shared_ptr<GlTextureBufferPool>(
new GlTextureBufferPool(width, height, format, keep_count));
}
// Obtains a buffers. May either be reused or created anew.
// A GlContext must be current when this is called.
GlTextureBufferSharedPtr GetBuffer();
int width() const { return width_; }
int height() const { return height_; }
GpuBufferFormat format() const { return format_; }
// This method is meant for testing.
std::pair<int, int> GetInUseAndAvailableCounts();
private:
GlTextureBufferPool(int width, int height, GpuBufferFormat format,
int keep_count);
// Return a buffer to the pool.
void Return(GlTextureBuffer* buf);
// If the total number of buffers is greater than keep_count, destroys any
// surplus buffers that are no longer in use.
void TrimAvailable() EXCLUSIVE_LOCKS_REQUIRED(mutex_);
const int width_;
const int height_;
const GpuBufferFormat format_;
const int keep_count_;
absl::Mutex mutex_;
int in_use_count_ GUARDED_BY(mutex_) = 0;
std::vector<std::unique_ptr<GlTextureBuffer>> available_ GUARDED_BY(mutex_);
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_POOL_H_
mediapipe/gpu/gl_thread_collector.h
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_THREAD_COLLECTOR_H_
#define MEDIAPIPE_GPU_GL_THREAD_COLLECTOR_H_
#include <cstdlib>
#if defined(MEDIAPIPE_USING_SWIFTSHADER) && !defined(NDEBUG)
#define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1
#endif
#if MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/deps/no_destructor.h"
#endif // MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR
namespace mediapipe {
#if MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR
class GlThreadCollector {
public:
static void ThreadStarting() { Collector().ChangeCount(1); }
static void ThreadEnding() { Collector().ChangeCount(-1); }
private:
GlThreadCollector() { std::atexit(WaitForThreadsToTerminate); }
static GlThreadCollector& Collector() {
static NoDestructor<GlThreadCollector> collector;
return *collector;
}
static void WaitForThreadsToTerminate() { Collector().Wait(); }
void ChangeCount(int delta) {
absl::MutexLock l(&mutex_);
active_threads_ += delta;
}
void Wait() {
auto done = [this]() {
mutex_.AssertReaderHeld();
return active_threads_ == 0;
};
absl::MutexLock l(&mutex_);
mutex_.Await(Condition(&done));
}
absl::Mutex mutex_;
int active_threads_ GUARDED_BY(mutex_) = 0;
friend NoDestructor<GlThreadCollector>;
};
#else
class GlThreadCollector {
public:
static void ThreadStarting() {}
static void ThreadEnding() {}
};
#endif // MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_THREAD_COLLECTOR_H_
mediapipe/gpu/gpu_buffer.h
+154
View File
@@ -0,0 +1,154 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_H_
#include <utility>
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#if defined(__APPLE__)
#include <CoreVideo/CoreVideo.h>
#include "mediapipe/framework/ios/CFHolder.h"
#if !TARGET_OS_OSX
#define MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER 1
#endif // TARGET_OS_OSX
#endif // defined(__APPLE__)
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#include "mediapipe/gpu/gl_texture_buffer.h"
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
namespace mediapipe {
// This class wraps a platform-specific buffer of GPU data.
// An instance of GpuBuffer acts as an opaque reference to the underlying
// data object.
class GpuBuffer {
public:
// Default constructor creates invalid object.
GpuBuffer() = default;
// Copy and move constructors and assignment operators are supported.
GpuBuffer(const GpuBuffer& other) = default;
GpuBuffer(GpuBuffer&& other) = default;
GpuBuffer& operator=(const GpuBuffer& other) = default;
GpuBuffer& operator=(GpuBuffer&& other) = default;
// Constructors from platform-specific representations, and accessors for the
// underlying platform-specific representation. Use with caution, since they
// are not portable. Applications and calculators should normally obtain
// GpuBuffers in a portable way from the framework, e.g. using
// GpuBufferMultiPool.
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
explicit GpuBuffer(CFHolder<CVPixelBufferRef> pixel_buffer)
: pixel_buffer_(std::move(pixel_buffer)) {}
explicit GpuBuffer(CVPixelBufferRef pixel_buffer)
: pixel_buffer_(pixel_buffer) {}
CVPixelBufferRef GetCVPixelBufferRef() const { return *pixel_buffer_; }
#else
explicit GpuBuffer(GlTextureBufferSharedPtr texture_buffer)
: texture_buffer_(std::move(texture_buffer)) {}
const GlTextureBufferSharedPtr& GetGlTextureBufferSharedPtr() const {
return texture_buffer_;
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
int width() const;
int height() const;
GpuBufferFormat format() const;
// Converts to true iff valid.
explicit operator bool() const { return operator!=(nullptr); }
bool operator==(const GpuBuffer& other) const;
bool operator!=(const GpuBuffer& other) const { return !operator==(other); }
// Allow comparison with nullptr.
bool operator==(std::nullptr_t other) const;
bool operator!=(std::nullptr_t other) const { return !operator==(other); }
// Allow assignment from nullptr.
GpuBuffer& operator=(std::nullptr_t other);
private:
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CFHolder<CVPixelBufferRef> pixel_buffer_;
#else
GlTextureBufferSharedPtr texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
};
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
inline int GpuBuffer::width() const {
return static_cast<int>(CVPixelBufferGetWidth(*pixel_buffer_));
}
inline int GpuBuffer::height() const {
return static_cast<int>(CVPixelBufferGetHeight(*pixel_buffer_));
}
inline GpuBufferFormat GpuBuffer::format() const {
return GpuBufferFormatForCVPixelFormat(
CVPixelBufferGetPixelFormatType(*pixel_buffer_));
}
inline bool GpuBuffer::operator==(std::nullptr_t other) const {
return pixel_buffer_ == other;
}
inline bool GpuBuffer::operator==(const GpuBuffer& other) const {
return pixel_buffer_ == other.pixel_buffer_;
}
inline GpuBuffer& GpuBuffer::operator=(std::nullptr_t other) {
pixel_buffer_.reset(other);
return *this;
}
#else
inline int GpuBuffer::width() const { return texture_buffer_->width(); }
inline int GpuBuffer::height() const { return texture_buffer_->height(); }
inline GpuBufferFormat GpuBuffer::format() const {
return texture_buffer_->format();
}
inline bool GpuBuffer::operator==(std::nullptr_t other) const {
return texture_buffer_ == other;
}
inline bool GpuBuffer::operator==(const GpuBuffer& other) const {
return texture_buffer_ == other.texture_buffer_;
}
inline GpuBuffer& GpuBuffer::operator=(std::nullptr_t other) {
texture_buffer_ = other;
return *this;
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_H_
mediapipe/gpu/gpu_buffer_format.cc
+223
View File
@@ -0,0 +1,223 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "absl/container/flat_hash_map.h"
#include "mediapipe/framework/deps/no_destructor.h"
#include "mediapipe/framework/port/logging.h"
namespace mediapipe {
#ifndef GL_RGBA16F
#define GL_RGBA16F 34842
#endif // GL_RGBA16F
#ifndef GL_HALF_FLOAT
#define GL_HALF_FLOAT 0x140B
#endif // GL_HALF_FLOAT
#ifdef GL_ES_VERSION_2_0
static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) {
switch (info->gl_internal_format) {
case GL_R16F:
case GL_R32F:
// Should this be GL_RED_EXT instead?
info->gl_internal_format = info->gl_format = GL_LUMINANCE;
return;
case GL_RG16F:
// Should this be GL_RG_EXT instead?
info->gl_internal_format = info->gl_format = GL_LUMINANCE_ALPHA;
return;
case GL_R8:
info->gl_internal_format = info->gl_format = GL_RED_EXT;
return;
case GL_RG8:
info->gl_internal_format = info->gl_format = GL_RG_EXT;
return;
default:
return;
}
}
#endif // GL_ES_VERSION_2_0
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
int plane) {
#if defined(__APPLE__) && TARGET_OS_OSX
constexpr GlVersion default_version = GlVersion::kGL;
#else
constexpr GlVersion default_version = GlVersion::kGLES3;
#endif // defined(__APPLE__) && TARGET_OS_OSX
return GlTextureInfoForGpuBufferFormat(format, plane, default_version);
}
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
int plane,
GlVersion gl_version) {
// TODO: check/add more cases using info from
// CVPixelFormatDescriptionCreateWithPixelFormatType.
static const mediapipe::NoDestructor<
absl::flat_hash_map<GpuBufferFormat, std::vector<GlTextureInfo>>>
gles3_format_info{{
{GpuBufferFormat::kBGRA32,
{
// internal_format, format, type, downscale
#ifdef __APPLE__
// On Apple platforms, the preferred transfer format is BGRA.
{GL_RGBA, GL_BGRA, GL_UNSIGNED_BYTE, 1},
#else
{GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE, 1},
#endif // __APPLE__
}},
{GpuBufferFormat::kOneComponent8,
{
// This should be GL_RED, but it would change the output for existing
// shaders. It would not be a good representation of a grayscale texture,
// unless we use texture swizzling. We could add swizzle parameters (e.g.
// GL_TEXTURE_SWIZZLE_R) in GLES 3 and desktop GL, and use GL_LUMINANCE
// in GLES 2. Or we could just punt and make it a red texture.
// {GL_R8, GL_RED, GL_UNSIGNED_BYTE, 1},
#if !TARGET_OS_OSX
{GL_LUMINANCE, GL_LUMINANCE, GL_UNSIGNED_BYTE, 1},
#endif // TARGET_OS_OSX
}},
#ifdef __APPLE__
// TODO: figure out GL_RED_EXT etc. on Android.
{GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange,
{
// Apple's documentation suggests GL_LUMINANCE and
// GL_LUMINANCE_ALPHA,
// but since they are deprecated in later versions of OpenGL, we
// use
// GL_RED and GL_RG. On GLES2 we can use GL_RED_EXT and GL_RG_EXT
// instead, though we are not sure if it may cause compatibility
// problems
// with very old devices.
{GL_R8, GL_RED, GL_UNSIGNED_BYTE, 1},
{GL_RG8, GL_RG, GL_UNSIGNED_BYTE, 2},
}},
{GpuBufferFormat::kBiPlanar420YpCbCr8FullRange,
{
{GL_R8, GL_RED, GL_UNSIGNED_BYTE, 1},
{GL_RG8, GL_RG, GL_UNSIGNED_BYTE, 2},
}},
#endif // __APPLE__
{GpuBufferFormat::kTwoComponentHalf16,
{
// TODO: use GL_HALF_FLOAT_OES on GLES2?
{GL_RG16F, GL_RG, GL_HALF_FLOAT, 1},
}},
{GpuBufferFormat::kGrayHalf16,
{
{GL_R16F, GL_RED, GL_HALF_FLOAT, 1},
}},
{GpuBufferFormat::kGrayFloat32,
{
{GL_R32F, GL_RED, GL_FLOAT, 1},
}},
{GpuBufferFormat::kRGB24,
{
{GL_RGB, GL_RGB, GL_UNSIGNED_BYTE, 1},
}},
{GpuBufferFormat::kRGBAHalf64,
{
{GL_RGBA16F, GL_RGBA, GL_HALF_FLOAT, 1},
}},
{GpuBufferFormat::kRGBAFloat128,
{
{GL_RGBA, GL_RGBA, GL_FLOAT, 1},
}},
}};
static const auto* gles2_format_info = ([] {
auto formats =
new absl::flat_hash_map<GpuBufferFormat, std::vector<GlTextureInfo>>(
*gles3_format_info);
#ifdef GL_ES_VERSION_2_0
for (auto& format_planes : *formats) {
for (auto& info : format_planes.second) {
AdaptGlTextureInfoForGLES2(&info);
}
}
#endif // GL_ES_VERSION_2_0
return formats;
})();
auto* format_info = gles3_format_info.get();
switch (gl_version) {
case GlVersion::kGLES2:
format_info = gles2_format_info;
break;
case GlVersion::kGLES3:
case GlVersion::kGL:
break;
}
auto iter = format_info->find(format);
CHECK(iter != format_info->end()) << "unsupported format";
const auto& planes = iter->second;
#ifndef __APPLE__
CHECK_EQ(planes.size(), 1)
<< "multiplanar formats are not supported on this platform";
#endif
CHECK_GE(plane, 0) << "invalid plane number";
CHECK_LT(plane, planes.size()) << "invalid plane number";
return planes[plane];
}
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
switch (format) {
case GpuBufferFormat::kBGRA32:
// TODO: verify we are handling order of channels correctly.
return ImageFormat::SRGBA;
case GpuBufferFormat::kGrayFloat32:
return ImageFormat::VEC32F1;
case GpuBufferFormat::kOneComponent8:
return ImageFormat::GRAY8;
case GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange:
case GpuBufferFormat::kBiPlanar420YpCbCr8FullRange:
// TODO: should either of these be YCBCR420P10?
return ImageFormat::YCBCR420P;
case GpuBufferFormat::kRGB24:
return ImageFormat::SRGB;
case GpuBufferFormat::kGrayHalf16:
case GpuBufferFormat::kTwoComponentHalf16:
case GpuBufferFormat::kRGBAHalf64:
case GpuBufferFormat::kRGBAFloat128:
case GpuBufferFormat::kUnknown:
return ImageFormat::UNKNOWN;
}
}
GpuBufferFormat GpuBufferFormatForImageFormat(ImageFormat::Format format) {
switch (format) {
case ImageFormat::SRGB:
return GpuBufferFormat::kRGB24;
case ImageFormat::SRGBA:
// TODO: verify we are handling order of channels correctly.
return GpuBufferFormat::kBGRA32;
case ImageFormat::VEC32F1:
return GpuBufferFormat::kGrayFloat32;
case ImageFormat::GRAY8:
return GpuBufferFormat::kOneComponent8;
case ImageFormat::YCBCR420P:
// TODO: or video range?
return GpuBufferFormat::kBiPlanar420YpCbCr8FullRange;
case ImageFormat::UNKNOWN:
default:
return GpuBufferFormat::kUnknown;
}
}
} // namespace mediapipe
mediapipe/gpu/gpu_buffer_format.h
+133
View File
@@ -0,0 +1,133 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_FORMAT_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_FORMAT_H_
#ifdef __APPLE__
#include <CoreVideo/CoreVideo.h>
#endif // defined(__APPLE__)
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/gpu/gl_base.h"
// The behavior of multi-char constants is implementation-defined, so out of an
// excess of caution we define them in this portable way.
#define MEDIAPIPE_FOURCC(a, b, c, d) \
(((a) << 24) + ((b) << 16) + ((c) << 8) + (d))
namespace mediapipe {
enum class GpuBufferFormat : uint32_t {
kUnknown = 0,
kBGRA32 = MEDIAPIPE_FOURCC('B', 'G', 'R', 'A'),
kGrayFloat32 = MEDIAPIPE_FOURCC('L', '0', '0', 'f'),
kGrayHalf16 = MEDIAPIPE_FOURCC('L', '0', '0', 'h'),
kOneComponent8 = MEDIAPIPE_FOURCC('L', '0', '0', '8'),
kTwoComponentHalf16 = MEDIAPIPE_FOURCC('2', 'C', '0', 'h'),
kBiPlanar420YpCbCr8VideoRange = MEDIAPIPE_FOURCC('4', '2', '0', 'v'),
kBiPlanar420YpCbCr8FullRange = MEDIAPIPE_FOURCC('4', '2', '0', 'f'),
kRGB24 = 0x00000018, // Note: prefer BGRA32 whenever possible.
kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'),
kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'),
};
// TODO: make this more generally applicable.
enum class GlVersion {
kGL = 1,
kGLES2 = 2,
kGLES3 = 3,
};
struct GlTextureInfo {
GLint gl_internal_format;
GLenum gl_format;
GLenum gl_type;
// For multiplane buffers, this represents how many times smaller than
// the nominal image size a plane is.
int downscale;
};
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
int plane);
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
int plane,
GlVersion gl_version);
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format);
GpuBufferFormat GpuBufferFormatForImageFormat(ImageFormat::Format format);
#ifdef __APPLE__
inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) {
switch (format) {
case GpuBufferFormat::kBGRA32:
return kCVPixelFormatType_32BGRA;
case GpuBufferFormat::kGrayHalf16:
return kCVPixelFormatType_OneComponent16Half;
case GpuBufferFormat::kGrayFloat32:
return kCVPixelFormatType_OneComponent32Float;
case GpuBufferFormat::kOneComponent8:
return kCVPixelFormatType_OneComponent8;
case GpuBufferFormat::kTwoComponentHalf16:
return kCVPixelFormatType_TwoComponent16Half;
case GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange:
return kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
case GpuBufferFormat::kBiPlanar420YpCbCr8FullRange:
return kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
case GpuBufferFormat::kRGB24:
return kCVPixelFormatType_24RGB;
case GpuBufferFormat::kRGBAHalf64:
return kCVPixelFormatType_64RGBAHalf;
case GpuBufferFormat::kRGBAFloat128:
return kCVPixelFormatType_128RGBAFloat;
case GpuBufferFormat::kUnknown:
return -1;
}
return -1;
}
inline GpuBufferFormat GpuBufferFormatForCVPixelFormat(OSType format) {
switch (format) {
case kCVPixelFormatType_32BGRA:
return GpuBufferFormat::kBGRA32;
case kCVPixelFormatType_DepthFloat32:
return GpuBufferFormat::kGrayFloat32;
case kCVPixelFormatType_OneComponent16Half:
return GpuBufferFormat::kGrayHalf16;
case kCVPixelFormatType_OneComponent32Float:
return GpuBufferFormat::kGrayFloat32;
case kCVPixelFormatType_OneComponent8:
return GpuBufferFormat::kOneComponent8;
case kCVPixelFormatType_TwoComponent16Half:
return GpuBufferFormat::kTwoComponentHalf16;
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
return GpuBufferFormat::kBiPlanar420YpCbCr8VideoRange;
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
return GpuBufferFormat::kBiPlanar420YpCbCr8FullRange;
case kCVPixelFormatType_24RGB:
return GpuBufferFormat::kRGB24;
case kCVPixelFormatType_64RGBAHalf:
return GpuBufferFormat::kRGBAHalf64;
case kCVPixelFormatType_128RGBAFloat:
return GpuBufferFormat::kRGBAFloat128;
}
return GpuBufferFormat::kUnknown;
}
#endif // __APPLE__
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_FORMAT_H_
mediapipe/gpu/gpu_buffer_multi_pool.cc
+150
View File
@@ -0,0 +1,150 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gpu_buffer_multi_pool.h"
#include <tuple>
#include "absl/memory/memory.h"
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#ifdef __APPLE__
#include "mediapipe/framework/ios/CFHolder.h"
#endif // __APPLE__
namespace mediapipe {
// Keep this many buffers allocated for a given frame size.
static constexpr int kKeepCount = 2;
// The maximum size of the GpuBufferMultiPool. When the limit is reached, the
// oldest BufferSpec will be dropped.
static constexpr int kMaxPoolCount = 20;
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GpuBufferMultiPool::SimplePool GpuBufferMultiPool::MakeSimplePool(
BufferSpec spec) {
OSType cv_format = CVPixelFormatForGpuBufferFormat(spec.format);
CHECK_NE(cv_format, -1) << "unsupported pixel format";
return MakeCFHolderAdopting(
CreateCVPixelBufferPool(spec.width, spec.height, cv_format, kKeepCount,
0.1 /* max age in seconds */));
}
GpuBuffer GpuBufferMultiPool::GetBufferFromSimplePool(
BufferSpec spec, const GpuBufferMultiPool::SimplePool& pool) {
#if TARGET_IPHONE_SIMULATOR
// On the simulator, syncing the texture with the pixelbuffer does not work,
// and we have to use glReadPixels. Since GL_UNPACK_ROW_LENGTH is not
// available in OpenGL ES 2, we should create the buffer so the pixels are
// contiguous.
//
// TODO: verify if we can use kIOSurfaceBytesPerRow to force the
// pool to give us contiguous data.
OSType cv_format = CVPixelFormatForGpuBufferFormat(spec.format);
CHECK_NE(cv_format, -1) << "unsupported pixel format";
CVPixelBufferRef buffer;
CVReturn err = CreateCVPixelBufferWithoutPool(spec.width, spec.height,
cv_format, &buffer);
CHECK(!err) << "Error creating pixel buffer: " << err;
return GpuBuffer(MakeCFHolderAdopting(buffer));
#else
CVPixelBufferRef buffer;
// TODO: allow the keepCount and the allocation threshold to be set
// by the application, and to be set independently.
static CFDictionaryRef auxAttributes =
CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(kKeepCount);
CVReturn err = CreateCVPixelBufferWithPool(
*pool, auxAttributes,
[this]() {
for (const auto& cache : texture_caches_) {
#if TARGET_OS_OSX
CVOpenGLTextureCacheFlush(*cache, 0);
#else
CVOpenGLESTextureCacheFlush(*cache, 0);
#endif // TARGET_OS_OSX
}
},
&buffer);
CHECK(!err) << "Error creating pixel buffer: " << err;
return GpuBuffer(MakeCFHolderAdopting(buffer));
#endif // TARGET_IPHONE_SIMULATOR
}
#else
GpuBufferMultiPool::SimplePool GpuBufferMultiPool::MakeSimplePool(
BufferSpec spec) {
return GlTextureBufferPool::Create(spec.width, spec.height, spec.format,
kKeepCount);
}
GpuBuffer GpuBufferMultiPool::GetBufferFromSimplePool(
BufferSpec spec, const GpuBufferMultiPool::SimplePool& pool) {
return GpuBuffer(pool->GetBuffer());
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GpuBuffer GpuBufferMultiPool::GetBuffer(int width, int height,
GpuBufferFormat format) {
absl::MutexLock lock(&mutex_);
BufferSpec key(width, height, format);
auto pool_it = pools_.find(key);
if (pool_it == pools_.end()) {
// Discard the oldest pool in order of creation.
// TODO: implement a better policy.
if (pools_.size() >= kMaxPoolCount) {
auto old_spec = buffer_specs_.front();
buffer_specs_.pop();
pools_.erase(old_spec);
}
buffer_specs_.push(key);
std::tie(pool_it, std::ignore) =
pools_.emplace(std::piecewise_construct, std::forward_as_tuple(key),
std::forward_as_tuple(MakeSimplePool(key)));
}
return GetBufferFromSimplePool(pool_it->first, pool_it->second);
}
GpuBufferMultiPool::~GpuBufferMultiPool() {
#ifdef __APPLE__
CHECK_EQ(texture_caches_.size(), 0)
<< "Failed to unregister texture caches before deleting pool";
#endif // defined(__APPLE__)
}
#ifdef __APPLE__
void GpuBufferMultiPool::RegisterTextureCache(CVTextureCacheType cache) {
MutexLock lock(&mutex_);
CHECK(std::find(texture_caches_.begin(), texture_caches_.end(), cache) ==
texture_caches_.end())
<< "Attempting to register a texture cache twice";
texture_caches_.emplace_back(cache);
}
void GpuBufferMultiPool::UnregisterTextureCache(CVTextureCacheType cache) {
MutexLock lock(&mutex_);
auto it = std::find(texture_caches_.begin(), texture_caches_.end(), cache);
CHECK(it != texture_caches_.end())
<< "Attempting to unregister an unknown texture cache";
texture_caches_.erase(it);
}
#endif // defined(__APPLE__)
} // namespace mediapipe
mediapipe/gpu/gpu_buffer_multi_pool.h
+123
View File
@@ -0,0 +1,123 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This class lets calculators allocate GpuBuffers of various sizes, caching
// and reusing them as needed. It does so by automatically creating and using
// platform-specific buffer pools for the requested sizes.
//
// This class is not meant to be used directly by calculators, but is instead
// used by GlCalculatorHelper to allocate buffers.
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_MULTI_POOL_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_MULTI_POOL_H_
#include <limits>
#include <queue>
#include <unordered_map>
#include "absl/synchronization/mutex.h"
#include "mediapipe/gpu/gpu_buffer.h"
#ifdef __APPLE__
#include "mediapipe/gpu/pixel_buffer_pool_util.h"
#endif // __APPLE__
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#include "mediapipe/gpu/gl_texture_buffer_pool.h"
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
namespace mediapipe {
class GpuSharedData;
struct BufferSpec {
BufferSpec(int w, int h, GpuBufferFormat f)
: width(w), height(h), format(f) {}
int width;
int height;
GpuBufferFormat format;
};
inline bool operator==(const BufferSpec& lhs, const BufferSpec& rhs) {
return lhs.width == rhs.width && lhs.height == rhs.height &&
lhs.format == rhs.format;
}
inline bool operator!=(const BufferSpec& lhs, const BufferSpec& rhs) {
return !operator==(lhs, rhs);
}
// This generates a "rol" instruction with both Clang and GCC.
static inline std::size_t RotateLeft(std::size_t x, int n) {
return (x << n) | (x >> (std::numeric_limits<size_t>::digits - n));
}
struct BufferSpecHash {
std::size_t operator()(const mediapipe::BufferSpec& spec) const {
// Width and height are expected to be smaller than half the width of
// size_t. We can combine them into a single integer, and then use
// std::hash, which is what go/hashing recommends for hashing numbers.
constexpr int kWidth = std::numeric_limits<size_t>::digits;
return std::hash<std::size_t>{}(
spec.width ^ RotateLeft(spec.height, kWidth / 2) ^
RotateLeft(static_cast<uint32_t>(spec.format), kWidth / 4));
}
};
class GpuBufferMultiPool {
public:
GpuBufferMultiPool() {}
explicit GpuBufferMultiPool(void* ignored) {}
~GpuBufferMultiPool();
// Obtains a buffer. May either be reused or created anew.
GpuBuffer GetBuffer(int width, int height,
GpuBufferFormat format = GpuBufferFormat::kBGRA32);
#ifdef __APPLE__
// TODO: add tests for the texture cache registration.
// Inform the pool of a cache that should be flushed when it is low on
// reusable buffers.
void RegisterTextureCache(CVTextureCacheType cache);
// Remove a texture cache from the list of caches to be flushed.
void UnregisterTextureCache(CVTextureCacheType cache);
#endif // defined(__APPLE__)
private:
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
typedef CFHolder<CVPixelBufferPoolRef> SimplePool;
#else
typedef std::shared_ptr<GlTextureBufferPool> SimplePool;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
SimplePool MakeSimplePool(BufferSpec spec);
GpuBuffer GetBufferFromSimplePool(BufferSpec spec, const SimplePool& pool);
absl::Mutex mutex_;
std::unordered_map<BufferSpec, SimplePool, BufferSpecHash> pools_
GUARDED_BY(mutex_);
// A queue of BufferSpecs to keep track of the age of each BufferSpec added to
// the pool.
std::queue<BufferSpec> buffer_specs_;
#ifdef __APPLE__
// Texture caches used with this pool.
std::vector<CFHolder<CVTextureCacheType>> texture_caches_ GUARDED_BY(mutex_);
#endif // defined(__APPLE__)
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_MULTI_POOL_H_
mediapipe/gpu/gpu_buffer_to_image_frame_calculator.cc
+106
View File
@@ -0,0 +1,106 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#define HAVE_GPU_BUFFER
#ifdef __APPLE__
#include "mediapipe/framework/ios/util.h"
#endif
#include "mediapipe/gpu/gl_calculator_helper.h"
namespace mediapipe {
// Convert an input image (GpuBuffer or ImageFrame) to ImageFrame.
class GpuBufferToImageFrameCalculator : public CalculatorBase {
public:
GpuBufferToImageFrameCalculator() {}
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlCalculatorHelper helper_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
};
REGISTER_CALCULATOR(GpuBufferToImageFrameCalculator);
// static
::mediapipe::Status GpuBufferToImageFrameCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Index(0).SetAny();
cc->Outputs().Index(0).Set<ImageFrame>();
// Note: we call this method even on platforms where we don't use the helper,
// to ensure the calculator's contract is the same. In particular, the helper
// enables support for the legacy side packet, which several graphs still use.
RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
return ::mediapipe::OkStatus();
}
::mediapipe::Status GpuBufferToImageFrameCalculator::Open(
CalculatorContext* cc) {
// Inform the framework that we always output at the same timestamp
// as we receive a packet at.
cc->SetOffset(TimestampDiff(0));
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
RETURN_IF_ERROR(helper_.Open(cc));
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return ::mediapipe::OkStatus();
}
::mediapipe::Status GpuBufferToImageFrameCalculator::Process(
CalculatorContext* cc) {
if (cc->Inputs().Index(0).Value().ValidateAsType<ImageFrame>().ok()) {
cc->Outputs().Index(0).AddPacket(cc->Inputs().Index(0).Value());
return ::mediapipe::OkStatus();
}
#ifdef HAVE_GPU_BUFFER
if (cc->Inputs().Index(0).Value().ValidateAsType<GpuBuffer>().ok()) {
const auto& input = cc->Inputs().Index(0).Get<GpuBuffer>();
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
std::unique_ptr<ImageFrame> frame =
CreateImageFrameForCVPixelBuffer(input.GetCVPixelBufferRef());
cc->Outputs().Index(0).Add(frame.release(), cc->InputTimestamp());
#else
helper_.RunInGlContext([this, &input, &cc]() {
auto src = helper_.CreateSourceTexture(input);
std::unique_ptr<ImageFrame> frame = absl::make_unique<ImageFrame>(
ImageFormatForGpuBufferFormat(input.format()), src.width(),
src.height(), ImageFrame::kGlDefaultAlignmentBoundary);
helper_.BindFramebuffer(src);
const auto info = GlTextureInfoForGpuBufferFormat(input.format(), 0);
glReadPixels(0, 0, src.width(), src.height(), info.gl_format,
info.gl_type, frame->MutablePixelData());
glFlush();
cc->Outputs().Index(0).Add(frame.release(), cc->InputTimestamp());
src.Release();
});
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return ::mediapipe::OkStatus();
}
#endif // defined(HAVE_GPU_BUFFER)
return ::mediapipe::Status(::mediapipe::StatusCode::kInvalidArgument,
"Input packets must be ImageFrame or GpuBuffer.");
}
} // namespace mediapipe
mediapipe/gpu/gpu_service.cc
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gpu_service.h"
namespace mediapipe {
const GraphService<::mediapipe::GpuResources> kGpuService("kGpuService");
} // namespace mediapipe
mediapipe/gpu/gpu_service.h
+30
View File
@@ -0,0 +1,30 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GPU_SERVICE_H_
#define MEDIAPIPE_GPU_GPU_SERVICE_H_
#include "mediapipe/framework/graph_service.h"
namespace mediapipe {
class GpuResources;
} // namespace mediapipe
namespace mediapipe {
extern const GraphService<::mediapipe::GpuResources> kGpuService;
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_SERVICE_H_
mediapipe/gpu/gpu_shared_data_internal.cc
+176
View File
@@ -0,0 +1,176 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#include "mediapipe/framework/deps/no_destructor.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/gpu/gl_context_options.pb.h"
#include "mediapipe/gpu/graph_support.h"
#if __APPLE__
#import "mediapipe/gpu/MediaPipeGraphGPUData.h"
#endif // __APPLE__
namespace mediapipe {
#if __APPLE__
static constexpr bool kGlContextUseDedicatedThread = false;
#elif defined(__EMSCRIPTEN__)
// Since we're forcing single-threaded execution, we just run everything
// in-place.
static constexpr bool kGlContextUseDedicatedThread = false;
#else
// TODO: in theory this is only needed on Android. In practice,
// when using SwiftShader on Linux, we get memory leaks if we attempt to get
// the current GL context on a random thread. For now let's keep the
// single-thread approach on Linux as a workaround.
static constexpr bool kGlContextUseDedicatedThread = true;
#endif // __APPLE__
// If true, use a single GL context shared by all calculators.
// If false, create a separate context per calculator.
// Context-per-calculator (i.e. setting this to false) is not fully supported,
// and it is only known to work on iOS.
static constexpr bool kGlCalculatorShareContext = true;
// Allow a GlContext to be used as an Executor. This makes it possible to run
// GPU-based calculators directly on the GlContext thread, avoiding two thread
// switches.
class GlContextExecutor : public Executor {
public:
explicit GlContextExecutor(GlContext* gl_context) : gl_context_(gl_context) {}
~GlContextExecutor() override {}
void Schedule(std::function<void()> task) override {
gl_context_->RunWithoutWaiting(std::move(task));
}
private:
GlContext* const gl_context_;
};
static const std::string& SharedContextKey() {
static const mediapipe::NoDestructor<std::string> kSharedContextKey("");
return *kSharedContextKey;
}
GpuResources::StatusOrGpuResources GpuResources::Create() {
return Create(kPlatformGlContextNone);
}
GpuResources::StatusOrGpuResources GpuResources::Create(
PlatformGlContext external_context) {
ASSIGN_OR_RETURN(
std::shared_ptr<GlContext> context,
GlContext::Create(external_context, kGlContextUseDedicatedThread));
std::shared_ptr<GpuResources> gpu_resources(
new GpuResources(std::move(context)));
return gpu_resources;
}
GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context) {
gl_key_context_[SharedContextKey()] = gl_context;
named_executors_[kGpuExecutorName] =
std::make_shared<GlContextExecutor>(gl_context.get());
#if __APPLE__
gpu_buffer_pool().RegisterTextureCache(gl_context->cv_texture_cache());
ios_gpu_data_ =
[[MediaPipeGraphGPUData alloc] initWithContext:gl_context.get()
multiPool:&gpu_buffer_pool_];
#endif // __APPLE__
}
GpuResources::~GpuResources() {
#if __APPLE__
// Note: on Apple platforms, this object contains Objective-C objects. The
// destructor will release them, but ARC must be on.
#if !__has_feature(objc_arc)
#error This file must be built with ARC.
#endif
for (auto& kv : gl_key_context_) {
gpu_buffer_pool().UnregisterTextureCache(kv.second->cv_texture_cache());
}
#endif
}
void GpuResources::PrepareGpuNode(CalculatorNode* node) {
CHECK(node->UsesGpu());
std::string node_id = node->GetCalculatorState().NodeName();
std::string node_type = node->GetCalculatorState().CalculatorType();
std::string context_key;
// TODO Allow calculators to request a separate context.
// For now, white-list a few calculators to run in their own context.
bool gets_own_context = (node_type == "ImageFrameToGpuBufferCalculator") ||
(node_type == "GpuBufferToImageFrameCalculator") ||
(node_type == "GlSurfaceSinkCalculator");
const auto& options = node->GetCalculatorState().Options<GlContextOptions>();
if (options.has_gl_context_name() && !options.gl_context_name().empty()) {
context_key = absl::StrCat("user:", options.gl_context_name());
} else if (gets_own_context) {
context_key = absl::StrCat("auto:", node_type);
} else if (kGlCalculatorShareContext) {
context_key = SharedContextKey();
} else {
context_key = absl::StrCat("auto:", node_id);
}
node_key_[node_id] = context_key;
if (kGlContextUseDedicatedThread) {
std::string executor_name =
absl::StrCat(kGpuExecutorName, "_", context_key);
node->SetExecutor(executor_name);
if (!ContainsKey(named_executors_, executor_name)) {
named_executors_.emplace(
executor_name,
std::make_shared<GlContextExecutor>(gl_context(context_key).get()));
}
}
gl_context(context_key)
->SetProfilingContext(
node->GetCalculatorState().GetSharedProfilingContext());
}
// TODO: expose and use an actual ID instead of using the
// canonicalized name.
const std::shared_ptr<GlContext>& GpuResources::gl_context(
CalculatorContext* cc) {
return gl_context(cc ? node_key_[cc->NodeName()] : SharedContextKey());
}
const std::shared_ptr<GlContext>& GpuResources::gl_context(
const std::string& key) {
auto it = gl_key_context_.find(key);
if (it == gl_key_context_.end()) {
it = gl_key_context_
.emplace(key,
GlContext::Create(*gl_key_context_[SharedContextKey()],
kGlContextUseDedicatedThread)
.ValueOrDie())
.first;
#if __APPLE__
gpu_buffer_pool_.RegisterTextureCache(it->second->cv_texture_cache());
#endif
}
return it->second;
}
GpuSharedData::GpuSharedData() : GpuSharedData(kPlatformGlContextNone) {}
#if __APPLE__
MediaPipeGraphGPUData* GpuResources::ios_gpu_data() { return ios_gpu_data_; }
#endif // __APPLE__
} // namespace mediapipe
mediapipe/gpu/gpu_shared_data_internal.h
+133
View File
@@ -0,0 +1,133 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Declares GpuSharedData, a private object that is used to store
// platform-specific resources shared by GPU calculators across a graph.
// Consider this file an implementation detail. None of this is part of the
// public API.
#ifndef MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
#define MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_node.h"
#include "mediapipe/framework/executor.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gpu_buffer_multi_pool.h"
#ifdef __APPLE__
#ifdef __OBJC__
@class MediaPipeGraphGPUData;
#else
struct MediaPipeGraphGPUData;
#endif // __OBJC__
#endif // defined(__APPLE__)
namespace mediapipe {
// TODO: rename to GpuService or GpuManager or something.
class GpuResources {
public:
using StatusOrGpuResources =
::mediapipe::StatusOr<std::shared_ptr<GpuResources>>;
static StatusOrGpuResources Create();
static StatusOrGpuResources Create(PlatformGlContext external_context);
// The destructor must be defined in the implementation file so that on iOS
// the correct ARC release calls are generated.
~GpuResources();
explicit GpuResources(PlatformGlContext external_context);
// Shared GL context for calculators.
// TODO: require passing a context or node identifier.
const std::shared_ptr<GlContext>& gl_context() {
return gl_context(nullptr);
};
const std::shared_ptr<GlContext>& gl_context(CalculatorContext* cc);
// Shared buffer pool.
GpuBufferMultiPool& gpu_buffer_pool() { return gpu_buffer_pool_; }
#ifdef __APPLE__
MediaPipeGraphGPUData* ios_gpu_data();
#endif // defined(__APPLE__)
void PrepareGpuNode(CalculatorNode* node);
// If the node requires custom GPU executors in the current configuration,
// returns the executor's names and the executors themselves.
const std::map<std::string, std::shared_ptr<Executor>>& GetGpuExecutors() {
return named_executors_;
}
private:
GpuResources() = delete;
explicit GpuResources(std::shared_ptr<GlContext> gl_context);
const std::shared_ptr<GlContext>& gl_context(const std::string& key);
const std::string& ContextKey(const std::string& canonical_node_name);
std::map<std::string, std::string> node_key_;
std::map<std::string, std::shared_ptr<GlContext>> gl_key_context_;
// The pool must be destructed before the gl_context, but after the
// ios_gpu_data, so the declaration order is important.
GpuBufferMultiPool gpu_buffer_pool_;
#ifdef __APPLE__
// Note that this is an Objective-C object.
MediaPipeGraphGPUData* ios_gpu_data_;
#endif // defined(__APPLE__)
std::map<std::string, std::shared_ptr<Executor>> named_executors_;
};
// Legacy struct to keep existing client code happy.
// TODO: eliminate!
struct GpuSharedData {
GpuSharedData();
explicit GpuSharedData(PlatformGlContext external_context)
: GpuSharedData(CreateGpuResourcesOrDie(external_context)) {}
explicit GpuSharedData(std::shared_ptr<GpuResources> gpu_resources)
: gpu_resources(gpu_resources),
gl_context(gpu_resources->gl_context()),
gpu_buffer_pool(gpu_resources->gpu_buffer_pool()) {}
std::shared_ptr<GpuResources> gpu_resources;
std::shared_ptr<GlContext> gl_context;
GpuBufferMultiPool& gpu_buffer_pool;
private:
static std::shared_ptr<GpuResources> CreateGpuResourcesOrDie(
PlatformGlContext external_context) {
auto status_or_resources = GpuResources::Create(external_context);
MEDIAPIPE_CHECK_OK(status_or_resources.status())
<< "could not create GpuResources";
return std::move(status_or_resources).ValueOrDie();
}
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
mediapipe/gpu/graph_support.h
+29
View File
@@ -0,0 +1,29 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// TODO: Update all the reference and delete this forwarding header.
#ifndef MEDIAPIPE_GPU_GRAPH_SUPPORT_H_
#define MEDIAPIPE_GPU_GRAPH_SUPPORT_H_
#include "mediapipe/gpu/gpu_service.h"
namespace mediapipe {
static constexpr char kGpuSharedTagName[] = "GPU_SHARED";
static constexpr char kGpuSharedSidePacketName[] = "gpu_shared";
static constexpr char kGpuExecutorName[] = "__gpu";
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GRAPH_SUPPORT_H_
mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc
+86
View File
@@ -0,0 +1,86 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#ifdef __APPLE__
#include "mediapipe/framework/ios/util.h"
#endif
namespace mediapipe {
// Convert ImageFrame to GpuBuffer.
class ImageFrameToGpuBufferCalculator : public CalculatorBase {
public:
ImageFrameToGpuBufferCalculator() {}
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
private:
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GlCalculatorHelper helper_;
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
};
REGISTER_CALCULATOR(ImageFrameToGpuBufferCalculator);
// static
::mediapipe::Status ImageFrameToGpuBufferCalculator::GetContract(
CalculatorContract* cc) {
cc->Inputs().Index(0).Set<ImageFrame>();
cc->Outputs().Index(0).Set<GpuBuffer>();
// Note: we call this method even on platforms where we don't use the helper,
// to ensure the calculator's contract is the same. In particular, the helper
// enables support for the legacy side packet, which several graphs still use.
RETURN_IF_ERROR(GlCalculatorHelper::UpdateContract(cc));
return ::mediapipe::OkStatus();
}
::mediapipe::Status ImageFrameToGpuBufferCalculator::Open(
CalculatorContext* cc) {
// Inform the framework that we always output at the same timestamp
// as we receive a packet at.
cc->SetOffset(TimestampDiff(0));
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
RETURN_IF_ERROR(helper_.Open(cc));
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return ::mediapipe::OkStatus();
}
::mediapipe::Status ImageFrameToGpuBufferCalculator::Process(
CalculatorContext* cc) {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CFHolder<CVPixelBufferRef> buffer;
RETURN_IF_ERROR(CreateCVPixelBufferForImageFramePacket(
cc->Inputs().Index(0).Value(), &buffer));
cc->Outputs().Index(0).Add(new GpuBuffer(buffer), cc->InputTimestamp());
#else
const auto& input = cc->Inputs().Index(0).Get<ImageFrame>();
helper_.RunInGlContext([this, &input, &cc]() {
auto src = helper_.CreateSourceTexture(input);
auto output = src.GetFrame<GpuBuffer>();
glFlush();
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
src.Release();
});
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return ::mediapipe::OkStatus();
}
} // namespace mediapipe
mediapipe/gpu/scale_mode.proto
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
// We wrap the enum in a message to avoid namespace collisions.
message ScaleMode {
// This enum mirrors the ScaleModes supported by Quad Renderer.
enum Mode {
DEFAULT = 0;
// Stretch the frame to the exact provided output dimensions.
STRETCH = 1;
// Scale the frame up to fit the drawing area, preserving aspect ratio; may
// letterbox.
FIT = 2;
// Scale the frame up to fill the drawing area, preserving aspect ratio; may
// crop.
FILL_AND_CROP = 3;
}
}
mediapipe/gpu/shader_util.cc
+191
View File
@@ -0,0 +1,191 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/gpu/shader_util.h"
#include <stdlib.h>
#include "mediapipe/framework/port/logging.h"
#if DEBUG
#define GL_DEBUG_LOG(type, object, action) \
do { \
GLint log_length = 0; \
glGet##type##iv(object, GL_INFO_LOG_LENGTH, &log_length); \
if (log_length > 0) { \
GLchar* log = static_cast<GLchar*>(malloc(log_length)); \
glGet##type##InfoLog(object, log_length, &log_length, log); \
LOG(INFO) << #type " " action " log:\n" << log; \
free(log); \
} \
} while (0)
#else
#define GL_DEBUG_LOG(type, object, action)
#endif
#define GL_ERROR_LOG(type, object, action) \
do { \
GLint log_length = 0; \
glGet##type##iv(object, GL_INFO_LOG_LENGTH, &log_length); \
if (log_length > 0) { \
GLchar* log = static_cast<GLchar*>(malloc(log_length)); \
glGet##type##InfoLog(object, log_length, &log_length, log); \
LOG(ERROR) << #type " " action " log:\n" << log; \
free(log); \
} \
} while (0)
namespace mediapipe {
constexpr int kMaxShaderInfoLength = 1024;
GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader) {
GLint status;
*shader = glCreateShader(target);
if (*shader == 0) {
return GL_FALSE;
}
glShaderSource(*shader, 1, &source, NULL);
glCompileShader(*shader);
GL_DEBUG_LOG(Shader, *shader, "compile");
glGetShaderiv(*shader, GL_COMPILE_STATUS, &status);
LOG_IF(ERROR, status == GL_FALSE) << "Failed to compile shader:\n" << source;
if (status == GL_FALSE) {
int length = 0;
GLchar cmessage[kMaxShaderInfoLength];
glGetShaderInfoLog(*shader, kMaxShaderInfoLength, &length, cmessage);
LOG(ERROR) << "Error message: " << std::string(cmessage, length);
}
return status;
}
GLint GlhLinkProgram(GLuint program) {
GLint status;
glLinkProgram(program);
GL_DEBUG_LOG(Program, program, "link");
glGetProgramiv(program, GL_LINK_STATUS, &status);
LOG_IF(ERROR, status == GL_FALSE) << "Failed to link program " << program;
return status;
}
GLint GlhValidateProgram(GLuint program) {
GLint status;
glValidateProgram(program);
GL_DEBUG_LOG(Program, program, "validate");
glGetProgramiv(program, GL_VALIDATE_STATUS, &status);
LOG_IF(ERROR, status == GL_FALSE) << "Failed to validate program " << program;
return status;
}
GLint GlhCreateProgram(const GLchar* vert_src, const GLchar* frag_src,
GLsizei attr_count, const GLchar* const* attr_names,
const GLint* attr_locations, GLuint* program) {
GLuint vert_shader = 0;
GLuint frag_shader = 0;
GLint ok = GL_TRUE;
*program = glCreateProgram();
if (*program == 0) {
return GL_FALSE;
}
ok = ok && GlhCompileShader(GL_VERTEX_SHADER, vert_src, &vert_shader);
ok = ok && GlhCompileShader(GL_FRAGMENT_SHADER, frag_src, &frag_shader);
if (ok) {
glAttachShader(*program, vert_shader);
glAttachShader(*program, frag_shader);
// Attribute location binding must be set before linking.
for (int i = 0; i < attr_count; i++) {
glBindAttribLocation(*program, attr_locations[i], attr_names[i]);
}
ok = GlhLinkProgram(*program);
}
if (vert_shader) glDeleteShader(vert_shader);
if (frag_shader) glDeleteShader(frag_shader);
if (!ok) {
glDeleteProgram(*program);
*program = 0;
}
return ok;
}
bool CompileShader(GLenum shader_type, const std::string& shader_source,
GLuint* shader) {
*shader = glCreateShader(shader_type);
if (*shader == 0) {
VLOG(2) << "Unable to create shader of type: " << shader_type;
return false;
}
const char* shader_source_cstr = shader_source.c_str();
glShaderSource(*shader, 1, &shader_source_cstr, NULL);
glCompileShader(*shader);
GLint compiled;
glGetShaderiv(*shader, GL_COMPILE_STATUS, &compiled);
if (!compiled) {
VLOG(2) << "Unable to compile shader:\n" << shader_source;
GL_ERROR_LOG(Shader, *shader, "compile");
glDeleteShader(*shader);
*shader = 0;
return false;
}
return true;
}
bool CreateShaderProgram(
GLuint vertex_shader, GLuint fragment_shader,
const std::unordered_map<GLuint, std::string>& attributes,
GLuint* shader_program) {
*shader_program = glCreateProgram();
if (*shader_program == 0) {
VLOG(2) << "Unable to create shader program";
return false;
}
glAttachShader(*shader_program, vertex_shader);
glAttachShader(*shader_program, fragment_shader);
for (const auto& it : attributes) {
glBindAttribLocation(*shader_program, it.first, it.second.c_str());
}
glLinkProgram(*shader_program);
GLint is_linked = 0;
glGetProgramiv(*shader_program, GL_LINK_STATUS, &is_linked);
if (!is_linked) {
glDeleteProgram(*shader_program);
*shader_program = 0;
return false;
}
return true;
}
} // namespace mediapipe
mediapipe/gpu/shader_util.h
+56
View File
@@ -0,0 +1,56 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_SHADER_UTIL_H_
#define MEDIAPIPE_GPU_SHADER_UTIL_H_
#include <string>
#include <unordered_map>
#include "mediapipe/gpu/gl_base.h"
namespace mediapipe {
// TODO: Remove the C-style helpers.
// Compiles a GLSL shader, logs errors, returns the compile status
// (GL_TRUE for success, GL_FALSE for failure).
GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader);
// Links a GLSL program, logs errors, returns the link status
// (GL_TRUE for success, GL_FALSE for failure).
GLint GlhLinkProgram(GLuint program);
// Validates a GLSL program, logs errors, returns the validate status
// (GL_TRUE for success, GL_FALSE for failure).
GLint GlhValidateProgram(GLuint program);
// Creates a GLSL program by compiling and linking the provided shaders.
// Also obtains the locations of the requested attributes.
// Return GL_TRUE for success, GL_FALSE for failure.
GLint GlhCreateProgram(const GLchar* vert_src, const GLchar* frag_src,
GLsizei attr_count, const GLchar* const* attr_names,
const GLint* attr_locations, GLuint* program);
// Compiles a shader specified by shader_source. Returns true on success.
bool CompileShader(GLenum shader_type, const std::string& shader_source,
GLuint* shader);
// Creates a shader program using the supplied vertex shader, fragment shader
// and attributes and stores in program. Returns true on success.
bool CreateShaderProgram(
GLuint vertex_shader, GLuint fragment_shader,
const std::unordered_map<GLuint, std::string>& attributes, GLuint* program);
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_SHADER_UTIL_H_