talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 4cee4a2c2317fb190680c17e31ebbb03bb73b71c
This commit is contained in:
MediaPipe Team
2020-09-17 11:09:17 -04:00
committed by chuoling
parent 1db91b550a
commit a908d668c7
142 changed files with 40878 additions and 574 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/gpu/BUILD
+22 -12
View File
@@ -159,7 +159,7 @@ cc_library(
"//conditions:default": [],
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc",
"-fobjc-arc", # enable reference-counting
],
}),
visibility = ["//visibility:public"],
@@ -407,20 +407,29 @@ cc_library(
}),
)
cc_library(
name = "gl_texture_buffer_pool",
srcs = ["gl_texture_buffer_pool.cc"],
hdrs = ["gl_texture_buffer_pool.h"],
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_texture_buffer",
":gpu_buffer",
":gpu_shared_data_header",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_node",
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
],
)
cc_library(
name = "gpu_buffer_multi_pool",
srcs = ["gpu_buffer_multi_pool.cc"] + select({
"//conditions:default": [
"gl_texture_buffer_pool.cc",
],
"//mediapipe:ios": [],
"//mediapipe:macos": [
"gl_texture_buffer_pool.cc",
],
}),
srcs = ["gpu_buffer_multi_pool.cc"],
hdrs = ["gpu_buffer_multi_pool.h"] + select({
"//conditions:default": [
"gl_texture_buffer_pool.h",
],
"//mediapipe:ios": [
# The inclusions check does not see that this is provided by
@@ -429,7 +438,6 @@ cc_library(
"pixel_buffer_pool_util.h",
],
"//mediapipe:macos": [
"gl_texture_buffer_pool.h",
],
}),
copts = select({
@@ -452,6 +460,7 @@ cc_library(
] + select({
"//conditions:default": [
":gl_texture_buffer",
":gl_texture_buffer_pool",
],
"//mediapipe:ios": [
":pixel_buffer_pool_util",
@@ -460,6 +469,7 @@ cc_library(
"//mediapipe:macos": [
":pixel_buffer_pool_util",
":gl_texture_buffer",
":gl_texture_buffer_pool",
],
}),
)
mediapipe/gpu/gl_calculator_helper.cc
+6 -1
View File
@@ -14,14 +14,19 @@
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/legacy_calculator_support.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_service.h"
#ifdef __APPLE__
#include "mediapipe/objc/util.h"
#endif
namespace mediapipe {
GlTexture::GlTexture(GLuint name, int width, int height)
mediapipe/gpu/gl_calculator_helper.h
+20 -7
View File
@@ -15,8 +15,6 @@
#ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
#define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_H_
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
@@ -29,6 +27,7 @@
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/graph_support.h"
#ifdef __APPLE__
#include <CoreVideo/CoreVideo.h>
@@ -50,6 +49,8 @@ typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
#endif // __APPLE__
using ImageFrameSharedPtr = std::shared_ptr<ImageFrame>;
// TODO: remove this and Process below, or make Process available
// on Android.
typedef std::function<void(const GlTexture& src, const GlTexture& dst)>
@@ -121,7 +122,7 @@ class GlCalculatorHelper {
// where it is supported (iOS, for now) they take advantage of memory sharing
// between the CPU and GPU, avoiding memory copies.
// Creates a texture representing an input frame.
// Creates a texture representing an input frame, and manages sync token.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
@@ -137,7 +138,7 @@ class GlCalculatorHelper {
void GetGpuBufferDimensions(const GpuBuffer& pixel_buffer, int* width,
int* height);
// Creates a texture representing an output frame.
// Creates a texture representing an output frame, and manages sync token.
// TODO: This should either return errors or a status.
GlTexture CreateDestinationTexture(
int output_width, int output_height,
@@ -152,11 +153,22 @@ class GlCalculatorHelper {
GlContext& GetGlContext() const;
// Check if the calculator helper has been previously initialized.
bool Initialized() { return impl_ != nullptr; }
private:
std::unique_ptr<GlCalculatorHelperImpl> impl_;
};
// Represents an OpenGL texture.
// Represents an OpenGL texture, and is a 'view' into the memory pool.
// It's more like a GlTextureLock, because it's main purpose (in conjunction
// with the helper): to manage GL sync points in the gl command queue.
//
// This class should be the main way to interface with GL memory within a single
// calculator. This is the preferred way to utilize the memory pool inside of
// the helper, because GlTexture manages efficiently releasing memory back into
// the pool. A GPU backed Image can be extracted from the unerlying
// memory.
class GlTexture {
public:
GlTexture() {}
@@ -170,11 +182,12 @@ class GlTexture {
GLuint name() const { return name_; }
// Returns a buffer that can be sent to another calculator.
// Can be used with GpuBuffer or ImageFrame.
// & manages sync token
// Can be used with GpuBuffer or ImageFrame or Image
template <typename T>
std::unique_ptr<T> GetFrame() const;
// Releases texture memory
// Releases texture memory & manages sync token
void Release();
private:
mediapipe/gpu/gl_calculator_helper_impl.h
+2 -2
View File
@@ -42,10 +42,10 @@ class GlCalculatorHelperImpl {
CalculatorContext* calculator_context);
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer);
GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer);
// Note: multi-plane support is currently only available on iOS.
GlTexture CreateSourceTexture(const GpuBuffer& pixel_buffer, int plane);
GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer, int plane);
// Creates a framebuffer and returns the texture that it is bound to.
GlTexture CreateDestinationTexture(int output_width, int output_height,
mediapipe/gpu/gl_context_egl.cc
+21 -12
View File
@@ -177,6 +177,27 @@ GlContext::StatusOrGlContext GlContext::Create(EGLContext share_context,
}
void GlContext::DestroyContext() {
#ifdef __ANDROID__
if (HasContext()) {
// Detach the current program to work around b/166322604.
auto detach_program = [this] {
GlContext::ContextBinding saved_context;
GetCurrentContextBinding(&saved_context);
// Note: cannot use ThisContextBinding because it calls shared_from_this,
// which is not available during destruction.
if (eglMakeCurrent(display_, surface_, surface_, context_)) {
glUseProgram(0);
} else {
LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase
<< std::hex << eglGetError();
}
return SetCurrentContextBinding(saved_context);
};
auto status = thread_ ? thread_->Run(detach_program) : detach_program();
LOG_IF(ERROR, !status.ok()) << status;
}
#endif // __ANDROID__
if (thread_) {
// Delete thread-local storage.
// TODO: in theory our EglThreadExitCallback should suffice for
@@ -191,18 +212,6 @@ void GlContext::DestroyContext() {
.IgnoreError();
}
#ifdef __ANDROID__
if (HasContext()) {
// Detach the current program to work around b/166322604.
if (eglMakeCurrent(display_, surface_, surface_, context_)) {
glUseProgram(0);
} else {
LOG(ERROR) << "eglMakeCurrent() returned error " << std::showbase
<< std::hex << eglGetError();
}
}
#endif // __ANDROID__
// Destroy the context and surface.
if (IsCurrent()) {
if (!eglMakeCurrent(display_, EGL_NO_SURFACE, EGL_NO_SURFACE,
mediapipe/gpu/gpu_buffer.h
-1
View File
@@ -72,7 +72,6 @@ class GpuBuffer {
int width() const;
int height() const;
GpuBufferFormat format() const;
// Converts to true iff valid.
mediapipe/gpu/gpu_buffer_multi_pool.cc
+3 -3
View File
@@ -44,8 +44,8 @@ static constexpr int kRequestCountScrubInterval = 50;
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
CvPixelBufferPoolWrapper::CvPixelBufferPoolWrapper(const BufferSpec& spec,
CFTimeInterval maxAge) {
CvPixelBufferPoolWrapper::CvPixelBufferPoolWrapper(
const GpuBufferMultiPool::BufferSpec& spec, CFTimeInterval maxAge) {
OSType cv_format = CVPixelFormatForGpuBufferFormat(spec.format);
CHECK_NE(cv_format, -1) << "unsupported pixel format";
pool_ = MakeCFHolderAdopting(
@@ -90,7 +90,7 @@ std::string CvPixelBufferPoolWrapper::GetDebugString() const {
void CvPixelBufferPoolWrapper::Flush() { CVPixelBufferPoolFlush(*pool_, 0); }
GpuBufferMultiPool::SimplePool GpuBufferMultiPool::MakeSimplePool(
const BufferSpec& spec) {
const GpuBufferMultiPool::BufferSpec& spec) {
return std::make_shared<CvPixelBufferPoolWrapper>(spec,
kMaxInactiveBufferAge);
}
mediapipe/gpu/gpu_buffer_multi_pool.h
+55 -50
View File
@@ -40,56 +40,7 @@
namespace mediapipe {
struct GpuSharedData;
struct BufferSpec {
BufferSpec(int w, int h, GpuBufferFormat f)
: width(w), height(h), format(f) {}
int width;
int height;
GpuBufferFormat format;
};
inline bool operator==(const BufferSpec& lhs, const BufferSpec& rhs) {
return lhs.width == rhs.width && lhs.height == rhs.height &&
lhs.format == rhs.format;
}
inline bool operator!=(const BufferSpec& lhs, const BufferSpec& rhs) {
return !operator==(lhs, rhs);
}
// This generates a "rol" instruction with both Clang and GCC.
static inline std::size_t RotateLeft(std::size_t x, int n) {
return (x << n) | (x >> (std::numeric_limits<size_t>::digits - n));
}
struct BufferSpecHash {
std::size_t operator()(const mediapipe::BufferSpec& spec) const {
// Width and height are expected to be smaller than half the width of
// size_t. We can combine them into a single integer, and then use
// std::hash, which is what go/hashing recommends for hashing numbers.
constexpr int kWidth = std::numeric_limits<size_t>::digits;
return std::hash<std::size_t>{}(
spec.width ^ RotateLeft(spec.height, kWidth / 2) ^
RotateLeft(static_cast<uint32_t>(spec.format), kWidth / 4));
}
};
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
class CvPixelBufferPoolWrapper {
public:
CvPixelBufferPoolWrapper(const BufferSpec& spec, CFTimeInterval maxAge);
GpuBuffer GetBuffer(std::function<void(void)> flush);
int GetBufferCount() const { return count_; }
std::string GetDebugString() const;
void Flush();
private:
CFHolder<CVPixelBufferPoolRef> pool_;
int count_ = 0;
};
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
class CvPixelBufferPoolWrapper;
class GpuBufferMultiPool {
public:
@@ -114,6 +65,31 @@ class GpuBufferMultiPool {
void FlushTextureCaches();
#endif // defined(__APPLE__)
// This generates a "rol" instruction with both Clang and GCC.
inline static std::size_t RotateLeft(std::size_t x, int n) {
return (x << n) | (x >> (std::numeric_limits<size_t>::digits - n));
}
struct BufferSpec {
BufferSpec(int w, int h, mediapipe::GpuBufferFormat f)
: width(w), height(h), format(f) {}
int width;
int height;
mediapipe::GpuBufferFormat format;
};
struct BufferSpecHash {
std::size_t operator()(const BufferSpec& spec) const {
// Width and height are expected to be smaller than half the width of
// size_t. We can combine them into a single integer, and then use
// std::hash, which is what go/hashing recommends for hashing numbers.
constexpr int kWidth = std::numeric_limits<size_t>::digits;
return std::hash<std::size_t>{}(
spec.width ^ RotateLeft(spec.height, kWidth / 2) ^
RotateLeft(static_cast<uint32_t>(spec.format), kWidth / 4));
}
};
private:
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
using SimplePool = std::shared_ptr<CvPixelBufferPoolWrapper>;
@@ -175,6 +151,35 @@ class GpuBufferMultiPool {
#endif // defined(__APPLE__)
};
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
class CvPixelBufferPoolWrapper {
public:
CvPixelBufferPoolWrapper(const GpuBufferMultiPool::BufferSpec& spec,
CFTimeInterval maxAge);
GpuBuffer GetBuffer(std::function<void(void)> flush);
int GetBufferCount() const { return count_; }
std::string GetDebugString() const;
void Flush();
private:
CFHolder<CVPixelBufferPoolRef> pool_;
int count_ = 0;
};
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
// BufferSpec equality operators
inline bool operator==(const GpuBufferMultiPool::BufferSpec& lhs,
const GpuBufferMultiPool::BufferSpec& rhs) {
return lhs.width == rhs.width && lhs.height == rhs.height &&
lhs.format == rhs.format;
}
inline bool operator!=(const GpuBufferMultiPool::BufferSpec& lhs,
const GpuBufferMultiPool::BufferSpec& rhs) {
return !operator==(lhs, rhs);
}
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GPU_BUFFER_MULTI_POOL_H_
mediapipe/gpu/gpu_shared_data_internal.cc
+5
View File
@@ -110,6 +110,7 @@ GpuResources::~GpuResources() {
std::string node_type = node->GetCalculatorState().CalculatorType();
std::string context_key;
#ifndef __EMSCRIPTEN__
// TODO Allow calculators to request a separate context.
// For now, white-list a few calculators to run in their own context.
bool gets_own_context = (node_type == "ImageFrameToGpuBufferCalculator") ||
@@ -126,6 +127,10 @@ GpuResources::~GpuResources() {
} else {
context_key = absl::StrCat("auto:", node_id);
}
#else
// On Emscripten we currently do not support multiple contexts.
context_key = SharedContextKey();
#endif // !__EMSCRIPTEN__
node_key_[node_id] = context_key;
ASSIGN_OR_RETURN(std::shared_ptr<GlContext> context,