talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 53a42bf7ad836321123cb7b6c80b0f2e13fbf83e
This commit is contained in:
MediaPipe Team
2020-04-06 19:14:13 -07:00
committed by jqtang
parent 1722d4b8a2
commit a3d36eee32
127 changed files with 3910 additions and 4783 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/gpu/BUILD
+12 -7
View File
@@ -59,7 +59,7 @@ GL_BASE_LINK_OPTS = select({
# runtime. Weak GLESv3 symbols will still be resolved if we
# load it early enough.
],
"//mediapipe:apple": [
"//mediapipe:ios": [
"-framework OpenGLES",
"-framework CoreVideo",
],
@@ -111,7 +111,7 @@ cc_library(
# Note: need the frameworks on Apple platforms to get the headers.
linkopts = select({
"//conditions:default": [],
"//mediapipe:apple": [
"//mediapipe:ios": [
"-framework OpenGLES",
"-framework CoreVideo",
],
@@ -147,7 +147,7 @@ cc_library(
"//conditions:default": [
"gl_context_egl.cc",
],
"//mediapipe:apple": [
"//mediapipe:ios": [
"gl_context_eagl.cc",
],
"//mediapipe:macos": [
@@ -214,7 +214,7 @@ cc_library(
"//conditions:default": [
":gl_texture_buffer",
],
"//mediapipe:apple": [
"//mediapipe:ios": [
"//mediapipe/objc:CFHolder",
],
"//mediapipe:macos": [
@@ -246,9 +246,14 @@ objc_library(
"-Wno-shorten-64-to-32",
],
sdk_frameworks = [
"Accelerate",
"CoreGraphics",
"CoreVideo",
],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/objc:util",
],
)
objc_library(
@@ -408,7 +413,7 @@ cc_library(
"//conditions:default": [
"gl_texture_buffer_pool.cc",
],
"//mediapipe:apple": [],
"//mediapipe:ios": [],
"//mediapipe:macos": [
"gl_texture_buffer_pool.cc",
],
@@ -417,7 +422,7 @@ cc_library(
"//conditions:default": [
"gl_texture_buffer_pool.h",
],
"//mediapipe:apple": [
"//mediapipe:ios": [
# The inclusions check does not see that this is provided by
# pixel_buffer_pool_util, so we include it here too. This is
# b/28066691.
@@ -441,7 +446,7 @@ cc_library(
"//conditions:default": [
":gl_texture_buffer",
],
"//mediapipe:apple": [
"//mediapipe:ios": [
":pixel_buffer_pool_util",
"//mediapipe/objc:CFHolder",
],
mediapipe/gpu/gl_context.cc
+100 -4
View File
@@ -117,6 +117,7 @@ void* GlContext::DedicatedThread::ThreadBody(void* instance) {
void GlContext::DedicatedThread::ThreadBody() {
SetThreadName("mediapipe_gl_runner");
#ifndef __EMSCRIPTEN__
GlThreadCollector::ThreadStarting();
#endif
@@ -276,6 +277,11 @@ bool GlContext::HasGlExtension(absl::string_view extension) const {
absl::string_view version_string(
reinterpret_cast<const char*>(glGetString(GL_VERSION)));
// We will decide later whether we want to use the version numbers we query
// for, or instead derive that information from the context creation result,
// which we cache here.
GLint gl_major_version_from_context_creation = gl_major_version_;
// Let's try getting the numeric version if possible.
glGetIntegerv(GL_MAJOR_VERSION, &gl_major_version_);
GLenum err = glGetError();
@@ -293,6 +299,23 @@ bool GlContext::HasGlExtension(absl::string_view extension) const {
}
}
// If our platform-specific CreateContext already set a major GL version,
// then we use that. Otherwise, we use the queried-for result. We do this
// as a workaround for a Swiftshader on Android bug where the ES2 context
// can report major version 3 instead of 2 when queried. Therefore we trust
// the result from context creation more than from query. See b/152519932
// for more details.
if (gl_major_version_from_context_creation > 0 &&
gl_major_version_ != gl_major_version_from_context_creation) {
LOG(WARNING) << "Requested a context with major GL version "
<< gl_major_version_from_context_creation
<< " but context reports major version " << gl_major_version_
<< ". Setting to " << gl_major_version_from_context_creation
<< ".0";
gl_major_version_ = gl_major_version_from_context_creation;
gl_minor_version_ = 0;
}
LOG(INFO) << "GL version: " << gl_major_version_ << "." << gl_minor_version_
<< " (" << glGetString(GL_VERSION) << ")";
if (gl_major_version_ >= 3) {
@@ -613,7 +636,17 @@ std::shared_ptr<GlSyncPoint> GlContext::CreateSyncToken() {
#if MEDIAPIPE_DISABLE_GL_SYNC_FOR_DEBUG
token.reset(new GlNopSyncPoint(shared_from_this()));
#else
if (SymbolAvailable(&glWaitSync)) {
#ifdef __EMSCRIPTEN__
// In Emscripten the glWaitSync function is non-null depending on linkopts,
// but only works in a WebGL2 context, so fall back to use Finish if it is a
// WebGL1/ES2 context.
// TODO: apply this more generally once b/152794517 is fixed.
bool useFenceSync = gl_major_version() > 2;
#else
bool useFenceSync = SymbolAvailable(&glWaitSync);
#endif // __EMSCRIPTEN__
if (useFenceSync) {
token.reset(new GlFenceSyncPoint(shared_from_this()));
} else {
token.reset(new GlFinishSyncPoint(shared_from_this()));
@@ -633,8 +666,30 @@ std::shared_ptr<GlSyncPoint> GlContext::TestOnly_CreateSpecificSyncToken(
return nullptr;
}
// Atomically set var to the greater of its current value or target.
template <typename T>
static void assign_larger_value(std::atomic<T>* var, T target) {
T current = var->load();
while (current < target && !var->compare_exchange_weak(current, target)) {
}
}
// Note: this can get called from an arbitrary thread which is dealing with a
// GlFinishSyncPoint originating from this context.
void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) {
if (gl_finish_count_ > count_to_pass) return;
// If we've been asked to do a glFinish, note the count we need to reach and
// signal the context our thread may currently be blocked on.
{
absl::MutexLock lock(&mutex_);
assign_larger_value(&gl_finish_count_target_, count_to_pass + 1);
wait_for_gl_finish_cv_.SignalAll();
if (context_waiting_on_) {
context_waiting_on_->wait_for_gl_finish_cv_.SignalAll();
}
}
auto finish_task = [this, count_to_pass]() {
// When a GlFinishSyncToken is created it takes the current finish count
// from the GlContext, and we must wait for gl_finish_count_ to pass it.
@@ -646,6 +701,7 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) {
GlFinishCalled();
}
};
if (IsCurrent()) {
// If we are already on the current context, we cannot call
// RunWithoutWaiting, since that task will not run until this function
@@ -653,13 +709,53 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) {
finish_task();
return;
}
std::shared_ptr<GlContext> other = GetCurrent();
if (other) {
// If another context is current, make a note that it is blocked on us, so
// it can signal the right condition variable if it is asked to do a
// glFinish.
absl::MutexLock other_lock(&other->mutex_);
DCHECK(!other->context_waiting_on_);
other->context_waiting_on_ = this;
}
// We do not schedule this action using Run because we don't necessarily
// want to wait for it to complete. If another job calls GlFinishCalled
// sooner, we are done.
RunWithoutWaiting(std::move(finish_task));
absl::MutexLock lock(&mutex_);
while (gl_finish_count_ <= count_to_pass) {
wait_for_gl_finish_cv_.Wait(&mutex_);
{
absl::MutexLock lock(&mutex_);
while (gl_finish_count_ <= count_to_pass) {
if (other && other->gl_finish_count_ < other->gl_finish_count_target_) {
// If another context's dedicated thread is current, it is blocked
// waiting for this context to issue a glFinish call. But this context
// may also block waiting for the other context to do the same: this can
// happen when two contexts are handling each other's GlFinishSyncPoints
// (e.g. a producer and a consumer). To avoid a deadlock a context that
// is waiting on another context must still service Wait calls it may
// receive from its own GlFinishSyncPoints.
//
// We unlock this context's mutex to avoid holding both at the same
// time.
mutex_.Unlock();
{
glFinish();
other->GlFinishCalled();
}
mutex_.Lock();
// Because we temporarily unlocked mutex_, we cannot wait on the
// condition variable wait away; we need to go back to re-checking the
// condition. Otherwise we might miss a signal.
continue;
}
wait_for_gl_finish_cv_.Wait(&mutex_);
}
}
if (other) {
// The other context is no longer waiting on us.
absl::MutexLock other_lock(&other->mutex_);
other->context_waiting_on_ = nullptr;
}
}
mediapipe/gpu/gl_context.h
+3
View File
@@ -380,6 +380,9 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
// Changes should be guarded by mutex_. However, we use simple atomic
// loads for efficiency on the fast path.
std::atomic<int64_t> gl_finish_count_ = ATOMIC_VAR_INIT(0);
std::atomic<int64_t> gl_finish_count_target_ = ATOMIC_VAR_INIT(0);
GlContext* context_waiting_on_ ABSL_GUARDED_BY(mutex_) = nullptr;
// This mutex is held by a thread while this GL context is current on that
// thread. Since it may be held for extended periods of time, it should not
mediapipe/gpu/gl_texture_buffer_pool.cc
+33 -15
View File
@@ -26,19 +26,28 @@ GlTextureBufferPool::GlTextureBufferPool(int width, int height,
keep_count_(keep_count) {}
GlTextureBufferSharedPtr GlTextureBufferPool::GetBuffer() {
absl::MutexLock lock(&mutex_);
std::unique_ptr<GlTextureBuffer> buffer;
if (available_.empty()) {
buffer = GlTextureBuffer::Create(width_, height_, format_);
if (!buffer) return nullptr;
} else {
buffer = std::move(available_.back());
available_.pop_back();
buffer->Reuse();
bool reuse = false;
{
absl::MutexLock lock(&mutex_);
if (available_.empty()) {
buffer = GlTextureBuffer::Create(width_, height_, format_);
if (!buffer) return nullptr;
} else {
buffer = std::move(available_.back());
available_.pop_back();
reuse = true;
}
++in_use_count_;
}
++in_use_count_;
// This needs to wait on consumer sync points, therefore it should not be
// done while holding the mutex.
if (reuse) {
buffer->Reuse();
}
// Return a shared_ptr with a custom deleter that adds the buffer back
// to our available list.
@@ -60,15 +69,24 @@ std::pair<int, int> GlTextureBufferPool::GetInUseAndAvailableCounts() {
}
void GlTextureBufferPool::Return(GlTextureBuffer* buf) {
absl::MutexLock lock(&mutex_);
--in_use_count_;
available_.emplace_back(buf);
TrimAvailable();
std::vector<std::unique_ptr<GlTextureBuffer>> trimmed;
{
absl::MutexLock lock(&mutex_);
--in_use_count_;
available_.emplace_back(buf);
TrimAvailable(&trimmed);
}
// The trimmed buffers will be released without holding the lock.
}
void GlTextureBufferPool::TrimAvailable() {
void GlTextureBufferPool::TrimAvailable(
std::vector<std::unique_ptr<GlTextureBuffer>>* trimmed) {
int keep = std::max(keep_count_ - in_use_count_, 0);
if (available_.size() > keep) {
auto trim_it = std::next(available_.begin(), keep);
if (trimmed) {
std::move(available_.begin(), trim_it, std::back_inserter(*trimmed));
}
available_.resize(keep);
}
}
mediapipe/gpu/gl_texture_buffer_pool.h
+2 -1
View File
@@ -60,7 +60,8 @@ class GlTextureBufferPool
// If the total number of buffers is greater than keep_count, destroys any
// surplus buffers that are no longer in use.
void TrimAvailable() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
void TrimAvailable(std::vector<std::unique_ptr<GlTextureBuffer>>* trimmed)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
const int width_;
const int height_;
mediapipe/gpu/pixel_buffer_pool_util.mm
+11 -21
View File
@@ -16,6 +16,8 @@
#import <Foundation/Foundation.h>
#include "mediapipe/objc/util.h"
#if !defined(ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK) && !defined(NDEBUG)
#define ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK 1
#endif // defined(ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK)
@@ -27,17 +29,13 @@ CVPixelBufferPoolRef CreateCVPixelBufferPool(
CFTimeInterval maxAge) {
CVPixelBufferPoolRef pool = NULL;
NSDictionary *sourcePixelBufferOptions = @{
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
(id)kCVPixelBufferWidthKey : @(width),
(id)kCVPixelBufferHeightKey : @(height),
#if TARGET_OS_OSX
(id)kCVPixelFormatOpenGLCompatibility : @(YES),
#else
(id)kCVPixelFormatOpenGLESCompatibility : @(YES),
#endif // TARGET_OS_OSX
(id)kCVPixelBufferIOSurfacePropertiesKey : @{ /*empty dictionary*/ }
};
NSMutableDictionary *sourcePixelBufferOptions =
[(__bridge NSDictionary*)GetCVPixelBufferAttributesForGlCompatibility() mutableCopy];
[sourcePixelBufferOptions addEntriesFromDictionary:@{
(id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
(id)kCVPixelBufferWidthKey : @(width),
(id)kCVPixelBufferHeightKey : @(height),
}];
NSMutableDictionary *pixelBufferPoolOptions = [[NSMutableDictionary alloc] init];
pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = @(keepCount);
@@ -131,14 +129,6 @@ static void FreeRefConReleaseCallback(void* refCon, const void* baseAddress) {
CVReturn CreateCVPixelBufferWithoutPool(
int width, int height, OSType pixelFormat, CVPixelBufferRef* outBuffer) {
NSDictionary *attributes = @{
#if TARGET_OS_OSX
(id)kCVPixelFormatOpenGLCompatibility : @(YES),
#else
(id)kCVPixelFormatOpenGLESCompatibility : @(YES),
#endif // TARGET_OS_OSX
(id)kCVPixelBufferIOSurfacePropertiesKey : @{ /*empty dictionary*/ }
};
#if TARGET_IPHONE_SIMULATOR
// On the simulator, syncing the texture with the pixelbuffer does not work,
// and we have to use glReadPixels. Since GL_UNPACK_ROW_LENGTH is not
@@ -151,12 +141,12 @@ CVReturn CreateCVPixelBufferWithoutPool(
void* data = malloc(bytes_per_row * height);
return CVPixelBufferCreateWithBytes(
kCFAllocatorDefault, width, height, pixelFormat, data, bytes_per_row,
FreeRefConReleaseCallback, data, (__bridge CFDictionaryRef)attributes,
FreeRefConReleaseCallback, data, GetCVPixelBufferAttributesForGlCompatibility(),
outBuffer);
#else
return CVPixelBufferCreate(
kCFAllocatorDefault, width, height, pixelFormat,
(__bridge CFDictionaryRef)attributes, outBuffer);
GetCVPixelBufferAttributesForGlCompatibility(), outBuffer);
#endif
}