talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: ff83882955f1a1e2a043ff4e71278be9d7217bbe
This commit is contained in:
MediaPipe Team
2021-05-05 14:56:16 -04:00
committed by chuoling
parent ecb5b5f44a
commit a9b643e0f5
210 changed files with 5312 additions and 3838 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/gpu/BUILD
+11 -1
View File
@@ -14,7 +14,7 @@
load("//mediapipe/gpu:metal.bzl", "metal_library")
load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
licenses(["notice"])
@@ -240,6 +240,12 @@ cc_library(
],
)
mediapipe_proto_library(
name = "gpu_origin_proto",
srcs = ["gpu_origin.proto"],
visibility = ["//visibility:public"],
)
objc_library(
name = "pixel_buffer_pool_util",
srcs = ["pixel_buffer_pool_util.mm"],
@@ -460,6 +466,8 @@ cc_library(
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_node",
"//mediapipe/framework/port:logging",
"//mediapipe/util:resource_cache",
"@com_google_absl//absl/hash",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
] + select({
@@ -760,8 +768,10 @@ cc_library(
deps = [
":gl_calculator_helper",
":gl_quad_renderer",
":gpu_buffer",
":shader_util",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gl_surface_sink_calculator_cc_proto",
mediapipe/gpu/gl_context.cc
+7 -1
View File
@@ -563,8 +563,14 @@ class GlFenceSyncPoint : public GlSyncPoint {
void WaitOnGpu() override {
if (!sync_) return;
// TODO: do not wait if we are already on the same context?
// TODO: do not wait if we are already on the same context?
// WebGL2 specifies a waitSync call, but since cross-context
// synchronization is not supported, it's actually a no-op. Firefox prints
// a warning when it's called, so let's just skip the call. See
// b/184637485 for details.
#ifndef __EMSCRIPTEN__
glWaitSync(sync_, 0, GL_TIMEOUT_IGNORED);
#endif
}
bool IsReady() override {
mediapipe/gpu/gl_surface_sink_calculator.cc
+50 -28
View File
@@ -13,6 +13,7 @@
// limitations under the License.
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
@@ -21,9 +22,11 @@
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_quad_renderer.h"
#include "mediapipe/gpu/gl_surface_sink_calculator.pb.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/shader_util.h"
namespace mediapipe {
namespace api2 {
enum { kAttribVertex, kAttribTexturePosition, kNumberOfAttributes };
@@ -37,45 +40,52 @@ enum { kAttribVertex, kAttribTexturePosition, kNumberOfAttributes };
// GPU_SHARED: shared GPU resources.
//
// See GlSurfaceSinkCalculatorOptions for options.
class GlSurfaceSinkCalculator : public CalculatorBase {
class GlSurfaceSinkCalculator : public Node {
public:
GlSurfaceSinkCalculator() : initialized_(false) {}
~GlSurfaceSinkCalculator() override;
static constexpr Input<
OneOf<mediapipe::Image, mediapipe::GpuBuffer>>::Optional kInVideo{
"VIDEO"};
static constexpr Input<
OneOf<mediapipe::Image, mediapipe::GpuBuffer>>::Optional kIn{""};
static constexpr SideInput<std::unique_ptr<mediapipe::EglSurfaceHolder>>
kSurface{"SURFACE"};
static absl::Status GetContract(CalculatorContract* cc);
MEDIAPIPE_NODE_INTERFACE(GlSurfaceSinkCalculator, kInVideo, kIn, kSurface);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
~GlSurfaceSinkCalculator();
static absl::Status UpdateContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) final;
absl::Status Process(CalculatorContext* cc) final;
private:
GlCalculatorHelper helper_;
EglSurfaceHolder* surface_holder_;
bool initialized_;
std::unique_ptr<QuadRenderer> renderer_;
FrameScaleMode scale_mode_ = FrameScaleMode::kFillAndCrop;
mediapipe::GlCalculatorHelper helper_;
mediapipe::EglSurfaceHolder* surface_holder_;
bool initialized_ = false;
std::unique_ptr<mediapipe::QuadRenderer> renderer_;
mediapipe::FrameScaleMode scale_mode_ =
mediapipe::FrameScaleMode::kFillAndCrop;
};
REGISTER_CALCULATOR(GlSurfaceSinkCalculator);
MEDIAPIPE_REGISTER_NODE(GlSurfaceSinkCalculator);
// static
absl::Status GlSurfaceSinkCalculator::GetContract(CalculatorContract* cc) {
TagOrIndex(&(cc->Inputs()), "VIDEO", 0).Set<GpuBuffer>();
cc->InputSidePackets()
.Tag("SURFACE")
.Set<std::unique_ptr<EglSurfaceHolder>>();
absl::Status GlSurfaceSinkCalculator::UpdateContract(CalculatorContract* cc) {
RET_CHECK(kInVideo(cc).IsConnected() ^ kIn(cc).IsConnected())
<< "Only one of VIDEO or index 0 input is expected.";
// Currently we pass GL context information and other stuff as external
// inputs, which are handled by the helper.
return GlCalculatorHelper::UpdateContract(cc);
return mediapipe::GlCalculatorHelper::UpdateContract(cc);
}
absl::Status GlSurfaceSinkCalculator::Open(CalculatorContext* cc) {
surface_holder_ = cc->InputSidePackets()
.Tag("SURFACE")
.Get<std::unique_ptr<EglSurfaceHolder>>()
.get();
surface_holder_ = kSurface(cc).Get().get();
scale_mode_ = FrameScaleModeFromProto(
cc->Options<GlSurfaceSinkCalculatorOptions>().frame_scale_mode(),
FrameScaleMode::kFillAndCrop);
cc->Options<mediapipe::GlSurfaceSinkCalculatorOptions>()
.frame_scale_mode(),
mediapipe::FrameScaleMode::kFillAndCrop);
// Let the helper access the GL context information.
return helper_.Open(cc);
@@ -90,9 +100,20 @@ absl::Status GlSurfaceSinkCalculator::Process(CalculatorContext* cc) {
return absl::OkStatus();
}
const auto& input = TagOrIndex(cc->Inputs(), "VIDEO", 0).Get<GpuBuffer>();
mediapipe::Packet packet;
if (kInVideo(cc).IsConnected())
packet = kInVideo(cc).packet();
else
packet = kIn(cc).packet();
mediapipe::GpuBuffer input;
if (packet.ValidateAsType<mediapipe::GpuBuffer>().ok())
input = packet.Get<mediapipe::GpuBuffer>();
if (packet.ValidateAsType<mediapipe::Image>().ok())
input = packet.Get<mediapipe::Image>().GetGpuBuffer();
if (!initialized_) {
renderer_ = absl::make_unique<QuadRenderer>();
renderer_ = absl::make_unique<mediapipe::QuadRenderer>();
MP_RETURN_IF_ERROR(renderer_->GlSetup());
initialized_ = true;
}
@@ -125,7 +146,7 @@ absl::Status GlSurfaceSinkCalculator::Process(CalculatorContext* cc) {
MP_RETURN_IF_ERROR(
renderer_->GlRender(src.width(), src.height(), dst_width, dst_height,
scale_mode_, FrameRotation::kNone,
scale_mode_, mediapipe::FrameRotation::kNone,
/*flip_horizontal=*/false, /*flip_vertical=*/false,
/*flip_texture=*/surface_holder_->flip_y));
@@ -145,7 +166,7 @@ absl::Status GlSurfaceSinkCalculator::Process(CalculatorContext* cc) {
GlSurfaceSinkCalculator::~GlSurfaceSinkCalculator() {
if (renderer_) {
// TODO: use move capture when we have C++14 or better.
QuadRenderer* renderer = renderer_.release();
mediapipe::QuadRenderer* renderer = renderer_.release();
helper_.RunInGlContext([renderer] {
renderer->GlTeardown();
delete renderer;
@@ -153,4 +174,5 @@ GlSurfaceSinkCalculator::~GlSurfaceSinkCalculator() {
}
}
} // namespace api2
} // namespace mediapipe
mediapipe/gpu/gpu_buffer_multi_pool.cc
+8 -114
View File
@@ -157,125 +157,19 @@ GpuBuffer GpuBufferMultiPool::GetBufferFromSimplePool(
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
void GpuBufferMultiPool::EntryList::Prepend(Entry* entry) {
if (head_ == nullptr) {
head_ = tail_ = entry;
} else {
entry->next = head_;
head_->prev = entry;
head_ = entry;
}
++size_;
}
void GpuBufferMultiPool::EntryList::Append(Entry* entry) {
if (tail_ == nullptr) {
head_ = tail_ = entry;
} else {
tail_->next = entry;
entry->prev = tail_;
tail_ = entry;
}
++size_;
}
void GpuBufferMultiPool::EntryList::Remove(Entry* entry) {
if (entry == head_) {
head_ = entry->next;
} else {
entry->prev->next = entry->next;
}
if (entry == tail_) {
tail_ = entry->prev;
} else {
entry->next->prev = entry->prev;
}
entry->prev = nullptr;
entry->next = nullptr;
--size_;
}
void GpuBufferMultiPool::EntryList::InsertAfter(Entry* entry, Entry* after) {
if (after != nullptr) {
entry->next = after->next;
if (entry->next) entry->next->prev = entry;
entry->prev = after;
after->next = entry;
++size_;
} else
Prepend(entry);
}
void GpuBufferMultiPool::Evict(std::vector<SimplePool>* evicted) {
// Remove excess entries.
while (entry_list_.size() > kMaxPoolCount) {
Entry* victim = entry_list_.tail();
evicted->emplace_back(std::move(victim->pool));
entry_list_.Remove(victim);
pools_.erase(victim->spec);
}
// Every kRequestCountScrubInterval requests, halve the request counts, and
// remove entries which have fallen to 0.
// This keeps sporadic requests from accumulating and eventually exceeding
// the minimum request threshold for allocating a pool. Also, it means that
// if the request regimen changes (e.g. a graph was always requesting a large
// size, but then switches to a small size to save memory or CPU), the pool
// can quickly adapt to it.
if (total_request_count_ >= kRequestCountScrubInterval) {
total_request_count_ = 0;
VLOG(2) << "begin pool scrub";
for (Entry* entry = entry_list_.head(); entry != nullptr;) {
VLOG(2) << "entry for: " << entry->spec.width << "x" << entry->spec.height
<< " request_count: " << entry->request_count
<< " has pool: " << (entry->pool != nullptr);
entry->request_count /= 2;
Entry* next = entry->next;
if (entry->request_count == 0) {
evicted->emplace_back(std::move(entry->pool));
entry_list_.Remove(entry);
pools_.erase(entry->spec);
}
entry = next;
}
}
}
GpuBufferMultiPool::SimplePool GpuBufferMultiPool::RequestPool(
const BufferSpec& key) {
const BufferSpec& spec) {
SimplePool pool;
std::vector<SimplePool> evicted;
{
absl::MutexLock lock(&mutex_);
auto pool_it = pools_.find(key);
Entry* entry;
if (pool_it == pools_.end()) {
std::tie(pool_it, std::ignore) =
pools_.emplace(std::piecewise_construct, std::forward_as_tuple(key),
std::forward_as_tuple(key));
entry = &pool_it->second;
CHECK_EQ(entry->request_count, 0);
entry->request_count = 1;
entry_list_.Append(entry);
if (entry->prev != nullptr) CHECK_GE(entry->prev->request_count, 1);
} else {
entry = &pool_it->second;
++entry->request_count;
Entry* larger = entry->prev;
while (larger != nullptr &&
larger->request_count < entry->request_count) {
larger = larger->prev;
}
if (larger != entry->prev) {
entry_list_.Remove(entry);
entry_list_.InsertAfter(entry, larger);
}
}
if (!entry->pool && entry->request_count >= kMinRequestsBeforePool) {
entry->pool = MakeSimplePool(key);
}
pool = entry->pool;
++total_request_count_;
Evict(&evicted);
pool =
cache_.Lookup(spec, [this](const BufferSpec& spec, int request_count) {
return (request_count >= kMinRequestsBeforePool)
? MakeSimplePool(spec)
: nullptr;
});
evicted = cache_.Evict(kMaxPoolCount, kRequestCountScrubInterval);
}
// Evicted pools, and their buffers, will be released without holding the
// lock.
mediapipe/gpu/gpu_buffer_multi_pool.h
+15 -59
View File
@@ -22,12 +22,10 @@
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_MULTI_POOL_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_MULTI_POOL_H_
#include <deque>
#include <limits>
#include <unordered_map>
#include "absl/hash/hash.h"
#include "absl/synchronization/mutex.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/util/resource_cache.h"
#ifdef __APPLE__
#include "mediapipe/gpu/pixel_buffer_pool_util.h"
@@ -65,31 +63,24 @@ class GpuBufferMultiPool {
void FlushTextureCaches();
#endif // defined(__APPLE__)
// This generates a "rol" instruction with both Clang and GCC.
inline static std::size_t RotateLeft(std::size_t x, int n) {
return (x << n) | (x >> (std::numeric_limits<size_t>::digits - n));
}
// This class is not intended as part of the public api of this class. It is
// public only because it is used as a map key type, and the map
// implementation needs access to, e.g., the equality operator.
struct BufferSpec {
BufferSpec(int w, int h, mediapipe::GpuBufferFormat f)
: width(w), height(h), format(f) {}
template <typename H>
friend H AbslHashValue(H h, const BufferSpec& spec) {
return H::combine(std::move(h), spec.width, spec.height,
static_cast<uint32_t>(spec.format));
}
int width;
int height;
mediapipe::GpuBufferFormat format;
};
struct BufferSpecHash {
std::size_t operator()(const BufferSpec& spec) const {
// Width and height are expected to be smaller than half the width of
// size_t. We can combine them into a single integer, and then use
// std::hash.
constexpr int kWidth = std::numeric_limits<size_t>::digits;
return std::hash<std::size_t>{}(
spec.width ^ RotateLeft(spec.height, kWidth / 2) ^
RotateLeft(static_cast<uint32_t>(spec.format), kWidth / 4));
}
};
private:
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
using SimplePool = std::shared_ptr<CvPixelBufferPoolWrapper>;
@@ -97,53 +88,18 @@ class GpuBufferMultiPool {
using SimplePool = std::shared_ptr<GlTextureBufferPool>;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
struct Entry {
Entry(const BufferSpec& spec) : spec(spec) {}
Entry* prev = nullptr;
Entry* next = nullptr;
BufferSpec spec;
int request_count = 0;
SimplePool pool;
};
// Unlike std::list, this is an intrusive list, meaning that the prev and next
// pointers live inside the element. Apart from not requiring an extra
// allocation, this means that once we look up an entry by key in the pools_
// map we do not need to look it up separately in the list.
//
class EntryList {
public:
void Prepend(Entry* entry);
void Append(Entry* entry);
void Remove(Entry* entry);
void InsertAfter(Entry* entry, Entry* after);
Entry* head() { return head_; }
Entry* tail() { return tail_; }
size_t size() { return size_; }
private:
Entry* head_ = nullptr;
Entry* tail_ = nullptr;
size_t size_ = 0;
};
SimplePool MakeSimplePool(const BufferSpec& spec);
// Requests a simple buffer pool for the given spec. This may return nullptr
// if we have not yet reached a sufficient number of requests to allocate a
// pool, in which case the caller should invoke GetBufferWithoutPool instead
// of GetBufferFromSimplePool.
SimplePool RequestPool(const BufferSpec& key);
SimplePool RequestPool(const BufferSpec& spec);
GpuBuffer GetBufferFromSimplePool(BufferSpec spec, const SimplePool& pool);
GpuBuffer GetBufferWithoutPool(const BufferSpec& spec);
void Evict(std::vector<SimplePool>* evicted)
ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
absl::Mutex mutex_;
std::unordered_map<BufferSpec, Entry, BufferSpecHash> pools_
ABSL_GUARDED_BY(mutex_);
EntryList entry_list_ ABSL_GUARDED_BY(mutex_);
int total_request_count_ = 0;
mediapipe::ResourceCache<BufferSpec, SimplePool, absl::Hash<BufferSpec>>
cache_ ABSL_GUARDED_BY(mutex_);
#ifdef __APPLE__
// Texture caches used with this pool.
mediapipe/gpu/gpu_origin.proto
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2021 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
message GpuOrigin {
enum Mode {
DEFAULT = 0;
// OpenGL: bottom-left origin
// Metal : top-left origin
CONVENTIONAL = 1;
// OpenGL: top-left origin
// Metal : top-left origin
TOP_LEFT = 2;
}
}