talksik/mediapipe
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: d4a11282d20fe4d2e137f9032cf349750030dcb9
This commit is contained in:
MediaPipe Team
2021-11-03 17:27:30 -07:00
committed by jqtang
parent 1faeaae7e5
commit d4bb35fe5a
72 changed files with 1089 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mediapipe/gpu/gl_calculator_helper_impl_common.cc
+3 -3
View File
@@ -112,13 +112,13 @@ GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer, int plane) {
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetGlTextureReadView(plane));
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(plane));
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const ImageFrame& image_frame) {
auto gpu_buffer = GpuBuffer::CopyingImageFrame(image_frame);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetGlTextureReadView(0));
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(0));
}
template <>
@@ -149,7 +149,7 @@ GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
GpuBuffer gpu_buffer =
gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetGlTextureWriteView(0));
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView<GlTextureView>(0));
}
} // namespace mediapipe
mediapipe/gpu/gl_texture_buffer.cc
+6 -3
View File
@@ -224,7 +224,8 @@ void GlTextureBuffer::WaitForConsumersOnGpu() {
// precisely, on only one GL context.
}
GlTextureView GlTextureBuffer::GetGlTextureReadView(
GlTextureView GlTextureBuffer::GetReadView(
mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) const {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
@@ -241,7 +242,8 @@ GlTextureView GlTextureBuffer::GetGlTextureReadView(
nullptr);
}
GlTextureView GlTextureBuffer::GetGlTextureWriteView(
GlTextureView GlTextureBuffer::GetWriteView(
mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) {
auto gl_context = GlContext::GetCurrent();
CHECK(gl_context);
@@ -341,7 +343,8 @@ std::unique_ptr<ImageFrame> GlTextureBuffer::AsImageFrame() const {
ImageFormat::Format image_format = ImageFormatForGpuBufferFormat(format());
auto output = absl::make_unique<ImageFrame>(
image_format, width(), height(), ImageFrame::kGlDefaultAlignmentBoundary);
auto view = GetGlTextureReadView(nullptr, 0);
auto view =
GetReadView(mediapipe::internal::types<GlTextureView>{}, nullptr, 0);
ReadTexture(view, format(), output->MutablePixelData(),
output->PixelDataSize());
return output;
mediapipe/gpu/gl_texture_buffer.h
+11 -6
View File
@@ -32,7 +32,9 @@ namespace mediapipe {
class GlCalculatorHelperImpl;
// Implements a GPU memory buffer as an OpenGL texture. For internal use.
class GlTextureBuffer : public mediapipe::internal::GpuBufferStorage {
class GlTextureBuffer
: public mediapipe::internal::GpuBufferStorageImpl<
GlTextureBuffer, mediapipe::internal::ViewProvider<GlTextureView>> {
public:
// This is called when the texture buffer is deleted. It is passed a sync
// token created at that time on the GlContext. If the GlTextureBuffer has
@@ -86,11 +88,12 @@ class GlTextureBuffer : public mediapipe::internal::GpuBufferStorage {
int height() const { return height_; }
GpuBufferFormat format() const { return format_; }
GlTextureView GetGlTextureReadView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override;
GlTextureView GetGlTextureWriteView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override;
void ViewDoneWriting(const GlTextureView& view) override;
GlTextureView GetReadView(mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override;
GlTextureView GetWriteView(mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override;
std::unique_ptr<ImageFrame> AsImageFrame() const override;
// If this texture is going to be used outside of the context that produced
@@ -142,6 +145,8 @@ class GlTextureBuffer : public mediapipe::internal::GpuBufferStorage {
// Returns true on success.
bool CreateInternal(const void* data, int alignment = 4);
void ViewDoneWriting(const GlTextureView& view);
friend class GlCalculatorHelperImpl;
GLuint name_ = 0;
mediapipe/gpu/gpu_buffer.h
+63 -59
View File
@@ -15,6 +15,7 @@
#ifndef MEDIAPIPE_GPU_GPU_BUFFER_H_
#define MEDIAPIPE_GPU_GPU_BUFFER_H_
#include <memory>
#include <utility>
#include "mediapipe/framework/formats/image_frame.h"
@@ -23,6 +24,10 @@
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_buffer_storage.h"
// Note: these headers are needed for the legacy storage APIs. Do not add more
// storage-specific headers here. See WebGpuTextureBuffer/View for an example
// of adding a new storage and view.
#if defined(__APPLE__)
#include <CoreVideo/CoreVideo.h>
@@ -31,9 +36,7 @@
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#include "mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.h"
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
#else
#include "mediapipe/gpu/gl_texture_buffer.h"
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
@@ -60,19 +63,28 @@ class GpuBuffer {
// are not portable. Applications and calculators should normally obtain
// GpuBuffers in a portable way from the framework, e.g. using
// GpuBufferMultiPool.
explicit GpuBuffer(
std::shared_ptr<mediapipe::internal::GpuBufferStorage> storage)
: storage_(std::move(storage)) {}
// Note: these constructors and accessors for specific storage types exist
// for backwards compatibility reasons. Do not add new ones.
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
explicit GpuBuffer(CFHolder<CVPixelBufferRef> pixel_buffer)
: pixel_buffer_(std::move(pixel_buffer)) {}
: storage_(std::make_shared<GpuBufferStorageCvPixelBuffer>(
std::move(pixel_buffer))) {}
explicit GpuBuffer(CVPixelBufferRef pixel_buffer)
: pixel_buffer_(pixel_buffer) {}
: storage_(
std::make_shared<GpuBufferStorageCvPixelBuffer>(pixel_buffer)) {}
CVPixelBufferRef GetCVPixelBufferRef() const { return *pixel_buffer_; }
CVPixelBufferRef GetCVPixelBufferRef() const {
auto p = storage_->down_cast<GpuBufferStorageCvPixelBuffer>();
if (p) return **p;
return nullptr;
}
#else
explicit GpuBuffer(GlTextureBufferSharedPtr texture_buffer)
: texture_buffer_(std::move(texture_buffer)) {}
const GlTextureBufferSharedPtr& GetGlTextureBufferSharedPtr() const {
return texture_buffer_;
GlTextureBufferSharedPtr GetGlTextureBufferSharedPtr() const {
return internal_storage<GlTextureBuffer>();
}
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
@@ -93,14 +105,26 @@ class GpuBuffer {
// Allow assignment from nullptr.
GpuBuffer& operator=(std::nullptr_t other);
GlTextureView GetGlTextureReadView(int plane) const {
return current_storage().GetGlTextureReadView(
std::make_shared<GpuBuffer>(*this), plane);
// Gets a read view of the specified type. The arguments depend on the
// specific view type; see the corresponding ViewProvider.
template <class View, class... Args>
auto GetReadView(Args... args) const {
return current_storage()
.down_cast<mediapipe::internal::ViewProvider<View>>()
->GetReadView(mediapipe::internal::types<View>{},
std::make_shared<GpuBuffer>(*this),
std::forward<Args>(args)...);
}
GlTextureView GetGlTextureWriteView(int plane) {
return current_storage().GetGlTextureWriteView(
std::make_shared<GpuBuffer>(*this), plane);
// Gets a write view of the specified type. The arguments depend on the
// specific view type; see the corresponding ViewProvider.
template <class View, class... Args>
auto GetWriteView(Args... args) {
return current_storage()
.down_cast<mediapipe::internal::ViewProvider<View>>()
->GetWriteView(mediapipe::internal::types<View>{},
std::make_shared<GpuBuffer>(*this),
std::forward<Args>(args)...);
}
// Make a GpuBuffer copying the data from an ImageFrame.
@@ -115,77 +139,57 @@ class GpuBuffer {
return current_storage().AsImageFrame();
}
// Attempts to access an underlying storage object of the specified type.
// This method is meant for internal use: user code should access the contents
// using views.
template <class T>
std::shared_ptr<T> internal_storage() const {
if (storage_->down_cast<T>()) return std::static_pointer_cast<T>(storage_);
return nullptr;
}
private:
class PlaceholderGpuBufferStorage
: public mediapipe::internal::GpuBufferStorage {
: public mediapipe::internal::GpuBufferStorageImpl<
PlaceholderGpuBufferStorage> {
public:
int width() const override { return 0; }
int height() const override { return 0; }
virtual GpuBufferFormat format() const override {
return GpuBufferFormat::kUnknown;
}
GlTextureView GetGlTextureReadView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override {
return {};
}
GlTextureView GetGlTextureWriteView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override {
return {};
}
void ViewDoneWriting(const GlTextureView& view) override{};
std::unique_ptr<ImageFrame> AsImageFrame() const override {
return nullptr;
}
};
mediapipe::internal::GpuBufferStorage& no_storage() const {
static PlaceholderGpuBufferStorage placeholder;
std::shared_ptr<mediapipe::internal::GpuBufferStorage>& no_storage() const {
static auto placeholder =
std::static_pointer_cast<mediapipe::internal::GpuBufferStorage>(
std::make_shared<PlaceholderGpuBufferStorage>());
return placeholder;
}
const mediapipe::internal::GpuBufferStorage& current_storage() const {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
if (pixel_buffer_ != nullptr) return pixel_buffer_;
#else
if (texture_buffer_) return *texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return no_storage();
return *storage_;
}
mediapipe::internal::GpuBufferStorage& current_storage() {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
if (pixel_buffer_ != nullptr) return pixel_buffer_;
#else
if (texture_buffer_) return *texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return no_storage();
}
mediapipe::internal::GpuBufferStorage& current_storage() { return *storage_; }
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
GpuBufferStorageCvPixelBuffer pixel_buffer_;
#else
GlTextureBufferSharedPtr texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
std::shared_ptr<mediapipe::internal::GpuBufferStorage> storage_ =
no_storage();
};
inline bool GpuBuffer::operator==(std::nullptr_t other) const {
return &current_storage() == &no_storage();
return storage_ == no_storage();
}
inline bool GpuBuffer::operator==(const GpuBuffer& other) const {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return pixel_buffer_ == other.pixel_buffer_;
#else
return texture_buffer_ == other.texture_buffer_;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
return storage_ == other.storage_;
}
inline GpuBuffer& GpuBuffer::operator=(std::nullptr_t other) {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
pixel_buffer_.reset(other);
#else
texture_buffer_ = other;
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
storage_ = no_storage();
return *this;
}
mediapipe/gpu/gpu_buffer_storage.h
+57 -11
View File
@@ -12,27 +12,73 @@ class GpuBuffer;
namespace mediapipe {
namespace internal {
using mediapipe::GlTextureView;
using mediapipe::GpuBuffer;
using mediapipe::GpuBufferFormat;
template <class... T>
struct types {};
class GlTextureViewManager {
template <class V>
class ViewProvider;
// Note: this specialization temporarily lives here for backwards compatibility
// reasons. New specializations should be put in the same file as their view.
template <>
class ViewProvider<GlTextureView> {
public:
virtual ~GlTextureViewManager() = default;
virtual GlTextureView GetGlTextureReadView(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) const = 0;
virtual GlTextureView GetGlTextureWriteView(
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) = 0;
virtual void ViewDoneWriting(const GlTextureView& view) = 0;
virtual ~ViewProvider() = default;
// Note that the view type is encoded in an argument to allow overloading,
// so a storage class can implement GetRead/WriteView for multiple view types.
// We cannot use a template function because it cannot be virtual; we want to
// have a virtual function here to enforce that different storages supporting
// the same view implement the same signature.
// Note that we allow different views to have custom signatures, providing
// additional view-specific arguments that may be needed.
virtual GlTextureView GetReadView(types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const = 0;
virtual GlTextureView GetWriteView(types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) = 0;
};
class GpuBufferStorage : public GlTextureViewManager {
class GpuBufferStorage {
public:
virtual ~GpuBufferStorage() = default;
virtual int width() const = 0;
virtual int height() const = 0;
virtual GpuBufferFormat format() const = 0;
virtual std::unique_ptr<ImageFrame> AsImageFrame() const = 0;
// We can't use dynamic_cast since we want to support building without RTTI.
// The public methods delegate to the type-erased private virtual method.
template <class T>
T* down_cast() {
return static_cast<T*>(
const_cast<void*>(down_cast(tool::GetTypeHash<T>())));
}
template <class T>
const T* down_cast() const {
return static_cast<const T*>(down_cast(tool::GetTypeHash<T>()));
}
private:
virtual const void* down_cast(size_t type_hash) const = 0;
virtual size_t storage_type_hash() const = 0;
};
template <class T, class... U>
class GpuBufferStorageImpl : public GpuBufferStorage, public U... {
private:
virtual const void* down_cast(size_t type_hash) const override {
return down_cast_impl(type_hash, types<T, U...>{});
}
size_t storage_type_hash() const override { return tool::GetTypeHash<T>(); }
const void* down_cast_impl(size_t type_hash, types<>) const {
return nullptr;
}
template <class V, class... W>
const void* down_cast_impl(size_t type_hash, types<V, W...>) const {
if (type_hash == tool::GetTypeHash<V>()) return static_cast<const V*>(this);
return down_cast_impl(type_hash, types<W...>{});
}
};
} // namespace internal
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc
+6 -3
View File
@@ -11,7 +11,8 @@ typedef CVOpenGLTextureRef CVTextureType;
typedef CVOpenGLESTextureRef CVTextureType;
#endif // TARGET_OS_OSX
GlTextureView GpuBufferStorageCvPixelBuffer::GetGlTextureReadView(
GlTextureView GpuBufferStorageCvPixelBuffer::GetReadView(
mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) const {
CVReturn err;
auto gl_context = GlContext::GetCurrent();
@@ -58,11 +59,13 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetGlTextureReadView(
#endif // TARGET_OS_OSX
}
GlTextureView GpuBufferStorageCvPixelBuffer::GetGlTextureWriteView(
GlTextureView GpuBufferStorageCvPixelBuffer::GetWriteView(
mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer, int plane) {
// For this storage there is currently no difference between read and write
// views, so we delegate to the read method.
return GetGlTextureReadView(std::move(gpu_buffer), plane);
return GetReadView(mediapipe::internal::types<GlTextureView>{},
std::move(gpu_buffer), plane);
}
void GpuBufferStorageCvPixelBuffer::ViewDoneWriting(const GlTextureView& view) {
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.h
+12 -6
View File
@@ -12,7 +12,9 @@ namespace mediapipe {
class GlContext;
class GpuBufferStorageCvPixelBuffer
: public mediapipe::internal::GpuBufferStorage,
: public mediapipe::internal::GpuBufferStorageImpl<
GpuBufferStorageCvPixelBuffer,
mediapipe::internal::ViewProvider<GlTextureView>>,
public CFHolder<CVPixelBufferRef> {
public:
using CFHolder<CVPixelBufferRef>::CFHolder;
@@ -28,12 +30,16 @@ class GpuBufferStorageCvPixelBuffer
return GpuBufferFormatForCVPixelFormat(
CVPixelBufferGetPixelFormatType(**this));
}
GlTextureView GetGlTextureReadView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override;
GlTextureView GetGlTextureWriteView(std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override;
GlTextureView GetReadView(mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) const override;
GlTextureView GetWriteView(mediapipe::internal::types<GlTextureView>,
std::shared_ptr<GpuBuffer> gpu_buffer,
int plane) override;
std::unique_ptr<ImageFrame> AsImageFrame() const override;
void ViewDoneWriting(const GlTextureView& view) override;
private:
void ViewDoneWriting(const GlTextureView& view);
};
} // namespace mediapipe