Project import generated by Copybara.

GitOrigin-RevId: 19a829ffd755edb43e54d20c0e7b9348512d5108

mediapipe/gpu/BUILD

@@ -38,14 +38,19 @@ cc_library(
     srcs = ["gpu_service.cc"],
     hdrs = ["gpu_service.h"],
     visibility = ["//visibility:public"],
-    deps = ["//mediapipe/framework:graph_service"],
+    deps = ["//mediapipe/framework:graph_service"] + select({
+        "//conditions:default": [
+            ":gpu_shared_data_internal",
+        ],
+        "//mediapipe/gpu:disable_gpu": [],
+    }),
 )
 
 cc_library(
     name = "graph_support",
     hdrs = ["graph_support.h"],
     visibility = ["//visibility:public"],
-    deps = [":gpu_service"],
+    deps = ["//mediapipe/framework:graph_service"],
 )
 
 GL_BASE_LINK_OPTS = select({
@@ -366,7 +371,6 @@ objc_library(
     hdrs = ["pixel_buffer_pool_util.h"],
     copts = [
         "-Wno-shorten-64-to-32",
-        "-std=c++17",
     ],
     sdk_frameworks = [
         "Accelerate",
@@ -389,7 +393,6 @@ objc_library(
     copts = [
         "-x objective-c++",
         "-Wno-shorten-64-to-32",
-        "-std=c++17",
     ],
     features = ["-layering_check"],
     sdk_frameworks = [
@@ -425,7 +428,6 @@ objc_library(
     copts = [
         "-x objective-c++",
         "-Wno-shorten-64-to-32",
-        "-std=c++17",
     ],
     sdk_frameworks = [
         "CoreVideo",
@@ -691,7 +693,6 @@ objc_library(
     name = "gl_calculator_helper_ios",
     copts = [
         "-Wno-shorten-64-to-32",
-        "-std=c++17",
     ],
     visibility = ["//visibility:public"],
     deps = [
@@ -707,7 +708,6 @@ objc_library(
     hdrs = ["MPPMetalHelper.h"],
     copts = [
         "-Wno-shorten-64-to-32",
-        "-std=c++17",
     ],
     features = ["-layering_check"],
     sdk_frameworks = [
@@ -801,7 +801,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":gl_calculator_helper",
-        ":gpu_buffer_storage_image_frame",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:status",
@@ -927,7 +926,6 @@ mediapipe_cc_proto_library(
 objc_library(
     name = "metal_copy_calculator",
     srcs = ["MetalCopyCalculator.mm"],
-    copts = ["-std=c++17"],
     features = ["-layering_check"],
     sdk_frameworks = [
         "CoreVideo",
@@ -946,7 +944,6 @@ objc_library(
 objc_library(
     name = "metal_rgb_weight_calculator",
     srcs = ["MetalRgbWeightCalculator.mm"],
-    copts = ["-std=c++17"],
     features = ["-layering_check"],
     sdk_frameworks = [
         "CoreVideo",
@@ -964,7 +961,6 @@ objc_library(
 objc_library(
     name = "metal_sobel_calculator",
     srcs = ["MetalSobelCalculator.mm"],
-    copts = ["-std=c++17"],
     features = ["-layering_check"],
     sdk_frameworks = [
         "CoreVideo",
@@ -982,7 +978,6 @@ objc_library(
 objc_library(
     name = "metal_sobel_compute_calculator",
     srcs = ["MetalSobelComputeCalculator.mm"],
-    copts = ["-std=c++17"],
     features = ["-layering_check"],
     sdk_frameworks = [
         "CoreVideo",
@@ -1018,7 +1013,6 @@ objc_library(
 objc_library(
     name = "mps_threshold_calculator",
     srcs = ["MPSThresholdCalculator.mm"],
-    copts = ["-std=c++17"],
     sdk_frameworks = [
         "CoreVideo",
         "Metal",
@@ -1053,7 +1047,6 @@ objc_library(
     ],
     copts = [
         "-Wno-shorten-64-to-32",
-        "-std=c++17",
     ],
     data = [
         "//mediapipe/objc:testdata/googlelogo_color_272x92dp.png",

mediapipe/gpu/gl_context.cc

@@ -23,6 +23,7 @@
 
 #include "absl/base/dynamic_annotations.h"
 #include "absl/memory/memory.h"
+#include "absl/status/status.h"
 #include "absl/synchronization/mutex.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/ret_check.h"
@@ -358,6 +359,7 @@ absl::Status GlContext::FinishInitialization(bool create_thread) {
 GlContext::GlContext() {}
 
 GlContext::~GlContext() {
+  destructing_ = true;
   // Note: on Apple platforms, this object contains Objective-C objects.
   // The destructor will release them, but ARC must be on.
 #ifdef __OBJC__
@@ -366,11 +368,16 @@ GlContext::~GlContext() {
 #endif
 #endif  // __OBJC__
 
+  auto clear_attachments = [this] {
+    attachments_.clear();
+    if (profiling_helper_) {
+      profiling_helper_->LogAllTimestamps();
+    }
+  };
+
   if (thread_) {
-    auto status = thread_->Run([this] {
-      if (profiling_helper_) {
-        profiling_helper_->LogAllTimestamps();
-      }
+    auto status = thread_->Run([this, clear_attachments] {
+      clear_attachments();
       return ExitContext(nullptr);
     });
     LOG_IF(ERROR, !status.ok())
@@ -378,6 +385,17 @@ GlContext::~GlContext() {
     if (thread_->IsCurrentThread()) {
       thread_.release()->SelfDestruct();
     }
+  } else {
+    if (IsCurrent()) {
+      clear_attachments();
+    } else {
+      ContextBinding saved_context;
+      auto status = SwitchContextAndRun([&clear_attachments] {
+        clear_attachments();
+        return absl::OkStatus();
+      });
+      LOG_IF(ERROR, !status.ok()) << status;
+    }
   }
   DestroyContext();
 }
@@ -501,6 +519,14 @@ absl::Status GlContext::SwitchContext(ContextBinding* saved_context,
   }
 }
 
+GlContext::ContextBinding GlContext::ThisContextBinding() {
+  GlContext::ContextBinding result = ThisContextBindingPlatform();
+  if (!destructing_) {
+    result.context_object = shared_from_this();
+  }
+  return result;
+}
+
 absl::Status GlContext::EnterContext(ContextBinding* saved_context) {
   DCHECK(HasContext());
   return SwitchContext(saved_context, ThisContextBinding());

mediapipe/gpu/gl_context.h

@@ -21,6 +21,7 @@
 #include <functional>
 #include <memory>
 
+#include "absl/container/flat_hash_map.h"
 #include "absl/synchronization/mutex.h"
 #include "mediapipe/framework/executor.h"
 #include "mediapipe/framework/mediapipe_profiling.h"
@@ -285,6 +286,48 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
   // Sets default texture filtering parameters.
   void SetStandardTextureParams(GLenum target, GLint internal_format);
 
+  template <class T>
+  using AttachmentPtr = std::unique_ptr<T, std::function<void(void*)>>;
+
+  template <class T, class... Args>
+  static std::enable_if_t<!std::is_array<T>::value, AttachmentPtr<T>>
+  MakeAttachmentPtr(Args&&... args) {
+    return {new T(std::forward<Args>(args)...),
+            [](void* ptr) { delete static_cast<T*>(ptr); }};
+  }
+
+  class AttachmentBase {};
+
+  template <class T>
+  class Attachment : public AttachmentBase {
+   public:
+    using FactoryT = std::function<AttachmentPtr<T>(GlContext&)>;
+    Attachment(FactoryT factory) : factory_(factory) {}
+
+    Attachment(const Attachment&) = delete;
+    Attachment(Attachment&&) = delete;
+    Attachment& operator=(const Attachment&) = delete;
+    Attachment& operator=(Attachment&&) = delete;
+
+    T& Get(GlContext& ctx) const { return ctx.GetCachedAttachment(*this); }
+
+    const FactoryT& factory() const { return factory_; }
+
+   private:
+    FactoryT factory_;
+  };
+
+  // TOOD: const result?
+  template <class T>
+  T& GetCachedAttachment(const Attachment<T>& attachment) {
+    DCHECK(IsCurrent());
+    AttachmentPtr<void>& entry = attachments_[&attachment];
+    if (entry == nullptr) {
+      entry = attachment.factory()(*this);
+    }
+    return *static_cast<T*>(entry.get());
+  }
+
   // These are used for testing specific SyncToken implementations. Do not use
   // outside of tests.
   enum class SyncTokenTypeForTest {
@@ -387,6 +430,8 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
 
   // A binding that can be used to make this GlContext current.
   ContextBinding ThisContextBinding();
+  // Fill in platform-specific fields. Must _not_ set context_obj.
+  ContextBinding ThisContextBindingPlatform();
   // Fills in a ContextBinding with platform-specific information about which
   // context is current on this thread.
   static void GetCurrentContextBinding(ContextBinding* binding);
@@ -409,6 +454,8 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
   // better mechanism?
   bool can_linear_filter_float_textures_;
 
+  absl::flat_hash_map<const AttachmentBase*, AttachmentPtr<void>> attachments_;
+
   // Number of glFinish calls completed on the GL thread.
   // Changes should be guarded by mutex_. However, we use simple atomic
   // loads for efficiency on the fast path.
@@ -428,6 +475,8 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
   absl::CondVar wait_for_gl_finish_cv_ ABSL_GUARDED_BY(mutex_);
 
   std::unique_ptr<mediapipe::GlProfilingHelper> profiling_helper_ = nullptr;
+
+  bool destructing_ = false;
 };
 
 // For backward compatibility. TODO: migrate remaining callers.

mediapipe/gpu/gl_context_eagl.cc

@@ -84,9 +84,8 @@ void GlContext::DestroyContext() {
   }
 }
 
-GlContext::ContextBinding GlContext::ThisContextBinding() {
+GlContext::ContextBinding GlContext::ThisContextBindingPlatform() {
   GlContext::ContextBinding result;
-  result.context_object = shared_from_this();
   result.context = context_;
   return result;
 }

mediapipe/gpu/gl_context_egl.cc

@@ -269,9 +269,8 @@ void GlContext::DestroyContext() {
 #endif  // __ANDROID__
 }
 
-GlContext::ContextBinding GlContext::ThisContextBinding() {
+GlContext::ContextBinding GlContext::ThisContextBindingPlatform() {
   GlContext::ContextBinding result;
-  result.context_object = shared_from_this();
   result.display = display_;
   result.draw_surface = surface_;
   result.read_surface = surface_;

mediapipe/gpu/gl_context_nsgl.cc

@@ -134,9 +134,8 @@ void GlContext::DestroyContext() {
   }
 }
 
-GlContext::ContextBinding GlContext::ThisContextBinding() {
+GlContext::ContextBinding GlContext::ThisContextBindingPlatform() {
   GlContext::ContextBinding result;
-  result.context_object = shared_from_this();
   result.context = context_;
   return result;
 }

mediapipe/gpu/gl_context_webgl.cc

@@ -173,9 +173,8 @@ void GlContext::DestroyContext() {
   }
 }
 
-GlContext::ContextBinding GlContext::ThisContextBinding() {
+GlContext::ContextBinding GlContext::ThisContextBindingPlatform() {
   GlContext::ContextBinding result;
-  result.context_object = shared_from_this();
   result.context = context_;
   return result;
 }

mediapipe/gpu/gl_quad_renderer.cc

@@ -111,7 +111,7 @@ absl::Status QuadRenderer::GlRender(float frame_width, float frame_height,
                                     FrameScaleMode scale_mode,
                                     FrameRotation rotation,
                                     bool flip_horizontal, bool flip_vertical,
-                                    bool flip_texture) {
+                                    bool flip_texture) const {
   RET_CHECK(program_) << "Must setup the program before rendering.";
 
   glUseProgram(program_);

mediapipe/gpu/gl_quad_renderer.h

@@ -72,7 +72,7 @@ class QuadRenderer {
   absl::Status GlRender(float frame_width, float frame_height, float view_width,
                         float view_height, FrameScaleMode scale_mode,
                         FrameRotation rotation, bool flip_horizontal,
-                        bool flip_vertical, bool flip_texture);
+                        bool flip_vertical, bool flip_texture) const;
   // Deletes the rendering program. Must be called withn the GL context where
   // it was created.
   void GlTeardown();

mediapipe/gpu/gpu_buffer_format.cc

@@ -144,7 +144,7 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
            }},
           {GpuBufferFormat::kRGBAFloat128,
            {
-               {GL_RGBA, GL_RGBA, GL_FLOAT, 1},
+               {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1},
            }},
       }};
 

mediapipe/gpu/gpu_service.cc

@@ -16,6 +16,7 @@
 
 namespace mediapipe {
 
-const GraphService<GpuResources> kGpuService("kGpuService");
+const GraphService<GpuResources> kGpuService(
+    "kGpuService", GraphServiceBase::kAllowDefaultInitialization);
 
 }  // namespace mediapipe

mediapipe/gpu/gpu_service.h

@@ -17,9 +17,18 @@
 
 #include "mediapipe/framework/graph_service.h"
 
+#if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gpu_shared_data_internal.h"
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
 namespace mediapipe {
 
-class GpuResources;
+#if MEDIAPIPE_DISABLE_GPU
+class GpuResources {
+  GpuResources() = delete;
+};
+#endif  // !MEDIAPIPE_DISABLE_GPU
+
 extern const GraphService<GpuResources> kGpuService;
 
 }  // namespace mediapipe

mediapipe/gpu/gpu_shared_data_internal.cc

@@ -105,7 +105,7 @@ GpuResources::~GpuResources() {
 }
 
 absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) {
-  CHECK(node->UsesGpu());
+  CHECK(ContainsKey(node->Contract().ServiceRequests(), kGpuService.key));
   std::string node_id = node->GetCalculatorState().NodeName();
   std::string node_type = node->GetCalculatorState().CalculatorType();
   std::string context_key;

mediapipe/gpu/graph_support.h

@@ -16,10 +16,14 @@
 #ifndef MEDIAPIPE_GPU_GRAPH_SUPPORT_H_
 #define MEDIAPIPE_GPU_GRAPH_SUPPORT_H_
 
-#include "mediapipe/gpu/gpu_service.h"
+#include "mediapipe/framework/graph_service.h"
 
 namespace mediapipe {
 
+// Forward declaration to avoid depending on GpuResources here.
+class GpuResources;
+extern const GraphService<GpuResources> kGpuService;
+
 static constexpr char kGpuSharedTagName[] = "GPU_SHARED";
 static constexpr char kGpuSharedSidePacketName[] = "gpu_shared";
 static constexpr char kGpuExecutorName[] = "__gpu";

mediapipe/gpu/image_frame_to_gpu_buffer_calculator.cc

@@ -16,7 +16,10 @@
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/gpu/gl_calculator_helper.h"
-#include "mediapipe/gpu/gpu_buffer_storage_image_frame.h"
+
+#ifdef __APPLE__
+#include "mediapipe/objc/util.h"
+#endif
 
 namespace mediapipe {
 
@@ -31,7 +34,9 @@ class ImageFrameToGpuBufferCalculator : public CalculatorBase {
   absl::Status Process(CalculatorContext* cc) override;
 
  private:
+#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
   GlCalculatorHelper helper_;
+#endif  // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
 };
 REGISTER_CALCULATOR(ImageFrameToGpuBufferCalculator);
 
@@ -51,25 +56,28 @@ absl::Status ImageFrameToGpuBufferCalculator::Open(CalculatorContext* cc) {
   // Inform the framework that we always output at the same timestamp
   // as we receive a packet at.
   cc->SetOffset(TimestampDiff(0));
+#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
   MP_RETURN_IF_ERROR(helper_.Open(cc));
+#endif  // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
   return absl::OkStatus();
 }
 
 absl::Status ImageFrameToGpuBufferCalculator::Process(CalculatorContext* cc) {
-  auto image_frame = std::const_pointer_cast<ImageFrame>(
-      mediapipe::SharedPtrWithPacket<ImageFrame>(
-          cc->Inputs().Index(0).Value()));
-  auto gpu_buffer = MakePacket<GpuBuffer>(
-                        std::make_shared<mediapipe::GpuBufferStorageImageFrame>(
-                            std::move(image_frame)))
-                        .At(cc->InputTimestamp());
-  // Request GPU access to ensure the data is available to the GPU.
-  // TODO: have a better way to do this, or defer until later.
-  helper_.RunInGlContext([&gpu_buffer] {
-    auto view = gpu_buffer.Get<GpuBuffer>().GetReadView<GlTextureView>(0);
+#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
+  CFHolder<CVPixelBufferRef> buffer;
+  MP_RETURN_IF_ERROR(CreateCVPixelBufferForImageFramePacket(
+      cc->Inputs().Index(0).Value(), &buffer));
+  cc->Outputs().Index(0).Add(new GpuBuffer(buffer), cc->InputTimestamp());
+#else
+  const auto& input = cc->Inputs().Index(0).Get<ImageFrame>();
+  helper_.RunInGlContext([this, &input, &cc]() {
+    auto src = helper_.CreateSourceTexture(input);
+    auto output = src.GetFrame<GpuBuffer>();
+    glFlush();
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    src.Release();
   });
-  cc->Outputs().Index(0).AddPacket(std::move(gpu_buffer));
-
+#endif  // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
   return absl::OkStatus();
 }