Project import generated by Copybara.

GitOrigin-RevId: 1e13be30e2c6838d4a2ff768a39c414bc80534bb

mediapipe/gpu/BUILD

@@ -385,9 +385,7 @@ objc_library(
         "CoreVideo",
     ],
     visibility = ["//visibility:public"],
-    deps = [
-        "//mediapipe/objc:util",
-    ],
+    deps = ["//mediapipe/objc:util"],
 )
 
 objc_library(
@@ -421,10 +419,11 @@ objc_library(
         ":gpu_buffer_multi_pool",
         ":gpu_shared_data_header",
         ":graph_support",
+        "//mediapipe/gpu:gl_context_options_cc_proto",
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework/port:ret_check",
-        "//mediapipe/gpu:gl_context_options_cc_proto",
         "@google_toolbox_for_mac//:GTM_Defines",
+    ] + [
     ],
 )
 

mediapipe/gpu/gl_animation_overlay_calculator.proto

@@ -0,0 +1,41 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message GlAnimationOverlayCalculatorOptions {
+  extend CalculatorOptions {
+    optional GlAnimationOverlayCalculatorOptions ext = 174760573;
+  }
+
+  // Default aspect ratio of rendering target width over height.
+  // This specific value is for 3:4 view. Do not change this default value.
+  optional float aspect_ratio = 1 [default = 0.75];
+  // Default vertical field of view in degrees. This specific default value
+  // is arbitrary. Do not change this default value. If you want to use
+  // a different vertical_fov_degrees, set it in the options.
+  optional float vertical_fov_degrees = 2 [default = 70.0];
+
+  // Perspective projection matrix z-clipping near plane value.
+  optional float z_clipping_plane_near = 3 [default = 0.1];
+  // Perspective projection matrix z-clipping far plane value.
+  optional float z_clipping_plane_far = 4 [default = 1000.0];
+
+  // Speed at which to play the animation (in frames per second).
+  optional float animation_speed_fps = 5 [default = 25.0];
+}

mediapipe/gpu/gl_calculator_helper_impl_common.cc

@@ -78,9 +78,6 @@ void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) {
   }
   glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
   glViewport(0, 0, dst.width(), dst.height());
-
-  glActiveTexture(GL_TEXTURE0);
-  glBindTexture(dst.target(), dst.name());
   glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(),
                          dst.name(), 0);
 

mediapipe/gpu/gl_context.cc

@@ -569,55 +569,64 @@ class GlFinishSyncPoint : public GlSyncPoint {
   int64_t gl_finish_count_ = -1;
 };
 
-class GlFenceSyncPoint : public GlSyncPoint {
+// Just handles a GLsync. No context management.
+class GlSyncWrapper {
  public:
-  explicit GlFenceSyncPoint(const std::shared_ptr<GlContext>& gl_context)
-      : GlSyncPoint(gl_context) {
-    gl_context_->Run([this] {
-      sync_ = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
-      // Defer the flush for WebGL until the glClientWaitSync call as it's a
-      // costly IPC call in Chrome's WebGL implementation.
+  GlSyncWrapper() : sync_(nullptr) {}
+  explicit GlSyncWrapper(GLsync sync) : sync_(sync) {}
+
+  void Create() {
+    Clear();
+    sync_ = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
+    // Defer the flush for WebGL until the glClientWaitSync call as it's a
+    // costly IPC call in Chrome's WebGL implementation.
 #ifndef __EMSCRIPTEN__
-      glFlush();
+    glFlush();
 #endif
-    });
   }
 
-  ~GlFenceSyncPoint() {
-    if (sync_) {
-      GLsync sync = sync_;
-      gl_context_->RunWithoutWaiting([sync] { glDeleteSync(sync); });
-    }
+  ~GlSyncWrapper() { Clear(); }
+
+  GlSyncWrapper(const GlSyncWrapper&) = delete;
+  GlSyncWrapper(GlSyncWrapper&& other) : sync_(nullptr) {
+    *this = std::move(other);
+  }
+  GlSyncWrapper& operator=(const GlSyncWrapper&) = delete;
+  GlSyncWrapper& operator=(GlSyncWrapper&& other) {
+    using std::swap;
+    swap(sync_, other.sync_);
+    return *this;
+  }
+  GlSyncWrapper& operator=(std::nullptr_t) {
+    Clear();
+    return *this;
   }
 
-  GlFenceSyncPoint(const GlFenceSyncPoint&) = delete;
-  GlFenceSyncPoint& operator=(const GlFenceSyncPoint&) = delete;
+  operator bool() const { return sync_ != nullptr; }
+  bool operator==(std::nullptr_t) const { return sync_ == nullptr; }
+  bool operator!=(std::nullptr_t) const { return sync_ != nullptr; }
 
-  void Wait() override {
+  void Wait() {
     if (!sync_) return;
-    gl_context_->Run([this] {
-      GLuint flags = 0;
-      uint64_t timeout = std::numeric_limits<uint64_t>::max();
+    GLuint flags = 0;
+    uint64_t timeout = std::numeric_limits<uint64_t>::max();
 #ifdef __EMSCRIPTEN__
-      // Setting GL_SYNC_FLUSH_COMMANDS_BIT ensures flush happens before we wait
-      // on the fence. This is necessary since we defer the flush on WebGL.
-      flags = GL_SYNC_FLUSH_COMMANDS_BIT;
-      // WebGL only supports small implementation dependent timeout values. In
-      // particular, Chrome only supports a timeout of 0.
-      timeout = 0;
+    // Setting GL_SYNC_FLUSH_COMMANDS_BIT ensures flush happens before we wait
+    // on the fence. This is necessary since we defer the flush on WebGL.
+    flags = GL_SYNC_FLUSH_COMMANDS_BIT;
+    // WebGL only supports small implementation dependent timeout values. In
+    // particular, Chrome only supports a timeout of 0.
+    timeout = 0;
 #endif
-      GLenum result = glClientWaitSync(sync_, flags, timeout);
-      if (result == GL_ALREADY_SIGNALED || result == GL_CONDITION_SATISFIED) {
-        glDeleteSync(sync_);
-        sync_ = nullptr;
-      }
-      // TODO: do something if the wait fails?
-    });
+    GLenum result = glClientWaitSync(sync_, flags, timeout);
+    if (result == GL_ALREADY_SIGNALED || result == GL_CONDITION_SATISFIED) {
+      Clear();
+    }
+    // TODO: do something if the wait fails?
   }
 
-  void WaitOnGpu() override {
+  void WaitOnGpu() {
     if (!sync_) return;
-      // TODO: do not wait if we are already on the same context?
       // WebGL2 specifies a waitSync call, but since cross-context
       // synchronization is not supported, it's actually a no-op. Firefox prints
       // a warning when it's called, so let's just skip the call. See
@@ -627,36 +636,122 @@ class GlFenceSyncPoint : public GlSyncPoint {
 #endif
   }
 
+  bool IsReady() {
+    if (!sync_) return true;
+    GLuint flags = 0;
+#ifdef __EMSCRIPTEN__
+    // Setting GL_SYNC_FLUSH_COMMANDS_BIT ensures flush happens before we wait
+    // on the fence. This is necessary since we defer the flush on WebGL.
+    flags = GL_SYNC_FLUSH_COMMANDS_BIT;
+#endif
+    GLenum result = glClientWaitSync(sync_, flags, 0);
+    if (result == GL_ALREADY_SIGNALED || result == GL_CONDITION_SATISFIED) {
+      Clear();
+      return true;
+    }
+    return false;
+  }
+
+ private:
+  void Clear() {
+    if (sync_) {
+      glDeleteSync(sync_);
+      sync_ = nullptr;
+    }
+  }
+
+  GLsync sync_;
+};
+
+class GlFenceSyncPoint : public GlSyncPoint {
+ public:
+  explicit GlFenceSyncPoint(const std::shared_ptr<GlContext>& gl_context)
+      : GlSyncPoint(gl_context) {
+    gl_context_->Run([this] { sync_.Create(); });
+  }
+
+  ~GlFenceSyncPoint() {
+    if (sync_) {
+      gl_context_->RunWithoutWaiting(
+          [sync = new GlSyncWrapper(std::move(sync_))] { delete sync; });
+    }
+  }
+
+  GlFenceSyncPoint(const GlFenceSyncPoint&) = delete;
+  GlFenceSyncPoint& operator=(const GlFenceSyncPoint&) = delete;
+
+  void Wait() override {
+    if (!sync_) return;
+    gl_context_->Run([this] {
+      // TODO: must this run on the original context??
+      sync_.Wait();
+    });
+  }
+
+  void WaitOnGpu() override {
+    if (!sync_) return;
+    // TODO: do not wait if we are already on the same context?
+    sync_.WaitOnGpu();
+  }
+
   bool IsReady() override {
     if (!sync_) return true;
     bool ready = false;
     // TODO: we should not block on the original context if possible.
-    gl_context_->Run([this, &ready] {
-      GLuint flags = 0;
-#ifdef __EMSCRIPTEN__
-      // Setting GL_SYNC_FLUSH_COMMANDS_BIT ensures flush happens before we wait
-      // on the fence. This is necessary since we defer the flush on WebGL.
-      flags = GL_SYNC_FLUSH_COMMANDS_BIT;
-#endif
-      GLenum result = glClientWaitSync(sync_, flags, 0);
-      if (result == GL_ALREADY_SIGNALED || result == GL_CONDITION_SATISFIED) {
-        glDeleteSync(sync_);
-        sync_ = nullptr;
-        ready = true;
-      }
-    });
+    gl_context_->Run([this, &ready] { ready = sync_.IsReady(); });
     return ready;
   }
 
  private:
-  GLsync sync_;
+  GlSyncWrapper sync_;
+};
+
+class GlExternalFenceSyncPoint : public GlSyncPoint {
+ public:
+  // The provided GlContext is used as a fallback when a context is needed (e.g.
+  // for deletion), but it's not the context the sync was created on, so we pass
+  // nullptr to GlSyncPoint.
+  explicit GlExternalFenceSyncPoint(
+      const std::shared_ptr<GlContext>& graph_service_gl_context)
+      : GlSyncPoint(nullptr),
+        graph_service_gl_context_(graph_service_gl_context) {
+    sync_.Create();
+  }
+
+  ~GlExternalFenceSyncPoint() {
+    if (sync_) {
+      graph_service_gl_context_->RunWithoutWaiting(
+          [sync = new GlSyncWrapper(std::move(sync_))] { delete sync; });
+    }
+  }
+
+  GlExternalFenceSyncPoint(const GlExternalFenceSyncPoint&) = delete;
+  GlExternalFenceSyncPoint& operator=(const GlExternalFenceSyncPoint&) = delete;
+
+  void Wait() override {
+    // TODO: can we assume this is always called with a GLContext being current?
+    sync_.Wait();
+  }
+
+  void WaitOnGpu() override { sync_.WaitOnGpu(); }
+
+  bool IsReady() override {
+    // TODO: can we assume this is always called with a GLContext being current?
+    return sync_.IsReady();
+  }
+
+ private:
+  GlSyncWrapper sync_;
+  std::shared_ptr<GlContext> graph_service_gl_context_;
 };
 
 void GlMultiSyncPoint::Add(std::shared_ptr<GlSyncPoint> new_sync) {
-  for (auto& sync : syncs_) {
-    if (sync->GetContext() == new_sync->GetContext()) {
-      sync = std::move(new_sync);
-      return;
+  if (new_sync->GetContext() != nullptr) {
+    for (auto& sync : syncs_) {
+      if (sync->GetContext() == new_sync->GetContext()) {
+        sync = std::move(new_sync);
+        return;
+      }
     }
   }
   syncs_.emplace_back(std::move(new_sync));
@@ -701,22 +796,24 @@ class GlNopSyncPoint : public GlSyncPoint {
 };
 #endif
 
-std::shared_ptr<GlSyncPoint> GlContext::CreateSyncToken() {
-  std::shared_ptr<GlSyncPoint> token;
-#if MEDIAPIPE_DISABLE_GL_SYNC_FOR_DEBUG
-  token.reset(new GlNopSyncPoint(shared_from_this()));
-#else
-
+bool GlContext::ShouldUseFenceSync() const {
 #ifdef __EMSCRIPTEN__
   // In Emscripten the glWaitSync function is non-null depending on linkopts,
   // but only works in a WebGL2 context, so fall back to use Finish if it is a
   // WebGL1/ES2 context.
   // TODO: apply this more generally once b/152794517 is fixed.
-  bool useFenceSync = gl_major_version() > 2;
+  return gl_major_version() > 2;
 #else
-  bool useFenceSync = SymbolAvailable(&glWaitSync);
+  return SymbolAvailable(&glWaitSync);
 #endif  // __EMSCRIPTEN__
-  if (useFenceSync) {
+}
+
+std::shared_ptr<GlSyncPoint> GlContext::CreateSyncToken() {
+  std::shared_ptr<GlSyncPoint> token;
+#if MEDIAPIPE_DISABLE_GL_SYNC_FOR_DEBUG
+  token.reset(new GlNopSyncPoint(shared_from_this()));
+#else
+  if (ShouldUseFenceSync()) {
     token.reset(new GlFenceSyncPoint(shared_from_this()));
   } else {
     token.reset(new GlFinishSyncPoint(shared_from_this()));
@@ -725,6 +822,19 @@ std::shared_ptr<GlSyncPoint> GlContext::CreateSyncToken() {
   return token;
 }
 
+std::shared_ptr<GlSyncPoint>
+GlContext::CreateSyncTokenForCurrentExternalContext(
+    const std::shared_ptr<GlContext>& delegate_graph_context) {
+  CHECK(delegate_graph_context);
+  if (delegate_graph_context->ShouldUseFenceSync()) {
+    return std::shared_ptr<GlSyncPoint>(
+        new GlExternalFenceSyncPoint(delegate_graph_context));
+  } else {
+    glFinish();
+    return nullptr;
+  }
+}
+
 std::shared_ptr<GlSyncPoint> GlContext::TestOnly_CreateSpecificSyncToken(
     SyncTokenTypeForTest type) {
   std::shared_ptr<GlSyncPoint> token;

mediapipe/gpu/gl_context.h

@@ -94,6 +94,7 @@ class GlSyncPoint {
   // Returns whether the sync point has been reached. Does not block.
   virtual bool IsReady() = 0;
 
+  // Returns the GlContext object associated with this sync point, if any.
   const std::shared_ptr<GlContext>& GetContext() { return gl_context_; }
 
  protected:
@@ -302,6 +303,15 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
     return *static_cast<T*>(entry.get());
   }
 
+  // Creates a synchronization token for the current, non-GlContext-owned
+  // context. This can be passed to MediaPipe so it can synchronize with the
+  // commands issued in the external context up to this point.
+  // Note: if the current context does not support sync fences, this calls
+  // glFinish and returns nullptr.
+  // TODO: return GlNopSyncPoint instead?
+  static std::shared_ptr<GlSyncPoint> CreateSyncTokenForCurrentExternalContext(
+      const std::shared_ptr<GlContext>& delegate_graph_context);
+
   // These are used for testing specific SyncToken implementations. Do not use
   // outside of tests.
   enum class SyncTokenTypeForTest {
@@ -313,6 +323,8 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
  private:
   GlContext();
 
+  bool ShouldUseFenceSync() const;
+
 #if defined(__EMSCRIPTEN__)
   absl::Status CreateContext(EMSCRIPTEN_WEBGL_CONTEXT_HANDLE share_context);
   absl::Status CreateContextInternal(

mediapipe/gpu/shader_util.cc

@@ -50,7 +50,8 @@ namespace mediapipe {
 
 constexpr int kMaxShaderInfoLength = 1024;
 
-GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader) {
+GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader,
+                       bool force_log_errors) {
   *shader = glCreateShader(target);
   if (*shader == 0) {
     return GL_FALSE;
@@ -61,8 +62,11 @@ GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader) {
   GL_DEBUG_LOG(Shader, *shader, "compile");
 
 #if UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING
-  return GL_TRUE;
-#else
+  if (!force_log_errors) {
+    return GL_TRUE;
+  }
+#endif  // UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING
+
   GLint status;
 
   glGetShaderiv(*shader, GL_COMPILE_STATUS, &status);
@@ -75,15 +79,17 @@ GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader) {
     LOG(ERROR) << "Error message: " << std::string(cmessage, length);
   }
   return status;
-#endif  // UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING
 }
 
-GLint GlhLinkProgram(GLuint program) {
+GLint GlhLinkProgram(GLuint program, bool force_log_errors) {
   glLinkProgram(program);
 
 #if UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING
-  return GL_TRUE;
-#else
+  if (!force_log_errors) {
+    return GL_TRUE;
+  }
+#endif  // UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING
+
   GLint status;
 
   GL_DEBUG_LOG(Program, program, "link");
@@ -92,7 +98,6 @@ GLint GlhLinkProgram(GLuint program) {
   LOG_IF(ERROR, status == GL_FALSE) << "Failed to link program " << program;
 
   return status;
-#endif  // UNSAFE_EMSCRIPTEN_SKIP_GL_ERROR_HANDLING
 }
 
 GLint GlhValidateProgram(GLuint program) {
@@ -110,7 +115,8 @@ GLint GlhValidateProgram(GLuint program) {
 
 GLint GlhCreateProgram(const GLchar* vert_src, const GLchar* frag_src,
                        GLsizei attr_count, const GLchar* const* attr_names,
-                       const GLint* attr_locations, GLuint* program) {
+                       const GLint* attr_locations, GLuint* program,
+                       bool force_log_errors) {
   GLuint vert_shader = 0;
   GLuint frag_shader = 0;
   GLint ok = GL_TRUE;
@@ -120,8 +126,10 @@ GLint GlhCreateProgram(const GLchar* vert_src, const GLchar* frag_src,
     return GL_FALSE;
   }
 
-  ok = ok && GlhCompileShader(GL_VERTEX_SHADER, vert_src, &vert_shader);
-  ok = ok && GlhCompileShader(GL_FRAGMENT_SHADER, frag_src, &frag_shader);
+  ok = ok && GlhCompileShader(GL_VERTEX_SHADER, vert_src, &vert_shader,
+                              force_log_errors);
+  ok = ok && GlhCompileShader(GL_FRAGMENT_SHADER, frag_src, &frag_shader,
+                              force_log_errors);
 
   if (ok) {
     glAttachShader(*program, vert_shader);

mediapipe/gpu/shader_util.h

@@ -25,11 +25,12 @@ namespace mediapipe {
 // TODO: Remove the C-style helpers.
 // Compiles a GLSL shader, logs errors, returns the compile status
 // (GL_TRUE for success, GL_FALSE for failure).
-GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader);
+GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader,
+                       bool force_log_errors = false);
 
 // Links a GLSL program, logs errors, returns the link status
 // (GL_TRUE for success, GL_FALSE for failure).
-GLint GlhLinkProgram(GLuint program);
+GLint GlhLinkProgram(GLuint program, bool force_log_errors = false);
 
 // Validates a GLSL program, logs errors, returns the validate status
 // (GL_TRUE for success, GL_FALSE for failure).
@@ -40,7 +41,8 @@ GLint GlhValidateProgram(GLuint program);
 // Return GL_TRUE for success, GL_FALSE for failure.
 GLint GlhCreateProgram(const GLchar* vert_src, const GLchar* frag_src,
                        GLsizei attr_count, const GLchar* const* attr_names,
-                       const GLint* attr_locations, GLuint* program);
+                       const GLint* attr_locations, GLuint* program,
+                       bool force_log_errors = false);
 
 // Compiles a shader specified by shader_source. Returns true on success.
 bool CompileShader(GLenum shader_type, const std::string& shader_source,