[video_player] Add support GstEGLImage to improve playback performance (#67)

Signed-off-by: Makoto Sato <[email protected]>

AUTHORS

@@ -6,3 +6,4 @@
 
 Sony Group Corporation
 Hidenori Matsubayashi ([email protected])
+Makoto Sato ([email protected])

packages/video_player/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.9.7
+* Improve playback performance if GstEGLImage is avalable.
+
 ## 0.9.6
 * Update for video_player_platform_interface v5.1.1 / video_player v2.4.7 / flutter 3.3.0
 

packages/video_player/README.md

@@ -44,3 +44,10 @@ playbin uri=<file> video-sink="videoconvert ! video/x-raw,format=RGBA ! fakesink
 
 #### e.g. customization for i.MX 8M platforms:
 playbin uri=<file> video-sink="imxvideoconvert_g2d ! video/x-raw,format=RGBA ! fakesink"
+
+### Enable GstEGLImage
+If GstEGLImage is enabled on your target device, adding the following code to `<user's project>/elinux/CMakeLists.txt` may improve playback performance.
+```
+add_definitions(-DUSE_EGL_IMAGE_DMABUF)
+set(USE_EGL_IMAGE_DMABUF "on")
+```

packages/video_player/elinux/CMakeLists.txt

@@ -9,6 +9,9 @@ set(PLUGIN_NAME "video_player_elinux_plugin")
 find_package(PkgConfig)
 pkg_check_modules(GLIB REQUIRED glib-2.0)
 pkg_check_modules(GSTREAMER REQUIRED gstreamer-1.0)
+if(USE_EGL_IMAGE_DMABUF)
+pkg_check_modules(GSTREAMER_GL REQUIRED gstreamer-gl-1.0)
+endif()
 
 add_library(${PLUGIN_NAME} SHARED
   "video_player_elinux_plugin.cc"
@@ -27,12 +30,24 @@ target_include_directories(${PLUGIN_NAME}
     ${GLIB_INCLUDE_DIRS}
     ${GSTREAMER_INCLUDE_DIRS}
 )
+if(USE_EGL_IMAGE_DMABUF)
+target_include_directories(${PLUGIN_NAME}
+  PRIVATE
+    ${GSTREAMER_GL_INCLUDE_DIRS}
+)
+endif()
 
 target_link_libraries(${PLUGIN_NAME}
   PRIVATE
     ${GLIB_LIBRARIES}
     ${GSTREAMER_LIBRARIES}
 )
+if(USE_EGL_IMAGE_DMABUF)
+target_link_libraries(${PLUGIN_NAME}
+  PRIVATE
+    ${GSTREAMER_GL_LIBRARIES}
+)
+endif()
 
 # List of absolute paths to libraries that should be bundled with the plugin
 set(video_player_elinux_bundled_libraries

packages/video_player/elinux/gst_video_player.cc

@@ -35,6 +35,9 @@ GstVideoPlayer::GstVideoPlayer(
 }
 
 GstVideoPlayer::~GstVideoPlayer() {
+#ifdef USE_EGL_IMAGE_DMABUF
+  UnrefEGLImage();
+#endif  // USE_EGL_IMAGE_DMABUF
   Stop();
   DestroyPipeline();
 }
@@ -167,6 +170,45 @@ int64_t GstVideoPlayer::GetCurrentPosition() {
   return position / GST_MSECOND;
 }
 
+#ifdef USE_EGL_IMAGE_DMABUF
+void* GstVideoPlayer::GetEGLImage(void* egl_display, void* egl_context) {
+  std::shared_lock<std::shared_mutex> lock(mutex_buffer_);
+  if (!gst_.buffer) {
+    return nullptr;
+  }
+
+  GstMemory* memory = gst_buffer_peek_memory(gst_.buffer, 0);
+  if (gst_is_dmabuf_memory(memory)) {
+    UnrefEGLImage();
+
+    gint fd = gst_dmabuf_memory_get_fd(memory);
+    gst_gl_display_egl_ =
+        gst_gl_display_egl_new_with_egl_display(reinterpret_cast<gpointer>(egl_display));
+    gst_gl_ctx_ = gst_gl_context_new_wrapped(
+        GST_GL_DISPLAY_CAST(gst_gl_display_egl_), reinterpret_cast<guintptr>(egl_context),
+        GST_GL_PLATFORM_EGL, GST_GL_API_GLES2);
+
+    gst_gl_context_activate(gst_gl_ctx_, TRUE);
+
+    gst_egl_image_ =
+        gst_egl_image_from_dmabuf(gst_gl_ctx_, fd, &gst_video_info_, 0, 0);
+    return reinterpret_cast<void*>(gst_egl_image_get_image(gst_egl_image_));
+  }
+  return nullptr;
+}
+
+void GstVideoPlayer::UnrefEGLImage() {
+  if (gst_egl_image_) {
+    gst_egl_image_unref(gst_egl_image_);
+    gst_object_unref(gst_gl_ctx_);
+    gst_object_unref(gst_gl_display_egl_);
+    gst_egl_image_ = NULL;
+    gst_gl_ctx_ = NULL;
+    gst_gl_display_egl_ = NULL;
+  }
+}
+#endif  // USE_EGL_IMAGE_DMABUF
+
 const uint8_t* GstVideoPlayer::GetFrameBuffer() {
   std::shared_lock<std::shared_mutex> lock(mutex_buffer_);
   if (!gst_.buffer) {
@@ -346,6 +388,14 @@ void GstVideoPlayer::GetVideoSize(int32_t& width, int32_t& height) {
 
   gst_structure_get_int(structure, "width", &width);
   gst_structure_get_int(structure, "height", &height);
+
+#ifdef USE_EGL_IMAGE_DMABUF
+  gboolean res = gst_video_info_from_caps(&gst_video_info_, caps);
+  if (!res) {
+    std::cerr << "Failed to get a gst_video_info" << std::endl;
+    return;
+  }
+#endif  // USE_EGL_IMAGE_DMABUF
 }
 
 // static

packages/video_player/elinux/gst_video_player.h

@@ -7,6 +7,13 @@
 
 #include <gst/gst.h>
 
+#ifdef USE_EGL_IMAGE_DMABUF
+#include <gst/allocators/gstdmabuf.h>
+#include <gst/gl/egl/egl.h>
+#include <gst/gl/gl.h>
+#include <gst/video/video.h>
+#endif  // USE_EGL_IMAGE_DMABUF
+
 #include <memory>
 #include <mutex>
 #include <shared_mutex>
@@ -33,6 +40,9 @@ class GstVideoPlayer {
   int64_t GetDuration();
   int64_t GetCurrentPosition();
   const uint8_t* GetFrameBuffer();
+#ifdef USE_EGL_IMAGE_DMABUF
+  void* GetEGLImage(void* egl_display, void* egl_context);
+#endif  // USE_EGL_IMAGE_DMABUF
   int32_t GetWidth() const { return width_; };
   int32_t GetHeight() const { return height_; };
 
@@ -56,6 +66,9 @@ class GstVideoPlayer {
   void DestroyPipeline();
   void Preroll();
   void GetVideoSize(int32_t& width, int32_t& height);
+#ifdef USE_EGL_IMAGE_DMABUF
+  void UnrefEGLImage();
+#endif  // USE_EGL_IMAGE_DMABUF
 
   GstVideoElements gst_;
   std::string uri_;
@@ -70,6 +83,13 @@ class GstVideoPlayer {
   std::mutex mutex_event_completed_;
   std::shared_mutex mutex_buffer_;
   std::unique_ptr<VideoPlayerStreamHandler> stream_handler_;
+
+#ifdef USE_EGL_IMAGE_DMABUF
+  GstVideoInfo gst_video_info_;
+  GstEGLImage* gst_egl_image_ = NULL;
+  GstGLContext* gst_gl_ctx_ = NULL;
+  GstGLDisplayEGL* gst_gl_display_egl_ = NULL;
+#endif  // USE_EGL_IMAGE_DMABUF
 };
 
 #endif  // PACKAGES_VIDEO_PLAYER_VIDEO_PLAYER_ELINUX_GST_VIDEO_PLAYER_H_

packages/video_player/elinux/video_player_elinux_plugin.cc

@@ -86,6 +86,9 @@ class VideoPlayerPlugin : public flutter::Plugin {
     std::unique_ptr<GstVideoPlayer> player;
     std::unique_ptr<flutter::TextureVariant> texture;
     std::unique_ptr<FlutterDesktopPixelBuffer> buffer;
+#ifdef USE_EGL_IMAGE_DMABUF
+    std::unique_ptr<FlutterDesktopEGLImage> egl_image;
+#endif  // USE_EGL_IMAGE_DMABUF
     std::unique_ptr<flutter::EventChannel<flutter::EncodableValue>>
         event_channel;
     std::unique_ptr<flutter::EventSink<flutter::EncodableValue>> event_sink;
@@ -293,6 +296,20 @@ void VideoPlayerPlugin::HandleCreateMethodCall(
   }
 
   auto instance = std::make_unique<FlutterVideoPlayer>();
+#ifdef USE_EGL_IMAGE_DMABUF
+  instance->egl_image = std::make_unique<FlutterDesktopEGLImage>();
+  instance->texture =
+      std::make_unique<flutter::TextureVariant>(flutter::EGLImageTexture(
+          [instance = instance.get()](
+              size_t width, size_t height, void* egl_display,
+              void* egl_context) -> const FlutterDesktopEGLImage* {
+            instance->egl_image->width = instance->player->GetWidth();
+            instance->egl_image->height = instance->player->GetHeight();
+            instance->egl_image->egl_image =
+                instance->player->GetEGLImage(egl_display, egl_context);
+            return instance->egl_image.get();
+          }));
+#else
   instance->buffer = std::make_unique<FlutterDesktopPixelBuffer>();
   instance->texture =
       std::make_unique<flutter::TextureVariant>(flutter::PixelBufferTexture(
@@ -303,6 +320,7 @@ void VideoPlayerPlugin::HandleCreateMethodCall(
             instance->buffer->buffer = instance->player->GetFrameBuffer();
             return instance->buffer.get();
           }));
+#endif  // USE_EGL_IMAGE_DMABUF
   const auto texture_id =
       texture_registrar_->RegisterTexture(instance->texture.get());
   instance->texture_id = texture_id;

packages/video_player/pubspec.yaml

@@ -1,6 +1,6 @@
 name: video_player_elinux
 description: Flutter plugin for displaying inline video with other Flutter widgets on Embedded Linux.
-version: 0.9.6
+version: 0.9.7
 homepage: https://github.com/sony/flutter-elinux-plugins
 repository: https://github.com/sony/flutter-elinux-plugins/tree/main/packages/video_player