// Copyright 2021 Sony Group Corporation. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "gst_video_player.h" #include #include #include #include GstVideoPlayer::GstVideoPlayer( const std::string& uri, std::unique_ptr handler) : stream_handler_(std::move(handler)) { gst_.pipeline = nullptr; gst_.playbin = nullptr; gst_.video_convert = nullptr; gst_.video_sink = nullptr; gst_.output = nullptr; gst_.bus = nullptr; gst_.buffer = nullptr; uri_ = ParseUri(uri); if (!CreatePipeline()) { std::cerr << "Failed to create a pipeline" << std::endl; DestroyPipeline(); return; } // Prerolls before getting information from the pipeline. Preroll(); // Sets internal video size and buffier. GetVideoSize(width_, height_); pixels_.reset(new uint32_t[width_ * height_]); stream_handler_->OnNotifyInitialized(); } GstVideoPlayer::~GstVideoPlayer() { Stop(); DestroyPipeline(); } // static void GstVideoPlayer::GstLibraryLoad() { gst_init(NULL, NULL); } // static void GstVideoPlayer::GstLibraryUnload() { gst_deinit(); } bool GstVideoPlayer::Play() { if (gst_element_set_state(gst_.pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { std::cerr << "Failed to change the state to PLAYING" << std::endl; return false; } return true; } bool GstVideoPlayer::Pause() { if (gst_element_set_state(gst_.pipeline, GST_STATE_PAUSED) == GST_STATE_CHANGE_FAILURE) { std::cerr << "Failed to change the state to PAUSED" << std::endl; return false; } return true; } bool GstVideoPlayer::Stop() { if (gst_element_set_state(gst_.pipeline, GST_STATE_READY) == GST_STATE_CHANGE_FAILURE) { std::cerr << "Failed to change the state to READY" << std::endl; return false; } return true; } bool GstVideoPlayer::SetVolume(double volume) { if (!gst_.playbin) { return false; } volume_ = volume; g_object_set(gst_.playbin, "volume", volume, NULL); return true; } bool GstVideoPlayer::SetPlaybackRate(double rate) { if (!gst_.playbin) { return false; } if (rate <= 0) { std::cerr << "Rate " << rate << " is not supported" << std::endl; return false; } if (!gst_element_seek(gst_.pipeline, rate, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, GST_SEEK_TYPE_SET, GetCurrentPosition() * GST_MSECOND, GST_SEEK_TYPE_SET, GST_CLOCK_TIME_NONE)) { std::cerr << "Failed to set playback rate to " << rate << " (gst_element_seek failed)" << std::endl; return false; } playback_rate_ = rate; mute_ = (rate < 0.5 || rate > 2); g_object_set(gst_.playbin, "mute", mute_, NULL); return true; } bool GstVideoPlayer::SetSeek(int64_t position) { auto nanosecond = position * 1000 * 1000; if (!gst_element_seek( gst_.pipeline, playback_rate_, GST_FORMAT_TIME, (GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_KEY_UNIT), GST_SEEK_TYPE_SET, nanosecond, GST_SEEK_TYPE_SET, GST_CLOCK_TIME_NONE)) { std::cerr << "Failed to seek " << nanosecond << std::endl; return false; } return true; } int64_t GstVideoPlayer::GetDuration() { GstFormat fmt = GST_FORMAT_TIME; int64_t duration_msec; if (!gst_element_query_duration(gst_.pipeline, fmt, &duration_msec)) { std::cerr << "Failed to get duration" << std::endl; return -1; } duration_msec /= GST_MSECOND; return duration_msec; } int64_t GstVideoPlayer::GetCurrentPosition() { gint64 position = 0; if (!gst_element_query_position(gst_.pipeline, GST_FORMAT_TIME, &position)) { std::cerr << "Failed to get current position" << std::endl; } return position / GST_MSECOND; } const uint8_t* GstVideoPlayer::GetFrameBuffer() { std::shared_lock lock(mutex_buffer_); if (!gst_.buffer) { return nullptr; } const uint32_t pixel_bytes = width_ * height_ * 4; gst_buffer_extract(gst_.buffer, 0, pixels_.get(), pixel_bytes); return reinterpret_cast(pixels_.get()); } // Creats a video pipeline using playbin. // $ playbin uri= video-sink="videoconvert ! video/x-raw,format=RGBA ! // fakesink" bool GstVideoPlayer::CreatePipeline() { gst_.pipeline = gst_pipeline_new("pipeline"); if (!gst_.pipeline) { std::cerr << "Failed to create a pipeline" << std::endl; return false; } gst_.playbin = gst_element_factory_make("playbin", "playbin"); if (!gst_.playbin) { std::cerr << "Failed to create a source" << std::endl; return false; } gst_.video_convert = gst_element_factory_make("videoconvert", "videoconvert"); if (!gst_.video_convert) { std::cerr << "Failed to create a videoconvert" << std::endl; return false; } gst_.video_sink = gst_element_factory_make("fakesink", "videosink"); if (!gst_.video_sink) { std::cerr << "Failed to create a videosink" << std::endl; return false; } gst_.output = gst_bin_new("output"); if (!gst_.output) { std::cerr << "Failed to create an output" << std::endl; return false; } gst_.bus = gst_pipeline_get_bus(GST_PIPELINE(gst_.pipeline)); if (!gst_.bus) { std::cerr << "Failed to create a bus" << std::endl; return false; } gst_bus_set_sync_handler(gst_.bus, (GstBusSyncHandler)HandleGstMessage, this, NULL); // Sets properties to fakesink to get the callback of a decoded frame. g_object_set(G_OBJECT(gst_.video_sink), "sync", TRUE, "qos", FALSE, NULL); g_object_set(G_OBJECT(gst_.video_sink), "signal-handoffs", TRUE, NULL); g_signal_connect(G_OBJECT(gst_.video_sink), "handoff", G_CALLBACK(HandoffHandler), this); gst_bin_add_many(GST_BIN(gst_.output), gst_.video_convert, gst_.video_sink, NULL); // Adds caps to the converter to convert the color format to RGBA. auto* caps = gst_caps_from_string("video/x-raw,format=RGBA"); auto link_ok = gst_element_link_filtered(gst_.video_convert, gst_.video_sink, caps); gst_caps_unref(caps); if (!link_ok) { std::cerr << "Failed to link elements" << std::endl; return false; } auto* sinkpad = gst_element_get_static_pad(gst_.video_convert, "sink"); auto* ghost_sinkpad = gst_ghost_pad_new("sink", sinkpad); gst_pad_set_active(ghost_sinkpad, TRUE); gst_element_add_pad(gst_.output, ghost_sinkpad); // Sets properties to playbin. g_object_set(gst_.playbin, "uri", uri_.c_str(), NULL); g_object_set(gst_.playbin, "video-sink", gst_.output, NULL); gst_bin_add_many(GST_BIN(gst_.pipeline), gst_.playbin, NULL); return true; } void GstVideoPlayer::Preroll() { if (!gst_.playbin) { return; } auto result = gst_element_set_state(gst_.pipeline, GST_STATE_PAUSED); if (result == GST_STATE_CHANGE_FAILURE) { std::cerr << "Failed to change the state to PAUSED" << std::endl; return; } // Waits until the state becomes GST_STATE_PAUSED. if (result == GST_STATE_CHANGE_ASYNC) { GstState state; result = gst_element_get_state(gst_.pipeline, &state, NULL, GST_CLOCK_TIME_NONE); if (result == GST_STATE_CHANGE_FAILURE) { std::cerr << "Failed to get the current state" << std::endl; } } } void GstVideoPlayer::DestroyPipeline() { if (gst_.video_sink) { g_object_set(G_OBJECT(gst_.video_sink), "signal-handoffs", FALSE, NULL); } if (gst_.pipeline) { gst_element_set_state(gst_.pipeline, GST_STATE_NULL); } if (gst_.buffer) { gst_buffer_unref(gst_.buffer); gst_.buffer = nullptr; } if (gst_.bus) { gst_object_unref(gst_.bus); gst_.bus = nullptr; } if (gst_.pipeline) { gst_object_unref(gst_.pipeline); gst_.pipeline = nullptr; } if (gst_.playbin) { gst_.playbin = nullptr; } if (gst_.output) { gst_.output = nullptr; } if (gst_.video_sink) { gst_.video_sink = nullptr; } if (gst_.video_convert) { gst_.video_convert = nullptr; } } std::string GstVideoPlayer::ParseUri(const std::string& uri) { if (gst_uri_is_valid(uri.c_str())) { return uri; } const auto* filename_uri = gst_filename_to_uri(uri.c_str(), NULL); if (!filename_uri) { std::cerr << "Faild to open " << uri.c_str() << std::endl; return uri; } std::string result_uri(filename_uri); delete filename_uri; return result_uri; } void GstVideoPlayer::GetVideoSize(int32_t& width, int32_t& height) { if (!gst_.pipeline || !gst_.video_sink) { std::cerr << "Failed to get video size. The pileline hasn't initialized yet."; return; } auto* sink_pad = gst_element_get_static_pad(gst_.video_sink, "sink"); if (!sink_pad) { std::cerr << "Failed to get a pad"; return; } auto* caps = gst_pad_get_current_caps(sink_pad); auto* structure = gst_caps_get_structure(caps, 0); if (!structure) { std::cerr << "Failed to get a structure"; return; } gst_structure_get_int(structure, "width", &width); gst_structure_get_int(structure, "height", &height); } // static void GstVideoPlayer::HandoffHandler(GstElement* fakesink, GstBuffer* buf, GstPad* new_pad, gpointer user_data) { auto* self = reinterpret_cast(user_data); auto* caps = gst_pad_get_current_caps(new_pad); auto* structure = gst_caps_get_structure(caps, 0); int width; int height; gst_structure_get_int(structure, "width", &width); gst_structure_get_int(structure, "height", &height); if (width != self->width_ || height != self->height_) { self->width_ = width; self->height_ = height; self->pixels_.reset(new uint32_t[width * height]); std::cout << "Pixel buffer size: width = " << width << ", height = " << height << std::endl; } std::lock_guard lock(self->mutex_buffer_); if (self->gst_.buffer) { gst_buffer_unref(self->gst_.buffer); self->gst_.buffer = nullptr; } self->gst_.buffer = gst_buffer_ref(buf); self->stream_handler_->OnNotifyFrameDecoded(); } // static gboolean GstVideoPlayer::HandleGstMessage(GstBus* bus, GstMessage* message, gpointer user_data) { switch (GST_MESSAGE_TYPE(message)) { case GST_MESSAGE_EOS: { auto* self = reinterpret_cast(user_data); self->stream_handler_->OnNotifyCompleted(); if (self->auto_repeat_) { self->SetSeek(0); } break; } case GST_MESSAGE_WARNING: { gchar* debug; GError* error; gst_message_parse_warning(message, &error, &debug); g_printerr("WARNING from element %s: %s\n", GST_OBJECT_NAME(message->src), error->message); g_printerr("Warning details: %s\n", debug); g_free(debug); g_error_free(error); break; } case GST_MESSAGE_ERROR: { gchar* debug; GError* error; gst_message_parse_error(message, &error, &debug); g_printerr("ERROR from element %s: %s\n", GST_OBJECT_NAME(message->src), error->message); g_printerr("Error details: %s\n", debug); g_free(debug); g_error_free(error); break; } default: break; } return TRUE; }