qgstvideorenderersink.cpp

Go to the documentation of this file.

// Copyright (C) 2016 Jolla Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qgstvideorenderersink_p.h"
 
#include <QtMultimedia/qvideoframe.h>
#include <QtMultimedia/qvideosink.h>
#include <QtMultimedia/private/qvideoframe_p.h>
#include <QtGui/rhi/qrhi.h>
#include <QtGui/qguiapplication.h>
#include <QtGui/qopenglcontext.h>
#include <QtCore/qcoreapplication.h>
#include <QtCore/qdebug.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/private/qfactoryloader_p.h>
#include <QtCore/private/quniquehandle_p.h>
 
#include <common/qgst_debug_p.h>
#include <common/qgstreamermetadata_p.h>
#include <common/qgstreamervideosink_p.h>
#include <common/qgstutils_p.h>
#include <common/qgstvideobuffer_p.h>
 
#include <gst/video/video.h>
#include <gst/video/gstvideometa.h>
 
 
#if QT_CONFIG(gstreamer_gl)
#include <gst/gl/gl.h>
#endif // #if QT_CONFIG(gstreamer_gl)
 
// DMA support
#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
#  include <gst/allocators/gstdmabuf.h>
#endif
 
// NOLINTBEGIN(readability-convert-member-functions-to-static)
 
Q_STATIC_LOGGING_CATEGORY(qLcGstVideoRenderer, "qt.multimedia.gstvideorenderer");
 
QT_BEGIN_NAMESPACE
 
QGstVideoRenderer::QGstVideoRenderer(QGstreamerRelayVideoSink *sink)
    : m_sink(sink), m_surfaceCaps(createSurfaceCaps(sink))
{
    QObject::connect(
            sink, &QGstreamerRelayVideoSink::aboutToBeDestroyed, this,
            [this] {
                QMutexLocker locker(&m_sinkMutex);
                m_sink = nullptr;
            },
            Qt::DirectConnection);
}
 
QGstVideoRenderer::~QGstVideoRenderer() = default;
 
QGstCaps QGstVideoRenderer::createSurfaceCaps([[maybe_unused]] QGstreamerRelayVideoSink *sink)
{
    QGstCaps caps = QGstCaps::create();
 
    // All the formats that both we and gstreamer support
    auto formats = QList<QVideoFrameFormat::PixelFormat>()
                   << QVideoFrameFormat::Format_YUV420P
                   << QVideoFrameFormat::Format_YUV422P
                   << QVideoFrameFormat::Format_YV12
                   << QVideoFrameFormat::Format_UYVY
                   << QVideoFrameFormat::Format_YUYV
                   << QVideoFrameFormat::Format_NV12
                   << QVideoFrameFormat::Format_NV21
                   << QVideoFrameFormat::Format_AYUV
                   << QVideoFrameFormat::Format_P010
                   << QVideoFrameFormat::Format_XRGB8888
                   << QVideoFrameFormat::Format_XBGR8888
                   << QVideoFrameFormat::Format_RGBX8888
                   << QVideoFrameFormat::Format_BGRX8888
                   << QVideoFrameFormat::Format_ARGB8888
                   << QVideoFrameFormat::Format_ABGR8888
                   << QVideoFrameFormat::Format_RGBA8888
                   << QVideoFrameFormat::Format_BGRA8888
                   << QVideoFrameFormat::Format_Y8
                   << QVideoFrameFormat::Format_Y16
        ;
    caps.addPixelFormats(formats);
#if QT_CONFIG(gstreamer_gl)
    QRhi *rhi = sink->rhi();
    if (rhi && rhi->backend() == QRhi::OpenGLES2) {
        caps.addPixelFormats(formats, GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
#  if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
        if (sink->eglDisplay() && sink->eglImageTargetTexture2D()) {
            caps.addPixelFormats(formats, GST_CAPS_FEATURE_MEMORY_DMABUF);
        }
#  endif
    }
#endif
    return caps;
}
 
void QGstVideoRenderer::customEvent(QEvent *event)
{
QT_WARNING_PUSH
QT_WARNING_DISABLE_GCC("-Wswitch") // case value not in enumerated type ‘QEvent::Type’
 
    switch (event->type()) {
    case renderFramesEvent: {
        // LATER: we currently show every frame. however it may be reasonable to drop frames
        // here if the queue contains more than one frame
        while (std::optional<RenderBufferState> nextState = m_bufferQueue.dequeue())
            handleNewBuffer(std::move(*nextState));
        return;
    }
    case stopEvent: {
        m_currentPipelineFrame = {};
        updateCurrentVideoFrame(m_currentVideoFrame);
        return;
    }
 
    default:
        return;
    }
QT_WARNING_POP
}
 
 
void QGstVideoRenderer::handleNewBuffer(RenderBufferState state)
{
    auto videoBuffer =
            std::make_unique<QGstVideoBuffer>(state.buffer, state.videoInfo, state.format);
    QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(videoBuffer), state.format);
    QGstUtils::setFrameTimeStampsFromBuffer(&frame, state.buffer.get());
    m_currentPipelineFrame = std::move(frame);
 
    if (!m_isActive) {
        qCDebug(qLcGstVideoRenderer) << "    showing empty video frame";
        updateCurrentVideoFrame({});
        return;
    }
 
    updateCurrentVideoFrame(m_currentPipelineFrame);
}
 
const QGstCaps &QGstVideoRenderer::caps()
{
    return m_surfaceCaps;
}
 
bool QGstVideoRenderer::start(const QGstCaps& caps)
{
    qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::start" << caps;
 
    auto optionalVideoInfo = caps.videoInfo();
    if (optionalVideoInfo) {
        m_videoInfo = std::move(*optionalVideoInfo);
        m_format = qVideoFrameFormatFromGstVideoInfo(m_videoInfo);
    } else {
        m_format = {};
        m_videoInfo = {};
    }
    m_capsMemoryFormat = caps.memoryFormat();
 
    // NOTE: m_format will not be fully populated until GST_EVENT_TAG is processed
 
    return true;
}
 
void QGstVideoRenderer::stop()
{
    qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::stop";
 
    m_bufferQueue.clear();
    QCoreApplication::postEvent(this, new QEvent(stopEvent));
}
 
void QGstVideoRenderer::unlock()
{
    qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::unlock";
}
 
bool QGstVideoRenderer::proposeAllocation(GstQuery *query)
{
    if (!query || GST_QUERY_TYPE(query) != GST_QUERY_ALLOCATION)
        return false;
 
    GstCaps *queryCaps = nullptr;
    gboolean needPool = false;
    gst_query_parse_allocation(query, &queryCaps, &needPool);
    GstVideoInfo info;
    int size = 0;
    if (queryCaps && gst_video_info_from_caps(&info, queryCaps)) {
        size = info.size;
    } else {
        qWarning(qLcGstVideoRenderer) << "QGstVideoRenderer::proposeAllocation failed to "
                                          "get size from query caps";
        return true;
    }
 
    constexpr int defaultMinBuffers = 3;
    static const int env
            = qEnvironmentVariableIntValue("QT_GSTREAMER_PROPOSE_ALLOCATION_MIN_BUFFERS");
    static const int minBuffers = env ? env : defaultMinBuffers;
    qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::proposeAllocation: "
                                    "needPool:" << needPool
                                 << "size:" << size
                                 << "minBuffers:" << minBuffers;
 
    // This call is needed to avoid enabling of copy threshold by v4l2 decoders, which can result
    // in a mix of dmabuf and system memory buffers. The query sender should use its own buffer
    // pool, and will only take the size value and our suggested minimum buffers into account.
    // The driver can force a higher minimum if minBuffers is set too low, making 3 sufficient.
    gst_query_add_allocation_pool(query, nullptr, size, minBuffers, 0);
 
    // Advertise GstVideoMeta support, often needed for DMA buffer negotiation.
    gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, nullptr);
 
    return true;
}
 
GstFlowReturn QGstVideoRenderer::render(GstBuffer *buffer)
{
    qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::render";
 
    if (m_flushing) {
        qCDebug(qLcGstVideoRenderer)
                << "    buffer received while flushing the sink ... discarding buffer";
        return GST_FLOW_FLUSHING;
    }
 
    GstVideoCropMeta *meta = gst_buffer_get_video_crop_meta(buffer);
    if (meta) {
        QRect vp(meta->x, meta->y, meta->width, meta->height);
        if (m_format.viewport() != vp) {
            qCDebug(qLcGstVideoRenderer)
                    << Q_FUNC_INFO << " Update viewport on Metadata: [" << meta->height << "x"
                    << meta->width << " | " << meta->x << "x" << meta->y << "]";
            // Update viewport if data is not the same
            m_format.setViewport(vp);
        }
    }
 
    // Some gst elements, like v4l2h264dec, can provide Direct Memory Access buffers (DMA-BUF)
    // without specifying it in their caps. So we check the memory format manually:
    QGstCaps::MemoryFormat bufferMemoryFormat = [&] {
        if (m_capsMemoryFormat != QGstCaps::CpuMemory)
            return m_capsMemoryFormat;
 
        return qMemoryFormatFromGstBuffer(buffer);
    }();
 
    qCDebug(qLcGstVideoRenderer) << "m_capsMemoryFormat" << m_capsMemoryFormat
                                 << "bufferMemoryFormat" << bufferMemoryFormat;
 
    QVideoFrameFormat bufferVideoFrameFormat = m_format;
 
    // When rendering DMAbuf frames with OpenGLES, EGL seems to do implicit YUV->RGB conversion for
    // UYVY and YUYV (YUY2), so we need to change the pixel format to Format_RGBA8888 to select the
    // correct shader.
    const bool setFormat_RGBA8888 = [&] {
#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
        if ((m_format.pixelFormat() == QVideoFrameFormat::Format_UYVY
             || m_format.pixelFormat() == QVideoFrameFormat::Format_YUYV)
            && bufferMemoryFormat == QGstCaps::DMABuf
            && m_sink && m_sink->eglDisplay() && m_sink->eglImageTargetTexture2D()) {
 
            QRhi *rhi = m_sink->rhi();
            if (!rhi || rhi->backend() != QRhi::OpenGLES2)
                return false;
 
            auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());
            QOpenGLContext *glContext = nativeHandles ? nativeHandles->context : nullptr;
            return glContext && glContext->isOpenGLES();
        }
#endif
        return false;
    }();
    if (setFormat_RGBA8888) {
        // TODO: Replace with new private setter of pixel format.
        qCDebug(qLcGstVideoRenderer) << "Setting pixel format to Format_RGBA8888";
        bufferVideoFrameFormat = QVideoFrameFormat(m_format.frameSize(),
                                                   QVideoFrameFormat::Format_RGBA8888);
        bufferVideoFrameFormat.setStreamFrameRate(m_format.streamFrameRate());
        bufferVideoFrameFormat.setColorRange(m_format.colorRange());
        bufferVideoFrameFormat.setColorTransfer(m_format.colorTransfer());
        bufferVideoFrameFormat.setColorSpace(m_format.colorSpace());
    }
 
    RenderBufferState state{
        QGstBufferHandle{ buffer, QGstBufferHandle::NeedsRef },
        bufferVideoFrameFormat,
        m_videoInfo,
        bufferMemoryFormat,
    };
 
    qCDebug(qLcGstVideoRenderer) << "    sending video frame";
 
    qsizetype sizeOfQueue = m_bufferQueue.enqueue(std::move(state));
    if (sizeOfQueue == 1)
        // we only need to wake up, if we don't have a pending frame
        QCoreApplication::postEvent(this, new QEvent(renderFramesEvent));
 
    return GST_FLOW_OK;
}
 
bool QGstVideoRenderer::query(GstQuery *query)
{
#if QT_CONFIG(gstreamer_gl)
    if (GST_QUERY_TYPE(query) == GST_QUERY_CONTEXT) {
        const gchar *type = nullptr;
        gst_query_parse_context_type(query, &type);
 
        QLatin1StringView typeStr(type);
        if (typeStr != QLatin1StringView("gst.gl.GLDisplay")
            && typeStr != QLatin1StringView("gst.gl.local_context")) {
            return false;
        }
 
        QMutexLocker locker(&m_sinkMutex);
        if (!m_sink)
            return false;
 
        auto *gstGlContext = typeStr == QLatin1StringView("gst.gl.GLDisplay")
                ? m_sink->gstGlDisplayContext() : m_sink->gstGlLocalContext();
        if (!gstGlContext)
            return false;
 
        gst_query_set_context(query, gstGlContext);
 
        return true;
    }
#else
    Q_UNUSED(query);
#endif
    return false;
}
 
void QGstVideoRenderer::gstEvent(GstEvent *event)
{
    qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent:" << event;
 
    switch (GST_EVENT_TYPE(event)) {
    case GST_EVENT_TAG:
        return gstEventHandleTag(event);
    case GST_EVENT_EOS:
        return gstEventHandleEOS(event);
    case GST_EVENT_FLUSH_START:
        return gstEventHandleFlushStart(event);
    case GST_EVENT_FLUSH_STOP:
        return gstEventHandleFlushStop(event);
 
    default:
        return;
    }
}
 
void QGstVideoRenderer::setActive(bool isActive)
{
    if (isActive == m_isActive)
        return;
 
    m_isActive = isActive;
    if (isActive)
        updateCurrentVideoFrame(m_currentPipelineFrame);
    else
        updateCurrentVideoFrame({});
}
 
void QGstVideoRenderer::updateCurrentVideoFrame(QVideoFrame frame)
{
    m_currentVideoFrame = std::move(frame);
    if (m_sink)
        m_sink->setVideoFrame(m_currentVideoFrame);
}
 
void QGstVideoRenderer::gstEventHandleTag(GstEvent *event)
{
    GstTagList *taglist = nullptr;
    gst_event_parse_tag(event, &taglist);
    if (!taglist)
        return;
 
    qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEventHandleTag:" << taglist;
 
    QGString value;
    if (!gst_tag_list_get_string(taglist, GST_TAG_IMAGE_ORIENTATION, &value))
        return;
 
    RotationResult parsed = parseRotationTag(value.get());
 
    m_format.setMirrored(parsed.flip);
    m_format.setRotation(parsed.rotation);
}
 
void QGstVideoRenderer::gstEventHandleEOS(GstEvent *)
{
    stop();
}
 
void QGstVideoRenderer::gstEventHandleFlushStart(GstEvent *)
{
    // "data is to be discarded"
    m_flushing = true;
    m_bufferQueue.clear();
}
 
void QGstVideoRenderer::gstEventHandleFlushStop(GstEvent *)
{
    // "data is allowed again"
    m_flushing = false;
}
 
static GstVideoSinkClass *gvrs_sink_parent_class;
static thread_local QGstreamerRelayVideoSink *gvrs_current_sink;
 
#define VO_SINK(s) QGstVideoRendererSink *sink(reinterpret_cast<QGstVideoRendererSink *>(s))
 
QGstVideoRendererSinkElement QGstVideoRendererSink::createSink(QGstreamerRelayVideoSink *sink)
{
    setSink(sink);
    QGstVideoRendererSink *gstSink = reinterpret_cast<QGstVideoRendererSink *>(
            g_object_new(QGstVideoRendererSink::get_type(), nullptr));
 
    return QGstVideoRendererSinkElement{
        gstSink,
        QGstElement::NeedsRef,
    };
}
 
void QGstVideoRendererSink::setSink(QGstreamerRelayVideoSink *sink)
{
    gvrs_current_sink = sink;
}
 
GType QGstVideoRendererSink::get_type()
{
    static const GTypeInfo info =
    {
        sizeof(QGstVideoRendererSinkClass),                // class_size
        base_init,                                         // base_init
        nullptr,                                           // base_finalize
        class_init,                                        // class_init
        nullptr,                                           // class_finalize
        nullptr,                                           // class_data
        sizeof(QGstVideoRendererSink),                     // instance_size
        0,                                                 // n_preallocs
        instance_init,                                     // instance_init
        nullptr                                            // value_table
    };
 
    static const GType type = g_type_register_static(GST_TYPE_VIDEO_SINK, "QGstVideoRendererSink",
                                                     &info, GTypeFlags(0));
 
    return type;
}
 
void QGstVideoRendererSink::class_init(gpointer g_class, gpointer class_data)
{
    Q_UNUSED(class_data);
 
    gvrs_sink_parent_class = reinterpret_cast<GstVideoSinkClass *>(g_type_class_peek_parent(g_class));
 
    GstVideoSinkClass *video_sink_class = reinterpret_cast<GstVideoSinkClass *>(g_class);
    video_sink_class->show_frame = QGstVideoRendererSink::show_frame;
 
    GstBaseSinkClass *base_sink_class = reinterpret_cast<GstBaseSinkClass *>(g_class);
    base_sink_class->get_caps = QGstVideoRendererSink::get_caps;
    base_sink_class->set_caps = QGstVideoRendererSink::set_caps;
    base_sink_class->propose_allocation = QGstVideoRendererSink::propose_allocation;
    base_sink_class->stop = QGstVideoRendererSink::stop;
    base_sink_class->unlock = QGstVideoRendererSink::unlock;
    base_sink_class->query = QGstVideoRendererSink::query;
    base_sink_class->event = QGstVideoRendererSink::event;
 
    GstElementClass *element_class = reinterpret_cast<GstElementClass *>(g_class);
    element_class->change_state = QGstVideoRendererSink::change_state;
    gst_element_class_set_metadata(element_class,
        "Qt built-in video renderer sink",
        "Sink/Video",
        "Qt default built-in video renderer sink",
        "The Qt Company");
 
    GObjectClass *object_class = reinterpret_cast<GObjectClass *>(g_class);
    object_class->finalize = QGstVideoRendererSink::finalize;
}
 
void QGstVideoRendererSink::base_init(gpointer g_class)
{
    static GstStaticPadTemplate sink_pad_template = GST_STATIC_PAD_TEMPLATE(
            "sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS(
                    "video/x-raw, "
                    "framerate = (fraction) [ 0, MAX ], "
                    "width = (int) [ 1, MAX ], "
                    "height = (int) [ 1, MAX ]"));
 
    gst_element_class_add_pad_template(
            GST_ELEMENT_CLASS(g_class), gst_static_pad_template_get(&sink_pad_template));
}
 
void QGstVideoRendererSink::instance_init(GTypeInstance *instance, gpointer g_class)
{
    Q_UNUSED(g_class);
    VO_SINK(instance);
 
    Q_ASSERT(gvrs_current_sink);
 
    sink->renderer = new QGstVideoRenderer(gvrs_current_sink);
    sink->renderer->moveToThread(gvrs_current_sink->thread());
    gvrs_current_sink = nullptr;
}
 
void QGstVideoRendererSink::finalize(GObject *object)
{
    VO_SINK(object);
 
    delete sink->renderer;
 
    // Chain up
    G_OBJECT_CLASS(gvrs_sink_parent_class)->finalize(object);
}
 
GstStateChangeReturn QGstVideoRendererSink::change_state(
        GstElement *element, GstStateChange transition)
{
    GstStateChangeReturn ret =
            GST_ELEMENT_CLASS(gvrs_sink_parent_class)->change_state(element, transition);
    qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::change_state:" << transition << ret;
    return ret;
}
 
GstCaps *QGstVideoRendererSink::get_caps(GstBaseSink *base, GstCaps *filter)
{
    VO_SINK(base);
 
    QGstCaps caps = sink->renderer->caps();
    if (filter)
        caps = QGstCaps(gst_caps_intersect(caps.caps(), filter), QGstCaps::HasRef);
 
    return caps.release();
}
 
gboolean QGstVideoRendererSink::set_caps(GstBaseSink *base, GstCaps *gcaps)
{
    VO_SINK(base);
    auto caps = QGstCaps(gcaps, QGstCaps::NeedsRef);
 
    qCDebug(qLcGstVideoRenderer) << "set_caps:" << caps;
 
    if (!caps) {
        sink->renderer->stop();
        return TRUE;
    }
 
    return sink->renderer->start(caps);
}
 
gboolean QGstVideoRendererSink::propose_allocation(GstBaseSink *base, GstQuery *query)
{
    VO_SINK(base);
    return sink->renderer->proposeAllocation(query);
}
 
gboolean QGstVideoRendererSink::stop(GstBaseSink *base)
{
    VO_SINK(base);
    sink->renderer->stop();
    return TRUE;
}
 
gboolean QGstVideoRendererSink::unlock(GstBaseSink *base)
{
    VO_SINK(base);
    sink->renderer->unlock();
    return TRUE;
}
 
GstFlowReturn QGstVideoRendererSink::show_frame(GstVideoSink *base, GstBuffer *buffer)
{
    VO_SINK(base);
    return sink->renderer->render(buffer);
}
 
gboolean QGstVideoRendererSink::query(GstBaseSink *base, GstQuery *query)
{
    VO_SINK(base);
    if (sink->renderer->query(query))
        return TRUE;
 
    return GST_BASE_SINK_CLASS(gvrs_sink_parent_class)->query(base, query);
}
 
gboolean QGstVideoRendererSink::event(GstBaseSink *base, GstEvent * event)
{
    VO_SINK(base);
    sink->renderer->gstEvent(event);
    return GST_BASE_SINK_CLASS(gvrs_sink_parent_class)->event(base, event);
}
 
QGstVideoRendererSinkElement::QGstVideoRendererSinkElement(QGstVideoRendererSink *element,
                                                           RefMode mode)
    : QGstBaseSink{
          qGstCheckedCast<GstBaseSink>(element),
          mode,
      }
{
}
 
void QGstVideoRendererSinkElement::setActive(bool isActive)
{
    qGstVideoRendererSink()->renderer->setActive(isActive);
}
 
QGstVideoRendererSink *QGstVideoRendererSinkElement::qGstVideoRendererSink() const
{
    return reinterpret_cast<QGstVideoRendererSink *>(element());
}
 
QT_END_NAMESPACE