qgstreamervideosink.cpp

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include <common/qgstreamervideosink_p.h>
#include <common/qgstvideorenderersink_p.h>
#include <common/qgst_debug_p.h>
#include <common/qgstutils_p.h>
#include <rhi/qrhi.h>
 
#include <QtCore/qdebug.h>
#include <QtCore/qloggingcategory.h>
 
#include <optional>
#include <variant>
 
#if QT_CONFIG(gstreamer_gl)
#  include <QtGui/qguiapplication.h>
#  include <QtGui/qguiapplication_platform.h>
#  include <QtGui/qopenglcontext.h>
#  include <QtGui/qwindow.h>
#  include <gst/gl/gstglconfig.h>
#  include <gst/gl/gstgldisplay.h>
 
#  if QT_CONFIG(gstreamer_gl_x11)
#    include <gst/gl/x11/gstgldisplay_x11.h>
#  endif
#  if QT_CONFIG(gstreamer_gl_egl)
#    include <common/qgstreameregldisplay_p.h>
#    include <QtMultimedia/private/qeglimagefunctions_p.h>
#    include <QtMultimedia/private/qmultimedia_gl_support_p.h>
#    include <gst/gl/egl/gstgldisplay_egl.h>
#    include <EGL/egl.h>
#    include <EGL/eglext.h>
#  endif
#  if QT_CONFIG(gstreamer_gl_wayland)
#    include <gst/gl/wayland/gstgldisplay_wayland.h>
#  endif
#endif // #if QT_CONFIG(gstreamer_gl)
 
QT_BEGIN_NAMESPACE
 
Q_STATIC_LOGGING_CATEGORY(qLcGstVideoSink, "qt.multimedia.gstvideosink");
 
QGstreamerRelayVideoSink::QGstreamerRelayVideoSink(QObject *parent)
    : QObject{
          parent,
      },
      m_sinkBin{
          QGstBin::create("videoSinkBin"),
      }
{
    // This is a hack for some iMX and NVidia platforms. These require the use of a special video
    // conversion element in the pipeline before the video sink, as they unfortunately
    // output some proprietary format from the decoder even though it's sometimes marked as
    // a regular supported video/x-raw format.
    //
    // To fix this, simply insert the element into the pipeline if it's available. Otherwise
    // we simply use an identity element.
    QGstElementFactoryHandle factory;
 
    // QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT allows users to override the
    // conversion element. Ideally we construct the element programatically, though.
    QByteArray preprocessOverride = qgetenv("QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT");
    if (!preprocessOverride.isEmpty()) {
        qCDebug(qLcGstVideoSink) << "requesting conversion element from environment:"
                                 << preprocessOverride;
 
        m_gstPreprocess = QGstBin::createFromPipelineDescription(preprocessOverride, nullptr,
                                                                 /*ghostUnlinkedPads=*/true);
        if (!m_gstPreprocess)
            qCWarning(qLcGstVideoSink) << "Cannot create conversion element:" << preprocessOverride;
    }
 
    if (!m_gstPreprocess) {
        // This is a hack for some iMX and NVidia platforms. These require the use of a special
        // video conversion element in the pipeline before the video sink, as they unfortunately
        // output some proprietary format from the decoder even though it's sometimes marked as
        // a regular supported video/x-raw format.
        static constexpr auto decodersToTest = {
            "imxvideoconvert_g2d",
            "nvvidconv",
        };
 
        for (const char *decoder : decodersToTest) {
            factory = QGstElement::findFactory(decoder);
            if (factory)
                break;
        }
 
        if (factory) {
            qCDebug(qLcGstVideoSink)
                    << "instantiating conversion element:"
                    << g_type_name(gst_element_factory_get_element_type(factory.get()));
 
            m_gstPreprocess = QGstElement::createFromFactory(factory, "preprocess");
        }
    }
 
    bool disablePixelAspectRatio =
            qEnvironmentVariableIntValue("QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO") != 0;
    if (disablePixelAspectRatio) {
        // Enabling the pixel aspect ratio may expose a gstreamer bug on cameras that don't expose a
        // pixel-aspect-ratio via `VIDIOC_CROPCAP`. This can cause the caps negotiation to fail.
        // Using the QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO environment variable, one can disable
        // pixel-aspect-ratio handling
        //
        // compare: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6242
        m_gstCapsFilter =
                QGstElement::createFromFactory("identity", "nullPixelAspectRatioCapsFilter");
    } else {
        m_gstCapsFilter =
                QGstElement::createFromFactory("capsfilter", "pixelAspectRatioCapsFilter");
        QGstCaps capsFilterCaps{
            gst_caps_new_simple("video/x-raw", "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL),
            QGstCaps::HasRef,
        };
        g_object_set(m_gstCapsFilter.element(), "caps", capsFilterCaps.caps(), NULL);
    }
 
    if (m_gstPreprocess) {
        m_sinkBin.add(m_gstPreprocess, m_gstCapsFilter);
        qLinkGstElements(m_gstPreprocess, m_gstCapsFilter);
        m_sinkBin.addGhostPad(m_gstPreprocess, "sink");
    } else {
        m_sinkBin.add(m_gstCapsFilter);
        m_sinkBin.addGhostPad(m_gstCapsFilter, "sink");
    }
 
    updateGstContexts();
}
 
QGstreamerRelayVideoSink::~QGstreamerRelayVideoSink()
{
    Q_EMIT aboutToBeDestroyed();
 
    unrefGstContexts();
}
 
QGstElement QGstreamerRelayVideoSink::gstSink()
{
    if (!m_gstVideoSink) {
        if (!m_gstQtSink)
            createQtSink();
 
        updateSinkElement(m_gstQtSink);
    }
 
    return m_sinkBin;
}
 
void QGstreamerRelayVideoSink::setActive(bool isActive)
{
    if (m_isActive == isActive)
        return;
    m_isActive = isActive;
 
    if (m_gstQtSink)
        m_gstQtSink.setActive(isActive);
}
 
void QGstreamerRelayVideoSink::setAsync(bool isAsync)
{
    m_sinkIsAsync = isAsync;
}
 
void QGstreamerRelayVideoSink::connectPluggableVideoSink(QGstreamerPluggableVideoSink *pluggableSink)
{
    Q_ASSERT(pluggableSink);
    m_pluggableVideoSink = pluggableSink;
    m_pluggableVideoSink->setVideoFrame(m_currentVideoFrame);
    m_pluggableVideoSink->setSubtitleText(m_currentSubtitleText);
    m_pluggableVideoSink->setNativeSize(m_currentNativeSize);
    connect(this, &QGstreamerRelayVideoSink::videoFrameChanged,
            m_pluggableVideoSink, &QPlatformVideoSink::setVideoFrame);
    connect(this, &QGstreamerRelayVideoSink::subtitleTextChanged,
            m_pluggableVideoSink, &QPlatformVideoSink::setSubtitleText);
    connect(this, &QGstreamerRelayVideoSink::nativeSizeChanged,
            m_pluggableVideoSink, &QPlatformVideoSink::setNativeSize);
 
    // Update pipeline contexts using the rendering rhi
 
    m_pluggableVideoSink->withRhi([&](QRhi *rhi) {
        renderingRhiChanged(rhi);
    });
 
    m_rhiConnection =
            connect(m_pluggableVideoSink, &QRhiVideoSink::rhiChanged, this, [this](QRhi *rhi) {
        renderingRhiChanged(rhi);
    });
}
 
void QGstreamerRelayVideoSink::disconnectPluggableVideoSink()
{
    if (m_pluggableVideoSink) {
        disconnect(this, &QGstreamerRelayVideoSink::videoFrameChanged,
                   m_pluggableVideoSink, &QPlatformVideoSink::setVideoFrame);
        disconnect(this, &QGstreamerRelayVideoSink::subtitleTextChanged,
                   m_pluggableVideoSink, &QPlatformVideoSink::setSubtitleText);
        disconnect(this, &QGstreamerRelayVideoSink::nativeSizeChanged,
                   m_pluggableVideoSink, &QPlatformVideoSink::setNativeSize);
        disconnect(m_rhiConnection);
        m_pluggableVideoSink = nullptr;
    }
}
 
void QGstreamerRelayVideoSink::setVideoFrame(const QVideoFrame &frame)
{
    Q_EMIT videoFrameChanged(frame);
    m_currentVideoFrame = frame;
}
 
void QGstreamerRelayVideoSink::setSubtitleText(const QString &subtitleText)
{
    Q_EMIT subtitleTextChanged(subtitleText);
    m_currentSubtitleText = subtitleText;
}
 
void QGstreamerRelayVideoSink::setNativeSize(QSize size)
{
    Q_EMIT nativeSizeChanged(size);
    m_currentNativeSize = size;
}
 
void QGstreamerRelayVideoSink::createQtSink()
{
    Q_ASSERT(!m_gstQtSink);
 
    m_gstQtSink = QGstVideoRendererSink::createSink(this);
    if (!m_sinkIsAsync)
        m_gstQtSink.set("async", false);
    m_gstQtSink.setActive(m_isActive);
}
 
void QGstreamerRelayVideoSink::updateSinkElement(QGstVideoRendererSinkElement newSink)
{
    if (newSink == m_gstVideoSink)
        return;
 
    m_gstCapsFilter.src().modifyPipelineInIdleProbe([&] {
        if (m_gstVideoSink)
            m_sinkBin.stopAndRemoveElements(m_gstVideoSink);
 
        m_gstVideoSink = std::move(newSink);
        m_sinkBin.add(m_gstVideoSink);
        qLinkGstElements(m_gstCapsFilter, m_gstVideoSink);
        GstEvent *event = gst_event_new_reconfigure();
        gst_element_send_event(m_gstVideoSink.element(), event);
        m_gstVideoSink.syncStateWithParent();
    });
 
    m_sinkBin.dumpPipelineGraph("updateSinkElement");
}
 
void QGstreamerRelayVideoSink::unrefGstContexts()
{
    m_gstGlDisplayContext.reset();
    m_gstGlLocalContext.reset();
}
 
#if QT_CONFIG(gstreamer_gl)
 
namespace {
 
using AnyNativeGLContextType = std::variant<
#  if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(gstreamer_gl_x11) && QT_CONFIG(xcb_glx_plugin)
        EGLContext, GLXContext
#  elif QT_CONFIG(gstreamer_gl_egl)
        EGLContext
#  elif QT_CONFIG(gstreamer_gl_x11) && QT_CONFIG(xcb_glx_plugin)
        GLXContext
#  else
        void *
#  endif
        >;
 
std::optional<AnyNativeGLContextType>
resolveNativeGlContext([[maybe_unused]] QOpenGLContext *glContext, GstGLPlatform glPlatform)
{
    switch (glPlatform) {
#  if QT_CONFIG(gstreamer_gl_egl)
    case GST_GL_PLATFORM_EGL:
        if (auto *eglContext = glContext->nativeInterface<QNativeInterface::QEGLContext>())
            return eglContext->nativeContext();
        return std::nullopt;
#    endif
#  if QT_CONFIG(gstreamer_gl_x11) && QT_CONFIG(xcb_glx_plugin)
    case GST_GL_PLATFORM_GLX:
        if (auto *glxContext = glContext->nativeInterface<QNativeInterface::QGLXContext>())
            return glxContext->nativeContext();
        return std::nullopt;
#  endif
    default:
        return std::nullopt;
    }
}
 
struct ResolveGstGlDisplayResult
{
    QGstGLDisplayHandle display;
    GstGLPlatform platform;
};
 
std::optional<ResolveGstGlDisplayResult> resolveGstGlDisplay([[maybe_unused]] QRhi &rhi)
{
    // use the egl display if we have one
#  if QT_CONFIG(gstreamer_gl_egl)
    if (EGLDisplay eglDisplay = QtMultimediaPrivate::resolveEglDisplay(rhi)) {
        QGstGLDisplayHandle gstGlDisplay{
            GST_GL_DISPLAY_CAST(gst_gl_display_egl_new_with_egl_display(eglDisplay)),
            QGstGLDisplayHandle::HasRef,
        };
        if (gstGlDisplay)
            return ResolveGstGlDisplayResult{
                std::move(gstGlDisplay),
                GST_GL_PLATFORM_EGL,
            };
    }
#    endif
 
#  if QT_CONFIG(gstreamer_gl_wayland)
    if (auto *waylandApp = qGuiApp->nativeInterface<QNativeInterface::QWaylandApplication>()) {
        QGstGLDisplayHandle gstGlDisplay{
            GST_GL_DISPLAY_CAST(gst_gl_display_wayland_new_with_display(waylandApp->display())),
            QGstGLDisplayHandle::HasRef,
        };
        if (gstGlDisplay)
            return ResolveGstGlDisplayResult{
                std::move(gstGlDisplay),
                GST_GL_PLATFORM_EGL,
            };
    }
#  endif
 
#  if QT_CONFIG(gstreamer_gl_x11) && QT_CONFIG(xcb_glx_plugin)
    if (auto *x11App = qGuiApp->nativeInterface<QNativeInterface::QX11Application>()) {
        QGstGLDisplayHandle gstGlDisplay{
            GST_GL_DISPLAY_CAST(gst_gl_display_x11_new_with_display(x11App->display())),
            QGstGLDisplayHandle::HasRef,
        };
        if (gstGlDisplay)
            return ResolveGstGlDisplayResult{
                std::move(gstGlDisplay),
                GST_GL_PLATFORM_GLX,
            };
    }
#  endif
 
    return std::nullopt;
}
 
QGstGLContextHandle resolveGstGLContextHandle(const QGstGLDisplayHandle &gstGlDisplay,
                                              AnyNativeGLContextType nativeContext,
                                              GstGLPlatform glPlatform)
{
    GstGLAPI glApi = QOpenGLContext::openGLModuleType() == QOpenGLContext::LibGL ? GST_GL_API_OPENGL
                                                                                 : GST_GL_API_GLES2;
 
    guintptr nativeContextArgument = std::visit([](auto nativeContext) {
        return guintptr(nativeContext);
    }, nativeContext);
 
    QGstGLContextHandle appContext{
        gst_gl_context_new_wrapped(gstGlDisplay.get(), nativeContextArgument, glPlatform, glApi),
        QGstGLContextHandle::HasRef,
    };
 
    return appContext;
}
 
} // namespace
 
#  endif // #if QT_CONFIG(gstreamer_gl)
 
void QGstreamerRelayVideoSink::updateGstContexts()
{
    QRhi *currentRhi = rhi();
 
    unrefGstContexts();
 
#if QT_CONFIG(gstreamer_gl)
    if (!currentRhi || currentRhi->backend() != QRhi::OpenGLES2)
        return;
    QOpenGLContext *glContext =
            static_cast<const QRhiGles2NativeHandles *>(currentRhi->nativeHandles())->context;
    Q_ASSERT(glContext);
 
    std::optional resolvedDisplay = resolveGstGlDisplay(*currentRhi);
    if (!resolvedDisplay) {
        qWarning() << "Could not resolve GstGLDisplay";
        return;
    }
 
    std::optional nativeContext = resolveNativeGlContext(glContext, resolvedDisplay->platform);
    if (!nativeContext) {
        qWarning() << "Could not find native gl context";
        return;
    }
 
    QGstGLContextHandle appContext = resolveGstGLContextHandle(
            resolvedDisplay->display, *nativeContext, resolvedDisplay->platform);
    if (!appContext) {
        qWarning() << "Could not create wrappped context for platform:"
                   << resolvedDisplay->platform;
        return;
    }
 
    gst_gl_context_activate(appContext.get(), true);
 
    QUniqueGErrorHandle error;
    gst_gl_context_fill_info(appContext.get(), &error);
    if (error) {
        qWarning() << "Could not fill context info:" << error;
        error = {};
    }
 
    QGstGLContextHandle displayContext;
    gst_gl_display_create_context(resolvedDisplay->display.get(), appContext.get(), &displayContext,
                                  &error);
    if (error)
        qWarning() << "Could not create display context:" << error;
 
    appContext.reset();
 
    m_gstGlDisplayContext = QGstContextHandle{
        gst_context_new(GST_GL_DISPLAY_CONTEXT_TYPE, false),
        QGstContextHandle::HasRef,
    };
    gst_context_set_gl_display(m_gstGlDisplayContext.get(), resolvedDisplay->display.get());
 
    m_gstGlLocalContext = QGstContextHandle{
        gst_context_new("gst.gl.local_context", false),
        QGstContextHandle::HasRef,
    };
 
    GstStructure *structure = gst_context_writable_structure(m_gstGlLocalContext.get());
    gst_structure_set(structure, "context", GST_TYPE_GL_CONTEXT, displayContext.get(), nullptr);
    displayContext.reset();
 
    // Note: after updating the context, we switch the sink and send gst_event_new_reconfigure()
    // upstream. this will cause the context to be queried again.
#endif // #if QT_CONFIG(gstreamer_gl)
}
 
void QGstreamerRelayVideoSink::renderingRhiChanged(QRhi *rhi)
{
    m_rhi = rhi;
    updateGstContexts();
}
 
QT_END_NAMESPACE
 
#include "moc_qgstreamervideosink_p.cpp"