qgstreamerintegration.cpp

Go to the documentation of this file.

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include <qgstreamerintegration_p.h>
#include <qgstreamerformatinfo_p.h>
#include <qgstreamervideodevices_p.h>
#include <audio/qgstreameraudiodevice_p.h>
#include <audio/qgstreameraudiodecoder_p.h>
#include <common/qgstreameraudioinput_p.h>
#include <common/qgstreameraudiooutput_p.h>
#include <common/qgstreamermediaplayer_p.h>
#include <common/qgstreamervideosink_p.h>
#include <mediacapture/qgstreamercamera_p.h>
#include <mediacapture/qgstreamerimagecapture_p.h>
#include <mediacapture/qgstreamermediacapturesession_p.h>
#include <mediacapture/qgstreamermediarecorder_p.h>
#include <uri_handler/qgstreamer_qiodevice_handler_p.h>
#include <uri_handler/qgstreamer_qrc_handler_p.h>
 
#if QT_CONFIG(gstreamer_qt_api)
#  include <QtMultimedia/qgstreamervideosource.h>
#endif
#include <QtCore/qloggingcategory.h>
#include <QtMultimedia/private/qmediaplayer_p.h>
#include <QtMultimedia/private/qmediacapturesession_p.h>
#include <QtMultimedia/private/qcameradevice_p.h>
#include <QtMultimedia/private/qvideoframe_p.h>
 
QT_BEGIN_NAMESPACE
 
static_assert(GST_CHECK_VERSION(1, 20, 0), "Minimum required GStreamer version is 1.20");
 
static thread_local bool inCustomCameraConstruction = false;
static thread_local QGstElement pendingCameraElement{};
 
QGStreamerPlatformSpecificInterfaceImplementation::
        ~QGStreamerPlatformSpecificInterfaceImplementation() = default;
 
QAudioDevice QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerAudioInput(
        const QByteArray &gstreamerPipeline)
{
    return qMakeCustomGStreamerAudioInput(gstreamerPipeline);
}
 
QAudioDevice QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerAudioOutput(
        const QByteArray &gstreamerPipeline)
{
    return qMakeCustomGStreamerAudioOutput(gstreamerPipeline);
}
 
QCamera *QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerCamera(
        const QByteArray &gstreamerPipeline, QObject *parent)
{
    QCameraDevicePrivate *info = new QCameraDevicePrivate;
    info->id = gstreamerPipeline;
    QCameraDevice device = info->create();
 
    inCustomCameraConstruction = true;
    auto guard = qScopeGuard([] {
        inCustomCameraConstruction = false;
    });
 
    return new QCamera(device, parent);
}
 
QCamera *
QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerCamera(GstElement *element,
                                                                             QObject *parent)
{
    QCameraDevicePrivate *info = new QCameraDevicePrivate;
    info->id = "Custom Camera from GstElement";
    QCameraDevice device = info->create();
 
    pendingCameraElement = QGstElement{
        element,
        QGstElement::NeedsRef,
    };
 
    inCustomCameraConstruction = true;
    auto guard = qScopeGuard([] {
        inCustomCameraConstruction = false;
        Q_ASSERT(!pendingCameraElement);
    });
 
    return new QCamera(device, parent);
}
 
GstPipeline *QGStreamerPlatformSpecificInterfaceImplementation::gstPipeline(QMediaPlayer *player)
{
    auto *priv = reinterpret_cast<QMediaPlayerPrivate *>(QMediaPlayerPrivate::get(player));
    if (!priv)
        return nullptr;
 
    QGstreamerMediaPlayer *gstreamerPlayer = dynamic_cast<QGstreamerMediaPlayer *>(priv->control);
    return gstreamerPlayer ? gstreamerPlayer->pipeline().pipeline() : nullptr;
}
 
GstPipeline *
QGStreamerPlatformSpecificInterfaceImplementation::gstPipeline(QMediaCaptureSession *session)
{
    auto *priv = QMediaCaptureSessionPrivate::get(session);
    if (!priv)
        return nullptr;
 
    QGstreamerMediaCaptureSession *gstreamerCapture =
            dynamic_cast<QGstreamerMediaCaptureSession *>(priv->captureSession.get());
    return gstreamerCapture ? gstreamerCapture->pipeline().pipeline() : nullptr;
}
 
GstBuffer *QGStreamerPlatformSpecificInterfaceImplementation::getRawGstBuffer(QVideoFrame &frame)
{
    QHwVideoBuffer *hwBuffer = QVideoFramePrivate::hwBuffer(frame);
    if (!hwBuffer)
        return nullptr;
    QGstVideoBuffer *gstBuffer = dynamic_cast<QGstVideoBuffer *>(hwBuffer);
    return gstBuffer ? gstBuffer->gstBuffer() : nullptr;
}
 
QVideoFrame QGStreamerPlatformSpecificInterfaceImplementation::createFrameFromGstBuffer(
        GstBuffer *buffer, const GstVideoInfo &videoInfo)
{
    if (!buffer)
        return QVideoFrame();
 
    QGstVideoInfo qtVideoInfo{ videoInfo, std::nullopt };
 
    return qCreateFrameFromGstBuffer(QGstBufferHandle{ buffer, QGstBufferHandle::NeedsRef },
                                     qtVideoInfo);
}
 
QVideoFrame QGStreamerPlatformSpecificInterfaceImplementation::createFrameFromGstBuffer(
        GstBuffer *buffer, const GstVideoInfoDmaDrm &videoInfoDmaDrm)
{
#if !QT_GSTREAMER_SUPPORTS_GST_VIDEO_FORMAT_DMA_DRM
    Q_UNUSED(buffer);
    Q_UNUSED(videoInfoDmaDrm);
    qWarning() << QStringLiteral("GstVideoInfoDmaDrm unsupported, minimum GStreamer version required: 1.24");
    return QVideoFrame();
#else
    if (!buffer)
        return QVideoFrame();
 
    QGstVideoInfo qtVideoInfo{ { }, videoInfoDmaDrm.drm_modifier };
    if (!gst_video_info_dma_drm_to_video_info(&videoInfoDmaDrm, &qtVideoInfo.gstVideoInfo))
        qWarning() << "Failed to create QGstVideoInfo from GstVideoInfoDmaDrm";
 
    return qCreateFrameFromGstBuffer(QGstBufferHandle{ buffer, QGstBufferHandle::NeedsRef },
                                     qtVideoInfo);
#endif
}
 
Q_STATIC_LOGGING_CATEGORY(lcGstreamer, "qt.multimedia.gstreamer")
 
namespace {
 
void rankDownPlugin(GstRegistry *reg, const char *name)
{
    QGstPluginFeatureHandle pluginFeature{
        gst_registry_lookup_feature(reg, name),
        QGstPluginFeatureHandle::HasRef,
    };
    if (pluginFeature)
        gst_plugin_feature_set_rank(pluginFeature.get(), GST_RANK_PRIMARY - 1);
}
 
// https://gstreamer.freedesktop.org/documentation/vaapi/index.html
constexpr auto vaapiPluginNames = {
    "vaapidecodebin", "vaapih264dec", "vaapih264enc",  "vaapih265dec",
    "vaapijpegdec",   "vaapijpegenc", "vaapimpeg2dec", "vaapipostproc",
    "vaapisink",      "vaapivp8dec",  "vaapivp9dec",
};
 
// https://gstreamer.freedesktop.org/documentation/va/index.html
constexpr auto vaPluginNames = {
    "vaav1dec",  "vacompositor", "vadeinterlace", "vah264dec", "vah264enc", "vah265dec",
    "vajpegdec", "vampeg2dec",   "vapostproc",    "vavp8dec",  "vavp9dec",
};
 
// https://gstreamer.freedesktop.org/documentation/nvcodec/index.html
constexpr auto nvcodecPluginNames = {
    "cudaconvert",     "cudaconvertscale", "cudadownload",     "cudaipcsink",      "cudaipcsrc",
    "cudascale",       "cudaupload",       "nvautogpuh264enc", "nvautogpuh265enc", "nvav1dec",
    "nvcudah264enc",   "nvcudah265enc",    "nvd3d11h264enc",   "nvd3d11h265enc",   "nvh264dec",
    "nvh264enc",       "nvh265dec",        "nvh265enc",        "nvjpegdec",        "nvjpegenc",
    "nvmpeg2videodec", "nvmpeg4videodec",  "nvmpegvideodec",   "nvvp8dec",         "nvvp9dec",
};
 
} // namespace
 
QGstreamerIntegration::QGstreamerIntegration()
    : QPlatformMediaIntegration(QLatin1String("gstreamer"))
{
    gst_init(nullptr, nullptr);
 
    const QGString version{ gst_version_string() };
    qCInfo(lcGstreamer) << "Using Qt multimedia with GStreamer version:" << version.asStringView();
 
    GstRegistry *reg = gst_registry_get();
 
    if constexpr (!GST_CHECK_VERSION(1, 22, 0)) {
        for (const char *name : vaapiPluginNames)
            rankDownPlugin(reg, name);
    }
 
    if (qEnvironmentVariableIsSet("QT_GSTREAMER_DISABLE_VA")) {
        for (const char *name : vaPluginNames)
            rankDownPlugin(reg, name);
    }
 
    if (qEnvironmentVariableIsSet("QT_GSTREAMER_DISABLE_NVCODEC")) {
        for (const char *name : nvcodecPluginNames)
            rankDownPlugin(reg, name);
    }
 
    qGstRegisterQRCHandler(nullptr);
    qGstRegisterQIODeviceHandler(nullptr);
}
 
QGstreamerIntegration::~QGstreamerIntegration()
{
    // by default we don't deinit, as the application may have initialized gstreamer
    // (gst_init/deinit is not refcounted).
    // however it's useful to force deinitialization for leak detection in qt's unit tests.
    if (qEnvironmentVariableIsSet("QT_GSTREAMER_DEINIT"))
        gst_deinit();
}
 
QPlatformMediaFormatInfo *QGstreamerIntegration::createFormatInfo()
{
    return new QGstreamerFormatInfo();
}
 
QPlatformVideoDevices *QGstreamerIntegration::createVideoDevices()
{
    return new QGstreamerVideoDevices(this);
}
 
const QGstreamerFormatInfo *QGstreamerIntegration::gstFormatsInfo()
{
    return static_cast<const QGstreamerFormatInfo *>(formatInfo());
}
 
q23::expected<QPlatformAudioDecoder *, QString> QGstreamerIntegration::createAudioDecoder(QAudioDecoder *decoder)
{
    return QGstreamerAudioDecoder::create(decoder);
}
 
q23::expected<QPlatformMediaCaptureSession *, QString> QGstreamerIntegration::createCaptureSession()
{
    return QGstreamerMediaCaptureSession::create();
}
 
q23::expected<QPlatformMediaPlayer *, QString> QGstreamerIntegration::createPlayer(QMediaPlayer *player)
{
    return QGstreamerMediaPlayer::create(player);
}
 
q23::expected<QPlatformCamera *, QString> QGstreamerIntegration::createCamera(QCamera *camera)
{
    if (inCustomCameraConstruction) {
        QGstElement element = std::exchange(pendingCameraElement, {});
        return element ? new QGstreamerCustomCamera{ camera, std::move(element) }
                       : new QGstreamerCustomCamera{ camera };
    }
 
    return QGstreamerCamera::create(camera);
}
 
q23::expected<QPlatformMediaRecorder *, QString> QGstreamerIntegration::createRecorder(QMediaRecorder *recorder)
{
    return new QGstreamerMediaRecorder(recorder);
}
 
q23::expected<QPlatformImageCapture *, QString> QGstreamerIntegration::createImageCapture(QImageCapture *imageCapture)
{
    return QGstreamerImageCapture::create(imageCapture);
}
 
q23::expected<QPlatformVideoSink *, QString> QGstreamerIntegration::createVideoSink(QVideoSink *sink)
{
    return new QGstreamerPluggableVideoSink(sink);
}
 
q23::expected<QPlatformAudioInput *, QString> QGstreamerIntegration::createAudioInput(QAudioInput *q)
{
    return QGstreamerAudioInput::create(q);
}
 
q23::expected<QPlatformAudioOutput *, QString> QGstreamerIntegration::createAudioOutput(QAudioOutput *q)
{
    return QGstreamerAudioOutput::create(q);
}
 
q23::expected<QPlatformCamera *, QString>
QGstreamerIntegration::createGStreamerVideoSource(QGStreamerVideoSource *videoSource,
                                                  const GstElementOrDescription &elementOrDesc)
{
#if QT_CONFIG(gstreamer_qt_api)
    using namespace Qt::Literals;
    auto createImpl = [videoSource](const auto &arg) -> q23::expected<QPlatformCamera *, QString> {
        QGstElement element;
        if constexpr (std::is_same_v<decltype(arg), const QString &>) {
            if (arg.isEmpty())
                return q23::unexpected{ u"GstBin description is empty"_s };
            element = QGstBin::createFromPipelineDescription(arg.toUtf8(), /*name=*/nullptr,
                                                             /* ghostUnlinkedPads=*/true);
            if (!element)
                return q23::unexpected{ u"Failed to create GstBin from description"_s };
        } else {
            if (!arg)
                return q23::unexpected{ u"GstElement is null"_s };
 
            element = QGstElement{ arg, QGstElement::NeedsRef };
        }
 
        return new QGstreamerCustomCamera(videoSource, std::move(element));
    };
    return std::visit(createImpl, elementOrDesc);
#else
    return QPlatformMediaIntegration::createGStreamerVideoSource(videoSource, elementOrDesc);
#endif
}
 
GstDevice *QGstreamerIntegration::videoDevice(const QByteArray &id)
{
    const auto devices = videoDevices();
    return devices ? static_cast<QGstreamerVideoDevices *>(devices)->videoDevice(id) : nullptr;
}
 
QAbstractPlatformSpecificInterface *QGstreamerIntegration::platformSpecificInterface()
{
    return &m_platformSpecificImplementation;
}
 
QT_END_NAMESPACE