qgstreameraudiodecoder.cpp

Go to the documentation of this file.

// Copyright (C) 2020 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
//#define DEBUG_DECODER
 
#include <audio/qgstreameraudiodecoder_p.h>
 
#include <common/qgst_debug_p.h>
#include <common/qgstreamermessage_p.h>
#include <common/qgstutils_p.h>
#include <uri_handler/qgstreamer_qiodevice_handler_p.h>
 
#include <gst/gstvalue.h>
#include <gst/base/gstbasesrc.h>
 
#include <QtCore/qdatetime.h>
#include <QtCore/qdebug.h>
#include <QtCore/qsize.h>
#include <QtCore/qtimer.h>
#include <QtCore/qdebug.h>
#include <QtCore/qdir.h>
#include <QtCore/qstandardpaths.h>
#include <QtCore/qurl.h>
#include <QtCore/qloggingcategory.h>
 
QT_BEGIN_NAMESPACE
 
Q_STATIC_LOGGING_CATEGORY(qLcGstreamerAudioDecoder, "qt.multimedia.gstreameraudiodecoder");
 
typedef enum {
    GST_PLAY_FLAG_VIDEO         = 0x00000001,
    GST_PLAY_FLAG_AUDIO         = 0x00000002,
    GST_PLAY_FLAG_TEXT          = 0x00000004,
    GST_PLAY_FLAG_VIS           = 0x00000008,
    GST_PLAY_FLAG_SOFT_VOLUME   = 0x00000010,
    GST_PLAY_FLAG_NATIVE_AUDIO  = 0x00000020,
    GST_PLAY_FLAG_NATIVE_VIDEO  = 0x00000040,
    GST_PLAY_FLAG_DOWNLOAD      = 0x00000080,
    GST_PLAY_FLAG_BUFFERING     = 0x000000100
} GstPlayFlags;
 
 
q23::expected<QPlatformAudioDecoder *, QString> QGstreamerAudioDecoder::create(QAudioDecoder *parent)
{
    static const auto error = qGstErrorMessageIfElementsNotAvailable("audioconvert", "playbin");
    if (error)
        return q23::unexpected{ *error };
 
    return new QGstreamerAudioDecoder(parent);
}
 
QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent)
    : QPlatformAudioDecoder(parent),
      m_playbin{
          QGstPipeline::createFromFactory("playbin3", "playbin"),
      },
      m_audioConvert{
          QGstElement::createFromFactory("audioconvert", "audioconvert"),
      }
{
    // Sort out messages
    m_playbin.installMessageFilter(this);
 
    // Set the rest of the pipeline up
    setAudioFlags(true);
 
    m_outputBin = QGstBin::create("audio-output-bin");
    m_outputBin.add(m_audioConvert);
 
    // add ghostpad
    m_outputBin.addGhostPad(m_audioConvert, "sink");
 
    g_object_set(m_playbin.object(), "audio-sink", m_outputBin.element(), NULL);
 
    // Set volume to 100%
    gdouble volume = 1.0;
    m_playbin.set("volume", volume);
}
 
QGstreamerAudioDecoder::~QGstreamerAudioDecoder()
{
    stop();
 
    m_playbin.removeMessageFilter(this);
}
 
bool QGstreamerAudioDecoder::processBusMessage(const QGstreamerMessage &message)
{
    qCDebug(qLcGstreamerAudioDecoder) << "received bus message:" << message;
 
    switch (message.type()) {
    case GST_MESSAGE_DURATION:
        return processBusMessageDuration(message);
 
    case GST_MESSAGE_ERROR:
        return processBusMessageError(message);
 
    case GST_MESSAGE_WARNING:
        return processBusMessageWarning(message);
 
    case GST_MESSAGE_INFO:
        return processBusMessageInfo(message);
 
    case GST_MESSAGE_EOS:
        return processBusMessageEOS(message);
 
    case GST_MESSAGE_STATE_CHANGED:
        return processBusMessageStateChanged(message);
 
    case GST_MESSAGE_STREAMS_SELECTED:
        return processBusMessageStreamsSelected(message);
 
    default:
        return false;
    }
}
 
bool QGstreamerAudioDecoder::canReadQrc() const
{
    return true;
}
 
bool QGstreamerAudioDecoder::processBusMessageError(const QGstreamerMessage &message)
{
    qCDebug(qLcGstreamerAudioDecoder) << "    error" << QCompactGstMessageAdaptor(message);
 
    QUniqueGErrorHandle err;
    QGString debug;
    gst_message_parse_error(message.message(), &err, &debug);
 
    if (message.source() == m_playbin) {
        if (err.get()->domain == GST_STREAM_ERROR
            && err.get()->code == GST_STREAM_ERROR_CODEC_NOT_FOUND)
            processInvalidMedia(QAudioDecoder::FormatError,
                                tr("Cannot play stream of type: <unknown>"));
        else
            processInvalidMedia(QAudioDecoder::ResourceError,
                                QString::fromUtf8(err.get()->message));
    } else {
        QAudioDecoder::Error qerror = QAudioDecoder::ResourceError;
        if (err.get()->domain == GST_STREAM_ERROR) {
            switch (err.get()->code) {
            case GST_STREAM_ERROR_DECRYPT:
            case GST_STREAM_ERROR_DECRYPT_NOKEY:
                qerror = QAudioDecoder::AccessDeniedError;
                break;
            case GST_STREAM_ERROR_FORMAT:
            case GST_STREAM_ERROR_DEMUX:
            case GST_STREAM_ERROR_DECODE:
            case GST_STREAM_ERROR_WRONG_TYPE:
            case GST_STREAM_ERROR_TYPE_NOT_FOUND:
            case GST_STREAM_ERROR_CODEC_NOT_FOUND:
                qerror = QAudioDecoder::FormatError;
                break;
            default:
                break;
            }
        } else if (err.get()->domain == GST_CORE_ERROR) {
            switch (err.get()->code) {
            case GST_CORE_ERROR_MISSING_PLUGIN:
                qerror = QAudioDecoder::FormatError;
                break;
            default:
                break;
            }
        }
 
        processInvalidMedia(qerror, QString::fromUtf8(err.get()->message));
    }
 
    return false;
}
 
bool QGstreamerAudioDecoder::processBusMessageDuration(const QGstreamerMessage &)
{
    updateDuration();
    return false;
}
 
bool QGstreamerAudioDecoder::processBusMessageWarning(const QGstreamerMessage &message)
{
    qCWarning(qLcGstreamerAudioDecoder) << "Warning:" << QCompactGstMessageAdaptor(message);
    return false;
}
 
bool QGstreamerAudioDecoder::processBusMessageInfo(const QGstreamerMessage &message)
{
    if (qLcGstreamerAudioDecoder().isDebugEnabled())
        qCWarning(qLcGstreamerAudioDecoder) << "Info:" << QCompactGstMessageAdaptor(message);
    return false;
}
 
bool QGstreamerAudioDecoder::processBusMessageEOS(const QGstreamerMessage &)
{
    m_playbin.setState(GST_STATE_NULL);
    finished();
    return false;
}
 
bool QGstreamerAudioDecoder::processBusMessageStateChanged(const QGstreamerMessage &message)
{
    if (message.source() != m_playbin)
        return false;
 
    GstState oldState;
    GstState newState;
    GstState pending;
 
    gst_message_parse_state_changed(message.message(), &oldState, &newState, &pending);
 
    bool isDecoding = false;
    switch (newState) {
    case GST_STATE_VOID_PENDING:
    case GST_STATE_NULL:
    case GST_STATE_READY:
        break;
    case GST_STATE_PLAYING:
        isDecoding = true;
        break;
    case GST_STATE_PAUSED:
        isDecoding = true;
 
        // gstreamer doesn't give a reliable indication the duration
        // information is ready, GST_MESSAGE_DURATION is not sent by most elements
        // the duration is queried up to 5 times with increasing delay
        m_durationQueries = 5;
        updateDuration();
        break;
    }
 
    setIsDecoding(isDecoding);
    return false;
}
 
bool QGstreamerAudioDecoder::processBusMessageStreamsSelected(const QGstreamerMessage &message)
{
    using namespace Qt::StringLiterals;
 
    QGstStreamCollectionHandle collection;
    gst_message_parse_streams_selected(const_cast<GstMessage *>(message.message()), &collection);
 
    bool hasAudio = false;
    qForeachStreamInCollection(collection, [&](GstStream *stream) {
        GstStreamType type = gst_stream_get_stream_type(stream);
        if (type == GstStreamType::GST_STREAM_TYPE_AUDIO)
            hasAudio = true;
    });
 
    if (!hasAudio)
        processInvalidMedia(QAudioDecoder::FormatError, u"No audio track in media"_s);
 
    return false;
}
 
QUrl QGstreamerAudioDecoder::source() const
{
    return mSource;
}
 
void QGstreamerAudioDecoder::setSource(const QUrl &fileName)
{
    stop();
    mDevice = nullptr;
 
    bool isSignalRequired = (mSource != fileName);
    mSource = fileName;
    if (isSignalRequired)
        sourceChanged();
}
 
QIODevice *QGstreamerAudioDecoder::sourceDevice() const
{
    return mDevice;
}
 
void QGstreamerAudioDecoder::setSourceDevice(QIODevice *device)
{
    stop();
    mSource.clear();
    bool isSignalRequired = (mDevice != device);
    mDevice = device;
    if (isSignalRequired)
        sourceChanged();
}
 
void QGstreamerAudioDecoder::start()
{
    addAppSink();
 
    if (!mSource.isEmpty()) {
        m_playbin.set("uri", mSource.toEncoded().constData());
    } else if (mDevice) {
        // make sure we can read from device
        if (!mDevice->isOpen() || !mDevice->isReadable()) {
            processInvalidMedia(QAudioDecoder::ResourceError, QLatin1String("Unable to read from specified device"));
            return;
        }
 
        QUrl streamURL = qGstRegisterQIODevice(mDevice);
        m_playbin.set("uri", streamURL.toEncoded().constData());
    } else {
        return;
    }
 
    // Set audio format
    if (m_appSink) {
        if (mFormat.isValid()) {
            setAudioFlags(false);
            auto caps = QGstUtils::capsForAudioFormat(mFormat);
            m_appSink.setCaps(caps);
        } else {
            // We want whatever the native audio format is
            setAudioFlags(true);
            m_appSink.setCaps({});
        }
    }
 
    if (m_playbin.setState(GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) {
        qWarning() << "GStreamer; Unable to start decoding process";
        m_playbin.dumpGraph("failed");
        return;
    }
}
 
void QGstreamerAudioDecoder::stop()
{
    m_playbin.setState(GST_STATE_NULL);
    m_currentSessionId += 1;
    removeAppSink();
 
    // GStreamer thread is stopped. Can safely access m_buffersAvailable
    if (m_buffersAvailable != 0) {
        m_buffersAvailable = 0;
        bufferAvailableChanged(false);
    }
 
    positionChanged(kInvalidPosition);
    durationChanged(kInvalidDuration);
 
    setIsDecoding(false);
}
 
QAudioFormat QGstreamerAudioDecoder::audioFormat() const
{
    return mFormat;
}
 
void QGstreamerAudioDecoder::setAudioFormat(const QAudioFormat &format)
{
    if (mFormat != format) {
        mFormat = format;
        formatChanged(mFormat);
    }
}
 
QAudioBuffer QGstreamerAudioDecoder::read()
{
    using namespace std::chrono;
 
    QAudioBuffer audioBuffer;
 
    if (m_buffersAvailable == 0)
        return audioBuffer;
 
    m_buffersAvailable -= 1;
 
    if (m_buffersAvailable == 0)
        bufferAvailableChanged(false);
 
    QGstSampleHandle sample = m_appSink.pullSample();
    GstBuffer *buffer = gst_sample_get_buffer(sample.get());
    GstMapInfo mapInfo;
    gst_buffer_map(buffer, &mapInfo, GST_MAP_READ);
    const char *bufferData = (const char *)mapInfo.data;
    int bufferSize = mapInfo.size;
    QAudioFormat format = QGstUtils::audioFormatForSample(sample.get());
 
    if (format.isValid()) {
        // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer.
        // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer.
        nanoseconds position = getPositionFromBuffer(buffer);
        // QAudioBuffer expects startTime in microseconds, but the platform
        // QPlatformAudioDecoder stores/uses milliseconds resolution. Convert
        // the nanoseconds position to microseconds for the buffer, and to
        // milliseconds for the stored platform position.
        const auto startUs = std::chrono::round<std::chrono::microseconds>(position);
        audioBuffer = QAudioBuffer{ QByteArray(bufferData, bufferSize), format, startUs.count() };
        positionChanged(std::chrono::round<std::chrono::milliseconds>(position));
    }
    gst_buffer_unmap(buffer, &mapInfo);
 
    return audioBuffer;
}
 
void QGstreamerAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString)
{
    stop();
    error(errorCode, errorString);
}
 
GstFlowReturn QGstreamerAudioDecoder::newSample(GstAppSink *)
{
    // "Note that the preroll buffer will also be returned as the first buffer when calling
    // gst_app_sink_pull_buffer()."
 
    QMetaObject::invokeMethod(this, [this, sessionId = m_currentSessionId] {
        if (sessionId != m_currentSessionId)
            return; // stop()ed before message is executed
 
        m_buffersAvailable += 1;
        bufferAvailableChanged(true);
        bufferReady();
    });
 
    return GST_FLOW_OK;
}
 
GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *sink, gpointer user_data)
{
    QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder *>(user_data);
    qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::new_sample";
    return decoder->newSample(sink);
}
 
void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio)
{
    int flags = m_playbin.getInt("flags");
    // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired
    // it prevents audio format conversion
    flags &= ~(GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_NATIVE_VIDEO | GST_PLAY_FLAG_TEXT | GST_PLAY_FLAG_VIS | GST_PLAY_FLAG_NATIVE_AUDIO);
    flags |= GST_PLAY_FLAG_AUDIO;
    if (wantNativeAudio)
        flags |= GST_PLAY_FLAG_NATIVE_AUDIO;
    m_playbin.set("flags", flags);
}
 
void QGstreamerAudioDecoder::addAppSink()
{
    using namespace std::chrono_literals;
 
    if (m_appSink)
        return;
 
    qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::addAppSink";
    m_appSink = QGstAppSink::create("decoderAppSink");
    GstAppSinkCallbacks callbacks{};
    callbacks.new_sample = new_sample;
    m_appSink.setCallbacks(callbacks, this, nullptr);
 
#if GST_CHECK_VERSION(1, 24, 0)
    static constexpr auto maxBufferTime = 500ms;
    m_appSink.setMaxBufferTime(maxBufferTime);
#else
    static constexpr int maxBuffers = 16;
    m_appSink.setMaxBuffers(maxBuffers);
#endif
 
    static constexpr bool sync = false;
    m_appSink.setSync(sync);
 
    m_audioConvert.src().modifyPipelineInIdleProbe([&] {
        m_outputBin.add(m_appSink);
        qLinkGstElements(m_audioConvert, m_appSink);
    });
}
 
void QGstreamerAudioDecoder::removeAppSink()
{
    if (!m_appSink)
        return;
 
    qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::removeAppSink";
 
    m_audioConvert.src().modifyPipelineInIdleProbe([&] {
        qUnlinkGstElements(m_audioConvert, m_appSink);
        m_outputBin.stopAndRemoveElements(m_appSink);
    });
 
    m_appSink = {};
}
 
void QGstreamerAudioDecoder::updateDuration()
{
    std::optional<std::chrono::milliseconds> dur = m_playbin.durationInMs();
    if (!dur)
        dur = kInvalidDuration;
 
    durationChanged(*dur);
 
    if (dur->count() > 0)
        m_durationQueries = 0;
 
    if (m_durationQueries > 0) {
        //increase delay between duration requests
        int delay = 25 << (5 - m_durationQueries);
        QTimer::singleShot(delay, this, &QGstreamerAudioDecoder::updateDuration);
        m_durationQueries--;
    }
}
 
std::chrono::nanoseconds QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer *buffer)
{
    using namespace std::chrono;
    using namespace std::chrono_literals;
    nanoseconds position{ GST_BUFFER_TIMESTAMP(buffer) };
    if (position >= 0ns)
        return position;
    else
        return std::chrono::nanoseconds{ kInvalidPosition };
}
 
QT_END_NAMESPACE
 
#include "moc_qgstreameraudiodecoder_p.cpp"