qgstreamermediaplayer.cpp

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include <common/qgstreamermediaplayer_p.h>
 
#include <audio/qgstreameraudiodevice_p.h>
#include <common/qglist_helper_p.h>
#include <common/qgst_debug_p.h>
#include <common/qgstutils_p.h>
#include <common/qgst_discoverer_p.h>
#include <common/qgst_play_p.h>
#include <common/qgstpipeline_p.h>
#include <common/qgstreameraudiooutput_p.h>
#include <common/qgstreamermessage_p.h>
#include <common/qgstreamermetadata_p.h>
#include <common/qgstreamervideooutput_p.h>
#include <common/qgstreamervideosink_p.h>
#include <uri_handler/qgstreamer_qiodevice_handler_p.h>
#include <qgstreamerformatinfo_p.h>
 
#include <QtMultimedia/private/qthreadlocalrhi_p.h>
#include <QtMultimedia/qaudiodevice.h>
#include <QtConcurrent/qtconcurrentrun.h>
#include <QtCore/qdeadlinetimer.h>
#include <QtCore/qdebug.h>
#include <QtMultimedia/private/qmultimedia_ranges_p.h>
#include <QtCore/qiodevice.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/qthread.h>
#include <QtCore/qurl.h>
#include <QtCore/private/quniquehandle_p.h>
 
Q_STATIC_LOGGING_CATEGORY(qLcMediaPlayer, "qt.multimedia.player");
 
QT_BEGIN_NAMESPACE
namespace ranges = QtMultimediaPrivate::ranges;
 
namespace {
 
std::optional<QGstreamerMediaPlayer::TrackType> toTrackType(const QGstCaps &caps)
{
    using TrackType = QGstreamerMediaPlayer::TrackType;
 
    QByteArrayView type = caps.at(0).name();
 
    if (type.startsWith("video/x-raw"))
        return TrackType::VideoStream;
    if (type.startsWith("audio/x-raw"))
        return TrackType::AudioStream;
    if (type.startsWith("text"))
        return TrackType::SubtitleStream;
 
    return std::nullopt;
}
 
} // namespace
 
QFuture<QGstreamerMediaPlayer::DiscoverResult> QGstreamerMediaPlayer::discover(QUrl url)
{
    return QtConcurrent::run([url = std::move(url)] {
        QGst::QGstDiscoverer discoverer;
        return discoverer.discover(url);
    });
}
 
void QGstreamerMediaPlayer::handleDiscoverResult(const DiscoverResult &discoveryResult,
                                                 const QUrl &url)
{
    using namespace Qt::Literals;
    using namespace std::chrono;
    using namespace std::chrono_literals;
 
    if (discoveryResult) {
        // Make sure GstPlay is ready if play() is called from slots during discovery
        qCDebug(qLcMediaPlayer) << "gst_play_set_uri";
        gst_play_set_uri(m_gstPlay.get(), url.toEncoded().constData());
 
        m_trackMetaData.fill({});
        seekableChanged(discoveryResult->isSeekable);
        if (discoveryResult->duration)
            m_duration = round<milliseconds>(*discoveryResult->duration);
        else
            m_duration = 0ms;
        durationChanged(m_duration);
 
        m_metaData = QGst::toContainerMetadata(*discoveryResult);
 
        videoAvailableChanged(!discoveryResult->videoStreams.empty());
        audioAvailableChanged(!discoveryResult->audioStreams.empty());
 
        m_nativeSize.clear();
        for (const auto &videoInfo : discoveryResult->videoStreams) {
            m_trackMetaData[0].emplace_back(QGst::toStreamMetadata(videoInfo));
            QSize nativeSize = QGstUtils::qCalculateFrameSizeGStreamer(videoInfo.size,
                                                                       videoInfo.pixelAspectRatio);
            m_nativeSize.emplace_back(nativeSize);
        }
        for (const auto &audioInfo : discoveryResult->audioStreams)
            m_trackMetaData[1].emplace_back(QGst::toStreamMetadata(audioInfo));
        for (const auto &subtitleInfo : discoveryResult->subtitleStreams)
            m_trackMetaData[2].emplace_back(QGst::toStreamMetadata(subtitleInfo));
 
        using Key = QMediaMetaData::Key;
        auto copyKeysToRootMetadata = [&](const QMediaMetaData &reference, QSpan<const Key> keys) {
            for (QMediaMetaData::Key key : keys) {
                QVariant referenceValue = reference.value(key);
                if (referenceValue.isValid())
                    m_metaData.insert(key, referenceValue);
            }
        };
 
        // FIXME: we duplicate some metadata for the first audio / video track
        // in future we will want to use e.g. the currently selected track
        if (!m_trackMetaData[0].empty())
            copyKeysToRootMetadata(m_trackMetaData[0].front(),
                                   {
                                           Key::HasHdrContent,
                                           Key::Orientation,
                                           Key::Resolution,
                                           Key::VideoBitRate,
                                           Key::VideoCodec,
                                           Key::VideoFrameRate,
                                   });
 
        if (!m_trackMetaData[1].empty())
            copyKeysToRootMetadata(m_trackMetaData[1].front(),
                                   {
                                           Key::AudioBitRate,
                                           Key::AudioCodec,
                                   });
 
        if (!m_url.isEmpty())
            m_metaData.insert(QMediaMetaData::Key::Url, m_url);
 
        qCDebug(qLcMediaPlayer) << "metadata:" << m_metaData;
        qCDebug(qLcMediaPlayer) << "video metadata:" << m_trackMetaData[0];
        qCDebug(qLcMediaPlayer) << "audio metadata:" << m_trackMetaData[1];
        qCDebug(qLcMediaPlayer) << "subtitle metadata:" << m_trackMetaData[2];
 
        metaDataChanged();
        tracksChanged();
        m_activeTrack = {
            isVideoAvailable() ? 0 : -1,
            isAudioAvailable() ? 0 : -1,
            -1,
        };
        updateVideoTrackEnabled();
        updateAudioTrackEnabled();
        updateNativeSizeOnVideoOutput();
        positionChanged(0ms);
 
        // Handle the last play/pause/stop call made during async media loading.
        m_hasPendingMedia = false;
        if (m_requestedPlaybackState) {
            switch (*m_requestedPlaybackState) {
            case QMediaPlayer::PlayingState:
                play();
                break;
            case QMediaPlayer::PausedState:
                pause();
                break;
            default:
                break;
            }
        }
 
    } else {
        qCDebug(qLcMediaPlayer) << "Discovery error:" << discoveryResult.error();
        m_resourceErrorState = ResourceErrorState::ErrorOccurred;
        setInvalidMediaWithError(QMediaPlayer::Error::ResourceError,
                                 u"Resource cannot be discovered"_s);
        m_hasPendingMedia = false;
        resetStateForEmptyOrInvalidMedia();
    };
}
 
void QGstreamerMediaPlayer::decoderPadAddedCustomSource(const QGstElement &src, const QGstPad &pad)
{
    // GStreamer or application thread
    if (src != decoder)
        return;
 
    qCDebug(qLcMediaPlayer) << "Added pad" << pad.name() << "from" << src.name();
 
    QGstCaps caps = pad.queryCaps();
 
    std::optional<QGstreamerMediaPlayer::TrackType> type = toTrackType(caps);
    if (!type)
        return;
 
    customPipelinePads[*type] = pad;
 
    switch (*type) {
    case VideoStream: {
        QGstElement sink = gstVideoOutput->gstreamerVideoSink()
                ? gstVideoOutput->gstreamerVideoSink()->gstSink()
                : QGstElement::createFromPipelineDescription("fakesink");
 
        customPipeline.add(sink);
        pad.link(sink.sink());
        customPipelineSinks[VideoStream] = sink;
        sink.syncStateWithParent();
        return;
    }
    case AudioStream: {
        QGstElement sink = gstAudioOutput ? gstAudioOutput->gstElement()
                                          : QGstElement::createFromPipelineDescription("fakesink");
        customPipeline.add(sink);
        pad.link(sink.sink());
        customPipelineSinks[AudioStream] = sink;
        sink.syncStateWithParent();
        return;
    }
    case SubtitleStream: {
        QGstElement sink = gstVideoOutput->gstreamerVideoSink()
                ? gstVideoOutput->gstreamerVideoSink()->gstSink()
                : QGstElement::createFromPipelineDescription("fakesink");
        customPipeline.add(sink);
        pad.link(sink.sink());
        customPipelineSinks[SubtitleStream] = sink;
        sink.syncStateWithParent();
        return;
    }
 
    default:
        Q_UNREACHABLE();
    }
}
 
void QGstreamerMediaPlayer::decoderPadRemovedCustomSource(const QGstElement &src,
                                                          const QGstPad &pad)
{
    if (src != decoder)
        return;
 
    // application thread!
    Q_ASSERT(thread()->isCurrentThread());
 
    qCDebug(qLcMediaPlayer) << "Removed pad" << pad.name() << "from" << src.name() << "for stream"
                            << pad.streamId();
 
    auto found = ranges::find(customPipelinePads, pad);
    if (found == customPipelinePads.end())
        return;
 
    TrackType type = TrackType(found - customPipelinePads.cbegin());
 
    switch (type) {
    case VideoStream:
    case AudioStream:
    case SubtitleStream: {
        if (customPipelineSinks[VideoStream]) {
            customPipeline.stopAndRemoveElements(customPipelineSinks[VideoStream]);
            customPipelineSinks[VideoStream] = {};
        }
        return;
 
    default:
        Q_UNREACHABLE();
    }
    }
}
 
void QGstreamerMediaPlayer::resetStateForEmptyOrInvalidMedia()
{
    using namespace std::chrono_literals;
    m_nativeSize.clear();
 
    bool metadataNeedsSignal = !m_metaData.isEmpty();
    bool tracksNeedsSignal = ranges::any_of(m_trackMetaData, [](const auto &container) {
        return !container.empty();
    });
 
    m_metaData.clear();
    m_trackMetaData.fill({});
    m_duration = 0ms;
    seekableChanged(false);
 
    videoAvailableChanged(false);
    audioAvailableChanged(false);
 
    m_activeTrack.fill(-1);
 
    if (metadataNeedsSignal)
        metaDataChanged();
    if (tracksNeedsSignal)
        tracksChanged();
}
 
void QGstreamerMediaPlayer::updateNativeSizeOnVideoOutput()
{
    int activeVideoTrack = activeTrack(TrackType::VideoStream);
    bool hasVideoTrack = activeVideoTrack != -1;
 
    QSize nativeSize = hasVideoTrack ? m_nativeSize[activeTrack(TrackType::VideoStream)] : QSize{};
 
    QVariant orientation = hasVideoTrack
            ? m_trackMetaData[TrackType::VideoStream][activeTrack(TrackType::VideoStream)].value(
                    QMediaMetaData::Key::Orientation)
            : QVariant{};
 
    if (orientation.isValid()) {
        auto rotation = orientation.value<QtVideo::Rotation>();
        gstVideoOutput->setRotation(rotation);
    }
    gstVideoOutput->setNativeSize(nativeSize);
}
 
void QGstreamerMediaPlayer::seekToCurrentPosition()
{
    qCDebug(qLcMediaPlayer) << "gst_play_seek";
    gst_play_seek(m_gstPlay.get(), gst_play_get_position(m_gstPlay.get()));
}
 
void QGstreamerMediaPlayer::updateVideoTrackEnabled()
{
    bool hasTrack = m_activeTrack[TrackType::VideoStream] != -1;
    bool hasSink = gstVideoOutput->gstreamerVideoSink() != nullptr;
 
    gstVideoOutput->setActive(hasTrack);
    gst_play_set_video_track_enabled(m_gstPlay.get(), hasTrack && hasSink);
}
 
void QGstreamerMediaPlayer::updateAudioTrackEnabled()
{
    bool hasTrack = m_activeTrack[TrackType::AudioStream] != -1;
    bool hasAudioOut = gstAudioOutput;
 
    gst_play_set_audio_track_enabled(m_gstPlay.get(), hasTrack && hasAudioOut);
}
 
void QGstreamerMediaPlayer::updateBufferProgress(float newProgress)
{
    if (qFuzzyIsNull(newProgress - m_bufferProgress))
        return;
 
    m_bufferProgress = newProgress;
    bufferProgressChanged(m_bufferProgress);
}
 
void QGstreamerMediaPlayer::disconnectDecoderHandlers()
{
    auto handlers = std::initializer_list<QGObjectHandlerScopedConnection *>{ &sourceSetup };
    for (QGObjectHandlerScopedConnection *handler : handlers)
        handler->disconnect();
}
 
q23::expected<QPlatformMediaPlayer *, QString> QGstreamerMediaPlayer::create(QMediaPlayer *parent)
{
    auto videoOutput = QGstreamerVideoOutput::create();
    if (!videoOutput)
        return q23::unexpected{ videoOutput.error() };
 
    return new QGstreamerMediaPlayer(videoOutput.value(), parent);
}
 
template <typename T>
void setSeekAccurate(T *config, gboolean accurate)
{
    gst_play_config_set_seek_accurate(config, accurate);
}
 
QGstreamerMediaPlayer::QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput,
                                             QMediaPlayer *parent)
    : QObject(parent),
      QPlatformMediaPlayer(parent),
      gstVideoOutput(videoOutput),
      m_gstPlay{
          gst_play_new(nullptr),
          QGstPlayHandle::HasRef,
      },
      m_playbin{
          GST_PIPELINE_CAST(gst_play_get_pipeline(m_gstPlay.get())),
          QGstPipeline::HasRef,
      },
      m_gstPlayBus{
          QGstBusHandle{ gst_play_get_message_bus(m_gstPlay.get()), QGstBusHandle::HasRef },
      }
{
#if 1
    // LATER: remove this hack after meta-freescale decides not to pull in outdated APIs
 
    // QTBUG-131300: nxp deliberately reverted to an old gst-play API before the gst-play API
    // stabilized. compare:
    // https://github.com/nxp-imx/gst-plugins-bad/commit/ff04fa9ca1b79c98e836d8cdb26ac3502dafba41
 
    // Update: Looks like first unaffected Yocto release series is 5.3 Whinlatter, so this hack can
    // likely be removed when we drop Boot to Qt support for the LTS 5.0 Scarthgap (EOL April 2028).
    // Link to first unaffected nxp gstreamer branch:
    // https://github.com/nxp-imx/gst-plugins-bad/blame/MM_04.10.0_2505_L6.12.20/gst-libs/gst/play/gstplay.c#L4690C9-L4690C9
    constexpr bool useNxpWorkaround = std::is_same_v<decltype(&gst_play_config_set_seek_accurate),
                                                     void (*)(GstPlay *, gboolean)>;
 
    QUniqueGstStructureHandle config{
        gst_play_get_config(m_gstPlay.get()),
    };
 
    if constexpr (useNxpWorkaround)
        setSeekAccurate(m_gstPlay.get(), true);
    else
        setSeekAccurate(config.get(), true);
 
    gst_play_set_config(m_gstPlay.get(), config.release());
#else
    QUniqueGstStructureHandle config{
        gst_play_get_config(m_gstPlay.get()),
    };
    gst_play_config_set_seek_accurate(config.get(), true);
    gst_play_set_config(m_gstPlay.get(), config.release());
#endif
 
    gstVideoOutput->setParent(this);
 
    // NOTE: Creating a GStreamer video sink to be owned by the media player, any sink created by
    // user would be a pluggable sink connected to this
    m_gstVideoSink = new QGstreamerRelayVideoSink(this);
    gstVideoOutput->setVideoSink(m_gstVideoSink);
 
    m_playbin.set("video-sink", gstVideoOutput->gstElement());
    m_playbin.set("text-sink", gstVideoOutput->gstSubtitleElement());
    m_playbin.set("audio-sink", QGstElement::createFromPipelineDescription("fakesink"));
 
    m_gstPlayBus.installMessageFilter(this);
 
    // we start without subtitles
    gst_play_set_subtitle_track_enabled(m_gstPlay.get(), false);
 
    sourceSetup = m_playbin.connect("source-setup", GCallback(sourceSetupCallback), this);
 
    m_activeTrack.fill(-1);
 
    // TODO: how to detect stalled media?
}
 
QGstreamerMediaPlayer::~QGstreamerMediaPlayer()
{
    using namespace std::chrono_literals;
    using namespace std::chrono;
    qCDebug(qLcMediaPlayer) << "~QGstreamerMediaPlayer" << m_callStopInDestructor
                            << m_expectedStoppedMessages << state();
 
    if (customPipeline)
        cleanupCustomPipeline();
 
    m_gstPlayBus.removeMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this));
 
    if (m_callStopInDestructor) {
        m_expectedStoppedMessages += 1;
        qCDebug(qLcMediaPlayer) << "gst_play_stop";
        gst_play_stop(m_gstPlay.get());
    }
 
    if (m_expectedStoppedMessages) {
        auto processNextMessage = [this](nanoseconds timeout) {
            QGstreamerMessage message{
                gst_bus_timed_pop_filtered(m_gstPlayBus.get(), timeout.count(),
                                           GST_MESSAGE_APPLICATION), QGstreamerMessage::HasRef
            };
            if (!message || !gst_play_is_play_message(message.message()))
                return;
            GstPlayMessage type;
            gst_play_message_parse_type(message.message(), &type);
            qCDebug(qLcMediaPlayer) << QGstPlayMessageAdaptor{ message };
            if (type == GST_PLAY_MESSAGE_END_OF_STREAM) {
                m_expectedStoppedMessages += 1;
                return;
            }
            if (type == GST_PLAY_MESSAGE_STATE_CHANGED) {
                GstPlayState state;
                gst_play_message_parse_state_changed(message.message(), &state);
                if (state == GST_PLAY_STATE_STOPPED) {
                    if (m_expectedStoppedMessages > 0)
                        m_expectedStoppedMessages -= 1;
                }
            }
        };
 
        constexpr auto timeout = 200ms;
        QDeadlineTimer deadline{timeout};
        while (!deadline.hasExpired() && m_expectedStoppedMessages)
            processNextMessage(deadline.remainingTimeAsDuration());
 
        if (m_expectedStoppedMessages)
            qWarning() << "Did not receive expected STOPPED state change messages from GstPlay "
                          "in QGstreamerMediaPlayer destructor:" << m_expectedStoppedMessages;
    }
 
    // NOTE: gst_play_stop is not sufficient, un-reffing m_gstPlay can deadlock
    m_playbin.setStateSync(GST_STATE_NULL);
 
    m_playbin.set("video-sink", QGstElement::createFromPipelineDescription("fakesink"));
    m_playbin.set("text-sink", QGstElement::createFromPipelineDescription("fakesink"));
    m_playbin.set("audio-sink", QGstElement::createFromPipelineDescription("fakesink"));
 
    gst_bus_set_flushing(m_gstPlayBus.get(), TRUE);
}
 
void QGstreamerMediaPlayer::updatePositionFromPipeline()
{
    using namespace std::chrono;
 
    positionChanged(round<milliseconds>(nanoseconds{
            gst_play_get_position(m_gstPlay.get()),
    }));
}
 
bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message)
{
    if (isCustomSource()) {
        constexpr bool traceBusMessages = true;
        if (traceBusMessages)
            qCDebug(qLcMediaPlayer) << "received bus message:" << message;
 
        switch (message.type()) {
        case GST_MESSAGE_WARNING:
            qWarning() << "received bus message:" << message;
            break;
 
        case GST_MESSAGE_INFO:
            qInfo() << "received bus message:" << message;
            break;
 
        case GST_MESSAGE_ERROR:
            qWarning() << "received bus message:" << message;
            customPipeline.dumpPipelineGraph("GST_MESSAGE_ERROR");
            break;
 
        case GST_MESSAGE_LATENCY:
            customPipeline.recalculateLatency();
            break;
 
        default:
            break;
        }
        return false;
    }
 
    switch (message.type()) {
    case GST_MESSAGE_APPLICATION:
        if (gst_play_is_play_message(message.message()))
            return processBusMessageApplication(message);
        return false;
 
    default:
        qCDebug(qLcMediaPlayer) << message;
 
        return false;
    }
 
    return false;
}
 
bool QGstreamerMediaPlayer::processBusMessageApplication(const QGstreamerMessage &message)
{
    using namespace std::chrono;
    GstPlayMessage type;
    gst_play_message_parse_type(message.message(), &type);
    qCDebug(qLcMediaPlayer) << QGstPlayMessageAdaptor{ message };
 
    switch (type) {
    case GST_PLAY_MESSAGE_URI_LOADED: {
        mediaStatusChanged(QMediaPlayer::LoadedMedia);
        return false;
    }
 
    case GST_PLAY_MESSAGE_POSITION_UPDATED: {
        if (state() == QMediaPlayer::PlaybackState::PlayingState) {
 
            constexpr bool usePayload = false;
            if constexpr (usePayload) {
                GstClockTime position;
                gst_play_message_parse_position_updated(message.message(), &position);
                positionChanged(round<milliseconds>(nanoseconds{ position }));
            } else {
                GstClockTime position = gst_play_get_position(m_gstPlay.get());
                positionChanged(round<milliseconds>(nanoseconds{ position }));
            }
        }
        return false;
    }
    case GST_PLAY_MESSAGE_DURATION_CHANGED: {
        GstClockTime duration;
        gst_play_message_parse_duration_updated(message.message(), &duration);
        milliseconds durationInMs = round<milliseconds>(nanoseconds{ duration });
        durationChanged(durationInMs);
 
        m_metaData.insert(QMediaMetaData::Duration, int(durationInMs.count()));
        metaDataChanged();
 
        return false;
    }
    case GST_PLAY_MESSAGE_BUFFERING: {
        guint percent;
        gst_play_message_parse_buffering_percent(message.message(), &percent);
        updateBufferProgress(percent * 0.01f);
        return false;
    }
    case GST_PLAY_MESSAGE_STATE_CHANGED: {
        m_playOrPauseCalledSinceLastStateChangedOrEosMessage = false;
        GstPlayState state;
        gst_play_message_parse_state_changed(message.message(), &state);
 
        switch (state) {
        case GstPlayState::GST_PLAY_STATE_STOPPED:
            if (m_expectedStoppedMessages > 0)
                m_expectedStoppedMessages -= 1;
 
            if (stateChangeToSkip) {
                qCDebug(qLcMediaPlayer) << "    skipping StoppedState transition";
 
                stateChangeToSkip -= 1;
                return false;
            }
            stateChanged(QMediaPlayer::StoppedState);
            updateBufferProgress(0);
            return false;
 
        case GstPlayState::GST_PLAY_STATE_PAUSED:
            stateChanged(QMediaPlayer::PausedState);
            mediaStatusChanged(QMediaPlayer::BufferedMedia);
            gstVideoOutput->setActive(true);
            updateBufferProgress(1);
            return false;
        case GstPlayState::GST_PLAY_STATE_BUFFERING:
            mediaStatusChanged(QMediaPlayer::BufferingMedia);
            return false;
        case GstPlayState::GST_PLAY_STATE_PLAYING:
            stateChanged(QMediaPlayer::PlayingState);
            mediaStatusChanged(QMediaPlayer::BufferedMedia);
            gstVideoOutput->setActive(true);
            updateBufferProgress(1);
 
            return false;
        default:
            return false;
        }
    }
    case GST_PLAY_MESSAGE_MEDIA_INFO_UPDATED: {
        using namespace QGstPlaySupport;
 
        QUniqueGstPlayMediaInfoHandle info{};
        gst_play_message_parse_media_info_updated(message.message(), &info);
 
        seekableChanged(gst_play_media_info_is_seekable(info.get()));
 
        const gchar *title = gst_play_media_info_get_title(info.get());
        m_metaData.insert(QMediaMetaData::Title, QString::fromUtf8(title));
 
        metaDataChanged();
        tracksChanged();
 
        return false;
    }
    case GST_PLAY_MESSAGE_END_OF_STREAM: {
        m_expectedStoppedMessages += 1;
        // Set m_callStopInDestructor to false except in the edge case where play or pause is called
        // after reaching EOS, but before we've processed this EOS message
        if (!m_playOrPauseCalledSinceLastStateChangedOrEosMessage)
            m_callStopInDestructor = false;
        m_playOrPauseCalledSinceLastStateChangedOrEosMessage = false;
 
        if (doLoop()) {
            positionChanged(m_duration);
            m_callStopInDestructor = true;
            qCDebug(qLcMediaPlayer) << "EOS: restarting loop (gst_play_play)";
            gst_play_play(m_gstPlay.get());
            positionChanged(0ms);
 
            // we will still get a GST_PLAY_MESSAGE_STATE_CHANGED message, which we will just ignore
            // for now
            stateChangeToSkip += 1;
        } else {
            qCDebug(qLcMediaPlayer) << "EOS: done";
            positionChanged(m_duration);
            mediaStatusChanged(QMediaPlayer::EndOfMedia);
            stateChanged(QMediaPlayer::StoppedState);
            gstVideoOutput->setActive(false);
        }
 
        return false;
    }
    case GST_PLAY_MESSAGE_ERROR:
    case GST_PLAY_MESSAGE_WARNING:
    case GST_PLAY_MESSAGE_VIDEO_DIMENSIONS_CHANGED:
    case GST_PLAY_MESSAGE_VOLUME_CHANGED:
    case GST_PLAY_MESSAGE_MUTE_CHANGED:
    case GST_PLAY_MESSAGE_SEEK_DONE:
        return false;
 
    default:
        Q_UNREACHABLE_RETURN(false);
    }
}
 
qint64 QGstreamerMediaPlayer::duration() const
{
    return m_duration.count();
}
 
bool QGstreamerMediaPlayer::hasMedia() const
{
    return !m_url.isEmpty() || m_stream;
}
 
bool QGstreamerMediaPlayer::hasValidMedia() const
{
    if (!hasMedia())
        return false;
 
    switch (mediaStatus()) {
    case QMediaPlayer::MediaStatus::NoMedia:
    case QMediaPlayer::MediaStatus::InvalidMedia:
        return false;
 
    default:
        return true;
    }
}
 
float QGstreamerMediaPlayer::bufferProgress() const
{
    return m_bufferProgress;
}
 
QMediaTimeRange QGstreamerMediaPlayer::availablePlaybackRanges() const
{
    return QMediaTimeRange();
}
 
qreal QGstreamerMediaPlayer::playbackRate() const
{
    return gst_play_get_rate(m_gstPlay.get());
}
 
void QGstreamerMediaPlayer::setPlaybackRate(qreal rate)
{
    if (isCustomSource()) {
        static std::once_flag flag;
        std::call_once(flag, [] {
            // CAVEAT: unsynchronised with pipeline state. Potentially prone to race conditions
            qWarning()
                    << "setPlaybackRate with custom gstreamer pipelines can cause pipeline hangs. "
                       "Use with care";
        });
 
        customPipeline.setPlaybackRate(rate);
        return;
    }
 
    if (rate == playbackRate())
        return;
 
    qCDebug(qLcMediaPlayer) << "gst_play_set_rate" << rate;
    gst_play_set_rate(m_gstPlay.get(), rate);
    playbackRateChanged(rate);
}
 
void QGstreamerMediaPlayer::setPosition(qint64 pos)
{
    std::chrono::milliseconds posInMs{ pos };
 
    setPosition(posInMs);
}
 
void QGstreamerMediaPlayer::setPosition(std::chrono::milliseconds pos)
{
    using namespace std::chrono;
 
    if (isCustomSource()) {
        static std::once_flag flag;
        std::call_once(flag, [] {
            // CAVEAT: unsynchronised with pipeline state. Potentially prone to race conditions
            qWarning() << "setPosition with custom gstreamer pipelines can cause pipeline hangs. "
                          "Use with care";
        });
 
        customPipeline.setPosition(pos);
        return;
    }
 
    if (m_hasPendingMedia) {
        return;
    }
 
    qCDebug(qLcMediaPlayer) << "gst_play_seek" << pos;
    gst_play_seek(m_gstPlay.get(), nanoseconds(pos).count());
 
    if (mediaStatus() == QMediaPlayer::EndOfMedia)
        mediaStatusChanged(QMediaPlayer::LoadedMedia);
    positionChanged(pos);
}
 
void QGstreamerMediaPlayer::play()
{
    if (isCustomSource()) {
        gstVideoOutput->setActive(true);
        customPipeline.setState(GST_STATE_PLAYING);
        stateChanged(QMediaPlayer::PlayingState);
        return;
    }
 
    if (m_hasPendingMedia) {
        // Async media loading in progress via QGstDiscoverer, m_discoveryHandler will fulfill the
        // last requested playback state later.
        m_requestedPlaybackState = QMediaPlayer::PlayingState;
        return;
    }
 
    QMediaPlayer::PlaybackState currentState = state();
    if (currentState == QMediaPlayer::PlayingState || !hasValidMedia())
        return;
 
    if (currentState != QMediaPlayer::PausedState)
        resetCurrentLoop();
 
    if (mediaStatus() == QMediaPlayer::EndOfMedia) {
        positionChanged(0);
        mediaStatusChanged(QMediaPlayer::LoadedMedia);
    }
 
    if (m_pendingSeek) {
        qCDebug(qLcMediaPlayer) << "gst_play_seek";
        gst_play_seek(m_gstPlay.get(), m_pendingSeek->count());
        m_pendingSeek = std::nullopt;
    }
 
    gstVideoOutput->setActive(true);
    m_callStopInDestructor = true;
    m_playOrPauseCalledSinceLastStateChangedOrEosMessage = true;
    qCDebug(qLcMediaPlayer) << "gst_play_play";
    gst_play_play(m_gstPlay.get());
    stateChanged(QMediaPlayer::PlayingState);
}
 
void QGstreamerMediaPlayer::pause()
{
    if (isCustomSource()) {
        gstVideoOutput->setActive(true);
        customPipeline.setState(GST_STATE_PAUSED);
        stateChanged(QMediaPlayer::PausedState);
        return;
    }
 
    if (m_hasPendingMedia) {
        m_requestedPlaybackState = QMediaPlayer::PausedState;
        return;
    }
 
    if (state() == QMediaPlayer::PausedState || !hasMedia()
        || m_resourceErrorState != ResourceErrorState::NoError)
        return;
 
    gstVideoOutput->setActive(true);
 
    m_callStopInDestructor = true;
    m_playOrPauseCalledSinceLastStateChangedOrEosMessage = true;
    qCDebug(qLcMediaPlayer) << "gst_play_pause";
    gst_play_pause(m_gstPlay.get());
 
    mediaStatusChanged(QMediaPlayer::BufferedMedia);
    stateChanged(QMediaPlayer::PausedState);
}
 
void QGstreamerMediaPlayer::stop()
{
    if (isCustomSource()) {
        customPipeline.setState(GST_STATE_READY);
        stateChanged(QMediaPlayer::StoppedState);
        gstVideoOutput->setActive(false);
        return;
    }
 
    if (m_hasPendingMedia) {
        m_requestedPlaybackState = QMediaPlayer::StoppedState;
        return;
    }
 
    using namespace std::chrono_literals;
    if (state() == QMediaPlayer::StoppedState) {
        if (position() != 0) {
            m_pendingSeek = 0ms;
            positionChanged(0ms);
            mediaStatusChanged(QMediaPlayer::LoadedMedia);
        }
        return;
    }
 
    gstVideoOutput->setActive(false);
    m_expectedStoppedMessages += 1;
    m_callStopInDestructor = false;
    qCDebug(qLcMediaPlayer) << "gst_play_stop";
    gst_play_stop(m_gstPlay.get());
 
    stateChanged(QMediaPlayer::StoppedState);
 
    mediaStatusChanged(QMediaPlayer::LoadedMedia);
    positionChanged(0ms);
}
 
const QGstPipeline &QGstreamerMediaPlayer::pipeline() const
{
    if (isCustomSource())
        return customPipeline;
 
    return m_playbin;
}
 
bool QGstreamerMediaPlayer::canPlayQrc() const
{
    return true;
}
 
bool QGstreamerMediaPlayer::pitchCompensation() const
{
    return true;
}
 
QPlatformMediaPlayer::PitchCompensationAvailability
QGstreamerMediaPlayer::pitchCompensationAvailability() const
{
    return PitchCompensationAvailability::AlwaysOn;
}
 
QUrl QGstreamerMediaPlayer::media() const
{
    return m_url;
}
 
const QIODevice *QGstreamerMediaPlayer::mediaStream() const
{
    return m_stream;
}
 
void QGstreamerMediaPlayer::sourceSetupCallback([[maybe_unused]] GstElement *playbin,
                                                GstElement *source, QGstreamerMediaPlayer *)
{
    // gst_play thread
 
    const gchar *typeName = g_type_name_from_instance((GTypeInstance *)source);
    qCDebug(qLcMediaPlayer) << "Setting up source:" << typeName;
 
    if (typeName == std::string_view("GstRTSPSrc")) {
        QGstElement s(source, QGstElement::NeedsRef);
        int latency{40};
        bool ok{false};
        int v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_LATENCY", &ok);
        if (ok)
            latency = v;
        qCDebug(qLcMediaPlayer) << "    -> setting source latency to:" << latency << "ms";
        s.set("latency", latency);
 
        bool drop{true};
        v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_DROP_ON_LATENCY", &ok);
        if (ok && v == 0)
            drop = false;
        qCDebug(qLcMediaPlayer) << "    -> setting drop-on-latency to:" << drop;
        s.set("drop-on-latency", drop);
 
        bool retrans{false};
        v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_DO_RETRANSMISSION", &ok);
        if (ok && v != 0)
            retrans = true;
        qCDebug(qLcMediaPlayer) << "    -> setting do-retransmission to:" << retrans;
        s.set("do-retransmission", retrans);
    }
}
 
void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream)
{
    if (customPipeline)
        cleanupCustomPipeline();
 
    m_resourceErrorState = ResourceErrorState::NoError;
    m_url = content;
    m_stream = stream;
 
    // cancel any pending discovery continuations
    m_discoveryHandler.cancel();
    m_discoverFuture.cancel();
 
    QUrl streamURL;
    if (stream)
        streamURL = qGstRegisterQIODevice(stream);
 
    if (content.isEmpty() && !stream) {
        mediaStatusChanged(QMediaPlayer::NoMedia);
        resetStateForEmptyOrInvalidMedia();
        return;
    }
 
    if (isCustomSource()) {
        setMediaCustomSource(content);
    } else {
        mediaStatusChanged(QMediaPlayer::LoadingMedia);
        m_hasPendingMedia = true;
        m_requestedPlaybackState = std::nullopt;
 
        const QUrl &playUrl = stream ? streamURL : content;
        m_discoverFuture = discover(playUrl);
 
        m_discoveryHandler =
                m_discoverFuture.then(this,[this, playUrl](const DiscoverResult &result) {
                    handleDiscoverResult(result, playUrl);
                });
    }
}
 
void QGstreamerMediaPlayer::setMediaCustomSource(const QUrl &content)
{
    using namespace Qt::Literals;
    using namespace std::chrono;
    using namespace std::chrono_literals;
 
    {
        // FIXME: claim sinks
        // TODO: move ownership of sinks to gst_play after using them
        m_playbin.set("video-sink", QGstElement::createFromPipelineDescription("fakesink"));
        m_playbin.set("text-sink", QGstElement::createFromPipelineDescription("fakesink"));
        m_playbin.set("audio-sink", QGstElement::createFromPipelineDescription("fakesink"));
 
        if (gstVideoOutput->gstreamerVideoSink()) {
            if (QGstElement sink = gstVideoOutput->gstreamerVideoSink()->gstSink())
                sink.removeFromParent();
        }
    }
 
    customPipeline = QGstPipeline::create("customPipeline");
    customPipeline.installMessageFilter(this);
    positionUpdateTimer = std::make_unique<QTimer>();
 
    QObject::connect(positionUpdateTimer.get(), &QTimer::timeout, this, [this] {
        Q_ASSERT(customPipeline);
        auto position = customPipeline.position();
 
        positionChanged(round<milliseconds>(position));
    });
 
    positionUpdateTimer->start(100ms);
 
    QByteArray gstLaunchString =
            content.toString(QUrl::RemoveScheme | QUrl::PrettyDecoded).toLatin1();
    qCDebug(qLcMediaPlayer) << "generating" << gstLaunchString;
    QGstElement element = QGstElement::createFromPipelineDescription(gstLaunchString);
    if (!element) {
        emit error(QMediaPlayer::ResourceError, u"Could not create custom pipeline"_s);
        return;
    }
 
    decoder = element;
    customPipeline.add(decoder);
 
    QGstBin elementBin{
        qGstSafeCast<GstBin>(element.element()),
        QGstBin::NeedsRef,
    };
    if (elementBin) // bins are expected to provide unconnected src pads
        elementBin.addUnlinkedGhostPads(GstPadDirection::GST_PAD_SRC);
 
    // for all other elements
    padAdded = decoder.onPadAdded<&QGstreamerMediaPlayer::decoderPadAddedCustomSource>(this);
    padRemoved = decoder.onPadRemoved<&QGstreamerMediaPlayer::decoderPadRemovedCustomSource>(this);
 
    customPipeline.setStateSync(GstState::GST_STATE_PAUSED);
 
    auto srcPadVisitor = [](GstElement *element, GstPad *pad, void *self) -> gboolean {
        reinterpret_cast<QGstreamerMediaPlayer *>(self)->decoderPadAddedCustomSource(
                QGstElement{ element, QGstElement::NeedsRef }, QGstPad{ pad, QGstPad::NeedsRef });
        return true;
    };
 
    gst_element_foreach_pad(element.element(), srcPadVisitor, this);
 
    mediaStatusChanged(QMediaPlayer::LoadedMedia);
 
    customPipeline.dumpGraph("setMediaCustomPipeline");
}
 
void QGstreamerMediaPlayer::cleanupCustomPipeline()
{
    customPipeline.setStateSync(GST_STATE_NULL);
    customPipeline.removeMessageFilter(this);
 
    for (QGstElement &sink : customPipelineSinks)
        if (sink)
            customPipeline.remove(sink);
 
    positionUpdateTimer = {};
    customPipeline = {};
}
 
void QGstreamerMediaPlayer::setAudioOutput(QPlatformAudioOutput *output)
{
    if (isCustomSource()) {
        qWarning() << "QMediaPlayer::setAudioOutput not supported when using custom sources";
        return;
    }
 
    if (gstAudioOutput == output)
        return;
 
    auto *gstOutput = static_cast<QGstreamerAudioOutput *>(output);
    if (gstOutput)
        gstOutput->setAsync(true);
 
    gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output);
    if (gstAudioOutput)
        m_playbin.set("audio-sink", gstAudioOutput->gstElement());
    else
        m_playbin.set("audio-sink", QGstElement::createFromPipelineDescription("fakesink"));
    updateAudioTrackEnabled();
 
    // FIXME: we need to have a gst_play API to change the sinks on the fly.
    // finishStateChange a hack to avoid assertion failures in gstreamer
    if (!qmediaplayerDestructorCalled)
        m_playbin.finishStateChange();
}
 
QMediaMetaData QGstreamerMediaPlayer::metaData() const
{
    return m_metaData;
}
 
void QGstreamerMediaPlayer::setVideoSink(QVideoSink *sink)
{
    // Disconnect previous sink
    m_gstVideoSink->disconnectPluggableVideoSink();
 
    if (!sink)
        return;
 
    // Connect pluggable sink to native sink
    auto pluggableSink = dynamic_cast<QGstreamerPluggableVideoSink *>(sink->platformVideoSink());
    Q_ASSERT(pluggableSink);
    m_gstVideoSink->connectPluggableVideoSink(pluggableSink);
}
 
int QGstreamerMediaPlayer::trackCount(QPlatformMediaPlayer::TrackType type)
{
    QSpan<const QMediaMetaData> tracks = m_trackMetaData[type];
    return tracks.size();
}
 
QMediaMetaData QGstreamerMediaPlayer::trackMetaData(QPlatformMediaPlayer::TrackType type, int index)
{
    QSpan<const QMediaMetaData> tracks = m_trackMetaData[type];
    if (index < tracks.size())
        return tracks[index];
    return {};
}
 
int QGstreamerMediaPlayer::activeTrack(TrackType type)
{
    return m_activeTrack[type];
}
 
void QGstreamerMediaPlayer::setActiveTrack(TrackType type, int index)
{
    if (m_activeTrack[type] == index)
        return;
 
    int formerTrack = m_activeTrack[type];
    m_activeTrack[type] = index;
 
    switch (type) {
    case TrackType::VideoStream: {
        if (index != -1)
            gst_play_set_video_track(m_gstPlay.get(), index);
        updateVideoTrackEnabled();
        updateNativeSizeOnVideoOutput();
        break;
    }
    case TrackType::AudioStream: {
        if (index != -1)
            gst_play_set_audio_track(m_gstPlay.get(), index);
        updateAudioTrackEnabled();
        break;
    }
    case TrackType::SubtitleStream: {
        if (index != -1)
            gst_play_set_subtitle_track(m_gstPlay.get(), index);
        gst_play_set_subtitle_track_enabled(m_gstPlay.get(), index != -1);
        break;
    }
    default:
        Q_UNREACHABLE();
    };
 
    if (formerTrack != -1 && index != -1)
        // it can take several seconds for gstreamer to switch the track. so we seek to the current
        // position
        seekToCurrentPosition();
}
 
QT_END_NAMESPACE