qffmpegstreamdecoder.cpp

Go to the documentation of this file.

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qffmpegstreamdecoder_p.h"
 
#include "playbackengine/qffmpegmediadataholder_p.h"
#include <QtCore/qloggingcategory.h>
#include <QtCore/qregularexpression.h>
#include <QtCore/qspan.h>
 
QT_BEGIN_NAMESPACE
 
Q_STATIC_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder");
 
namespace QFFmpeg {
 
using namespace Qt::Literals;
 
StreamDecoder::StreamDecoder(const PlaybackEngineObjectID &id, const CodecContext &codecContext,
                             TrackPosition absSeekPos)
    : PlaybackEngineObject(id),
      m_codecContext(codecContext),
      m_trackType(*MediaDataHolder::trackTypeFromMediaType(codecContext.context()->codec_type)),
      m_sessionCtx{ absSeekPos }
{
    qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType
                              << "absSeekPos:" << absSeekPos.get();
}
 
StreamDecoder::~StreamDecoder()
{
    avcodec_flush_buffers(m_codecContext.context());
}
 
void StreamDecoder::seek(quint64 sessionID, TrackPosition pos, const LoopOffset &offset)
{
    updateSession(sessionID, [this, pos, offset]() {
        m_sessionCtx = { offset.loopStartTimeUs.asDuration() + pos };
        avcodec_flush_buffers(m_codecContext.context());
    });
}
 
void StreamDecoder::onFinalPacketReceived(PlaybackEngineObjectID sourceID)
{
    if (checkSessionID(sourceID.sessionID))
        decode({});
}
 
void StreamDecoder::decode(Packet packet)
{
    if (packet.isValid() && !checkSessionID(packet.sourceID().sessionID)) {
        qCDebug(qLcStreamDecoder) << "Packet session outdated. Source id:" << packet.sourceID()
                                  << "current id" << id();
        // no need to report packetProcessed: demuxer must be cleaned up
        return;
    }
 
    m_sessionCtx.packets.enqueue(std::move(packet));
    scheduleNextStep();
}
 
void StreamDecoder::doNextStep()
{
    Packet packet = m_sessionCtx.packets.dequeue();
 
    auto decodePacket = [this](const Packet &packet) {
        if (trackType() == TrackType::SubtitleStream)
            decodeSubtitle(packet);
        else
            decodeMedia(packet);
    };
 
    if (packet.isValid() && packet.loopOffset().loopIndex != m_sessionCtx.offset.loopIndex) {
        decodePacket({});
 
        qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:"
                                  << packet.loopOffset().loopIndex;
 
        avcodec_flush_buffers(m_codecContext.context());
        m_sessionCtx.offset = packet.loopOffset();
    }
 
    decodePacket(packet);
 
    setAtEnd(!packet.isValid());
 
    if (packet.isValid())
        emit packetProcessed(std::move(packet));
 
    scheduleNextStep();
}
 
StreamDecoder::TrackType StreamDecoder::trackType() const
{
    return m_trackType;
}
 
qint32 StreamDecoder::maxQueueSize(TrackType type)
{
    switch (type) {
 
    case TrackType::VideoStream:
        return 3;
    case TrackType::AudioStream:
        return 9;
    case TrackType::SubtitleStream:
        return 6; /*main packet and closing packet*/
    default:
        Q_UNREACHABLE_RETURN(-1);
    }
}
 
void StreamDecoder::onFrameProcessed(const Frame &frame)
{
    if (!checkID(frame.sourceID()))
        return;
 
    --m_sessionCtx.pendingFramesCount;
    Q_ASSERT(m_sessionCtx.pendingFramesCount >= 0);
 
    scheduleNextStep();
}
 
bool StreamDecoder::canDoNextStep() const
{
    const qint32 maxCount = maxQueueSize(m_trackType);
 
    return !m_sessionCtx.packets.empty() && m_sessionCtx.pendingFramesCount < maxCount
            && PlaybackEngineObject::canDoNextStep();
}
 
void StreamDecoder::onFrameFound(const Frame &frame)
{
    if (frame.isValid() && frame.absoluteEnd() < m_sessionCtx.absSeekPos)
        return;
 
    Q_ASSERT(m_sessionCtx.pendingFramesCount >= 0);
    ++m_sessionCtx.pendingFramesCount;
    emit requestHandleFrame(frame);
}
 
void StreamDecoder::decodeMedia(const Packet &packet)
{
    auto sendPacketResult = sendAVPacket(packet);
 
    if (sendPacketResult == AVERROR(EAGAIN)) {
        // Doc says:
        //  AVERROR(EAGAIN): input is not accepted in the current state - user
        //                   must read output with avcodec_receive_frame() (once
        //                   all output is read, the packet should be resent, and
        //                   the call will not fail with EAGAIN).
        receiveAVFrames();
        sendPacketResult = sendAVPacket(packet);
 
        if (sendPacketResult != AVERROR(EAGAIN))
            qWarning() << "Unexpected FFmpeg behavior";
    }
 
    if (sendPacketResult == 0)
        receiveAVFrames(!packet.isValid());
}
 
int StreamDecoder::sendAVPacket(const Packet &packet)
{
    return avcodec_send_packet(m_codecContext.context(), packet.isValid() ? packet.avPacket() : nullptr);
}
 
void StreamDecoder::receiveAVFrames(bool flushPacket)
{
    while (true) {
        auto avFrame = makeAVFrame();
 
        const auto receiveFrameResult = avcodec_receive_frame(m_codecContext.context(), avFrame.get());
 
        if (receiveFrameResult == AVERROR_EOF || receiveFrameResult == AVERROR(EAGAIN)) {
            if (flushPacket && receiveFrameResult == AVERROR(EAGAIN)) {
            // The documentation says that in the EAGAIN state output is not available. The new
            // input must be sent. It does not say that this state can also be returned for
            // Android MediaCodec when the ff_AMediaCodec_dequeueOutputBuffer call times out.
            // The flush packet means it is the end of the stream. No more packets are available,
            // so getting EAGAIN is unexpected here. At this point, the EAGAIN status was probably
            // caused by a timeout in the ffmpeg implementation, not by too few packets. That is
            // why there will be another try of calling avcodec_receive_frame
                qWarning() << "Unexpected FFmpeg behavior: EAGAIN state for avcodec_receive_frame "
                           << "at end of the stream";
                flushPacket = false;
                continue;
            }
            break;
        }
 
        if (receiveFrameResult < 0) {
            emit error(QMediaPlayer::FormatError, err2str(receiveFrameResult));
            break;
        }
 
 
        // Avoid starvation on FFmpeg decoders with fixed size frame pool
        if (m_trackType == TrackType::VideoStream)
            avFrame = copyFromHwPool(std::move(avFrame));
 
        onFrameFound({ m_sessionCtx.offset, std::move(avFrame), m_codecContext, id() });
    }
}
 
void StreamDecoder::decodeSubtitle(const Packet &packet)
{
    if (!packet.isValid())
        return;
    //    qCDebug(qLcDecoder) << "    decoding subtitle" << "has delay:" <<
    //    (codec->codec->capabilities & AV_CODEC_CAP_DELAY);
    AVSubtitle subtitle;
    memset(&subtitle, 0, sizeof(subtitle));
    int gotSubtitle = 0;
 
    const int res =
            avcodec_decode_subtitle2(m_codecContext.context(), &subtitle, &gotSubtitle, packet.avPacket());
    //    qCDebug(qLcDecoder) << "       subtitle got:" << res << gotSubtitle << subtitle.format <<
    //    Qt::hex << (quint64)subtitle.pts;
    if (res < 0 || !gotSubtitle)
        return;
 
    // apparently the timestamps in the AVSubtitle structure are not always filled in
    // if they are missing, use the packets pts and duration values instead
    TrackPosition start = 0, end = 0;
    if (subtitle.pts == AV_NOPTS_VALUE) {
        start = m_codecContext.toTrackPosition(AVStreamPosition(packet.avPacket()->pts));
        end = start + m_codecContext.toTrackDuration(AVStreamDuration(packet.avPacket()->duration));
    } else {
        auto pts = timeStampUs(subtitle.pts, AVRational{ 1, AV_TIME_BASE });
        start = TrackPosition(*pts + qint64(subtitle.start_display_time) * 1000);
        end = TrackPosition(*pts + qint64(subtitle.end_display_time) * 1000);
    }
 
    if (end <= start) {
        qWarning() << "Invalid subtitle time";
        return;
    }
    //        qCDebug(qLcDecoder) << "    got subtitle (" << start << "--" << end << "):";
    QString text = subtitleTextFromAVSubtitle(subtitle);
 
    onFrameFound({ m_sessionCtx.offset, text, start, end - start, id() });
 
    // TODO: maybe optimize
    onFrameFound({ m_sessionCtx.offset, QString(), end, TrackDuration(0), id() });
}
 
static QString extractSubtitleText(const AVSubtitleRect &r)
{
    if (r.text)
        return QString::fromUtf8(r.text);
 
    const char *ass = r.ass;
    int nCommas = 0;
    while (*ass) {
        if (nCommas == 8)
            break;
        if (*ass == ',')
            ++nCommas;
        ++ass;
    }
    return QString::fromUtf8(ass);
}
 
QString subtitleTextFromAVSubtitle(const AVSubtitle &subtitle)
{
    QString text;
 
    for (const AVSubtitleRect *r : QSpan(subtitle.rects, subtitle.num_rects)) {
        if (r != subtitle.rects[0])
            text += u'\n';
 
        text += extractSubtitleText(*r);
    }
    static const QRegularExpression lineBreakRe(u"\\\\N|\\\\n|\\r\\n"_s);
    text.replace(lineBreakRe, u"\n"_s);
    if (text.endsWith(u'\n'))
        text.chop(1);
    return text;
}
 
} // namespace QFFmpeg
 
QT_END_NAMESPACE
 
#include "moc_qffmpegstreamdecoder_p.cpp"