qffmpegrecordingengine.cpp

Go to the documentation of this file.

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpegrecordingengine_p.h"
 
#include "qffmpegaudioencoder_p.h"
#include "qffmpegaudioinput_p.h"
#include "qffmpegencodinginitializer_p.h"
#include "qffmpegmediaformatinfo_p.h"
#include "qffmpegrecordingengineutils_p.h"
 
#include "private/qmultimediautils_p.h"
#include "private/qplatformaudiobufferinput_p.h"
#include "private/qplatformvideosource_p.h"
#include "private/qplatformvideoframeinput_p.h"
 
#include "qdebug.h"
#include "qffmpegvideoencoder_p.h"
#include "qffmpegmediametadata_p.h"
#include "qffmpegmuxer_p.h"
#include "qloggingcategory.h"
 
extern "C" {
#include "libavutil/opt.h"
}
 
QT_BEGIN_NAMESPACE
namespace ranges = QtMultimediaPrivate::ranges;
 
Q_STATIC_LOGGING_CATEGORY(qLcFFmpegEncoder, "qt.multimedia.ffmpeg.encoder");
 
namespace QFFmpeg
{
 
namespace {
 
bool supportsNegativeCtsOffsets(const AVOutputFormat *format)
{
    if (!format || !format->priv_class)
        return false;
 
    // Check if muxer can store negative composition offsets in a version 1
    // ctts box, exposed as the negative_cts_offsets movflag
    const AVClass *avClass = format->priv_class;
    return av_opt_find(static_cast<void *>(&avClass), "negative_cts_offsets", "movflags", 0, 0)
            != nullptr;
}
 
AVDictionaryHolder muxerOptions(const AVFormatContext *formatContext)
{
    AVDictionaryHolder options;
 
    if (supportsNegativeCtsOffsets(formatContext->oformat)) {
        // Codecs that reorder frames (H264, H265) emit packets with dts starting negative (frame
        // reorder delay). Muxer can store that delay as negative composition offsets
        av_dict_set(options, "movflags", "+negative_cts_offsets", 0);
        // Disable avformat shifting to keep packets unmodified for muxer
        av_dict_set(options, "avoid_negative_ts", "disabled", 0);
    }
 
    return options;
}
 
} // namespace
 
using namespace Qt::StringLiterals;
 
RecordingEngine::RecordingEngine(const QMediaEncoderSettings &settings,
                 std::unique_ptr<EncodingFormatContext> context)
    : m_settings(settings), m_formatContext(std::move(context)), m_muxer(new Muxer(this))
{
    Q_ASSERT(m_formatContext);
    Q_ASSERT(m_formatContext->isAVIOOpen());
    m_startEncodingPromise.start();
}
 
RecordingEngine::~RecordingEngine()
{
    Q_ASSERT(m_state == State::Finalizing);
}
 
void RecordingEngine::addAudioInput(QFFmpegAudioInput *input)
{
    Q_ASSERT(input);
    Q_ASSERT(m_state == State::FormatsInitializing);
 
    if (input->device.isNull()) {
        Q_EMIT streamInitializationError(QMediaRecorder::ResourceError,
                                       QLatin1StringView("Audio device is null"));
        return;
    }
 
    const QAudioFormat format = input->device.preferredFormat();
 
    if (!format.isValid()) {
        Q_EMIT streamInitializationError(
                QMediaRecorder::FormatError,
                QLatin1StringView("Audio device has invalid preferred format"));
        return;
    }
 
    AudioEncoder *audioEncoder = createAudioEncoder(format);
    if (!audioEncoder)
        return;
 
    connectEncoderToSource(audioEncoder, input);
}
 
void RecordingEngine::addAudioBufferInput(QPlatformAudioBufferInput *input,
                                          const QAudioBuffer &firstBuffer)
{
    Q_ASSERT(input);
    Q_ASSERT(m_state == State::FormatsInitializing);
 
    const QAudioFormat format = firstBuffer.isValid() ? firstBuffer.format() : input->audioFormat();
 
    AudioEncoder *audioEncoder = createAudioEncoder(format);
    if (!audioEncoder)
        return;
 
    // set the buffer before connecting to avoid potential races
    if (firstBuffer.isValid())
        audioEncoder->addBuffer(firstBuffer);
 
    connectEncoderToSource(audioEncoder, input);
}
 
AudioEncoder *RecordingEngine::createAudioEncoder(const QAudioFormat &format)
{
    Q_ASSERT(format.isValid());
 
    const AVCodecID codecID = QFFmpegMediaFormatInfo::codecIdForAudioCodec(m_settings.audioCodec());
    if (codecID == AV_CODEC_ID_NONE) {
        Q_EMIT streamInitializationError(QMediaRecorder::FormatError,
                                         u"No audio encoder available for the selected format"_s);
        return nullptr;
    }
 
    auto audioEncoder = new AudioEncoder(*this, format, m_settings);
 
    m_audioEncoders.emplace_back(audioEncoder);
    connect(audioEncoder, &EncoderThread::endOfSourceStream, this,
            &RecordingEngine::handleSourceEndOfStream);
    if (m_autoStop)
        audioEncoder->setAutoStop(true);
 
    return audioEncoder;
}
 
void RecordingEngine::addVideoSource(QPlatformVideoSource *source, const QVideoFrame &firstFrame)
{
    Q_ASSERT(m_state == State::FormatsInitializing);
 
    QVideoFrameFormat frameFormat =
            firstFrame.isValid() ? firstFrame.surfaceFormat() : source->frameFormat();
 
    Q_ASSERT(frameFormat.isValid());
 
    std::optional<AVPixelFormat> hwPixelFormat = source->ffmpegHWPixelFormat()
            ? AVPixelFormat(*source->ffmpegHWPixelFormat())
            : std::optional<AVPixelFormat>{};
 
    qCDebug(qLcFFmpegEncoder) << "adding video source" << source->metaObject()->className() << ":"
                              << "pixelFormat=" << frameFormat.pixelFormat()
                              << "frameSize=" << frameFormat.frameSize()
                              << "frameRate=" << frameFormat.streamFrameRate()
                              << "ffmpegHWPixelFormat=" << (hwPixelFormat ? *hwPixelFormat : AV_PIX_FMT_NONE);
 
    auto videoEncoder = new VideoEncoder(*this, m_settings, frameFormat, hwPixelFormat);
    m_videoEncoders.emplace_back(videoEncoder);
    if (m_autoStop)
        videoEncoder->setAutoStop(true);
 
    connect(videoEncoder, &EncoderThread::endOfSourceStream, this,
            &RecordingEngine::handleSourceEndOfStream);
 
    // set the frame before connecting to avoid potential races
    if (firstFrame.isValid())
        videoEncoder->addFrame(firstFrame);
 
    connectEncoderToSource(videoEncoder, source);
}
 
bool RecordingEngine::startEncoders()
{
    Q_ASSERT(m_state == State::FormatsInitializing);
    Q_ASSERT(m_formatsInitializer);
 
    if (m_audioEncoders.empty() && m_videoEncoders.empty()) {
        Q_EMIT sessionError(QMediaRecorder::ResourceError,
                          QLatin1StringView("No valid stream found for encoding"));
        return false;
    }
 
    m_formatsInitializer.reset();
    m_state = State::EncodersInitializing;
 
    std::vector<QFuture<bool>> resolutions;
    forEachEncoder([&resolutions](EncoderThread *encoder) { //
        encoder->start();
        resolutions.push_back(encoder->resolvedFuture());
    });
 
    QtFuture::whenAll(resolutions.begin(), resolutions.end())
            .then(this, [this](const QList<QFuture<bool>> &resolutions) {
                handleEncodersResolved(resolutions);
            });
 
    return true;
}
 
bool RecordingEngine::initialize(const std::vector<QAudioBufferSource *> &audioSources,
                                 const std::vector<QPlatformVideoSource *> &videoSources)
{
    Q_ASSERT(m_state == State::None);
 
    m_state = State::FormatsInitializing;
    m_formatsInitializer = std::make_unique<EncodingInitializer>(*this);
    return m_formatsInitializer->start(audioSources, videoSources);
}
 
RecordingEngine::EncodingFinalizer::EncodingFinalizer(RecordingEngine &recordingEngine)
    : m_recordingEngine(recordingEngine)
{
    setObjectName(QStringLiteral("EncodingFinalizer"));
    Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
    connect(this, &QThread::finished, this, &QObject::deleteLater);
}
 
void RecordingEngine::EncodingFinalizer::run()
{
    Q_ASSERT(m_recordingEngine.m_state == State::Finalizing);
 
    // Block until handleEncodersResolved() (or finalize(), if the encoders had
    // already all resolved) has finished with the encoders
    const bool headerWritten = m_recordingEngine.waitForEncodingStart();
 
    m_recordingEngine.stopAndDeleteThreads();
 
    if (headerWritten) {
        const int res = av_write_trailer(m_recordingEngine.avFormatContext());
        if (res < 0) {
            qCWarning(qLcFFmpegEncoder) << "could not write trailer" << res << AVError(res);
            Q_EMIT m_recordingEngine.sessionError(QMediaRecorder::FormatError,
                                                QLatin1String("Cannot write trailer: ")
                                                        + err2str(res));
        }
    }
    // else ffmpeg might crash
 
    // close AVIO before emitting finalizationDone.
    m_recordingEngine.m_formatContext->closeAVIO();
 
    qCDebug(qLcFFmpegEncoder) << "Media recording finalized";
    Q_EMIT m_recordingEngine.finalizationDone();
    m_recordingEngine.deleteLater(); // defer destruction to m_recordingEngine's thread
}
 
void RecordingEngine::finalize()
{
    qCDebug(qLcFFmpegEncoder) << "Media recording finalizing";
 
    Q_ASSERT(m_state == State::FormatsInitializing || m_state == State::EncodersInitializing
             || m_state == State::Encoding);
 
    Q_ASSERT((m_state == State::FormatsInitializing) == !!m_formatsInitializer);
 
    if (m_state == State::FormatsInitializing)
        markEncodingStart(false);
 
    m_formatsInitializer.reset();
 
    forEachEncoder(&disconnectEncoderFromSource);
 
    m_state = State::Finalizing;
 
    m_finalizer = new EncodingFinalizer(*this);
    m_finalizer->start();
}
 
void RecordingEngine::setPaused(bool paused)
{
    forEachEncoder(&EncoderThread::setPaused, paused);
}
 
void RecordingEngine::setAutoStop(bool autoStop)
{
    m_autoStop = autoStop;
    forEachEncoder(&EncoderThread::setAutoStop, autoStop);
    handleSourceEndOfStream();
}
 
void RecordingEngine::setMetaData(const QMediaMetaData &metaData)
{
    m_metaData = metaData;
}
 
void RecordingEngine::newTimeStamp(qint64 time)
{
    QMutexLocker locker(&m_timeMutex);
    if (time > m_timeRecorded) {
        m_timeRecorded = time;
        Q_EMIT durationChanged(time);
    }
}
 
bool RecordingEngine::isEndOfSourceStreams() const
{
    return allOfEncoders(&EncoderThread::isEndOfSourceStream);
}
 
void RecordingEngine::handleSourceEndOfStream()
{
    // Once finalizing, the EncodingFinalizer thread owns the encoders and deletes
    // them, so we must not access them. The recording is stopping anyway.
    // This can be reached via queued end-of-stream signals emitted before finalize().
    if (m_state == State::Finalizing)
        return;
 
    if (m_autoStop && isEndOfSourceStreams())
        Q_EMIT autoStopped();
}
 
void RecordingEngine::handleEncodersResolved(const QList<QFuture<bool>> &resolutions)
{
    Q_ASSERT(m_state == State::EncodersInitializing || m_state == State::Finalizing);
 
    bool allEncodersSucceeded = ranges::all_of(resolutions,
                                            [](const QFuture<bool> &future) {
                                                return future.result();
                                            });
 
    if (allEncodersSucceeded) {
        Q_ASSERT(allOfEncoders(&EncoderThread::isInitialized));
 
        qCDebug(qLcFFmpegEncoder) << "Encoders initialized; writing header";
 
        avFormatContext()->metadata = QFFmpegMetaData::toAVMetaData(m_metaData);
 
        AVDictionaryHolder options = muxerOptions(avFormatContext());
        const int res = avformat_write_header(avFormatContext(), options);
        if (res < 0) {
            qWarning() << "could not write header, error:" << res << AVError(res);
            allEncodersSucceeded = false;
            Q_EMIT sessionError(QMediaRecorder::ResourceError,
                              QStringLiteral("Cannot start writing the stream"));
        } else {
            qCDebug(qLcFFmpegEncoder) << "Stream header is successfully written";
            // finalize() may already have moved m_state to Finalizing, don't go backwards
            if (m_state == State::EncodersInitializing)
                m_state = State::Encoding;
            m_muxer->start();
        }
    }
 
    // Unblock encoders in init()
    markEncodingStart(allEncodersSucceeded);
}
 
void RecordingEngine::stopAndDeleteThreads()
{
    m_audioEncoders.clear();
    m_videoEncoders.clear();
    m_muxer.reset();
}
 
void RecordingEngine::markEncodingStart(bool startOk)
{
    m_startEncodingPromise.addResult(startOk);
    m_startEncodingPromise.finish();
}
 
template <typename F, typename... Args>
void RecordingEngine::forEachEncoder(F &&f, Args &&...args)
{
    for (auto &audioEncoder : m_audioEncoders)
        std::invoke(f, audioEncoder.get(), args...);
    for (auto &videoEncoder : m_videoEncoders)
        std::invoke(f, videoEncoder.get(), args...);
}
 
template <typename F>
bool RecordingEngine::allOfEncoders(F &&f) const
{
    auto predicate = [&f](const auto &encoder) { return std::invoke(f, encoder.get()); };
 
    return ranges::all_of(m_audioEncoders, predicate)
            && ranges::all_of(m_videoEncoders, predicate);
}
 
} // namespace QFFmpeg
 
QT_END_NAMESPACE
 
#include "moc_qffmpegrecordingengine_p.cpp"