qffmpegencoderoptions.cpp

Go to the documentation of this file.

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
#include "qffmpegencoderoptions_p.h"
 
#include <QtFFmpegMediaPluginImpl/private/qffmpegmediaformatinfo_p.h>
#include <QtFFmpegMediaPluginImpl/private/qffmpegrecordingengineutils_p.h>
 
#include <QtMultimedia/qaudioformat.h>
#include <QtCore/private/qflatmap_p.h>
 
#if QT_CONFIG(vaapi)
#include <va/va.h>
#endif
 
#ifdef Q_OS_ANDROID
extern "C" {
#include <libavcodec/avcodec.h>
}
#endif
 
#include <libavutil/channel_layout.h>
 
QT_BEGIN_NAMESPACE
 
// unfortunately there is no common way to specify options for the encoders. The code here tries to map our settings sensibly
// to options available in different encoders
 
// For constant quality options, we're trying to map things to approx those bit rates for 1080p@30fps (in Mbps):
//         VeryLow  Low     Normal      High    VeryHigh
// H264:   0.8M     1.5M    3.5M        6M      10M
// H265:   0.5M     1.0M    2.5M        4M      7M
 
[[maybe_unused]]
static int bitrateForSettings(const QMediaEncoderSettings &settings, bool hdr = false)
{
    // calculate an acceptable bitrate depending on video codec, resolution, framerate and requested quality
    // The calculations are rather heuristic here, trying to take into account how well codecs compress using
    // the tables above.
 
    // The table here is for 30FPS
    const double bitsPerPixel[int(QMediaFormat::VideoCodec::LastVideoCodec)+1][QMediaRecorder::VeryHighQuality+1] = {
        { 1.2, 2.25, 5, 9, 15 }, // MPEG1,
        { 0.8, 1.5, 3.5, 6, 10 }, // MPEG2
        { 0.4, 0.75, 1.75, 3, 5 }, // MPEG4
        { 0.4, 0.75, 1.75, 3, 5 }, // H264
        { 0.3, 0.5, 0.2, 2, 3 }, // H265
        { 0.4, 0.75, 1.75, 3, 5 }, // VP8
        { 0.3, 0.5, 0.2, 2, 3 }, // VP9
        { 0.2, 0.4, 0.9, 1.5, 2.5 }, // AV1
        { 0.4, 0.75, 1.75, 3, 5 }, // Theora
        { 0.8, 1.5, 3.5, 6, 10 }, // WMV
        { 16, 24, 32, 40, 48 }, // MotionJPEG
    };
 
    QSize s = settings.videoResolution();
    double bitrate = bitsPerPixel[int(settings.videoCodec())][settings.quality()]*s.width()*s.height();
 
    // Frame rate can be 0 with variable-rate sources
    const double frameRate = settings.videoFrameRate() > 0.
            ? settings.videoFrameRate()
            : static_cast<double>(QFFmpeg::DefaultVideoFrameRate);
 
    if (settings.videoCodec() != QMediaFormat::VideoCodec::MotionJPEG) {
        // We assume that doubling the framerate requires 1.5 times the amount of data (not twice, as intraframe
        // differences will be smaller). 4 times the frame rate uses thus 2.25 times the data, etc.
        float rateMultiplier = log2(frameRate / 30.);
        bitrate *= pow(1.5, rateMultiplier);
    } else {
        // MotionJPEG doesn't optimize between frames, so we have a linear dependency on framerate
        bitrate *= frameRate / 30.;
    }
 
    // HDR requires 10bits per pixel instead of 8, so apply a factor of 1.25.
    if (hdr)
        bitrate *= 1.25;
    return bitrate;
}
 
static void apply_openh264(const QMediaEncoderSettings &settings, AVCodecContext *codec,
                           AVDictionary **opts)
{
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding
        || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
        av_dict_set(opts, "rc_mode", "bitrate", 0);
    } else {
        av_dict_set(opts, "rc_mode", "quality", 0);
        static const int q[] = { 51, 48, 38, 25, 5 };
        codec->qmax = codec->qmin = q[settings.quality()];
    }
}
 
static void apply_x264(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
{
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
    } else {
        const char *scales[] = {
            "29", "26", "23", "21", "19"
        };
        av_dict_set(opts, "crf", scales[settings.quality()], 0);
    }
}
 
static void apply_x265(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
{
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
    } else {
        const char *scales[QMediaRecorder::VeryHighQuality+1] = {
            "40", "34", "28", "26", "24",
        };
        av_dict_set(opts, "crf", scales[settings.quality()], 0);
    }
}
 
static void apply_libvpx(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
{
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
    } else {
        const char *scales[QMediaRecorder::VeryHighQuality+1] = {
            "38", "34", "31", "28", "25",
        };
        av_dict_set(opts, "crf", scales[settings.quality()], 0);
        av_dict_set(opts, "b", nullptr, 0);
    }
    av_dict_set(opts, "row-mt", "1", 0); // better multithreading
}
 
static void apply_mpeg4(const QMediaEncoderSettings &settings, AVCodecContext *codec,
                        AVDictionary **opts)
{
    // compare https://trac.ffmpeg.org/wiki/Encode/MPEG-4
 
    QMediaRecorder::EncodingMode encodingMode = settings.encodingMode();
    switch (encodingMode) {
    case QMediaRecorder::ConstantBitRateEncoding:
    case QMediaRecorder::QMediaRecorder::AverageBitRateEncoding: {
        codec->bit_rate = settings.videoBitRate();
        if (encodingMode == QMediaRecorder::ConstantBitRateEncoding)
            codec->rc_min_rate = codec->rc_max_rate = settings.videoBitRate();
 
        break;
    }
    case QMediaRecorder::ConstantQualityEncoding: {
        constexpr auto scales = std::array{
            31, // VeryLowQuality
            23, // LowQuality
            16, // NormalQuality
            9, // HighQuality
            1, // VeryHighQuality
        };
        av_dict_set_int(opts, "qscale", scales[settings.quality()], 0);
        codec->global_quality = scales[settings.quality()] * FF_QP2LAMBDA;
        codec->flags |= AV_CODEC_FLAG_QSCALE;
        break;
    }
    case QMediaRecorder::TwoPassEncoding: {
        qWarning("Two pass encoding is not supported for MPEG4");
        break;
    }
    default: {
        Q_UNREACHABLE_RETURN();
    }
    }
}
 
#if defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)
static void apply_v4l2m2m(const QMediaEncoderSettings &settings, AVCodecContext *codec,
                           AVDictionary ** /*opts*/)
{
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding
        || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
        return;
    }
 
    // Constant quality: v4l2m2m encoders don't read global_quality/AV_CODEC_FLAG_QSCALE,
    // so approximate a fixed QP via qmin==qmax over the codec-specific V4L2 QP range.
    // Bitrate is still required by the driver's frame-level rate controller, so keep
    // supplying a heuristic ceiling alongside the QP pin.
    codec->bit_rate = bitrateForSettings(settings);
 
    const int quality = settings.quality();
    switch (settings.videoCodec()) {
    case QMediaFormat::VideoCodec::H264: {
        static constexpr int q[] = { 45, 38, 28, 20, 10 }; // 0..51
        codec->qmin = codec->qmax = q[quality];
        break;
    }
    case QMediaFormat::VideoCodec::MPEG4: {
        static constexpr int q[] = { 28, 22, 16, 10, 4 }; // 1..31
        codec->qmin = codec->qmax = q[quality];
        break;
    }
    case QMediaFormat::VideoCodec::VP8: {
        static constexpr int q[] = { 108, 88, 64, 40, 16 }; // 0..127
        codec->qmin = codec->qmax = q[quality];
        break;
    }
    case QMediaFormat::VideoCodec::VP9: {
        static constexpr int q[] = { 216, 176, 128, 80, 32 }; // 0..255
        codec->qmin = codec->qmax = q[quality];
        break;
    }
    default:
        // e.g. HEVC/H263: driver exposes no QP control, bit_rate heuristic above is all we can do
        break;
    }
}
#endif
 
#ifdef Q_OS_DARWIN
static void apply_videotoolbox(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
{
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
    } else {
        // only use quality on macOS/ARM, as FFmpeg doesn't support it on the other platforms and would throw
        // an error when initializing the codec
#if defined(Q_OS_MACOS) && defined(Q_PROCESSOR_ARM_64)
        // Videotoolbox describes quality as a number from 0 to 1, with low == 0.25, normal 0.5, high 0.75 and lossless = 1
        // ffmpeg uses a different scale going from 0 to 11800.
        // Values here are adjusted to agree approximately with the target bit rates listed above
        const int scales[] = {
            3000, 4800, 5900, 6900, 7700,
        };
        codec->global_quality = scales[settings.quality()];
        codec->flags |= AV_CODEC_FLAG_QSCALE;
#else
        codec->bit_rate = bitrateForSettings(settings);
#endif
    }
 
    // Videotooldox hw acceleration fails of some hardwares,
    // allow_sw makes sw encoding available if hw encoding failed.
    // Under the hood, ffmpeg sets
    // kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder instead of
    // kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder
    av_dict_set(opts, "allow_sw", "1", 0);
}
#endif
 
#if QT_CONFIG(vaapi)
static void apply_vaapi(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **/*opts*/)
{
    // See also vaapi_encode_init_rate_control() in libavcodec
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
        codec->rc_max_rate = settings.videoBitRate();
    } else if (settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
    } else {
        const int *quality = nullptr;
        // unfortunately, all VA codecs use different quality scales :/
        switch (settings.videoCodec()) {
        case QMediaFormat::VideoCodec::MPEG2: {
            static const int q[] = { 20, 15, 10, 8, 6 };
            quality = q;
            break;
        }
        case QMediaFormat::VideoCodec::MPEG4:
        case QMediaFormat::VideoCodec::H264: {
            static const int q[] = { 29, 26, 23, 21, 19 };
            quality = q;
            break;
        }
        case QMediaFormat::VideoCodec::H265: {
            static const int q[] = { 40, 34, 28, 26, 24 };
            quality = q;
            break;
        }
        case QMediaFormat::VideoCodec::VP8: {
            static const int q[] = { 56, 48, 40, 34, 28 };
            quality = q;
            break;
        }
        case QMediaFormat::VideoCodec::VP9: {
            static const int q[] = { 124, 112, 100, 88, 76 };
            quality = q;
            break;
        }
        case QMediaFormat::VideoCodec::MotionJPEG: {
            static const int q[] = { 40, 60, 80, 90, 95 };
            quality = q;
            break;
        }
        case QMediaFormat::VideoCodec::AV1:
        case QMediaFormat::VideoCodec::Theora:
        case QMediaFormat::VideoCodec::WMV:
        default:
            break;
        }
 
        if (quality)
            codec->global_quality = quality[settings.quality()];
    }
}
#endif
 
static void apply_nvenc(const QMediaEncoderSettings &settings, AVCodecContext *codec,
                        AVDictionary **opts)
{
    switch (settings.encodingMode()) {
    case QMediaRecorder::EncodingMode::AverageBitRateEncoding:
        av_dict_set(opts, "vbr", "1", 0);
        codec->bit_rate = settings.videoBitRate();
        break;
    case QMediaRecorder::EncodingMode::ConstantBitRateEncoding:
        av_dict_set(opts, "cbr", "1", 0);
        codec->bit_rate = settings.videoBitRate();
        codec->rc_max_rate = codec->rc_min_rate = codec->bit_rate;
        break;
    case QMediaRecorder::EncodingMode::ConstantQualityEncoding: {
        static const char *q[] = { "51", "48", "35", "15", "1" };
        av_dict_set(opts, "cq", q[settings.quality()], 0);
    } break;
    default:
        break;
    }
}
 
#ifdef Q_OS_WINDOWS
static void apply_mf(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts)
{
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) {
        codec->bit_rate = settings.videoBitRate();
        av_dict_set(opts, "rate_control", "cbr", 0);
    } else {
        av_dict_set(opts, "rate_control", "quality", 0);
        const char *scales[] = {
            "25", "50", "75", "90", "100"
        };
        av_dict_set(opts, "quality", scales[settings.quality()], 0);
    }
}
#endif
 
#ifdef Q_OS_ANDROID
static void apply_mediacodec(const QMediaEncoderSettings &settings, AVCodecContext *codec,
                             AVDictionary **opts)
{
    if (settings.videoBitRate() != -1)
        codec->bit_rate = settings.videoBitRate();
    else
        codec->bit_rate = bitrateForSettings(settings);
 
    const int quality[] = { 25, 50, 75, 90, 100 };
    codec->global_quality = quality[settings.quality()];
 
    switch (settings.encodingMode()) {
    case QMediaRecorder::EncodingMode::AverageBitRateEncoding:
        av_dict_set(opts, "bitrate_mode", "vbr", 1);
        break;
    case QMediaRecorder::EncodingMode::ConstantBitRateEncoding:
        av_dict_set(opts, "bitrate_mode", "cbr", 1);
        break;
    case QMediaRecorder::EncodingMode::ConstantQualityEncoding:
        //        av_dict_set(opts, "bitrate_mode", "cq", 1);
        av_dict_set(opts, "bitrate_mode", "cbr", 1);
        break;
    default:
        break;
    }
 
    switch (settings.videoCodec()) {
    case QMediaFormat::VideoCodec::H264: {
        const char *levels[] = { "2.2", "3.2", "4.2", "5.2", "6.2" };
        av_dict_set(opts, "level", levels[settings.quality()], 1);
        codec->profile = AV_PROFILE_H264_HIGH;
        break;
    }
    case QMediaFormat::VideoCodec::H265: {
        // Set the level only for FFmpeg versions that correctly recognize level values.
        // Affected revisions: from n7.1 https://github.com/FFmpeg/FFmpeg/commit/7753a9d62725d5bd8313e2d249acbe1c8af79ab1
        // up to https://github.com/FFmpeg/FFmpeg/commit/020d9f2b4886aa620252da4db7a4936378d6eb3a
        if (avcodec_version() < 4000612 || avcodec_version() > 4002660) {
            const char *levels[] = { "h2.1", "h3.1", "h4.1", "h5.1", "h6.1" };
            av_dict_set(opts, "level", levels[settings.quality()], 1);
        }
 
        codec->profile = AV_PROFILE_HEVC_MAIN;
        break;
    }
    default:
        break;
    }
}
#endif
 
namespace QFFmpeg {
 
using ApplyVideoCodecOptions = void (*)(const QMediaEncoderSettings &settings,
                                        AVCodecContext *codec, AVDictionary **opts);
 
using namespace Qt::StringLiterals;
 
using VideoCodecOptionsTableType =
        QFlatMap<QLatin1StringView, ApplyVideoCodecOptions, std::less<>,
                 std::vector<QLatin1StringView>, std::vector<ApplyVideoCodecOptions>>;
 
const VideoCodecOptionsTableType videoCodecOptionTable{
    { "libx264"_L1, apply_x264 },
    { "libx265"_L1, apply_x265 },
    { "libvpx"_L1, apply_libvpx },
    { "libvpx_vp9"_L1, apply_libvpx },
    { "libopenh264"_L1, apply_openh264 },
    { "h264_nvenc"_L1, apply_nvenc },
    { "hevc_nvenc"_L1, apply_nvenc },
    { "av1_nvenc"_L1, apply_nvenc },
    { "mpeg4"_L1, apply_mpeg4 },
#if defined(Q_OS_LINUX) || defined(Q_OS_ANDROID)
    { "h264_v4l2m2m"_L1, apply_v4l2m2m },
    { "hevc_v4l2m2m"_L1, apply_v4l2m2m },
    { "mpeg4_v4l2m2m"_L1, apply_v4l2m2m },
    { "vp8_v4l2m2m"_L1, apply_v4l2m2m },
#endif
#ifdef Q_OS_DARWIN
    { "h264_videotoolbox"_L1, apply_videotoolbox },
    { "hevc_videotoolbox"_L1, apply_videotoolbox },
    { "prores_videotoolbox"_L1, apply_videotoolbox },
    { "vp9_videotoolbox"_L1, apply_videotoolbox },
#endif
#if QT_CONFIG(vaapi)
    { "mpeg2_vaapi"_L1, apply_vaapi },
    { "mjpeg_vaapi"_L1, apply_vaapi },
    { "h264_vaapi"_L1, apply_vaapi },
    { "hevc_vaapi"_L1, apply_vaapi },
    { "vp8_vaapi"_L1, apply_vaapi },
    { "vp9_vaapi"_L1, apply_vaapi },
#endif
#ifdef Q_OS_WINDOWS
    { "hevc_mf"_L1, apply_mf },
    { "h264_mf"_L1, apply_mf },
#endif
#ifdef Q_OS_ANDROID
    { "hevc_mediacodec"_L1, apply_mediacodec },
    { "h264_mediacodec"_L1, apply_mediacodec },
#endif
 
};
 
void applyVideoEncoderOptions(const QMediaEncoderSettings &settings, QLatin1StringView codecName,
                              AVCodecContext *codec, AVDictionary **opts)
{
    av_dict_set(opts, "threads", "auto", 0); // we always want automatic threading
 
    auto entry = videoCodecOptionTable.find(codecName);
 
    if (entry != videoCodecOptionTable.end())
        entry->second(settings, codec, opts);
}
 
void applyAudioEncoderOptions(const QMediaEncoderSettings &settings,
                              QLatin1StringView /*codecName*/, AVCodecContext *codec,
                              AVDictionary ** /*opts*/)
{
    codec->thread_count = -1; // we always want automatic threading
    if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding
        || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding)
        codec->bit_rate = settings.audioBitRate();
 
    if (settings.audioSampleRate() != -1)
        codec->sample_rate = settings.audioSampleRate();
 
    if (settings.audioChannelCount() != -1) {
        auto mask = QFFmpegMediaFormatInfo::avChannelLayout(
                QAudioFormat::defaultChannelConfigForChannelCount(settings.audioChannelCount()));
 
#if QT_FFMPEG_HAS_AV_CHANNEL_LAYOUT
        av_channel_layout_from_mask(&codec->ch_layout, mask);
#else
        codec->channel_layout = mask;
        codec->channels = qPopulationCount(codec->channel_layout);
#endif
    }
}
 
} // namespace QFFmpeg
 
QT_END_NAMESPACE