qffmpeghwaccel.cpp

Go to the documentation of this file.

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qffmpeghwaccel_p.h"
 
#ifdef Q_OS_WINDOWS
#  include "qffmpeghwaccel_d3d11_p.h"
#  include <QtCore/private/qsystemlibrary_p.h>
#endif
 
#include "qffmpeg_p.h"
#include "qffmpegcodecstorage_p.h"
#include "qffmpegmediaintegration_p.h"
#include "qffmpegvideobuffer_p.h"
#include "qffmpeg_ranges_p.h"
 
#include <QtCore/qscopedvaluerollback.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/qelapsedtimer.h>
#include <QtMultimedia/private/qmultimedia_ranges_p.h>
 
#ifdef Q_OS_LINUX
#  include <QtCore/qfile.h>
#  include <QtCore/qlibrary.h>
#  include <QtCore/qdir.h>
#  include <QtCore/qjsondocument.h>
#  include <QtCore/qjsonobject.h>
#endif
 
#include <algorithm>
#include <unordered_set>
 
/* Infrastructure for HW acceleration goes into this file. */
 
QT_BEGIN_NAMESPACE
 
using namespace Qt::StringLiterals;
 
Q_STATIC_LOGGING_CATEGORY(qLHWAccel, "qt.multimedia.ffmpeg.hwaccel");
 
namespace QFFmpeg {
 
static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators = {
#if defined(Q_OS_ANDROID)
    AV_HWDEVICE_TYPE_MEDIACODEC,
#elif defined(Q_OS_LINUX)
    AV_HWDEVICE_TYPE_CUDA,
    AV_HWDEVICE_TYPE_VAAPI,
 
    // TODO: investigate VDPAU advantages.
    // nvenc/nvdec codecs use AV_HWDEVICE_TYPE_CUDA by default, but they can also use VDPAU
    // if it's included into the ffmpeg build and vdpau drivers are installed.
    // AV_HWDEVICE_TYPE_VDPAU
#elif defined (Q_OS_WIN)
    AV_HWDEVICE_TYPE_D3D11VA,
#elif defined (Q_OS_DARWIN)
    AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
#endif
};
 
static AVBufferUPtr loadHWContext(AVHWDeviceType type)
{
    AVBufferRef *hwContext = nullptr;
    qCDebug(qLHWAccel) << "    Checking HW context:" << type;
    int ret = av_hwdevice_ctx_create(&hwContext, type, nullptr, nullptr, 0);
 
    if (ret == 0) {
        qCDebug(qLHWAccel) << "    Using above hw context.";
        return AVBufferUPtr(hwContext);
    }
    qCDebug(qLHWAccel) << "    Could not create hw context:" << QFFmpeg::AVError(ret);
    return nullptr;
}
 
#if defined(Q_OS_LINUX)
// Mesa's llvmpipe and Google's SwiftShader are pure-software Vulkan implementations: using them
// for "hw" encoding/decoding buys nothing over FFmpeg's native CPU codecs.
static bool isSoftwareVulkanIcdManifest(const QString &manifestPath)
{
    QFile file(manifestPath);
    if (!file.open(QIODevice::ReadOnly))
        return false;
 
    const QJsonObject icd = QJsonDocument::fromJson(file.readAll())[u"ICD"_s].toObject();
    const QString libraryPath = icd[u"library_path"_s].toString();
 
    return libraryPath.contains(u"lvp"_s, Qt::CaseInsensitive)
            || libraryPath.contains(u"swiftshader"_s, Qt::CaseInsensitive);
}
 
static bool hasSoftwareVulkanIcdRegistered()
{
    namespace views = QtMultimediaPrivate::views;
    namespace ranges = QtMultimediaPrivate::ranges;
 
    const std::vector<QString> manifestPaths = [] {
        // VK_DRIVER_FILES is the current loader env var name, VK_ICD_FILENAMES the legacy one;
        // either, if set, replaces the default search path entirely.
        QByteArray overrideEnv = qgetenv("VK_DRIVER_FILES");
        if (overrideEnv.isEmpty())
            overrideEnv = qgetenv("VK_ICD_FILENAMES");
        if (!overrideEnv.isEmpty())
            return overrideEnv.split(':') | views::filter([](const QByteArray &path) {
                return !path.isEmpty();
            }) | views::transform([](const QByteArray &path) {
                return QString::fromUtf8(path);
            }) | ranges::to<std::vector<QString>>();
 
        const std::initializer_list<QDir> searchDirs = {
            QDir(QDir::homePath() + u"/.local/share/vulkan/icd.d"_s),
            u"/usr/local/share/vulkan/icd.d"_s,
            u"/usr/share/vulkan/icd.d"_s,
            u"/etc/vulkan/icd.d"_s,
        };
 
        std::vector<QString> paths;
        for (const QDir &dir : searchDirs) {
            const QFileInfoList entries = dir.entryInfoList(QStringList(u"*.json"_s), QDir::Files);
            auto absolutePaths = entries | views::transform([](const QFileInfo &entry) {
                return entry.absoluteFilePath();
            });
            paths.insert(paths.end(), absolutePaths.begin(), absolutePaths.end());
        }
        return paths;
    }();
 
    return ranges::any_of(manifestPaths, isSoftwareVulkanIcdManifest);
}
#endif // Q_OS_LINUX
 
// FFmpeg might crash on loading non-existing hw devices.
// Let's roughly precheck drivers/libraries.
static bool precheckDriver(AVHWDeviceType type)
{
    // precheckings might need some improvements
#if defined(Q_OS_LINUX)
    if (type == AV_HWDEVICE_TYPE_CUDA) {
        if (!QFile::exists(QLatin1String("/proc/driver/nvidia/version")))
            return false;
 
        // QTBUG-122199
        // CUDA backend requires libnvcuvid in libavcodec
        QLibrary lib(u"libnvcuvid.so"_s);
        if (!lib.load())
            return false;
        lib.unload();
        return true;
    }
 
    if (type == AV_HWDEVICE_TYPE_VULKAN && hasSoftwareVulkanIcdRegistered()) {
        qCDebug(qLHWAccel) << "A software Vulkan ICD (llvmpipe/SwiftShader) is registered on this "
                              "system; skipping Vulkan hwaccel, as it offers no real acceleration "
                              "benefit over native CPU codecs";
        return false;
    }
#elif defined(Q_OS_WINDOWS)
    if (type == AV_HWDEVICE_TYPE_D3D11VA)
        return QSystemLibrary(QLatin1String("d3d11.dll")).load();
 
#if QT_FFMPEG_HAS_D3D12VA
    if (type == AV_HWDEVICE_TYPE_D3D12VA)
        return QSystemLibrary(QLatin1String("d3d12.dll")).load();
#endif
 
    if (type == AV_HWDEVICE_TYPE_DXVA2)
        return QSystemLibrary(QLatin1String("d3d9.dll")).load();
 
    // TODO: check nvenc/nvdec and revisit the checking
    if (type == AV_HWDEVICE_TYPE_CUDA)
        return QSystemLibrary(QLatin1String("nvml.dll")).load();
#else
     Q_UNUSED(type);
#endif
 
    return true;
}
 
static bool checkHwType(AVHWDeviceType type)
{
    if (!precheckDriver(type)) {
        qCDebug(qLHWAccel) << "Drivers for hw device" << type << "is not installed";
        return false;
    }
 
#if QT_FFMPEG_HAS_D3D12VA
    if (type == AV_HWDEVICE_TYPE_D3D12VA)
        return false; // QTBUG-146635: opening d3d12va codecs fails
#endif
 
    if (type == AV_HWDEVICE_TYPE_MEDIACODEC ||
        type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX ||
        type == AV_HWDEVICE_TYPE_D3D11VA ||
#if QT_FFMPEG_HAS_D3D12VA
        type == AV_HWDEVICE_TYPE_D3D12VA ||
#endif
        type == AV_HWDEVICE_TYPE_DXVA2)
        return true; // Don't waste time; it's expected to work fine if the precheck is OK
 
    QScopedValueRollback rollback(FFmpegLogsEnabledInThread);
    FFmpegLogsEnabledInThread = false;
 
    return loadHWContext(type) != nullptr;
}
 
using HWDeviceTypesRange = FFmpegValueIteratorRange<AVHWDeviceType, av_hwdevice_iterate_types>;
 
static const std::vector<AVHWDeviceType> &deviceTypes()
{
    static const auto types = []() {
        qCDebug(qLHWAccel) << "Check device types";
        QElapsedTimer timer;
        timer.start();
 
        // gather hw pix formats
        std::unordered_set<AVPixelFormat> hwPixFormats;
        for (const Codec codec : allCodecs) {
            forEachAVPixelFormat(codec, [&](AVPixelFormat format) {
                if (isHwPixelFormat(format))
                    hwPixFormats.insert(format);
            });
        }
 
        // create a device types list
        std::vector<AVHWDeviceType> result;
        for (const AVHWDeviceType type : HWDeviceTypesRange())
            if (hwPixFormats.count(pixelFormatForHwDevice(type)) && checkHwType(type))
                result.push_back(type);
        result.shrink_to_fit();
 
        // reorder the list accordingly preferredHardwareAccelerators
        auto it = result.begin();
        for (const auto preffered : preferredHardwareAccelerators) {
            auto found = std::find(it, result.end(), preffered);
            if (found != result.end())
                std::rotate(it++, found, std::next(found));
        }
 
        using namespace std::chrono;
        qCDebug(qLHWAccel) << "Device types checked. Spent time:" << duration_cast<microseconds>(timer.durationElapsed());
 
        return result;
    }();
 
    return types;
}
 
static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName)
{
    const auto definedDeviceTypes = qgetenv(envVarName);
 
    if (definedDeviceTypes.isNull())
        return deviceTypes();
 
    std::vector<AVHWDeviceType> result;
    const auto definedDeviceTypesString = QString::fromUtf8(definedDeviceTypes).toLower();
    for (const auto &deviceType : definedDeviceTypesString.split(u',')) {
        if (!deviceType.isEmpty()) {
            const auto foundType = av_hwdevice_find_type_by_name(deviceType.toUtf8().data());
            if (foundType == AV_HWDEVICE_TYPE_NONE)
                qWarning() << "Unknown hw device type" << deviceType;
            else
                result.emplace_back(foundType);
        }
    }
 
    result.shrink_to_fit();
    return result;
}
 
std::pair<std::optional<Codec>, HWAccelUPtr> HWAccel::findDecoderWithHwAccel(AVCodecID id)
{
    for (auto type : decodingDeviceTypes()) {
        const std::optional<Codec> codec = findAVDecoder(id, pixelFormatForHwDevice(type));
 
        if (!codec)
            continue;
 
        qCDebug(qLHWAccel) << "Found potential codec" << codec->name() << "for hw accel" << type
                           << "; Checking the hw device...";
 
        HWAccelUPtr hwAccel = create(type);
 
        if (!hwAccel)
            continue;
 
        qCDebug(qLHWAccel) << "HW device is OK";
 
        return { codec, std::move(hwAccel) };
    }
 
    qCDebug(qLHWAccel) << "No hw acceleration found for codec id" << id;
 
    return { std::nullopt, nullptr };
}
 
static bool isNoConversionFormat(AVPixelFormat f)
{
    bool needsConversion = true;
    QFFmpegVideoBuffer::toQtPixelFormat(f, &needsConversion);
    return !needsConversion;
};
 
// Used for the AVCodecContext::get_format callback
AVPixelFormat getFormat(AVCodecContext *codecContext, const AVPixelFormat *fmt)
{
    QSpan<const AVPixelFormat> suggestedFormats = makeSpan(fmt);
    // First check HW accelerated codecs, the HW device context must be set
    if (codecContext->hw_device_ctx) {
        auto *device_ctx = (AVHWDeviceContext *)codecContext->hw_device_ctx->data;
        std::optional<ValueAndScore<AVPixelFormat>> formatAndScore;
 
        // to be rewritten via findBestAVFormat
        const Codec codec{ codecContext->codec };
        for (const AVCodecHWConfig *config : codec.hwConfigs()) {
            if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX))
                continue;
 
            if (device_ctx->type != config->device_type)
                continue;
 
            const bool isDeprecated = (config->methods & AV_CODEC_HW_CONFIG_METHOD_AD_HOC) != 0;
            const bool shouldCheckCodecFormats = config->pix_fmt == AV_PIX_FMT_NONE;
 
            auto scoresGettor = [&](AVPixelFormat format) {
                // check in supported codec->pix_fmts (avcodec_get_supported_config with
                // AV_CODEC_CONFIG_PIX_FORMAT since n7.1); no reason to use findAVPixelFormat as
                // we're already in the hw_config loop
                const auto pixelFormats = codec.pixelFormats();
                if (shouldCheckCodecFormats && !ranges::contains(pixelFormats, format))
                    return NotSuitableAVScore;
 
                if (!shouldCheckCodecFormats && config->pix_fmt != format)
                    return NotSuitableAVScore;
 
                auto result = DefaultAVScore;
 
                if (isDeprecated)
                    result -= 10000;
                if (isHwPixelFormat(format))
                    result += 10;
 
                return result;
            };
 
            const auto found = findBestAVValueWithScore(suggestedFormats, scoresGettor);
            if (found) {
                if (!formatAndScore || found->score > formatAndScore->score)
                    formatAndScore = found;
            }
        }
 
        if (formatAndScore) {
            AVPixelFormat format = formatAndScore->value;
            TextureConverter::applyDecoderPreset(format, *codecContext);
            qCDebug(qLHWAccel) << "Selected format" << format << "for hw" << device_ctx->type;
            return format;
        }
    }
 
    // prefer video formats we can handle directly
    const auto noConversionFormat = findIf(suggestedFormats, &isNoConversionFormat);
    if (noConversionFormat) {
        qCDebug(qLHWAccel) << "Selected format with no conversion" << *noConversionFormat;
        return *noConversionFormat;
    }
 
    const AVPixelFormat format = !suggestedFormats.empty() ? suggestedFormats[0] : AV_PIX_FMT_NONE;
    qCDebug(qLHWAccel) << "Selected format with conversion" << format;
 
    // take the native format, this will involve one additional format conversion on the CPU side
    return format;
}
 
HWAccel::~HWAccel() = default;
 
HWAccelUPtr HWAccel::create(AVHWDeviceType deviceType)
{
    if (auto ctx = loadHWContext(deviceType))
        return HWAccelUPtr(new HWAccel(std::move(ctx)));
    else
        return {};
}
 
AVPixelFormat HWAccel::format(AVFrame *frame)
{
    if (!frame->hw_frames_ctx)
        return AVPixelFormat(frame->format);
 
    auto *hwFramesContext = (AVHWFramesContext *)frame->hw_frames_ctx->data;
    Q_ASSERT(hwFramesContext);
    return AVPixelFormat(hwFramesContext->sw_format);
}
 
QSpan<const AVHWDeviceType> HWAccel::encodingDeviceTypes()
{
    static const auto &result = deviceTypes("QT_FFMPEG_ENCODING_HW_DEVICE_TYPES");
    return result;
}
 
QSpan<const AVHWDeviceType> HWAccel::decodingDeviceTypes()
{
    static const auto &result = deviceTypes("QT_FFMPEG_DECODING_HW_DEVICE_TYPES");
    return result;
}
 
AVHWDeviceContext *HWAccel::hwDeviceContext() const
{
    return m_hwDeviceContext ? (AVHWDeviceContext *)m_hwDeviceContext->data : nullptr;
}
 
AVPixelFormat HWAccel::hwFormat() const
{
    return pixelFormatForHwDevice(deviceType());
}
 
const AVHWFramesConstraints *HWAccel::constraints() const
{
    std::call_once(m_constraintsOnceFlag, [this]() {
        if (auto context = hwDeviceContextAsBuffer())
            m_constraints.reset(av_hwdevice_get_hwframe_constraints(context, nullptr));
    });
 
    return m_constraints.get();
}
 
bool HWAccel::matchesSizeContraints(QSize size) const
{
    const auto constraints = this->constraints();
    if (!constraints)
        return true;
 
    return size.width() >= constraints->min_width
            && size.height() >= constraints->min_height
            && size.width() <= constraints->max_width
            && size.height() <= constraints->max_height;
}
 
AVHWDeviceType HWAccel::deviceType() const
{
    return m_hwDeviceContext ? hwDeviceContext()->type : AV_HWDEVICE_TYPE_NONE;
}
 
void HWAccel::createFramesContext(AVPixelFormat swFormat, const QSize &size)
{
    if (m_hwFramesContext) {
        qWarning() << "Frames context has been already created!";
        return;
    }
 
    if (!m_hwDeviceContext)
        return;
 
    m_hwFramesContext.reset(av_hwframe_ctx_alloc(m_hwDeviceContext.get()));
    auto *c = (AVHWFramesContext *)m_hwFramesContext->data;
    c->format = hwFormat();
    c->sw_format = swFormat;
    c->width = size.width();
    c->height = size.height();
    qCDebug(qLHWAccel) << "init frames context";
    int err = av_hwframe_ctx_init(m_hwFramesContext.get());
    if (err < 0)
        qWarning() << "failed to init HW frame context" << err << AVError(err);
    else
        qCDebug(qLHWAccel) << "Initialized frames context" << size << c->format << c->sw_format;
}
 
void HWAccel::destroyFramesContext()
{
    m_hwFramesContext = nullptr;
}
 
void HWAccel::updateFramesContext(AVPixelFormat swFormat, const QSize &size)
{
    if (m_hwFramesContext) {
        auto *c = (AVHWFramesContext *)m_hwFramesContext->data;
        if (c->sw_format != swFormat || QSize(c->width, c->height) != size)
            m_hwFramesContext.reset();
        else
            return;
    }
 
    createFramesContext(swFormat, size);
}
 
AVHWFramesContext *HWAccel::hwFramesContext() const
{
    return m_hwFramesContext ? (AVHWFramesContext *)m_hwFramesContext->data : nullptr;
}
 
static void deleteHwFrameContextData(AVHWFramesContext *context)
{
    std::unique_ptr<HwFrameContextData> contextData(
            static_cast<HwFrameContextData *>(context->user_opaque));
    Q_ASSERT(contextData);
 
    if (contextData->avDeleter) {
        context->user_opaque = contextData->avUserOpaque;
        context->free = contextData->avDeleter;
 
        context->free(context);
    }
}
 
HwFrameContextData &HwFrameContextData::ensure(AVFrame &hwFrame)
{
    Q_ASSERT(hwFrame.hw_frames_ctx && hwFrame.hw_frames_ctx->data);
 
    auto context = reinterpret_cast<AVHWFramesContext *>(hwFrame.hw_frames_ctx->data);
 
    if (context->free != deleteHwFrameContextData) {
        // In most cases, we expect null context->free and context->user_opaque.
        // However, FFmpeg decoding implementations for specific graphic card backends
        // may set a custom deleter. Let's save the deleter and its data (user_opaque)
        // to invoke it in deleteHwFrameContextData.
        context->user_opaque = new HwFrameContextData{ context->free, context->user_opaque };
        context->free = deleteHwFrameContextData;
    } else {
        Q_ASSERT(context->user_opaque);
    }
 
    return *static_cast<HwFrameContextData *>(context->user_opaque);
}
 
AVFrameUPtr copyFromHwPool(AVFrameUPtr frame)
{
#ifdef Q_OS_WINDOWS
    return copyFromHwPoolD3D11(std::move(frame));
#else
    return frame;
#endif
}
 
} // namespace QFFmpeg
 
QT_END_NAMESPACE