qffmpeghwaccel_d3d11.cpp

Go to the documentation of this file.

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qffmpeghwaccel_d3d11_p.h"
 
#include "qffmpegvideobuffer_p.h"
 
#include <QtMultimedia/qvideoframeformat.h>
#include <QtMultimedia/private/qvideotexturehelper_p.h>
 
#include <QtGui/rhi/qrhi.h>
#include <QtCore/qloggingcategory.h>
#include <QtCore/private/qcomptr_p.h>
 
#include <libavutil/hwcontext_d3d11va.h>
#include <d3d11_1.h>
 
#include <mutex>
 
namespace std {
 
template <>
class lock_guard<AVD3D11VADeviceContext>
{
public:
    explicit lock_guard(const AVD3D11VADeviceContext &ctx) : m_ctx(ctx)
    {
        m_ctx.lock(m_ctx.lock_ctx);
    }
 
    ~lock_guard() { m_ctx.unlock(m_ctx.lock_ctx); }
 
    lock_guard(const lock_guard &) = delete;
    lock_guard &operator=(const lock_guard &) = delete;
 
private:
    const AVD3D11VADeviceContext &m_ctx;
};
 
} // namespace std
 
QT_BEGIN_NAMESPACE
 
namespace QFFmpeg {
 
namespace {
 
Q_LOGGING_CATEGORY(qLcMediaFFmpegHWAccel, "qt.multimedia.hwaccel");
 
ComPtr<ID3D11Texture2D> getAvFrameTexture(const AVFrame *frame)
{
    return reinterpret_cast<ID3D11Texture2D *>(frame->data[0]);
}
 
int getAvFramePoolIndex(const AVFrame *frame)
{
    return static_cast<int>(reinterpret_cast<intptr_t>(frame->data[1]));
}
 
const AVD3D11VADeviceContext *getHwDeviceContext(const AVHWDeviceContext *ctx)
{
    return static_cast<AVD3D11VADeviceContext *>(ctx->hwctx);
}
 
AVBufferRef *wrapTextureAsBuffer(const ComPtr<ID3D11Texture2D> &tex)
{
    Q_ASSERT(tex);
 
    AVD3D11FrameDescriptor *avFrameDesc =
            static_cast<AVD3D11FrameDescriptor *>(av_mallocz(sizeof(AVD3D11FrameDescriptor)));
    avFrameDesc->index = 0;
    avFrameDesc->texture = tex.Get();
 
    return av_buffer_create(reinterpret_cast<uint8_t *>(avFrameDesc),
                            sizeof(AVD3D11FrameDescriptor *), [](void *opaque, uint8_t *data) {
        static_cast<ID3D11Texture2D *>(opaque)->Release();
        av_free(data);
    }, tex.Get(), 0);
}
 
ComPtr<ID3D11Texture2D> copyTexture(const AVD3D11VADeviceContext *hwDevCtx, const AVFrame *src)
{
    const int poolIndex = getAvFramePoolIndex(src);
    const ComPtr<ID3D11Texture2D> poolTex = getAvFrameTexture(src);
 
    D3D11_TEXTURE2D_DESC texDesc{};
    poolTex->GetDesc(&texDesc);
 
    texDesc.ArraySize = 1;
    texDesc.MiscFlags = 0;
    texDesc.BindFlags = 0;
 
    ComPtr<ID3D11Texture2D> destTex;
    if (hwDevCtx->device->CreateTexture2D(&texDesc, nullptr, &destTex) != S_OK) {
        qCCritical(qLcMediaFFmpegHWAccel) << "Unable to copy frame from decoder pool";
        return {};
    }
 
    hwDevCtx->device_context->CopySubresourceRegion(destTex.Get(), 0, 0, 0, 0, poolTex.Get(),
                                                    poolIndex, nullptr);
 
    return destTex;
}
 
class ImportedTextureHandles : public QVideoFrameTexturesHandles
{
public:
    ImportedTextureHandles(TextureConverterBackendPtr &&converterBackend,
                           QVideoFrameTexturesHandlesUPtr &&handles)
        : m_parentConverterBackend(std::move(converterBackend)), m_handles(std::move(handles))
    {
    }
 
    quint64 textureHandle(QRhi &rhi, int plane) override
    {
        return m_handles ? m_handles->textureHandle(rhi, plane) : 0;
    }
 
private:
    TextureConverterBackendPtr
            m_parentConverterBackend; // ensures the backend is deleted after the texture
    QVideoFrameTexturesHandlesUPtr m_handles;
};
 
} // namespace
 
std::shared_ptr<D3D11TextureConverter> D3D11TextureConverter::create(QRhi *rhi)
{
    if (!rhi)
        return nullptr;
 
    if (!QWindowsD3D11RhiTextureImporter::isRhiBackendSupported(*rhi))
        return nullptr;
 
    return std::shared_ptr<D3D11TextureConverter>(new D3D11TextureConverter(rhi));
}
 
D3D11TextureConverter::D3D11TextureConverter(QRhi *rhi) : TextureConverterBackend(rhi) { }
 
bool D3D11TextureConverter::isSupported(AVFrame &hwFrame, QRhi &rhi)
{
    if (!QWindowsD3D11RhiTextureImporter::isRhiBackendSupported(rhi))
        return false;
 
    const QVideoFrameFormat::PixelFormat pixelFormat =
            QFFmpegVideoBuffer::toQtPixelFormat(HWAccel::format(&hwFrame));
 
    if (QWindowsD3D11RhiTextureImporter::isPixelFormatSupportedByRhiBackend(rhi, pixelFormat))
        return true;
 
    switch (rhi.backend()) {
    case QRhi::D3D12:
#if QT_CONFIG(opengl)
    case QRhi::OpenGLES2:
#endif
        return QWindowsD3D11VideoProcessor::isPixelFormatSupported(pixelFormat);
    default:
        return false;
    }
}
 
QVideoFrameTexturesHandlesUPtr
D3D11TextureConverter::createTextureHandles(AVFrame *frame)
{
    if (!frame || !frame->hw_frames_ctx || frame->format != AV_PIX_FMT_D3D11)
        return nullptr;
 
    if (!isSupported(*frame, *rhi))
        return nullptr;
 
    const auto *ctx = avFrameDeviceContext(frame);
 
    if (!ctx || ctx->type != AV_HWDEVICE_TYPE_D3D11VA)
        return nullptr;
 
    const auto *avDeviceCtx = getHwDeviceContext(ctx);
    if (!avDeviceCtx)
        return nullptr;
 
    Q_ASSERT(QWindowsD3D11RhiTextureImporter::isRhiBackendSupported(*rhi));
 
    const ComPtr<ID3D11Texture2D> ffmpegTex = getAvFrameTexture(frame);
    const int index = getAvFramePoolIndex(frame);
    const QVideoFrameFormat::PixelFormat pixelFormat =
            QFFmpegVideoBuffer::toQtPixelFormat(HWAccel::format(frame));
 
    bool canImportDirectly =
            QWindowsD3D11RhiTextureImporter::isPixelFormatSupportedByRhiBackend(*rhi, pixelFormat);
 
    const QSize size = frameSize(frame);
    QVideoFrameTexturesHandlesUPtr handles;
    {
        // Lock the FFmpeg device context while we read from FFmpeg's frame pool because the
        // underlying ID3D11DeviceContext is not thread safe.
        std::lock_guard<AVD3D11VADeviceContext> guard(*avDeviceCtx);
 
        if (canImportDirectly) {
            // Import a copy of the slice, cropping away FFmpeg's surface alignment padding.
            handles =
                    m_importer.importTexture(*rhi, avDeviceCtx->device, avDeviceCtx->device_context,
                                             ffmpegTex, size, pixelFormat, index);
        } else {
            // The video processor reads the (possibly padded) pool slice directly, so no
            // separate cropping copy is needed here.
            const ComPtr<ID3D11Texture2D> converted = m_videoProcessor.convertToBGRA(
                    avDeviceCtx->device, avDeviceCtx->device_context, ffmpegTex, size,
                    fromAvColorSpace(frame->colorspace), fromAvColorRange(frame->color_range),
                    fromAvColorTransfer(frame->color_trc), index);
            if (converted) {
                handles = m_importer.importTexture(*rhi, avDeviceCtx->device,
                                                   avDeviceCtx->device_context, converted, size,
                                                   QVideoFrameFormat::Format_BGRA8888);
            }
        }
    }
 
    if (!handles)
        return nullptr;
 
    return std::make_unique<ImportedTextureHandles>(shared_from_this(), std::move(handles));
}
 
void D3D11TextureConverter::SetupDecoderTextures(AVCodecContext *s)
{
    int ret = avcodec_get_hw_frames_parameters(s, s->hw_device_ctx, AV_PIX_FMT_D3D11,
                                               &s->hw_frames_ctx);
    if (ret < 0) {
        qCDebug(qLcMediaFFmpegHWAccel) << "Failed to allocate HW frames context" << AVError{ ret };
        return;
    }
 
    const auto *frames_ctx = reinterpret_cast<const AVHWFramesContext *>(s->hw_frames_ctx->data);
    auto *hwctx = static_cast<AVD3D11VAFramesContext *>(frames_ctx->hwctx);
    hwctx->MiscFlags = D3D11_RESOURCE_MISC_SHARED;
    hwctx->BindFlags = D3D11_BIND_DECODER | D3D11_BIND_SHADER_RESOURCE;
    ret = av_hwframe_ctx_init(s->hw_frames_ctx);
    if (ret < 0) {
        qCDebug(qLcMediaFFmpegHWAccel)
                << "Failed to initialize HW frames context" << AVError{ ret };
        av_buffer_unref(&s->hw_frames_ctx);
    }
}
 
AVFrameUPtr copyFromHwPoolD3D11(AVFrameUPtr src)
{
    if (!src || !src->hw_frames_ctx || src->format != AV_PIX_FMT_D3D11)
        return src;
 
    const AVHWDeviceContext *avDevCtx = avFrameDeviceContext(src.get());
    if (!avDevCtx || avDevCtx->type != AV_HWDEVICE_TYPE_D3D11VA)
        return src;
 
    AVFrameUPtr dest = makeAVFrame();
    if (int status = av_frame_copy_props(dest.get(), src.get()); status != 0) {
        qCCritical(qLcMediaFFmpegHWAccel)
                << "Unable to copy frame from decoder pool" << AVError{ status };
        return src;
    }
 
    const AVD3D11VADeviceContext *hwDevCtx = getHwDeviceContext(avDevCtx);
    ComPtr<ID3D11Texture2D> destTex = [&] {
        std::lock_guard<AVD3D11VADeviceContext> guard(*hwDevCtx);
        return copyTexture(hwDevCtx, src.get());
    }();
 
    if (!destTex)
        return src;
 
    dest->buf[0] = wrapTextureAsBuffer(destTex);
    dest->data[0] = reinterpret_cast<uint8_t *>(destTex.Detach());
    dest->data[1] = reinterpret_cast<uint8_t *>(0); // This texture is not a texture array
 
    dest->width = src->width;
    dest->height = src->height;
    dest->format = src->format;
    dest->hw_frames_ctx = av_buffer_ref(src->hw_frames_ctx);
 
    return dest;
}
 
} // namespace QFFmpeg
 
QT_END_NAMESPACE