qffmpegvideobuffer.cpp

Go to the documentation of this file.

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qffmpegvideobuffer_p.h"
#include "private/qvideotexturehelper_p.h"
#include "private/qmultimediautils_p.h"
#include "qffmpeghwaccel_p.h"
#include "qloggingcategory.h"
#include <QtCore/qthread.h>
 
extern "C" {
#include <libavutil/pixdesc.h>
#include <libavutil/hdr_dynamic_metadata.h>
#include <libavutil/mastering_display_metadata.h>
}
 
QT_BEGIN_NAMESPACE
 
using namespace QFFmpeg;
 
Q_STATIC_LOGGING_CATEGORY(qLcFFmpegVideoBuffer, "qt.multimedia.ffmpeg.videobuffer");
 
static bool isFrameFlipped(const AVFrame& frame) {
    for (int i = 0; i < AV_NUM_DATA_POINTERS && frame.data[i]; ++i) {
        if (frame.linesize[i] < 0)
            return true;
    }
 
    return false;
}
 
QFFmpegVideoBuffer::QFFmpegVideoBuffer(AVFrameUPtr frame, AVRational pixelAspectRatio)
    : m_frame(frame.get()),
      m_size(qCalculateFrameSize(frameSize(frame), { pixelAspectRatio.num, pixelAspectRatio.den }))
{
    if (frame->hw_frames_ctx) {
        m_hwFrame = std::move(frame);
        m_pixelFormat = toQtPixelFormat(HWAccel::format(m_hwFrame.get()));
        return;
    }
 
    m_swFrame = std::move(frame);
    m_pixelFormat = toQtPixelFormat(AVPixelFormat(m_swFrame->format));
 
    convertSWFrame();
}
 
QFFmpegVideoBuffer::~QFFmpegVideoBuffer() = default;
 
void QFFmpegVideoBuffer::convertSWFrame()
{
    Q_ASSERT(m_swFrame);
 
    const auto actualAVPixelFormat = AVPixelFormat(m_swFrame->format);
    const auto targetAVPixelFormat = toAVPixelFormat(m_pixelFormat);
 
    const QSize actualSize = frameSize(m_swFrame);
    if (actualAVPixelFormat != targetAVPixelFormat || isFrameFlipped(*m_swFrame)
        || m_size != actualSize) {
        Q_ASSERT(toQtPixelFormat(targetAVPixelFormat) == m_pixelFormat);
        // convert the format into something we can handle
        SwsContextUPtr scaleContext = createSwsContext(actualSize, actualAVPixelFormat, m_size,
                                                       targetAVPixelFormat, SWS_BICUBIC);
 
        auto newFrame = makeAVFrame();
        newFrame->width = m_size.width();
        newFrame->height = m_size.height();
        newFrame->format = targetAVPixelFormat;
        const int status = av_frame_get_buffer(newFrame.get(), 0);
        if (status < 0) {
            qCWarning(qLcFFmpegVideoBuffer)
                    << "Failed to allocate frame buffer:" << AVError(status);
            return;
        }
 
        sws_scale(scaleContext.get(), m_swFrame->data, m_swFrame->linesize, 0, m_swFrame->height,
                  newFrame->data, newFrame->linesize);
        if (m_frame == m_swFrame.get())
            m_frame = newFrame.get();
        m_swFrame = std::move(newFrame);
    }
}
 
void QFFmpegVideoBuffer::initTextureConverter(QRhi &rhi)
{
    if (!m_hwFrame)
        return;
 
    // don't use the result reference here
    ensureTextureConverter(rhi);
}
 
QFFmpeg::TextureConverter &QFFmpegVideoBuffer::ensureTextureConverter(QRhi &rhi)
{
    Q_ASSERT(m_hwFrame);
 
    HwFrameContextData &frameContextData = HwFrameContextData::ensure(*m_hwFrame);
    TextureConverter *converter = frameContextData.textureConverterMapper.get(&rhi);
 
    if (!converter) {
        bool added = false;
        std::tie(converter, added) =
                frameContextData.textureConverterMapper.tryMap(rhi, TextureConverter(rhi));
        // no issues are expected if it's already added in another thread, however,it's worth to
        // check it
        Q_ASSERT(converter && added);
    }
 
    return *converter;
}
 
QRhi *QFFmpegVideoBuffer::associatedCurrentThreadRhi() const
{
    if (!m_hwFrame)
        return nullptr;
 
    HwFrameContextData &frameContextData = HwFrameContextData::ensure(*m_hwFrame);
    return frameContextData.textureConverterMapper.findRhi(
            [](QRhi &rhi) { return rhi.thread()->isCurrentThread(); });
}
 
QVideoFrameFormat::ColorSpace QFFmpegVideoBuffer::colorSpace() const
{
    return fromAvColorSpace(m_frame->colorspace);
}
 
QVideoFrameFormat::ColorTransfer QFFmpegVideoBuffer::colorTransfer() const
{
    return fromAvColorTransfer(m_frame->color_trc);
}
 
QVideoFrameFormat::ColorRange QFFmpegVideoBuffer::colorRange() const
{
    return fromAvColorRange(m_frame->color_range);
}
 
float QFFmpegVideoBuffer::maxNits()
{
    float maxNits = -1;
    for (int i = 0; i < m_frame->nb_side_data; ++i) {
        AVFrameSideData *sd = m_frame->side_data[i];
        // TODO: Longer term we might want to also support HDR10+ dynamic metadata
        if (sd->type == AV_FRAME_DATA_MASTERING_DISPLAY_METADATA) {
            auto *data = reinterpret_cast<AVMasteringDisplayMetadata *>(sd->data);
            // max_luminance is a rational in cd/m2 (nits) already, valid with has_luminance
            if (!data->has_luminance)
                continue;
            auto maybeLum = QFFmpeg::toReal(data->max_luminance);
            if (maybeLum)
                maxNits = float(*maybeLum);
        }
    }
    return maxNits;
}
 
QAbstractVideoBuffer::MapData QFFmpegVideoBuffer::map(QVideoFrame::MapMode mode)
{
    if (!m_swFrame) {
        Q_ASSERT(m_hwFrame && m_hwFrame->hw_frames_ctx);
        m_swFrame = makeAVFrame();
        /* retrieve data from GPU to CPU */
        int ret = av_hwframe_transfer_data(m_swFrame.get(), m_hwFrame.get(), 0);
        if (ret < 0) {
            qWarning() << "Error transferring the data to system memory:" << ret;
            return {};
        }
        convertSWFrame();
    }
 
    m_mode = mode;
 
    MapData mapData;
    auto *desc = QVideoTextureHelper::textureDescription(pixelFormat());
    mapData.planeCount = desc->nplanes;
    for (int i = 0; i < mapData.planeCount; ++i) {
        Q_ASSERT(m_swFrame->linesize[i] >= 0);
 
        mapData.data[i] = m_swFrame->data[i];
        mapData.bytesPerLine[i] = m_swFrame->linesize[i];
        mapData.dataSize[i] = mapData.bytesPerLine[i]*desc->heightForPlane(m_swFrame->height, i);
    }
 
    if ((mode & QVideoFrame::WriteOnly) != 0 && m_hwFrame)
        m_hwFrame.reset();
 
    return mapData;
}
 
void QFFmpegVideoBuffer::unmap()
{
    // nothing to do here for SW buffers.
    // Set NotMapped mode to ensure map/unmap/mapMode consisteny.
    m_mode = QVideoFrame::NotMapped;
}
 
QVideoFrameTexturesUPtr QFFmpegVideoBuffer::mapTextures(QRhi &rhi, QVideoFrameTexturesUPtr& oldTextures)
{
    Q_ASSERT(rhi.thread()->isCurrentThread());
 
    return createTexturesFromHwFrame(rhi, oldTextures);
}
 
QVideoFrameTexturesUPtr QFFmpegVideoBuffer::createTexturesFromHwFrame(QRhi &rhi, QVideoFrameTexturesUPtr& oldTextures) {
 
    if (!m_hwFrame)
        return {};
 
    // QTBUG-132200:
    // We aim to set initTextureConverterForAnyRhi=true for as much platforms as we can,
    // and remove the check after all platforms work fine on CI. If the flag is enabled,
    // QVideoFrame::toImage can work faster, and we can test hw texture conversion on CI.
    // Currently, enabling the flag fails some CI platforms.
    constexpr bool initTextureConverterForAnyRhi = false;
 
    TextureConverter *converter = initTextureConverterForAnyRhi
            ? &ensureTextureConverter(rhi)
            : HwFrameContextData::ensure(*m_hwFrame).textureConverterMapper.get(&rhi);
 
    if (!converter)
        return {};
 
    if (!converter->init(*m_hwFrame))
        return {};
 
    const QVideoFrameTextures *oldTexturesRaw = oldTextures.get();
    if (QVideoFrameTexturesUPtr newTextures = converter->createTextures(*m_hwFrame, oldTextures))
        return newTextures;
 
    Q_ASSERT(oldTextures.get() == oldTexturesRaw);
 
    QVideoFrameTexturesHandlesUPtr oldTextureHandles =
            oldTextures ? oldTextures->takeHandles() : nullptr;
    QVideoFrameTexturesHandlesUPtr newTextureHandles =
            converter->createTextureHandles(*m_hwFrame, std::move(oldTextureHandles));
 
    if (newTextureHandles) {
        QVideoFrameTexturesUPtr newTextures = QVideoTextureHelper::createTexturesFromHandles(
                std::move(newTextureHandles), rhi, m_pixelFormat, frameSize(m_hwFrame));
 
        return newTextures;
    }
 
    static thread_local int lastFormat = 0;
    if (std::exchange(lastFormat, m_hwFrame->format) != m_hwFrame->format) // prevent logging spam
        qWarning() << "    failed to get textures for frame; format:" << m_hwFrame->format;
 
    return {};
}
 
QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::pixelFormat() const
{
    return m_pixelFormat;
}
 
QSize QFFmpegVideoBuffer::size() const
{
    return m_size;
}
 
QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat avPixelFormat, bool *needsConversion)
{
    if (needsConversion)
        *needsConversion = false;
 
    switch (avPixelFormat) {
    default:
        break;
    case AV_PIX_FMT_NONE:
        Q_ASSERT(!"Invalid avPixelFormat!");
        return QVideoFrameFormat::Format_Invalid;
    case AV_PIX_FMT_ARGB:
        return QVideoFrameFormat::Format_ARGB8888;
    case AV_PIX_FMT_0RGB:
        return QVideoFrameFormat::Format_XRGB8888;
    case AV_PIX_FMT_BGRA:
        return QVideoFrameFormat::Format_BGRA8888;
    case AV_PIX_FMT_BGR0:
        return QVideoFrameFormat::Format_BGRX8888;
    case AV_PIX_FMT_ABGR:
        return QVideoFrameFormat::Format_ABGR8888;
    case AV_PIX_FMT_0BGR:
        return QVideoFrameFormat::Format_XBGR8888;
    case AV_PIX_FMT_RGBA:
        return QVideoFrameFormat::Format_RGBA8888;
    case AV_PIX_FMT_RGB0:
        return QVideoFrameFormat::Format_RGBX8888;
 
    case AV_PIX_FMT_YUV422P:
        return QVideoFrameFormat::Format_YUV422P;
    case AV_PIX_FMT_YUV420P:
        return QVideoFrameFormat::Format_YUV420P;
    case AV_PIX_FMT_YUV420P10:
        return QVideoFrameFormat::Format_YUV420P10;
    case AV_PIX_FMT_UYVY422:
        return QVideoFrameFormat::Format_UYVY;
    case AV_PIX_FMT_YUYV422:
        return QVideoFrameFormat::Format_YUYV;
    case AV_PIX_FMT_NV12:
        return QVideoFrameFormat::Format_NV12;
    case AV_PIX_FMT_NV21:
        return QVideoFrameFormat::Format_NV21;
    case AV_PIX_FMT_GRAY8:
        return QVideoFrameFormat::Format_Y8;
    case AV_PIX_FMT_GRAY16:
        return QVideoFrameFormat::Format_Y16;
 
    case AV_PIX_FMT_P010:
        return QVideoFrameFormat::Format_P010;
    case AV_PIX_FMT_P016:
        return QVideoFrameFormat::Format_P016;
    case AV_PIX_FMT_MEDIACODEC:
        return QVideoFrameFormat::Format_SamplerExternalOES;
    }
 
    if (needsConversion)
        *needsConversion = true;
 
    const AVPixFmtDescriptor *descriptor = av_pix_fmt_desc_get(avPixelFormat);
 
    if (descriptor->flags & AV_PIX_FMT_FLAG_RGB)
        return QVideoFrameFormat::Format_RGBA8888;
 
    if (descriptor->comp[0].depth > 8)
        return QVideoFrameFormat::Format_P016;
    return QVideoFrameFormat::Format_YUV420P;
}
 
AVPixelFormat QFFmpegVideoBuffer::toAVPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat)
{
    switch (pixelFormat) {
    default:
    case QVideoFrameFormat::Format_Invalid:
    case QVideoFrameFormat::Format_AYUV:
    case QVideoFrameFormat::Format_AYUV_Premultiplied:
    case QVideoFrameFormat::Format_YV12:
    case QVideoFrameFormat::Format_IMC1:
    case QVideoFrameFormat::Format_IMC2:
    case QVideoFrameFormat::Format_IMC3:
    case QVideoFrameFormat::Format_IMC4:
        return AV_PIX_FMT_NONE;
    case QVideoFrameFormat::Format_Jpeg:
        // We're using the data from the converted QImage here, which is in BGRA.
        return AV_PIX_FMT_BGRA;
    case QVideoFrameFormat::Format_ARGB8888:
        return AV_PIX_FMT_ARGB;
    case QVideoFrameFormat::Format_ARGB8888_Premultiplied:
    case QVideoFrameFormat::Format_XRGB8888:
        return AV_PIX_FMT_0RGB;
    case QVideoFrameFormat::Format_BGRA8888:
        return AV_PIX_FMT_BGRA;
    case QVideoFrameFormat::Format_BGRA8888_Premultiplied:
    case QVideoFrameFormat::Format_BGRX8888:
        return AV_PIX_FMT_BGR0;
    case QVideoFrameFormat::Format_ABGR8888:
        return AV_PIX_FMT_ABGR;
    case QVideoFrameFormat::Format_XBGR8888:
        return AV_PIX_FMT_0BGR;
    case QVideoFrameFormat::Format_RGBA8888:
        return AV_PIX_FMT_RGBA;
    // to be added in 6.8:
    // case QVideoFrameFormat::Format_RGBA8888_Premultiplied:
    case QVideoFrameFormat::Format_RGBX8888:
        return AV_PIX_FMT_RGB0;
 
    case QVideoFrameFormat::Format_YUV422P:
        return AV_PIX_FMT_YUV422P;
    case QVideoFrameFormat::Format_YUV420P:
        return AV_PIX_FMT_YUV420P;
    case QVideoFrameFormat::Format_YUV420P10:
        return AV_PIX_FMT_YUV420P10;
    case QVideoFrameFormat::Format_UYVY:
        return AV_PIX_FMT_UYVY422;
    case QVideoFrameFormat::Format_YUYV:
        return AV_PIX_FMT_YUYV422;
    case QVideoFrameFormat::Format_NV12:
        return AV_PIX_FMT_NV12;
    case QVideoFrameFormat::Format_NV21:
        return AV_PIX_FMT_NV21;
    case QVideoFrameFormat::Format_Y8:
        return AV_PIX_FMT_GRAY8;
    case QVideoFrameFormat::Format_Y16:
        return AV_PIX_FMT_GRAY16;
 
    case QVideoFrameFormat::Format_P010:
        return AV_PIX_FMT_P010;
    case QVideoFrameFormat::Format_P016:
        return AV_PIX_FMT_P016;
 
    case QVideoFrameFormat::Format_SamplerExternalOES:
        return AV_PIX_FMT_MEDIACODEC;
    }
}
 
QT_END_NAMESPACE