qvideowindow.cpp

Go to the documentation of this file.

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qvideowindow_p.h"
 
#include <QtMultimedia/private/qmemoryvideobuffer_p.h>
#include <QtMultimedia/private/qhwvideobuffer_p.h>
#include <QtMultimedia/private/qmultimediautils_p.h>
#include <QtMultimedia/private/qvideoframe_p.h>
 
#include <QtGui/qevent.h>
#include <QtGui/qpainter.h>
#include <QtGui/private/qguiapplication_p.h>
#include <QtGui/qpa/qplatformintegration.h>
 
#include <QtCore/qfile.h>
#include <QtCore/private/qoperatingsystemversion_p.h>
 
QT_BEGIN_NAMESPACE
 
namespace {
 
std::optional<QRhi::Implementation> g_graphicsApiOverride;
 
QSurface::SurfaceType surfaceTypeFor(QRhi::Implementation graphicsApi)
{
    switch (graphicsApi) {
    case QRhi::OpenGLES2:
        return QSurface::OpenGLSurface;
    case QRhi::Vulkan:
        return QSurface::VulkanSurface;
    case QRhi::Metal:
        return QSurface::MetalSurface;
    case QRhi::D3D11:
    case QRhi::D3D12:
        return QSurface::Direct3DSurface;
    default:
        return QSurface::RasterSurface;
    }
}
 
QSurface::SurfaceType platformSurfaceType()
{
    switch (QOperatingSystemVersion::currentType()) {
    case QOperatingSystemVersion::Windows:
        return QSurface::Direct3DSurface;
    case QOperatingSystemVersion::MacOS:
        return QSurface::MetalSurface;
    default:
        auto *integration = QGuiApplicationPrivate::platformIntegration();
 
        if (!integration->hasCapability(QPlatformIntegration::OpenGL))
            return QSurface::RasterSurface;
 
        if (QCoreApplication::testAttribute(Qt::AA_ForceRasterWidgets))
            return QSurface::RasterSurface;
 
        return QSurface::OpenGLSurface;
    }
}
 
} // namespace
 
void QVideoWindowPrivate::setGraphicsApiOverride(std::optional<QRhi::Implementation> api)
{
    g_graphicsApiOverride = api;
}
 
QVideoWindowPrivate::QVideoWindowPrivate(QVideoWindow *q) : q(q), m_sink(new QVideoSink)
{
    Q_ASSERT(q);
 
    if (QGuiApplicationPrivate::platformIntegration()->hasCapability(QPlatformIntegration::RhiBasedRendering)) {
        if (g_graphicsApiOverride) {
            m_graphicsApi = *g_graphicsApiOverride;
            q->setSurfaceType(surfaceTypeFor(m_graphicsApi));
        } else {
            auto surfaceType = ::platformSurfaceType();
            q->setSurfaceType(surfaceType);
            switch (surfaceType) {
            case QSurface::RasterSurface:
            case QSurface::OpenVGSurface:
            default:
                // can't use those surfaces, need to render in SW
                m_graphicsApi = QRhi::Null;
                break;
            case QSurface::OpenGLSurface:
                m_graphicsApi = QRhi::OpenGLES2;
                break;
            case QSurface::VulkanSurface:
                m_graphicsApi = QRhi::Vulkan;
                break;
            case QSurface::MetalSurface:
                m_graphicsApi = QRhi::Metal;
                break;
            case QSurface::Direct3DSurface:
                m_graphicsApi = QRhi::D3D11;
                break;
            }
        }
    }
 
    QObject::connect(m_sink.get(), &QVideoSink::videoFrameChanged, q, &QVideoWindow::setVideoFrame);
}
 
QVideoWindowPrivate::~QVideoWindowPrivate()
{
    QObject::disconnect(m_sink.get(), &QVideoSink::videoFrameChanged,
            q, &QVideoWindow::setVideoFrame);
}
 
static const float g_vw_quad[] = {
    // 4 clockwise rotation of texture vertexes (the second pair)
    // Rotation 0
    -1.f, -1.f,   0.f, 0.f,
    -1.f, 1.f,    0.f, 1.f,
    1.f, -1.f,    1.f, 0.f,
    1.f, 1.f,     1.f, 1.f,
    // Rotation 90
    -1.f, -1.f,   0.f, 1.f,
    -1.f, 1.f,    1.f, 1.f,
    1.f, -1.f,    0.f, 0.f,
    1.f, 1.f,     1.f, 0.f,
 
    // Rotation 180
    -1.f, -1.f,   1.f, 1.f,
    -1.f, 1.f,    1.f, 0.f,
    1.f, -1.f,    0.f, 1.f,
    1.f, 1.f,     0.f, 0.f,
    // Rotation 270
    -1.f, -1.f,   1.f, 0.f,
    -1.f, 1.f,    0.f, 0.f,
    1.f, -1.f,    1.f, 1.f,
    1.f, 1.f,     0.f, 1.f
};
 
static QShader vwGetShader(const QString &name)
{
    QFile f(name);
    if (f.open(QIODevice::ReadOnly))
        return QShader::fromSerialized(f.readAll());
 
    return QShader();
}
 
void QVideoWindowPrivate::initRhi()
{
    if (m_graphicsApi == QRhi::Null)
        return;
 
    [[maybe_unused]] QRhi::Flags rhiFlags = {};//QRhi::EnableDebugMarkers | QRhi::EnableProfiling;
 
#if QT_CONFIG(opengl)
    if (m_graphicsApi == QRhi::OpenGLES2) {
        m_fallbackSurface.reset(QRhiGles2InitParams::newFallbackSurface(q->format()));
        QRhiGles2InitParams params;
        params.fallbackSurface = m_fallbackSurface.get();
        params.window = q;
        params.format = q->format();
        m_rhi.reset(QRhi::create(QRhi::OpenGLES2, &params, rhiFlags));
    }
#endif
 
#if QT_CONFIG(vulkan)
    if (m_graphicsApi == QRhi::Vulkan) {
        QRhiVulkanInitParams params;
        params.inst = q->vulkanInstance();
        params.window = q;
        m_rhi.reset(QRhi::create(QRhi::Vulkan, &params, rhiFlags));
    }
#endif
 
#ifdef Q_OS_WIN
    if (m_graphicsApi == QRhi::D3D11) {
        QRhiD3D11InitParams params;
        params.enableDebugLayer = true;
        m_rhi.reset(QRhi::create(QRhi::D3D11, &params, rhiFlags));
    }
    if (m_graphicsApi == QRhi::D3D12) {
        QRhiD3D12InitParams params;
        m_rhi.reset(QRhi::create(QRhi::D3D12, &params, rhiFlags));
    }
#endif
 
#if QT_CONFIG(metal)
    if (m_graphicsApi == QRhi::Metal) {
        QRhiMetalInitParams params;
        m_rhi.reset(QRhi::create(QRhi::Metal, &params, rhiFlags));
    }
#endif
    if (!m_rhi)
        return;
 
    m_swapChain.reset(m_rhi->newSwapChain());
    m_swapChain->setWindow(q);
    m_renderPass.reset(m_swapChain->newCompatibleRenderPassDescriptor());
    m_swapChain->setRenderPassDescriptor(m_renderPass.get());
 
    m_vertexBuf.reset(m_rhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(g_vw_quad)));
    m_vertexBuf->create();
    m_vertexBufReady = false;
 
    m_uniformBuf.reset(m_rhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, sizeof(QVideoTextureHelper::UniformData)));
    m_uniformBuf->create();
 
    m_textureSampler.reset(m_rhi->newSampler(QRhiSampler::Linear, QRhiSampler::Linear, QRhiSampler::None,
                                             QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge));
    m_textureSampler->create();
 
    m_shaderResourceBindings.reset(m_rhi->newShaderResourceBindings());
    m_subtitleResourceBindings.reset(m_rhi->newShaderResourceBindings());
 
    m_subtitleUniformBuf.reset(m_rhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, sizeof(QVideoTextureHelper::UniformData)));
    m_subtitleUniformBuf->create();
}
 
void QVideoWindowPrivate::setupGraphicsPipeline(QRhiGraphicsPipeline *pipeline, QRhiShaderResourceBindings *bindings, const QVideoFrameFormat &fmt)
{
 
    pipeline->setTopology(QRhiGraphicsPipeline::TriangleStrip);
    QShader vs = vwGetShader(QVideoTextureHelper::vertexShaderFileName(fmt));
    Q_ASSERT(vs.isValid());
    QShader fs = vwGetShader(QVideoTextureHelper::fragmentShaderFileName(
            fmt, m_rhi.get(), m_swapChain->format()));
    Q_ASSERT(fs.isValid());
    pipeline->setShaderStages({
        { QRhiShaderStage::Vertex, vs },
        { QRhiShaderStage::Fragment, fs }
    });
    QRhiVertexInputLayout inputLayout;
    inputLayout.setBindings({
        { 4 * sizeof(float) }
    });
    inputLayout.setAttributes({
        { 0, 0, QRhiVertexInputAttribute::Float2, 0 },
        { 0, 1, QRhiVertexInputAttribute::Float2, 2 * sizeof(float) }
    });
    pipeline->setVertexInputLayout(inputLayout);
    pipeline->setShaderResourceBindings(bindings);
    pipeline->setRenderPassDescriptor(m_renderPass.get());
    pipeline->create();
}
 
void QVideoWindowPrivate::updateTextures(QRhiResourceUpdateBatch *rub)
{
    // We render a 1x1 black pixel when we don't have a video
    if (!m_texturePool.currentFrame().isValid())
        m_texturePool.setCurrentFrame(QVideoFramePrivate::createFrame(
                std::make_unique<QMemoryVideoBuffer>(QByteArray{ 4, 0 }, 4),
                QVideoFrameFormat(QSize(1, 1), QVideoFrameFormat::Format_RGBA8888)));
 
    if (!m_texturePool.texturesDirty())
        return;
 
    QVideoFrameTextures *textures = m_texturePool.updateTextures(*m_rhi, *rub);
    if (!textures)
        return;
 
    QRhiShaderResourceBinding bindings[4];
    auto *b = bindings;
    *(b++) = QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage,
                                                   m_uniformBuf.get());
 
    auto fmt = m_texturePool.currentFrame().surfaceFormat();
    auto textureDesc = QVideoTextureHelper::textureDescription(fmt.pixelFormat());
 
    for (int i = 0; i < textureDesc->nplanes; ++i)
        (*b++) = QRhiShaderResourceBinding::sampledTexture(
                i + 1, QRhiShaderResourceBinding::FragmentStage, textures->texture(i),
                m_textureSampler.get());
    m_shaderResourceBindings->setBindings(bindings, b);
    m_shaderResourceBindings->create();
 
    if (fmt != format) {
        format = fmt;
        if (!m_graphicsPipeline)
            m_graphicsPipeline.reset(m_rhi->newGraphicsPipeline());
 
        setupGraphicsPipeline(m_graphicsPipeline.get(), m_shaderResourceBindings.get(), format);
    }
}
 
void QVideoWindowPrivate::updateSubtitle(QRhiResourceUpdateBatch *rub, const QSize &frameSize)
{
    m_subtitleDirty = false;
    m_hasSubtitle = !m_texturePool.currentFrame().subtitleText().isEmpty();
    if (!m_hasSubtitle)
        return;
 
    m_subtitleLayout.update(frameSize, m_texturePool.currentFrame().subtitleText());
    QSize size = m_subtitleLayout.bounds.size().toSize();
 
    QImage img = m_subtitleLayout.toImage();
 
    m_subtitleTexture.reset(m_rhi->newTexture(QRhiTexture::RGBA8, size));
    m_subtitleTexture->create();
    rub->uploadTexture(m_subtitleTexture.get(), img);
 
    QRhiShaderResourceBinding bindings[2];
 
    bindings[0] = QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage,
                                                           m_subtitleUniformBuf.get());
 
    bindings[1] = QRhiShaderResourceBinding::sampledTexture(1, QRhiShaderResourceBinding::FragmentStage,
                                                            m_subtitleTexture.get(), m_textureSampler.get());
    m_subtitleResourceBindings->setBindings(bindings, bindings + 2);
    m_subtitleResourceBindings->create();
 
    if (!m_subtitlePipeline) {
        m_subtitlePipeline.reset(m_rhi->newGraphicsPipeline());
 
        QRhiGraphicsPipeline::TargetBlend blend;
        blend.enable = true;
        m_subtitlePipeline->setTargetBlends({ blend });
        setupGraphicsPipeline(m_subtitlePipeline.get(), m_subtitleResourceBindings.get(), QVideoFrameFormat(QSize(1, 1), QVideoFrameFormat::Format_RGBA8888));
    }
}
 
void QVideoWindowPrivate::init()
{
    if (initialized)
        return;
    initialized = true;
 
    initRhi();
 
    if (!m_rhi)
        backingStore = new QBackingStore(q);
    else
        m_sink->setRhi(m_rhi.get());
}
 
void QVideoWindowPrivate::resizeSwapChain()
{
    m_hasSwapChain = m_swapChain->createOrResize();
}
 
void QVideoWindowPrivate::releaseSwapChain()
{
    if (m_hasSwapChain) {
        m_hasSwapChain = false;
        m_swapChain->destroy();
    }
}
 
void QVideoWindowPrivate::render()
{
    if (!initialized)
        init();
 
    if (!q->isExposed() || !isExposed)
        return;
 
    QRect rect(0, 0, q->width(), q->height());
 
    if (backingStore) {
        if (backingStore->size() != q->size())
            backingStore->resize(q->size());
 
        backingStore->beginPaint(rect);
 
        QPaintDevice *device = backingStore->paintDevice();
        if (!device)
            return;
        QPainter painter(device);
 
        QVideoFrame frame = m_texturePool.currentFrame();
        frame.paint(&painter, rect, { Qt::black, aspectRatioMode });
        painter.end();
 
        backingStore->endPaint();
        backingStore->flush(rect);
        return;
    }
 
    const VideoTransformation frameTransformation =
            qNormalizedFrameTransformation(m_texturePool.currentFrame().surfaceFormat());
    const QSize frameSize = qRotatedFramePresentationSize(m_texturePool.currentFrame());
    const QSize scaled = frameSize.scaled(rect.size(), aspectRatioMode);
    QRect videoRect = QRect(QPoint(0, 0), scaled);
    videoRect.moveCenter(rect.center());
    QRect subtitleRect = videoRect.intersected(rect);
 
    if (!m_hasSwapChain || (m_swapChain->currentPixelSize() != m_swapChain->surfacePixelSize()))
        resizeSwapChain();
 
    const auto requiredSwapChainFormat =
            qGetRequiredSwapChainFormat(m_texturePool.currentFrame().surfaceFormat());
    if (qShouldUpdateSwapChainFormat(m_swapChain.get(), requiredSwapChainFormat)) {
        releaseSwapChain();
        m_swapChain->setFormat(requiredSwapChainFormat);
        resizeSwapChain();
    }
 
    if (!m_hasSwapChain)
        return;
 
    QRhi::FrameOpResult r = m_rhi->beginFrame(m_swapChain.get());
 
    if (r == QRhi::FrameOpSwapChainOutOfDate) {
        resizeSwapChain();
        if (!m_hasSwapChain)
            return;
        r = m_rhi->beginFrame(m_swapChain.get());
    }
    if (r != QRhi::FrameOpSuccess) {
        qWarning("beginFrame failed with %d, retry", r);
        q->requestUpdate();
        return;
    }
 
    QRhiResourceUpdateBatch *rub = m_rhi->nextResourceUpdateBatch();
 
    if (!m_vertexBufReady) {
        m_vertexBufReady = true;
        rub->uploadStaticBuffer(m_vertexBuf.get(), g_vw_quad);
    }
 
    updateTextures(rub);
 
    if (m_subtitleDirty || m_subtitleLayout.videoSize != subtitleRect.size())
        updateSubtitle(rub, subtitleRect.size());
 
    const float mirrorFrame = frameTransformation.mirroredHorizontallyAfterRotation ? -1.f : 1.f;
    const float xscale = mirrorFrame * float(videoRect.width()) / float(rect.width());
    const float yscale = -1.f * float(videoRect.height()) / float(rect.height());
 
    QMatrix4x4 transform;
    transform.scale(xscale, yscale);
 
    float maxNits = 100;
    if (m_swapChain->format() == QRhiSwapChain::HDRExtendedSrgbLinear) {
        auto info = m_swapChain->hdrInfo();
        if (info.limitsType == QRhiSwapChainHdrInfo::ColorComponentValue)
            maxNits = 100 * info.limits.colorComponentValue.maxColorComponentValue;
        else
            maxNits = info.limits.luminanceInNits.maxLuminance;
    }
 
    QByteArray uniformData;
    QVideoTextureHelper::updateUniformData(&uniformData, m_rhi.get(),
                                           m_texturePool.currentFrame().surfaceFormat(),
                                           m_texturePool.currentFrame(), transform, 1.f, maxNits);
    rub->updateDynamicBuffer(m_uniformBuf.get(), 0, uniformData.size(), uniformData.constData());
 
    if (m_hasSubtitle) {
        QMatrix4x4 st;
        st.translate(0, -2.f * (float(m_subtitleLayout.bounds.center().y())  + float(subtitleRect.top()))/ float(rect.height()) + 1.f);
        st.scale(float(m_subtitleLayout.bounds.width())/float(rect.width()),
                -1.f * float(m_subtitleLayout.bounds.height())/float(rect.height()));
 
        QByteArray uniformData;
        QVideoFrameFormat fmt(m_subtitleLayout.bounds.size().toSize(), QVideoFrameFormat::Format_ARGB8888);
        QVideoTextureHelper::updateUniformData(&uniformData, m_rhi.get(), fmt, QVideoFrame(), st,
                                               1.f);
        rub->updateDynamicBuffer(m_subtitleUniformBuf.get(), 0, uniformData.size(), uniformData.constData());
    }
 
    QRhiCommandBuffer *cb = m_swapChain->currentFrameCommandBuffer();
    cb->beginPass(m_swapChain->currentFrameRenderTarget(), Qt::black, { 1.0f, 0 }, rub);
    cb->setGraphicsPipeline(m_graphicsPipeline.get());
    auto size = m_swapChain->currentPixelSize();
    cb->setViewport({ 0, 0, float(size.width()), float(size.height()) });
    cb->setShaderResources(m_shaderResourceBindings.get());
 
    const quint32 vertexOffset = quint32(sizeof(float)) * 16 * frameTransformation.rotationIndex();
    const QRhiCommandBuffer::VertexInput vbufBinding(m_vertexBuf.get(), vertexOffset);
    cb->setVertexInput(0, 1, &vbufBinding);
    cb->draw(4);
 
    if (m_hasSubtitle) {
        cb->setGraphicsPipeline(m_subtitlePipeline.get());
        cb->setShaderResources(m_subtitleResourceBindings.get());
        const QRhiCommandBuffer::VertexInput vbufBinding(m_vertexBuf.get(), 0);
        cb->setVertexInput(0, 1, &vbufBinding);
        cb->draw(4);
    }
 
    cb->endPass();
 
    if (m_postRenderHookForTests)
        m_postRenderHookForTests(*m_rhi, *cb);
 
    m_rhi->endFrame(m_swapChain.get());
 
    m_texturePool.onFrameEndInvoked();
}
 
void QVideoWindowPrivate::setPostRenderHookForTests(
        std::function<void(QRhi &, QRhiCommandBuffer &)> hook)
{
    m_postRenderHookForTests = std::move(hook);
}
 
/*!
    \class QVideoWindow
    \internal
*/
QVideoWindow::QVideoWindow(QScreen *screen)
    : QWindow(screen)
    , d(new QVideoWindowPrivate(this))
{
}
 
QVideoWindow::QVideoWindow(QWindow *parent)
    : QWindow(parent)
    , d(new QVideoWindowPrivate(this))
{
}
 
QVideoWindow::~QVideoWindow() = default;
 
QVideoSink *QVideoWindow::videoSink() const
{
    return d->m_sink.get();
}
 
Qt::AspectRatioMode QVideoWindow::aspectRatioMode() const
{
    return d->aspectRatioMode;
}
 
void QVideoWindow::setAspectRatioMode(Qt::AspectRatioMode mode)
{
    if (d->aspectRatioMode == mode)
        return;
    d->aspectRatioMode = mode;
    Q_EMIT aspectRatioModeChanged(mode);
}
 
bool QVideoWindow::event(QEvent *e)
{
    switch (e->type()) {
    case QEvent::UpdateRequest:
        d->render();
        return true;
 
    case QEvent::PlatformSurface:
        // this is the proper time to tear down the swapchain (while the native window and surface are still around)
        if (static_cast<QPlatformSurfaceEvent *>(e)->surfaceEventType() == QPlatformSurfaceEvent::SurfaceAboutToBeDestroyed) {
            d->releaseSwapChain();
            d->isExposed = false;
        }
        break;
    case QEvent::Expose:
        d->isExposed = isExposed();
        if (d->isExposed)
            d->render();
        return true;
 
    default:
        break;
    }
 
    return QWindow::event(e);
}
 
void QVideoWindow::resizeEvent(QResizeEvent *resizeEvent)
{
    if (!d->backingStore)
        return;
    if (!d->initialized)
        d->init();
    d->backingStore->resize(resizeEvent->size());
}
 
void QVideoWindow::setVideoFrame(const QVideoFrame &frame)
{
    if (d->m_texturePool.currentFrame().subtitleText() != frame.subtitleText())
        d->m_subtitleDirty = true;
    d->m_texturePool.setCurrentFrame(frame);
    if (d->isExposed)
        requestUpdate();
}
 
QT_END_NAMESPACE
 
#include "moc_qvideowindow_p.cpp"