qandroidvideooutput.cpp

Go to the documentation of this file.

// Copyright (C) 2021 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qandroidvideooutput_p.h"
#include "androidsurfacetexture_p.h"
 
#include <rhi/qrhi.h>
#if QT_CONFIG(opengl)
#include <QtGui/private/qopenglextensions_p.h>
#include <qopenglcontext.h>
#include <qopenglfunctions.h>
#endif
#include <private/qhwvideobuffer_p.h>
#include <private/qvideoframeconverter_p.h>
#include <private/qplatformvideosink_p.h>
#include <private/qvideoframe_p.h>
#include <qvideosink.h>
#include <qvideoframeformat.h>
#include <qthread.h>
#include <qfile.h>
 
QT_BEGIN_NAMESPACE
 
class QAndroidVideoFrameTextures : public QVideoFrameTextures
{
public:
    QAndroidVideoFrameTextures(QRhi *rhi, QSize size, quint64 handle)
    {
        m_tex.reset(rhi->newTexture(QRhiTexture::RGBA8, size, 1));
        m_tex->createFrom({quint64(handle), 0});
    }
 
    QRhiTexture *texture(uint plane) const override
    {
        return plane == 0 ? m_tex.get() : nullptr;
    }
 
private:
    std::unique_ptr<QRhiTexture> m_tex;
};
 
class AndroidTextureVideoBuffer final : public QHwVideoBuffer
{
public:
    AndroidTextureVideoBuffer(std::unique_ptr<QRhiTexture> tex, const QSize &size)
        : m_size(size), m_tex(std::move(tex))
    {}
 
    MapData map(QVideoFrame::MapMode mode) override;
 
    void unmap() override
    {
        m_image = {};
        m_mapMode = QVideoFrame::NotMapped;
    }
 
    QVideoFrameTexturesUPtr mapTextures(QRhi &rhi) override
    {
        return std::make_unique<QAndroidVideoFrameTextures>(&rhi, m_size, m_tex->nativeTexture().object);
    }
 
private:
    QSize m_size;
    std::unique_ptr<QRhiTexture> m_tex;
    QImage m_image;
    QVideoFrame::MapMode m_mapMode = QVideoFrame::NotMapped;
};
 
class ImageFromVideoFrameHelper final : public QHwVideoBuffer
{
public:
    ImageFromVideoFrameHelper(AndroidTextureVideoBuffer &atvb) : m_atvb(atvb)
    {}
    QVideoFrameTexturesUPtr mapTextures(QRhi &rhi) override
    {
        return m_atvb.mapTextures(rhi);
    }
 
    MapData map(QVideoFrame::MapMode) override { return {}; }
    void unmap() override {}
 
private:
    AndroidTextureVideoBuffer &m_atvb;
};
 
QAbstractVideoBuffer::MapData AndroidTextureVideoBuffer::map(QVideoFrame::MapMode mode)
{
    QAbstractVideoBuffer::MapData mapData;
 
    if (m_mapMode == QVideoFrame::NotMapped && mode == QVideoFrame::ReadOnly) {
        m_mapMode = QVideoFrame::ReadOnly;
        m_image = qImageFromVideoFrame(QVideoFramePrivate::createFrame(
                std::make_unique<ImageFromVideoFrameHelper>(*this),
                QVideoFrameFormat(m_size, QVideoFrameFormat::Format_RGBA8888)));
        mapData.planeCount = 1;
        mapData.bytesPerLine[0] = m_image.bytesPerLine();
        mapData.dataSize[0] = static_cast<int>(m_image.sizeInBytes());
        mapData.data[0] = m_image.bits();
    }
 
    return mapData;
}
 
static const float g_quad[] = {
    -1.f, -1.f, 0.f, 0.f,
    -1.f,  1.f, 0.f, 1.f,
    1.f,  1.f, 1.f, 1.f,
    1.f, -1.f, 1.f, 0.f
};
 
class TextureCopy
{
    static QShader getShader(const QString &name)
    {
        QFile f(name);
        if (f.open(QIODevice::ReadOnly))
            return QShader::fromSerialized(f.readAll());
        return {};
    }
 
public:
    TextureCopy(QRhi *rhi, QRhiTexture *externalTex)
        : m_rhi(rhi)
    {
        m_vertexBuffer.reset(m_rhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(g_quad)));
        m_vertexBuffer->create();
 
        m_uniformBuffer.reset(m_rhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, 64 + 64 + 4 + 4));
        m_uniformBuffer->create();
 
        m_sampler.reset(m_rhi->newSampler(QRhiSampler::Nearest, QRhiSampler::Nearest, QRhiSampler::None,
                                          QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge));
        m_sampler->create();
 
        m_srb.reset(m_rhi->newShaderResourceBindings());
        m_srb->setBindings({
                QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage, m_uniformBuffer.get()),
                QRhiShaderResourceBinding::sampledTexture(1, QRhiShaderResourceBinding::FragmentStage, externalTex, m_sampler.get())
        });
        m_srb->create();
 
        m_vertexShader = getShader(QStringLiteral(":/qt-project.org/multimedia/shaders/externalsampler.vert.qsb"));
        Q_ASSERT(m_vertexShader.isValid());
        m_fragmentShader = getShader(QStringLiteral(":/qt-project.org/multimedia/shaders/externalsampler.frag.qsb"));
        Q_ASSERT(m_fragmentShader.isValid());
    }
 
#if QT_CONFIG(opengl)
    std::unique_ptr<QRhiTexture> copyExternalGlTexture(QSize size, const QMatrix4x4 &externalTexMatrix);
#endif
 
private:
    QRhi *m_rhi = nullptr;
    std::unique_ptr<QRhiBuffer> m_vertexBuffer;
    std::unique_ptr<QRhiBuffer> m_uniformBuffer;
    std::unique_ptr<QRhiSampler> m_sampler;
    std::unique_ptr<QRhiShaderResourceBindings> m_srb;
    QShader m_vertexShader;
    QShader m_fragmentShader;
};
 
#if QT_CONFIG(opengl)
static std::unique_ptr<QRhiGraphicsPipeline> newGraphicsPipeline(QRhi *rhi,
                                                                 QRhiShaderResourceBindings *shaderResourceBindings,
                                                                 QRhiRenderPassDescriptor *renderPassDescriptor,
                                                                 QShader vertexShader,
                                                                 QShader fragmentShader)
{
    std::unique_ptr<QRhiGraphicsPipeline> gp(rhi->newGraphicsPipeline());
    gp->setTopology(QRhiGraphicsPipeline::TriangleFan);
    gp->setShaderStages({
            { QRhiShaderStage::Vertex, vertexShader },
            { QRhiShaderStage::Fragment, fragmentShader }
    });
    QRhiVertexInputLayout inputLayout;
    inputLayout.setBindings({
            { 4 * sizeof(float) }
    });
    inputLayout.setAttributes({
            { 0, 0, QRhiVertexInputAttribute::Float2, 0 },
            { 0, 1, QRhiVertexInputAttribute::Float2, 2 * sizeof(float) }
    });
    gp->setVertexInputLayout(inputLayout);
    gp->setShaderResourceBindings(shaderResourceBindings);
    gp->setRenderPassDescriptor(renderPassDescriptor);
    gp->create();
 
    return gp;
}
 
std::unique_ptr<QRhiTexture> TextureCopy::copyExternalGlTexture(QSize size, const QMatrix4x4 &externalTexMatrix)
{
    std::unique_ptr<QRhiTexture> tex(m_rhi->newTexture(QRhiTexture::RGBA8, size, 1, QRhiTexture::RenderTarget));
    if (!tex->create()) {
        qWarning("Failed to create frame texture");
        return {};
    }
 
    std::unique_ptr<QRhiTextureRenderTarget> renderTarget(m_rhi->newTextureRenderTarget({ { tex.get() } }));
    std::unique_ptr<QRhiRenderPassDescriptor> renderPassDescriptor(renderTarget->newCompatibleRenderPassDescriptor());
    renderTarget->setRenderPassDescriptor(renderPassDescriptor.get());
    renderTarget->create();
 
    QRhiResourceUpdateBatch *rub = m_rhi->nextResourceUpdateBatch();
    rub->uploadStaticBuffer(m_vertexBuffer.get(), g_quad);
 
    QMatrix4x4 identity;
    char *p = m_uniformBuffer->beginFullDynamicBufferUpdateForCurrentFrame();
    memcpy(p, identity.constData(), 64);
    memcpy(p + 64, externalTexMatrix.constData(), 64);
    float opacity = 1.0f;
    memcpy(p + 64 + 64, &opacity, 4);
    m_uniformBuffer->endFullDynamicBufferUpdateForCurrentFrame();
 
    auto graphicsPipeline = newGraphicsPipeline(m_rhi, m_srb.get(), renderPassDescriptor.get(),
                                                m_vertexShader, m_fragmentShader);
 
    const QRhiCommandBuffer::VertexInput vbufBinding(m_vertexBuffer.get(), 0);
 
    QRhiCommandBuffer *cb = nullptr;
    if (m_rhi->beginOffscreenFrame(&cb) != QRhi::FrameOpSuccess)
        return {};
 
    cb->beginPass(renderTarget.get(), Qt::transparent, { 1.0f, 0 }, rub);
    cb->setGraphicsPipeline(graphicsPipeline.get());
    cb->setViewport({0, 0, float(size.width()), float(size.height())});
    cb->setShaderResources(m_srb.get());
    cb->setVertexInput(0, 1, &vbufBinding);
    cb->draw(4);
    cb->endPass();
    m_rhi->endOffscreenFrame();
 
    QOpenGLContext *ctx = QOpenGLContext::currentContext();
    QOpenGLFunctions *f = ctx->functions();
    static_cast<QOpenGLExtensions *>(f)->flushShared();
 
    return tex;
}
 
static QMatrix4x4 extTransformMatrix(AndroidSurfaceTexture *surfaceTexture)
{
    QMatrix4x4 m = surfaceTexture->getTransformMatrix();
    // flip it back, see
    // http://androidxref.com/9.0.0_r3/xref/frameworks/native/libs/gui/GLConsumer.cpp#866
    // (NB our matrix ctor takes row major)
    static const QMatrix4x4 flipV(1.0f,  0.0f, 0.0f, 0.0f,
                                  0.0f, -1.0f, 0.0f, 1.0f,
                                  0.0f,  0.0f, 1.0f, 0.0f,
                                  0.0f,  0.0f, 0.0f, 1.0f);
    m *= flipV;
    return m;
}
 
#endif
 
class AndroidTextureThread : public QThread
{
    Q_OBJECT
public:
    AndroidTextureThread(QAndroidTextureVideoOutput * vo)
        : QThread()
        , m_videoOutput(vo)
    {
    }
 
    ~AndroidTextureThread() {
        QMetaObject::invokeMethod(this,
                &AndroidTextureThread::clearSurfaceTexture, Qt::BlockingQueuedConnection);
        this->quit();
        this->wait();
    }
 
    void start()
    {
        QThread::start();
        moveToThread(this);
    }
 
#if QT_CONFIG(opengl)
    void initRhi(QOpenGLContext *context)
    {
        QRhiGles2InitParams params;
        params.shareContext = context;
        params.fallbackSurface = QRhiGles2InitParams::newFallbackSurface();
        m_rhi.reset(QRhi::create(QRhi::OpenGLES2, &params));
    }
#endif
 
public Q_SLOTS:
    void onFrameAvailable(quint64 index)
    {
        // Check if 'm_surfaceTexture' is not reset and if the current index is the same that
        // was used for creating connection because there can be pending frames in queue.
        if (m_surfaceTexture && m_surfaceTexture->index() == index) {
#if QT_CONFIG(opengl)
            m_surfaceTexture->updateTexImage();
            auto matrix = extTransformMatrix(m_surfaceTexture.get());
            auto tex = m_textureCopy->copyExternalGlTexture(m_size, matrix);
            auto *buffer = new AndroidTextureVideoBuffer(std::move(tex), m_size);
            QVideoFrame frame(buffer, QVideoFrameFormat(m_size, QVideoFrameFormat::Format_RGBA8888));
            Q_EMIT newFrame(frame);
#endif
        }
    }
 
    void clearFrame() { Q_EMIT newFrame({}); }
 
    void setFrameSize(QSize size) { m_size = size; }
 
    void clearSurfaceTexture()
    {
        m_surfaceTexture.reset();
        m_texture.reset();
        m_textureCopy.reset();
        m_rhi.reset();
    }
 
    AndroidSurfaceTexture *createSurfaceTexture([[maybe_unused]] QRhi *rhi)
    {
        if (m_surfaceTexture)
            return m_surfaceTexture.get();
 
#if QT_CONFIG(opengl)
        QOpenGLContext *ctx = rhi
                ? static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles())->context
                : nullptr;
        initRhi(ctx);
 
        m_texture.reset(m_rhi->newTexture(QRhiTexture::RGBA8, m_size, 1, QRhiTexture::ExternalOES));
        m_texture->create();
        m_surfaceTexture = std::make_unique<AndroidSurfaceTexture>(m_texture->nativeTexture().object);
        if (m_surfaceTexture->surfaceTexture()) {
            const quint64 index = m_surfaceTexture->index();
            connect(m_surfaceTexture.get(), &AndroidSurfaceTexture::frameAvailable, this,
                    [this, index] () { this->onFrameAvailable(index); });
 
            m_textureCopy = std::make_unique<TextureCopy>(m_rhi.get(), m_texture.get());
 
        } else {
            m_texture.reset();
            m_surfaceTexture.reset();
        }
 
        return m_surfaceTexture.get();
#else
        qCritical()
            << "Attempting to use Qt Multimedia VideoOutput on a Qt build without OpenGL support. "
            << "This is not supported by the native Android media backend.";
        m_rhi.reset(QRhi::create(QRhi::Null, nullptr));
        return nullptr;
#endif
    }
 
Q_SIGNALS:
    void newFrame(const QVideoFrame &);
 
private:
    QAndroidTextureVideoOutput * m_videoOutput;
    std::shared_ptr<QRhi> m_rhi;
    std::unique_ptr<AndroidSurfaceTexture> m_surfaceTexture;
    std::unique_ptr<QRhiTexture> m_texture;
    std::unique_ptr<TextureCopy> m_textureCopy;
    QSize m_size;
};
 
QAndroidTextureVideoOutput::QAndroidTextureVideoOutput(QVideoSink *sink, QObject *parent)
    : QAndroidVideoOutput(parent)
    , m_sink(sink)
{
    if (!m_sink) {
        qDebug() << "Cannot create QAndroidTextureVideoOutput without a sink.";
        m_surfaceThread = nullptr;
        return;
    }
 
    startNewSurfaceThread();
}
 
void QAndroidTextureVideoOutput::startNewSurfaceThread()
{
    m_surfaceThread = std::make_shared<AndroidTextureThread>(this);
    connect(m_surfaceThread.get(), &AndroidTextureThread::newFrame,
            this, &QAndroidTextureVideoOutput::newFrame);
    m_surfaceThread->start();
}
 
QAndroidTextureVideoOutput::~QAndroidTextureVideoOutput()
{
    // Make sure that no more VideFrames will be created by surfaceThread
    QMetaObject::invokeMethod(m_surfaceThread.get(),
        &AndroidTextureThread::clearSurfaceTexture, Qt::BlockingQueuedConnection);
}
 
void QAndroidTextureVideoOutput::setSubtitle(const QString &subtitle)
{
    if (m_sink) {
        auto *sink = m_sink->platformVideoSink();
        if (sink)
            sink->setSubtitleText(subtitle);
    }
}
 
bool QAndroidTextureVideoOutput::shouldTextureBeUpdated() const
{
    return m_sink->rhi() && m_surfaceCreatedWithoutRhi;
}
 
AndroidSurfaceTexture *QAndroidTextureVideoOutput::surfaceTexture()
{
    if (!m_sink)
        return nullptr;
 
    AndroidSurfaceTexture *surface = nullptr;
    QMetaObject::invokeMethod(m_surfaceThread.get(), [&]() {
                auto rhi = m_sink->rhi();
                if (!rhi) {
                    m_surfaceCreatedWithoutRhi = true;
                }
                else if (m_surfaceCreatedWithoutRhi) {
                    m_surfaceThread->clearSurfaceTexture();
                    m_surfaceCreatedWithoutRhi = false;
                }
                surface = m_surfaceThread->createSurfaceTexture(rhi);
            },
            Qt::BlockingQueuedConnection);
    return surface;
}
 
void QAndroidTextureVideoOutput::setVideoSize(const QSize &size)
{
    if (m_nativeSize == size)
        return;
 
    m_nativeSize = size;
    QMetaObject::invokeMethod(m_surfaceThread.get(),
            [&](){ m_surfaceThread->setFrameSize(size); },
            Qt::BlockingQueuedConnection);
}
 
void QAndroidTextureVideoOutput::stop()
{
    m_nativeSize = {};
    QMetaObject::invokeMethod(m_surfaceThread.get(), [&](){ m_surfaceThread->clearFrame(); });
}
 
void QAndroidTextureVideoOutput::reset()
{
    if (m_sink)
        m_sink->platformVideoSink()->setVideoFrame({});
    QMetaObject::invokeMethod(m_surfaceThread.get(), &AndroidTextureThread::clearSurfaceTexture);
}
 
void QAndroidTextureVideoOutput::newFrame(const QVideoFrame &frame)
{
    if (m_sink)
        m_sink->setVideoFrame(frame);
}
 
QT_END_NAMESPACE
 
#include "qandroidvideooutput.moc"
#include "moc_qandroidvideooutput_p.cpp"