qwasmvideoframegrabber.cpp

Go to the documentation of this file.

// Copyright (C) 2026 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qwasmvideoframegrabber_p.h"
 
#include "qwasmvideooutput_p.h"
#include "qwasmgltexturevideobuffer_p.h"
 
#include <QtMultimedia/qvideosink.h>
#include <QtMultimedia/private/qmemoryvideobuffer_p.h>
#include <QtMultimedia/private/qvideoframe_p.h>
#include <QtGui/qopenglcontext.h>
#include <QtGui/rhi/qrhi_platform.h>
#include <QtCore/qdebug.h>
#include <QtCore/qpointer.h>
#include <QtCore/private/qstdweb_p.h>
 
#include <emscripten/emscripten.h>
#include <emscripten/em_js.h>
#include <emscripten/html5.h>
#include <emscripten/val.h>
 
#include <GLES2/gl2.h>
 
QT_BEGIN_NAMESPACE
 
using namespace emscripten;
using namespace Qt::Literals;
 
// EM_JS pointer arguments are BigInt on wasm64, Number() makes them usable on both targets.
 
// Upload the current video frame to the already-bound TEXTURE_2D.
EM_JS(void, em_texImage2DFromVideo, (const char *videoId, int *widthOut, int *heightOut), {
    videoId = Number(videoId);
    widthOut = Number(widthOut);
    heightOut = Number(heightOut);
    var gl = GL.currentContext.GLctx;
    var video = document.getElementById(UTF8ToString(videoId));
    if (!video) { return; }
    var frame;
    try { frame = new VideoFrame(video); } catch(e) { return; }
    HEAP32[widthOut / 4] = frame.displayWidth;
    HEAP32[heightOut / 4] = frame.displayHeight;
    gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, frame);
    frame.close();
});
 
QWasmVideoFrameGrabber::QWasmVideoFrameGrabber(QWasmVideoOutput *videoOutput)
    : m_videoOutput(videoOutput)
{
}
 
QWasmVideoFrameGrabber::~QWasmVideoFrameGrabber()
{
    // The pending callback holds a raw pointer to this object.
    if (m_animationFrameRequestId)
        emscripten_cancel_animation_frame(m_animationFrameRequestId);
}
 
void QWasmVideoFrameGrabber::detectWebGLContext()
{
    m_glContextHandle = 0;
    m_hasWebGLContext = false;
 
    QRhi *rhi = m_videoOutput->m_wasmSink ? m_videoOutput->m_wasmSink->rhi() : nullptr;
    if (!rhi || rhi->backend() != QRhi::OpenGLES2)
        return;
 
    const auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());
    if (!nativeHandles || !nativeHandles->context)
        return;
    QOpenGLContext *openGLContext = nativeHandles->context;
 
    auto tryGetHandleFromSurface = [&]() -> bool {
        QSurface *surface = openGLContext->surface();
        if (!surface)
            return false;
        // QWasmOpenGLContext::makeCurrent() ends in
        // emscripten_webgl_make_context_current(), so the handle we want is the
        // one that is current afterwards.
        if (QOpenGLContext::currentContext() != openGLContext
            && !openGLContext->makeCurrent(surface))
            return false;
        m_glContextHandle = emscripten_webgl_get_current_context();
        m_hasWebGLContext = (m_glContextHandle != 0);
        return m_hasWebGLContext;
    };
 
    if (!tryGetHandleFromSurface())
        qWarning() << Q_FUNC_INFO << "could not determine the WebGL context for the current RHI";
}
 
// software path, copies VideoFrame data into a QMemoryVideoBuffer
void QWasmVideoFrameGrabber::processVideoFrame()
{
    emscripten::val videoElement = m_videoOutput->currentVideoElement();
 
    // The VideoFrame constructor throws InvalidStateError when the browser compositor
    // has not yet committed the first decoded frame, even if readyState == 4 and
    // videoWidth > 0. Use a JS try-catch so the exception does not propagate into
    // the wasm runtime and abort the application.
    emscripten::val oneVideoFrame = emscripten::val::take_ownership(
            static_cast<EM_VAL>(EM_ASM_PTR({
                try {
                    return Emval.toHandle(new VideoFrame(Emval.toValue($0)));
                } catch(e) {
                    return Emval.toHandle(null);
                }
            }, videoElement.as_handle())));
 
    if (oneVideoFrame.isNull() || oneVideoFrame.isUndefined()) {
        qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "VideoFrame not ready yet, skipping";
        return;
    }
 
    emscripten::val options = emscripten::val::object();
    emscripten::val rectOptions = emscripten::val::object();
 
    int displayWidth = oneVideoFrame["displayWidth"].as<int>();
    int displayHeight = oneVideoFrame["displayHeight"].as<int>();
 
    rectOptions.set("width", displayWidth);
    rectOptions.set("height", displayHeight);
    options.set("rect", rectOptions);
 
    emscripten::val frameBytesAllocationSize = oneVideoFrame.call<emscripten::val>("allocationSize", options);
    emscripten::val frameBuffer =
            emscripten::val::global("Uint8Array").new_(frameBytesAllocationSize);
    // copyTo resolves asynchronously, the output may be gone by then.
    QPointer<QWasmVideoOutput> videoOutput = m_videoOutput;
 
    qstdweb::PromiseCallbacks copyToCallback;
    copyToCallback.thenFunc = [videoOutput, oneVideoFrame, frameBuffer,
                                displayWidth, displayHeight]
            (emscripten::val frameLayout)
    {
        if (!videoOutput) {
            oneVideoFrame.call<emscripten::val>("close");
            return;
        }
        if (frameLayout.isNull() || frameLayout.isUndefined()) {
            qCDebug(qWasmMediaVideoOutput) << "theres no frameLayout";
            return;
        }
 
        // frameBuffer now has a new frame, send to Qt
        const QSize frameSize(displayWidth, displayHeight);
 
        QByteArray frameBytes = QByteArray::fromEcmaUint8Array(frameBuffer);
 
        QVideoFrameFormat::PixelFormat pixelFormat =
                fromJsPixelFormat(oneVideoFrame["format"].as<std::string>());
        if (pixelFormat == QVideoFrameFormat::Format_Invalid)
            pixelFormat = QVideoFrameFormat::Format_RGBA8888;
        QVideoFrameFormat frameFormat = QVideoFrameFormat(frameSize, pixelFormat);
 
        if (videoOutput->m_useCameraRotation)
            frameFormat.setRotation(videoOutput->m_rotateBy);
        if (videoOutput->m_streamFrameRate > 0)
            frameFormat.setStreamFrameRate(videoOutput->m_streamFrameRate);
        auto buffer = std::make_unique<QMemoryVideoBuffer>(
                std::move(frameBytes),
                frameLayout[0]["stride"].as<int>());
 
        QVideoFrame vFrame =
                QVideoFramePrivate::createFrame(std::move(buffer), std::move(frameFormat));
 
        if (!videoOutput->m_wasmSink) {
            qWarning() << "ERROR ALERT!! video sink not set";
            return;
        }
        videoOutput->m_wasmSink->setVideoFrame(vFrame);
        oneVideoFrame.call<emscripten::val>("close");
    };
    copyToCallback.catchFunc = [oneVideoFrame](emscripten::val error)
    {
        qCDebug(qWasmMediaVideoOutput) << "copyTo error"
                               << QString::fromStdString(error["name"].as<std::string>())
                               << QString::fromStdString(error["message"].as<std::string>());
        oneVideoFrame.call<emscripten::val>("close");
    };
 
    qstdweb::Promise::make(oneVideoFrame, u"copyTo"_s, std::move(copyToCallback), frameBuffer, options);
}
 
// zero-readback path, uploads the VideoFrame straight into a GL texture
void QWasmVideoFrameGrabber::processWebGLVideoFrame()
{
    emscripten_webgl_make_context_current(m_glContextHandle);
 
    GLuint rawTextureId = 0;
    glGenTextures(1, &rawTextureId);
    QGlTextureHandle textureHandle{ rawTextureId };
 
    glBindTexture(GL_TEXTURE_2D, textureHandle.get());
 
    int width = 0, height = 0;
    em_texImage2DFromVideo(m_videoOutput->m_videoSurfaceId.c_str(), &width, &height);
 
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    glBindTexture(GL_TEXTURE_2D, 0);
 
    if (!textureHandle || width == 0 || height == 0) {
        qCWarning(qWasmMediaVideoOutput) << "VideoFrame upload failed";
        return;
    }
 
    std::unique_ptr<QHwVideoBuffer> hwBuffer =
            std::make_unique<QWasmGLTextureVideoBuffer>(
                    std::move(textureHandle), QSize(width, height),
                    m_glContextHandle,
                    m_videoOutput->m_wasmSink ? m_videoOutput->m_wasmSink->rhi() : nullptr);
 
    QVideoFrameFormat frameFormat(QSize(width, height), QVideoFrameFormat::Format_RGBA8888);
    if (m_videoOutput->m_streamFrameRate > 0)
        frameFormat.setStreamFrameRate(m_videoOutput->m_streamFrameRate);
    QVideoFrame vFrame =
            QVideoFramePrivate::createFrame(std::move(hwBuffer), std::move(frameFormat));
 
    m_videoOutput->m_wasmSink->setVideoFrame(vFrame);
}
 
void QWasmVideoFrameGrabber::startFrameLoop()
{
    if (!m_webGLContextChecked) {
        m_webGLContextChecked = true;
        detectWebGLContext();
    }
 
    m_videoOutput->detectIosCameraRotation();
 
    // canplaythrough and playing both land here, one loop per grabber is enough.
    if (m_animationFrameRequestId)
        return;
 
    // Single-shot callback: re-registers each frame so multiple QWasmVideoOutput
    // instances can coexist. emscripten_request_animation_frame_loop allows only one
    // active loop globally and would cancel another instance.
    static EM_BOOL (*frame)(double, void *) = [](double frameTime, void *context) -> EM_BOOL {
 
        Q_UNUSED(frameTime);
 
        QWasmVideoFrameGrabber *frameGrabber = reinterpret_cast<QWasmVideoFrameGrabber *>(context);
        frameGrabber->m_animationFrameRequestId = 0;
        QWasmVideoOutput *videoOutput = frameGrabber->m_videoOutput;
        if (!videoOutput || videoOutput->m_isStopped) {
            qCWarning(qWasmMediaVideoOutput) << "frame loop exit: isStopped=" << (videoOutput ? videoOutput->m_isStopped : true)
                                             << "mode=" << (videoOutput ? videoOutput->m_currentVideoMode : -1);
            return false;
        }
 
        if (videoOutput->m_currentVideoMode == QWasmVideoOutput::VideoDisplay
            && videoOutput->m_currentMediaStatus != QWasmVideoOutput::MediaStatus::LoadedMedia) {
            frameGrabber->m_animationFrameRequestId = emscripten_request_animation_frame(frame, context);
            return true;
        }
 
        emscripten::val videoElement = videoOutput->currentVideoElement();
        if (videoElement.isNull() || videoElement.isUndefined()) {
            qCWarning(qWasmMediaVideoOutput) << "frame loop exit: video element null, mode=" << videoOutput->m_currentVideoMode;
            return false;
        }
 
        if (videoElement["paused"].as<bool>() || videoElement["ended"].as<bool>()
            || videoElement["readyState"].as<int>() < 2) {
            qCDebug(qWasmMediaVideoOutput) << "frame loop waiting: mode=" << videoOutput->m_currentVideoMode
                                           << "paused=" << videoElement["paused"].as<bool>()
                                           << "ended=" << videoElement["ended"].as<bool>()
                                           << "readyState=" << videoElement["readyState"].as<int>();
            frameGrabber->m_animationFrameRequestId = emscripten_request_animation_frame(frame, context);
            return true;
        }
 
        if (frameGrabber->m_glContextHandle)
            frameGrabber->processWebGLVideoFrame();
        else
            frameGrabber->processVideoFrame();
 
        frameGrabber->m_animationFrameRequestId = emscripten_request_animation_frame(frame, context);
        return true;
    };
    if ((!m_videoOutput->m_isStopped
         && m_videoOutput->m_video["className"].as<std::string>() == "Camera"
         && m_videoOutput->m_cameraIsReady)
        || (!m_videoOutput->m_isStopped
            && m_videoOutput->m_currentVideoMode == QWasmVideoOutput::SurfaceCapture)
        || m_videoOutput->isReady())
        m_animationFrameRequestId = emscripten_request_animation_frame(frame, this);
}
 
QVideoFrameFormat::PixelFormat QWasmVideoFrameGrabber::fromJsPixelFormat(std::string_view videoFormat)
{
    if (videoFormat == "I420")
        return QVideoFrameFormat::Format_YUV420P;
    // no equivalent pixel format
    //   else if (videoFormat == "I420A") // AYUV ?
    else if (videoFormat == "I422")
        return QVideoFrameFormat::Format_YUV422P;
    // no equivalent pixel format
    //     else if (videoFormat == "I444")
    else if (videoFormat == "NV12")
        return QVideoFrameFormat::Format_NV12;
    else if (videoFormat == "RGBA")
        return QVideoFrameFormat::Format_RGBA8888;
    else if (videoFormat == "RGBX")
        return QVideoFrameFormat::Format_RGBX8888;
    else if (videoFormat == "BGRA")
        return QVideoFrameFormat::Format_BGRA8888;
    else if (videoFormat == "BGRX")
        return QVideoFrameFormat::Format_BGRX8888;
 
    return QVideoFrameFormat::Format_Invalid;
}
 
QT_END_NAMESPACE