qpipewire_videostream.cpp

Go to the documentation of this file.

// Copyright (C) 2026 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qpipewire_videostream_p.h"
 
#include <QtMultimedia/private/qmemoryvideobuffer_p.h>
#include <QtMultimedia/private/qpipewire_propertydict_p.h>
#include <QtMultimedia/private/qpipewire_videoformat_support_p.h>
#include <QtMultimedia/private/qvideoframe_p.h>
#include <QtCore/qloggingcategory.h>
 
#include <spa/param/video/format-utils.h>
#include <spa/pod/builder.h>
 
#if __has_include(<spa/param/buffers.h>)
#  include <spa/param/buffers.h>
#else
#  include <QtMultimedia/private/qpipewire_spa_compat_p.h>
#endif
 
#include <array>
 
QT_BEGIN_NAMESPACE
 
namespace QtPipeWire {
 
using namespace Qt::StringLiterals;
 
Q_STATIC_LOGGING_CATEGORY(lcPipewireCameraStream, "qt.multimedia.pipewire.camera.stream")
 
namespace {
 
// The raw formats a camera realistically produces, offered when the caller did
// not ask for one in particular.
constexpr std::array cameraPixelFormats{
    SPA_VIDEO_FORMAT_NV12, SPA_VIDEO_FORMAT_YUY2, SPA_VIDEO_FORMAT_UYVY,
    SPA_VIDEO_FORMAT_I420, SPA_VIDEO_FORMAT_RGB,  SPA_VIDEO_FORMAT_BGR,
    SPA_VIDEO_FORMAT_RGBA, SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_GRAY8,
};
 
} // namespace
 
QPipeWireVideoStream::QPipeWireVideoStream(QCameraContextManager &context, QByteArray targetObject,
                                           QVideoFrameFormat::PixelFormat requestedPixelFormat,
                                           QSize requestedSize, qreal requestedFrameRate)
    : m_context(context),
      m_targetObject(std::move(targetObject)),
      m_requestedPixelFormat(requestedPixelFormat),
      m_requestedSize(requestedSize),
      m_requestedFrameRate(requestedFrameRate)
{
}
 
QPipeWireVideoStream::~QPipeWireVideoStream()
{
    if (!m_stream)
        return;
 
    m_context.pwInstance().runWithEventLoopLock([&] {
        pw_stream_disconnect(m_stream.get());
        spa_hook_remove(&m_streamListener);
        m_stream = {};
    });
}
 
const pw_stream_events &QPipeWireVideoStream::streamEvents()
{
    QT_WARNING_PUSH
    // QTBUG-129587: libpipewire 1.2.5 grew fields we do not set
    QT_WARNING_DISABLE_GCC("-Wmissing-field-initializers")
    QT_WARNING_DISABLE_CLANG("-Wmissing-field-initializers")
    // clang-format off
    static constexpr pw_stream_events events = {
        .version = PW_VERSION_STREAM_EVENTS,
        .state_changed = [](void *data, pw_stream_state o, pw_stream_state s, const char *err) {
            reinterpret_cast<QPipeWireVideoStream *>(data)->onStateChanged(o, s, err);
        },
        .param_changed = [](void *data, uint32_t id, const spa_pod *param) {
            reinterpret_cast<QPipeWireVideoStream *>(data)->onParamChanged(id, param);
        },
        .process = [](void *data) {
            reinterpret_cast<QPipeWireVideoStream *>(data)->onProcess();
        },
    };
    // clang-format on
    QT_WARNING_POP
 
    return events;
}
 
bool QPipeWireVideoStream::start()
{
    Q_ASSERT(!m_stream);
 
    if (!m_context.isConnected())
        return false;
 
    const spa_dict_item items[] = {
        SPA_DICT_ITEM_INIT(PW_KEY_MEDIA_TYPE, "Video"),
        SPA_DICT_ITEM_INIT(PW_KEY_MEDIA_CATEGORY, "Capture"),
        SPA_DICT_ITEM_INIT(PW_KEY_MEDIA_ROLE, "Camera"),
        SPA_DICT_ITEM_INIT(PW_KEY_TARGET_OBJECT, m_targetObject.constData()),
    };
 
    return m_context.pwInstance().runWithEventLoopLock([&] {
        PwPropertiesHandle props = makeProperties(items);
 
        m_stream = PwStreamHandle{
            pw_stream_new(m_context.coreConnection().get(), "qt-camera", props.release()),
        };
        if (!m_stream) {
            qCWarning(lcPipewireCameraStream) << "pw_stream_new failed";
            return false;
        }
 
        m_streamListener = {};
        pw_stream_add_listener(m_stream.get(), &m_streamListener, &streamEvents(), this);
 
        std::array<uint8_t, 1024> buffer{};
        spa_pod_builder builder = SPA_POD_BUILDER_INIT(buffer.data(), buffer.size());
 
        spa_pod_frame frame{};
        spa_pod_builder_push_object(&builder, &frame, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat);
        spa_pod_builder_add(&builder, SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), 0);
 
        switch (m_requestedPixelFormat) {
        case QVideoFrameFormat::Format_Jpeg: {
            spa_pod_builder_add(&builder, SPA_FORMAT_mediaSubtype,
                                SPA_POD_Id(SPA_MEDIA_SUBTYPE_mjpg), 0);
            break;
        }
        default: {
            spa_pod_builder_add(&builder, SPA_FORMAT_mediaSubtype,
                                SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), 0);
 
            spa_video_format requestedPixelFormatSpa = toSpaVideoFormat(m_requestedPixelFormat);
            switch (requestedPixelFormatSpa) {
            case SPA_VIDEO_FORMAT_UNKNOWN: {
                // advertise all alternatives, letting PipeWire pick one of the camera's supported formats
                spa_pod_builder_prop(&builder, SPA_FORMAT_VIDEO_format, 0);
                spa_pod_frame choice{};
                spa_pod_builder_push_choice(&builder, &choice, SPA_CHOICE_Enum, 0);
                // The first entry is both the default and one of the alternatives.
                spa_pod_builder_id(&builder, cameraPixelFormats.front());
                for (spa_video_format format : cameraPixelFormats)
                    spa_pod_builder_id(&builder, format);
                spa_pod_builder_pop(&builder, &choice);
                break;
            }
 
            default: {
                spa_pod_builder_add(&builder, SPA_FORMAT_VIDEO_format,
                                    SPA_POD_Id(toSpaVideoFormat(m_requestedPixelFormat)), 0);
                break;
            }
            }
            break;
        }
        }
 
        if (!m_requestedSize.isEmpty()) {
            const spa_rectangle exactSize = toSpaRectangle(m_requestedSize);
            spa_pod_builder_add(&builder, SPA_FORMAT_VIDEO_size, SPA_POD_Rectangle(&exactSize), 0);
        } else {
            constexpr spa_rectangle minSize{ .width = 1, .height = 1 };
            constexpr spa_rectangle maxSize{ .width = 8192, .height = 8192 };
 
            spa_pod_builder_add(&builder, SPA_FORMAT_VIDEO_size,
                                SPA_POD_CHOICE_RANGE_Rectangle(&maxSize, &minSize, &maxSize), 0);
        }
 
        // Cameras are treated as variable rate with the requested rate as the
        // upper bound
        constexpr spa_fraction minRate{ .num = 0, .denom = 1 };
        const FrameRate rate = rateFromFps(m_requestedFrameRate > 0. ? m_requestedFrameRate : 30.);
        spa_pod_builder_add(&builder, SPA_FORMAT_VIDEO_framerate,
                            SPA_POD_CHOICE_RANGE_Fraction(&rate.frac, &minRate, &rate.frac), 0);
 
        // Advertise MemFd alongside the default MemPtr: a real V4L2 camera
        // node hands out memfd-backed buffers, and without this the link
        // fails with "error use input buffers: -22" while a synthetic
        // source (e.g. a libcamera test node) that happens to use MemPtr
        // still works, masking the problem.
        spa_pod_frame buffersFrame{};
        spa_pod_builder_push_object(&builder, &buffersFrame, SPA_TYPE_OBJECT_ParamBuffers,
                                    SPA_PARAM_Buffers);
        spa_pod_builder_add(
                &builder, SPA_PARAM_BUFFERS_dataType,
                SPA_POD_CHOICE_FLAGS_Int((1u << SPA_DATA_MemPtr) | (1u << SPA_DATA_MemFd)), 0);
 
        const spa_pod *params[] = {
            static_cast<const spa_pod *>(spa_pod_builder_pop(&builder, &frame)),
            static_cast<const spa_pod *>(spa_pod_builder_pop(&builder, &buffersFrame)),
        };
 
        const int status = pw_stream_connect(
                m_stream.get(), PW_DIRECTION_INPUT, PW_ID_ANY,
                pw_stream_flags(PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_MAP_BUFFERS), params,
                2);
        if (status < 0) {
            qCWarning(lcPipewireCameraStream)
                    << "pw_stream_connect failed:" << make_error_code(-status).message();
            spa_hook_remove(&m_streamListener);
            m_stream = {};
            return false;
        }
 
        return true;
    });
}
 
QVideoFrameFormat QPipeWireVideoStream::frameFormat() const
{
    std::lock_guard guard{ m_formatMutex };
    return m_frameFormat;
}
 
void QPipeWireVideoStream::onStateChanged(pw_stream_state old, pw_stream_state state,
                                          const char *error)
{
    qCDebug(lcPipewireCameraStream) << "stream state" << pw_stream_state_as_string(old) << "->"
                                    << pw_stream_state_as_string(state)
                                    << (error ? QString::fromUtf8(error) : QString());
 
    if (state == PW_STREAM_STATE_ERROR) {
        emit errorOccurred(error ? QString::fromUtf8(error)
                                  : u"The PipeWire camera stream failed"_s);
    }
}
 
void QPipeWireVideoStream::onParamChanged(uint32_t id, const spa_pod *param)
{
    if (!param || id != SPA_PARAM_Format)
        return;
 
    spa_video_info info{};
    if (spa_format_parse(param, &info.media_type, &info.media_subtype) < 0)
        return;
 
    if (info.media_type != SPA_MEDIA_TYPE_video)
        return;
 
    QSize size;
    QVideoFrameFormat::PixelFormat pixelFormat = QVideoFrameFormat::Format_Invalid;
    spa_fraction framerate{};
 
    switch (info.media_subtype) {
    case SPA_MEDIA_SUBTYPE_raw: {
        if (spa_format_video_raw_parse(param, &info.info.raw) < 0)
            return;
        size = QSize(int(info.info.raw.size.width), int(info.info.raw.size.height));
        pixelFormat = toQtPixelFormat(info.info.raw.format);
        framerate = info.info.raw.framerate;
        break;
    }
    case SPA_MEDIA_SUBTYPE_mjpg: {
        if (spa_format_video_mjpg_parse(param, &info.info.mjpg) < 0)
            return;
        size = QSize(int(info.info.mjpg.size.width), int(info.info.mjpg.size.height));
        pixelFormat = QVideoFrameFormat::Format_Jpeg;
        framerate = info.info.mjpg.framerate;
        break;
    }
    default:
        return;
    }
 
    qCDebug(lcPipewireCameraStream)
            << "negotiated" << pixelFormat << size << framerate.num << "/" << framerate.denom;
 
    if (pixelFormat == QVideoFrameFormat::Format_Invalid) {
        qCWarning(lcPipewireCameraStream)
                << "the server picked a pixel format we cannot represent, media subtype"
                << info.media_subtype;
        Q_EMIT errorOccurred(u"The PipeWire camera stream picked an unsupported pixel format"_s);
    }
 
    QVideoFrameFormat frameFormat(size, pixelFormat);
    if (framerate.denom != 0)
        frameFormat.setStreamFrameRate(qreal(framerate.num) / qreal(framerate.denom));
 
    std::lock_guard guard{ m_formatMutex };
    m_frameFormat = std::move(frameFormat);
}
 
void QPipeWireVideoStream::onProcess()
{
    pw_buffer *pwBuffer = pw_stream_dequeue_buffer(m_stream.get());
    if (!pwBuffer) {
        // The producer is ahead of us; dropping is better than blocking the
        // graph.
        qCDebug(lcPipewireCameraStream) << "no buffer available";
        return;
    }
 
    const QVideoFrameFormat frameFormat = this->frameFormat();
 
    spa_buffer *buffer = pwBuffer->buffer;
 
    if (frameFormat.isValid() && buffer->n_datas > 0 && buffer->datas[0].data) {
        const spa_chunk *chunk = buffer->datas[0].chunk;
 
        // A compressed format has no fixed stride; guessing one from the
        // chunk size would be meaningless.
        int stride = chunk->stride;
        if (stride == 0 && frameFormat.pixelFormat() != QVideoFrameFormat::Format_Jpeg) {
            const int height = frameFormat.frameHeight();
            if (height > 0)
                stride = int(chunk->size / quint32(height));
        }
 
        QByteArray data{ static_cast<const char *>(buffer->datas[0].data) + chunk->offset,
                         qsizetype(chunk->size) };
 
        QVideoFrame frame = QVideoFramePrivate::createFrame(
                std::make_unique<QMemoryVideoBuffer>(std::move(data), stride), frameFormat);
 
        if (const auto *header = static_cast<const spa_meta_header *>(
                    spa_buffer_find_meta_data(buffer, SPA_META_Header, sizeof(spa_meta_header)))) {
            // pts is in nanoseconds since an arbitrary epoch.
            frame.setStartTime(qint64(header->pts) / 1000);
        }
 
        emit newVideoFrame(frame);
    }
 
    pw_stream_queue_buffer(m_stream.get(), pwBuffer);
}
 
} // namespace QtPipeWire
 
QT_END_NAMESPACE