qffmpegdarwinhwframehelpers.mm

Go to the documentation of this file.

// Copyright (C) 2026 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qffmpegdarwinhwframehelpers_p.h"
 
#include <QtCore/qscopeguard.h>
 
#define AVMediaType XAVMediaType
#include <QtFFmpegMediaPluginImpl/private/qffmpegvideobuffer_p.h>
#include <QtFFmpegMediaPluginImpl/private/qffmpeghwaccel_p.h>
#undef AVMediaType
 
#include <QtMultimedia/private/qvideoframe_p.h>
 
#include <CoreVideo/CVPixelBuffer.h>
#include <VideoToolbox/VTPixelTransferSession.h>
 
QT_BEGIN_NAMESPACE
 
using namespace Qt::StringLiterals;
 
namespace QFFmpeg {
 
namespace {
 
// Helper function to allocate a FFmpeg HwFrame, from a given CVPixelBufferRef.
// The FFmpeg HwFrame becomes a owning reference for this CVPixelBufferRef.
// Make sure this is compatible with the layout used in ffmpeg's hwcontext_videotoolbox
[[nodiscard]] q23::expected<AVFrameUPtr, QString> allocHWFrame(
    AVBufferRef *hwContext,
    QAVFHelpers::QSharedCVPixelBuffer sharedPixBuf)
{
    Q_ASSERT(sharedPixBuf);
 
    AVHWFramesContext *ctx = (AVHWFramesContext *)hwContext->data;
 
    if (QAVFHelpers::CVPixelBufferSize(sharedPixBuf.get()) != QSize(ctx->width, ctx->height))
        return q23::unexpected{
            u"Size of given CVPixelBufferRef does not match the FFmpeg hw frame context"_s };
 
    auto frame = QFFmpeg::makeAVFrame();
    if (!frame)
        return q23::unexpected{ u"Failed to allocate FFmpeg AVFrame"_s };
 
    frame->hw_frames_ctx = av_buffer_ref(hwContext);
    frame->extended_data = frame->data;
 
    CVPixelBufferRef pixbuf = sharedPixBuf.release();
    auto releasePixBufFn = [](void* opaquePtr, uint8_t *) {
        CVPixelBufferRelease(static_cast<CVPixelBufferRef>(opaquePtr));
    };
    frame->buf[0] = av_buffer_create(nullptr, 0, releasePixBufFn, pixbuf, 0);
 
    // It is convention to use 4th data plane for hardware frames.
    frame->data[3] = (uint8_t *)pixbuf;
    frame->width = ctx->width;
    frame->height = ctx->height;
    frame->format = AV_PIX_FMT_VIDEOTOOLBOX;
    return frame;
}
 
} // Anonymous namespace end
 
q23::expected<QAVFHelpers::QSharedCVPixelBuffer, QString> deepCopyCvPixelBuffer(
    CVPixelBufferRef source)
{
    Q_ASSERT(source);
    const QSize sourceSize = QAVFHelpers::CVPixelBufferSize(source);
    Q_ASSERT(!sourceSize.isEmpty());
    Q_ASSERT(CVPixelBufferGetPixelFormatType(source) != CvPixelFormatInvalid);
 
    // We request an IOSurface backing (and Metal compatibility) so the
    // transfer happens on the GPU and the buffer stays usable as a
    // hardware video frame downstream.
    NSDictionary *attributes = @{
        (id)kCVPixelBufferIOSurfacePropertiesKey : @{},
        (id)kCVPixelBufferMetalCompatibilityKey : @YES,
    };
 
    CVPixelBufferRef destination = nullptr;
    const CVReturn createResult = CVPixelBufferCreate(
        kCFAllocatorDefault,
        sourceSize.width(),
        sourceSize.height(),
        CVPixelBufferGetPixelFormatType(source),
        (__bridge CFDictionaryRef)attributes,
        &destination);
    if (createResult != kCVReturnSuccess || !destination)
        return q23::unexpected{
            u"Failed to allocate destination CVPixelBuffer (CVReturn %1)"_s.arg(createResult) };
 
    // Adopt ownership immediately so the buffer is released on every error path.
    QAVFHelpers::QSharedCVPixelBuffer destinationBuffer {
        destination,
        QAVFHelpers::QSharedCVPixelBuffer::RefMode::HasRef };
 
    // Creating a transfer session per frame is acceptable but not free; if this
    // shows up in profiles it can be hoisted into the per-stream object and
    // reused across frames (a single session handles varying sizes/formats).
    VTPixelTransferSessionRef transferSession = nullptr;
    const OSStatus sessionResult = VTPixelTransferSessionCreate(
        kCFAllocatorDefault,
        &transferSession);
    if (sessionResult != noErr || !transferSession)
        return q23::unexpected{
            u"Failed to create VTPixelTransferSession (OSStatus %1)"_s.arg(sessionResult) };
    auto sessionGuard = qScopeGuard([&] {
        VTPixelTransferSessionInvalidate(transferSession);
        CFRelease(transferSession);
    });
 
    const OSStatus transferResult =
        VTPixelTransferSessionTransferImage(transferSession, source, destinationBuffer.get());
    if (transferResult != noErr)
        return q23::unexpected{
            u"VTPixelTransferSessionTransferImage failed (OSStatus %1)"_s.arg(transferResult) };
 
    // The freshly created destination has no attachments; carry over the
    // source's colorimetry (YCbCr matrix, transfer function, ...) so that
    // QAVFHelpers::videoFormatForImageBuffer reports the same format as for the
    // original buffer.
    CVBufferPropagateAttachments(source, destinationBuffer.get());
 
    return destinationBuffer;
}
 
q23::expected<QVideoFrame, QString> qVideoFrameFromCvPixelBuffer(
    const QFFmpeg::HWAccel &hwAccel,
    std::chrono::microseconds presentationTimeStamp,
    const QAVFHelpers::QSharedCVPixelBuffer &imageBuffer,
    QVideoFrameFormat format)
{
    q23::expected<AVFrameUPtr, QString> avFrameResult = allocHWFrame(
        hwAccel.hwFramesContextAsBuffer(),
        imageBuffer);
    if (!avFrameResult)
        return q23::unexpected{ u"Failed to allocate FFmpeg HwFrame"_s };
    AVFrameUPtr &avFrame = *avFrameResult;
 
    avFrame->pts = presentationTimeStamp.count();
 
    return QVideoFramePrivate::createFrame(
        std::make_unique<QFFmpegVideoBuffer>(std::move(avFrame)),
        format);
}
 
} // namespace QFFmpeg
 
QT_END_NAMESPACE