qavfsamplebufferdelegate.mm

Go to the documentation of this file.

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include <QtFFmpegMediaPluginImpl/private/qavfsamplebufferdelegate_p.h>
 
#define AVMediaType XAVMediaType
extern "C" {
#include <libavutil/hwcontext_videotoolbox.h>
} // extern "C"
#undef AVMediaType
 
#include <QtCore/qloggingcategory.h>
#include <QtCore/qsize.h>
 
#include <QtMultimedia/private/qavfhelpers_p.h>
#include <QtMultimedia/private/qvideoframe_p.h>
 
#include <QtFFmpegMediaPluginImpl/private/qcvimagevideobuffer_p.h>
#include <QtFFmpegMediaPluginImpl/private/qffmpegdarwinhwframehelpers_p.h>
#define AVMediaType XAVMediaType
#include <QtFFmpegMediaPluginImpl/private/qffmpegvideobuffer_p.h>
#include <QtFFmpegMediaPluginImpl/private/qffmpeghwaccel_p.h>
#undef AVMediaType
 
#include <chrono>
#include <optional>
 
Q_STATIC_LOGGING_CATEGORY(qLcAvfSampleBufferDelegate, "qt.multimedia.avf.samplebufferdelegate");
 
using namespace Qt::StringLiterals;
 
QT_USE_NAMESPACE
 
@implementation QT_MANGLE_NAMESPACE(QAVFSampleBufferDelegate) {
@private
    std::function<void(const QVideoFrame &)> frameHandler;
    QFFmpeg::QAVFSampleBufferDelegateTransformProvider transformationProvider;
    std::unique_ptr<QFFmpeg::HWAccel> m_accel;
    std::chrono::microseconds startTime;
    std::optional<std::chrono::microseconds> baseTime;
    qreal frameRate;
}
 
- (instancetype)initWithFrameHandler:(std::function<void(const QVideoFrame &)>)handler
{
    if (!(self = [super init]))
        return nil;
 
    Q_ASSERT(handler);
 
    frameHandler = std::move(handler);
    return self;
}
 
- (void)discardFutureSamples
{
    frameHandler = nullptr;
}
 
- (void)setTransformationProvider:
    (const QFFmpeg::QAVFSampleBufferDelegateTransformProvider &)provider
{
    transformationProvider = std::move(provider);
}
 
- (void)captureOutput:(AVCaptureOutput *)captureOutput
        didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
               fromConnection:(AVCaptureConnection *)connection
{
    Q_UNUSED(captureOutput);
 
    if (!frameHandler)
        return;
 
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    if (!imageBuffer || CFGetTypeID(imageBuffer) != CVPixelBufferGetTypeID()) {
        qWarning() << "Cannot get image buffer from sample buffer";
        return;
    }
 
    auto pixelBuffer = QAVFHelpers::QSharedCVPixelBuffer(
        imageBuffer,
        QAVFHelpers::QSharedCVPixelBuffer::RefMode::NeedsRef);
 
    // The incoming buffer is recycled from a pool owned by AVFoundation, so we
    // copy it into a free-standing CVPixelBuffer that we fully own before
    // keeping it around in a QVideoFrame.
    q23::expected<QAVFHelpers::QSharedCVPixelBuffer, QString> copyResult =
        QFFmpeg::deepCopyCvPixelBuffer(pixelBuffer.get());
    if (!copyResult) {
        qCWarning(qLcAvfSampleBufferDelegate)
            << "Failed to copy incoming pixel buffer:" << copyResult.error();
        return;
    }
    pixelBuffer = std::move(*copyResult);
 
    QSize incomingFrameSize {
        static_cast<int>(CVPixelBufferGetWidth(pixelBuffer.get())),
        static_cast<int>(CVPixelBufferGetHeight(pixelBuffer.get())) };
    Q_ASSERT(!incomingFrameSize.isEmpty());
    CvPixelFormat incomingCvPixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer.get());
 
    Q_ASSERT(m_accel);
    m_accel->updateFramesContext(
        av_map_videotoolbox_format_to_pixfmt(incomingCvPixelFormat),
        incomingFrameSize);
 
    std::chrono::microseconds frameTime =
        QAVFHelpers::CMTimeToMicroseconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer));
    if (!baseTime) {
        baseTime = frameTime;
        startTime = frameTime;
    }
 
    QVideoFrameFormat format = QAVFHelpers::videoFormatForImageBuffer(pixelBuffer.get());
    if (!format.isValid()) {
        qWarning() << "Cannot get get video format for image buffer"
                   << CVPixelBufferGetWidth(pixelBuffer.get()) << 'x'
                   << CVPixelBufferGetHeight(pixelBuffer.get());
        return;
    }
 
    std::optional<QFFmpeg::QAVFSampleBufferDelegateTransform> transform;
    if (transformationProvider) {
        transform = transformationProvider(connection);
        const VideoTransformation &surfaceTransform = transform.value().surfaceTransform;
        format.setRotation(surfaceTransform.rotation);
        format.setMirrored(surfaceTransform.mirroredHorizontallyAfterRotation);
    }
 
    format.setStreamFrameRate(frameRate);
 
    QVideoFrame frame;
    q23::expected<QVideoFrame, QString> frameResult = QFFmpeg::qVideoFrameFromCvPixelBuffer(
        *m_accel,
        startTime - *baseTime,
        pixelBuffer,
        format);
    if (!frameResult)
        qCWarning(qLcAvfSampleBufferDelegate) << frameResult.error();
    else
        frame = *frameResult;
 
    if (!frame.isValid())
        frame = QVideoFramePrivate::createFrame(
            std::make_unique<QFFmpeg::CVImageVideoBuffer>(std::move(pixelBuffer)),
            std::move(format));
 
    if (transform.has_value()) {
        const VideoTransformation &presentationTransform = transform.value().presentationTransform;
        frame.setRotation(presentationTransform.rotation);
        frame.setMirrored(presentationTransform.mirroredHorizontallyAfterRotation);
    }
 
    frame.setStartTime((startTime - *baseTime).count());
    frame.setEndTime((frameTime - *baseTime).count());
    startTime = frameTime;
 
    frameHandler(frame);
}
 
// Sets the initial HWAccel. Should only be called once during
// initialization.
- (void)setHWAccel:(std::unique_ptr<QFFmpeg::HWAccel> &&)accel
{
    m_accel = std::move(accel);
}
 
- (void)setVideoFormatFrameRate:(qreal)rate
{
    frameRate = rate;
}
 
@end