qvideoframe.cpp

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qvideoframe.h"
 
#include "qvideoframe_p.h"
#include "qvideotexturehelper_p.h"
#include "qmultimediautils_p.h"
#include "qmemoryvideobuffer_p.h"
#include "qvideoframeconverter_p.h"
#include "qimagevideobuffer_p.h"
#include "qpainter.h"
#include <qtextlayout.h>
 
#include <qimage.h>
#include <qpair.h>
#include <qsize.h>
#include <qvariant.h>
#include <rhi/qrhi.h>
 
#include <QDebug>
 
QT_BEGIN_NAMESPACE
 
QT_DEFINE_QESDP_SPECIALIZATION_DTOR(QVideoFramePrivate);
 
QVideoFrame::QVideoFrame()
{
}
 
QVideoFrame::QVideoFrame(QAbstractVideoBuffer *buffer, const QVideoFrameFormat &format)
    : d(new QVideoFramePrivate(format))
{
    d->buffer.reset(buffer);
}
 
QAbstractVideoBuffer *QVideoFrame::videoBuffer() const
{
    return d ? d->buffer.get() : nullptr;
}
 
QVideoFrame::QVideoFrame(const QVideoFrameFormat &format)
    : d(new QVideoFramePrivate(format))
{
    auto *textureDescription = QVideoTextureHelper::textureDescription(format.pixelFormat());
    qsizetype bytes = textureDescription->bytesForSize(format.frameSize());
    if (bytes > 0) {
        QByteArray data;
        data.resize(bytes);
 
        // Check the memory was successfully allocated.
        if (!data.isEmpty())
            d->buffer = std::make_unique<QMemoryVideoBuffer>(data, textureDescription->strideForWidth(format.frameWidth()));
    }
}
 
QVideoFrame::QVideoFrame(const QImage &image)
{
    auto buffer = std::make_unique<QImageVideoBuffer>(image);
 
    // If the QImage::Format is not convertible to QVideoFrameFormat,
    // QImageVideoBuffer automatically converts image to a compatible
    // (A)RGB format.
    const QImage &bufferImage = buffer->underlyingImage();
 
    if (bufferImage.isNull())
        return;
 
    // `bufferImage` is now supported by QVideoFrameFormat::pixelFormatFromImageFormat()
    QVideoFrameFormat format = {
        bufferImage.size(), QVideoFrameFormat::pixelFormatFromImageFormat(bufferImage.format())
    };
 
    Q_ASSERT(format.isValid());
 
    d = new QVideoFramePrivate{ std::move(format), std::move(buffer) };
}
 
QVideoFrame::QVideoFrame(const QVideoFrame &other) = default;
 
QVideoFrame &QVideoFrame::operator =(const QVideoFrame &other) = default;
 
bool QVideoFrame::operator==(const QVideoFrame &other) const
{
    // Due to explicit sharing we just compare the QSharedData which in turn compares the pointers.
    return d == other.d;
}
 
bool QVideoFrame::operator!=(const QVideoFrame &other) const
{
    return d != other.d;
}
 
QVideoFrame::~QVideoFrame() = default;
 
bool QVideoFrame::isValid() const
{
    return (d && d->buffer) && d->format.pixelFormat() != QVideoFrameFormat::Format_Invalid;
}
 
QVideoFrameFormat::PixelFormat QVideoFrame::pixelFormat() const
{
    return d ? d->format.pixelFormat() : QVideoFrameFormat::Format_Invalid;
}
 
QVideoFrameFormat QVideoFrame::surfaceFormat() const
{
    return d ? d->format : QVideoFrameFormat{};
}
 
QVideoFrame::HandleType QVideoFrame::handleType() const
{
    return (d && d->buffer) ? d->buffer->handleType() : QVideoFrame::NoHandle;
}
 
QSize QVideoFrame::size() const
{
    return d ? d->format.frameSize() : QSize();
}
 
int QVideoFrame::width() const
{
    return size().width();
}
 
int QVideoFrame::height() const
{
    return size().height();
}
 
bool QVideoFrame::isMapped() const
{
    return d && d->buffer && d->buffer->mapMode() != QVideoFrame::NotMapped;
}
 
bool QVideoFrame::isWritable() const
{
    return d && d->buffer && (d->buffer->mapMode() & QVideoFrame::WriteOnly);
}
 
bool QVideoFrame::isReadable() const
{
    return d && d->buffer && (d->buffer->mapMode() & QVideoFrame::ReadOnly);
}
 
QVideoFrame::MapMode QVideoFrame::mapMode() const
{
    return (d && d->buffer) ? d->buffer->mapMode() : QVideoFrame::NotMapped;
}
 
bool QVideoFrame::map(QVideoFrame::MapMode mode)
{
 
    if (!d || !d->buffer)
        return false;
 
    QMutexLocker lock(&d->mapMutex);
    if (mode == QVideoFrame::NotMapped)
        return false;
 
    if (d->mappedCount > 0) {
        //it's allowed to map the video frame multiple times in read only mode
        if (d->buffer->mapMode() == QVideoFrame::ReadOnly
                && mode == QVideoFrame::ReadOnly) {
            d->mappedCount++;
            return true;
        }
 
        return false;
    }
 
    Q_ASSERT(d->mapData.data[0] == nullptr);
    Q_ASSERT(d->mapData.bytesPerLine[0] == 0);
    Q_ASSERT(d->mapData.nPlanes == 0);
    Q_ASSERT(d->mapData.size[0] == 0);
 
    d->mapData = d->buffer->map(mode);
    if (d->mapData.nPlanes == 0)
        return false;
 
    if (d->mapData.nPlanes == 1) {
        auto pixelFmt = d->format.pixelFormat();
        // If the plane count is 1 derive the additional planes for planar formats.
        switch (pixelFmt) {
        case QVideoFrameFormat::Format_Invalid:
        case QVideoFrameFormat::Format_ARGB8888:
        case QVideoFrameFormat::Format_ARGB8888_Premultiplied:
        case QVideoFrameFormat::Format_XRGB8888:
        case QVideoFrameFormat::Format_BGRA8888:
        case QVideoFrameFormat::Format_BGRA8888_Premultiplied:
        case QVideoFrameFormat::Format_BGRX8888:
        case QVideoFrameFormat::Format_ABGR8888:
        case QVideoFrameFormat::Format_XBGR8888:
        case QVideoFrameFormat::Format_RGBA8888:
        case QVideoFrameFormat::Format_RGBX8888:
        case QVideoFrameFormat::Format_AYUV:
        case QVideoFrameFormat::Format_AYUV_Premultiplied:
        case QVideoFrameFormat::Format_UYVY:
        case QVideoFrameFormat::Format_YUYV:
        case QVideoFrameFormat::Format_Y8:
        case QVideoFrameFormat::Format_Y16:
        case QVideoFrameFormat::Format_Jpeg:
        case QVideoFrameFormat::Format_SamplerExternalOES:
        case QVideoFrameFormat::Format_SamplerRect:
            // Single plane or opaque format.
            break;
        case QVideoFrameFormat::Format_YUV420P:
        case QVideoFrameFormat::Format_YUV420P10:
        case QVideoFrameFormat::Format_YUV422P:
        case QVideoFrameFormat::Format_YV12: {
            // The UV stride is usually half the Y stride and is 32-bit aligned.
            // However it's not always the case, at least on Windows where the
            // UV planes are sometimes not aligned.
            // We calculate the stride using the UV byte count to always
            // have a correct stride.
            const int height = this->height();
            const int yStride = d->mapData.bytesPerLine[0];
            const int uvHeight = pixelFmt == QVideoFrameFormat::Format_YUV422P ? height : height / 2;
            const int uvStride = (d->mapData.size[0] - (yStride * height)) / uvHeight / 2;
 
            // Three planes, the second and third vertically (and horizontally for other than Format_YUV422P formats) subsampled.
            d->mapData.nPlanes = 3;
            d->mapData.bytesPerLine[2] = d->mapData.bytesPerLine[1] = uvStride;
            d->mapData.size[0] = yStride * height;
            d->mapData.size[1] = uvStride * uvHeight;
            d->mapData.size[2] = uvStride * uvHeight;
            d->mapData.data[1] = d->mapData.data[0] + d->mapData.size[0];
            d->mapData.data[2] = d->mapData.data[1] + d->mapData.size[1];
            break;
        }
        case QVideoFrameFormat::Format_NV12:
        case QVideoFrameFormat::Format_NV21:
        case QVideoFrameFormat::Format_IMC2:
        case QVideoFrameFormat::Format_IMC4:
        case QVideoFrameFormat::Format_P010:
        case QVideoFrameFormat::Format_P016: {
            // Semi planar, Full resolution Y plane with interleaved subsampled U and V planes.
            d->mapData.nPlanes = 2;
            d->mapData.bytesPerLine[1] = d->mapData.bytesPerLine[0];
            int size = d->mapData.size[0];
            d->mapData.size[0] = (d->mapData.bytesPerLine[0] * height());
            d->mapData.size[1] = size - d->mapData.size[0];
            d->mapData.data[1] = d->mapData.data[0] + d->mapData.size[0];
            break;
        }
        case QVideoFrameFormat::Format_IMC1:
        case QVideoFrameFormat::Format_IMC3: {
            // Three planes, the second and third vertically and horizontally subsumpled,
            // but with lines padded to the width of the first plane.
            d->mapData.nPlanes = 3;
            d->mapData.bytesPerLine[2] = d->mapData.bytesPerLine[1] = d->mapData.bytesPerLine[0];
            d->mapData.size[0] = (d->mapData.bytesPerLine[0] * height());
            d->mapData.size[1] = (d->mapData.bytesPerLine[0] * height() / 2);
            d->mapData.size[2] = (d->mapData.bytesPerLine[0] * height() / 2);
            d->mapData.data[1] = d->mapData.data[0] + d->mapData.size[0];
            d->mapData.data[2] = d->mapData.data[1] + d->mapData.size[1];
            break;
        }
        }
    }
 
    d->mappedCount++;
 
    // unlock mapMutex to avoid potential deadlock imageMutex <--> mapMutex
    lock.unlock();
 
    if ((mode & QVideoFrame::WriteOnly) != 0) {
        QMutexLocker lock(&d->imageMutex);
        d->image = {};
    }
 
    return true;
}
 
void QVideoFrame::unmap()
{
    if (!d || !d->buffer)
        return;
 
    QMutexLocker lock(&d->mapMutex);
 
    if (d->mappedCount == 0) {
        qWarning() << "QVideoFrame::unmap() was called more times then QVideoFrame::map()";
        return;
    }
 
    d->mappedCount--;
 
    if (d->mappedCount == 0) {
        d->mapData = {};
        d->buffer->unmap();
    }
}
 
int QVideoFrame::bytesPerLine(int plane) const
{
    if (!d)
        return 0;
    return plane >= 0 && plane < d->mapData.nPlanes ? d->mapData.bytesPerLine[plane] : 0;
}
 
uchar *QVideoFrame::bits(int plane)
{
    if (!d)
        return nullptr;
    return plane >= 0 && plane < d->mapData.nPlanes ? d->mapData.data[plane] : nullptr;
}
 
const uchar *QVideoFrame::bits(int plane) const
{
    if (!d)
        return nullptr;
    return plane >= 0 && plane < d->mapData.nPlanes ?  d->mapData.data[plane] : nullptr;
}
 
int QVideoFrame::mappedBytes(int plane) const
{
    if (!d)
        return 0;
    return plane >= 0 && plane < d->mapData.nPlanes ? d->mapData.size[plane] : 0;
}
 
int QVideoFrame::planeCount() const
{
    if (!d)
        return 0;
    return d->format.planeCount();
}
 
qint64 QVideoFrame::startTime() const
{
    if (!d)
        return -1;
    return d->startTime;
}
 
void QVideoFrame::setStartTime(qint64 time)
{
    if (!d)
        return;
    d->startTime = time;
}
 
qint64 QVideoFrame::endTime() const
{
    if (!d)
        return -1;
    return d->endTime;
}
 
void QVideoFrame::setEndTime(qint64 time)
{
    if (!d)
        return;
    d->endTime = time;
}
 
#if QT_DEPRECATED_SINCE(6, 7)
#endif
 
 
void QVideoFrame::setRotation(QtVideo::Rotation angle)
{
    if (d)
        d->format.setRotation(angle);
}
 
QtVideo::Rotation QVideoFrame::rotation() const
{
    return d ? d->format.rotation() : QtVideo::Rotation::None;
}
 
void QVideoFrame::setMirrored(bool mirrored)
{
    if (d)
        d->format.setMirrored(mirrored);
}
 
bool QVideoFrame::mirrored() const
{
    return d && d->format.isMirrored();
}
 
void QVideoFrame::setStreamFrameRate(qreal rate)
{
    if (d)
        d->format.setStreamFrameRate(rate);
}
 
qreal QVideoFrame::streamFrameRate() const
{
    return d ? d->format.streamFrameRate() : 0.;
}
 
QImage QVideoFrame::toImage() const
{
    if (!isValid())
        return {};
 
    QMutexLocker lock(&d->imageMutex);
 
    if (d->image.isNull()) {
        const bool mirrorY = surfaceFormat().scanLineDirection() != QVideoFrameFormat::TopToBottom;
        d->image = qImageFromVideoFrame(*this, rotation(), mirrored(), mirrorY);
    }
 
    return d->image;
}
 
QString QVideoFrame::subtitleText() const
{
    return d ? d->subtitleText : QString();
}
 
void QVideoFrame::setSubtitleText(const QString &text)
{
    if (!d)
        return;
    d->subtitleText = text;
}
 
void QVideoFrame::paint(QPainter *painter, const QRectF &rect, const PaintOptions &options)
{
    if (!isValid()) {
        painter->fillRect(rect, options.backgroundColor);
        return;
    }
 
    QRectF targetRect = rect;
    QSizeF size = qRotatedFrameSize(*this);
 
    size.scale(targetRect.size(), options.aspectRatioMode);
 
    if (options.aspectRatioMode == Qt::KeepAspectRatio) {
        targetRect = QRect(0, 0, size.width(), size.height());
        targetRect.moveCenter(rect.center());
        // we might not be drawing every pixel, fill the leftovers black
        if (options.backgroundColor != Qt::transparent && rect != targetRect) {
            if (targetRect.top() > rect.top()) {
                QRectF top(rect.left(), rect.top(), rect.width(), targetRect.top() - rect.top());
                painter->fillRect(top, Qt::black);
            }
            if (targetRect.left() > rect.left()) {
                QRectF top(rect.left(), targetRect.top(), targetRect.left() - rect.left(), targetRect.height());
                painter->fillRect(top, Qt::black);
            }
            if (targetRect.right() < rect.right()) {
                QRectF top(targetRect.right(), targetRect.top(), rect.right() - targetRect.right(), targetRect.height());
                painter->fillRect(top, Qt::black);
            }
            if (targetRect.bottom() < rect.bottom()) {
                QRectF top(rect.left(), targetRect.bottom(), rect.width(), rect.bottom() - targetRect.bottom());
                painter->fillRect(top, Qt::black);
            }
        }
    }
 
    if (map(QVideoFrame::ReadOnly)) {
        const QTransform oldTransform = painter->transform();
        QTransform transform = oldTransform;
        transform.translate(targetRect.center().x() - size.width()/2,
                            targetRect.center().y() - size.height()/2);
        painter->setTransform(transform);
        QImage image = toImage();
        painter->drawImage({{}, size}, image, {{},image.size()});
        painter->setTransform(oldTransform);
 
        unmap();
    } else if (isValid()) {
        // #### error handling
    } else {
        painter->fillRect(rect, Qt::black);
    }
 
    if ((options.paintFlags & PaintOptions::DontDrawSubtitles) || d->subtitleText.isEmpty())
        return;
 
    // draw subtitles
    auto text = d->subtitleText;
    text.replace(QLatin1Char('\n'), QChar::LineSeparator);
 
    QVideoTextureHelper::SubtitleLayout layout;
    layout.update(targetRect.size().toSize(), this->subtitleText());
    layout.draw(painter, targetRect.topLeft());
}
 
#ifndef QT_NO_DEBUG_STREAM
static QString qFormatTimeStamps(qint64 start, qint64 end)
{
    // Early out for invalid.
    if (start < 0)
        return QLatin1String("[no timestamp]");
 
    bool onlyOne = (start == end);
 
    // [hh:]mm:ss.ms
    const int s_millis = start % 1000000;
    start /= 1000000;
    const int s_seconds = start % 60;
    start /= 60;
    const int s_minutes = start % 60;
    start /= 60;
 
    if (onlyOne) {
        if (start > 0)
            return QStringLiteral("@%1:%2:%3.%4")
                    .arg(start, 1, 10, QLatin1Char('0'))
                    .arg(s_minutes, 2, 10, QLatin1Char('0'))
                    .arg(s_seconds, 2, 10, QLatin1Char('0'))
                    .arg(s_millis, 2, 10, QLatin1Char('0'));
        return QStringLiteral("@%1:%2.%3")
                .arg(s_minutes, 2, 10, QLatin1Char('0'))
                .arg(s_seconds, 2, 10, QLatin1Char('0'))
                .arg(s_millis, 2, 10, QLatin1Char('0'));
    }
 
    if (end == -1) {
        // Similar to start-start, except it means keep displaying it?
        if (start > 0)
            return QStringLiteral("%1:%2:%3.%4 - forever")
                    .arg(start, 1, 10, QLatin1Char('0'))
                    .arg(s_minutes, 2, 10, QLatin1Char('0'))
                    .arg(s_seconds, 2, 10, QLatin1Char('0'))
                    .arg(s_millis, 2, 10, QLatin1Char('0'));
        return QStringLiteral("%1:%2.%3 - forever")
                .arg(s_minutes, 2, 10, QLatin1Char('0'))
                .arg(s_seconds, 2, 10, QLatin1Char('0'))
                .arg(s_millis, 2, 10, QLatin1Char('0'));
    }
 
    const int e_millis = end % 1000000;
    end /= 1000000;
    const int e_seconds = end % 60;
    end /= 60;
    const int e_minutes = end % 60;
    end /= 60;
 
    if (start > 0 || end > 0)
        return QStringLiteral("%1:%2:%3.%4 - %5:%6:%7.%8")
                .arg(start, 1, 10, QLatin1Char('0'))
                .arg(s_minutes, 2, 10, QLatin1Char('0'))
                .arg(s_seconds, 2, 10, QLatin1Char('0'))
                .arg(s_millis, 2, 10, QLatin1Char('0'))
                .arg(end, 1, 10, QLatin1Char('0'))
                .arg(e_minutes, 2, 10, QLatin1Char('0'))
                .arg(e_seconds, 2, 10, QLatin1Char('0'))
                .arg(e_millis, 2, 10, QLatin1Char('0'));
    return QStringLiteral("%1:%2.%3 - %4:%5.%6")
            .arg(s_minutes, 2, 10, QLatin1Char('0'))
            .arg(s_seconds, 2, 10, QLatin1Char('0'))
            .arg(s_millis, 2, 10, QLatin1Char('0'))
            .arg(e_minutes, 2, 10, QLatin1Char('0'))
            .arg(e_seconds, 2, 10, QLatin1Char('0'))
            .arg(e_millis, 2, 10, QLatin1Char('0'));
}
 
QDebug operator<<(QDebug dbg, QVideoFrame::HandleType type)
{
    QDebugStateSaver saver(dbg);
    dbg.nospace();
    switch (type) {
    case QVideoFrame::NoHandle:
        return dbg << "NoHandle";
    case QVideoFrame::RhiTextureHandle:
        return dbg << "RhiTextureHandle";
    }
    return dbg;
}
 
QDebug operator<<(QDebug dbg, const QVideoFrame& f)
{
    QDebugStateSaver saver(dbg);
    dbg.nospace();
    dbg << "QVideoFrame(" << f.size() << ", "
               << f.pixelFormat() << ", "
               << f.handleType() << ", "
               << f.mapMode() << ", "
               << qFormatTimeStamps(f.startTime(), f.endTime()).toLatin1().constData();
    dbg << ')';
    return dbg;
}
#endif
 
QT_END_NAMESPACE