qwindowscamera.cpp

Go to the documentation of this file.

// Copyright (C) 2022 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qwindowscamera_p.h"
#include "qsemaphore.h"
#include "qmutex.h"
 
#include <private/qcameradevice_p.h>
#include <private/qmemoryvideobuffer_p.h>
#include <private/qwindowsmultimediautils_p.h>
#include <private/qvideoframe_p.h>
#include <private/qcomobject_p.h>
#include <private/qwmf_support_p.h>
 
#include <QtCore/private/qsystemerror_p.h>
 
#include <mfapi.h>
#include <mfidl.h>
#include <mferror.h>
#include <mfreadwrite.h>
 
#include <system_error>
#include <utility>
 
QT_BEGIN_NAMESPACE
 
namespace QFFmpeg {
 
qsizetype indexOfClosestCameraFormat(QSpan<const QCameraFormat> candidates,
                                     const QCameraFormat &requested)
{
    qsizetype closest = -1;
    std::pair<float, float> closestDistance{};
 
    for (qsizetype i = 0; i != candidates.size(); ++i) {
        const QCameraFormat &candidate = candidates[i];
        if (candidate.pixelFormat() != requested.pixelFormat()
            || candidate.resolution() != requested.resolution()) {
            continue;
        }
 
        const std::pair distance{ qAbs(candidate.maxFrameRate() - requested.maxFrameRate()),
                                  qAbs(candidate.minFrameRate() - requested.minFrameRate()) };
        if (closest < 0 || distance < closestDistance) {
            closest = i;
            closestDistance = distance;
        }
    }
 
    return closest;
}
 
namespace {
 
class CameraReaderCallback : public QComObject<IMFSourceReaderCallback>
{
public:
    //from IMFSourceReaderCallback
    STDMETHODIMP OnReadSample(HRESULT status, DWORD, DWORD, LONGLONG timestamp, IMFSample *sample) override;
    STDMETHODIMP OnFlush(DWORD) override;
    STDMETHODIMP OnEvent(DWORD, IMFMediaEvent *) override { return S_OK; }
 
    void setActiveCamera(ActiveCamera *activeCamera)
    {
        QMutexLocker locker(&m_mutex);
        m_activeCamera = activeCamera;
    }
private:
    // Destructor is not public. Caller should call Release.
    ~CameraReaderCallback() override = default;
 
    ActiveCamera *m_activeCamera = nullptr;
    QMutex m_mutex;
};
 
ComPtr<IMFSourceReader> createCameraReader(IMFMediaSource *mediaSource,
                                           const ComPtr<CameraReaderCallback> &callback)
{
    ComPtr<IMFSourceReader> sourceReader;
    ComPtr<IMFAttributes> readerAttributes;
 
    HRESULT hr = MFCreateAttributes(readerAttributes.GetAddressOf(), 1);
    if (SUCCEEDED(hr)) {
        hr = readerAttributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, callback.Get());
        if (SUCCEEDED(hr)) {
            hr = MFCreateSourceReaderFromMediaSource(mediaSource, readerAttributes.Get(), sourceReader.GetAddressOf());
            if (SUCCEEDED(hr))
                return sourceReader;
        }
    }
 
    qWarning() << "Failed to create camera IMFSourceReader" << hr;
    return sourceReader;
}
 
ComPtr<IMFMediaSource> createCameraSource(const QString &deviceId)
{
    ComPtr<IMFMediaSource> mediaSource;
    ComPtr<IMFAttributes> sourceAttributes;
    HRESULT hr = MFCreateAttributes(sourceAttributes.GetAddressOf(), 2);
    if (SUCCEEDED(hr)) {
        hr = sourceAttributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE,
                                       MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID);
        if (SUCCEEDED(hr)) {
            hr = sourceAttributes->SetString(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK,
                                             reinterpret_cast<LPCWSTR>(deviceId.utf16()));
            if (SUCCEEDED(hr)) {
                hr = MFCreateDeviceSource(sourceAttributes.Get(), mediaSource.GetAddressOf());
                if (SUCCEEDED(hr))
                    return mediaSource;
            }
        }
    }
    qWarning() << "Failed to create camera IMFMediaSource" << hr;
    return mediaSource;
}
 
int calculateVideoFrameStride(IMFMediaType *videoType, int width)
{
    Q_ASSERT(videoType);
 
    GUID subtype = GUID_NULL;
    HRESULT hr = videoType->GetGUID(MF_MT_SUBTYPE, &subtype);
    if (SUCCEEDED(hr)) {
        LONG stride = 0;
        hr = MFGetStrideForBitmapInfoHeader(subtype.Data1, width, &stride);
        if (SUCCEEDED(hr))
            return int(qAbs(stride));
    }
 
    qWarning() << "Failed to calculate video stride" << QSystemError::windowsComString(hr);
    return 0;
}
 
bool setCameraReaderFormat(IMFSourceReader *sourceReader, IMFMediaType *videoType)
{
    Q_ASSERT(sourceReader);
    Q_ASSERT(videoType);
 
    HRESULT hr = sourceReader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr,
                                                   videoType);
    if (FAILED(hr))
        qWarning() << "Failed to set video format" << QSystemError::windowsComString(hr);
 
    return SUCCEEDED(hr);
}
 
ComPtr<IMFMediaType> findVideoType(IMFSourceReader *reader, const QCameraFormat &format)
{
    std::vector<ComPtr<IMFMediaType>> mediaTypes;
    std::vector<QCameraFormat> candidates;
 
    for (DWORD i = 0;; ++i) {
        ComPtr<IMFMediaType> mediaType;
        HRESULT hr = reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, i,
                                                mediaType.GetAddressOf());
        if (FAILED(hr))
            break;
 
        const std::optional<QCameraFormat> candidate =
                QWMF::cameraFormatFromMediaType(mediaType.Get());
        if (!candidate)
            continue;
 
        mediaTypes.push_back(std::move(mediaType));
        candidates.push_back(std::move(*candidate));
    }
 
    const qsizetype index = indexOfClosestCameraFormat(candidates, format);
    if (index < 0)
        return {};
 
    return mediaTypes.at(index);
}
 
} // namespace
 
class ActiveCamera
{
public:
    static std::unique_ptr<ActiveCamera> create(QWindowsCamera &wc, const QCameraDevice &device, const QCameraFormat &format)
    {
        auto ac = std::unique_ptr<ActiveCamera>(new ActiveCamera(wc));
        ac->m_source = createCameraSource(QString::fromUtf8(device.id()));
        if (!ac->m_source)
            return {};
 
        ac->m_readerCallback = makeComObject<CameraReaderCallback>();
        ac->m_readerCallback->setActiveCamera(ac.get());
        ac->m_reader = createCameraReader(ac->m_source.Get(), ac->m_readerCallback);
        if (!ac->m_reader)
            return {};
 
        if (!ac->setFormat(format))
            return {};
 
        return ac;
    }
 
    bool setFormat(const QCameraFormat &format)
    {
        flush();
 
        auto videoType = findVideoType(m_reader.Get(), format);
        if (videoType) {
            if (setCameraReaderFormat(m_reader.Get(), videoType.Get())) {
                m_frameFormat = { format.resolution(), format.pixelFormat() };
                m_frameFormat.setColorRange(QCameraFormatPrivate::getColorRange(format));
                m_frameFormat.setColorSpace(QCameraFormatPrivate::getColorSpace(format));
                m_videoFrameStride =
                        calculateVideoFrameStride(videoType.Get(), format.resolution().width());
            }
        }
 
        m_reader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, nullptr, nullptr, nullptr,
                             nullptr);
        return true;
    }
 
    void onReadSample(HRESULT status, LONGLONG timestamp, IMFSample *sample)
    {
        if (FAILED(status)) {
            const std::string msg{ std::system_category().message(status) };
            m_windowsCamera.updateError(QCamera::CameraError, QString::fromStdString(msg));
            return;
        }
 
        if (sample) {
            ComPtr<IMFMediaBuffer> mediaBuffer;
            if (SUCCEEDED(sample->ConvertToContiguousBuffer(mediaBuffer.GetAddressOf()))) {
 
                auto result = QWMF::withLockedBuffer(mediaBuffer,
                                                     [&](QSpan<BYTE> data, QSpan<BYTE> /*max*/) {
                    QByteArray bytes(data);
                    auto buffer = std::make_unique<QMemoryVideoBuffer>(std::move(bytes),
                                                                       m_videoFrameStride);
                    QVideoFrame frame =
                            QVideoFramePrivate::createFrame(std::move(buffer), m_frameFormat);
 
                    // WMF uses 100-nanosecond units, Qt uses microseconds
                    frame.setStartTime(timestamp / 10);
 
                    LONGLONG duration = -1;
                    if (SUCCEEDED(sample->GetSampleDuration(&duration)))
                        frame.setEndTime((timestamp + duration) / 10);
 
                    return frame;
                });
 
                if (result)
                    emit m_windowsCamera.newVideoFrame(result.value());
            }
        }
 
        m_reader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, nullptr,
                             nullptr, nullptr, nullptr);
    }
 
    void onFlush()
    {
        m_flushWait.release();
    }
 
    ~ActiveCamera()
    {
        flush();
        if (m_readerCallback)
            m_readerCallback->setActiveCamera(nullptr);
    }
 
private:
    explicit ActiveCamera(QWindowsCamera &wc) : m_windowsCamera(wc), m_flushWait(0) { }
 
    void flush()
    {
        if (m_reader && SUCCEEDED(m_reader->Flush(MF_SOURCE_READER_FIRST_VIDEO_STREAM))) {
            m_flushWait.acquire();
        }
    }
 
    QWindowsCamera &m_windowsCamera;
 
    QSemaphore m_flushWait;
 
    ComPtr<IMFMediaSource> m_source;
    ComPtr<IMFSourceReader> m_reader;
    ComPtr<CameraReaderCallback> m_readerCallback;
 
    QVideoFrameFormat m_frameFormat;
    int m_videoFrameStride = 0;
};
 
STDMETHODIMP CameraReaderCallback::OnReadSample(HRESULT status, DWORD, DWORD, LONGLONG timestamp, IMFSample *sample)
{
    QMutexLocker locker(&m_mutex);
    if (m_activeCamera)
        m_activeCamera->onReadSample(status, timestamp, sample);
 
    return S_OK;
}
 
STDMETHODIMP CameraReaderCallback::OnFlush(DWORD)
{
    QMutexLocker locker(&m_mutex);
    if (m_activeCamera)
        m_activeCamera->onFlush();
    return S_OK;
}
 
QWindowsCamera::QWindowsCamera(QCamera *camera)
    : QPlatformCamera(camera)
{
    m_cameraDevice = camera ? camera->cameraDevice() : QCameraDevice{};
}
 
QWindowsCamera::~QWindowsCamera()
{
    QWindowsCamera::setActive(false);
}
 
void QWindowsCamera::setActive(bool active)
{
    if (bool(m_active) == active)
        return;
 
    if (active) {
        if (m_cameraDevice.isNull())
            return;
 
        if (m_cameraFormat.isNull())
            m_cameraFormat = findBestCameraFormat(m_cameraDevice);
 
        m_active = ActiveCamera::create(*this, m_cameraDevice, m_cameraFormat);
        if (m_active)
            activeChanged(true);
 
    } else {
        m_active.reset();
        emit activeChanged(false);
    }
}
 
void QWindowsCamera::setCamera(const QCameraDevice &camera)
{
    bool active = bool(m_active);
    if (active)
        setActive(false);
    m_cameraDevice = camera;
    m_cameraFormat = {};
    if (active)
        setActive(true);
}
 
bool QWindowsCamera::setCameraFormat(const QCameraFormat &format)
{
    if (format.isNull())
        return false;
 
    bool ok = m_active ? m_active->setFormat(format) : true;
    if (ok)
        m_cameraFormat = format;
 
    return ok;
}
 
} // namespace QFFmpeg
 
QT_END_NAMESPACE