avfmediaencoder.mm

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies).
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
 
#include "avfmediaencoder_p.h"
 
#include <camera/avfcamera_p.h>
#include <camera/avfcamerarenderer_p.h>
#include <camera/avfcameraservice_p.h>
#include <camera/avfcamerasession_p.h>
#include <qdarwinformatsinfo_p.h>
 
#include <QtMultimedia/qaudiodevice.h>
#include <QtMultimedia/qmediadevices.h>
#include <QtMultimedia/private/qavfcameradebug_p.h>
#include <QtMultimedia/private/qavfcamerautility_p.h>
#include <QtMultimedia/private/qcoreaudioutils_p.h>
#include <QtMultimedia/private/qmediarecorder_p.h>
#include <QtMultimedia/private/qmediastoragelocation_p.h>
#include <QtMultimedia/private/qplatformaudioinput_p.h>
#include <QtMultimedia/private/qplatformaudiooutput_p.h>
#include <QtCore/qdebug.h>
#include <QtCore/qmath.h>
 
QT_USE_NAMESPACE
 
namespace {
 
bool qt_is_writable_file_URL(NSURL *fileURL)
{
    Q_ASSERT(fileURL);
 
    if (![fileURL isFileURL])
        return false;
 
    if (NSString *path = [[fileURL path] stringByExpandingTildeInPath]) {
        return [[NSFileManager defaultManager]
                isWritableFileAtPath:[path stringByDeletingLastPathComponent]];
    }
 
    return false;
}
 
bool qt_file_exists(NSURL *fileURL)
{
    Q_ASSERT(fileURL);
 
    if (NSString *path = [[fileURL path] stringByExpandingTildeInPath])
        return [[NSFileManager defaultManager] fileExistsAtPath:path];
 
    return false;
}
 
} // namespace
 
AVFMediaEncoder::AVFMediaEncoder(QMediaRecorder *parent)
    : QObject(parent)
    , QPlatformMediaRecorder(parent)
    , m_state(QMediaRecorder::StoppedState)
    , m_duration(0)
    , m_audioSettings(nil)
    , m_videoSettings(nil)
    //, m_restoreFPS(-1, -1)
{
    m_writer.reset([[QT_MANGLE_NAMESPACE(AVFMediaAssetWriter) alloc] initWithDelegate:this]);
    if (!m_writer) {
        qCDebug(qLcCamera) << Q_FUNC_INFO << "failed to create an asset writer";
        return;
    }
}
 
AVFMediaEncoder::~AVFMediaEncoder()
{
    [m_writer abort];
 
    if (m_audioSettings)
        [m_audioSettings release];
    if (m_videoSettings)
        [m_videoSettings release];
}
 
bool AVFMediaEncoder::isLocationWritable(const QUrl &location) const
{
    return location.scheme() == QLatin1String("file") || location.scheme().isEmpty();
}
 
QMediaRecorder::RecorderState AVFMediaEncoder::state() const
{
    return m_state;
}
 
qint64 AVFMediaEncoder::duration() const
{
    return m_duration;
}
 
void AVFMediaEncoder::updateDuration(qint64 duration)
{
    m_duration = duration;
    durationChanged(m_duration);
}
 
static NSDictionary *avfAudioSettings(const QMediaEncoderSettings &encoderSettings, const QAudioFormat &format)
{
    NSMutableDictionary *settings = [NSMutableDictionary dictionary];
 
    // Codec
    int codecId = QDarwinFormatInfo::audioFormatForCodec(encoderSettings.mediaFormat().audioCodec());
    settings[AVFormatIDKey] = @(codecId);
 
    // Setting AVEncoderQualityKey is not allowed when format ID is alac or lpcm
    if (codecId != kAudioFormatAppleLossless && codecId != kAudioFormatLinearPCM
        && encoderSettings.encodingMode() == QMediaRecorder::ConstantQualityEncoding) {
        // AudioQuality
        int quality;
        switch (encoderSettings.quality()) {
        case QMediaRecorder::VeryLowQuality:
            quality = AVAudioQualityMin;
            break;
        case QMediaRecorder::LowQuality:
            quality = AVAudioQualityLow;
            break;
        case QMediaRecorder::HighQuality:
            quality = AVAudioQualityHigh;
            break;
        case QMediaRecorder::VeryHighQuality:
            quality = AVAudioQualityMax;
            break;
        case QMediaRecorder::NormalQuality:
        default:
            quality = AVAudioQualityMedium;
            break;
        }
        settings[AVEncoderAudioQualityKey] = @(quality);
    } else {
        // BitRate
        bool isBitRateSupported = false;
        int bitRate = encoderSettings.audioBitRate();
        if (bitRate > 0) {
            QList<AudioValueRange> bitRates = qt_supported_bit_rates_for_format(codecId);
            for (int i = 0; i < bitRates.count(); i++) {
                if (bitRate >= bitRates[i].mMinimum &&
                    bitRate <= bitRates[i].mMaximum) {
                    isBitRateSupported = true;
                    break;
                }
            }
            if (isBitRateSupported)
                settings[AVEncoderBitRateKey] = @(encoderSettings.audioBitRate());
        }
    }
 
    // SampleRate
    int sampleRate = encoderSettings.audioSampleRate();
    bool isSampleRateSupported = false;
    if (sampleRate >= 8000 && sampleRate <= 192000) {
        QList<AudioValueRange> sampleRates = qt_supported_sample_rates_for_format(codecId);
        for (int i = 0; i < sampleRates.count(); i++) {
            if (sampleRate >= sampleRates[i].mMinimum && sampleRate <= sampleRates[i].mMaximum) {
                isSampleRateSupported = true;
                break;
            }
        }
    }
    if (!isSampleRateSupported)
        sampleRate = 44100;
    settings[AVSampleRateKey] = @(sampleRate);
 
    // Channels
    int channelCount = encoderSettings.audioChannelCount();
    bool isChannelCountSupported = false;
    if (channelCount > 0) {
        std::optional<QList<UInt32>> channelCounts = qt_supported_channel_counts_for_format(codecId);
        // An std::nullopt result indicates that
        // any number of channels can be encoded.
        if (channelCounts == std::nullopt) {
            isChannelCountSupported = true;
        } else {
            for (int i = 0; i < channelCounts.value().count(); i++) {
                if ((UInt32)channelCount == channelCounts.value()[i]) {
                    isChannelCountSupported = true;
                    break;
                }
            }
        }
 
        // if channel count is provided and it's bigger than 2
        // provide a supported channel layout
        if (isChannelCountSupported && channelCount > 2) {
            AudioChannelLayout channelLayout;
            memset(&channelLayout, 0, sizeof(AudioChannelLayout));
            auto channelLayoutTags = qt_supported_channel_layout_tags_for_format(codecId, channelCount);
            if (channelLayoutTags.size()) {
                channelLayout.mChannelLayoutTag = channelLayoutTags.first();
                settings[AVChannelLayoutKey] = [NSData dataWithBytes:&channelLayout length:sizeof(channelLayout)];
            } else {
                isChannelCountSupported = false;
            }
        }
 
        if (isChannelCountSupported)
            settings[AVNumberOfChannelsKey] = @(channelCount);
    }
 
    if (!isChannelCountSupported) {
        // fallback to providing channel layout if channel count is not specified or supported
        UInt32 size = 0;
        if (format.isValid()) {
            auto layout = QCoreAudioUtils::toAudioChannelLayout(format, &size);
            UInt32 layoutSize = offsetof(AudioChannelLayout, mChannelDescriptions)
                    + layout->mNumberChannelDescriptions * sizeof(AudioChannelDescription);
            settings[AVChannelLayoutKey] = [NSData dataWithBytes:layout.get() length:layoutSize];
        } else {
            // finally default to setting channel count to 1
            settings[AVNumberOfChannelsKey] = @(1);
        }
    }
 
    if (codecId == kAudioFormatAppleLossless)
        settings[AVEncoderBitDepthHintKey] = @(24);
 
    if (codecId == kAudioFormatLinearPCM) {
        settings[AVLinearPCMBitDepthKey] = @(16);
        settings[AVLinearPCMIsBigEndianKey] = @NO;
        settings[AVLinearPCMIsFloatKey] = @NO;
        settings[AVLinearPCMIsNonInterleaved] = @NO;
    }
 
    return settings;
}
 
static NSDictionary *avfVideoSettings(QMediaEncoderSettings &encoderSettings,
                                      AVCaptureDevice *device, AVCaptureConnection *connection,
                                      QSize nativeSize)
{
    if (!device)
        return nil;
 
 
    // ### re-add needFpsChange
//    AVFPSRange currentFps = qt_current_framerates(device, connection);
 
    NSMutableDictionary *videoSettings = [NSMutableDictionary dictionary];
 
    // -- Codec
 
    // AVVideoCodecKey is the only mandatory key
    auto codec = encoderSettings.mediaFormat().videoCodec();
    NSString *c = QDarwinFormatInfo::videoFormatForCodec(codec);
    [videoSettings setObject:c forKey:AVVideoCodecKey];
    [c release];
 
    // -- Resolution
 
    int w = encoderSettings.videoResolution().width();
    int h = encoderSettings.videoResolution().height();
 
    if (AVCaptureDeviceFormat *currentFormat = device.activeFormat) {
        CMFormatDescriptionRef formatDesc = currentFormat.formatDescription;
        CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(formatDesc);
        FourCharCode formatCodec = CMVideoFormatDescriptionGetCodecType(formatDesc);
 
        // We have to change the device's activeFormat in 3 cases:
        // - the requested recording resolution is higher than the current device resolution
        // - the requested recording resolution has a different aspect ratio than the current device aspect ratio
        // - the requested frame rate is not available for the current device format
        AVCaptureDeviceFormat *newFormat = nil;
        if ((w <= 0 || h <= 0)
                && encoderSettings.videoFrameRate() > 0
                && !qt_format_supports_framerate(currentFormat, encoderSettings.videoFrameRate())) {
 
            newFormat = qt_find_best_framerate_match(device,
                                                     formatCodec,
                                                     encoderSettings.videoFrameRate());
 
        } else if (w > 0 && h > 0) {
            AVCaptureDeviceFormat *f = qt_find_best_resolution_match(device,
                                                                     encoderSettings.videoResolution(),
                                                                     formatCodec);
 
            if (f) {
                CMVideoDimensions d = CMVideoFormatDescriptionGetDimensions(f.formatDescription);
                qreal fAspectRatio = qreal(d.width) / d.height;
 
                if (w > dim.width || h > dim.height
                        || qAbs((qreal(dim.width) / dim.height) - fAspectRatio) > 0.01) {
                    newFormat = f;
                }
            }
        }
 
        if (qt_set_active_format(device, newFormat, false /*### !needFpsChange*/)) {
            formatDesc = newFormat.formatDescription;
            dim = CMVideoFormatDescriptionGetDimensions(formatDesc);
        }
 
        if (w < 0 || h < 0) {
            w = dim.width;
            h = dim.height;
        }
 
 
        if (w > 0 && h > 0) {
            // Make sure the recording resolution has the same aspect ratio as the device's
            // current resolution
            qreal deviceAspectRatio = qreal(dim.width) / dim.height;
            qreal recAspectRatio = qreal(w) / h;
            if (qAbs(deviceAspectRatio - recAspectRatio) > 0.01) {
                if (recAspectRatio > deviceAspectRatio)
                    w = qRound(h * deviceAspectRatio);
                else
                    h = qRound(w / deviceAspectRatio);
            }
 
            // recording resolution can't be higher than the device's active resolution
            w = qMin(w, dim.width);
            h = qMin(h, dim.height);
        }
    }
 
    if (w > 0 && h > 0) {
        // Width and height must be divisible by 2
        w += w & 1;
        h += h & 1;
 
        bool isPortrait = nativeSize.width() < nativeSize.height();
        // Make sure the video has the right aspect ratio
        if (isPortrait && h < w)
            qSwap(w, h);
        else if (!isPortrait && w < h)
            qSwap(w, h);
 
        encoderSettings.setVideoResolution(QSize(w, h));
    } else {
        w = nativeSize.width();
        h = nativeSize.height();
        encoderSettings.setVideoResolution(nativeSize);
    }
    videoSettings[AVVideoWidthKey] = @(w);
    videoSettings[AVVideoHeightKey] = @(h);
 
    // -- FPS
 
    if (true /*needFpsChange*/) {
        const qreal fps = encoderSettings.videoFrameRate();
        qt_set_framerate_limits(device, connection, fps, fps);
    }
    encoderSettings.setVideoFrameRate(qt_current_framerates(device, connection).second);
 
    // -- Codec Settings
 
    NSMutableDictionary *codecProperties = [NSMutableDictionary dictionary];
    int bitrate = -1;
    float quality = -1.f;
 
    if (encoderSettings.encodingMode() == QMediaRecorder::ConstantQualityEncoding) {
        if (encoderSettings.quality() != QMediaRecorder::NormalQuality) {
            if (codec != QMediaFormat::VideoCodec::MotionJPEG) {
                qWarning("ConstantQualityEncoding is not supported for MotionJPEG");
            } else {
                switch (encoderSettings.quality()) {
                case QMediaRecorder::VeryLowQuality:
                    quality = 0.f;
                    break;
                case QMediaRecorder::LowQuality:
                    quality = 0.25f;
                    break;
                case QMediaRecorder::HighQuality:
                    quality = 0.75f;
                    break;
                case QMediaRecorder::VeryHighQuality:
                    quality = 1.f;
                    break;
                default:
                    quality = -1.f; // NormalQuality, let the system decide
                    break;
                }
            }
        }
    } else if (encoderSettings.encodingMode() == QMediaRecorder::AverageBitRateEncoding){
        if (codec != QMediaFormat::VideoCodec::H264 && codec != QMediaFormat::VideoCodec::H265)
            qWarning() << "AverageBitRateEncoding is not supported for codec" <<  QMediaFormat::videoCodecName(codec);
        else
            bitrate = encoderSettings.videoBitRate();
    } else {
        qWarning("Encoding mode is not supported");
    }
 
    if (bitrate != -1)
        codecProperties[AVVideoAverageBitRateKey] = @(bitrate);
    if (quality != -1.f)
        codecProperties[AVVideoQualityKey] = @(quality);
 
    videoSettings[AVVideoCompressionPropertiesKey] = codecProperties;
 
    return videoSettings;
}
 
void AVFMediaEncoder::applySettings(QMediaEncoderSettings &settings)
{
    unapplySettings();
 
    AVFCameraSession *session = m_service->session();
 
    // audio settings
    const auto audioInput = m_service->audioInput();
    const QAudioFormat audioFormat = audioInput ? audioInput->device.preferredFormat() : QAudioFormat();
    m_audioSettings = avfAudioSettings(settings, audioFormat);
    if (m_audioSettings)
        [m_audioSettings retain];
 
    // video settings
    AVCaptureDevice *device = session->videoCaptureDevice();
    if (!device)
        return;
    const AVFConfigurationLock lock(device); // prevents activeFormat from being overridden
    AVCaptureConnection *conn = [session->videoOutput()->videoDataOutput() connectionWithMediaType:AVMediaTypeVideo];
    auto nativeSize = session->videoOutput()->nativeSize();
    m_videoSettings = avfVideoSettings(settings, device, conn, nativeSize);
    if (m_videoSettings)
        [m_videoSettings retain];
}
 
void AVFMediaEncoder::unapplySettings()
{
    if (m_audioSettings) {
        [m_audioSettings release];
        m_audioSettings = nil;
    }
    if (m_videoSettings) {
        [m_videoSettings release];
        m_videoSettings = nil;
    }
}
 
void AVFMediaEncoder::setMetaData(const QMediaMetaData &metaData)
{
    m_metaData = metaData;
}
 
QMediaMetaData AVFMediaEncoder::metaData() const
{
    return m_metaData;
}
 
void AVFMediaEncoder::setCaptureSession(QPlatformMediaCaptureSession *session)
{
    AVFCameraService *captureSession = static_cast<AVFCameraService *>(session);
    if (m_service == captureSession)
        return;
 
    if (m_service)
        stop();
 
    m_service = captureSession;
    if (!m_service)
        return;
 
    connect(m_service, &AVFCameraService::cameraChanged, this, &AVFMediaEncoder::onCameraChanged);
    onCameraChanged();
}
 
void AVFMediaEncoder::record(QMediaEncoderSettings &settings)
{
    if (!m_service || !m_service->session()) {
        qWarning() << Q_FUNC_INFO << "Encoder is not set to a capture session";
        return;
    }
 
    if (!m_writer) {
        qCDebug(qLcCamera) << Q_FUNC_INFO << "Invalid recorder";
        return;
    }
 
    if (QMediaRecorder::RecordingState == m_state)
        return;
 
    AVFCamera *cameraControl = m_service->avfCameraControl();
    auto audioInput = m_service->audioInput();
 
    if (!cameraControl && !audioInput) {
        qWarning() << Q_FUNC_INFO << "Cannot record without any inputs";
        updateError(QMediaRecorder::ResourceError, tr("No inputs specified"));
        return;
    }
 
    // This is necessary to explicitly recreate m_audioInput inside AVFCameraSession.
    // Which in turn is necessary for the case when the microphone was disconnected
    // after stopping recording and reconnected.
    m_service->session()->updateAudioInput();
 
    m_service->session()->setActive(true);
    const bool audioOnly = settings.videoCodec() == QMediaFormat::VideoCodec::Unspecified;
    AVCaptureSession *session = m_service->session()->captureSession();
    float rotation = 0;
 
    if (!audioOnly) {
        if (!cameraControl || !cameraControl->isActive()) {
            qCDebug(qLcCamera) << Q_FUNC_INFO << "can not start record while camera is not active";
            updateError(QMediaRecorder::ResourceError,
                        QMediaRecorderPrivate::msgFailedStartRecording());
            return;
        }
    }
 
    const QString path(outputLocation().scheme() == QLatin1String("file") ?
                           outputLocation().path() : outputLocation().toString());
    const QUrl fileURL(QUrl::fromLocalFile(QMediaStorageLocation::generateFileName(
            path, audioOnly ? QStandardPaths::MusicLocation : QStandardPaths::MoviesLocation,
            settings.preferredSuffix())));
 
    NSURL *nsFileURL = fileURL.toNSURL();
    if (!nsFileURL) {
        qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL;
        updateError(QMediaRecorder::ResourceError, tr("Invalid output file URL"));
        return;
    }
    if (!qt_is_writable_file_URL(nsFileURL)) {
        qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL
                    << "(the location is not writable)";
        updateError(QMediaRecorder::ResourceError, tr("Non-writeable file location"));
        return;
    }
    if (qt_file_exists(nsFileURL)) {
        // We test for/handle this error here since AWAssetWriter will raise an
        // Objective-C exception, which is not good at all.
        qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL
                    << "(file already exists)";
        updateError(QMediaRecorder::ResourceError, tr("File already exists"));
        return;
    }
 
    applySettings(settings);
 
    QVideoOutputOrientationHandler::setIsRecording(true);
 
    // We stop session now so that no more frames for renderer's queue
    // generated, will restart in assetWriterStarted.
    [session stopRunning];
 
    if ([m_writer setupWithFileURL:nsFileURL
                    cameraService:m_service
                    audioSettings:m_audioSettings
                    videoSettings:m_videoSettings
                    fileFormat:settings.fileFormat()
                    transform:CGAffineTransformMakeRotation(qDegreesToRadians(rotation))]) {
 
        m_state = QMediaRecorder::RecordingState;
 
        actualLocationChanged(fileURL);
        stateChanged(m_state);
 
        // Apple recommends to call startRunning and do all
        // setup on a special queue, and that's what we had
        // initially (dispatch_async to writerQueue). Unfortunately,
        // writer's queue is not the only queue/thread that can
        // access/modify the session, and as a result we have
        // all possible data/race-conditions with Obj-C exceptions
        // at best and something worse in general.
        // Now we try to only modify session on the same thread.
        [m_writer start];
    } else {
        [session startRunning];
        updateError(QMediaRecorder::FormatError, QMediaRecorderPrivate::msgFailedStartRecording());
    }
}
 
void AVFMediaEncoder::pause()
{
    if (!m_service || !m_service->session() || state() != QMediaRecorder::RecordingState)
        return;
 
    toggleRecord(false);
    m_state = QMediaRecorder::PausedState;
    stateChanged(m_state);
}
 
void AVFMediaEncoder::resume()
{
    if (!m_service || !m_service->session() || state() != QMediaRecorder::PausedState)
        return;
 
    toggleRecord(true);
    m_state = QMediaRecorder::RecordingState;
    stateChanged(m_state);
}
 
void AVFMediaEncoder::stop()
{
    if (m_state != QMediaRecorder::StoppedState) {
        // Do not check the camera status, we can stop if we started.
        stopWriter();
    }
    QVideoOutputOrientationHandler::setIsRecording(false);
}
 
 
void AVFMediaEncoder::toggleRecord(bool enable)
{
    if (!m_service || !m_service->session())
        return;
 
    if (!enable)
        [m_writer pause];
    else
        [m_writer resume];
}
 
void AVFMediaEncoder::assetWriterStarted()
{
}
 
void AVFMediaEncoder::assetWriterFinished()
{
 
    const QMediaRecorder::RecorderState lastState = m_state;
 
    unapplySettings();
 
    if (m_service) {
        AVFCameraSession *session = m_service->session();
 
        if (session->videoOutput()) {
            session->videoOutput()->resetCaptureDelegate();
        }
        if (session->audioPreviewDelegate()) {
            [session->audioPreviewDelegate() resetAudioPreviewDelegate];
        }
        if (session->videoOutput() || session->audioPreviewDelegate())
            [session->captureSession() startRunning];
    }
 
    m_state = QMediaRecorder::StoppedState;
    if (m_state != lastState)
        stateChanged(m_state);
}
 
void AVFMediaEncoder::assetWriterError(QString err)
{
    updateError(QMediaRecorder::FormatError, err);
    if (m_state != QMediaRecorder::StoppedState)
        stopWriter();
}
 
void AVFMediaEncoder::onCameraChanged()
{
    if (m_service && m_service->avfCameraControl()) {
        AVFCamera *cameraControl = m_service->avfCameraControl();
        connect(cameraControl, SIGNAL(activeChanged(bool)),
                            SLOT(cameraActiveChanged(bool)));
    }
}
 
void AVFMediaEncoder::cameraActiveChanged(bool active)
{
    Q_ASSERT(m_service);
    AVFCamera *cameraControl = m_service->avfCameraControl();
    Q_ASSERT(cameraControl);
 
    if (!active) {
        return stopWriter();
    }
}
 
void AVFMediaEncoder::stopWriter()
{
    [m_writer stop];
}
 
#include "moc_avfmediaencoder_p.cpp"