qaudioengine_withplayer.cpp

Go to the documentation of this file.

// Copyright (C) 2025 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only
 
#include "qaudioengine_withplayer_p.h"
 
#include <QtMultimedia/qmediadevices.h>
#include <QtMultimedia/private/qaudiosystem_p.h>
#include <QtMultimedia/private/qmemory_resource_tlsf_p.h>
#include <QtMultimedia/private/qmultimedia_ranges_p.h>
#include <QtMultimedia/private/qrtaudioengine_p.h>
#include <QtMultimedia/private/qtmm_pmr_p.h>
#include <QtSpatialAudio/private/qambisonicdecoder_p.h>
#include <QtSpatialAudio/private/qspatialaudiosound_p.h>
 
#include <resonance_audio.h>
 
QT_BEGIN_NAMESPACE
 
enum class SourceId : int { };
 
class QResonanceAudioPlayer : public QtMultimediaPrivate::QRtAudioEngineVoice
{
public:
    using SharedPlaybackState = QAudioEnginePrivate::SharedPlaybackState;
 
    QResonanceAudioPlayer(QAudioEngineWithPlayer *player, VoiceId id);
 
    /// QRtAudioEngineVoice overrides
    VoicePlayResult play(QSpan<float>) noexcept Q_DECL_NONBLOCKING_FUNCTION override;
    bool isActive() noexcept Q_DECL_NONBLOCKING_FUNCTION override { return true; }
    const QAudioFormat &format() noexcept override { return m_format; }
 
    void addSound(SourceId, int numberOfChannels, SharedPlaybackState = nullptr);
    void setSoundPlaybackData(SourceId, SharedPlaybackState);
    void removeSound(SourceId);
 
    void setPaused(bool);
    void setOutputMode(QAudioEngine::OutputMode mode);
 
private:
    void processSlice(QSpan<float> hostBuffer);
    void processSliceFillResonanceBuffers();
    void processSliceProcessSurroundMode(QSpan<float>);
    void processSliceProcessWithoutReverb(QSpan<float>);
 
    constexpr static auto framesPerBuffer = QAudioEnginePrivate::framesPerBuffer;
    QAudioEngine::OutputMode m_outputMode;
    const std::shared_ptr<vraudio::ResonanceAudio> m_resonanceAudio;
    const QAudioFormat m_format;
    QAmbisonicDecoder m_ambisonicDecoder;
    bool m_paused = false;
 
    struct SoundState
    {
        SharedPlaybackState playbackState;
        int numberOfChannels;
    };
 
    static constexpr size_t poolSize =
            sizeof(QtMultimediaPrivate::pmr::map<SourceId, SoundState>::node_type)
                    * 512 // space for 512 sounds
            + sizeof(float) * qToUnderlying(framesPerBuffer) * 32 // leftover buffer for 32 channels
            + sizeof(float) * qToUnderlying(framesPerBuffer) * 32 // temporary slice buffer
            + 16384; // some extra space for internal fragmentation and allocator metadata
    QtMultimediaPrivate::QTlsfMemoryResource m_rtMemoryPool{ poolSize };
 
    QtMultimediaPrivate::pmr::map<SourceId, SoundState> m_sounds{ &m_rtMemoryPool };
    QtMultimediaPrivate::pmr::vector<float> m_leftoverBuffer{ &m_rtMemoryPool };
};
 
QResonanceAudioPlayer::QResonanceAudioPlayer(QAudioEngineWithPlayer *player, VoiceId id)
    : QRtAudioEngineVoice(id),
      m_outputMode(player->outputMode()),
      m_resonanceAudio(player->resonanceAudio),
      m_format(player->audioFormat()),
      m_ambisonicDecoder{
          QAmbisonicDecoder::HighQuality,
          m_format.sampleRate(),
          m_format.channelCount(),
          m_format.channelConfig(),
      }
{
}
 
QtMultimediaPrivate::QRtAudioEngineVoice::VoicePlayResult
QResonanceAudioPlayer::play(QSpan<float> outputBuffer) noexcept Q_DECL_NONBLOCKING_FUNCTION
{
    using namespace QtMultimediaPrivate;
 
    if (m_paused)
        return VoicePlayResult::Playing;
 
    const size_t samplesPerSlice = m_format.channelCount() * qToUnderlying(framesPerBuffer);
 
    // resonance-audio renders slices with fill (not additive) semantics, so every slice is
    // rendered into a zero-initialized scratch buffer first and mixed into outputBuffer
    // afterwards
    while (!outputBuffer.empty()) {
        if (m_leftoverBuffer.empty() && size_t(outputBuffer.size()) >= samplesPerSlice) {
            // fast path: the whole slice is consumed immediately, so we can use a stack-allocated
            // scratch buffer instead of carrying it via m_leftoverBuffer
            withTemporaryBuffer(samplesPerSlice * sizeof(float), [&](QSpan<std::byte> raw) {
                QSpan<float> scratch{
                    reinterpret_cast<float *>(raw.data()),
                    qsizetype(samplesPerSlice),
                };
                std::fill(scratch.begin(), scratch.end(), 0.0f);
                processSlice(scratch);
                for (size_t i = 0; i != samplesPerSlice; ++i)
                    outputBuffer[i] += scratch[i];
            });
            outputBuffer = drop(outputBuffer, samplesPerSlice);
            continue;
        }
 
        if (m_leftoverBuffer.empty()) {
            m_leftoverBuffer.assign(samplesPerSlice, 0.0f);
            processSlice(QSpan<float>(m_leftoverBuffer));
        }
 
        const auto samplesToMix = std::min<qsizetype>(m_leftoverBuffer.size(), outputBuffer.size());
        for (qsizetype i = 0; i != samplesToMix; ++i)
            outputBuffer[i] += m_leftoverBuffer[i];
 
        m_leftoverBuffer.erase(m_leftoverBuffer.begin(), m_leftoverBuffer.begin() + samplesToMix);
        outputBuffer = drop(outputBuffer, samplesToMix);
    }
 
    return VoicePlayResult::Playing;
}
 
void QResonanceAudioPlayer::addSound(SourceId id, int numberOfChannels, SharedPlaybackState state)
{
    auto [_, inserted] = m_sounds.insert_or_assign(id,
                                                   SoundState{
                                                           std::move(state),
                                                           numberOfChannels,
                                                   });
    Q_ASSERT(inserted);
}
 
void QResonanceAudioPlayer::setSoundPlaybackData(SourceId id, SharedPlaybackState state)
{
    auto it = m_sounds.find(id);
    Q_ASSERT(it != m_sounds.end());
    it->second.playbackState = std::move(state);
}
 
void QResonanceAudioPlayer::removeSound(SourceId sound)
{
    m_sounds.erase(sound);
}
 
void QResonanceAudioPlayer::setPaused(bool paused)
{
    m_paused = paused;
}
 
void QResonanceAudioPlayer::setOutputMode(QAudioEngine::OutputMode mode)
{
    m_outputMode = mode;
}
 
void QResonanceAudioPlayer::processSlice(QSpan<float> hostBuffer)
{
    using QtMultimediaPrivate::drop;
    constexpr auto framesPerBuffer = qToUnderlying(QAudioEnginePrivate::framesPerBuffer);
    const qsizetype samplesPerSlice = m_format.channelCount() * framesPerBuffer;
 
    Q_ASSERT(hostBuffer.size() == samplesPerSlice);
 
    // fill host API
    processSliceFillResonanceBuffers();
 
    // and process
    switch (m_outputMode) {
    case QAudioEngine::Surround:
        processSliceProcessSurroundMode(hostBuffer);
        break;
    default:
        processSliceProcessWithoutReverb(hostBuffer);
        break;
    }
}
 
 
void QResonanceAudioPlayer::processSliceFillResonanceBuffers()
{
    constexpr auto framesPerBuffer = qToUnderlying(QAudioEnginePrivate::framesPerBuffer);
    static constexpr std::array<float, 2 * framesPerBuffer> nullBuffer{};
    using QtMultimediaPrivate::take;
    vraudio::ResonanceAudioApi *api = m_resonanceAudio->api.get();
 
    for (auto &&[sourceId, sourceState] : m_sounds) {
        int numberOfChannels = sourceState.numberOfChannels;
        SharedPlaybackState &playbackState = sourceState.playbackState;
        if (playbackState) {
            Q_ASSERT(playbackState->format().channelCount() <= 2);
            std::array<float, 2 * framesPerBuffer> buf;
 
            playbackState->getBuffer(take(
                    QSpan<float>{ buf }, playbackState->format().channelCount() * framesPerBuffer));
            api->SetInterleavedBuffer(qToUnderlying(sourceId), buf.data(), numberOfChannels,
                                      framesPerBuffer);
        } else {
            api->SetInterleavedBuffer(qToUnderlying(sourceId), nullBuffer.data(), numberOfChannels,
                                      framesPerBuffer);
        }
    }
}
 
void QResonanceAudioPlayer::processSliceProcessSurroundMode(QSpan<float> outputBuffer)
{
    constexpr auto framesPerBuffer = qToUnderlying(QAudioEnginePrivate::framesPerBuffer);
    using QtMultimediaPrivate::take;
    namespace ranges = QtMultimediaPrivate::ranges;
    Q_ASSERT(outputBuffer.size() == m_format.channelCount() * framesPerBuffer);
 
    std::array<const float *, QAmbisonicDecoder::maxAmbisonicChannels> channels;
    std::array<const float *, 2> reverbBuffers{};
    int nFrames = m_resonanceAudio->getAmbisonicOutput(channels.data(), reverbBuffers.data(),
                                                       m_ambisonicDecoder.nInputChannels());
 
    if (nFrames < 0) {
        // If we get here, it means that resonanceAudio did not actually fill the buffer.
        // Sometimes this is expected, for example if resonanceAudio does not have any sources.
        // In this case we just fill the buffer with silence.
        ranges::fill(outputBuffer, 0);
        return;
    }
    auto nSamples = m_ambisonicDecoder.outputSamples(nFrames);
 
    Q_ASSERT(m_ambisonicDecoder.nOutputChannels() <= 8);
    QSpan currentOutput = take(outputBuffer, nSamples);
    m_ambisonicDecoder.processBufferWithReverb(
            QSpan{ channels.data(), m_ambisonicDecoder.nInputChannels() }, reverbBuffers,
            currentOutput);
}
 
void QResonanceAudioPlayer::processSliceProcessWithoutReverb(QSpan<float> outputBuffer)
{
    constexpr auto framesPerBuffer = qToUnderlying(QAudioEnginePrivate::framesPerBuffer);
 
    Q_ASSERT(outputBuffer.size() == m_format.channelCount() * framesPerBuffer);
    namespace ranges = QtMultimediaPrivate::ranges;
 
    bool ok = m_resonanceAudio->api->FillInterleavedOutputBuffer(
            m_format.channelCount(), framesPerBuffer, outputBuffer.data());
    if (!ok) {
        // If we get here, it means that resonanceAudio did not actually fill the buffer.
        // Sometimes this is expected, for example if resonanceAudio does not have any sources.
        // In this case we just fill the buffer with silence.
        if (m_sounds.empty()) {
            ranges::fill(outputBuffer, 0.f);
        } else {
            // If we get here, it means that something unexpected happened, so bail.
            qWarning() << "    processSliceProcessWithoutReverb failure!";
            return;
        }
    }
}
 
///////////////////////////////////////////////////////////////////////////////////////////////////
 
QAudioEngineWithPlayer::QAudioEngineWithPlayer(int sampleRate) : QAudioEnginePrivate(sampleRate)
{
}
 
QAudioEngineWithPlayer::~QAudioEngineWithPlayer()
{
    stop();
    m_playbackEngine = nullptr;
}
 
void QAudioEngineWithPlayer::start()
{
    QAudioDevice device = m_device;
    if (device.isNull())
        device = QMediaDevices::defaultAudioOutput();
 
    if (device.isNull()) {
        qWarning() << "QAudioEngine::start() failed: No audio output device found.";
        return;
    }
 
    QAudioFormat format;
    format.setSampleFormat(QAudioFormat::Float);
    format.setSampleRate(sampleRate());
    format.setChannelCount(std::min(2, device.maximumChannelCount()));
 
    m_playbackEngine = std::make_shared<QRtAudioEngine>(device, format, std::nullopt,
                                                        QAudioEnginePrivate::framesPerBuffer);
    m_format = format;
 
    auto voice = std::make_shared<QResonanceAudioPlayer>(this, QRtAudioEngine::allocateVoiceId());
 
    for (auto &[sound, state] : playbackStates)
        voice->addSound(SourceId{ sound->sourceId }, sound->nchannels, state);
 
    m_engineVoice = voice;
    m_playbackEngine->play(voice);
}
 
void QAudioEngineWithPlayer::stop()
{
    if (!m_engineVoice)
        return;
    m_playbackEngine->stop(m_engineVoice);
    m_engineVoice = {};
}
 
void QAudioEngineWithPlayer::setPaused(bool paused)
{
    if (paused == m_paused)
        return;
    m_paused = paused;
    if (!m_engineVoice)
        return;
 
    // CHECK: can we pause the stream? it seems a bit problematic, as it would prevent us from sending
    // rt-visitors to update the state of the player.
    m_playbackEngine->visitVoiceRt(m_engineVoice->voiceId(),
                                   [paused](QResonanceAudioPlayer &player) {
        player.setPaused(paused);
    });
}
 
bool QAudioEngineWithPlayer::isPaused() const
{
    return m_paused;
}
 
void QAudioEngineWithPlayer::setOutputDevice(const QAudioDevice &device)
{
    if (m_device == device)
        return;
    if (m_engineVoice) {
        qWarning() << "Changing device on a running engine not implemented";
        return;
    }
    m_device = device;
    Q_Q(QAudioEngine);
    emit q->outputDeviceChanged();
}
 
QAudioDevice QAudioEngineWithPlayer::outputDevice() const
{
    return m_device;
}
 
void QAudioEngineWithPlayer::addSound(QSpatialAudioSoundPrivate *sound)
{
    playbackStates.emplace(sound, nullptr);
 
    if (!m_engineVoice)
        return;
 
    m_playbackEngine->visitVoiceRt(
            m_engineVoice->voiceId(),
            [id = SourceId{ sound->sourceId },
             numberOfChannels = sound->nchannels](QResonanceAudioPlayer &player) mutable {
        player.addSound(id, numberOfChannels);
    });
}
 
void QAudioEngineWithPlayer::removeSound(QSpatialAudioSoundPrivate *sound)
{
    SharedPlaybackState oldState = std::move(playbackStates[sound]);
    playbackStates.erase(sound);
    if (!m_engineVoice)
        return;
 
    // pass oldState to the lambda to ensure that it is not freed on the real-time thread
    m_playbackEngine->visitVoiceRt(m_engineVoice->voiceId(),
                                   [id = SourceId{ sound->sourceId },
                                    oldState = std::move(oldState)](QResonanceAudioPlayer &player) {
        player.removeSound(id);
    });
}
 
void QAudioEngineWithPlayer::setSoundPlaybackData(QSpatialAudioSoundPrivate *sound,
                                                  SharedPlaybackState state)
{
    auto it = playbackStates.find(sound);
    Q_ASSERT(it != playbackStates.end());
 
    SharedPlaybackState oldState = std::exchange(it->second, state);
 
    if (!m_engineVoice)
        return;
 
    // pass oldState to the lambda to ensure that it is not freed on the real-time thread
    m_playbackEngine->visitVoiceRt(
            m_engineVoice->voiceId(),
            [id = SourceId{ sound->sourceId }, oldState = std::move(oldState),
             state = std::move(state)](QResonanceAudioPlayer &player) mutable {
        player.setSoundPlaybackData(id, std::move(state));
    });
}
 
void QAudioEngineWithPlayer::setOutputMode(QAudioEngine::OutputMode mode)
{
    if (outputMode() == mode)
        return;
    QAudioEnginePrivate::setOutputMode(mode);
    if (!m_engineVoice)
        return;
 
    m_playbackEngine->visitVoiceRt(m_engineVoice->voiceId(), [mode](QResonanceAudioPlayer &player) {
        player.setOutputMode(mode);
    });
}
 
void QAudioEngineWithPlayer::updateRoomEffects()
{
    for (auto [sound, key] : playbackStates)
        sound->updateRoomEffects();
}
 
QT_END_NAMESPACE