qsoundeffectwithplayer.cpp

Go to the documentation of this file.

// Copyright (C) 2025 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "qsoundeffectwithplayer_p.h"
 
#include <QtCore/q20map.h>
#include <QtCore/qmutex.h>
#include <QtCore/qscopeguard.h>
#include <QtCore/private/qobject_p.h>
 
#include <utility>
 
QT_BEGIN_NAMESPACE
 
namespace QtMultimediaPrivate {
 
namespace {
 
QSpan<const float> toFloatSpan(QSpan<const char> byteArray)
{
    return QSpan{
        reinterpret_cast<const float *>(byteArray.data()),
        qsizetype(byteArray.size_bytes() / sizeof(float)),
    };
}
 
struct AudioDeviceFormatLess
{
    bool operator()(const std::pair<QAudioDevice, QAudioFormat> &lhs,
                    const std::pair<QAudioDevice, QAudioFormat> &rhs) const
    {
        auto cmp = qCompareThreeWay(lhs.first.id(), rhs.first.id());
        if (cmp == Qt::strong_ordering::less)
            return true;
        if (cmp == Qt::strong_ordering::greater)
            return false;
 
        return std::tuple(lhs.second.sampleRate(), lhs.second.sampleFormat(),
                          lhs.second.channelCount())
                < std::tuple(rhs.second.sampleRate(), rhs.second.sampleFormat(),
                             rhs.second.channelCount());
    }
};
 
} // namespace
 
// we keep a pool of engines with one engine per device/format
std::shared_ptr<QRtAudioEngine>
QSoundEffectPrivateWithPlayer::getEngineFor(const QAudioDevice &device, const QAudioFormat &format)
{
    if (device.isNull()) {
        qWarning() << "QRtAudioEngine needs to be called with a valid device";
        return nullptr;
    }
 
    if (format.sampleFormat() != QAudioFormat::Float) {
        qWarning() << "QRtAudioEngine requires floating point samples";
        return nullptr;
    }
 
    if (!device.isFormatSupported(format)) {
        qWarning() << "QRtAudioEngine needs to be called with a supported fromat";
        return nullptr;
    }
 
    static QMutex s_playerRegistryMutex;
    static std::map<std::pair<QAudioDevice, QAudioFormat>, std::weak_ptr<QRtAudioEngine>,
                    AudioDeviceFormatLess>
            s_playerRegistry;
 
    auto guard = std::lock_guard{ s_playerRegistryMutex };
 
    auto key = std::pair{ device, format };
    auto found = s_playerRegistry.find(key);
    if (found != s_playerRegistry.end()) {
        auto player = found->second.lock();
        if (player && player->audioState() != QtAudio::StoppedState)
            return player;
    }
 
    // lazy clean up
    q20::erase_if(s_playerRegistry, [](auto &&keyValuePair) {
        return keyValuePair.second.expired();
    });
 
    // SoundEffect can prevent the stream from appear in an OS mixer
    auto endpointRole = AudioEndpointRole::SoundEffect;
 
    auto player = std::shared_ptr<QRtAudioEngine>(
            new QRtAudioEngine{ QRtAudioEngine::MoveToApplicationThread::Yes, device, format,
                                endpointRole },
            &QObjectPrivate::deleteInOwnThread);
    s_playerRegistry.insert_or_assign(key, player);
 
    return player;
}
 
///////////////////////////////////////////////////////////////////////////////////////////////////
 
QSoundEffectVoice::QSoundEffectVoice(VoiceId voiceId, std::shared_ptr<const QSample> sample,
                                     float volume, bool muted, int totalLoopCount,
                                     QAudioFormat engineFormat)
    : QRtAudioEngineVoice{ voiceId },
      m_sample{ std::move(sample) },
      m_engineFormat{ engineFormat },
      m_volume{ volume },
      m_muted{ muted },
      m_loopsRemaining{ totalLoopCount }
{
}
 
QSoundEffectVoice::~QSoundEffectVoice() = default;
 
VoicePlayResult QSoundEffectVoice::play(QSpan<float> outputBuffer) noexcept Q_DECL_NONBLOCKING_FUNCTION
{
    qsizetype playedFrames = playVoice(outputBuffer);
    m_currentFrame += playedFrames;
 
    if (m_currentFrame == m_totalFrames) {
        const bool isInfiniteLoop = loopsRemaining() == QSoundEffect::Infinite;
        bool continuePlaying = isInfiniteLoop;
 
        if (!isInfiniteLoop)
            continuePlaying = m_loopsRemaining.fetch_sub(1, std::memory_order_relaxed) > 1;
 
        if (continuePlaying) {
            if (!isInfiniteLoop)
                m_currentLoopChanged.set();
            m_currentFrame = 0;
            QSpan remainingOutputBuffer =
                    drop(outputBuffer, playedFrames * m_engineFormat.channelCount());
            return play(remainingOutputBuffer);
        }
        return VoicePlayResult::Finished;
    }
    return VoicePlayResult::Playing;
}
 
qsizetype QSoundEffectVoice::playVoice(QSpan<float> outputBuffer) noexcept Q_DECL_NONBLOCKING_FUNCTION
{
    const QAudioFormat &format = m_sample->format();
    const int totalSamples = m_totalFrames * format.channelCount();
    const int currentSample = format.channelCount() * m_currentFrame;
 
    const QSpan fullSample = toFloatSpan(m_sample->data());
    const QSpan playbackRange = take(drop(fullSample, currentSample), totalSamples);
 
    Q_ASSERT(!playbackRange.empty());
 
    const int sampleCh = format.channelCount();
    const int engineCh = m_engineFormat.channelCount();
    const qsizetype sampleSamples = playbackRange.size();
    const qsizetype outputSamples = outputBuffer.size();
    const qsizetype maxFrames = std::min(sampleSamples / sampleCh, outputSamples / engineCh);
    const qsizetype framesToPlay = maxFrames;
 
    enum ConversionType : uint8_t { SameChannels, MonoToStereo, StereoToMono };
    const ConversionType conversion = [&] {
        if (sampleCh == engineCh)
            return SameChannels;
        if (sampleCh == 1 && engineCh == 2)
            return MonoToStereo;
        if (sampleCh == 2 && engineCh == 1)
            return StereoToMono;
        Q_UNREACHABLE_RETURN(SameChannels);
    }();
 
    if (m_muted || m_volume == 0.f)
        return framesToPlay;
 
    // later: (auto)vectorize?
    switch (conversion) {
    case SameChannels:
        for (qsizetype frame = 0; frame < framesToPlay; ++frame) {
            const qsizetype sampleBase = frame * sampleCh;
            const qsizetype outputBase = frame * engineCh;
            for (int ch = 0; ch < sampleCh; ++ch) {
                outputBuffer[outputBase + ch] += playbackRange[sampleBase + ch] * m_volume;
            }
        }
        break;
    case MonoToStereo:
        for (qsizetype frame = 0; frame < framesToPlay; ++frame) {
            const qsizetype sampleBase = frame * sampleCh;
            const qsizetype outputBase = frame * engineCh;
            const float val = playbackRange[sampleBase] * m_volume;
            outputBuffer[outputBase] += val;
            outputBuffer[outputBase + 1] += val;
        }
        break;
    case StereoToMono:
        float scale = 0.5f * m_volume;
        for (qsizetype frame = 0; frame < framesToPlay; ++frame) {
            const qsizetype sampleBase = frame * sampleCh;
            const qsizetype outputBase = frame * engineCh;
            const float val = (playbackRange[sampleBase] + playbackRange[sampleBase + 1]) * scale;
            outputBuffer[outputBase] += val;
        }
        break;
    }
 
    return framesToPlay;
}
 
bool QSoundEffectVoice::isActive() noexcept Q_DECL_NONBLOCKING_FUNCTION
{
    if (m_currentFrame != m_totalFrames)
        return true;
 
    return loopsRemaining() != 0;
}
 
std::shared_ptr<QSoundEffectVoice>
QSoundEffectVoice::clone(std::optional<QAudioFormat> newEngineFormat) const
{
    auto clone = std::make_shared<QSoundEffectVoice>(QRtAudioEngine::allocateVoiceId(), m_sample,
                                                     m_volume, m_muted, loopsRemaining(),
                                                     newEngineFormat.value_or(m_engineFormat));
 
    // caveat: reading frame is not atomic, so we may have a race here ... is is rare, though,
    // not sure if we really care
    clone->m_currentFrame = m_currentFrame;
    return clone;
}
 
///////////////////////////////////////////////////////////////////////////////////////////////////
 
QSoundEffectPrivateWithPlayer::QSoundEffectPrivateWithPlayer(QSoundEffect *q,
                                                             QAudioDevice audioDevice)
    : q_ptr{ q }, m_audioDevice{ std::move(audioDevice) }
{
    resolveAudioDevice();
 
    QObject::connect(&m_mediaDevices, &QMediaDevices::audioOutputsChanged, this, [this] {
        const QAudioDevice defaultAudioDevice = QMediaDevices::defaultAudioOutput();
        if (defaultAudioDevice == m_defaultAudioDevice)
            return;
 
        m_defaultAudioDevice = defaultAudioDevice;
        if (m_audioDevice.isNull())
            setResolvedAudioDevice(m_defaultAudioDevice);
    });
 
    m_playerReleaseTimer.setTimerType(Qt::VeryCoarseTimer);
    m_playerReleaseTimer.setSingleShot(true);
}
 
QSoundEffectPrivateWithPlayer::~QSoundEffectPrivateWithPlayer()
{
    QObject::disconnect(m_playerStateConnection);
    stop();
    if (m_sampleLoadFuture)
        m_sampleLoadFuture->cancelChain();
}
 
bool QSoundEffectPrivateWithPlayer::setAudioDevice(QAudioDevice device)
{
    if (device == m_audioDevice)
        return false;
 
    m_audioDevice = std::move(device);
    resolveAudioDevice();
    return true;
}
 
void QSoundEffectPrivateWithPlayer::setResolvedAudioDevice(QAudioDevice device)
{
    if (m_resolvedAudioDevice == device)
        return;
 
    m_resolvedAudioDevice = std::move(device);
 
    if (m_player)
        for (const auto &voice : m_voices)
            m_player->stop(voice);
 
    std::vector<std::shared_ptr<QSoundEffectVoice>> voices;
    while (!m_voices.empty()) {
        auto node = m_voices.extract(m_voices.begin());
        voices.push_back(std::move(node.value()));
    }
 
    // the voices are replaced by clones on the new player, the originals are released through the
    // old player, so that their last reference is dropped on its thread
    auto oldPlayer = m_player;
    auto releaseVoices = qScopeGuard([&] {
        if (oldPlayer)
            for (auto &voice : voices)
                oldPlayer->release(std::move(voice));
    });
 
    if (m_sample) {
        bool hasPlayer = updatePlayer(m_sample);
        if (!hasPlayer) {
            setStatus(QSoundEffect::Error);
            return;
        }
 
        for (const auto &voice : voices)
            // we re-allocate a new voice ID and play on the new player
            play(voice->clone(m_player->audioSink().format()));
 
        setStatus(QSoundEffect::Ready);
    } else {
        setStatus(m_sampleLoadFuture ? QSoundEffect::Loading : QSoundEffect::Null);
    }
}
 
void QSoundEffectPrivateWithPlayer::resolveAudioDevice()
{
    if (m_audioDevice.isNull())
        m_defaultAudioDevice = QMediaDevices::defaultAudioOutput();
    setResolvedAudioDevice(m_audioDevice.isNull() ? m_defaultAudioDevice : m_audioDevice);
}
 
QAudioDevice QSoundEffectPrivateWithPlayer::audioDevice() const
{
    return m_audioDevice;
}
 
void QSoundEffectPrivateWithPlayer::setSource(QUrl url, QSampleCache &sampleCache)
{
    if (m_sampleLoadFuture) {
        m_sampleLoadFuture->cancelChain();
        m_sampleLoadFuture = std::nullopt;
    }
 
    if (m_player) {
        QObject::disconnect(m_voiceFinishedConnection);
        QObject::disconnect(m_playerStateConnection);
        m_playerReleaseTimer.callOnTimeout(this, [player = std::move(m_player)] {
            // we keep the player referenced for a little longer, so that later calls to
            // getEngineFor will be able to reuse the existing instance
        }, Qt::SingleShotConnection);
        m_playerReleaseTimer.start();
    }
 
    m_sample = {};
 
    if (url.isEmpty()) {
        setStatus(QSoundEffect::Null);
        return;
    }
 
    if (!url.isValid()) {
        setStatus(QSoundEffect::Error);
        return;
    }
 
    setStatus(QSoundEffect::Loading);
 
    m_sampleLoadFuture =
            sampleCache.requestSampleFuture(url).then(this, [this, url](SharedSamplePtr result) {
        if (result) {
            if (!formatIsSupported(result->format())) {
                qWarning("QSoundEffect: QSoundEffect only supports mono or stereo files");
                setStatus(QSoundEffect::Error);
                return;
            }
 
            bool hasPlayer = updatePlayer(result);
            m_sample = std::move(result);
            if (!hasPlayer) {
                qWarning("QSoundEffect: playback of this format is not supported on the selected "
                         "audio device");
                setStatus(QSoundEffect::Error);
                return;
            }
 
            setStatus(QSoundEffect::Ready);
            if (std::exchange(m_playPending, false)) {
                play();
            }
        } else {
            qWarning("QSoundEffect: Error decoding source %ls", qUtf16Printable(url.toString()));
            setStatus(QSoundEffect::Error);
        }
    });
}
 
void QSoundEffectPrivateWithPlayer::setStatus(QSoundEffect::Status status)
{
    if (status == m_status)
        return;
    m_status = status;
    Q_EMIT q_ptr->statusChanged();
}
 
QSoundEffect::Status QSoundEffectPrivateWithPlayer::status() const
{
    return m_status;
}
 
int QSoundEffectPrivateWithPlayer::loopCount() const
{
    return m_loopCount;
}
 
bool QSoundEffectPrivateWithPlayer::setLoopCount(int loopCount)
{
    if (loopCount == 0)
        loopCount = 1;
 
    if (loopCount == m_loopCount)
        return false;
 
    m_loopCount = loopCount;
 
    if (m_voices.empty())
        return true;
 
    const std::shared_ptr<QSoundEffectVoice> &voice = *m_voices.rbegin();
    voice->m_loopsRemaining.store(loopCount, std::memory_order_relaxed);
 
    setLoopsRemaining(loopCount);
 
    return true;
}
 
int QSoundEffectPrivateWithPlayer::loopsRemaining() const
{
    if (m_voices.empty())
        return 0;
 
    return m_loopsRemaining;
}
 
float QSoundEffectPrivateWithPlayer::volume() const
{
    return m_volume;
}
 
bool QSoundEffectPrivateWithPlayer::setVolume(float volume)
{
    if (m_volume == volume)
        return false;
 
    m_volume = volume;
    for (const auto &voice : m_voices) {
        m_player->visitVoiceRt(voice, [volume](QSoundEffectVoice &voice) {
            voice.m_volume = volume;
        });
    }
    return true;
}
 
bool QSoundEffectPrivateWithPlayer::muted() const
{
    return m_muted;
}
 
bool QSoundEffectPrivateWithPlayer::setMuted(bool muted)
{
    if (m_muted == muted)
        return false;
 
    m_muted = muted;
    for (const auto &voice : m_voices) {
        m_player->visitVoiceRt(voice, [muted](QSoundEffectVoice &voice) {
            voice.m_muted = muted;
        });
    }
    return true;
}
 
void QSoundEffectPrivateWithPlayer::play()
{
    if (!m_sample) {
        m_playPending = true;
        return;
    }
 
    // Error after a failed engine setup: retry, the device may be available again
    if (status() == QSoundEffect::Error && updatePlayer(m_sample))
        setStatus(QSoundEffect::Ready);
 
    if (status() != QSoundEffect::Ready)
        return;
 
    Q_ASSERT(m_player);
 
    if (m_player->audioState() == QtAudio::StoppedState && !updatePlayer(m_sample)) {
        setStatus(QSoundEffect::Error);
        return;
    }
 
    // each `play` will start a new voice
    auto voice = std::make_shared<QSoundEffectVoice>(QRtAudioEngine::allocateVoiceId(), m_sample,
                                                     m_volume, m_muted, m_loopCount,
                                                     m_player->audioSink().format());
 
    play(std::move(voice));
}
 
void QSoundEffectPrivateWithPlayer::stop()
{
    size_t activeVoices = m_voices.size();
    for (const auto &voice : m_voices)
        m_player->stop(voice->voiceId());
    setLoopsRemaining(0);
 
    while (!m_voices.empty()) {
        auto node = m_voices.extract(m_voices.begin());
        m_player->release(std::move(node.value()));
    }
    m_playPending = false;
    if (activeVoices)
        Q_EMIT q_ptr->playingChanged();
}
 
bool QSoundEffectPrivateWithPlayer::playing() const
{
    return !m_voices.empty();
}
 
void QSoundEffectPrivateWithPlayer::play(std::shared_ptr<QSoundEffectVoice> voice)
{
    voice->m_currentLoopChanged.callOnActivated(this, [this, voiceId = voice->voiceId()] {
        auto foundVoice = m_voices.find(voiceId);
        if (foundVoice == m_voices.end())
            return;
 
        if (voiceId != activeVoice())
            return;
 
        setLoopsRemaining((*foundVoice)->loopsRemaining());
    });
 
    // voice lives on the engline thread. they are not QObject, but may contain QObjects.
    voice->m_currentLoopChanged.moveToThread(m_player->thread());
 
    m_player->play(voice);
    m_voices.insert(std::move(voice));
    setLoopsRemaining(m_loopCount);
    if (m_voices.size() == 1)
        Q_EMIT q_ptr->playingChanged();
}
 
bool QSoundEffectPrivateWithPlayer::updatePlayer(const SharedSamplePtr &sample)
{
    Q_ASSERT(sample);
    Q_ASSERT(m_voices.empty());
    QObject::disconnect(m_voiceFinishedConnection);
    QObject::disconnect(m_playerStateConnection);
 
    m_player = {};
    if (m_resolvedAudioDevice.isNull())
        return false;
 
    m_player = [&]() -> std::shared_ptr<QRtAudioEngine> {
        auto player = getEngineFor(m_resolvedAudioDevice, sample->format());
        if (player)
            return player;
 
        QAudioFormat alternativeFormat = sample->format();
        switch (sample->format().channelCount()) {
        case 1:
            alternativeFormat.setChannelCount(2);
            break;
        case 2:
            alternativeFormat.setChannelCount(1);
            break;
        default:
            Q_UNREACHABLE_RETURN({});
        }
 
        return std::make_shared<QRtAudioEngine>(m_resolvedAudioDevice, alternativeFormat);
    }();
 
    if (!m_player || m_player->audioState() == QtAudio::StoppedState)
        return false;
 
    m_voiceFinishedConnection = QObject::connect(m_player.get(), &QRtAudioEngine::voiceFinished,
                                                 this, [this](VoiceId voiceId) {
        if (voiceId == activeVoice())
            setLoopsRemaining(0);
 
        auto found = m_voices.find(voiceId);
        if (found != m_voices.end()) {
            auto node = m_voices.extract(found);
            m_player->release(std::move(node.value()));
            if (m_voices.empty())
                Q_EMIT q_ptr->playingChanged();
        }
    });
 
    m_playerStateConnection = QObject::connect(m_player.get(), &QRtAudioEngine::stateChanged, this,
                                               [this](QtAudio::State state) {
        if (state != QtAudio::StoppedState || m_voices.empty())
            return;
 
        // When the player state fails something must have gone wrong, e.g. the audio device was
        // removed. We clear all voices and reset the loops remaining to 0.
        m_voices.clear();
        setLoopsRemaining(0);
        Q_EMIT q_ptr->playingChanged();
    });
    return true;
}
 
std::optional<VoiceId> QSoundEffectPrivateWithPlayer::activeVoice() const
{
    if (m_voices.empty())
        return std::nullopt;
    return (*m_voices.rbegin())->voiceId();
}
 
bool QSoundEffectPrivateWithPlayer::formatIsSupported(const QAudioFormat &fmt)
{
    switch (fmt.channelCount()) {
    case 1:
    case 2:
        return true;
    default:
        return false;
    }
}
 
void QSoundEffectPrivateWithPlayer::setLoopsRemaining(int loopsRemaining)
{
    if (loopsRemaining == m_loopsRemaining)
        return;
    m_loopsRemaining = loopsRemaining;
    Q_EMIT q_ptr->loopsRemainingChanged();
}
 
} // namespace QtMultimediaPrivate
 
QT_END_NAMESPACE