avfmediaplayer.mm

Go to the documentation of this file.

// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies).
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
 
#include "avfmediaplayer_p.h"
 
#include <avfvideosink_p.h>
#include <common/avfmetadata_p.h>
#include <mediaplayer/avfvideorenderercontrol_p.h>
 
#include <QtMultimedia/qaudiooutput.h>
#include <QtMultimedia/private/qplatformaudiooutput_p.h>
#if defined(Q_OS_IOS) || defined(Q_OS_VISIONOS) || defined(Q_OS_TVOS)
#  include <QtMultimedia/private/qavaudiosessionmanager_p.h>
#endif
 
#include <QtCore/qdir.h>
#include <QtCore/qfileinfo.h>
#include <QtCore/qmath.h>
#include <QtCore/qmimedatabase.h>
#include <QtCore/qmutex.h>
#include <QtCore/qpointer.h>
#include <QtCore/qthread.h>
#include <QtCore/private/qexpected_p.h>
 
#include <optional>
 
#import <AVFoundation/AVFoundation.h>
 
QT_USE_NAMESPACE
 
//AVAsset Keys
static NSString* const AVF_TRACKS_KEY       = @"tracks";
static NSString* const AVF_PLAYABLE_KEY     = @"playable";
 
//AVPlayerItem keys
static NSString* const AVF_STATUS_KEY                   = @"status";
static NSString* const AVF_BUFFER_LIKELY_KEEP_UP_KEY    = @"playbackLikelyToKeepUp";
 
//AVPlayer keys
static NSString* const AVF_RATE_KEY                     = @"rate";
static NSString* const AVF_CURRENT_ITEM_KEY             = @"currentItem";
static NSString* const AVF_CURRENT_ITEM_DURATION_KEY    = @"currentItem.duration";
 
static void *AVFMediaPlayerObserverRateObservationContext =
        &AVFMediaPlayerObserverRateObservationContext;
static void *AVFMediaPlayerObserverStatusObservationContext = &AVFMediaPlayerObserverStatusObservationContext;
static void *AVFMediaPlayerObserverPresentationSizeContext = &AVFMediaPlayerObserverPresentationSizeContext;
static void *AVFMediaPlayerObserverBufferLikelyToKeepUpContext = &AVFMediaPlayerObserverBufferLikelyToKeepUpContext;
static void *AVFMediaPlayerObserverTracksContext = &AVFMediaPlayerObserverTracksContext;
static void *AVFMediaPlayerObserverCurrentItemObservationContext = &AVFMediaPlayerObserverCurrentItemObservationContext;
static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFMediaPlayerObserverCurrentItemDurationObservationContext;
 
@interface AVFMediaPlayerObserver : NSObject<AVAssetResourceLoaderDelegate>
 
@property (readonly, getter=player) AVPlayer* m_player;
@property (readonly, getter=playerItem) AVPlayerItem* m_playerItem;
@property (readonly, getter=playerLayer) AVPlayerLayer* m_playerLayer;
@property (retain) AVPlayerItemTrack *videoTrack;
 
- (AVFMediaPlayerObserver *) initWithMediaPlayerSession:(AVFMediaPlayer *)session;
- (void) setURL:(NSURL *)url mimeType:(NSString *)mimeType;
- (void) unloadMedia;
- (void) prepareToPlayAsset:(AVURLAsset *)asset withKeys:(NSArray *)requestedKeys;
- (void) assetFailedToPrepareForPlayback:(NSError *)error;
- (void) playerItemDidReachEnd:(NSNotification *)notification;
- (void) observeValueForKeyPath:(NSString *)keyPath ofObject:(id)object
                         change:(NSDictionary *)change context:(void *)context;
- (void) clearSession;
- (void) notifySeekComplete;
- (void) dealloc;
- (BOOL) resourceLoader:(AVAssetResourceLoader *)resourceLoader shouldWaitForLoadingOfRequestedResource:(AVAssetResourceLoadingRequest *)loadingRequest;
@end
 
namespace {
 
struct GuardedPlatformPlayer
{
    mutable QMutex mutex;
    AVFMediaPlayer *player{};
 
    explicit operator bool() const
    {
        std::lock_guard guard(mutex);
        return player;
    }
 
    struct not_a_platform_player_t
    {
    };
 
    template <typename Functor>
    auto withPlatformPlayer(Functor &&f)
            -> q23::expected<std::invoke_result_t<Functor, AVFMediaPlayer *>,
                             not_a_platform_player_t>
    {
        std::unique_lock guard(mutex);
        if (!player)
            return q23::unexpected{ not_a_platform_player_t{} };
        if constexpr (std::is_void_v<std::invoke_result_t<Functor, AVFMediaPlayer *>>) {
            f(player);
            return {};
        } else {
            return f(player);
        }
    }
 
    template <typename Functor>
    void invokeWithPlatformPlayer(Functor f)
    {
        std::unique_lock guard(mutex);
        if (!player)
            return;
 
        if (player->thread()->isCurrentThread()) {
            guard.unlock();
            f(player);
        } else {
            QMetaObject::invokeMethod(player, [f = std::move(f), player = player]() {
                f(player);
            });
        }
    }
 
    void clear()
    {
        std::lock_guard<QMutex> guard(mutex);
        player = nullptr;
    }
};
} // namespace
 
#if defined(Q_OS_IOS) || defined(Q_OS_VISIONOS) || defined(Q_OS_TVOS)
using ActivationToken = QtMultimediaPrivate::QAVAudioSessionManager::ActivationToken;
#endif
 
@implementation AVFMediaPlayerObserver {
@private
    GuardedPlatformPlayer m_platformPlayer;
    AVPlayer *m_player;
    AVPlayerItem *m_playerItem;
    AVPlayerLayer *m_playerLayer;
    NSURL *m_URL;
    BOOL m_bufferIsLikelyToKeepUp;
    NSData *m_data;
    NSString *m_mimeType;
#if defined(Q_OS_IOS) || defined(Q_OS_VISIONOS) || defined(Q_OS_TVOS)
    std::optional<ActivationToken> m_sessionActivation;
#endif
}
 
@synthesize m_player, m_playerItem, m_playerLayer;
 
- (AVFMediaPlayerObserver *) initWithMediaPlayerSession:(AVFMediaPlayer *)session
{
    if (!(self = [super init]))
        return nil;
    m_platformPlayer.player = session;
    m_bufferIsLikelyToKeepUp = FALSE;
 
    m_playerLayer = [AVPlayerLayer playerLayerWithPlayer:nil];
    [m_playerLayer retain];
    m_playerLayer.videoGravity = AVLayerVideoGravityResizeAspectFill;
    m_playerLayer.anchorPoint = CGPointMake(0.0f, 0.0f);
    return self;
}
 
- (void) setURL:(NSURL *)url mimeType:(NSString *)mimeType
{
    [m_mimeType release];
    m_mimeType = [mimeType retain];
 
    if (m_URL != url)
    {
        [m_URL release];
        m_URL = [url copy];
 
        //Create an asset for inspection of a resource referenced by a given URL.
        //Load the values for the asset keys "tracks", "playable".
 
        // use __block to avoid maintaining strong references on variables captured by the
        // following block callback
        __block BOOL isAccessing = [m_URL startAccessingSecurityScopedResource];
        __block AVURLAsset *asset = [[AVURLAsset URLAssetWithURL:m_URL options:nil] retain];
        [asset.resourceLoader setDelegate:self queue:dispatch_get_main_queue()];
 
        __block NSArray *requestedKeys = [@[ AVF_TRACKS_KEY, AVF_PLAYABLE_KEY ] retain];
 
        __block AVFMediaPlayerObserver *blockSelf = [self retain];
 
        // Tells the asset to load the values of any of the specified keys that are not already loaded.
        [asset loadValuesAsynchronouslyForKeys:requestedKeys
                             completionHandler:^{
                                 dispatch_async(dispatch_get_main_queue(), ^{
                                     if (isAccessing)
                                         [m_URL stopAccessingSecurityScopedResource];
                                     [blockSelf prepareToPlayAsset:asset withKeys:requestedKeys];
                                     [asset release];
                                     [requestedKeys release];
                                     [blockSelf release];
                                 });
                             }];
    }
}
 
- (void) unloadMedia
{
    if (m_playerItem) {
        [m_playerItem removeObserver:self forKeyPath:@"presentationSize"];
        [m_playerItem removeObserver:self forKeyPath:AVF_STATUS_KEY];
        [m_playerItem removeObserver:self forKeyPath:AVF_BUFFER_LIKELY_KEEP_UP_KEY];
        [m_playerItem removeObserver:self forKeyPath:AVF_TRACKS_KEY];
 
        [[NSNotificationCenter defaultCenter] removeObserver:self
                                                    name:AVPlayerItemDidPlayToEndTimeNotification
                                                    object:m_playerItem];
        m_playerItem = nullptr;
    }
    if (m_player) {
        [m_player setRate:0.0];
        [m_player removeObserver:self forKeyPath:AVF_CURRENT_ITEM_DURATION_KEY];
        [m_player removeObserver:self forKeyPath:AVF_CURRENT_ITEM_KEY];
        [m_player removeObserver:self forKeyPath:AVF_RATE_KEY];
        [m_player replaceCurrentItemWithPlayerItem:nil];
 
        // Defer the release of AVPlayer to allow CoreMedia/VideoToolbox
        // dispatch queues to finish pending operations. Releasing the
        // player synchronously can cause sporadic crashes on macOS 14
        // when background threads still reference internal resources.
        AVPlayer *player = m_player;
        m_player = nullptr;
        dispatch_async(dispatch_get_main_queue(), ^{
            [player release];
        });
    }
    if (m_playerLayer)
        m_playerLayer.player = nil;
#if defined(Q_OS_IOS) || defined(Q_OS_VISIONOS) || defined(Q_OS_TVOS)
    m_sessionActivation.reset();
#endif
}
 
- (void) prepareToPlayAsset:(AVURLAsset *)asset
                   withKeys:(NSArray *)requestedKeys
{
    if (!m_platformPlayer)
        return;
 
#if defined(Q_OS_IOS) || defined(Q_OS_VISIONOS) || defined(Q_OS_TVOS)
    auto sessionActivation = QtMultimediaPrivate::QAVAudioSessionManager::instance()->activate({
            false,
            AVAudioSessionCategoryOptionMixWithOthers,
    });
#endif
 
    //Make sure that the value of each key has loaded successfully.
    for (NSString *thisKey in requestedKeys)
    {
        NSError *error = nil;
        AVKeyValueStatus keyStatus = [asset statusOfValueForKey:thisKey error:&error];
#ifdef QT_DEBUG_AVF
        qDebug() << Q_FUNC_INFO << [thisKey UTF8String] << " status: " << keyStatus;
#endif
        if (keyStatus == AVKeyValueStatusFailed)
        {
            [self assetFailedToPrepareForPlayback:error];
            return;
        }
    }
 
    //Use the AVAsset playable property to detect whether the asset can be played.
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << "isPlayable: " << [asset isPlayable];
#endif
    if (!asset.playable)
        qWarning() << "Asset reported to be not playable. Playback of this asset may not be possible.";
 
    //At this point we're ready to set up for playback of the asset.
    //Stop observing our prior AVPlayerItem, if we have one.
    if (m_playerItem)
    {
        //Remove existing player item key value observers and notifications.
        [self unloadMedia];
    }
 
    //Create a new instance of AVPlayerItem from the now successfully loaded AVAsset.
    m_playerItem = [AVPlayerItem playerItemWithAsset:asset];
    if (!m_playerItem) {
        qWarning() << "Failed to create player item";
        //Generate an error describing the failure.
        NSString *localizedDescription = NSLocalizedString(@"Item cannot be played", @"Item cannot be played description");
        NSString *localizedFailureReason = NSLocalizedString(@"The assets tracks were loaded, but couldn't create player item.", @"Item cannot be played failure reason");
        NSDictionary *errorDict = @{
            NSLocalizedDescriptionKey : localizedDescription,
            NSLocalizedFailureReasonErrorKey : localizedFailureReason,
        };
        NSError *assetCannotBePlayedError = [NSError errorWithDomain:@"StitchedStreamPlayer" code:0 userInfo:errorDict];
 
        [self assetFailedToPrepareForPlayback:assetCannotBePlayedError];
        return;
    }
 
    //Observe the player item "status" key to determine when it is ready to play.
    [m_playerItem addObserver:self
                   forKeyPath:AVF_STATUS_KEY
                      options:NSKeyValueObservingOptionInitial | NSKeyValueObservingOptionNew
                      context:AVFMediaPlayerObserverStatusObservationContext];
 
    [m_playerItem addObserver:self
                   forKeyPath:@"presentationSize"
                      options:NSKeyValueObservingOptionInitial | NSKeyValueObservingOptionNew
                      context:AVFMediaPlayerObserverPresentationSizeContext];
 
    [m_playerItem addObserver:self
                   forKeyPath:AVF_BUFFER_LIKELY_KEEP_UP_KEY
                      options:NSKeyValueObservingOptionInitial | NSKeyValueObservingOptionNew
                      context:AVFMediaPlayerObserverBufferLikelyToKeepUpContext];
 
    [m_playerItem addObserver:self
                   forKeyPath:AVF_TRACKS_KEY
                      options:NSKeyValueObservingOptionInitial | NSKeyValueObservingOptionNew
                      context:AVFMediaPlayerObserverTracksContext];
 
    //When the player item has played to its end time we'll toggle
    //the movie controller Pause button to be the Play button
    [[NSNotificationCenter defaultCenter] addObserver:self
                                             selector:@selector(playerItemDidReachEnd:)
                                                 name:AVPlayerItemDidPlayToEndTimeNotification
                                               object:m_playerItem];
 
    //Get a new AVPlayer initialized to play the specified player item.
    m_player = [AVPlayer playerWithPlayerItem:m_playerItem];
    [m_player retain];
 
    //Set the initial audio ouptut settings on new player object
    {
        m_platformPlayer.withPlatformPlayer([&](AVFMediaPlayer *player) {
            auto *audioOutput = player->m_audioOutput;
            m_player.volume = (audioOutput ? audioOutput->volume : 1.);
            m_player.muted = (audioOutput ? audioOutput->muted : true);
            player->updateAudioOutputDevice();
        });
    }
 
    //Assign the output layer to the new player
    m_playerLayer.player = m_player;
 
    //Observe the AVPlayer "currentItem" property to find out when any
    //AVPlayer replaceCurrentItemWithPlayerItem: replacement will/did
    //occur.
    [m_player addObserver:self
                          forKeyPath:AVF_CURRENT_ITEM_KEY
                          options:NSKeyValueObservingOptionInitial | NSKeyValueObservingOptionNew
                          context:AVFMediaPlayerObserverCurrentItemObservationContext];
 
    //Observe the AVPlayer "rate" property to update the scrubber control.
    [m_player addObserver:self
                          forKeyPath:AVF_RATE_KEY
                          options:NSKeyValueObservingOptionInitial | NSKeyValueObservingOptionNew
                          context:AVFMediaPlayerObserverRateObservationContext];
 
    //Observe the duration for getting the buffer state
    [m_player addObserver:self
                          forKeyPath:AVF_CURRENT_ITEM_DURATION_KEY
                          options:0
                          context:AVFMediaPlayerObserverCurrentItemDurationObservationContext];
#if defined(Q_OS_IOS) || defined(Q_OS_VISIONOS) || defined(Q_OS_TVOS)
    m_sessionActivation = std::move(sessionActivation);
#endif
}
 
-(void) assetFailedToPrepareForPlayback:(NSError *)error
{
    QMediaPlayer::Error errorCode = QMediaPlayer::FormatError;
    if (error) {
        NSError *underlyingError = error.userInfo[NSUnderlyingErrorKey];
        if (underlyingError && ![underlyingError.domain isEqualToString:AVFoundationErrorDomain])
            errorCode = QMediaPlayer::ResourceError;
    }
    m_platformPlayer.invokeWithPlatformPlayer([errorCode](AVFMediaPlayer *platformPlayer) {
        platformPlayer->processMediaLoadError(errorCode);
    });
 
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO;
    qDebug() << [[error localizedDescription] UTF8String];
    qDebug() << [[error localizedFailureReason] UTF8String];
    qDebug() << [[error localizedRecoverySuggestion] UTF8String];
#endif
}
 
- (void) playerItemDidReachEnd:(NSNotification *)notification
{
    Q_UNUSED(notification);
 
    m_platformPlayer.invokeWithPlatformPlayer([](AVFMediaPlayer *platformPlayer) {
        platformPlayer->processEOS();
    });
}
 
- (void) observeValueForKeyPath:(NSString*) path
                       ofObject:(id)object
                         change:(NSDictionary*)change
                        context:(void*)context
{
    //AVPlayerItem "status" property value observer.
    if (context == AVFMediaPlayerObserverStatusObservationContext)
    {
        AVPlayerStatus status = (AVPlayerStatus)[[change objectForKey:NSKeyValueChangeNewKey] integerValue];
        switch (status)
        {
            //Indicates that the status of the player is not yet known because
            //it has not tried to load new media resources for playback
            case AVPlayerStatusUnknown:
            {
                //QMetaObject::invokeMethod(m_session, "processLoadStateChange", Qt::AutoConnection);
            }
            break;
 
            case AVPlayerStatusReadyToPlay:
            {
                //Once the AVPlayerItem becomes ready to play, i.e.
                //[playerItem status] == AVPlayerItemStatusReadyToPlay,
                //its duration can be fetched from the item.
 
                m_platformPlayer.invokeWithPlatformPlayer([](AVFMediaPlayer *platformPlayer) {
                    platformPlayer->processLoadStateChange();
                });
            }
            break;
 
            case AVPlayerStatusFailed:
            {
                AVPlayerItem *playerItem = static_cast<AVPlayerItem*>(object);
                [self assetFailedToPrepareForPlayback:playerItem.error];
 
                m_platformPlayer.invokeWithPlatformPlayer([](AVFMediaPlayer *platformPlayer) {
                    platformPlayer->processLoadStateChange();
                });
            }
            break;
        }
    } else if (context == AVFMediaPlayerObserverPresentationSizeContext) {
        QSize size(m_playerItem.presentationSize.width, m_playerItem.presentationSize.height);
        m_platformPlayer.invokeWithPlatformPlayer([size](AVFMediaPlayer *platformPlayer) {
            platformPlayer->nativeSizeChanged(size);
        });
    } else if (context == AVFMediaPlayerObserverBufferLikelyToKeepUpContext)
    {
        const bool isPlaybackLikelyToKeepUp = [m_playerItem isPlaybackLikelyToKeepUp];
        if (isPlaybackLikelyToKeepUp != m_bufferIsLikelyToKeepUp) {
            m_bufferIsLikelyToKeepUp = isPlaybackLikelyToKeepUp;
            int bufferProgress = isPlaybackLikelyToKeepUp ? 100 : 0;
 
            m_platformPlayer.invokeWithPlatformPlayer(
                    [bufferProgress](AVFMediaPlayer *platformPlayer) {
                platformPlayer->processBufferStateChange(bufferProgress);
            });
        }
    }
    else if (context == AVFMediaPlayerObserverTracksContext)
    {
        m_platformPlayer.invokeWithPlatformPlayer([](AVFMediaPlayer *platformPlayer) {
            platformPlayer->updateTracks();
        });
    }
    //AVPlayer "rate" property value observer.
    else if (context == AVFMediaPlayerObserverRateObservationContext) {
        //QMetaObject::invokeMethod(m_session, "setPlaybackRate",  Qt::AutoConnection, Q_ARG(qreal, [m_player rate]));
    }
    //AVPlayer "currentItem" property observer.
    //Called when the AVPlayer replaceCurrentItemWithPlayerItem:
    //replacement will/did occur.
    else if (context == AVFMediaPlayerObserverCurrentItemObservationContext) {
        AVPlayerItem *newPlayerItem = [change objectForKey:NSKeyValueChangeNewKey];
        if (m_playerItem != newPlayerItem)
            m_playerItem = newPlayerItem;
    } else if (context == AVFMediaPlayerObserverCurrentItemDurationObservationContext) {
        const CMTime time = [m_playerItem duration];
        const qint64 dur =  static_cast<qint64>(float(time.value) / float(time.timescale) * 1000.0f);
 
        m_platformPlayer.invokeWithPlatformPlayer([dur](AVFMediaPlayer *platformPlayer) {
            platformPlayer->processDurationChange(dur);
        });
    } else {
        [super observeValueForKeyPath:path ofObject:object change:change context:context];
    }
}
 
- (void)clearSession
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO;
#endif
    m_platformPlayer.clear();
}
 
- (void)notifySeekComplete
{
    m_platformPlayer.withPlatformPlayer([](AVFMediaPlayer *player) {
        player->seekCompleted();
    });
}
 
- (void) dealloc
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO;
#endif
    [self unloadMedia];
 
    m_platformPlayer.clear();
 
    if (m_URL) {
        [m_URL release];
    }
 
    [m_mimeType release];
    [m_playerLayer release];
    // 'videoTrack' is a 'retain' property, but still needs a
    // manual 'release' (i.e. setting to nil):
    self.videoTrack = nil;
    [super dealloc];
}
 
- (BOOL) resourceLoader:(AVAssetResourceLoader *)resourceLoader shouldWaitForLoadingOfRequestedResource:(AVAssetResourceLoadingRequest *)loadingRequest
{
    Q_UNUSED(resourceLoader);
 
    if (![loadingRequest.request.URL.scheme isEqualToString:@"iodevice"])
        return NO;
 
    auto result = m_platformPlayer.withPlatformPlayer([&](AVFMediaPlayer *platformPlayer) {
        QIODevice *device = platformPlayer->mediaStream();
        if (!device)
            return NO;
 
        device->seek(loadingRequest.dataRequest.requestedOffset);
        if (loadingRequest.contentInformationRequest) {
            loadingRequest.contentInformationRequest.contentType = m_mimeType;
            loadingRequest.contentInformationRequest.contentLength = device->size();
            loadingRequest.contentInformationRequest.byteRangeAccessSupported = YES;
        }
 
        if (loadingRequest.dataRequest) {
            NSInteger requestedLength = loadingRequest.dataRequest.requestedLength;
            int maxBytes = qMin(32 * 1024, int(requestedLength));
            QByteArray buffer;
            buffer.resize(maxBytes);
 
            NSInteger submitted = 0;
            while (submitted < requestedLength) {
                qint64 len = device->read(buffer.data(), maxBytes);
                if (len < 1)
                    break;
 
                [loadingRequest.dataRequest respondWithData:buffer.toNSData()];
                submitted += len;
            }
 
            // Finish loading even if not all bytes submitted.
            [loadingRequest finishLoading];
        }
 
        return YES;
    });
 
    return result.value_or(NO);
}
@end
 
AVFMediaPlayer::AVFMediaPlayer(QMediaPlayer *player)
    : QObject(player),
      QPlatformMediaPlayer(player),
      m_mediaStream(nullptr),
      m_rate(1.0),
      m_requestedPosition(-1),
      m_duration(0),
      m_bufferProgress(0)
{
    m_observer = [[AVFMediaPlayerObserver alloc] initWithMediaPlayerSession:this];
    connect(&m_playbackTimer, &QTimer::timeout, this, &AVFMediaPlayer::processPositionChange);
    setVideoOutput(new AVFVideoRendererControl(this));
}
 
AVFMediaPlayer::~AVFMediaPlayer()
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO;
#endif
 
    // Unload media before the C++ side is torn down, so that
    // CoreMedia/VideoToolbox threads don't outlive our objects.
    [m_observer unloadMedia];
 
    //Detatch the session from the sessionObserver (which could still be alive trying to communicate with this session).
    [m_observer clearSession];
    [m_observer release];
}
 
void AVFMediaPlayer::setVideoSink(QVideoSink *sink)
{
    m_videoSink = sink ? static_cast<AVFVideoSink *>(sink->platformVideoSink()): nullptr;
    m_videoOutput->setVideoSink(m_videoSink);
}
 
void AVFMediaPlayer::setVideoOutput(AVFVideoRendererControl *output)
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << output;
#endif
 
    if (m_videoOutput == output)
        return;
 
    //Set the current output layer to null to stop rendering
    if (m_videoOutput) {
        m_videoOutput->setLayer(nullptr);
    }
 
    m_videoOutput = output;
 
    if (m_videoOutput && state() != QMediaPlayer::StoppedState)
        m_videoOutput->setLayer([m_observer playerLayer]);
}
 
AVAsset *AVFMediaPlayer::currentAssetHandle()
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO;
#endif
    AVAsset *currentAsset = [[m_observer playerItem] asset];
    return currentAsset;
}
 
QUrl AVFMediaPlayer::media() const
{
    return m_resources;
}
 
QIODevice *AVFMediaPlayer::mediaStream() const
{
    return m_mediaStream;
}
 
static void setURL(AVFMediaPlayerObserver *observer, const QUrl &url, const QString &mimeType = QString())
{
    QUrl resolvedUrl = url;
    // AVFoundation cannot handle file URLs with a relative path
    if (url.isLocalFile() && !QDir::isAbsolutePath(url.path()))
        resolvedUrl = QUrl::fromLocalFile(QFileInfo(url.path()).absoluteFilePath());
    NSURL *nsurl = resolvedUrl.toNSURL();
    [observer setURL:nsurl mimeType:[NSString stringWithUTF8String:mimeType.toLatin1().constData()]];
}
 
 
void AVFMediaPlayer::setMedia(const QUrl &content, QIODevice *stream)
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << content.request().url();
#endif
 
    [m_observer unloadMedia];
 
    m_resources = content;
    resetStream(stream);
 
    m_requestedPosition = -1;
    orientationChanged(QtVideo::Rotation::None, false);
    positionChanged(position());
    if (m_duration != 0) {
        m_duration = 0;
        durationChanged(0);
    }
    if (!m_metaData.isEmpty()) {
        m_metaData.clear();
        metaDataChanged();
    }
    resetBufferProgress();
    for (size_t i = 0; i < QPlatformMediaPlayer::NTrackTypes; ++i) {
        tracks[i].clear();
        nativeTracks[i].clear();
    }
    tracksChanged();
 
    if (!m_mediaStream && content.isEmpty()) {
        seekableChanged(false);
        audioAvailableChanged(false);
        videoAvailableChanged(false);
 
        mediaStatusChanged(QMediaPlayer::NoMedia);
        stateChanged(QMediaPlayer::StoppedState);
 
        return;
    }
 
    mediaStatusChanged(QMediaPlayer::LoadingMedia);
 
    if (m_mediaStream) {
        // If there is a data, try to load it,
        // otherwise wait for readyRead.
        if (m_mediaStream->size())
            loadMediaStream();
    } else {
        //Load AVURLAsset
        //initialize asset using content's URL
        setURL(m_observer, m_resources);
    }
 
    stateChanged(QMediaPlayer::StoppedState);
}
 
qint64 AVFMediaPlayer::position() const
{
    AVPlayerItem *playerItem = [m_observer playerItem];
 
    if (m_requestedPosition != -1)
        return m_requestedPosition;
 
    if (!playerItem)
        return 0;
 
    CMTime time = [playerItem currentTime];
    return static_cast<quint64>(float(time.value) / float(time.timescale) * 1000.0f);
}
 
qint64 AVFMediaPlayer::duration() const
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO;
#endif
    return m_duration;
}
 
float AVFMediaPlayer::bufferProgress() const
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO;
#endif
    return m_bufferProgress/100.;
}
 
QMediaTimeRange AVFMediaPlayer::availablePlaybackRanges() const
{
    AVPlayerItem *playerItem = [m_observer playerItem];
 
    if (!playerItem)
        return {};
 
    if (state() == QMediaPlayer::StoppedState)
        return {};
 
    QMediaTimeRange timeRanges;
 
    NSArray *ranges = [playerItem loadedTimeRanges];
    for (NSValue *timeRange in ranges) {
        CMTimeRange currentTimeRange = [timeRange CMTimeRangeValue];
        qint64 startTime = qint64(float(currentTimeRange.start.value) / currentTimeRange.start.timescale * 1000.0);
        timeRanges.addInterval(startTime, startTime + qint64(float(currentTimeRange.duration.value) / currentTimeRange.duration.timescale * 1000.0));
    }
    return timeRanges;
}
 
qreal AVFMediaPlayer::playbackRate() const
{
    return m_rate;
}
 
void AVFMediaPlayer::setAudioOutput(QPlatformAudioOutput *output)
{
    if (m_audioOutput == output)
        return;
    if (m_audioOutput)
        m_audioOutput->q->disconnect(this);
    m_audioOutput = output;
    if (m_audioOutput) {
        connect(m_audioOutput->q, &QAudioOutput::deviceChanged, this, &AVFMediaPlayer::updateAudioOutputDevice);
        connect(m_audioOutput->q, &QAudioOutput::volumeChanged, this, &AVFMediaPlayer::setVolume);
        connect(m_audioOutput->q, &QAudioOutput::mutedChanged, this, &AVFMediaPlayer::setMuted);
        //connect(m_audioOutput->q, &QAudioOutput::audioRoleChanged, this, &AVFMediaPlayer::setAudioRole);
    }
    updateAudioOutputDevice();
    setMuted(m_audioOutput ? m_audioOutput->muted : true);
    setVolume(m_audioOutput ? m_audioOutput->volume : 1.);
}
 
QMediaMetaData AVFMediaPlayer::metaData() const
{
    return m_metaData;
}
 
void AVFMediaPlayer::setPlaybackRate(qreal rate)
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << rate;
#endif
 
    if (QtPrivate::fuzzyCompare(m_rate, rate))
        return;
 
    m_rate = rate;
 
    AVPlayer *player = [m_observer player];
    if (player && state() == QMediaPlayer::PlayingState)
        [player setRate:m_rate];
 
    playbackRateChanged(m_rate);
}
 
void AVFMediaPlayer::setPosition(qint64 pos)
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << pos;
#endif
 
    if (pos == position())
        return;
 
    AVPlayerItem *playerItem = [m_observer playerItem];
    if (!playerItem) {
        m_requestedPosition = pos;
        positionChanged(m_requestedPosition);
        return;
    }
 
    if (!isSeekable()) {
        if (m_requestedPosition != -1) {
            m_requestedPosition = -1;
            positionChanged(position());
        }
        return;
    }
 
    pos = qMax(qint64(0), pos);
    if (duration() > 0)
        pos = qMin(pos, duration());
    m_requestedPosition = pos;
 
    CMTime newTime = [playerItem currentTime];
    newTime.value = (pos / 1000.0f) * newTime.timescale;
    AVFMediaPlayerObserver *observer = m_observer;
    [playerItem seekToTime:newTime toleranceBefore:kCMTimeZero toleranceAfter:kCMTimeZero
                           completionHandler:^(BOOL finished) {
                                if (finished)
                                    [observer notifySeekComplete];
                           }];
 
    positionChanged(pos);
 
    // Reset media status if the current status is EndOfMedia
    if (mediaStatus() == QMediaPlayer::EndOfMedia) {
        QMediaPlayer::MediaStatus newMediaStatus = (state() == QMediaPlayer::PausedState)
                ? QMediaPlayer::BufferedMedia
                : QMediaPlayer::LoadedMedia;
        mediaStatusChanged(newMediaStatus);
    }
}
 
void AVFMediaPlayer::play()
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << "currently: " << state();
#endif
 
    if (mediaStatus() == QMediaPlayer::NoMedia || mediaStatus() == QMediaPlayer::InvalidMedia)
        return;
 
    if (state() == QMediaPlayer::PlayingState)
        return;
 
    if (state() != QMediaPlayer::PausedState)
        resetCurrentLoop();
 
    if (m_videoOutput && m_videoSink)
        m_videoOutput->setLayer([m_observer playerLayer]);
 
    // Reset media status if the current status is EndOfMedia
    if (mediaStatus() == QMediaPlayer::EndOfMedia)
        setPosition(0);
 
    if (mediaStatus() == QMediaPlayer::LoadedMedia
        || mediaStatus() == QMediaPlayer::BufferedMedia) {
        // Setting the rate starts playback
        [[m_observer player] setRate:m_rate];
    }
 
    processLoadStateChange(QMediaPlayer::PlayingState);
 
    stateChanged(QMediaPlayer::PlayingState);
    m_playbackTimer.start(100);
}
 
void AVFMediaPlayer::pause()
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << "currently: " << state();
#endif
 
    if (mediaStatus() == QMediaPlayer::NoMedia || mediaStatus() == QMediaPlayer::InvalidMedia)
        return;
 
    if (state() == QMediaPlayer::PausedState)
        return;
 
    stateChanged(QMediaPlayer::PausedState);
 
    if (m_videoOutput && m_videoSink)
        m_videoOutput->setLayer([m_observer playerLayer]);
 
    [[m_observer player] pause];
 
    // Reset media status if the current status is EndOfMedia
    if (mediaStatus() == QMediaPlayer::EndOfMedia)
        setPosition(0);
 
    positionChanged(position());
    m_playbackTimer.stop();
}
 
void AVFMediaPlayer::stop()
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << "currently: " << state();
#endif
 
    if (state() == QMediaPlayer::StoppedState && mediaStatus() != QMediaPlayer::EndOfMedia)
        return;
 
    // AVPlayer doesn't have stop(), only pause() and play().
    [[m_observer player] pause];
    setPosition(0);
 
    if (m_videoOutput)
        m_videoOutput->setLayer(nullptr);
 
    resetBufferProgress();
 
    if (mediaStatus() == QMediaPlayer::BufferedMedia || mediaStatus() == QMediaPlayer::EndOfMedia)
        mediaStatusChanged(QMediaPlayer::LoadedMedia);
 
    stateChanged(QMediaPlayer::StoppedState);
    m_playbackTimer.stop();
}
 
void AVFMediaPlayer::setVolume(float volume)
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << volume;
#endif
 
    AVPlayer *player = [m_observer player];
    if (player)
        player.volume = volume;
}
 
void AVFMediaPlayer::setMuted(bool muted)
{
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << muted;
#endif
 
    AVPlayer *player = [m_observer player];
    if (player)
        player.muted = muted;
}
 
void AVFMediaPlayer::updateAudioOutputDevice()
{
#ifdef Q_OS_MACOS
    AVPlayer *player = [m_observer player];
    if (!player)
        return;
 
    if (!m_audioOutput || m_audioOutput->device.id().isEmpty()) {
        if (!m_audioOutput)
            player.muted = true;
        player.audioOutputDeviceUniqueID = nil;
    } else {
        NSString *str = QString::fromUtf8(m_audioOutput->device.id()).toNSString();
        player.audioOutputDeviceUniqueID = str;
    }
#endif
}
 
void AVFMediaPlayer::processEOS()
{
    if (doLoop()) {
        positionChanged(duration());
        setPosition(0);
        [[m_observer player] setRate:m_rate];
        return;
    }
 
    //AVPlayerItem has reached end of track/stream
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO;
#endif
    positionChanged(position());
 
    if (m_videoOutput)
        m_videoOutput->setLayer(nullptr);
 
    resetBufferProgress();
 
    stateChanged(QMediaPlayer::StoppedState);
    mediaStatusChanged(QMediaPlayer::EndOfMedia);
}
 
void AVFMediaPlayer::processLoadStateChange(QMediaPlayer::PlaybackState newState)
{
    AVPlayerStatus currentStatus = [[m_observer player] status];
 
#ifdef QT_DEBUG_AVF
    qDebug() << Q_FUNC_INFO << currentStatus << ", " << mediaStatus() << ", " << newState;
#endif
 
    if (mediaStatus() == QMediaPlayer::NoMedia)
        return;
 
    if (currentStatus == AVPlayerStatusReadyToPlay) {
 
        AVPlayerItem *playerItem = [m_observer playerItem];
 
        applyPitchCompensation(m_pitchCompensationEnabled);
 
        // get the meta data
        m_metaData = AVFMetaData::fromAsset(playerItem.asset);
        metaDataChanged();
        updateTracks();
 
        if (playerItem) {
            seekableChanged([[playerItem seekableTimeRanges] count] > 0);
 
            // Get the native size of the video, and reset the bounds of the player layer
            AVPlayerLayer *playerLayer = [m_observer playerLayer];
            if (m_observer.videoTrack && playerLayer) {
                if (!playerLayer.bounds.size.width || !playerLayer.bounds.size.height) {
                    playerLayer.bounds = CGRectMake(0.0f, 0.0f,
                                                    m_observer.videoTrack.assetTrack.naturalSize.width,
                                                    m_observer.videoTrack.assetTrack.naturalSize.height);
                }
            }
 
            if (m_requestedPosition != -1)
                setPosition(m_requestedPosition);
        }
 
        QMediaPlayer::MediaStatus newStatus = (newState != QMediaPlayer::StoppedState)
                ? QMediaPlayer::BufferedMedia
                : QMediaPlayer::LoadedMedia;
 
        if (newStatus != mediaStatus()) {
            if (newStatus == QMediaPlayer::BufferedMedia
                && mediaStatus() == QMediaPlayer::LoadingMedia) {
                // Emit intermediate transitions to match expected signal sequence
                mediaStatusChanged(QMediaPlayer::LoadedMedia);
                mediaStatusChanged(QMediaPlayer::BufferingMedia);
            } else if (newStatus == QMediaPlayer::BufferedMedia
                       && mediaStatus() == QMediaPlayer::LoadedMedia) {
                mediaStatusChanged(QMediaPlayer::BufferingMedia);
            }
            mediaStatusChanged(newStatus);
        }
    }
 
    if (newState == QMediaPlayer::PlayingState && [m_observer player]) {
        // Setting the rate is enough to start playback, no need to call play()
        [[m_observer player] setRate:m_rate];
        m_playbackTimer.start();
    }
}
 
 
void AVFMediaPlayer::processLoadStateChange()
{
    processLoadStateChange(state());
}
 
 
void AVFMediaPlayer::processLoadStateFailure()
{
    stateChanged(QMediaPlayer::StoppedState);
}
 
void AVFMediaPlayer::processBufferStateChange(int bufferProgress)
{
    if (state() == QMediaPlayer::StoppedState)
        return;
 
    if (bufferProgress == m_bufferProgress)
        return;
 
    auto status = mediaStatus();
    // Buffered -> unbuffered.
    if (!bufferProgress) {
        status = QMediaPlayer::StalledMedia;
    } else if (status == QMediaPlayer::StalledMedia) {
        status = QMediaPlayer::BufferedMedia;
        // Resume playback.
        if (state() == QMediaPlayer::PlayingState) {
            [[m_observer player] setRate:m_rate];
            m_playbackTimer.start();
        }
    }
 
    mediaStatusChanged(status);
 
    m_bufferProgress = bufferProgress;
    bufferProgressChanged(bufferProgress / 100.);
}
 
void AVFMediaPlayer::processDurationChange(qint64 duration)
{
    if (duration == m_duration)
        return;
 
    m_duration = duration;
    durationChanged(duration);
}
 
void AVFMediaPlayer::processPositionChange()
{
    if (state() == QMediaPlayer::StoppedState)
        return;
 
    positionChanged(position());
}
 
void AVFMediaPlayer::processMediaLoadError(QMediaPlayer::Error errorCode)
{
    if (m_requestedPosition != -1) {
        m_requestedPosition = -1;
        positionChanged(position());
    }
 
    setInvalidMediaWithError(errorCode, tr("Failed to load media"));
}
 
void AVFMediaPlayer::seekCompleted()
{
    m_requestedPosition = -1;
}
 
void AVFMediaPlayer::streamReady()
{
    loadMediaStream();
}
 
void AVFMediaPlayer::loadMediaStream()
{
    QString suffix;
    if (!m_resources.isEmpty())
        suffix = QFileInfo(m_resources.path()).suffix();
    if (suffix.isEmpty() && m_mediaStream)
        suffix = QMimeDatabase().mimeTypeForData(m_mediaStream).preferredSuffix();
    const QString url = QStringLiteral("iodevice:///iodevice.") + suffix;
    setURL(m_observer, QUrl(url), suffix);
}
 
void AVFMediaPlayer::streamDestroyed()
{
    resetStream(nullptr);
}
 
void AVFMediaPlayer::updateTracks()
{
    bool firstLoad = true;
    for (size_t i = 0; i < QPlatformMediaPlayer::NTrackTypes; ++i) {
        if (tracks[i].count())
            firstLoad = false;
        tracks[i].clear();
        nativeTracks[i].clear();
    }
    bool hasAudio = false;
    bool hasVideo = false;
    AVPlayerItem *playerItem = [m_observer playerItem];
    if (playerItem) {
        // Check each track for audio and video content
        NSArray *tracks =  playerItem.tracks;
        for (AVPlayerItemTrack *track in tracks) {
            AVAssetTrack *assetTrack = track.assetTrack;
            if (assetTrack) {
                std::optional<TrackType> qtTrack;
                if ([assetTrack.mediaType isEqualToString:AVMediaTypeAudio]) {
                    qtTrack = TrackType::AudioStream;
                    hasAudio = true;
                } else if ([assetTrack.mediaType isEqualToString:AVMediaTypeVideo]) {
                    qtTrack = TrackType::VideoStream;
                    hasVideo = true;
                    if (m_observer.videoTrack != track) {
                        m_observer.videoTrack = track;
                        bool isMirrored = false;
                        QtVideo::Rotation orientation = QtVideo::Rotation::None;
                        videoOrientationForAssetTrack(assetTrack, orientation, isMirrored);
                        orientationChanged(orientation, isMirrored);
                    }
                }
                else if ([assetTrack.mediaType isEqualToString:AVMediaTypeSubtitle]) {
                    qtTrack = TrackType::SubtitleStream;
                }
                if (qtTrack) {
                    QMediaMetaData metaData = AVFMetaData::fromAssetTrack(assetTrack);
                    this->tracks[*qtTrack].append(metaData);
                    nativeTracks[*qtTrack].append(track);
                }
            }
        }
        // subtitles are disabled by default
        if (firstLoad)
            setActiveTrack(TrackType::SubtitleStream, -1);
    }
    audioAvailableChanged(hasAudio);
    videoAvailableChanged(hasVideo);
    tracksChanged();
}
 
void AVFMediaPlayer::setActiveTrack(TrackType type, int index)
{
    const auto &t = nativeTracks[type];
    if (type == TrackType::SubtitleStream) {
        // subtitle streams are not always automatically enabled on macOS/iOS.
        // this hack ensures they get enables and we actually get the text
        AVPlayerItem *playerItem = m_observer.m_playerItem;
        if (playerItem) {
            AVAsset *asset = playerItem.asset;
            if (!asset)
                return;
            [asset loadMediaSelectionGroupForMediaCharacteristic:AVMediaCharacteristicLegible
                                               completionHandler:[=](AVMediaSelectionGroup *group,
                                                                     NSError *error) {
                if (error)
                    return; // FIXME: handle error
                auto *options = group.options;
                if (options.count)
                    [playerItem selectMediaOption:options.firstObject inMediaSelectionGroup:group];
            }];
        }
    }
    for (int i = 0; i < t.count(); ++i)
        t.at(i).enabled = (i == index);
    activeTracksChanged();
}
 
int AVFMediaPlayer::activeTrack(TrackType type)
{
    const auto &t = nativeTracks[type];
    for (int i = 0; i < t.count(); ++i)
        if (t.at(i).enabled)
            return i;
    return -1;
}
 
int AVFMediaPlayer::trackCount(TrackType type)
{
    return nativeTracks[type].count();
}
 
QMediaMetaData AVFMediaPlayer::trackMetaData(TrackType type, int trackNumber)
{
    const auto &t = tracks[type];
    if (trackNumber < 0 || trackNumber >= t.count())
        return QMediaMetaData();
    return t.at(trackNumber);
}
 
void AVFMediaPlayer::resetStream(QIODevice *stream)
{
    if (m_mediaStream) {
        disconnect(m_mediaStream, &QIODevice::readyRead, this, &AVFMediaPlayer::streamReady);
        disconnect(m_mediaStream, &QIODevice::destroyed, this, &AVFMediaPlayer::streamDestroyed);
    }
 
    m_mediaStream = stream;
 
    if (m_mediaStream) {
        connect(m_mediaStream, &QIODevice::readyRead, this, &AVFMediaPlayer::streamReady);
        connect(m_mediaStream, &QIODevice::destroyed, this, &AVFMediaPlayer::streamDestroyed);
    }
}
 
void AVFMediaPlayer::applyPitchCompensation(bool enabled)
{
    AVPlayerItem *playerItem = [m_observer playerItem];
    if (playerItem) {
        if (enabled)
            playerItem.audioTimePitchAlgorithm = AVAudioTimePitchAlgorithmSpectral;
        else
            playerItem.audioTimePitchAlgorithm = AVAudioTimePitchAlgorithmVarispeed;
    }
}
 
void AVFMediaPlayer::resetBufferProgress()
{
    if (m_bufferProgress != 0) {
        m_bufferProgress = 0;
        bufferProgressChanged(0);
    }
}
 
void AVFMediaPlayer::nativeSizeChanged(QSize size)
{
    if (!m_videoSink)
        return;
    m_videoSink->setNativeSize(size);
}
 
void AVFMediaPlayer::orientationChanged(QtVideo::Rotation rotation, bool mirrored)
{
    if (!m_videoOutput)
        return;
 
    m_videoOutput->setVideoRotation(rotation);
    m_videoOutput->setVideoMirrored(mirrored);
}
 
void AVFMediaPlayer::videoOrientationForAssetTrack(AVAssetTrack *videoTrack,
                                                   QtVideo::Rotation &angle,
                                                   bool &mirrored)
{
    angle = QtVideo::Rotation::None;
    mirrored = false;
    if (videoTrack) {
        CGAffineTransform transform = videoTrack.preferredTransform;
        if (CGAffineTransformIsIdentity(transform))
            return;
 
        // determinant < 0 means the transform includes a reflection (mirror)
        qreal det = transform.a * transform.d - transform.b * transform.c;
        mirrored = (det < 0.0);
 
        // Factor out mirror before computing rotation angle.
        // Negating the first column of a mirrored matrix yields a pure rotation.
        qreal ra = mirrored ? -transform.a : transform.a;
        qreal rb = mirrored ? -transform.b : transform.b;
 
        qreal degrees = qRadiansToDegrees(qAtan2(rb, ra));
        if (degrees < 0)
            degrees += 360.0;
 
        if (QtPrivate::fuzzyCompare(degrees, qreal(90))
            || QtPrivate::fuzzyCompare(degrees, qreal(-270))) {
            angle = QtVideo::Rotation::Clockwise90;
        } else if (QtPrivate::fuzzyCompare(degrees, qreal(-90))
                   || QtPrivate::fuzzyCompare(degrees, qreal(270))) {
            angle = QtVideo::Rotation::Clockwise270;
        } else if (QtPrivate::fuzzyCompare(degrees, qreal(180))
                   || QtPrivate::fuzzyCompare(degrees, qreal(-180))) {
            angle = QtVideo::Rotation::Clockwise180;
        }
    }
}
 
void AVFMediaPlayer::setPitchCompensation(bool enabled)
{
    if (m_pitchCompensationEnabled == enabled)
        return;
 
    applyPitchCompensation(enabled);
 
    m_pitchCompensationEnabled = enabled;
    pitchCompensationChanged(enabled);
}
 
bool AVFMediaPlayer::pitchCompensation() const
{
    return m_pitchCompensationEnabled;
}
 
QPlatformMediaPlayer::PitchCompensationAvailability
AVFMediaPlayer::pitchCompensationAvailability() const
{
    return QPlatformMediaPlayer::PitchCompensationAvailability::Available;
}
 
#include "moc_avfmediaplayer_p.cpp"