Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qmacscreencapturekit.mm
Go to the documentation of this file.
1// Copyright (C) 2026 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
5
6#include <QtCore/qmutex.h>
7
8#include <QtFFmpegMediaPluginImpl/private/qcvimagevideobuffer_p.h>
9#include <QtFFmpegMediaPluginImpl/private/qffmpegdarwinhwframehelpers_p.h>
10#define AVMediaType XAVMediaType
11#include <QtFFmpegMediaPluginImpl/private/qffmpeghwaccel_p.h>
12#include <QtFFmpegMediaPluginImpl/private/qffmpegvideobuffer_p.h>
13extern "C" {
14#include <libavutil/hwcontext_videotoolbox.h>
15}
16#undef AVMediaType
17
18#include <QtMultimedia/private/qavfcamerautility_p.h>
19#include <QtMultimedia/private/qavfhelpers_p.h>
20#include <QtMultimedia/private/qvideoframe_p.h>
21
22#include <CoreMedia/CMTime.h>
23#include <ScreenCaptureKit/ScreenCaptureKit.h>
24
25#include <chrono>
26
27using namespace Qt::Literals::StringLiterals;
28using QMacScreenCaptureKit = QT_PREPEND_NAMESPACE(QFFmpeg::QMacScreenCaptureKit);
29
31 QT_PREPEND_NAMESPACE(QFFmpeg::qLcMacScreenCapture),
32 "qt.multimedia.screencapture.macscreencapturekit");
33
34namespace {
35
36struct QMacScreenCaptureStreamDelegateHelper : public QObject {
37 Q_OBJECT
38signals:
39 void didStopWithError(QMacScreenCaptureKit::StreamId streamId, QString);
40};
41
42} // Anonymous namespace
43
44// Events are invoked on system background thread that we don't control.
45@implementation QT_MANGLE_NAMESPACE(QMacScreenCaptureStreamDelegate) {
46@public
47 QMacScreenCaptureKit::StreamId m_streamId;
48 QMacScreenCaptureStreamDelegateHelper m_helper;
49}
50
51- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error
52{
53 emit m_helper.didStopWithError(
54 m_streamId,
55 QString::fromNSString(error.localizedDescription));
56}
57
58@end
59
60QT_BEGIN_NAMESPACE
61
62namespace QFFmpeg {
63
64static void handleFrameOutput(
65 QMacScreenCaptureStreamOutput &scStreamOutput,
66 CMSampleBufferRef sampleBufferRef);
67} // namespace QFFmpeg
68
69QT_END_NAMESPACE
70
71// Invoked on background dispatch-queue.
72@implementation QT_MANGLE_NAMESPACE(QMacScreenCaptureStreamOutput) {
73@public
74 // Assigned at construction. We assume it is safe to never reset it, because
75 // we flush the background queue anytime we stop a stream.
76 QT_PREPEND_NAMESPACE(QFFmpeg::QMacScreenCaptureKit) *m_qScreenCaptureKit;
77
78 // Used to track when the underlying window size changed, in pixel-coordinates.
79 QSize m_previousFrameContentRect;
80
81 // Presentation timestamp of the first frame of the stream. Used to report
82 // frame times relative to the start of the stream.
83 std::optional<std::chrono::microseconds> m_baseTime;
84 std::unique_ptr<QT_PREPEND_NAMESPACE(QFFmpeg::HWAccel)> m_hwAccel;
85}
86
87- (void) stream:(SCStream *) stream
88didOutputSampleBuffer:(CMSampleBufferRef) sampleBufferRef
89 ofType:(SCStreamOutputType) type
90{
91 QT_USE_NAMESPACE
92 using namespace QFFmpeg;
93
94 // SCStreamOutputTypeScreen implies we are receiving video frames
95 // rather than audio samples. It doesn't exclude windows.
96 // Our stream is hardcoded to never report audio samples.
97 Q_ASSERT(type == SCStreamOutputTypeScreen);
98
99 handleFrameOutput(*self, sampleBufferRef);
100}
101
102@end
103
104QT_BEGIN_NAMESPACE
105
106namespace QFFmpeg {
107
108// Useful metadata grabbed from a CMSampleBufferRef.
110{
112
113 // The size of the captured content, in pixel-coordinates.
114 // Can be used to detect e.g. window size changes.
116};
117
118// Reads the frame status and content rect for the given CMSampleBufferRef.
119// The content rect usually means the window size at the time a frame was
120// outputted. The resolution is in pixel coordinates.
121// Error message is not user-facing.
122[[nodiscard]] static q23::expected<FrameInfo, QString> readFrameInfo(CMSampleBufferRef sampleBuffer)
123{
124 CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false);
125 if (!attachments || CFArrayGetCount(attachments) == 0)
126 return q23::unexpected{ u"CMSampleBuffer has no attachments array"_s };
127
128 CFDictionaryRef attachment = (CFDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
129 NSDictionary *dict = (__bridge NSDictionary *)attachment;
130
131 NSNumber *statusNumber = dict[(id)SCStreamFrameInfoStatus];
132 if (!statusNumber)
133 return q23::unexpected{ u"CMSampleBuffer has no frame status"_s };
134
135 FrameInfo info;
136 info.status = static_cast<SCFrameStatus>(statusNumber.intValue);
137
138 CGRect contentRect = CGRectZero;
139 NSDictionary *frameInfo = dict[(id)SCStreamFrameInfoContentRect];
140 if (frameInfo) {
141 contentRect = CGRectMakeWithDictionaryRepresentation(
142 (__bridge CFDictionaryRef)frameInfo, &contentRect)
143 ? contentRect
144 : CGRectZero;
145 }
146
147 NSNumber *scaleNumber = dict[(id)SCStreamFrameInfoScaleFactor];
148 CGFloat scaleFactor = scaleNumber ? scaleNumber.doubleValue : 1.0;
149
150 NSNumber *contentScaleNumber = dict[(id)SCStreamFrameInfoContentScale];
151 CGFloat contentScale = contentScaleNumber ? contentScaleNumber.doubleValue : 1.0;
152 if (contentScale <= 0.0)
153 contentScale = 1.0;
154
155 info.contentRect = QSize{
156 static_cast<int>(std::lround(contentRect.size.width * scaleFactor / contentScale)),
157 static_cast<int>(std::lround(contentRect.size.height * scaleFactor / contentScale)), };
158
159 return info;
160}
161
162// Invoked on background dispatch-queue.
163// Error message is not user-facing.
165 QMacScreenCaptureStreamOutput &scStreamOutput,
166 CMSampleBufferRef sampleBufferRef)
167{
168 CVImageBufferRef imageBufferRef = CMSampleBufferGetImageBuffer(sampleBufferRef);
169 if (!imageBufferRef)
170 return q23::unexpected(u"Cannot get CVImageBufferRef from CMSampleBufferRef"_s);
171 if (CFGetTypeID(imageBufferRef) != CVPixelBufferGetTypeID())
172 return q23::unexpected(u"Grabbed CVImageBufferRef that is not of type CVPixelBuffer"_s);
173
174 auto pixelBuffer = QAVFHelpers::QSharedCVPixelBuffer(
175 imageBufferRef,
176 QAVFHelpers::QSharedCVPixelBuffer::RefMode::NeedsRef);
177
178 // ScreenCaptureKit hands us buffers from its internal pool (see queueDepth),
179 // so copy into a free-standing CVPixelBuffer to decouple the frame's
180 // lifetime from the stream's pool.
181 q23::expected<QAVFHelpers::QSharedCVPixelBuffer, QString> copyResult = deepCopyCvPixelBuffer(
182 pixelBuffer.get());
183 if (!copyResult)
184 return q23::unexpected(u"Failed to copy incoming pixel buffer: "_s + copyResult.error());
185 pixelBuffer = std::move(*copyResult);
186
187 // If the new incoming frames have a different size, update the FFmpeg frames context.
188 QSize incomingFrameSize {
189 static_cast<int>(CVPixelBufferGetWidth(pixelBuffer.get())),
190 static_cast<int>(CVPixelBufferGetHeight(pixelBuffer.get())) };
191 Q_ASSERT(!incomingFrameSize.isEmpty());
192 CvPixelFormat incomingCvPixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer.get());
193 Q_ASSERT(scStreamOutput.m_hwAccel);
194 scStreamOutput.m_hwAccel->updateFramesContext(
195 av_map_videotoolbox_format_to_pixfmt(incomingCvPixelFormat),
196 incomingFrameSize);
197
198 // ScreenCaptureKit timestamps frames on the host clock. QVideoFrame times are
199 // relative to the start of the stream, so use the first frame as our zero point.
200 const std::chrono::microseconds frameTime =
201 QAVFHelpers::CMTimeToMicroseconds(CMSampleBufferGetPresentationTimeStamp(sampleBufferRef));
202 if (!scStreamOutput.m_baseTime)
203 scStreamOutput.m_baseTime = frameTime;
204 const std::chrono::microseconds presentationTime = frameTime - *scStreamOutput.m_baseTime;
205
206 QVideoFrameFormat format = QAVFHelpers::videoFormatForImageBuffer(pixelBuffer.get());
207 if (!format.isValid())
208 return q23::unexpected(u"Cannot get get video format for image buffer"_s);
209
210 format.setColorSpace(QMacScreenCaptureKit::colorSpace);
211 format.setColorRange(QMacScreenCaptureKit::colorRange);
212 format.setColorTransfer(QMacScreenCaptureKit::colorTransfer);
213
214 Q_ASSERT(scStreamOutput.m_hwAccel);
215 QVideoFrame frame;
216 q23::expected<QVideoFrame, QString> frameResult = QFFmpeg::qVideoFrameFromCvPixelBuffer(
217 *scStreamOutput.m_hwAccel,
218 presentationTime,
219 pixelBuffer,
220 format);
221 if (!frameResult)
222 qCWarning(qLcMacScreenCapture) << frameResult.error();
223 else
224 frame = *frameResult;
225
226 if (!frame.isValid()) {
227 frame = QVideoFramePrivate::createFrame(
228 std::make_unique<QFFmpeg::CVImageVideoBuffer>(std::move(pixelBuffer)),
229 std::move(format));
230 }
231
232 frame.setStartTime(presentationTime.count());
233
234 frame.setEndTime(-1);
235
236 return frame;
237}
238
239// Main frame handler.
240// Invoked on background dispatch-queue.
242 QFFmpeg::QMacScreenCaptureStreamOutput &streamOutput,
243 CMSampleBufferRef sampleBufferRef)
244{
245 Q_ASSERT(streamOutput.m_qScreenCaptureKit);
246
247 q23::expected<FrameInfo, QString> frameInfoResult = readFrameInfo(sampleBufferRef);
248 if (!frameInfoResult) {
249 qCDebug(qLcMacScreenCapture)
250 << "Error while reading frame info of CMSampleBufferRef:"
251 << frameInfoResult.error();
252 return;
253 }
254
255 const FrameInfo &frameInfo = *frameInfoResult;
256
257 // ScreenCaptureKit only hands us a new hardware buffer when the frame status
258 // is complete. For other statuses (e.g. idle when the captured content is
259 if (frameInfo.status != SCFrameStatusComplete)
260 return;
261
262 // The content rect is the updated resolution of the window we are capturing.
263 // If the window size is different from our current stream configuration,
264 // issue a reconfiguration.
265 //
266 // If the content rect is empty, it's usually an indication that the window has
267 // been minimized while capturing it. We keep the stream unchanged so that it
268 // is automatically resumed when the window is restored.
269 if (!frameInfo.contentRect.isEmpty()) {
270 if (streamOutput.m_previousFrameContentRect != frameInfo.contentRect)
271 streamOutput.m_qScreenCaptureKit->updateStream(frameInfo.contentRect);
272
273 streamOutput.m_previousFrameContentRect = frameInfo.contentRect;
274 }
275
276 q23::expected<QVideoFrame, QString> videoFrameResult = createQVideoFrame(
277 streamOutput,
278 sampleBufferRef);
279 if (!videoFrameResult) {
280 qCWarning(qLcMacScreenCapture)
281 << "Failed to create qVideoFrame from CMSampleBufferRef:"
282 << videoFrameResult.error();
283 return;
284 }
285
286 emit streamOutput.m_qScreenCaptureKit->newVideoFrameGenerated(
287 streamOutput.m_qScreenCaptureKit->streamId(),
288 std::move(*videoFrameResult));
289}
290
292 QSize resolutionPx,
293 QMacScreenCaptureKit::StreamSettings const& settings)
294{
295 // SCStreamConfiguration defines the output format, having zero resolution makes no sense.
296 Q_ASSERT(!resolutionPx.isEmpty());
297
298 // TODO: Possible improvements include specifying pixel format, HDR,
299 // capturing system audio...
300 auto scStreamConfig = AVFScopedPointer{ [[SCStreamConfiguration alloc] init] };
301 scStreamConfig.data().width = resolutionPx.width();
302 scStreamConfig.data().height = resolutionPx.height();
303 // We make a best-effort to always adjust our video output to match the window/screen size.
304 // So we leave scaling off to be pixel-perfect whenever we can.
305 scStreamConfig.data().scalesToFit = false;
306 scStreamConfig.data().queueDepth = QMacScreenCaptureKit::queueDepth;
307 scStreamConfig.data().pixelFormat = QMacScreenCaptureKit::cvPixelFormat;
308 scStreamConfig.data().colorSpaceName = QMacScreenCaptureKit::cgColorSpace();
309 scStreamConfig.data().ignoreShadowsSingleWindow = true;
310 scStreamConfig.data().captureResolution = SCCaptureResolutionBest;
311 if (@available(macOS 15.0, *))
312 scStreamConfig.data().captureDynamicRange = SCCaptureDynamicRangeSDR;
313
314 if (settings.frameRate) {
315 Q_ASSERT(settings.frameRate > 0);
316 scStreamConfig.data().minimumFrameInterval =
317 CMTimeMake(1, static_cast<int32_t>(std::round(*settings.frameRate)));
318 } else {
319 scStreamConfig.data().minimumFrameInterval = kCMTimeZero;
320 }
321
322 // These settings are meant to override whatever the default is.
323 if (settings.overrideIgnoreCursor)
324 scStreamConfig.data().showsCursor = false;
325
326 return scStreamConfig;
327}
328
330 QMacScreenCaptureStreamDelegate &streamDelegate,
331 QMacScreenCaptureKit::StreamId streamId,
332 const QMacScreenCaptureKit &macScreenCaptureKit)
333{
334 streamDelegate.m_streamId = streamId;
335 QObject::connect(
336 &streamDelegate.m_helper,
337 &QMacScreenCaptureStreamDelegateHelper::didStopWithError,
338 &macScreenCaptureKit,
339 &QMacScreenCaptureKit::streamStoppedWithError);
340}
341
344 QMacScreenCaptureKit &macScreenCaptureKit,
345 uint32_t cvPixelFormat,
346 QSize resolution)
347{
348 auto streamOutput = AVFScopedPointer{ [[QMacScreenCaptureStreamOutput alloc] init] };
349
350 streamOutput.data()->m_qScreenCaptureKit = &macScreenCaptureKit;
351
352 streamOutput.data()->m_previousFrameContentRect = resolution;
353
354 streamOutput.data()->m_hwAccel = HWAccel::create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX);
355 if (!streamOutput.data()->m_hwAccel)
356 return q23::unexpected(
357 u"Unable to create FFmpeg HW context when starting ScreenCaptureKit stream"_s);
358
359 streamOutput.data()->m_hwAccel->createFramesContext(
360 av_map_videotoolbox_format_to_pixfmt(cvPixelFormat),
361 resolution);
362
363 if (!streamOutput.data()->m_hwAccel->hwFramesContextAsBuffer())
364 return q23::unexpected(
365 u"Unable to create FFmpeg HW context when starting ScreenCaptureKit stream"_s);
366
367 return streamOutput;
368}
369
370// The strategy is to flush any remaining jobs on the background thread.
372{
373 if (!m_stream)
374 return;
375
376 // Issue a blocking stop command.
377 dispatch_semaphore_t semaphore = dispatch_semaphore_create(0);
378 [m_stream.data() stopCaptureWithCompletionHandler:[semaphore](NSError *error) {
379 if (error) {
380 qCWarning(qLcMacScreenCapture)
381 << "Error while stopping ScreenCaptureKit stream during teardown:"
382 << QString::fromNSString(error.localizedDescription);
383 }
384 dispatch_semaphore_signal(semaphore);
385 }];
386 dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
387 dispatch_release(semaphore);
388
389 // Flush the dispatch_queue. After this we assume it's safe to tear everything down.
390 if (m_dispatchQueue)
391 dispatch_sync(m_dispatchQueue.data(), []{});
392}
393
394// This will commonly fail if we are missing permissions for screen capturing.
395// It will also open the "Grant permissions" system dialog if we are missing
396// permissions.
397//
398// Thread-safe.
399//
400// Error-message is not user-facing
403{
404 // Block functions can only capture copyable types.
405 // Wrap the promise in a shared-ptr.
407
408 // This function call will open the permissions system dialog when applicable.
411 NSError *error)
412 {
413 if (error != nil) {
415 return;
416 }
417
419
423
427
429 }];
430
431 return promise->get_future();
432}
433
449
450// Note that we are using manual memory management here, because Obj-C block functions
451// do not support capturing move-only types.
456{
458
460 auto future = promise->get_future();
461
464
465 // SCContentFilter.contentRect is in screen-points, not pixels. Multiply by
466 // pointPixelScale.
468 // Rare edge cases have shown contentRect to sometimes be empty.
469 if (resolutionPx.isEmpty()) {
470 promise->set_value(q23::unexpected{ u"SCContentFilter contentRect reported as zero size"_s });
471 return future;
472 }
473
475
479
485 }
486
489 *captureKit,
492 if (!streamOutputResult) {
494 return future;
495 }
497
500
505
507 dispatch_queue_create("qt_screencapture", DISPATCH_QUEUE_SERIAL) };
508
509 NSError *addStreamError = nullptr;
514 if (addStreamError != nil) {
515 promise->set_value(q23::unexpected(u"Unable to add stream output to SCStream"_s));
516 return future;
517 }
518
523
524 // Block functions for the completion handler require
525 // that the callable is copyable. This means we can't capture
526 // move-only types. So we temporarily release the unique_ptr here,
527 // and switch to manual memory management and then adopt them
528 // back into AVFScopedPointer inside the callback.
529 // We assume the completion handler is always called, either with success or error.
533 (NSError *error)
534 {
536
537 if (error != nil) {
538 promise->set_value(q23::unexpected{ u"Error when starting screen capturing stream"_s });
539 return;
540 }
541
543 }];
544
545 return future;
546}
547
548// Frames may arrive on the background thread immediately after the
549// creation success event has been emitted, sometimes even out of order.
550// This function takes a function that allows us to establish
551// the connections on the newly constructed object before
552// the stream ever starts, so we never miss any frames.
567
568// Issues a stream configuration update, so that the stream will give us video frames
569// of a new resolution.
570void QMacScreenCaptureKit::startStreamReconfigure(
571 SCStream *scStream,
572 QSize resolutionPx,
573 StreamSettings const &streamSettings)
574{
575 Q_ASSERT(scStream);
576
577 AVFScopedPointer<SCStreamConfiguration> scStreamConfig = createStreamConfig(
578 resolutionPx,
579 streamSettings);
580
581 [scStream
582 updateConfiguration:scStreamConfig.data()
583 completionHandler:[](NSError *err) {
584 if (err) {
585 // TODO: Send potential error back to QMacScreenCaptureKit, but only
586 // if the error stops the stream.
587 qCWarning(qLcMacScreenCapture)
588 << "Error when reconfiguring ScreenCaptureKit stream:"
589 << QString::fromNSString(err.description);
590 return;
591 }
592 }];
593}
594
595// Reconfigures the stream with a new output resolution.
596// Does not stop the stream.
597// Input resolution is in pixel-coordinates.
598// Must be called from background dispatch_queue.
599void QMacScreenCaptureKit::updateStream(QSize resolutionPx)
600{
601 Q_ASSERT(m_dispatchQueue);
602 dispatch_assert_queue(m_dispatchQueue.data());
603
604 startStreamReconfigure(m_stream.data(), resolutionPx, m_streamSettings);
605}
606
607} // namespace QFFmpeg
608
609QT_END_NAMESPACE
610
611#include "moc_qmacscreencapturekit_p.cpp"
612#include "qmacscreencapturekit.moc"
void updateStream(QSize resolutionPx)
static AVFScopedPointer< SCStreamConfiguration > createStreamConfig(QSize resolutionPx, QMacScreenCaptureKit::StreamSettings const &settings)
Q_DECLARE_LOGGING_CATEGORY(qLcMacScreenCapture)
static void handleFrameOutput(QMacScreenCaptureStreamOutput &scStreamOutput, CMSampleBufferRef sampleBufferRef)
static q23::expected< QVideoFrame, QString > createQVideoFrame(QMacScreenCaptureStreamOutput &scStreamOutput, CMSampleBufferRef sampleBufferRef)
static q23::expected< AVFScopedPointer< QMacScreenCaptureStreamOutput >, QString > createStreamOutput(QMacScreenCaptureKit &macScreenCaptureKit, uint32_t cvPixelFormat, QSize resolution)
static void configureStreamDelegate(QMacScreenCaptureStreamDelegate &streamDelegate, QMacScreenCaptureKit::StreamId streamId, const QMacScreenCaptureKit &macScreenCaptureKit)
static q23::expected< FrameInfo, QString > readFrameInfo(CMSampleBufferRef sampleBuffer)
Q_LOGGING_CATEGORY_IMPL(QT_PREPEND_NAMESPACE(QFFmpeg::qLcMacScreenCapture), "qt.multimedia.screencapture.macscreencapturekit")