Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qmacscreencapturekit.mm
Go to the documentation of this file.
1// Copyright (C) 2026 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
5
6#include <QtCore/qmutex.h>
7
8#include <QtFFmpegMediaPluginImpl/private/qcvimagevideobuffer_p.h>
9#include <QtFFmpegMediaPluginImpl/private/qffmpegdarwinhwframehelpers_p.h>
10#define AVMediaType XAVMediaType
11#include <QtFFmpegMediaPluginImpl/private/qffmpeghwaccel_p.h>
12#include <QtFFmpegMediaPluginImpl/private/qffmpegvideobuffer_p.h>
13extern "C" {
14#include <libavutil/hwcontext_videotoolbox.h>
15}
16#undef AVMediaType
17
18#include <QtMultimedia/private/qavfcamerautility_p.h>
19#include <QtMultimedia/private/qavfhelpers_p.h>
20#include <QtMultimedia/private/qvideoframe_p.h>
21
22#include <CoreMedia/CMTime.h>
23#include <ScreenCaptureKit/ScreenCaptureKit.h>
24
25#include <chrono>
26
27using namespace Qt::Literals::StringLiterals;
28using QMacScreenCaptureKit = QT_PREPEND_NAMESPACE(QFFmpeg::QMacScreenCaptureKit);
29
31 QT_PREPEND_NAMESPACE(QFFmpeg::qLcMacScreenCapture),
32 "qt.multimedia.screencapture.macscreencapturekit");
33
34namespace {
35
36struct QMacScreenCaptureStreamDelegateHelper : public QObject {
37 Q_OBJECT
38signals:
39 void didStopWithError(QMacScreenCaptureKit::StreamId streamId, QString);
40};
41
42} // Anonymous namespace
43
44// Events are invoked on system background thread that we don't control.
45@implementation QT_MANGLE_NAMESPACE(QMacScreenCaptureStreamDelegate) {
46@public
47 QMacScreenCaptureKit::StreamId m_streamId;
48 QMacScreenCaptureStreamDelegateHelper m_helper;
49}
50
51- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error
52{
53 emit m_helper.didStopWithError(
54 m_streamId,
55 QString::fromNSString(error.localizedDescription));
56}
57
58@end
59
60QT_BEGIN_NAMESPACE
61
62namespace QFFmpeg {
63
64static void handleFrameOutput(
65 QMacScreenCaptureStreamOutput &scStreamOutput,
66 CMSampleBufferRef sampleBufferRef);
67} // namespace QFFmpeg
68
69QT_END_NAMESPACE
70
71// Invoked on background dispatch-queue.
72@implementation QT_MANGLE_NAMESPACE(QMacScreenCaptureStreamOutput) {
73@public
74 // Assigned at construction. We assume it is safe to never reset it, because
75 // we flush the background queue anytime we stop a stream.
76 QT_PREPEND_NAMESPACE(QFFmpeg::QMacScreenCaptureKit) *m_qScreenCaptureKit;
77
78 // Used to track when the underlying window size changed, in pixel-coordinates.
79 std::optional<QSize> m_previousFrameContentRect;
80 std::chrono::microseconds m_startTime;
81 std::optional<std::chrono::microseconds> m_baseTime;
82 std::unique_ptr<QT_PREPEND_NAMESPACE(QFFmpeg::HWAccel)> m_hwAccel;
83}
84
85- (void) stream:(SCStream *) stream
86didOutputSampleBuffer:(CMSampleBufferRef) sampleBufferRef
87 ofType:(SCStreamOutputType) type
88{
89 QT_USE_NAMESPACE
90 using namespace QFFmpeg;
91
92 // SCStreamOutputTypeScreen implies we are receiving video frames
93 // rather than audio samples. It doesn't exclude windows.
94 // Our stream is hardcoded to never report audio samples.
95 Q_ASSERT(type == SCStreamOutputTypeScreen);
96
97 handleFrameOutput(*self, sampleBufferRef);
98}
99
100@end
101
102QT_BEGIN_NAMESPACE
103
104namespace QFFmpeg {
105
106// Useful metadata grabbed from a CMSampleBufferRef.
108{
110
111 // The size of the captured content, in pixel-coordinates.
112 // Can be used to detect e.g. window size changes.
114};
115
116// Reads the frame status and content rect for the given CMSampleBufferRef.
117// The content rect usually means the window size at the time a frame was
118// outputted. The resolution is in pixel coordinates.
119// Error message is not user-facing.
120[[nodiscard]] static q23::expected<FrameInfo, QString> readFrameInfo(CMSampleBufferRef sampleBuffer)
121{
122 CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false);
123 if (!attachments || CFArrayGetCount(attachments) == 0)
124 return q23::unexpected{ u"CMSampleBuffer has no attachments array"_s };
125
126 CFDictionaryRef attachment = (CFDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
127 NSDictionary *dict = (__bridge NSDictionary *)attachment;
128
129 NSNumber *statusNumber = dict[(id)SCStreamFrameInfoStatus];
130 if (!statusNumber)
131 return q23::unexpected{ u"CMSampleBuffer has no frame status"_s };
132
133 FrameInfo info;
134 info.status = static_cast<SCFrameStatus>(statusNumber.intValue);
135
136 CGRect contentRect = CGRectZero;
137 NSDictionary *frameInfo = dict[(id)SCStreamFrameInfoContentRect];
138 if (frameInfo) {
139 contentRect = CGRectMakeWithDictionaryRepresentation(
140 (__bridge CFDictionaryRef)frameInfo, &contentRect)
141 ? contentRect
142 : CGRectZero;
143 }
144
145 NSNumber *scaleNumber = dict[(id)SCStreamFrameInfoScaleFactor];
146 CGFloat scaleFactor = scaleNumber ? scaleNumber.doubleValue : 1.0;
147
148 NSNumber *contentScaleNumber = dict[(id)SCStreamFrameInfoContentScale];
149 CGFloat contentScale = contentScaleNumber ? contentScaleNumber.doubleValue : 1.0;
150 if (contentScale <= 0.0)
151 contentScale = 1.0;
152
153 info.contentRect = QSize{
154 static_cast<int>(std::lround(contentRect.size.width * scaleFactor / contentScale)),
155 static_cast<int>(std::lround(contentRect.size.height * scaleFactor / contentScale)), };
156
157 return info;
158}
159
160// Invoked on background dispatch-queue.
161// Error message is not user-facing.
163 QMacScreenCaptureStreamOutput &scStreamOutput,
164 CMSampleBufferRef sampleBufferRef)
165{
166 CVImageBufferRef imageBufferRef = CMSampleBufferGetImageBuffer(sampleBufferRef);
167 if (!imageBufferRef)
168 return q23::unexpected(u"Cannot get CVImageBufferRef from CMSampleBufferRef"_s);
169 if (CFGetTypeID(imageBufferRef) != CVPixelBufferGetTypeID())
170 return q23::unexpected(u"Grabbed CVImageBufferRef that is not of type CVPixelBuffer"_s);
171
172 auto pixelBuffer = QAVFHelpers::QSharedCVPixelBuffer(
173 imageBufferRef,
174 QAVFHelpers::QSharedCVPixelBuffer::RefMode::NeedsRef);
175
176 // ScreenCaptureKit hands us buffers from its internal pool (see queueDepth),
177 // so copy into a free-standing CVPixelBuffer to decouple the frame's
178 // lifetime from the stream's pool.
179 q23::expected<QAVFHelpers::QSharedCVPixelBuffer, QString> copyResult = deepCopyCvPixelBuffer(
180 pixelBuffer.get());
181 if (!copyResult)
182 return q23::unexpected(u"Failed to copy incoming pixel buffer: "_s + copyResult.error());
183 pixelBuffer = std::move(*copyResult);
184
185 // If the new incoming frames have a different size, update the FFmpeg frames context.
186 QSize incomingFrameSize {
187 static_cast<int>(CVPixelBufferGetWidth(pixelBuffer.get())),
188 static_cast<int>(CVPixelBufferGetHeight(pixelBuffer.get())) };
189 Q_ASSERT(!incomingFrameSize.isEmpty());
190 CvPixelFormat incomingCvPixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer.get());
191 Q_ASSERT(scStreamOutput.m_hwAccel);
192 scStreamOutput.m_hwAccel->updateFramesContext(
193 av_map_videotoolbox_format_to_pixfmt(incomingCvPixelFormat),
194 incomingFrameSize);
195
196 // TODO: We can extract these values specifically with ScreenCaptureKit.
197 std::chrono::microseconds frameTime =
198 QAVFHelpers::CMTimeToMicroseconds(CMSampleBufferGetPresentationTimeStamp(sampleBufferRef));
199 if (!scStreamOutput.m_baseTime) {
200 scStreamOutput.m_baseTime = frameTime;
201 scStreamOutput.m_startTime = frameTime;
202 }
203
204 QVideoFrameFormat format = QAVFHelpers::videoFormatForImageBuffer(pixelBuffer.get());
205 if (!format.isValid())
206 return q23::unexpected(u"Cannot get get video format for image buffer"_s);
207
208 format.setColorSpace(QMacScreenCaptureKit::colorSpace);
209 format.setColorRange(QMacScreenCaptureKit::colorRange);
210 format.setColorTransfer(QMacScreenCaptureKit::colorTransfer);
211
212 Q_ASSERT(scStreamOutput.m_hwAccel);
213 QVideoFrame frame;
214 q23::expected<QVideoFrame, QString> frameResult = QFFmpeg::qVideoFrameFromCvPixelBuffer(
215 *scStreamOutput.m_hwAccel,
216 scStreamOutput.m_startTime - *scStreamOutput.m_baseTime,
217 pixelBuffer,
218 format);
219 if (!frameResult)
220 qCWarning(qLcMacScreenCapture) << frameResult.error();
221 else
222 frame = *frameResult;
223
224 if (!frame.isValid()) {
225 frame = QVideoFramePrivate::createFrame(
226 std::make_unique<QFFmpeg::CVImageVideoBuffer>(std::move(pixelBuffer)),
227 std::move(format));
228 }
229
230 frame.setStartTime((scStreamOutput.m_startTime - *scStreamOutput.m_baseTime).count());
231 frame.setEndTime((frameTime - *scStreamOutput.m_baseTime).count());
232 scStreamOutput.m_startTime = frameTime;
233
234 return frame;
235}
236
237// Main frame handler.
238// Invoked on background dispatch-queue.
240 QFFmpeg::QMacScreenCaptureStreamOutput &streamOutput,
241 CMSampleBufferRef sampleBufferRef)
242{
243 Q_ASSERT(streamOutput.m_qScreenCaptureKit);
244
245 q23::expected<FrameInfo, QString> frameInfoResult = readFrameInfo(sampleBufferRef);
246 if (!frameInfoResult) {
247 qCDebug(qLcMacScreenCapture)
248 << "Error while reading frame info of CMSampleBufferRef:"
249 << frameInfoResult.error();
250 return;
251 }
252
253 const FrameInfo &frameInfo = *frameInfoResult;
254
255 // ScreenCaptureKit only hands us a new hardware buffer when the frame status
256 // is complete. For other statuses (e.g. idle when the captured content is
257 if (frameInfo.status != SCFrameStatusComplete)
258 return;
259
260 // The content rect is the updated resolution of the window we are capturing.
261 // If the window size is different from our current stream configuration,
262 // issue a reconfiguration.
263 //
264 // If the content rect is empty, it's usually an indication that the window has
265 // been minimized while capturing it. We keep the stream unchanged so that it
266 // is automatically resumed when the window is restored.
267 if (!frameInfo.contentRect.isEmpty()) {
268 bool newContentRectIsDifferent =
269 streamOutput.m_previousFrameContentRect
270 && *streamOutput.m_previousFrameContentRect != frameInfo.contentRect;
271 if (newContentRectIsDifferent)
272 streamOutput.m_qScreenCaptureKit->updateStream(frameInfo.contentRect);
273
274 streamOutput.m_previousFrameContentRect = frameInfo.contentRect;
275 }
276
277 q23::expected<QVideoFrame, QString> videoFrameResult = createQVideoFrame(
278 streamOutput,
279 sampleBufferRef);
280 if (!videoFrameResult) {
281 qCWarning(qLcMacScreenCapture)
282 << "Failed to create qVideoFrame from CMSampleBufferRef:"
283 << videoFrameResult.error();
284 return;
285 }
286
287 emit streamOutput.m_qScreenCaptureKit->newVideoFrameGenerated(
288 streamOutput.m_qScreenCaptureKit->streamId(),
289 std::move(*videoFrameResult));
290}
291
293 QMacScreenCaptureStreamDelegate &streamDelegate,
294 QMacScreenCaptureKit::StreamId streamId,
295 const QMacScreenCaptureKit &macScreenCaptureKit)
296{
297 streamDelegate.m_streamId = streamId;
298 QObject::connect(
299 &streamDelegate.m_helper,
300 &QMacScreenCaptureStreamDelegateHelper::didStopWithError,
301 &macScreenCaptureKit,
302 &QMacScreenCaptureKit::streamStoppedWithError);
303}
304
307 QMacScreenCaptureKit &macScreenCaptureKit,
308 uint32_t cvPixelFormat,
309 QSize resolution)
310{
311 auto streamOutput = AVFScopedPointer{ [[QMacScreenCaptureStreamOutput alloc] init] };
312
313 streamOutput.data()->m_qScreenCaptureKit = &macScreenCaptureKit;
314
315 streamOutput.data()->m_hwAccel = HWAccel::create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX);
316 if (!streamOutput.data()->m_hwAccel)
317 return q23::unexpected(
318 u"Unable to create FFmpeg HW context when starting ScreenCaptureKit stream"_s);
319
320 streamOutput.data()->m_hwAccel->createFramesContext(
321 av_map_videotoolbox_format_to_pixfmt(cvPixelFormat),
322 resolution);
323
324 if (!streamOutput.data()->m_hwAccel->hwFramesContextAsBuffer())
325 return q23::unexpected(
326 u"Unable to create FFmpeg HW context when starting ScreenCaptureKit stream"_s);
327
328 return streamOutput;
329}
330
331// The strategy is to flush any remaining jobs on the background thread.
333{
334 if (!m_stream)
335 return;
336
337 // Issue a blocking stop command.
338 dispatch_semaphore_t semaphore = dispatch_semaphore_create(0);
339 [m_stream.data() stopCaptureWithCompletionHandler:[semaphore](NSError *error) {
340 if (error) {
341 qCWarning(qLcMacScreenCapture)
342 << "Error while stopping ScreenCaptureKit stream during teardown:"
343 << QString::fromNSString(error.localizedDescription);
344 }
345 dispatch_semaphore_signal(semaphore);
346 }];
347 dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
348 dispatch_release(semaphore);
349
350 // Flush the dispatch_queue. After this we assume it's safe to tear everything down.
351 if (m_dispatchQueue)
352 dispatch_sync(m_dispatchQueue.data(), []{});
353}
354
355// This will commonly fail if we are missing permissions for screen capturing.
356// It will also open the "Grant permissions" system dialog if we are missing
357// permissions.
358//
359// Thread-safe.
360//
361// Error-message is not user-facing
364{
365 // Block functions can only capture copyable types.
366 // Wrap the promise in a shared-ptr.
368
369 // This function call will open the permissions system dialog when applicable.
372 NSError *error)
373 {
374 if (error != nil) {
376 return;
377 }
378
380
384
388
390 }];
391
392 return promise->get_future();
393}
394
401{
403
406 excludingWindows: @[] ] };
407
408 // SCDisplay.frame is in screen-points, not pixels. Multiply by
409 // pointPixelScale.
411 streamId,
413 frameRate,
415}
416
417// Note that we are using manual memory management here, because Obj-C block functions
418// do not support capturing move-only types.
425{
427
429 auto future = promise->get_future();
430
433
434 // SCContentFilter.contentRect is in screen-points, not pixels. Multiply by
435 // pointPixelScale.
437 // Rare edge cases have shown contentRect to sometimes be empty.
438 if (resolutionPx.isEmpty()) {
439 promise->set_value(q23::unexpected{ u"SCContentFilter contentRect reported as zero size"_s });
440 return future;
441 }
442
445
449 if (connectionSetup) {
451 }
452
455 *captureKit,
458 if (!streamOutputResult) {
460 return future;
461 }
463
466
471
473 dispatch_queue_create("qt_screencapture", DISPATCH_QUEUE_SERIAL) };
474
475 NSError *addStreamError = nullptr;
480 if (addStreamError != nil) {
481 promise->set_value(q23::unexpected(u"Unable to add stream output to SCStream"_s));
482 return future;
483 }
484
489
490 // Block functions for the completion handler require
491 // that the callable is copyable. This means we can't capture
492 // move-only types. So we temporarily release the unique_ptr here,
493 // and switch to manual memory management and then adopt them
494 // back into AVFScopedPointer inside the callback.
495 // We assume the completion handler is always called, either with success or error.
499 (NSError *error)
500 {
502
503 if (error != nil) {
504 promise->set_value(q23::unexpected{ u"Error when starting screen capturing stream"_s });
505 return;
506 }
507
509 }];
510
511 return future;
512}
513
514// Frames may arrive on the background thread immediately after the
515// creation success event has been emitted, sometimes even out of order.
516// This function takes a function that allows us to establish
517// the connections on the newly constructed object before
518// the stream ever starts, so we never miss any frames.
537
538AVFScopedPointer<SCStreamConfiguration> QMacScreenCaptureKit::createStreamConfig(
539 QSize resolutionPx,
540 std::optional<qreal> frameRate)
541{
542 // SCStreamConfiguration defines the output format, having zero resolution makes no sense.
543 Q_ASSERT(!resolutionPx.isEmpty());
544
545 // TODO: Possible improvements include specifying pixel format, HDR,
546 // capturing system audio...
547 auto scStreamConfig = AVFScopedPointer{ [[SCStreamConfiguration alloc] init] };
548 scStreamConfig.data().width = resolutionPx.width();
549 scStreamConfig.data().height = resolutionPx.height();
550 // We make a best-effort to always adjust our video output to match the window/screen size.
551 // So we leave scaling off to be pixel-perfect whenever we can.
552 scStreamConfig.data().scalesToFit = false;
553 scStreamConfig.data().queueDepth = QMacScreenCaptureKit::queueDepth;
554 scStreamConfig.data().pixelFormat = QMacScreenCaptureKit::cvPixelFormat;
555 scStreamConfig.data().colorSpaceName = QMacScreenCaptureKit::cgColorSpace();
556 scStreamConfig.data().captureResolution = SCCaptureResolutionBest;
557 if (@available(macOS 15.0, *))
558 scStreamConfig.data().captureDynamicRange = SCCaptureDynamicRangeSDR;
559
560 if (frameRate) {
561 Q_ASSERT(frameRate > 0);
562 scStreamConfig.data().minimumFrameInterval =
563 CMTimeMake(1, static_cast<int32_t>(std::round(*frameRate)));
564 } else {
565 scStreamConfig.data().minimumFrameInterval = kCMTimeZero;
566 }
567
568 return scStreamConfig;
569}
570
571// Issues a stream configuration update, so that the stream will give us video frames
572// of a new resolution.
573void QMacScreenCaptureKit::startStreamReconfigure(
574 SCStream *scStream,
575 QSize resolutionPx,
576 std::optional<qreal> frameRate)
577{
578 Q_ASSERT(scStream);
579
580 AVFScopedPointer<SCStreamConfiguration> scStreamConfig =
581 QMacScreenCaptureKit::createStreamConfig(resolutionPx, frameRate);
582
583 [scStream
584 updateConfiguration:scStreamConfig.data()
585 completionHandler:[](NSError *err) {
586 if (err) {
587 // TODO: Send potential error back to QMacScreenCaptureKit, but only
588 // if the error stops the stream.
589 qCWarning(qLcMacScreenCapture)
590 << "Error when reconfiguring ScreenCaptureKit stream:"
591 << QString::fromNSString(err.description);
592 return;
593 }
594 }];
595}
596
597// Reconfigures the stream with a new output resolution.
598// Does not stop the stream.
599// Input resolution is in pixel-coordinates.
600// Must be called from background dispatch_queue.
601void QMacScreenCaptureKit::updateStream(QSize resolutionPx)
602{
603 Q_ASSERT(m_dispatchQueue);
604 dispatch_assert_queue(m_dispatchQueue.data());
605
606 startStreamReconfigure(m_stream.data(), resolutionPx, m_frameRate);
607}
608
609} // namespace QFFmpeg
610
611QT_END_NAMESPACE
612
613#include "moc_qmacscreencapturekit_p.cpp"
614#include "qmacscreencapturekit.moc"
void updateStream(QSize resolutionPx)
Q_DECLARE_LOGGING_CATEGORY(qLcMacScreenCapture)
static void handleFrameOutput(QMacScreenCaptureStreamOutput &scStreamOutput, CMSampleBufferRef sampleBufferRef)
QT_MANGLE_NAMESPACE(QMacScreenCaptureStreamDelegate) QMacScreenCaptureStreamDelegate
static q23::expected< QVideoFrame, QString > createQVideoFrame(QMacScreenCaptureStreamOutput &scStreamOutput, CMSampleBufferRef sampleBufferRef)
static q23::expected< AVFScopedPointer< QMacScreenCaptureStreamOutput >, QString > createStreamOutput(QMacScreenCaptureKit &macScreenCaptureKit, uint32_t cvPixelFormat, QSize resolution)
static void configureStreamDelegate(QMacScreenCaptureStreamDelegate &streamDelegate, QMacScreenCaptureKit::StreamId streamId, const QMacScreenCaptureKit &macScreenCaptureKit)
static q23::expected< FrameInfo, QString > readFrameInfo(CMSampleBufferRef sampleBuffer)
Q_LOGGING_CATEGORY_IMPL(QT_PREPEND_NAMESPACE(QFFmpeg::qLcMacScreenCapture), "qt.multimedia.screencapture.macscreencapturekit")