Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qmacscreencapturekit.mm
Go to the documentation of this file.
1// Copyright (C) 2026 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
5
6#include <QtCore/qmutex.h>
7
8#include <QtFFmpegMediaPluginImpl/private/qcvimagevideobuffer_p.h>
9#include <QtFFmpegMediaPluginImpl/private/qffmpegdarwinhwframehelpers_p.h>
10#define AVMediaType XAVMediaType
11#include <QtFFmpegMediaPluginImpl/private/qffmpeghwaccel_p.h>
12#include <QtFFmpegMediaPluginImpl/private/qffmpegvideobuffer_p.h>
13extern "C" {
14#include <libavutil/hwcontext_videotoolbox.h>
15}
16#undef AVMediaType
17
18#include <QtMultimedia/private/qavfcamerautility_p.h>
19#include <QtMultimedia/private/qavfhelpers_p.h>
20#include <QtMultimedia/private/qvideoframe_p.h>
21
22#include <CoreMedia/CMTime.h>
23#include <ScreenCaptureKit/ScreenCaptureKit.h>
24
25#include <chrono>
26
27using namespace Qt::Literals::StringLiterals;
28using QMacScreenCaptureKit = QT_PREPEND_NAMESPACE(QFFmpeg::QMacScreenCaptureKit);
29
31 QT_PREPEND_NAMESPACE(QFFmpeg::qLcMacScreenCapture),
32 "qt.multimedia.screencapture.macscreencapturekit");
33
34namespace {
35
36struct QMacScreenCaptureStreamDelegateHelper : public QObject {
37 Q_OBJECT
38signals:
39 void didStopWithError(QMacScreenCaptureKit::StreamId streamId, QString);
40};
41
42} // Anonymous namespace
43
44// Events are invoked on system background thread that we don't control.
45@implementation QT_MANGLE_NAMESPACE(QMacScreenCaptureStreamDelegate) {
46@public
47 QMacScreenCaptureKit::StreamId m_streamId;
48 QMacScreenCaptureStreamDelegateHelper m_helper;
49}
50
51- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error
52{
53 emit m_helper.didStopWithError(
54 m_streamId,
55 QString::fromNSString(error.localizedDescription));
56}
57
58@end
59
60QT_BEGIN_NAMESPACE
61
62namespace QFFmpeg {
63
64static void handleFrameOutput(
65 QMacScreenCaptureStreamOutput &scStreamOutput,
66 CMSampleBufferRef sampleBufferRef);
67} // namespace QFFmpeg
68
69QT_END_NAMESPACE
70
71// Invoked on background dispatch-queue.
72@implementation QT_MANGLE_NAMESPACE(QMacScreenCaptureStreamOutput) {
73@public
74 // Assigned at construction. We assume it is safe to never reset it, because
75 // we flush the background queue anytime we stop a stream.
76 QT_PREPEND_NAMESPACE(QFFmpeg::QMacScreenCaptureKit) *m_qScreenCaptureKit;
77
78 // Used to track when the underlying window size changed, in pixel-coordinates.
79 QSize m_previousFrameContentRect;
80 std::chrono::microseconds m_startTime;
81 std::optional<std::chrono::microseconds> m_baseTime;
82 std::unique_ptr<QT_PREPEND_NAMESPACE(QFFmpeg::HWAccel)> m_hwAccel;
83}
84
85- (void) stream:(SCStream *) stream
86didOutputSampleBuffer:(CMSampleBufferRef) sampleBufferRef
87 ofType:(SCStreamOutputType) type
88{
89 QT_USE_NAMESPACE
90 using namespace QFFmpeg;
91
92 // SCStreamOutputTypeScreen implies we are receiving video frames
93 // rather than audio samples. It doesn't exclude windows.
94 // Our stream is hardcoded to never report audio samples.
95 Q_ASSERT(type == SCStreamOutputTypeScreen);
96
97 handleFrameOutput(*self, sampleBufferRef);
98}
99
100@end
101
102QT_BEGIN_NAMESPACE
103
104namespace QFFmpeg {
105
106// Useful metadata grabbed from a CMSampleBufferRef.
108{
110
111 // The size of the captured content, in pixel-coordinates.
112 // Can be used to detect e.g. window size changes.
114};
115
116// Reads the frame status and content rect for the given CMSampleBufferRef.
117// The content rect usually means the window size at the time a frame was
118// outputted. The resolution is in pixel coordinates.
119// Error message is not user-facing.
120[[nodiscard]] static q23::expected<FrameInfo, QString> readFrameInfo(CMSampleBufferRef sampleBuffer)
121{
122 CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, false);
123 if (!attachments || CFArrayGetCount(attachments) == 0)
124 return q23::unexpected{ u"CMSampleBuffer has no attachments array"_s };
125
126 CFDictionaryRef attachment = (CFDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
127 NSDictionary *dict = (__bridge NSDictionary *)attachment;
128
129 NSNumber *statusNumber = dict[(id)SCStreamFrameInfoStatus];
130 if (!statusNumber)
131 return q23::unexpected{ u"CMSampleBuffer has no frame status"_s };
132
133 FrameInfo info;
134 info.status = static_cast<SCFrameStatus>(statusNumber.intValue);
135
136 CGRect contentRect = CGRectZero;
137 NSDictionary *frameInfo = dict[(id)SCStreamFrameInfoContentRect];
138 if (frameInfo) {
139 contentRect = CGRectMakeWithDictionaryRepresentation(
140 (__bridge CFDictionaryRef)frameInfo, &contentRect)
141 ? contentRect
142 : CGRectZero;
143 }
144
145 NSNumber *scaleNumber = dict[(id)SCStreamFrameInfoScaleFactor];
146 CGFloat scaleFactor = scaleNumber ? scaleNumber.doubleValue : 1.0;
147
148 NSNumber *contentScaleNumber = dict[(id)SCStreamFrameInfoContentScale];
149 CGFloat contentScale = contentScaleNumber ? contentScaleNumber.doubleValue : 1.0;
150 if (contentScale <= 0.0)
151 contentScale = 1.0;
152
153 info.contentRect = QSize{
154 static_cast<int>(std::lround(contentRect.size.width * scaleFactor / contentScale)),
155 static_cast<int>(std::lround(contentRect.size.height * scaleFactor / contentScale)), };
156
157 return info;
158}
159
160// Invoked on background dispatch-queue.
161// Error message is not user-facing.
163 QMacScreenCaptureStreamOutput &scStreamOutput,
164 CMSampleBufferRef sampleBufferRef)
165{
166 CVImageBufferRef imageBufferRef = CMSampleBufferGetImageBuffer(sampleBufferRef);
167 if (!imageBufferRef)
168 return q23::unexpected(u"Cannot get CVImageBufferRef from CMSampleBufferRef"_s);
169 if (CFGetTypeID(imageBufferRef) != CVPixelBufferGetTypeID())
170 return q23::unexpected(u"Grabbed CVImageBufferRef that is not of type CVPixelBuffer"_s);
171
172 auto pixelBuffer = QAVFHelpers::QSharedCVPixelBuffer(
173 imageBufferRef,
174 QAVFHelpers::QSharedCVPixelBuffer::RefMode::NeedsRef);
175
176 // ScreenCaptureKit hands us buffers from its internal pool (see queueDepth),
177 // so copy into a free-standing CVPixelBuffer to decouple the frame's
178 // lifetime from the stream's pool.
179 q23::expected<QAVFHelpers::QSharedCVPixelBuffer, QString> copyResult = deepCopyCvPixelBuffer(
180 pixelBuffer.get());
181 if (!copyResult)
182 return q23::unexpected(u"Failed to copy incoming pixel buffer: "_s + copyResult.error());
183 pixelBuffer = std::move(*copyResult);
184
185 // If the new incoming frames have a different size, update the FFmpeg frames context.
186 QSize incomingFrameSize {
187 static_cast<int>(CVPixelBufferGetWidth(pixelBuffer.get())),
188 static_cast<int>(CVPixelBufferGetHeight(pixelBuffer.get())) };
189 Q_ASSERT(!incomingFrameSize.isEmpty());
190 CvPixelFormat incomingCvPixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer.get());
191 Q_ASSERT(scStreamOutput.m_hwAccel);
192 scStreamOutput.m_hwAccel->updateFramesContext(
193 av_map_videotoolbox_format_to_pixfmt(incomingCvPixelFormat),
194 incomingFrameSize);
195
196 // TODO: We can extract these values specifically with ScreenCaptureKit.
197 std::chrono::microseconds frameTime =
198 QAVFHelpers::CMTimeToMicroseconds(CMSampleBufferGetPresentationTimeStamp(sampleBufferRef));
199 if (!scStreamOutput.m_baseTime) {
200 scStreamOutput.m_baseTime = frameTime;
201 scStreamOutput.m_startTime = frameTime;
202 }
203
204 QVideoFrameFormat format = QAVFHelpers::videoFormatForImageBuffer(pixelBuffer.get());
205 if (!format.isValid())
206 return q23::unexpected(u"Cannot get get video format for image buffer"_s);
207
208 format.setColorSpace(QMacScreenCaptureKit::colorSpace);
209 format.setColorRange(QMacScreenCaptureKit::colorRange);
210 format.setColorTransfer(QMacScreenCaptureKit::colorTransfer);
211
212 Q_ASSERT(scStreamOutput.m_hwAccel);
213 QVideoFrame frame;
214 q23::expected<QVideoFrame, QString> frameResult = QFFmpeg::qVideoFrameFromCvPixelBuffer(
215 *scStreamOutput.m_hwAccel,
216 scStreamOutput.m_startTime - *scStreamOutput.m_baseTime,
217 pixelBuffer,
218 format);
219 if (!frameResult)
220 qCWarning(qLcMacScreenCapture) << frameResult.error();
221 else
222 frame = *frameResult;
223
224 if (!frame.isValid()) {
225 frame = QVideoFramePrivate::createFrame(
226 std::make_unique<QFFmpeg::CVImageVideoBuffer>(std::move(pixelBuffer)),
227 std::move(format));
228 }
229
230 frame.setStartTime((scStreamOutput.m_startTime - *scStreamOutput.m_baseTime).count());
231 frame.setEndTime((frameTime - *scStreamOutput.m_baseTime).count());
232 scStreamOutput.m_startTime = frameTime;
233
234 return frame;
235}
236
237// Main frame handler.
238// Invoked on background dispatch-queue.
240 QFFmpeg::QMacScreenCaptureStreamOutput &streamOutput,
241 CMSampleBufferRef sampleBufferRef)
242{
243 Q_ASSERT(streamOutput.m_qScreenCaptureKit);
244
245 q23::expected<FrameInfo, QString> frameInfoResult = readFrameInfo(sampleBufferRef);
246 if (!frameInfoResult) {
247 qCDebug(qLcMacScreenCapture)
248 << "Error while reading frame info of CMSampleBufferRef:"
249 << frameInfoResult.error();
250 return;
251 }
252
253 const FrameInfo &frameInfo = *frameInfoResult;
254
255 // ScreenCaptureKit only hands us a new hardware buffer when the frame status
256 // is complete. For other statuses (e.g. idle when the captured content is
257 if (frameInfo.status != SCFrameStatusComplete)
258 return;
259
260 // The content rect is the updated resolution of the window we are capturing.
261 // If the window size is different from our current stream configuration,
262 // issue a reconfiguration.
263 //
264 // If the content rect is empty, it's usually an indication that the window has
265 // been minimized while capturing it. We keep the stream unchanged so that it
266 // is automatically resumed when the window is restored.
267 if (!frameInfo.contentRect.isEmpty()) {
268 if (streamOutput.m_previousFrameContentRect != frameInfo.contentRect)
269 streamOutput.m_qScreenCaptureKit->updateStream(frameInfo.contentRect);
270
271 streamOutput.m_previousFrameContentRect = frameInfo.contentRect;
272 }
273
274 q23::expected<QVideoFrame, QString> videoFrameResult = createQVideoFrame(
275 streamOutput,
276 sampleBufferRef);
277 if (!videoFrameResult) {
278 qCWarning(qLcMacScreenCapture)
279 << "Failed to create qVideoFrame from CMSampleBufferRef:"
280 << videoFrameResult.error();
281 return;
282 }
283
284 emit streamOutput.m_qScreenCaptureKit->newVideoFrameGenerated(
285 streamOutput.m_qScreenCaptureKit->streamId(),
286 std::move(*videoFrameResult));
287}
288
290 QMacScreenCaptureStreamDelegate &streamDelegate,
291 QMacScreenCaptureKit::StreamId streamId,
292 const QMacScreenCaptureKit &macScreenCaptureKit)
293{
294 streamDelegate.m_streamId = streamId;
295 QObject::connect(
296 &streamDelegate.m_helper,
297 &QMacScreenCaptureStreamDelegateHelper::didStopWithError,
298 &macScreenCaptureKit,
299 &QMacScreenCaptureKit::streamStoppedWithError);
300}
301
304 QMacScreenCaptureKit &macScreenCaptureKit,
305 uint32_t cvPixelFormat,
306 QSize resolution)
307{
308 auto streamOutput = AVFScopedPointer{ [[QMacScreenCaptureStreamOutput alloc] init] };
309
310 streamOutput.data()->m_qScreenCaptureKit = &macScreenCaptureKit;
311
312 streamOutput.data()->m_previousFrameContentRect = resolution;
313
314 streamOutput.data()->m_hwAccel = HWAccel::create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX);
315 if (!streamOutput.data()->m_hwAccel)
316 return q23::unexpected(
317 u"Unable to create FFmpeg HW context when starting ScreenCaptureKit stream"_s);
318
319 streamOutput.data()->m_hwAccel->createFramesContext(
320 av_map_videotoolbox_format_to_pixfmt(cvPixelFormat),
321 resolution);
322
323 if (!streamOutput.data()->m_hwAccel->hwFramesContextAsBuffer())
324 return q23::unexpected(
325 u"Unable to create FFmpeg HW context when starting ScreenCaptureKit stream"_s);
326
327 return streamOutput;
328}
329
330// The strategy is to flush any remaining jobs on the background thread.
332{
333 if (!m_stream)
334 return;
335
336 // Issue a blocking stop command.
337 dispatch_semaphore_t semaphore = dispatch_semaphore_create(0);
338 [m_stream.data() stopCaptureWithCompletionHandler:[semaphore](NSError *error) {
339 if (error) {
340 qCWarning(qLcMacScreenCapture)
341 << "Error while stopping ScreenCaptureKit stream during teardown:"
342 << QString::fromNSString(error.localizedDescription);
343 }
344 dispatch_semaphore_signal(semaphore);
345 }];
346 dispatch_semaphore_wait(semaphore, DISPATCH_TIME_FOREVER);
347 dispatch_release(semaphore);
348
349 // Flush the dispatch_queue. After this we assume it's safe to tear everything down.
350 if (m_dispatchQueue)
351 dispatch_sync(m_dispatchQueue.data(), []{});
352}
353
354// This will commonly fail if we are missing permissions for screen capturing.
355// It will also open the "Grant permissions" system dialog if we are missing
356// permissions.
357//
358// Thread-safe.
359//
360// Error-message is not user-facing
363{
364 // Block functions can only capture copyable types.
365 // Wrap the promise in a shared-ptr.
367
368 // This function call will open the permissions system dialog when applicable.
371 NSError *error)
372 {
373 if (error != nil) {
375 return;
376 }
377
379
383
387
389 }];
390
391 return promise->get_future();
392}
393
400{
402
405 excludingWindows: @[] ] };
406
407 // SCDisplay.frame is in screen-points, not pixels. Multiply by
408 // pointPixelScale.
410 streamId,
412 frameRate,
414}
415
416// Note that we are using manual memory management here, because Obj-C block functions
417// do not support capturing move-only types.
424{
426
428 auto future = promise->get_future();
429
432
433 // SCContentFilter.contentRect is in screen-points, not pixels. Multiply by
434 // pointPixelScale.
436 // Rare edge cases have shown contentRect to sometimes be empty.
437 if (resolutionPx.isEmpty()) {
438 promise->set_value(q23::unexpected{ u"SCContentFilter contentRect reported as zero size"_s });
439 return future;
440 }
441
444
448 if (connectionSetup) {
450 }
451
454 *captureKit,
457 if (!streamOutputResult) {
459 return future;
460 }
462
465
470
472 dispatch_queue_create("qt_screencapture", DISPATCH_QUEUE_SERIAL) };
473
474 NSError *addStreamError = nullptr;
479 if (addStreamError != nil) {
480 promise->set_value(q23::unexpected(u"Unable to add stream output to SCStream"_s));
481 return future;
482 }
483
488
489 // Block functions for the completion handler require
490 // that the callable is copyable. This means we can't capture
491 // move-only types. So we temporarily release the unique_ptr here,
492 // and switch to manual memory management and then adopt them
493 // back into AVFScopedPointer inside the callback.
494 // We assume the completion handler is always called, either with success or error.
498 (NSError *error)
499 {
501
502 if (error != nil) {
503 promise->set_value(q23::unexpected{ u"Error when starting screen capturing stream"_s });
504 return;
505 }
506
508 }];
509
510 return future;
511}
512
513// Frames may arrive on the background thread immediately after the
514// creation success event has been emitted, sometimes even out of order.
515// This function takes a function that allows us to establish
516// the connections on the newly constructed object before
517// the stream ever starts, so we never miss any frames.
536
537AVFScopedPointer<SCStreamConfiguration> QMacScreenCaptureKit::createStreamConfig(
538 QSize resolutionPx,
539 std::optional<qreal> frameRate)
540{
541 // SCStreamConfiguration defines the output format, having zero resolution makes no sense.
542 Q_ASSERT(!resolutionPx.isEmpty());
543
544 // TODO: Possible improvements include specifying pixel format, HDR,
545 // capturing system audio...
546 auto scStreamConfig = AVFScopedPointer{ [[SCStreamConfiguration alloc] init] };
547 scStreamConfig.data().width = resolutionPx.width();
548 scStreamConfig.data().height = resolutionPx.height();
549 // We make a best-effort to always adjust our video output to match the window/screen size.
550 // So we leave scaling off to be pixel-perfect whenever we can.
551 scStreamConfig.data().scalesToFit = false;
552 scStreamConfig.data().queueDepth = QMacScreenCaptureKit::queueDepth;
553 scStreamConfig.data().pixelFormat = QMacScreenCaptureKit::cvPixelFormat;
554 scStreamConfig.data().colorSpaceName = QMacScreenCaptureKit::cgColorSpace();
555 scStreamConfig.data().captureResolution = SCCaptureResolutionBest;
556 if (@available(macOS 15.0, *))
557 scStreamConfig.data().captureDynamicRange = SCCaptureDynamicRangeSDR;
558
559 if (frameRate) {
560 Q_ASSERT(frameRate > 0);
561 scStreamConfig.data().minimumFrameInterval =
562 CMTimeMake(1, static_cast<int32_t>(std::round(*frameRate)));
563 } else {
564 scStreamConfig.data().minimumFrameInterval = kCMTimeZero;
565 }
566
567 return scStreamConfig;
568}
569
570// Issues a stream configuration update, so that the stream will give us video frames
571// of a new resolution.
572void QMacScreenCaptureKit::startStreamReconfigure(
573 SCStream *scStream,
574 QSize resolutionPx,
575 std::optional<qreal> frameRate)
576{
577 Q_ASSERT(scStream);
578
579 AVFScopedPointer<SCStreamConfiguration> scStreamConfig =
580 QMacScreenCaptureKit::createStreamConfig(resolutionPx, frameRate);
581
582 [scStream
583 updateConfiguration:scStreamConfig.data()
584 completionHandler:[](NSError *err) {
585 if (err) {
586 // TODO: Send potential error back to QMacScreenCaptureKit, but only
587 // if the error stops the stream.
588 qCWarning(qLcMacScreenCapture)
589 << "Error when reconfiguring ScreenCaptureKit stream:"
590 << QString::fromNSString(err.description);
591 return;
592 }
593 }];
594}
595
596// Reconfigures the stream with a new output resolution.
597// Does not stop the stream.
598// Input resolution is in pixel-coordinates.
599// Must be called from background dispatch_queue.
600void QMacScreenCaptureKit::updateStream(QSize resolutionPx)
601{
602 Q_ASSERT(m_dispatchQueue);
603 dispatch_assert_queue(m_dispatchQueue.data());
604
605 startStreamReconfigure(m_stream.data(), resolutionPx, m_frameRate);
606}
607
608} // namespace QFFmpeg
609
610QT_END_NAMESPACE
611
612#include "moc_qmacscreencapturekit_p.cpp"
613#include "qmacscreencapturekit.moc"
void updateStream(QSize resolutionPx)
Q_DECLARE_LOGGING_CATEGORY(qLcMacScreenCapture)
static void handleFrameOutput(QMacScreenCaptureStreamOutput &scStreamOutput, CMSampleBufferRef sampleBufferRef)
QT_MANGLE_NAMESPACE(QMacScreenCaptureStreamDelegate) QMacScreenCaptureStreamDelegate
static q23::expected< QVideoFrame, QString > createQVideoFrame(QMacScreenCaptureStreamOutput &scStreamOutput, CMSampleBufferRef sampleBufferRef)
static q23::expected< AVFScopedPointer< QMacScreenCaptureStreamOutput >, QString > createStreamOutput(QMacScreenCaptureKit &macScreenCaptureKit, uint32_t cvPixelFormat, QSize resolution)
static void configureStreamDelegate(QMacScreenCaptureStreamDelegate &streamDelegate, QMacScreenCaptureKit::StreamId streamId, const QMacScreenCaptureKit &macScreenCaptureKit)
static q23::expected< FrameInfo, QString > readFrameInfo(CMSampleBufferRef sampleBuffer)
Q_LOGGING_CATEGORY_IMPL(QT_PREPEND_NAMESPACE(QFFmpeg::qLcMacScreenCapture), "qt.multimedia.screencapture.macscreencapturekit")