Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qffmpeghwaccel_cuda.cpp
Go to the documentation of this file.
1// Copyright (C) 2026 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qffmpeghwaccel_cuda_p.h"
5
6#include <QtFFmpegMediaPluginImpl/private/qffmpeg_p.h>
7
8#include <QtMultimedia/private/qhwvideobuffer_p.h>
9#include <QtMultimedia/private/qmultimedia_ranges_p.h>
10#include <QtMultimedia/private/qmultimediautils_p.h>
11
12#include <QtGui/rhi/qrhi.h>
13#include <QtGui/rhi/qrhi_platform.h>
14
15#include <QtCore/qlibrary.h>
16#include <QtCore/qloggingcategory.h>
17#include <QtCore/qscopeguard.h>
18#include <QtCore/quuid.h>
19#include <QtCore/private/quniquehandle_types_p.h>
20
21#include <array>
22#include <optional>
23
24#if QT_CONFIG(opengl)
25# include <QtGui/qopenglcontext.h>
26# include <QtGui/qopengl.h>
27# include <QtGui/qopenglfunctions.h>
28# ifndef GL_RG16F
29# define GL_RG16F 0x822F
30# endif
31# ifndef GL_RG16
32# define GL_RG16 0x822C
33# endif
34# ifndef GL_R16
35# define GL_R16 0x822A
36# endif
37#endif
38
39#if QT_CONFIG(vulkan)
40# include <QtMultimedia/private/qmultimedia_vulkan_support_p.h>
41# include <QtGui/qvulkanfunctions.h>
42# include <QtGui/qvulkaninstance.h>
43# include <vulkan/vulkan.h>
44#endif
45
46#if QT_CONFIG(opengl) || QT_CONFIG(vulkan)
47# define QT_FFMPEG_HAS_CUDA_TEXTURE_INTEROP 1
48#endif
49
50#if QT_CONFIG(opengl) && (!QT_CONFIG(opengles2) || QT_CONFIG(opengles3))
51# define QT_FFMPEG_HAS_CUDA_GL_TEXTURE_INTEROP 1
52#endif
53
54extern "C" {
55
56// dynlink_cuda.h declares the CUDA driver API without requiring a CUDA Toolkit install;
57// including it first also makes libavutil/hwcontext_cuda.h skip its own "<cuda.h>" fallback.
58// The header is vendored in src/3rdparty/nv-codec-headers, so it is always available.
59#include <ffnvcodec/dynlink_cuda.h>
60
61#include <libavutil/hwcontext.h>
62#include <libavutil/hwcontext_cuda.h>
63}
64
65QT_BEGIN_NAMESPACE
66
67using namespace Qt::StringLiterals;
68
69Q_STATIC_LOGGING_CATEGORY(qLHWAccelCUDA, "qt.multimedia.ffmpeg.hwaccelcuda")
70
71namespace QFFmpeg {
72
73template <typename T>
74using PerPlaneArray = std::array<T, QVideoTextureHelper::TextureDescription::maxPlanes>;
75
76#if QT_CONFIG(opengl) || QT_CONFIG(vulkan)
77
78namespace {
79
80class CudaDriverApi
81{
82public:
83 static const CudaDriverApi *get()
84 {
85 static std::optional<CudaDriverApi> singleton = []() -> std::optional<CudaDriverApi> {
86 CudaDriverApi api;
87 if (!api.isValid())
88 return std::nullopt;
89 return std::optional<CudaDriverApi>{ std::move(api) };
90 }();
91 return singleton ? &*singleton : nullptr;
92 }
93
94 bool pushCurrent(CUcontext ctx) const { return cuCtxPushCurrent_v2(ctx) == CUDA_SUCCESS; }
95 bool popCurrent() const
96 {
97 CUcontext popped = nullptr;
98 return cuCtxPopCurrent_v2(&popped) == CUDA_SUCCESS;
99 }
100
101 tcuGraphicsGLRegisterImage *cuGraphicsGLRegisterImage = nullptr;
102 tcuGraphicsUnregisterResource *cuGraphicsUnregisterResource = nullptr;
103 tcuGraphicsMapResources *cuGraphicsMapResources = nullptr;
104 tcuGraphicsUnmapResources *cuGraphicsUnmapResources = nullptr;
105 tcuGraphicsSubResourceGetMappedArray *cuGraphicsSubResourceGetMappedArray = nullptr;
106 tcuMemcpy2D_v2 *cuMemcpy2D_v2 = nullptr;
107 tcuCtxPushCurrent_v2 *cuCtxPushCurrent_v2 = nullptr;
108 tcuCtxPopCurrent_v2 *cuCtxPopCurrent_v2 = nullptr;
109 tcuCtxGetDevice *cuCtxGetDevice = nullptr;
110 tcuStreamSynchronize *cuStreamSynchronize = nullptr;
111 tcuDeviceGetUuid *cuDeviceGetUuid = nullptr;
112 tcuImportExternalMemory *cuImportExternalMemory = nullptr;
113 tcuDestroyExternalMemory *cuDestroyExternalMemory = nullptr;
114 tcuExternalMemoryGetMappedMipmappedArray *cuExternalMemoryGetMappedMipmappedArray = nullptr;
115 tcuMipmappedArrayGetLevel *cuMipmappedArrayGetLevel = nullptr;
116 tcuGetErrorString *cuGetErrorString = nullptr;
117
118private:
119 CudaDriverApi()
120 : m_library{
121 std::make_unique<QLibrary>(u"cuda"_s),
122 }
123 {
124 if (!m_library->load()) {
125 qCDebug(qLHWAccelCUDA)
126 << "libcuda.so.1 could not be loaded:" << m_library->errorString();
127 m_library = nullptr;
128 return;
129 }
130
131# define RESOLVE(name)
132 name = reinterpret_cast<decltype(name)>(m_library->resolve(#name));
133 if (!name) {
134 qCDebug(qLHWAccelCUDA) << "libcuda.so.1 is missing symbol:" << #name;
135 m_library = nullptr;
136 return;
137 }
138
139 RESOLVE(cuGraphicsGLRegisterImage)
140 RESOLVE(cuGraphicsUnregisterResource)
141 RESOLVE(cuGraphicsMapResources)
142 RESOLVE(cuGraphicsUnmapResources)
143 RESOLVE(cuGraphicsSubResourceGetMappedArray)
144 RESOLVE(cuMemcpy2D_v2)
145 RESOLVE(cuCtxPushCurrent_v2)
146 RESOLVE(cuCtxPopCurrent_v2)
147 RESOLVE(cuCtxGetDevice)
148 RESOLVE(cuStreamSynchronize)
149 RESOLVE(cuDeviceGetUuid)
150 RESOLVE(cuImportExternalMemory)
151 RESOLVE(cuDestroyExternalMemory)
152 RESOLVE(cuExternalMemoryGetMappedMipmappedArray)
153 RESOLVE(cuMipmappedArrayGetLevel)
154 RESOLVE(cuGetErrorString)
155# undef RESOLVE
156 }
157
158 bool isValid() const { return bool(m_library); }
159 std::unique_ptr<QLibrary> m_library;
160};
161
162class CudaContextGuard
163{
164public:
165 static std::optional<CudaContextGuard> create(const CudaDriverApi &api, CUcontext ctx)
166 {
167 if (!api.pushCurrent(ctx))
168 return std::nullopt;
169 return CudaContextGuard(api);
170 }
171
172 CudaContextGuard(CudaContextGuard &&other) noexcept : m_api(other.m_api)
173 {
174 other.m_api = nullptr;
175 }
176
177 CudaContextGuard &operator=(CudaContextGuard &&) = delete;
178
179 ~CudaContextGuard()
180 {
181 if (m_api)
182 m_api->popCurrent();
183 }
184
185 Q_DISABLE_COPY(CudaContextGuard)
186
187private:
188 explicit CudaContextGuard(const CudaDriverApi &api) : m_api(&api) { }
189
190 const CudaDriverApi *m_api;
191};
192
193class CudaResult
194{
195public:
196 CudaResult(CUresult result, const CudaDriverApi &api) : m_result(result), m_api(api) { }
197
198private:
199 friend QDebug operator<<(QDebug debug, const CudaResult &r)
200 {
201 const QDebugStateSaver saver(debug);
202 const char *errorString = nullptr;
203 r.m_api.cuGetErrorString(r.m_result, &errorString);
204 debug.nospace().noquote() << "CUresult(0x" << Qt::hex << qToUnderlying(r.m_result)
205 << Qt::dec << ", "
206 << (errorString ? errorString : "unknown error") << ')';
207 return debug;
208 }
209
210 CUresult m_result;
211 const CudaDriverApi &m_api;
212};
213
214// The layout of one plane of a CUDA-decoded pixel format, backend-agnostic: just enough to
215// compute plane sizes and the byte width of the CUDA memcpy.
216struct CudaPlaneGeometry
217{
218 int sampleSize = 1; // bytes per sample: 1 for 8-bit formats, 2 for 10/16-bit-in-16-bit
219 int numChannels = 1; // 1 (luma) or 2 (interleaved chroma)
220 // The plane's size is the frame size divided by these factors, e.g. 2/2 for a 4:2:0
221 // chroma plane. Default to 1, not 0, since these are divisors.
222 int widthDivisor = 1;
223 int heightDivisor = 1;
224
225 int bytesPerTexel() const { return sampleSize * numChannels; }
226};
227
228struct CudaFrameGeometry
229{
230 int planeCount = 0;
231 PerPlaneArray<CudaPlaneGeometry> planes{};
232};
233
234// NVDEC only ever decodes into semi-planar (2-plane, chroma-subsampled) formats - never
235// fully-planar ones - so this table only needs to cover that family.
236std::optional<CudaFrameGeometry> cudaFrameGeometry(AVPixelFormat swFormat)
237{
238 switch (swFormat) {
239 case AV_PIX_FMT_NV12:
240 case AV_PIX_FMT_NV21:
241 return CudaFrameGeometry{
242 2,
243 { {
244 CudaPlaneGeometry{ 1, 1, 1, 1 },
245 CudaPlaneGeometry{ 1, 2, 2, 2 },
246 } },
247 };
248 case AV_PIX_FMT_P010:
249 case AV_PIX_FMT_P016:
250 return CudaFrameGeometry{
251 2,
252 { {
253 CudaPlaneGeometry{ 2, 1, 1, 1 },
254 CudaPlaneGeometry{ 2, 2, 2, 2 },
255 } },
256 };
257 default:
258 return std::nullopt;
259 }
260}
261
262// Exposes each plane's texture as a native handle: a GLuint on OpenGL, a VkImage on Vulkan.
263class CudaTextureHandles final : public QVideoFrameTexturesHandles
264{
265public:
266 CudaTextureHandles(int nPlanes, PerPlaneArray<quint64> handles,
267 std::shared_ptr<void> parentKeepAlive)
268 : m_nPlanes(nPlanes), m_handles(handles), m_parentKeepAlive(std::move(parentKeepAlive))
269 {
270 }
271
272 quint64 textureHandle(QRhi &, int plane) override
273 {
274 return plane < m_nPlanes ? m_handles[plane] : 0;
275 }
276
277private:
278 const int m_nPlanes;
279 const PerPlaneArray<quint64> m_handles;
280 const std::shared_ptr<void>
281 m_parentKeepAlive; // keeps the converter (and its GL/Vulkan resources) alive
282};
283
284// ---------------------------------------------------------------------------------------------
285// OpenGL backend
286// ---------------------------------------------------------------------------------------------
287
288# if QT_FFMPEG_HAS_CUDA_GL_TEXTURE_INTEROP
289
290struct GlPlaneFormat
291{
292 GLenum internalFormat;
293 GLenum format;
294 GLenum type;
295};
296
297GlPlaneFormat glPlaneFormat(const CudaPlaneGeometry &geometry)
298{
299 const GLenum type = geometry.sampleSize == 1 ? GL_UNSIGNED_BYTE : GL_UNSIGNED_SHORT;
300 if (geometry.numChannels == 1)
301 return GlPlaneFormat{
302 GLenum(geometry.sampleSize == 1 ? GL_R8 : GL_R16),
303 GLenum(GL_RED),
304 type,
305 };
306 return GlPlaneFormat{
307 GLenum(geometry.sampleSize == 1 ? GL_RG8 : GL_RG16),
308 GLenum(GL_RG),
309 type,
310 };
311}
312
313struct GlPlaneResource
314{
315 ~GlPlaneResource() { Q_ASSERT(texture == 0 && !cudaResource); }
316
317 void release(const CudaDriverApi *cudaApi, QOpenGLFunctions &functions)
318 {
319 if (cudaResource && cudaApi)
320 cudaApi->cuGraphicsUnregisterResource(cudaResource);
321 if (texture)
322 functions.glDeleteTextures(1, &texture);
323 cudaResource = nullptr;
324 texture = 0;
325 }
326
327 GLuint texture = 0;
328 CUgraphicsResource cudaResource = nullptr;
329};
330
331class CUDAGLTextureConverter final : public TextureConverterBackend
332{
333public:
334 static std::shared_ptr<TextureConverterBackend> create(QRhi *rhi)
335 {
336 if (!rhi)
337 return nullptr;
338
339 auto *api = CudaDriverApi::get();
340 if (!api) {
341 qCDebug(qLHWAccelCUDA) << "CUDA driver API is not available, disabling";
342 return nullptr;
343 }
344
345 return std::make_shared<CUDAGLTextureConverter>(rhi, api);
346 }
347
348 explicit CUDAGLTextureConverter(QRhi *rhi, const CudaDriverApi *api)
349 : TextureConverterBackend(rhi), m_api{ api }
350 {
351 }
352
353 ~CUDAGLTextureConverter() override { releasePlaneResources(); }
354
355 QVideoFrameTexturesHandlesUPtr
356 createTextureHandles(AVFrame *frame, QVideoFrameTexturesHandlesUPtr /*oldHandles*/) override
357 {
358 if (frame->format != AV_PIX_FMT_CUDA || !frame->hw_frames_ctx)
359 return nullptr;
360
361 auto *framesCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
362 const std::optional<CudaFrameGeometry> geometry = cudaFrameGeometry(framesCtx->sw_format);
363 if (!geometry) {
364 qWarning() << "CUDATextureConverter does not support sw_format" << framesCtx->sw_format;
365 return nullptr;
366 }
367
368 auto *deviceCtx = avFrameDeviceContext(frame);
369 if (!deviceCtx)
370 return nullptr;
371 auto *cudaCtx = reinterpret_cast<AVCUDADeviceContext *>(deviceCtx->hwctx);
372
373 std::optional<CudaContextGuard> guard = CudaContextGuard::create(*m_api, cudaCtx->cuda_ctx);
374 if (!guard) {
375 qWarning() << "Failed to push the CUDA context current; disabling";
376 rhi = nullptr;
377 return nullptr;
378 }
379 m_cudaContext = cudaCtx->cuda_ctx;
380
381 if (!ensurePlaneResources(frame))
382 return nullptr;
383
384 PerPlaneArray<CUgraphicsResource> resources{};
385 for (int i = 0; i < geometry->planeCount; ++i)
386 resources[i] = m_planes[i]->cudaResource;
387 const CUresult mapResult =
388 m_api->cuGraphicsMapResources(geometry->planeCount, resources.data(), nullptr);
389 if (mapResult != CUDA_SUCCESS) {
390 qWarning() << "cuGraphicsMapResources failed:" << CudaResult(mapResult, *m_api);
391 return nullptr;
392 }
393
394 auto unmapGuard = qScopeGuard([&] {
395 m_api->cuGraphicsUnmapResources(geometry->planeCount, resources.data(), nullptr);
396 });
397
398 PerPlaneArray<quint64> handles{};
399 for (int i = 0; i < geometry->planeCount; ++i) {
400 const CudaPlaneGeometry &planeGeometry = geometry->planes[i];
401 const QSize planeSize(frame->width / planeGeometry.widthDivisor,
402 frame->height / planeGeometry.heightDivisor);
403
404 CUarray array = nullptr;
405 const CUresult subResourceResult =
406 m_api->cuGraphicsSubResourceGetMappedArray(&array, resources[i], 0, 0);
407 if (subResourceResult != CUDA_SUCCESS) {
408 qWarning() << "cuGraphicsSubResourceGetMappedArray failed for plane" << i << ":"
409 << CudaResult(subResourceResult, *m_api);
410 return nullptr;
411 }
412
413 const CUDA_MEMCPY2D copy{
414 .srcXInBytes = 0,
415 .srcY = 0,
416 .srcMemoryType = CU_MEMORYTYPE_DEVICE,
417 .srcHost = nullptr,
418 .srcDevice = reinterpret_cast<CUdeviceptr>(frame->data[i]),
419 .srcArray = nullptr,
420 .srcPitch = size_t(frame->linesize[i]),
421 .dstXInBytes = 0,
422 .dstY = 0,
423 .dstMemoryType = CU_MEMORYTYPE_ARRAY,
424 .dstHost = nullptr,
425 .dstDevice = 0,
426 .dstArray = array,
427 .dstPitch = 0,
428 .WidthInBytes = size_t(planeSize.width()) * size_t(planeGeometry.bytesPerTexel()),
429 .Height = size_t(planeSize.height()),
430 };
431
432 const CUresult copyResult = m_api->cuMemcpy2D_v2(&copy);
433 if (copyResult != CUDA_SUCCESS) {
434 qWarning() << "cuMemcpy2D_v2 failed for plane" << i << ":"
435 << CudaResult(copyResult, *m_api);
436 return nullptr;
437 }
438
439 handles[i] = quint64(m_planes[i]->texture);
440 }
441
442 return std::make_unique<CudaTextureHandles>(geometry->planeCount, handles,
443 shared_from_this());
444 }
445
446private:
447 bool ensurePlaneResources(AVFrame *frame)
448 {
449 auto *framesCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
450 const std::optional<CudaFrameGeometry> maybeGeometry =
451 cudaFrameGeometry(framesCtx->sw_format);
452 Q_ASSERT(maybeGeometry); // already validated by createTextureHandles
453 const CudaFrameGeometry &geometry = *maybeGeometry;
454
455 const QSize frameSize(frame->width, frame->height);
456 const bool canReuse = frameSize == m_planeResourcesSize
457 && framesCtx->sw_format == m_planeResourcesFormat
458 && ranges::all_of(QSpan(m_planes.data(), geometry.planeCount),
459 [](const auto &plane) {
460 return bool(plane);
461 });
462 if (canReuse)
463 return true;
464
465 releasePlaneResources();
466 m_planeResourcesSize = frameSize;
467 m_planeResourcesFormat = framesCtx->sw_format;
468
469 auto *glContext =
470 static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles())->context;
471 QOpenGLFunctions functions(glContext);
472
473 for (int i = 0; i < geometry.planeCount; ++i) {
474 const CudaPlaneGeometry &planeGeometry = geometry.planes[i];
475 const QSize planeSize(m_planeResourcesSize.width() / planeGeometry.widthDivisor,
476 m_planeResourcesSize.height() / planeGeometry.heightDivisor);
477 const GlPlaneFormat format = glPlaneFormat(planeGeometry);
478
479 auto plane = std::make_unique<GlPlaneResource>();
480 functions.glGenTextures(1, &plane->texture);
481 functions.glBindTexture(GL_TEXTURE_2D, plane->texture);
482 functions.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
483 functions.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
484 functions.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
485 functions.glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
486 functions.glTexImage2D(GL_TEXTURE_2D, 0, format.internalFormat, planeSize.width(),
487 planeSize.height(), 0, format.format, format.type, nullptr);
488 functions.glBindTexture(GL_TEXTURE_2D, 0);
489
490 const CUresult result = m_api->cuGraphicsGLRegisterImage(
491 &plane->cudaResource, plane->texture, GL_TEXTURE_2D,
492 CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD);
493 if (result != CUDA_SUCCESS) {
494 qWarning() << "cuGraphicsGLRegisterImage failed for plane" << i << ":"
495 << CudaResult(result, *m_api);
496 plane->release(m_api, functions);
497 return false;
498 }
499
500 m_planes[i] = std::move(plane);
501 }
502
503 return true;
504 }
505
506 void releasePlaneResources()
507 {
508 if (!rhi)
509 return;
510
511 std::optional<CudaContextGuard> guard = m_cudaContext && m_api
512 ? CudaContextGuard::create(*m_api, m_cudaContext)
513 : std::nullopt;
514
515 rhi->makeThreadLocalNativeContextCurrent();
516 auto *glContext =
517 static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles())->context;
518 QOpenGLFunctions functions(glContext);
519
520 for (auto &plane : m_planes) {
521 if (!plane)
522 continue;
523 plane->release(guard ? m_api : nullptr, functions);
524 plane.reset();
525 }
526 }
527
528 const CudaDriverApi *const m_api;
529 PerPlaneArray<std::unique_ptr<GlPlaneResource>> m_planes;
530 QSize m_planeResourcesSize;
531 AVPixelFormat m_planeResourcesFormat = AV_PIX_FMT_NONE;
532 CUcontext m_cudaContext = nullptr;
533};
534
535# endif // QT_FFMPEG_HAS_CUDA_GL_TEXTURE_INTEROP
536
537// ---------------------------------------------------------------------------------------------
538// Vulkan backend
539// ---------------------------------------------------------------------------------------------
540
541# if QT_CONFIG(vulkan)
542
543// Pass a device to resolve a device-level function (e.g. of VK_KHR_external_memory_fd), which
544// yields nullptr if the extension is not enabled on that device. Without a device, an
545// instance-level function is resolved.
546template <typename Func>
547Func resolveVulkanFunctionOrWarn(QVulkanInstance &inst, const char *name,
548 VkDevice dev = VK_NULL_HANDLE)
549{
550 using namespace QtMultimediaPrivate;
551 Func func = dev ? resolveVulkanDeviceFunction<Func>(inst, dev, name)
552 : resolveVulkanFunction<Func>(inst, name);
553 if (!func)
554 qCDebug(qLHWAccelCUDA) << "Failed to resolve Vulkan function" << name;
555 return func;
556}
557
558// vkGetPhysicalDeviceProperties2 is core since Vulkan 1.1, with vkGetPhysicalDeviceProperties2KHR
559// as the (functionally identical) extension entry point on older API versions/instances.
560PFN_vkGetPhysicalDeviceProperties2 resolveVkGetPhysicalDeviceProperties2(QVulkanInstance &inst)
561{
562 using namespace QtMultimediaPrivate;
563 PFN_vkGetPhysicalDeviceProperties2 func =
564 resolveVulkanFunction<PFN_vkGetPhysicalDeviceProperties2>(
565 inst, "vkGetPhysicalDeviceProperties2");
566 if (!func) {
567 func = resolveVulkanFunctionOrWarn<PFN_vkGetPhysicalDeviceProperties2>(
568 inst, "vkGetPhysicalDeviceProperties2KHR");
569 }
570 return func;
571}
572
573QUuid uuidFromCUuuid(const CUuuid &cuUuid)
574{
575 return QUuid::fromRfc4122(QByteArrayView(reinterpret_cast<const char *>(cuUuid.bytes), 16));
576}
577
578// Resolves the Vulkan state/functions needed to export a plane's VkImage memory as a POSIX
579// fd (VK_KHR_external_memory_fd) for import into CUDA. Bound to one QRhi::Vulkan instance.
580class CudaVulkanInterop
581{
582public:
583 explicit CudaVulkanInterop(const QRhiVulkanNativeHandles &handles)
584 : inst(handles.inst),
585 physDev(handles.physDev),
586 dev(handles.dev),
587 deviceFuncs(handles.inst->deviceFunctions(handles.dev)),
588 instFuncs(handles.inst->functions()),
589 vkGetMemoryFdKHR(resolveVulkanFunctionOrWarn<PFN_vkGetMemoryFdKHR>(
590 *handles.inst, "vkGetMemoryFdKHR", handles.dev)),
591 vkGetPhysicalDeviceProperties2(resolveVkGetPhysicalDeviceProperties2(*handles.inst))
592 {
593 }
594
595 static std::unique_ptr<CudaVulkanInterop> create(QRhi *rhi)
596 {
597 Q_ASSERT(rhi->backend() == QRhi::Vulkan);
598 const auto *handles = static_cast<const QRhiVulkanNativeHandles *>(rhi->nativeHandles());
599 if (!handles || !handles->inst || !handles->dev || !handles->physDev)
600 return nullptr;
601
602 auto interop = std::make_unique<CudaVulkanInterop>(*handles);
603 if (!interop->deviceFuncs || !interop->instFuncs)
604 return nullptr;
605
606 if (!interop->vkGetMemoryFdKHR || !interop->vkGetPhysicalDeviceProperties2) {
607 qCDebug(qLHWAccelCUDA)
608 << "VK_KHR_external_memory_fd or vkGetPhysicalDeviceProperties2 is either "
609 "unavailable or not enabled on this QRhi's VkDevice/VkInstance, disabling. "
610 "To use the CUDA texture interop with Vulkan, enable the device extensions "
611 "VK_KHR_external_memory and VK_KHR_external_memory_fd when creating the "
612 "QRhi, e.g. with QQuickGraphicsConfiguration::setDeviceExtensions() or "
613 "QRhiVulkanInitParams::deviceExtensions";
614 return nullptr;
615 }
616
617 return interop;
618 }
619
620 enum class MatchResult : uint8_t {
621 SameGpu,
622 DifferentGpu,
623 Unknown, // Vulkan device UUID could not be queried
624 };
625 MatchResult matchesCudaDevice(const CUuuid &cudaUuid) const
626 {
627 using namespace QtMultimediaPrivate;
628
629 VkPhysicalDeviceIDProperties idProps{
630 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES,
631 .pNext = nullptr,
632 .deviceUUID = {},
633 .driverUUID = {},
634 .deviceLUID = {},
635 .deviceNodeMask = 0,
636 .deviceLUIDValid = VK_FALSE,
637 };
638 VkPhysicalDeviceProperties2 props2{
639 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2,
640 .pNext = &idProps,
641 .properties = {},
642 };
643 vkGetPhysicalDeviceProperties2(physDev, &props2);
644 const QUuid vkDeviceUuid = uuidFromVkUuid(idProps.deviceUUID);
645 if (vkDeviceUuid.isNull())
646 return MatchResult::Unknown;
647 const QUuid cuDeviceUuid = uuidFromCUuuid(cudaUuid);
648 return (vkDeviceUuid == cuDeviceUuid) ? MatchResult::SameGpu : MatchResult::DifferentGpu;
649 }
650
651 std::optional<uint32_t> findDeviceLocalMemoryTypeIndex(uint32_t memoryTypeBits) const
652 {
653 VkPhysicalDeviceMemoryProperties memProps{};
654 instFuncs->vkGetPhysicalDeviceMemoryProperties(physDev, &memProps);
655 for (uint32_t i = 0; i < memProps.memoryTypeCount; ++i) {
656 const bool allowed = memoryTypeBits & (1u << i);
657 const bool deviceLocal =
658 memProps.memoryTypes[i].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
659 if (allowed && deviceLocal)
660 return i;
661 }
662 return std::nullopt;
663 }
664
665 QVulkanInstance *const inst;
666 const VkPhysicalDevice physDev;
667 const VkDevice dev;
668 QVulkanDeviceFunctions *const deviceFuncs;
669 QVulkanFunctions *const instFuncs;
670 const PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR;
671 const PFN_vkGetPhysicalDeviceProperties2 vkGetPhysicalDeviceProperties2;
672};
673
674struct VulkanPlaneFormat
675{
676 VkFormat vkFormat;
677 CUarray_format cuArrayFormat;
678};
679
680VulkanPlaneFormat vulkanPlaneFormat(const CudaPlaneGeometry &geometry)
681{
682 const CUarray_format cuFormat =
683 geometry.sampleSize == 1 ? CU_AD_FORMAT_UNSIGNED_INT8 : CU_AD_FORMAT_UNSIGNED_INT16;
684 if (geometry.numChannels == 1)
685 return VulkanPlaneFormat{
686 geometry.sampleSize == 1 ? VK_FORMAT_R8_UNORM : VK_FORMAT_R16_UNORM,
687 cuFormat,
688 };
689 return VulkanPlaneFormat{
690 geometry.sampleSize == 1 ? VK_FORMAT_R8G8_UNORM : VK_FORMAT_R16G16_UNORM,
691 cuFormat,
692 };
693}
694
695// Vulkan interop (VK_KHR_external_memory_fd + cuImportExternalMemory).
696struct VulkanPlaneResource
697{
698 ~VulkanPlaneResource()
699 {
700 Q_ASSERT(image == VK_NULL_HANDLE && memory == VK_NULL_HANDLE && !externalMemory);
701 }
702
703 void release(const CudaDriverApi *cudaApi, CudaVulkanInterop &vulkan)
704 {
705 if (externalMemory && cudaApi)
706 cudaApi->cuDestroyExternalMemory(externalMemory);
707 if (memory != VK_NULL_HANDLE)
708 vulkan.deviceFuncs->vkFreeMemory(vulkan.dev, memory, nullptr);
709 if (image != VK_NULL_HANDLE)
710 vulkan.deviceFuncs->vkDestroyImage(vulkan.dev, image, nullptr);
711 externalMemory = nullptr;
712 mipmappedArray = nullptr;
713 array = nullptr;
714 memory = VK_NULL_HANDLE;
715 image = VK_NULL_HANDLE;
716 }
717
718 VkImage image = VK_NULL_HANDLE;
719 VkDeviceMemory memory = VK_NULL_HANDLE;
720 CUexternalMemory externalMemory = nullptr;
721 CUmipmappedArray mipmappedArray = nullptr;
722 CUarray array = nullptr; // level-0 view of mipmappedArray, cached for the copy
723};
724
725class CUDAVulkanTextureConverter final : public TextureConverterBackend
726{
727 using MatchResult = CudaVulkanInterop::MatchResult;
728
729public:
730 static std::shared_ptr<TextureConverterBackend> create(QRhi *rhi)
731 {
732 if (!rhi)
733 return nullptr;
734
735 auto *api = CudaDriverApi::get();
736 if (!api) {
737 qCDebug(qLHWAccelCUDA) << "CUDA driver API is not available, disabling";
738 return nullptr;
739 }
740
741 auto vulkanApi = CudaVulkanInterop::create(rhi);
742 if (!vulkanApi) {
743 qCDebug(qLHWAccelCUDA) << "CUDA-Vulkan interop is not available, disabling";
744 return nullptr;
745 }
746
747 return std::make_shared<CUDAVulkanTextureConverter>(rhi, api, std::move(vulkanApi));
748 }
749
750 explicit CUDAVulkanTextureConverter(QRhi *rhi, const CudaDriverApi *api,
751 std::unique_ptr<CudaVulkanInterop> vulkanApi)
752 : TextureConverterBackend(rhi), m_api{ api }, m_vulkan{ std::move(vulkanApi) }
753 {
754 }
755
756 ~CUDAVulkanTextureConverter() override { releasePlaneResources(); }
757
758 QVideoFrameTexturesHandlesUPtr
759 createTextureHandles(AVFrame *frame, QVideoFrameTexturesHandlesUPtr /*oldHandles*/) override
760 {
761 if (frame->format != AV_PIX_FMT_CUDA || !frame->hw_frames_ctx)
762 return nullptr;
763
764 auto *framesCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
765 const std::optional<CudaFrameGeometry> geometry = cudaFrameGeometry(framesCtx->sw_format);
766 if (!geometry) {
767 qWarning() << "CUDATextureConverter does not support sw_format" << framesCtx->sw_format;
768 return nullptr;
769 }
770
771 auto *deviceCtx = avFrameDeviceContext(frame);
772 if (!deviceCtx)
773 return nullptr;
774 auto *cudaCtx = reinterpret_cast<AVCUDADeviceContext *>(deviceCtx->hwctx);
775
776 std::optional<CudaContextGuard> guard = CudaContextGuard::create(*m_api, cudaCtx->cuda_ctx);
777 if (!guard) {
778 qWarning() << "Failed to push the CUDA context current; disabling";
779 rhi = nullptr;
780 return nullptr;
781 }
782 m_cudaContext = cudaCtx->cuda_ctx;
783
784 if (!ensurePlaneResources(frame))
785 return nullptr;
786
787 PerPlaneArray<quint64> handles{};
788 for (int i = 0; i < geometry->planeCount; ++i) {
789 const CudaPlaneGeometry &planeGeometry = geometry->planes[i];
790 const QSize planeSize(frame->width / planeGeometry.widthDivisor,
791 frame->height / planeGeometry.heightDivisor);
792
793 const CUDA_MEMCPY2D copy{
794 .srcXInBytes = 0,
795 .srcY = 0,
796 .srcMemoryType = CU_MEMORYTYPE_DEVICE,
797 .srcHost = nullptr,
798 .srcDevice = reinterpret_cast<CUdeviceptr>(frame->data[i]),
799 .srcArray = nullptr,
800 .srcPitch = size_t(frame->linesize[i]),
801 .dstXInBytes = 0,
802 .dstY = 0,
803 .dstMemoryType = CU_MEMORYTYPE_ARRAY,
804 .dstHost = nullptr,
805 .dstDevice = 0,
806 .dstArray = m_planes[i]->array,
807 .dstPitch = 0,
808 .WidthInBytes = size_t(planeSize.width()) * size_t(planeGeometry.bytesPerTexel()),
809 .Height = size_t(planeSize.height()),
810 };
811
812 const CUresult copyResult = m_api->cuMemcpy2D_v2(&copy);
813 if (copyResult != CUDA_SUCCESS) {
814 qWarning() << "cuMemcpy2D_v2 failed for plane" << i << ":"
815 << CudaResult(copyResult, *m_api);
816 return nullptr;
817 }
818
819 handles[i] = quint64(m_planes[i]->image);
820 }
821
822 // No external-semaphore pipelining (see the class docstring): block until the
823 // copies actually land before handing the VkImages to the RHI. cuMemcpy2D_v2
824 // runs on the default stream, so synchronizing that stream is sufficient.
825 const CUresult syncResult = m_api->cuStreamSynchronize(nullptr);
826 if (syncResult != CUDA_SUCCESS) {
827 qWarning() << "cuStreamSynchronize failed:" << CudaResult(syncResult, *m_api);
828 return nullptr;
829 }
830
831 return std::make_unique<CudaTextureHandles>(geometry->planeCount, handles,
832 shared_from_this());
833 }
834
835private:
836 bool ensurePlaneResources(AVFrame *frame)
837 {
838 auto *framesCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data);
839 const std::optional<CudaFrameGeometry> maybeGeometry =
840 cudaFrameGeometry(framesCtx->sw_format);
841 Q_ASSERT(maybeGeometry); // already validated by createTextureHandles
842 const CudaFrameGeometry &geometry = *maybeGeometry;
843
844 const QSize frameSize(frame->width, frame->height);
845 const bool canReuse = frameSize == m_planeResourcesSize
846 && framesCtx->sw_format == m_planeResourcesFormat
847 && ranges::all_of(QSpan(m_planes.data(), geometry.planeCount),
848 [](const auto &plane) {
849 return bool(plane);
850 });
851 if (canReuse)
852 return true;
853
854 releasePlaneResources();
855 m_planeResourcesSize = frameSize;
856 m_planeResourcesFormat = framesCtx->sw_format;
857
858 if (m_vulkanDeviceMatches == MatchResult::Unknown) {
859 CUdevice device = 0;
860 CUuuid cudaUuid{};
861 const bool ok = m_api->cuCtxGetDevice(&device) == CUDA_SUCCESS
862 && m_api->cuDeviceGetUuid(&cudaUuid, device) == CUDA_SUCCESS;
863 m_vulkanDeviceMatches =
864 ok ? m_vulkan->matchesCudaDevice(cudaUuid) : MatchResult::Unknown;
865 if (m_vulkanDeviceMatches == MatchResult::Unknown) {
866 qWarning() << "CUDATextureConverter: could not determine the QRhi's Vulkan "
867 "physical device UUID (missing "
868 "VK_KHR_get_physical_device_properties2 on the VkInstance?); "
869 "disabling";
870 } else if (m_vulkanDeviceMatches == MatchResult::DifferentGpu) {
871 qWarning() << "CUDATextureConverter: the CUDA device does not match the QRhi's "
872 "Vulkan physical device (e.g. a multi-GPU system); disabling";
873 }
874 }
875 if (m_vulkanDeviceMatches != MatchResult::SameGpu)
876 return false;
877
878 for (int i = 0; i < geometry.planeCount; ++i) {
879 const CudaPlaneGeometry &planeGeometry = geometry.planes[i];
880 const QSize planeSize(m_planeResourcesSize.width() / planeGeometry.widthDivisor,
881 m_planeResourcesSize.height() / planeGeometry.heightDivisor);
882 const VulkanPlaneFormat format = vulkanPlaneFormat(planeGeometry);
883
884 auto plane = std::make_unique<VulkanPlaneResource>();
885 auto releaseGuard = qScopeGuard([&] {
886 plane->release(m_api, *m_vulkan);
887 });
888
889 const VkExternalMemoryImageCreateInfo extImageInfo{
890 .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
891 .pNext = nullptr,
892 .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
893 };
894
895 // TRANSFER_SRC is not needed by the RHI (it only samples this texture), but it
896 // costs nothing to declare and lets tests validate the copy via a readback.
897 const VkImageCreateInfo imageInfo{
898 .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
899 .pNext = &extImageInfo,
900 .flags = 0,
901 .imageType = VK_IMAGE_TYPE_2D,
902 .format = format.vkFormat,
903 .extent =
904 VkExtent3D{
905 .width = uint32_t(planeSize.width()),
906 .height = uint32_t(planeSize.height()),
907 .depth = 1u,
908 },
909 .mipLevels = 1,
910 .arrayLayers = 1,
911 .samples = VK_SAMPLE_COUNT_1_BIT,
912 .tiling = VK_IMAGE_TILING_OPTIMAL,
913 .usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT
914 | VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
915 .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
916 .queueFamilyIndexCount = 0,
917 .pQueueFamilyIndices = nullptr,
918 .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
919 };
920
921 if (m_vulkan->deviceFuncs->vkCreateImage(m_vulkan->dev, &imageInfo, nullptr,
922 &plane->image)
923 != VK_SUCCESS) {
924 qWarning() << "CUDATextureConverter: vkCreateImage failed for plane" << i;
925 return false;
926 }
927
928 VkMemoryRequirements memReq{};
929 m_vulkan->deviceFuncs->vkGetImageMemoryRequirements(m_vulkan->dev, plane->image,
930 &memReq);
931
932 const std::optional<uint32_t> memoryTypeIndex =
933 m_vulkan->findDeviceLocalMemoryTypeIndex(memReq.memoryTypeBits);
934 if (!memoryTypeIndex) {
935 qWarning() << "CUDATextureConverter: no suitable Vulkan memory type for plane" << i;
936 return false;
937 }
938
939 const VkExportMemoryAllocateInfo exportInfo{
940 .sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO,
941 .pNext = nullptr,
942 .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
943 };
944
945 const VkMemoryAllocateInfo allocInfo{
946 .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
947 .pNext = &exportInfo,
948 .allocationSize = memReq.size,
949 .memoryTypeIndex = *memoryTypeIndex,
950 };
951
952 if (m_vulkan->deviceFuncs->vkAllocateMemory(m_vulkan->dev, &allocInfo, nullptr,
953 &plane->memory)
954 != VK_SUCCESS) {
955 qWarning() << "CUDATextureConverter: vkAllocateMemory failed for plane" << i;
956 return false;
957 }
958
959 if (m_vulkan->deviceFuncs->vkBindImageMemory(m_vulkan->dev, plane->image, plane->memory,
960 0)
961 != VK_SUCCESS) {
962 qWarning() << "CUDATextureConverter: vkBindImageMemory failed for plane" << i;
963 return false;
964 }
965
966 const VkMemoryGetFdInfoKHR getFdInfo{
967 .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
968 .pNext = nullptr,
969 .memory = plane->memory,
970 .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT,
971 };
972 QUniqueFileDescriptorHandle fd;
973 if (m_vulkan->vkGetMemoryFdKHR(m_vulkan->dev, &getFdInfo, &fd) != VK_SUCCESS) {
974 qWarning() << "CUDATextureConverter: vkGetMemoryFdKHR failed for plane" << i;
975 return false;
976 }
977
978 const CUDA_EXTERNAL_MEMORY_HANDLE_DESC extMemDesc{
979 .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
980 .handle = { .fd = fd.get() },
981 .size = memReq.size,
982 .flags = 0,
983 .reserved = {},
984 };
985 const CUresult importResult =
986 m_api->cuImportExternalMemory(&plane->externalMemory, &extMemDesc);
987 if (importResult != CUDA_SUCCESS) {
988 qWarning() << "CUDATextureConverter: cuImportExternalMemory failed for plane" << i
989 << ":" << CudaResult(importResult, *m_api);
990 return false;
991 }
992 std::ignore = fd.release(); // import succeeded, ownership transferred to CUDA
993
994 const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC arrayDesc{
995 .offset = 0,
996 .arrayDesc =
997 CUDA_ARRAY3D_DESCRIPTOR{
998 .Width = size_t(planeSize.width()),
999 .Height = size_t(planeSize.height()),
1000 .Depth = 0,
1001 .Format = format.cuArrayFormat,
1002 .NumChannels = unsigned(planeGeometry.numChannels),
1003 .Flags = 0,
1004 },
1005 .numLevels = 1,
1006 .reserved = {},
1007 };
1008
1009 const CUresult mappedArrayResult = m_api->cuExternalMemoryGetMappedMipmappedArray(
1010 &plane->mipmappedArray, plane->externalMemory, &arrayDesc);
1011 if (mappedArrayResult != CUDA_SUCCESS) {
1012 qWarning() << "CUDATextureConverter: cuExternalMemoryGetMappedMipmappedArray "
1013 "failed for plane"
1014 << i << ":" << CudaResult(mappedArrayResult, *m_api);
1015 return false;
1016 }
1017
1018 const CUresult arrayLevelResult =
1019 m_api->cuMipmappedArrayGetLevel(&plane->array, plane->mipmappedArray, 0);
1020 if (arrayLevelResult != CUDA_SUCCESS) {
1021 qWarning() << "CUDATextureConverter: cuMipmappedArrayGetLevel failed for plane" << i
1022 << ":" << CudaResult(arrayLevelResult, *m_api);
1023 return false;
1024 }
1025
1026 releaseGuard.dismiss();
1027 m_planes[i] = std::move(plane);
1028 }
1029
1030 return true;
1031 }
1032
1033 void releasePlaneResources()
1034 {
1035 if (!rhi)
1036 return;
1037
1038 std::optional<CudaContextGuard> guard = m_cudaContext && m_api
1039 ? CudaContextGuard::create(*m_api, m_cudaContext)
1040 : std::nullopt;
1041
1042 for (auto &plane : m_planes) {
1043 if (!plane)
1044 continue;
1045 plane->release(guard ? m_api : nullptr, *m_vulkan);
1046 plane.reset();
1047 }
1048 }
1049
1050 const CudaDriverApi *const m_api;
1051 const std::unique_ptr<CudaVulkanInterop> m_vulkan;
1052 MatchResult m_vulkanDeviceMatches = MatchResult::Unknown;
1053 PerPlaneArray<std::unique_ptr<VulkanPlaneResource>> m_planes;
1054 QSize m_planeResourcesSize;
1055 AVPixelFormat m_planeResourcesFormat = AV_PIX_FMT_NONE;
1056 CUcontext m_cudaContext = nullptr;
1057};
1058
1059# endif // QT_CONFIG(vulkan)
1060
1061} // namespace
1062
1063#endif // QT_FFMPEG_HAS_CUDA_TEXTURE_INTEROP
1064
1065// ---------------------------------------------------------------------------------------------
1066// Factory
1067// ---------------------------------------------------------------------------------------------
1068
1069std::shared_ptr<TextureConverterBackend> CUDATextureConverterFactory::create(QRhi *rhi)
1070{
1071 if (rhi) {
1072 switch (rhi->backend()) {
1073#if QT_FFMPEG_HAS_CUDA_GL_TEXTURE_INTEROP
1074 case QRhi::OpenGLES2:
1075 return CUDAGLTextureConverter::create(rhi);
1076#endif
1077#if QT_CONFIG(vulkan)
1078 case QRhi::Vulkan:
1079 return CUDAVulkanTextureConverter::create(rhi);
1080#endif
1081 default:
1082 break;
1083 }
1084 }
1085
1086 return nullptr;
1087}
1088
1089} // namespace QFFmpeg
1090
1091QT_END_NAMESPACE
Q_STATIC_LOGGING_CATEGORY
QT_BEGIN_NAMESPACE Q_STATIC_LOGGING_CATEGORY(lcSynthesizedIterableAccess, "qt.iterable.synthesized", QtWarningMsg)