Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qgstvideobuffer.cpp
Go to the documentation of this file.
1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include "qgstvideobuffer_p.h"
5#include "qgstreamervideosink_p.h"
6#include <private/qvideotexturehelper_p.h>
7#include <qpa/qplatformnativeinterface.h>
8#include <qguiapplication.h>
9#include <QtCore/qloggingcategory.h>
10
11#include <gst/video/video.h>
12#include <gst/video/video-frame.h>
13#include <gst/video/gstvideometa.h>
14#include <gst/pbutils/gstpluginsbaseversion.h>
15
16#include <common/qgstutils_p.h>
17
18#if QT_CONFIG(gstreamer_gl)
19# include <QtGui/rhi/qrhi.h>
20# include <QtGui/qopenglcontext.h>
21# include <QtGui/qopenglfunctions.h>
22# include <QtGui/qopengl.h>
23
24# include <gst/gl/gstglconfig.h>
25# include <gst/gl/gstglmemory.h>
26# include <gst/gl/gstglsyncmeta.h>
27
28# if QT_CONFIG(gstreamer_gl_egl)
29# include <EGL/egl.h>
30# include <EGL/eglext.h>
31# endif
32
33# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
34# include <gst/allocators/gstdmabuf.h>
35# include <drm_fourcc.h>
36# endif
37#endif
38
39QT_BEGIN_NAMESPACE
40
41Q_STATIC_LOGGING_CATEGORY(qLcGstVideoBuffer, "qt.multimedia.gstreamer.videobuffer");
42
43QGstVideoBuffer::QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info,
44 QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat,
45 QGstCaps::MemoryFormat memoryFormat)
46 : QHwVideoBuffer((sink && sink->rhi() && memoryFormat != QGstCaps::CpuMemory)
47 ? QVideoFrame::RhiTextureHandle
48 : QVideoFrame::NoHandle,
49 sink ? sink->rhi() : nullptr),
50 m_memoryFormat(memoryFormat),
51 m_frameFormat(frameFormat),
52 m_videoInfo(info),
53 m_buffer(std::move(buffer))
54{
55#if QT_CONFIG(gstreamer_gl_egl)
56 if (sink) {
57 eglDisplay = sink->eglDisplay();
58 eglImageTargetTexture2D = sink->eglImageTargetTexture2D();
59 }
60#endif
61 Q_UNUSED(m_memoryFormat);
62 Q_UNUSED(eglDisplay);
63 Q_UNUSED(eglImageTargetTexture2D);
64}
65
66QGstVideoBuffer::~QGstVideoBuffer()
67{
68 Q_ASSERT(m_mode == QVideoFrame::NotMapped);
69}
70
71QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode)
72{
73 const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0)
74 | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0));
75
76 MapData mapData;
77 if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped)
78 return mapData;
79
80 if (m_videoInfo.finfo->n_planes == 0) { // Encoded
81 if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) {
82 mapData.planeCount = 1;
83 mapData.bytesPerLine[0] = -1;
84 mapData.dataSize[0] = m_frame.map[0].size;
85 mapData.data[0] = static_cast<uchar *>(m_frame.map[0].data);
86
87 m_mode = mode;
88 }
89 } else if (gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer.get(), flags)) {
90 mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame);
91
92 for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) {
93 mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i);
94 mapData.data[i] = static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i));
95 mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i);
96 }
97
98 m_mode = mode;
99 }
100 return mapData;
101}
102
103void QGstVideoBuffer::unmap()
104{
105 if (m_mode != QVideoFrame::NotMapped) {
106 if (m_videoInfo.finfo->n_planes == 0)
107 gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]);
108 else
109 gst_video_frame_unmap(&m_frame);
110 }
111 m_mode = QVideoFrame::NotMapped;
112}
113
114bool QGstVideoBuffer::isDmaBuf() const
115{
116 return m_memoryFormat == QGstCaps::DMABuf;
117}
118
119#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
120
121static int
122fourccFromVideoInfo(const GstVideoInfo * info, int plane, bool singleEGLImage)
123{
124 GstVideoFormat format = GST_VIDEO_INFO_FORMAT (info);
125#if G_BYTE_ORDER == G_LITTLE_ENDIAN
126 const gint argb_fourcc = DRM_FORMAT_ARGB8888;
127 const gint rgba_fourcc = DRM_FORMAT_ABGR8888;
128 const gint rgb_fourcc = DRM_FORMAT_BGR888;
129 const gint rg_fourcc = DRM_FORMAT_GR88;
130#else
131 const gint argb_fourcc = DRM_FORMAT_BGRA8888;
132 const gint rgba_fourcc = DRM_FORMAT_RGBA8888;
133 const gint rgb_fourcc = DRM_FORMAT_RGB888;
134 const gint rg_fourcc = DRM_FORMAT_RG88;
135#endif
136
137 qCDebug(qLcGstVideoBuffer) << "Getting DRM fourcc for"
138 << gst_video_format_to_string(format)
139 << "plane" << plane;
140
141 switch (format) {
142 case GST_VIDEO_FORMAT_RGB16:
143 case GST_VIDEO_FORMAT_BGR16:
144 return DRM_FORMAT_RGB565;
145
146 case GST_VIDEO_FORMAT_RGB:
147 case GST_VIDEO_FORMAT_BGR:
148 return rgb_fourcc;
149
150 case GST_VIDEO_FORMAT_BGRx:
151 case GST_VIDEO_FORMAT_BGRA:
152 return argb_fourcc;
153
154 case GST_VIDEO_FORMAT_AYUV:
155 if (singleEGLImage) return DRM_FORMAT_AYUV;
156 [[fallthrough]];
157 case GST_VIDEO_FORMAT_RGBx:
158 case GST_VIDEO_FORMAT_RGBA:
159 case GST_VIDEO_FORMAT_ARGB:
160 case GST_VIDEO_FORMAT_xRGB:
161 case GST_VIDEO_FORMAT_ABGR:
162 case GST_VIDEO_FORMAT_xBGR:
163 return rgba_fourcc;
164
165 case GST_VIDEO_FORMAT_GRAY8:
166 return DRM_FORMAT_R8;
167
168 case GST_VIDEO_FORMAT_YUY2:
169 return DRM_FORMAT_YUYV;
170
171 case GST_VIDEO_FORMAT_UYVY:
172 return DRM_FORMAT_UYVY;
173
174 case GST_VIDEO_FORMAT_GRAY16_LE:
175 case GST_VIDEO_FORMAT_GRAY16_BE:
176 if (singleEGLImage) return DRM_FORMAT_R16;
177 return rg_fourcc;
178
179 case GST_VIDEO_FORMAT_NV12:
180 if (singleEGLImage) return DRM_FORMAT_NV12;
181 [[fallthrough]];
182 case GST_VIDEO_FORMAT_NV21:
183 if (singleEGLImage) return DRM_FORMAT_NV21;
184 return plane == 0 ? DRM_FORMAT_R8 : rg_fourcc;
185
186 case GST_VIDEO_FORMAT_I420:
187 if (singleEGLImage) return DRM_FORMAT_YUV420;
188 [[fallthrough]];
189 case GST_VIDEO_FORMAT_YV12:
190 if (singleEGLImage) return DRM_FORMAT_YVU420;
191 [[fallthrough]];
192 case GST_VIDEO_FORMAT_Y41B:
193 if (singleEGLImage) return DRM_FORMAT_YUV411;
194 [[fallthrough]];
195 case GST_VIDEO_FORMAT_Y42B:
196 if (singleEGLImage) return DRM_FORMAT_YUV422;
197 [[fallthrough]];
198 case GST_VIDEO_FORMAT_Y444:
199 if (singleEGLImage) return DRM_FORMAT_YUV444;
200 return DRM_FORMAT_R8;
201
202#if GST_CHECK_PLUGINS_BASE_VERSION(1,16,0)
203 case GST_VIDEO_FORMAT_BGR10A2_LE:
204 return DRM_FORMAT_BGRA1010102;
205#endif
206
207 case GST_VIDEO_FORMAT_P010_10LE:
208 case GST_VIDEO_FORMAT_P010_10BE:
209 if (singleEGLImage) return DRM_FORMAT_P010;
210 return plane == 0 ? DRM_FORMAT_R16 : DRM_FORMAT_RG1616;
211
212 default:
213 qWarning() << "Unsupported format for DMABuf:" << gst_video_format_to_string(format);
214 return -1;
215 }
216}
217#endif
218
219#if QT_CONFIG(gstreamer_gl)
220struct GlTextures
221{
222 uint count = 0;
223 bool owned = false;
224 std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{};
225};
226
227class QGstQVideoFrameTextures : public QVideoFrameTextures
228{
229public:
230 QGstQVideoFrameTextures(QRhi *rhi,
231 QSize size,
232 QVideoFrameFormat::PixelFormat format,
233 GlTextures &textures,
234 QGstCaps::MemoryFormat memoryFormat)
235 : m_rhi(rhi)
236 , m_glTextures(textures)
237 {
238 QRhiTexture::Flags textureFlags = {};
239 if (QVideoTextureHelper::forceGlTextureExternalOesIsSet()
240 && m_rhi && rhi->backend() == QRhi::OpenGLES2)
241 textureFlags = {QRhiTexture::ExternalOES};
242
243 bool isDmaBuf = memoryFormat == QGstCaps::DMABuf;
244 auto fallbackPolicy = isDmaBuf
245 ? QVideoTextureHelper::TextureDescription::FallbackPolicy::Disable
246 : QVideoTextureHelper::TextureDescription::FallbackPolicy::Enable;
247
248 auto desc = QVideoTextureHelper::textureDescription(format);
249 for (uint i = 0; i < textures.count; ++i) {
250 // Pass nullptr to rhiPlaneSize to disable fallback in its call to rhiTextureFormat
251 QSize planeSize = desc->rhiPlaneSize(size, i, isDmaBuf ? nullptr : m_rhi);
252 QRhiTexture::Format format = desc->rhiTextureFormat(i, m_rhi, fallbackPolicy);
253 m_textures[i].reset(rhi->newTexture(format, planeSize, 1, textureFlags));
254 m_textures[i]->createFrom({textures.names[i], 0});
255 }
256 }
257
258 ~QGstQVideoFrameTextures() override
259 {
260 m_rhi->makeThreadLocalNativeContextCurrent();
261 auto ctx = QOpenGLContext::currentContext();
262 if (m_glTextures.owned && ctx)
263 ctx->functions()->glDeleteTextures(int(m_glTextures.count), m_glTextures.names.data());
264 }
265
266 QRhiTexture *texture(uint plane) const override
267 {
268 return plane < m_glTextures.count ? m_textures[plane].get() : nullptr;
269 }
270
271private:
272 QRhi *m_rhi = nullptr;
273 GlTextures m_glTextures;
274 std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes];
275};
276
277static GlTextures mapFromGlTexture(const QGstBufferHandle &bufferHandle, GstVideoFrame &frame,
278 GstVideoInfo &videoInfo)
279{
280 qCDebug(qLcGstVideoBuffer) << "mapFromGlTexture";
281
282 GstBuffer *buffer = bufferHandle.get();
283 auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0));
284 if (!mem)
285 return {};
286
287 if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) {
288 qWarning() << "Could not map GL textures";
289 return {};
290 }
291
292 auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer);
293 GstBuffer *sync_buffer = nullptr;
294 if (!sync_meta) {
295 sync_buffer = gst_buffer_new();
296 sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer);
297 }
298 gst_gl_sync_meta_set_sync_point (sync_meta, mem->context);
299 gst_gl_sync_meta_wait (sync_meta, mem->context);
300 if (sync_buffer)
301 gst_buffer_unref(sync_buffer);
302
303 GlTextures textures;
304 textures.count = frame.info.finfo->n_planes;
305
306 for (uint i = 0; i < textures.count; ++i)
307 textures.names[i] = *(guint32 *)frame.data[i];
308
309 gst_video_frame_unmap(&frame);
310
311 return textures;
312}
313
314# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
315static GlTextures mapFromDmaBuffer(QRhi *rhi, const QGstBufferHandle &bufferHandle,
316 GstVideoFrame &frame, GstVideoInfo &videoInfo,
317 Qt::HANDLE eglDisplay, QFunctionPointer eglImageTargetTexture2D)
318{
319 qCDebug(qLcGstVideoBuffer) << "mapFromDmaBuffer, glGetError returns" << Qt::hex << glGetError()
320 << ", eglGetError() returns" << eglGetError();
321
322 GstBuffer *buffer = bufferHandle.get();
323
324 Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0)));
325 Q_ASSERT(eglDisplay);
326 Q_ASSERT(eglImageTargetTexture2D);
327
328 auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles());
329 auto glContext = nativeHandles->context;
330 if (!glContext) {
331 qWarning() << "no GL context";
332 return {};
333 }
334
335 if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ))) {
336 qWarning() << "gst_video_frame_map failed, couldn't map DMA video frame";
337 return {};
338 }
339
340 constexpr int maxPlanes = 4;
341 const int nPlanes = GST_VIDEO_FRAME_N_PLANES(&frame);
342 const int nMemoryBlocks = gst_buffer_n_memory(buffer);
343 const bool externalOes =
344 QVideoTextureHelper::forceGlTextureExternalOesIsSet();
345 static const bool singleEGLImage =
346 externalOes || qEnvironmentVariableIsSet("QT_GSTREAMER_FORCE_SINGLE_EGLIMAGE");
347
348 qCDebug(qLcGstVideoBuffer) << "nPlanes:" << nPlanes
349 << "nMemoryBlocks:" << nMemoryBlocks
350 << "externalOes:" << externalOes
351 << "singleEGLImage:" << singleEGLImage;
352 Q_ASSERT(nPlanes >= 1
353 && nPlanes <= maxPlanes
354 && (nMemoryBlocks == 1 || nMemoryBlocks == nPlanes));
355
356 GlTextures textures = {};
357 textures.owned = true;
358 textures.count = singleEGLImage ? 1 : nPlanes;
359
360 QOpenGLFunctions functions(glContext);
361 functions.glGenTextures(int(textures.count), textures.names.data());
362 qCDebug(qLcGstVideoBuffer) << "called glGenTextures, glGetError returns"
363 << Qt::hex << glGetError();
364
365 std::array<int, maxPlanes> fds{-1, -1, -1, -1};
366 for (int i = 0; i < nMemoryBlocks && i < maxPlanes; ++i) {
367 fds[i] = gst_dmabuf_memory_get_fd(gst_buffer_peek_memory(buffer, i));
368 }
369 auto fdForPlane = [&](int plane) {
370 if (plane < 0 || plane >= maxPlanes || plane >= nMemoryBlocks)
371 return fds[0];
372 return (fds[plane] >= 0) ? fds[plane] : fds[0];
373 };
374
375 int nEGLImages = singleEGLImage ? 1 : nPlanes;
376 for (int plane = 0; plane < nEGLImages; ++plane) {
377 constexpr int maxAttrCount = 31;
378 std::array<EGLAttrib, maxAttrCount> attr;
379 int i = 0;
380
381 gint width = singleEGLImage ? GST_VIDEO_FRAME_WIDTH(&frame)
382 : GST_VIDEO_FRAME_COMP_WIDTH(&frame, plane);
383 gint height = singleEGLImage ? GST_VIDEO_FRAME_HEIGHT(&frame)
384 : GST_VIDEO_FRAME_COMP_HEIGHT(&frame, plane);
385 attr[i++] = EGL_WIDTH;
386 attr[i++] = width;
387 attr[i++] = EGL_HEIGHT;
388 attr[i++] = height;
389 attr[i++] = EGL_LINUX_DRM_FOURCC_EXT;
390 attr[i++] = fourccFromVideoInfo(&videoInfo, plane, singleEGLImage);
391
392 attr[i++] = EGL_DMA_BUF_PLANE0_FD_EXT;
393 attr[i++] = fdForPlane(plane);
394 attr[i++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT;
395 attr[i++] = (EGLAttrib)(GST_VIDEO_FRAME_PLANE_OFFSET(&frame, plane));
396 attr[i++] = EGL_DMA_BUF_PLANE0_PITCH_EXT;
397 attr[i++] = (EGLAttrib)(GST_VIDEO_FRAME_PLANE_STRIDE(&frame, plane));
398
399 if (singleEGLImage && nPlanes > 1) {
400 attr[i++] = EGL_DMA_BUF_PLANE1_FD_EXT;
401 attr[i++] = fdForPlane(1);
402 attr[i++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT;
403 attr[i++] = (EGLAttrib)(GST_VIDEO_FRAME_PLANE_OFFSET(&frame, 1));
404 attr[i++] = EGL_DMA_BUF_PLANE1_PITCH_EXT;
405 attr[i++] = (EGLAttrib)(GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 1));
406 }
407
408 if (singleEGLImage && nPlanes > 2) {
409 attr[i++] = EGL_DMA_BUF_PLANE2_FD_EXT;
410 attr[i++] = fdForPlane(2);
411 attr[i++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT;
412 attr[i++] = (EGLAttrib)(GST_VIDEO_FRAME_PLANE_OFFSET(&frame, 2));
413 attr[i++] = EGL_DMA_BUF_PLANE2_PITCH_EXT;
414 attr[i++] = (EGLAttrib)(GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 2));
415 }
416
417 if (singleEGLImage && nPlanes > 3) {
418 attr[i++] = EGL_DMA_BUF_PLANE3_FD_EXT;
419 attr[i++] = fdForPlane(3);
420 attr[i++] = EGL_DMA_BUF_PLANE3_OFFSET_EXT;
421 attr[i++] = (EGLAttrib)(GST_VIDEO_FRAME_PLANE_OFFSET(&frame, 3));
422 attr[i++] = EGL_DMA_BUF_PLANE3_PITCH_EXT;
423 attr[i++] = (EGLAttrib)(GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 3));
424 }
425
426 attr[i++] = EGL_NONE;
427 Q_ASSERT(i <= maxAttrCount);
428
429 EGLImage image = eglCreateImage(eglDisplay,
430 EGL_NO_CONTEXT,
431 EGL_LINUX_DMA_BUF_EXT,
432 nullptr,
433 attr.data());
434 if (image == EGL_NO_IMAGE_KHR) {
435 qWarning() << "could not create EGL image for plane" << plane
436 << ", eglError"<< Qt::hex << eglGetError();
437 continue;
438 }
439 qCDebug(qLcGstVideoBuffer) << "called eglCreateImage, glGetError returns"
440 << Qt::hex << glGetError()
441 << ", eglGetError() returns" << eglGetError();
442
443 #ifdef GL_OES_EGL_image_external
444 GLenum target = externalOes ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
445 #else
446 GLenum target = GL_TEXTURE_2D;
447 #endif
448 functions.glBindTexture(target, textures.names[plane]);
449
450 auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D;
451 EGLImageTargetTexture2D(target, image);
452 qCDebug(qLcGstVideoBuffer) << "called glEGLImageTargetTexture2DOES, glGetError returns"
453 << Qt::hex << glGetError()
454 << ", eglGetError() returns" << eglGetError();
455
456 eglDestroyImage(eglDisplay, image);
457 }
458
459 gst_video_frame_unmap(&frame);
460
461 return textures;
462}
463#endif
464#endif
465
466QVideoFrameTexturesUPtr QGstVideoBuffer::mapTextures(QRhi &rhi, QVideoFrameTexturesUPtr& /*oldTextures*/)
467{
468#if QT_CONFIG(gstreamer_gl)
469 GlTextures textures = {};
470 if (m_memoryFormat == QGstCaps::GLTexture)
471 textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo);
472
473# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
474 else if (m_memoryFormat == QGstCaps::DMABuf && eglDisplay)
475 textures = mapFromDmaBuffer(&rhi, m_buffer, m_frame, m_videoInfo, eglDisplay,
476 eglImageTargetTexture2D);
477
478# endif
479 if (textures.count > 0)
480 return std::make_unique<QGstQVideoFrameTextures>(&rhi, QSize{m_videoInfo.width, m_videoInfo.height},
481 m_frameFormat.pixelFormat(), textures, m_memoryFormat);
482#endif
483 return {};
484}
485
486QT_END_NAMESPACE
QGstCaps::MemoryFormat
MemoryFormat
Definition qgst_p.h:343
QGstCaps::DMABuf
@ DMABuf
Definition qgst_p.h:343
QGstVideoBuffer::~QGstVideoBuffer
~QGstVideoBuffer() override
Definition qgstvideobuffer.cpp:66
QGstVideoBuffer::isDmaBuf
bool isDmaBuf() const override
Definition qgstvideobuffer.cpp:114
QGstVideoBuffer::mapTextures
QVideoFrameTexturesUPtr mapTextures(QRhi &, QVideoFrameTexturesUPtr &) override
Definition qgstvideobuffer.cpp:466
QGstVideoBuffer::unmap
void unmap() override
Releases the memory mapped by the map() function.
Definition qgstvideobuffer.cpp:103
QGstVideoBuffer::QGstVideoBuffer
QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat, QGstCaps::MemoryFormat format)
Definition qgstvideobuffer.cpp:43
QGstVideoBuffer::map
MapData map(QVideoFrame::MapMode mode) override
Maps the planes of a video buffer to memory.
Definition qgstvideobuffer.cpp:71