5#include <private/qvideotexturehelper_p.h>
6#include <QtCore/qloggingcategory.h>
8#include <gst/video/video.h>
9#include <gst/video/video-frame.h>
10#include <gst/video/gstvideometa.h>
11#include <gst/pbutils/gstpluginsbaseversion.h>
13#include <common/qgstutils_p.h>
16#if QT_CONFIG(gstreamer_gl)
17# include <QtGui/rhi/qrhi.h>
18# include <QtGui/qopenglcontext.h>
19# include <QtGui/qopenglfunctions.h>
20# include <QtGui/qopengl.h>
22# include <gst/gl/gstglconfig.h>
23# include <gst/gl/gstglmemory.h>
24# include <gst/gl/gstglsyncmeta.h>
26# if QT_CONFIG(gstreamer_gl_egl)
28# include <EGL/eglext.h>
31# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
32# include <QtMultimedia/private/qmultimedia_drm_support_p.h>
34# include <common/qgstreameregldisplay_p.h>
35# include <gst/allocators/gstdmabuf.h>
41Q_STATIC_LOGGING_CATEGORY(qLcGstVideoBuffer,
"qt.multimedia.gstreamer.videobuffer");
44 const QVideoFrameFormat &frameFormat)
52 m_type = m_memoryFormat != QGstCaps::CpuMemory ? QVideoFrame::RhiTextureHandle
53 : QVideoFrame::NoHandle;
58 Q_ASSERT(m_mode == QVideoFrame::NotMapped);
63 const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0)
64 | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0));
67 if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped)
70 const GstVideoInfo &gstVideoInfo = m_videoInfo.gstVideoInfo;
71 if (!gstVideoInfo.finfo || gstVideoInfo.finfo->n_planes == 0) {
72 if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) {
73 mapData.planeCount = 1;
74 mapData.bytesPerLine[0] = -1;
75 mapData.dataSize[0] = m_frame.map[0].size;
76 mapData.data[0] =
static_cast<uchar *>(m_frame.map[0].data);
80 }
else if (gst_video_frame_map(&m_frame, &gstVideoInfo, m_buffer.get(), flags)) {
81 mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame);
83 for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) {
84 mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i);
85 mapData.data[i] =
static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i));
86 mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i);
96 if (m_mode != QVideoFrame::NotMapped) {
97 if (!m_videoInfo.gstVideoInfo.finfo || m_videoInfo.gstVideoInfo.finfo->n_planes == 0)
98 gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]);
100 gst_video_frame_unmap(&m_frame);
102 m_mode = QVideoFrame::NotMapped;
110#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
112using QtMultimediaPrivate::DRMFormat;
114static std::optional<DRMFormat>
115fourccFromGstVideoFormat(
const GstVideoFormat format,
int plane,
bool singleEGLImage)
117#if G_BYTE_ORDER == G_LITTLE_ENDIAN
118 constexpr DRMFormat argb_fourcc = DRMFormat::ARGB8888;
119 constexpr DRMFormat rgba_fourcc = DRMFormat::ABGR8888;
120 constexpr DRMFormat rgb_fourcc = DRMFormat::BGR888;
121 constexpr DRMFormat rg_fourcc = DRMFormat::GR88;
123 constexpr DRMFormat argb_fourcc = DRMFormat::BGRA8888;
124 constexpr DRMFormat rgba_fourcc = DRMFormat::RGBA8888;
125 constexpr DRMFormat rgb_fourcc = DRMFormat::RGB888;
126 constexpr DRMFormat rg_fourcc = DRMFormat::RG88;
130 case GST_VIDEO_FORMAT_RGB16:
131 case GST_VIDEO_FORMAT_BGR16:
132 return DRMFormat::RGB565;
134 case GST_VIDEO_FORMAT_RGB:
135 case GST_VIDEO_FORMAT_BGR:
138 case GST_VIDEO_FORMAT_BGRx:
139 case GST_VIDEO_FORMAT_BGRA:
142 case GST_VIDEO_FORMAT_AYUV:
143 if (singleEGLImage)
return DRMFormat::AYUV;
145 case GST_VIDEO_FORMAT_RGBx:
146 case GST_VIDEO_FORMAT_RGBA:
147 case GST_VIDEO_FORMAT_ARGB:
148 case GST_VIDEO_FORMAT_xRGB:
149 case GST_VIDEO_FORMAT_ABGR:
150 case GST_VIDEO_FORMAT_xBGR:
153 case GST_VIDEO_FORMAT_GRAY8:
154 return DRMFormat::R8;
156 case GST_VIDEO_FORMAT_YUY2:
157 return DRMFormat::YUYV;
159 case GST_VIDEO_FORMAT_UYVY:
160 return DRMFormat::UYVY;
162 case GST_VIDEO_FORMAT_GRAY16_LE:
163 case GST_VIDEO_FORMAT_GRAY16_BE:
164 if (singleEGLImage)
return DRMFormat::R16;
167 case GST_VIDEO_FORMAT_NV12:
168 if (singleEGLImage)
return DRMFormat::NV12;
170 case GST_VIDEO_FORMAT_NV21:
171 if (singleEGLImage)
return DRMFormat::NV21;
172 return plane == 0 ? DRMFormat::R8 : rg_fourcc;
174 case GST_VIDEO_FORMAT_I420:
175 if (singleEGLImage)
return DRMFormat::YUV420;
177 case GST_VIDEO_FORMAT_YV12:
178 if (singleEGLImage)
return DRMFormat::YVU420;
180 case GST_VIDEO_FORMAT_Y41B:
181 if (singleEGLImage)
return DRMFormat::YUV411;
183 case GST_VIDEO_FORMAT_Y42B:
184 if (singleEGLImage)
return DRMFormat::YUV422;
186 case GST_VIDEO_FORMAT_Y444:
187 if (singleEGLImage)
return DRMFormat::YUV444;
188 return DRMFormat::R8;
190 case GST_VIDEO_FORMAT_BGR10A2_LE:
191 return DRMFormat::BGRA1010102;
193 case GST_VIDEO_FORMAT_P010_10LE:
194 case GST_VIDEO_FORMAT_P010_10BE:
195 if (singleEGLImage)
return DRMFormat::P010;
196 return plane == 0 ? DRMFormat::R16 : DRMFormat::RG1616;
203static void logGlAndEglErrors(
const char *context)
205 if (!qLcGstVideoBuffer().isDebugEnabled())
208 const GLenum glError = glGetError();
209 const EGLint eglError = eglGetError();
210 if (glError == GL_NO_ERROR && eglError == EGL_SUCCESS)
213 qCDebug(qLcGstVideoBuffer).nospace()
214 << context <<
": GL error 0x" << Qt::hex << glError
215 <<
", EGL error 0x" << eglError;
219#if QT_CONFIG(gstreamer_gl)
224 std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{};
227class QGstQVideoFrameTextures :
public QVideoFrameTextures
230 QGstQVideoFrameTextures(QRhi *rhi,
232 QVideoFrameFormat::PixelFormat format,
233 GlTextures &textures,
234 QGstCaps::MemoryFormat memoryFormat)
236 , m_glTextures(textures)
238 QRhiTexture::Flags textureFlags = {};
239 if (QVideoTextureHelper::forceGlTextureExternalOesIsSet()
240 && m_rhi && rhi->backend() == QRhi::OpenGLES2)
241 textureFlags = {QRhiTexture::ExternalOES};
243 bool isDmaBuf = memoryFormat == QGstCaps::DMABuf;
244 auto fallbackPolicy = isDmaBuf
245 ? QVideoTextureHelper::TextureDescription::FallbackPolicy::Disable
246 : QVideoTextureHelper::TextureDescription::FallbackPolicy::Enable;
248 auto desc = QVideoTextureHelper::textureDescription(format);
249 for (uint i = 0; i < textures.count; ++i) {
251 QSize planeSize = desc->rhiPlaneSize(size, i, isDmaBuf ?
nullptr : m_rhi);
252 QRhiTexture::Format format = desc->rhiTextureFormat(i, m_rhi, fallbackPolicy);
253 m_textures[i].reset(rhi->newTexture(format, planeSize, 1, textureFlags));
254 m_textures[i]->createFrom({textures.names[i], 0});
258 ~QGstQVideoFrameTextures() override
260 m_rhi->makeThreadLocalNativeContextCurrent();
261 auto ctx = QOpenGLContext::currentContext();
262 if (m_glTextures.owned && ctx)
263 ctx->functions()->glDeleteTextures(
int(m_glTextures.count), m_glTextures.names.data());
266 QRhiTexture *texture(uint plane)
const override
268 return plane < m_glTextures.count ? m_textures[plane].get() :
nullptr;
272 QRhi *m_rhi =
nullptr;
273 GlTextures m_glTextures;
274 std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes];
277static GlTextures mapFromGlTexture(
const QGstBufferHandle &bufferHandle, GstVideoFrame &frame,
278 GstVideoInfo &videoInfo)
280 qCDebug(qLcGstVideoBuffer) <<
"Mapping textures from GL memory";
282 GstBuffer *buffer = bufferHandle.get();
283 auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0));
287 if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) {
288 qWarning() <<
"Could not map GL textures";
292 auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer);
293 GstBuffer *sync_buffer =
nullptr;
295 sync_buffer = gst_buffer_new();
296 sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer);
298 gst_gl_sync_meta_set_sync_point (sync_meta, mem->context);
299 gst_gl_sync_meta_wait (sync_meta, mem->context);
301 gst_buffer_unref(sync_buffer);
304 textures.count = frame.info.finfo->n_planes;
306 for (uint i = 0; i < textures.count; ++i)
307 textures.names[i] = *(guint32 *)frame.data[i];
309 gst_video_frame_unmap(&frame);
314# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
315static GlTextures mapFromDmaBuffer(QRhi *rhi,
const QGstBufferHandle &bufferHandle,
316 const QGstVideoInfo &videoInfo, Qt::HANDLE eglDisplay,
317 QFunctionPointer eglImageTargetTexture2D)
319 qCDebug(qLcGstVideoBuffer) <<
"Importing textures from DMA buffer";
320 logGlAndEglErrors(
"mapFromDmaBuffer");
322 GstBuffer *buffer = bufferHandle.get();
324 Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0)));
325 Q_ASSERT(eglDisplay);
326 Q_ASSERT(eglImageTargetTexture2D);
328 Q_ASSERT(rhi->backend() == QRhi::OpenGLES2);
330 auto *nativeHandles =
static_cast<
const QRhiGles2NativeHandles *>(rhi->nativeHandles());
331 auto glContext = nativeHandles->context;
333 qCWarning(qLcGstVideoBuffer) <<
"no GL context";
337 const GstVideoInfo &gstVideoInfo = videoInfo.gstVideoInfo;
338 if (!gstVideoInfo.finfo) {
339 qCWarning(qLcGstVideoBuffer) <<
"Missing valid GstVideoInfo for DMABuf GstBuffer";
343 if (videoInfo.dmaDrmModifier && *videoInfo.dmaDrmModifier != 0) {
344 qCWarning(qLcGstVideoBuffer) <<
"Unsupported non-linear DMABuf modifier:"
345 << Qt::hex << *videoInfo.dmaDrmModifier;
349 const GstVideoMeta *videoMeta = gst_buffer_get_video_meta(buffer);
350 const GstVideoFormat videoInfoFormat = GST_VIDEO_INFO_FORMAT(&gstVideoInfo);
351 GstVideoFormat format = videoMeta ? videoMeta->format : videoInfoFormat;
352 if (format == GST_VIDEO_FORMAT_UNKNOWN)
353 format = videoInfoFormat;
355 const int nPlanes = videoMeta ? videoMeta->n_planes : GST_VIDEO_INFO_N_PLANES(&gstVideoInfo);
356 const int nMemoryBlocks = gst_buffer_n_memory(buffer);
357 static const bool externalOes = QVideoTextureHelper::forceGlTextureExternalOesIsSet();
358 static const bool singleEGLImage =
359 externalOes || qEnvironmentVariableIntValue(
"QT_GSTREAMER_FORCE_SINGLE_EGLIMAGE") != 0;
361 qCDebug(qLcGstVideoBuffer) <<
"format:" << gst_video_format_to_string(format)
362 <<
"nPlanes:" << nPlanes
363 <<
"nMemoryBlocks:" << nMemoryBlocks
364 <<
"externalOes:" << externalOes
365 <<
"singleEGLImage:" << singleEGLImage;
367 constexpr int maxPlanes = 4;
368 Q_ASSERT(nPlanes >= 1
369 && nPlanes <= maxPlanes
370 && (nMemoryBlocks == 1 || nMemoryBlocks == nPlanes));
372 const int nEGLImages = singleEGLImage ? 1 : nPlanes;
373 std::array<EGLAttrib, maxPlanes> planeFourcc{};
374 for (
int plane = 0; plane < nEGLImages; ++plane) {
375 const std::optional<DRMFormat> fourcc = fourccFromGstVideoFormat(format, plane, singleEGLImage);
377 qCWarning(qLcGstVideoBuffer) <<
"Unsupported format for DMABuf:"
378 << gst_video_format_to_string(format) <<
"plane:" << plane
379 <<
"singleEGLImage" << singleEGLImage;
382 planeFourcc[plane] = EGLAttrib(quint32(*fourcc));
385 GlTextures textures = {};
386 textures.owned =
true;
387 textures.count = nEGLImages;
389 QOpenGLFunctions functions(glContext);
390 functions.glGenTextures(
int(textures.count), textures.names.data());
391 logGlAndEglErrors(
"glGenTextures");
393 std::array<
int, maxPlanes> fds{-1, -1, -1, -1};
394 for (
int i = 0; i < nMemoryBlocks && i < maxPlanes; ++i) {
395 fds[i] = gst_dmabuf_memory_get_fd(gst_buffer_peek_memory(buffer, i));
398 auto fdForPlane = [&](
int plane) -> EGLAttrib {
399 if (plane < 0 || plane >= maxPlanes || plane >= nMemoryBlocks)
401 return (fds[plane] >= 0) ? fds[plane] : fds[0];
404 auto compWidth = [&](
int plane) -> EGLAttrib {
405 return singleEGLImage ? GST_VIDEO_INFO_WIDTH(&gstVideoInfo)
406 : GST_VIDEO_INFO_COMP_WIDTH(&gstVideoInfo, plane);
409 auto compHeight = [&](
int plane) -> EGLAttrib {
410 return singleEGLImage ? GST_VIDEO_INFO_HEIGHT(&gstVideoInfo)
411 : GST_VIDEO_INFO_COMP_HEIGHT(&gstVideoInfo, plane);
414 auto planeOffset = [&](
int plane) -> EGLAttrib {
416 if (nPlanes == nMemoryBlocks)
419 return videoMeta->offset[plane];
420 return GST_VIDEO_INFO_PLANE_OFFSET(&gstVideoInfo, plane);
423 auto planeStride = [&](
int plane) -> EGLAttrib {
425 return videoMeta->stride[plane];
426 return GST_VIDEO_INFO_PLANE_STRIDE(&gstVideoInfo, plane);
429 for (
int plane = 0; plane < nEGLImages; ++plane) {
430 constexpr int maxAttrCount = 31;
431 std::array<EGLAttrib, maxAttrCount> attr;
434 const int width = compWidth(plane);
435 const int height = compHeight(plane);
437 attr[i++] = EGL_WIDTH;
439 attr[i++] = EGL_HEIGHT;
441 attr[i++] = EGL_LINUX_DRM_FOURCC_EXT;
442 attr[i++] = planeFourcc[plane];
444 attr[i++] = EGL_DMA_BUF_PLANE0_FD_EXT;
445 attr[i++] = fdForPlane(plane);
446 attr[i++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT;
447 attr[i++] = planeOffset(plane);
448 attr[i++] = EGL_DMA_BUF_PLANE0_PITCH_EXT;
449 attr[i++] = planeStride(plane);
451 if (singleEGLImage && nPlanes > 1) {
452 attr[i++] = EGL_DMA_BUF_PLANE1_FD_EXT;
453 attr[i++] = fdForPlane(1);
454 attr[i++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT;
455 attr[i++] = planeOffset(1);
456 attr[i++] = EGL_DMA_BUF_PLANE1_PITCH_EXT;
457 attr[i++] = planeStride(1);
460 if (singleEGLImage && nPlanes > 2) {
461 attr[i++] = EGL_DMA_BUF_PLANE2_FD_EXT;
462 attr[i++] = fdForPlane(2);
463 attr[i++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT;
464 attr[i++] = planeOffset(2);
465 attr[i++] = EGL_DMA_BUF_PLANE2_PITCH_EXT;
466 attr[i++] = planeStride(2);
469 if (singleEGLImage && nPlanes > 3) {
470 attr[i++] = EGL_DMA_BUF_PLANE3_FD_EXT;
471 attr[i++] = fdForPlane(3);
472 attr[i++] = EGL_DMA_BUF_PLANE3_OFFSET_EXT;
473 attr[i++] = planeOffset(3);
474 attr[i++] = EGL_DMA_BUF_PLANE3_PITCH_EXT;
475 attr[i++] = planeStride(3);
478 attr[i++] = EGL_NONE;
479 Q_ASSERT(i <= maxAttrCount);
481 EGLImage image = eglCreateImage(eglDisplay,
483 EGL_LINUX_DMA_BUF_EXT,
486 if (image == EGL_NO_IMAGE_KHR) {
487 qCWarning(qLcGstVideoBuffer) <<
"could not create EGL image for plane" << plane
488 <<
", EGL error 0x" << Qt::hex << eglGetError();
491 logGlAndEglErrors(
"eglCreateImage");
493 #ifdef GL_OES_EGL_image_external
494 GLenum target = externalOes ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
496 GLenum target = GL_TEXTURE_2D;
498 functions.glBindTexture(target, textures.names[plane]);
500 auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D;
501 EGLImageTargetTexture2D(target, image);
502 logGlAndEglErrors(
"glEGLImageTargetTexture2DOES");
504 eglDestroyImage(eglDisplay, image);
514#if QT_CONFIG(gstreamer_gl)
515 GlTextures textures = {};
516 if (m_memoryFormat == QGstCaps::GLTexture)
517 textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo.gstVideoInfo);
519# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
520 else if (m_memoryFormat == QGstCaps::DMABuf && qGstEglCanMapDmaBuf()
521 && rhi.backend() == QRhi::OpenGLES2)
522 textures = mapFromDmaBuffer(&rhi, m_buffer, m_videoInfo, qGstEglDisplay(),
523 qGstEglImageTargetTexture2D());
526 if (textures.count > 0)
527 return std::make_unique<QGstQVideoFrameTextures>(
528 &rhi, QSize{ m_videoInfo.gstVideoInfo.width, m_videoInfo.gstVideoInfo.height },
529 m_frameFormat.pixelFormat(), textures, m_memoryFormat);
QGstVideoBuffer(QGstBufferHandle buffer, const QGstVideoInfo &videoInfo, const QVideoFrameFormat &frameFormat)
~QGstVideoBuffer() override
bool isDmaBuf() const override
QVideoFrameTexturesUPtr mapTextures(QRhi &, QVideoFrameTexturesUPtr &) override
void unmap() override
Releases the memory mapped by the map() function.
MapData map(QVideoFrame::MapMode mode) override
Maps the planes of a video buffer to memory.
Combined button and popup list for selecting options.