5#include <private/qvideotexturehelper_p.h>
6#include <QtCore/qloggingcategory.h>
8#include <gst/video/video.h>
9#include <gst/video/video-frame.h>
10#include <gst/video/gstvideometa.h>
11#include <gst/pbutils/gstpluginsbaseversion.h>
13#include <common/qgstutils_p.h>
16#if QT_CONFIG(gstreamer_gl)
17# include <QtGui/rhi/qrhi.h>
18# include <QtGui/qopenglcontext.h>
19# include <QtGui/qopenglfunctions.h>
20# include <QtGui/qopengl.h>
22# include <gst/gl/gstglconfig.h>
23# include <gst/gl/gstglmemory.h>
24# include <gst/gl/gstglsyncmeta.h>
26# if QT_CONFIG(gstreamer_gl_egl)
28# include <EGL/eglext.h>
31# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
32# include <QtMultimedia/private/qdmabuftextureimporter_p.h>
33# include <QtMultimedia/private/qmultimedia_drm_support_p.h>
35# include <common/qgstreameregldisplay_p.h>
36# include <gst/allocators/gstdmabuf.h>
42Q_STATIC_LOGGING_CATEGORY(qLcGstVideoBuffer,
"qt.multimedia.gstreamer.videobuffer");
45 const QVideoFrameFormat &frameFormat)
53 m_type = m_memoryFormat != QGstCaps::CpuMemory ? QVideoFrame::RhiTextureHandle
54 : QVideoFrame::NoHandle;
59 Q_ASSERT(m_mode == QVideoFrame::NotMapped);
64 const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0)
65 | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0));
68 if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped)
71 const GstVideoInfo &gstVideoInfo = m_videoInfo.gstVideoInfo;
72 if (!gstVideoInfo.finfo || gstVideoInfo.finfo->n_planes == 0) {
73 if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) {
74 mapData.planeCount = 1;
75 mapData.bytesPerLine[0] = -1;
76 mapData.dataSize[0] = m_frame.map[0].size;
77 mapData.data[0] =
static_cast<uchar *>(m_frame.map[0].data);
81 }
else if (gst_video_frame_map(&m_frame, &gstVideoInfo, m_buffer.get(), flags)) {
82 mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame);
84 for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) {
85 mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i);
86 mapData.data[i] =
static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i));
87 mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i);
97 if (m_mode != QVideoFrame::NotMapped) {
98 if (!m_videoInfo.gstVideoInfo.finfo || m_videoInfo.gstVideoInfo.finfo->n_planes == 0)
99 gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]);
101 gst_video_frame_unmap(&m_frame);
103 m_mode = QVideoFrame::NotMapped;
111#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
113using QtMultimediaPrivate::DRMFormat;
115static std::optional<DRMFormat>
116fourccFromGstVideoFormat(
const GstVideoFormat format,
int plane,
bool singleEGLImage)
118#if G_BYTE_ORDER == G_LITTLE_ENDIAN
119 constexpr DRMFormat argb_fourcc = DRMFormat::ARGB8888;
120 constexpr DRMFormat rgba_fourcc = DRMFormat::ABGR8888;
121 constexpr DRMFormat rgb_fourcc = DRMFormat::BGR888;
122 constexpr DRMFormat rg_fourcc = DRMFormat::GR88;
124 constexpr DRMFormat argb_fourcc = DRMFormat::BGRA8888;
125 constexpr DRMFormat rgba_fourcc = DRMFormat::RGBA8888;
126 constexpr DRMFormat rgb_fourcc = DRMFormat::RGB888;
127 constexpr DRMFormat rg_fourcc = DRMFormat::RG88;
131 case GST_VIDEO_FORMAT_RGB16:
132 case GST_VIDEO_FORMAT_BGR16:
133 return DRMFormat::RGB565;
135 case GST_VIDEO_FORMAT_RGB:
136 case GST_VIDEO_FORMAT_BGR:
139 case GST_VIDEO_FORMAT_BGRx:
140 case GST_VIDEO_FORMAT_BGRA:
143 case GST_VIDEO_FORMAT_AYUV:
144 if (singleEGLImage)
return DRMFormat::AYUV;
146 case GST_VIDEO_FORMAT_RGBx:
147 case GST_VIDEO_FORMAT_RGBA:
148 case GST_VIDEO_FORMAT_ARGB:
149 case GST_VIDEO_FORMAT_xRGB:
150 case GST_VIDEO_FORMAT_ABGR:
151 case GST_VIDEO_FORMAT_xBGR:
154 case GST_VIDEO_FORMAT_GRAY8:
155 return DRMFormat::R8;
157 case GST_VIDEO_FORMAT_YUY2:
158 return DRMFormat::YUYV;
160 case GST_VIDEO_FORMAT_UYVY:
161 return DRMFormat::UYVY;
163 case GST_VIDEO_FORMAT_GRAY16_LE:
164 case GST_VIDEO_FORMAT_GRAY16_BE:
165 if (singleEGLImage)
return DRMFormat::R16;
168 case GST_VIDEO_FORMAT_NV12:
169 if (singleEGLImage)
return DRMFormat::NV12;
171 case GST_VIDEO_FORMAT_NV21:
172 if (singleEGLImage)
return DRMFormat::NV21;
173 return plane == 0 ? DRMFormat::R8 : rg_fourcc;
175 case GST_VIDEO_FORMAT_I420:
176 if (singleEGLImage)
return DRMFormat::YUV420;
178 case GST_VIDEO_FORMAT_YV12:
179 if (singleEGLImage)
return DRMFormat::YVU420;
181 case GST_VIDEO_FORMAT_Y41B:
182 if (singleEGLImage)
return DRMFormat::YUV411;
184 case GST_VIDEO_FORMAT_Y42B:
185 if (singleEGLImage)
return DRMFormat::YUV422;
187 case GST_VIDEO_FORMAT_Y444:
188 if (singleEGLImage)
return DRMFormat::YUV444;
189 return DRMFormat::R8;
191 case GST_VIDEO_FORMAT_BGR10A2_LE:
192 return DRMFormat::BGRA1010102;
194 case GST_VIDEO_FORMAT_P010_10LE:
195 case GST_VIDEO_FORMAT_P010_10BE:
196 if (singleEGLImage)
return DRMFormat::P010;
197 return plane == 0 ? DRMFormat::R16 : DRMFormat::RG1616;
204static void logGlAndEglErrors(
const char *context)
206 if (!qLcGstVideoBuffer().isDebugEnabled())
209 const GLenum glError = glGetError();
210 const EGLint eglError = eglGetError();
211 if (glError == GL_NO_ERROR && eglError == EGL_SUCCESS)
214 qCDebug(qLcGstVideoBuffer).nospace()
215 << context <<
": GL error 0x" << Qt::hex << glError
216 <<
", EGL error 0x" << eglError;
220#if QT_CONFIG(gstreamer_gl)
225 std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{};
228class QGstQVideoFrameTextures :
public QVideoFrameTextures
231 QGstQVideoFrameTextures(QRhi *rhi,
233 QVideoFrameFormat::PixelFormat format,
234 GlTextures &textures,
235 QGstCaps::MemoryFormat memoryFormat)
237 , m_glTextures(textures)
239 QRhiTexture::Flags textureFlags = {};
240 if (QVideoTextureHelper::forceGlTextureExternalOesIsSet()
241 && m_rhi && rhi->backend() == QRhi::OpenGLES2)
242 textureFlags = {QRhiTexture::ExternalOES};
244 bool isDmaBuf = memoryFormat == QGstCaps::DMABuf;
245 auto fallbackPolicy = isDmaBuf
246 ? QVideoTextureHelper::TextureDescription::FallbackPolicy::Disable
247 : QVideoTextureHelper::TextureDescription::FallbackPolicy::Enable;
249 auto desc = QVideoTextureHelper::textureDescription(format);
250 for (uint i = 0; i < textures.count; ++i) {
252 QSize planeSize = desc->rhiPlaneSize(size, i, isDmaBuf ?
nullptr : m_rhi);
253 QRhiTexture::Format format = desc->rhiTextureFormat(i, m_rhi, fallbackPolicy);
254 m_textures[i].reset(rhi->newTexture(format, planeSize, 1, textureFlags));
255 m_textures[i]->createFrom({textures.names[i], 0});
259 ~QGstQVideoFrameTextures() override
261 m_rhi->makeThreadLocalNativeContextCurrent();
262 auto ctx = QOpenGLContext::currentContext();
263 if (m_glTextures.owned && ctx)
264 ctx->functions()->glDeleteTextures(
int(m_glTextures.count), m_glTextures.names.data());
267 QRhiTexture *texture(uint plane)
const override
269 return plane < m_glTextures.count ? m_textures[plane].get() :
nullptr;
273 QRhi *m_rhi =
nullptr;
274 GlTextures m_glTextures;
275 std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes];
278static GlTextures mapFromGlTexture(
const QGstBufferHandle &bufferHandle, GstVideoFrame &frame,
279 GstVideoInfo &videoInfo)
281 qCDebug(qLcGstVideoBuffer) <<
"Mapping textures from GL memory";
283 GstBuffer *buffer = bufferHandle.get();
284 auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0));
288 if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) {
289 qWarning() <<
"Could not map GL textures";
293 auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer);
294 GstBuffer *sync_buffer =
nullptr;
296 sync_buffer = gst_buffer_new();
297 sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer);
299 gst_gl_sync_meta_set_sync_point (sync_meta, mem->context);
300 gst_gl_sync_meta_wait (sync_meta, mem->context);
302 gst_buffer_unref(sync_buffer);
305 textures.count = frame.info.finfo->n_planes;
307 for (uint i = 0; i < textures.count; ++i)
308 textures.names[i] = *(guint32 *)frame.data[i];
310 gst_video_frame_unmap(&frame);
315# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
316static GlTextures mapFromDmaBuffer(QRhi *rhi,
const QGstBufferHandle &bufferHandle,
317 const QGstVideoInfo &videoInfo, Qt::HANDLE eglDisplay)
319 qCDebug(qLcGstVideoBuffer) <<
"Importing textures from DMA buffer";
320 logGlAndEglErrors(
"mapFromDmaBuffer");
322 GstBuffer *buffer = bufferHandle.get();
324 Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0)));
325 Q_ASSERT(eglDisplay);
327 Q_ASSERT(rhi->backend() == QRhi::OpenGLES2);
329 auto *nativeHandles =
static_cast<
const QRhiGles2NativeHandles *>(rhi->nativeHandles());
330 auto glContext = nativeHandles->context;
332 qCWarning(qLcGstVideoBuffer) <<
"no GL context";
336 const GstVideoInfo &gstVideoInfo = videoInfo.gstVideoInfo;
337 if (!gstVideoInfo.finfo) {
338 qCWarning(qLcGstVideoBuffer) <<
"Missing valid GstVideoInfo for DMABuf GstBuffer";
342 if (videoInfo.dmaDrmModifier && *videoInfo.dmaDrmModifier != 0) {
343 qCWarning(qLcGstVideoBuffer) <<
"Unsupported non-linear DMABuf modifier:"
344 << Qt::hex << *videoInfo.dmaDrmModifier;
348 const GstVideoMeta *videoMeta = gst_buffer_get_video_meta(buffer);
349 const GstVideoFormat videoInfoFormat = GST_VIDEO_INFO_FORMAT(&gstVideoInfo);
350 GstVideoFormat format = videoMeta ? videoMeta->format : videoInfoFormat;
351 if (format == GST_VIDEO_FORMAT_UNKNOWN)
352 format = videoInfoFormat;
354 const int nPlanes = videoMeta ? videoMeta->n_planes : GST_VIDEO_INFO_N_PLANES(&gstVideoInfo);
355 const int nMemoryBlocks = gst_buffer_n_memory(buffer);
356 static const bool externalOes = QVideoTextureHelper::forceGlTextureExternalOesIsSet();
357 static const bool singleEGLImage =
358 externalOes || qEnvironmentVariableIntValue(
"QT_GSTREAMER_FORCE_SINGLE_EGLIMAGE") != 0;
360 qCDebug(qLcGstVideoBuffer) <<
"format:" << gst_video_format_to_string(format)
361 <<
"nPlanes:" << nPlanes
362 <<
"nMemoryBlocks:" << nMemoryBlocks
363 <<
"externalOes:" << externalOes
364 <<
"singleEGLImage:" << singleEGLImage;
366 constexpr int maxPlanes = 4;
367 Q_ASSERT(nPlanes >= 1
368 && nPlanes <= maxPlanes
369 && (nMemoryBlocks == 1 || nMemoryBlocks == nPlanes));
371 const int nEGLImages = singleEGLImage ? 1 : nPlanes;
372 std::array<EGLAttrib, maxPlanes> planeFourcc{};
373 for (
int plane = 0; plane < nEGLImages; ++plane) {
374 const std::optional<DRMFormat> fourcc = fourccFromGstVideoFormat(format, plane, singleEGLImage);
376 qCWarning(qLcGstVideoBuffer) <<
"Unsupported format for DMABuf:"
377 << gst_video_format_to_string(format) <<
"plane:" << plane
378 <<
"singleEGLImage" << singleEGLImage;
381 planeFourcc[plane] = EGLAttrib(quint32(*fourcc));
384 GlTextures textures = {};
385 textures.owned =
true;
386 textures.count = nEGLImages;
388 QOpenGLFunctions functions(glContext);
389 functions.glGenTextures(
int(textures.count), textures.names.data());
390 logGlAndEglErrors(
"glGenTextures");
392 std::array<
int, maxPlanes> fds{-1, -1, -1, -1};
393 for (
int i = 0; i < nMemoryBlocks && i < maxPlanes; ++i) {
394 fds[i] = gst_dmabuf_memory_get_fd(gst_buffer_peek_memory(buffer, i));
397 auto fdForPlane = [&](
int plane) -> EGLAttrib {
398 if (plane < 0 || plane >= maxPlanes || plane >= nMemoryBlocks)
400 return (fds[plane] >= 0) ? fds[plane] : fds[0];
403 auto compWidth = [&](
int plane) -> EGLAttrib {
404 return singleEGLImage ? GST_VIDEO_INFO_WIDTH(&gstVideoInfo)
405 : GST_VIDEO_INFO_COMP_WIDTH(&gstVideoInfo, plane);
408 auto compHeight = [&](
int plane) -> EGLAttrib {
409 return singleEGLImage ? GST_VIDEO_INFO_HEIGHT(&gstVideoInfo)
410 : GST_VIDEO_INFO_COMP_HEIGHT(&gstVideoInfo, plane);
413 auto planeOffset = [&](
int plane) -> EGLAttrib {
415 if (nPlanes == nMemoryBlocks)
418 return videoMeta->offset[plane];
419 return GST_VIDEO_INFO_PLANE_OFFSET(&gstVideoInfo, plane);
422 auto planeStride = [&](
int plane) -> EGLAttrib {
424 return videoMeta->stride[plane];
425 return GST_VIDEO_INFO_PLANE_STRIDE(&gstVideoInfo, plane);
428 for (
int plane = 0; plane < nEGLImages; ++plane) {
429 constexpr int maxAttrCount = 31;
430 std::array<EGLAttrib, maxAttrCount> attr;
433 const int width = compWidth(plane);
434 const int height = compHeight(plane);
436 attr[i++] = EGL_WIDTH;
438 attr[i++] = EGL_HEIGHT;
440 attr[i++] = EGL_LINUX_DRM_FOURCC_EXT;
441 attr[i++] = planeFourcc[plane];
443 attr[i++] = EGL_DMA_BUF_PLANE0_FD_EXT;
444 attr[i++] = fdForPlane(plane);
445 attr[i++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT;
446 attr[i++] = planeOffset(plane);
447 attr[i++] = EGL_DMA_BUF_PLANE0_PITCH_EXT;
448 attr[i++] = planeStride(plane);
450 if (singleEGLImage && nPlanes > 1) {
451 attr[i++] = EGL_DMA_BUF_PLANE1_FD_EXT;
452 attr[i++] = fdForPlane(1);
453 attr[i++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT;
454 attr[i++] = planeOffset(1);
455 attr[i++] = EGL_DMA_BUF_PLANE1_PITCH_EXT;
456 attr[i++] = planeStride(1);
459 if (singleEGLImage && nPlanes > 2) {
460 attr[i++] = EGL_DMA_BUF_PLANE2_FD_EXT;
461 attr[i++] = fdForPlane(2);
462 attr[i++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT;
463 attr[i++] = planeOffset(2);
464 attr[i++] = EGL_DMA_BUF_PLANE2_PITCH_EXT;
465 attr[i++] = planeStride(2);
468 if (singleEGLImage && nPlanes > 3) {
469 attr[i++] = EGL_DMA_BUF_PLANE3_FD_EXT;
470 attr[i++] = fdForPlane(3);
471 attr[i++] = EGL_DMA_BUF_PLANE3_OFFSET_EXT;
472 attr[i++] = planeOffset(3);
473 attr[i++] = EGL_DMA_BUF_PLANE3_PITCH_EXT;
474 attr[i++] = planeStride(3);
477 attr[i++] = EGL_NONE;
478 Q_ASSERT(i <= maxAttrCount);
480 EGLImage image = eglCreateImage(eglDisplay,
482 EGL_LINUX_DMA_BUF_EXT,
485 if (image == EGL_NO_IMAGE_KHR) {
486 qCWarning(qLcGstVideoBuffer) <<
"could not create EGL image for plane" << plane
487 <<
", EGL error 0x" << Qt::hex << eglGetError();
490 logGlAndEglErrors(
"eglCreateImage");
492 #ifdef GL_OES_EGL_image_external
493 GLenum target = externalOes ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
495 GLenum target = GL_TEXTURE_2D;
497 functions.glBindTexture(target, textures.names[plane]);
499 using namespace QtMultimediaPrivate;
501 QEglImageFunctions::instance().glEGLImageTargetTexture2DOES(target, image);
502 logGlAndEglErrors(
"glEGLImageTargetTexture2DOES");
504 eglDestroyImage(eglDisplay, image);
514#if QT_CONFIG(gstreamer_gl)
515 GlTextures textures = {};
516 if (m_memoryFormat == QGstCaps::GLTexture)
517 textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo.gstVideoInfo);
519# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
520 else if (m_memoryFormat == QGstCaps::DMABuf && qGstEglCanMapDmaBuf()
521 && rhi.backend() == QRhi::OpenGLES2)
522 textures = mapFromDmaBuffer(&rhi, m_buffer, m_videoInfo, qGstEglDisplay());
524 if (textures.count > 0)
525 return std::make_unique<QGstQVideoFrameTextures>(
526 &rhi, QSize{ m_videoInfo.gstVideoInfo.width, m_videoInfo.gstVideoInfo.height },
527 m_frameFormat.pixelFormat(), textures, m_memoryFormat);
QGstVideoBuffer(QGstBufferHandle buffer, const QGstVideoInfo &videoInfo, const QVideoFrameFormat &frameFormat)
~QGstVideoBuffer() override
bool isDmaBuf() const override
QVideoFrameTexturesUPtr mapTextures(QRhi &, QVideoFrameTexturesUPtr &) override
void unmap() override
Releases the memory mapped by the map() function.
MapData map(QVideoFrame::MapMode mode) override
Maps the planes of a video buffer to memory.
Combined button and popup list for selecting options.