5#include <private/qvideotexturehelper_p.h>
6#include <QtCore/qloggingcategory.h>
8#include <gst/video/video.h>
9#include <gst/video/video-frame.h>
10#include <gst/video/gstvideometa.h>
11#include <gst/pbutils/gstpluginsbaseversion.h>
13#include <common/qgstutils_p.h>
16#if QT_CONFIG(gstreamer_gl)
17# include <QtGui/rhi/qrhi.h>
18# include <QtGui/qopenglcontext.h>
19# include <QtGui/qopenglfunctions.h>
20# include <QtGui/qopengl.h>
22# include <gst/gl/gstglconfig.h>
23# include <gst/gl/gstglmemory.h>
24# include <gst/gl/gstglsyncmeta.h>
26# if QT_CONFIG(gstreamer_gl_egl)
28# include <EGL/eglext.h>
31# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
32# include <QtMultimedia/private/qdmabuftextureimporter_p.h>
33# include <QtMultimedia/private/qeglimagefunctions_p.h>
34# include <QtMultimedia/private/qmultimedia_drm_support_p.h>
35# include <QtMultimedia/private/qmultimedia_gl_support_p.h>
37# include <common/qgstreameregldisplay_p.h>
38# include <gst/allocators/gstdmabuf.h>
44Q_STATIC_LOGGING_CATEGORY(qLcGstVideoBuffer,
"qt.multimedia.gstreamer.videobuffer");
47 const QVideoFrameFormat &frameFormat)
58 Q_ASSERT(m_mode == QVideoFrame::NotMapped);
63 const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0)
64 | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0));
67 if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped)
70 const GstVideoInfo &gstVideoInfo = m_videoInfo.gstVideoInfo;
71 if (!gstVideoInfo.finfo || gstVideoInfo.finfo->n_planes == 0) {
72 if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) {
73 mapData.planeCount = 1;
74 mapData.bytesPerLine[0] = -1;
75 mapData.dataSize[0] = m_frame.map[0].size;
76 mapData.data[0] =
static_cast<uchar *>(m_frame.map[0].data);
80 }
else if (gst_video_frame_map(&m_frame, &gstVideoInfo, m_buffer.get(), flags)) {
81 mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame);
83 for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) {
84 mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i);
85 mapData.data[i] =
static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i));
86 mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i);
96 if (m_mode != QVideoFrame::NotMapped) {
97 if (!m_videoInfo.gstVideoInfo.finfo || m_videoInfo.gstVideoInfo.finfo->n_planes == 0)
98 gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]);
100 gst_video_frame_unmap(&m_frame);
102 m_mode = QVideoFrame::NotMapped;
110#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
112using QtMultimediaPrivate::DRMFormat;
113using QtMultimediaPrivate::EGLError;
114using QtMultimediaPrivate::GLError;
116static std::optional<DRMFormat>
117fourccFromGstVideoFormat(
const GstVideoFormat format,
int plane,
bool singleEGLImage)
119#if G_BYTE_ORDER == G_LITTLE_ENDIAN
120 constexpr DRMFormat argb_fourcc = DRMFormat::ARGB8888;
121 constexpr DRMFormat rgba_fourcc = DRMFormat::ABGR8888;
122 constexpr DRMFormat rgb_fourcc = DRMFormat::BGR888;
123 constexpr DRMFormat rg_fourcc = DRMFormat::GR88;
125 constexpr DRMFormat argb_fourcc = DRMFormat::BGRA8888;
126 constexpr DRMFormat rgba_fourcc = DRMFormat::RGBA8888;
127 constexpr DRMFormat rgb_fourcc = DRMFormat::RGB888;
128 constexpr DRMFormat rg_fourcc = DRMFormat::RG88;
132 case GST_VIDEO_FORMAT_RGB16:
133 case GST_VIDEO_FORMAT_BGR16:
134 return DRMFormat::RGB565;
136 case GST_VIDEO_FORMAT_RGB:
137 case GST_VIDEO_FORMAT_BGR:
140 case GST_VIDEO_FORMAT_BGRx:
141 case GST_VIDEO_FORMAT_BGRA:
144 case GST_VIDEO_FORMAT_AYUV:
145 if (singleEGLImage)
return DRMFormat::AYUV;
147 case GST_VIDEO_FORMAT_RGBx:
148 case GST_VIDEO_FORMAT_RGBA:
149 case GST_VIDEO_FORMAT_ARGB:
150 case GST_VIDEO_FORMAT_xRGB:
151 case GST_VIDEO_FORMAT_ABGR:
152 case GST_VIDEO_FORMAT_xBGR:
155 case GST_VIDEO_FORMAT_GRAY8:
156 return DRMFormat::R8;
158 case GST_VIDEO_FORMAT_YUY2:
159 return DRMFormat::YUYV;
161 case GST_VIDEO_FORMAT_UYVY:
162 return DRMFormat::UYVY;
164 case GST_VIDEO_FORMAT_GRAY16_LE:
165 case GST_VIDEO_FORMAT_GRAY16_BE:
166 if (singleEGLImage)
return DRMFormat::R16;
169 case GST_VIDEO_FORMAT_NV12:
170 if (singleEGLImage)
return DRMFormat::NV12;
172 case GST_VIDEO_FORMAT_NV21:
173 if (singleEGLImage)
return DRMFormat::NV21;
174 return plane == 0 ? DRMFormat::R8 : rg_fourcc;
176 case GST_VIDEO_FORMAT_I420:
177 if (singleEGLImage)
return DRMFormat::YUV420;
179 case GST_VIDEO_FORMAT_YV12:
180 if (singleEGLImage)
return DRMFormat::YVU420;
182 case GST_VIDEO_FORMAT_Y41B:
183 if (singleEGLImage)
return DRMFormat::YUV411;
185 case GST_VIDEO_FORMAT_Y42B:
186 if (singleEGLImage)
return DRMFormat::YUV422;
188 case GST_VIDEO_FORMAT_Y444:
189 if (singleEGLImage)
return DRMFormat::YUV444;
190 return DRMFormat::R8;
192 case GST_VIDEO_FORMAT_BGR10A2_LE:
193 return DRMFormat::BGRA1010102;
195 case GST_VIDEO_FORMAT_P010_10LE:
196 case GST_VIDEO_FORMAT_P010_10BE:
197 if (singleEGLImage)
return DRMFormat::P010;
198 return plane == 0 ? DRMFormat::R16 : DRMFormat::RG1616;
205static void logGlAndEglErrors(
const char *context)
207 if (!qLcGstVideoBuffer().isDebugEnabled())
210 const GLError glError = GLError(glGetError());
211 const EGLError eglError = EGLError(eglGetError());
212 if (glError == GLError::NoError && eglError == EGLError::Success)
215 qCDebug(qLcGstVideoBuffer).nospace()
216 << context <<
": GL error " << glError <<
", EGL error " << eglError;
220#if QT_CONFIG(gstreamer_gl)
225 std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{};
228class QGstQVideoFrameTextures :
public QVideoFrameTextures
231 QGstQVideoFrameTextures(QRhi *rhi,
233 QVideoFrameFormat::PixelFormat format,
234 GlTextures &textures,
235 QGstCaps::MemoryFormat memoryFormat)
237 , m_glTextures(textures)
239 QRhiTexture::Flags textureFlags = {};
240 if (QVideoTextureHelper::forceGlTextureExternalOesIsSet()
241 && m_rhi && rhi->backend() == QRhi::OpenGLES2)
242 textureFlags = {QRhiTexture::ExternalOES};
244 bool isDmaBuf = memoryFormat == QGstCaps::DMABuf;
245 auto fallbackPolicy = isDmaBuf
246 ? QVideoTextureHelper::TextureDescription::FallbackPolicy::Disable
247 : QVideoTextureHelper::TextureDescription::FallbackPolicy::Enable;
249 auto desc = QVideoTextureHelper::textureDescription(format);
250 for (uint i = 0; i < textures.count; ++i) {
252 QSize planeSize = desc->rhiPlaneSize(size, i, isDmaBuf ?
nullptr : m_rhi);
253 QRhiTexture::Format format = desc->rhiTextureFormat(i, m_rhi, fallbackPolicy);
254 m_textures[i].reset(rhi->newTexture(format, planeSize, 1, textureFlags));
255 m_textures[i]->createFrom({textures.names[i], 0});
259 ~QGstQVideoFrameTextures() override
261 m_rhi->makeThreadLocalNativeContextCurrent();
262 auto ctx = QOpenGLContext::currentContext();
263 if (m_glTextures.owned && ctx)
264 ctx->functions()->glDeleteTextures(
int(m_glTextures.count), m_glTextures.names.data());
267 QRhiTexture *texture(uint plane)
const override
269 return plane < m_glTextures.count ? m_textures[plane].get() :
nullptr;
273 QRhi *m_rhi =
nullptr;
274 GlTextures m_glTextures;
275 std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes];
278static GlTextures mapFromGlTexture(
const QGstBufferHandle &bufferHandle, GstVideoFrame &frame,
279 GstVideoInfo &videoInfo)
281 qCDebug(qLcGstVideoBuffer) <<
"Mapping textures from GL memory";
283 GstBuffer *buffer = bufferHandle.get();
284 auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0));
288 if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) {
289 qWarning() <<
"Could not map GL textures";
293 auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer);
294 GstBuffer *sync_buffer =
nullptr;
296 sync_buffer = gst_buffer_new();
297 sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer);
299 gst_gl_sync_meta_set_sync_point (sync_meta, mem->context);
300 gst_gl_sync_meta_wait (sync_meta, mem->context);
302 gst_buffer_unref(sync_buffer);
305 textures.count = frame.info.finfo->n_planes;
307 for (uint i = 0; i < textures.count; ++i)
308 textures.names[i] = *(guint32 *)frame.data[i];
310 gst_video_frame_unmap(&frame);
315# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
316static GlTextures mapFromDmaBuffer(QRhi *rhi,
const QGstBufferHandle &bufferHandle,
317 const QGstVideoInfo &videoInfo, EGLDisplay eglDisplay)
319 qCDebug(qLcGstVideoBuffer) <<
"Importing textures from DMA buffer";
320 logGlAndEglErrors(
"mapFromDmaBuffer");
322 GstBuffer *buffer = bufferHandle.get();
324 Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0)));
325 Q_ASSERT(eglDisplay);
327 Q_ASSERT(rhi->backend() == QRhi::OpenGLES2);
329 auto *nativeHandles =
static_cast<
const QRhiGles2NativeHandles *>(rhi->nativeHandles());
330 auto glContext = nativeHandles->context;
332 qCWarning(qLcGstVideoBuffer) <<
"no GL context";
336 const GstVideoInfo &gstVideoInfo = videoInfo.gstVideoInfo;
337 if (!gstVideoInfo.finfo) {
338 qCWarning(qLcGstVideoBuffer) <<
"Missing valid GstVideoInfo for DMABuf GstBuffer";
342 if (videoInfo.dmaDrmModifier && *videoInfo.dmaDrmModifier != 0) {
343 qCWarning(qLcGstVideoBuffer) <<
"Unsupported non-linear DMABuf modifier:"
344 << Qt::hex << *videoInfo.dmaDrmModifier;
348 const GstVideoMeta *videoMeta = gst_buffer_get_video_meta(buffer);
349 const GstVideoFormat videoInfoFormat = GST_VIDEO_INFO_FORMAT(&gstVideoInfo);
350 GstVideoFormat format = videoMeta ? videoMeta->format : videoInfoFormat;
351 if (format == GST_VIDEO_FORMAT_UNKNOWN)
352 format = videoInfoFormat;
354 const int nPlanes = videoMeta ? videoMeta->n_planes : GST_VIDEO_INFO_N_PLANES(&gstVideoInfo);
355 const int nMemoryBlocks = gst_buffer_n_memory(buffer);
356 static const bool externalOes = QVideoTextureHelper::forceGlTextureExternalOesIsSet();
357 static const bool singleEGLImage =
358 externalOes || qEnvironmentVariableIntValue(
"QT_GSTREAMER_FORCE_SINGLE_EGLIMAGE") != 0;
360 qCDebug(qLcGstVideoBuffer) <<
"format:" << gst_video_format_to_string(format)
361 <<
"nPlanes:" << nPlanes
362 <<
"nMemoryBlocks:" << nMemoryBlocks
363 <<
"externalOes:" << externalOes
364 <<
"singleEGLImage:" << singleEGLImage;
366 constexpr int maxPlanes = 4;
367 Q_ASSERT(nPlanes >= 1
368 && nPlanes <= maxPlanes
369 && (nMemoryBlocks == 1 || nMemoryBlocks == nPlanes));
371 const int nEGLImages = singleEGLImage ? 1 : nPlanes;
372 std::array<EGLAttrib, maxPlanes> planeFourcc{};
373 for (
int plane = 0; plane < nEGLImages; ++plane) {
374 const std::optional<DRMFormat> fourcc = fourccFromGstVideoFormat(format, plane, singleEGLImage);
376 qCWarning(qLcGstVideoBuffer) <<
"Unsupported format for DMABuf:"
377 << gst_video_format_to_string(format) <<
"plane:" << plane
378 <<
"singleEGLImage" << singleEGLImage;
381 planeFourcc[plane] = EGLAttrib(quint32(*fourcc));
384 GlTextures textures = {};
385 textures.owned =
true;
386 textures.count = nEGLImages;
388 QOpenGLFunctions functions(glContext);
389 functions.glGenTextures(
int(textures.count), textures.names.data());
390 logGlAndEglErrors(
"glGenTextures");
392 std::array<
int, maxPlanes> fds{-1, -1, -1, -1};
393 for (
int i = 0; i < nMemoryBlocks && i < maxPlanes; ++i) {
394 fds[i] = gst_dmabuf_memory_get_fd(gst_buffer_peek_memory(buffer, i));
397 auto fdForPlane = [&](
int plane) -> EGLAttrib {
398 if (plane < 0 || plane >= maxPlanes || plane >= nMemoryBlocks)
400 return (fds[plane] >= 0) ? fds[plane] : fds[0];
403 auto compWidth = [&](
int plane) -> EGLAttrib {
404 return singleEGLImage ? GST_VIDEO_INFO_WIDTH(&gstVideoInfo)
405 : GST_VIDEO_INFO_COMP_WIDTH(&gstVideoInfo, plane);
408 auto compHeight = [&](
int plane) -> EGLAttrib {
409 return singleEGLImage ? GST_VIDEO_INFO_HEIGHT(&gstVideoInfo)
410 : GST_VIDEO_INFO_COMP_HEIGHT(&gstVideoInfo, plane);
413 auto planeOffset = [&](
int plane) -> EGLAttrib {
415 if (nPlanes == nMemoryBlocks)
418 return videoMeta->offset[plane];
419 return GST_VIDEO_INFO_PLANE_OFFSET(&gstVideoInfo, plane);
422 auto planeStride = [&](
int plane) -> EGLAttrib {
424 return videoMeta->stride[plane];
425 return GST_VIDEO_INFO_PLANE_STRIDE(&gstVideoInfo, plane);
428 for (
int plane = 0; plane < nEGLImages; ++plane) {
429 constexpr int maxAttrCount = 31;
430 std::array<EGLAttrib, maxAttrCount> attr;
433 const int width = compWidth(plane);
434 const int height = compHeight(plane);
436 attr[i++] = EGL_WIDTH;
438 attr[i++] = EGL_HEIGHT;
440 attr[i++] = EGL_LINUX_DRM_FOURCC_EXT;
441 attr[i++] = planeFourcc[plane];
443 attr[i++] = EGL_DMA_BUF_PLANE0_FD_EXT;
444 attr[i++] = fdForPlane(plane);
445 attr[i++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT;
446 attr[i++] = planeOffset(plane);
447 attr[i++] = EGL_DMA_BUF_PLANE0_PITCH_EXT;
448 attr[i++] = planeStride(plane);
450 if (singleEGLImage && nPlanes > 1) {
451 attr[i++] = EGL_DMA_BUF_PLANE1_FD_EXT;
452 attr[i++] = fdForPlane(1);
453 attr[i++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT;
454 attr[i++] = planeOffset(1);
455 attr[i++] = EGL_DMA_BUF_PLANE1_PITCH_EXT;
456 attr[i++] = planeStride(1);
459 if (singleEGLImage && nPlanes > 2) {
460 attr[i++] = EGL_DMA_BUF_PLANE2_FD_EXT;
461 attr[i++] = fdForPlane(2);
462 attr[i++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT;
463 attr[i++] = planeOffset(2);
464 attr[i++] = EGL_DMA_BUF_PLANE2_PITCH_EXT;
465 attr[i++] = planeStride(2);
468 if (singleEGLImage && nPlanes > 3) {
469 attr[i++] = EGL_DMA_BUF_PLANE3_FD_EXT;
470 attr[i++] = fdForPlane(3);
471 attr[i++] = EGL_DMA_BUF_PLANE3_OFFSET_EXT;
472 attr[i++] = planeOffset(3);
473 attr[i++] = EGL_DMA_BUF_PLANE3_PITCH_EXT;
474 attr[i++] = planeStride(3);
477 attr[i++] = EGL_NONE;
478 Q_ASSERT(i <= maxAttrCount);
480 using namespace QtMultimediaPrivate;
481 const auto &elgImageFunctions = QEglImageFunctions::instance();
483 EGLImageHandle image = elgImageFunctions.eglCreateImage(
484 eglDisplay, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT,
nullptr,
485 QSpan<
const EGLAttrib>(attr.data(), i));
487 qCWarning(qLcGstVideoBuffer) <<
"could not create EGL image for plane" << plane
488 <<
", EGL error" << EGLError(eglGetError());
491 logGlAndEglErrors(
"eglCreateImage");
493 #ifdef GL_OES_EGL_image_external
494 GLenum target = externalOes ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
496 GLenum target = GL_TEXTURE_2D;
498 functions.glBindTexture(target, textures.names[plane]);
500 elgImageFunctions.glEGLImageTargetTexture2DOES(target, image.get());
501 logGlAndEglErrors(
"glEGLImageTargetTexture2DOES");
511 using namespace QtMultimediaPrivate;
513#if QT_CONFIG(gstreamer_gl)
514 GlTextures textures = {};
515 if (m_memoryFormat == QGstCaps::GLTexture)
516 textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo.gstVideoInfo);
518# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
519 else if (m_memoryFormat == QGstCaps::DMABuf && qGstEglCanMapDmaBuf(&rhi)
520 && rhi.backend() == QRhi::OpenGLES2)
521 textures = mapFromDmaBuffer(&rhi, m_buffer, m_videoInfo, resolveEglDisplay(rhi));
523 if (textures.count > 0)
524 return std::make_unique<QGstQVideoFrameTextures>(
525 &rhi, QSize{ m_videoInfo.gstVideoInfo.width, m_videoInfo.gstVideoInfo.height },
526 m_frameFormat.pixelFormat(), textures, m_memoryFormat);
QGstVideoBuffer(QGstBufferHandle buffer, const QGstVideoInfo &videoInfo, const QVideoFrameFormat &frameFormat)
~QGstVideoBuffer() override
bool isDmaBuf() const override
void unmap() override
Releases the memory mapped by the map() function.
MapData map(QVideoFrame::MapMode mode) override
Maps the planes of a video buffer to memory.
QVideoFrameTexturesUPtr mapTextures(QRhi &) override
Combined button and popup list for selecting options.