5#include <private/qvideotexturehelper_p.h>
6#include <QtCore/qloggingcategory.h>
8#include <gst/video/video.h>
9#include <gst/video/video-frame.h>
10#include <gst/video/gstvideometa.h>
11#include <gst/pbutils/gstpluginsbaseversion.h>
13#include <common/qgstutils_p.h>
16#if QT_CONFIG(gstreamer_gl)
17# include <QtGui/rhi/qrhi.h>
18# include <QtGui/qopenglcontext.h>
19# include <QtGui/qopenglfunctions.h>
20# include <QtGui/qopengl.h>
22# include <gst/gl/gstglconfig.h>
23# include <gst/gl/gstglmemory.h>
24# include <gst/gl/gstglsyncmeta.h>
26# if QT_CONFIG(gstreamer_gl_egl)
28# include <EGL/eglext.h>
31# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
32# include <QtMultimedia/private/qdmabuftextureimporter_p.h>
33# include <QtMultimedia/private/qeglimagefunctions_p.h>
34# include <QtMultimedia/private/qmultimedia_drm_support_p.h>
35# include <QtMultimedia/private/qmultimedia_gl_support_p.h>
37# include <common/qgstreameregldisplay_p.h>
38# include <gst/allocators/gstdmabuf.h>
44Q_STATIC_LOGGING_CATEGORY(qLcGstVideoBuffer,
"qt.multimedia.gstreamer.videobuffer");
47 const QVideoFrameFormat &frameFormat)
55 m_type = m_memoryFormat != QGstCaps::CpuMemory ? QVideoFrame::RhiTextureHandle
56 : QVideoFrame::NoHandle;
61 Q_ASSERT(m_mode == QVideoFrame::NotMapped);
66 const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0)
67 | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0));
70 if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped)
73 const GstVideoInfo &gstVideoInfo = m_videoInfo.gstVideoInfo;
74 if (!gstVideoInfo.finfo || gstVideoInfo.finfo->n_planes == 0) {
75 if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) {
76 mapData.planeCount = 1;
77 mapData.bytesPerLine[0] = -1;
78 mapData.dataSize[0] = m_frame.map[0].size;
79 mapData.data[0] =
static_cast<uchar *>(m_frame.map[0].data);
83 }
else if (gst_video_frame_map(&m_frame, &gstVideoInfo, m_buffer.get(), flags)) {
84 mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame);
86 for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) {
87 mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i);
88 mapData.data[i] =
static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i));
89 mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i);
99 if (m_mode != QVideoFrame::NotMapped) {
100 if (!m_videoInfo.gstVideoInfo.finfo || m_videoInfo.gstVideoInfo.finfo->n_planes == 0)
101 gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]);
103 gst_video_frame_unmap(&m_frame);
105 m_mode = QVideoFrame::NotMapped;
113#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
115using QtMultimediaPrivate::DRMFormat;
116using QtMultimediaPrivate::EGLError;
117using QtMultimediaPrivate::GLError;
119static std::optional<DRMFormat>
120fourccFromGstVideoFormat(
const GstVideoFormat format,
int plane,
bool singleEGLImage)
122#if G_BYTE_ORDER == G_LITTLE_ENDIAN
123 constexpr DRMFormat argb_fourcc = DRMFormat::ARGB8888;
124 constexpr DRMFormat rgba_fourcc = DRMFormat::ABGR8888;
125 constexpr DRMFormat rgb_fourcc = DRMFormat::BGR888;
126 constexpr DRMFormat rg_fourcc = DRMFormat::GR88;
128 constexpr DRMFormat argb_fourcc = DRMFormat::BGRA8888;
129 constexpr DRMFormat rgba_fourcc = DRMFormat::RGBA8888;
130 constexpr DRMFormat rgb_fourcc = DRMFormat::RGB888;
131 constexpr DRMFormat rg_fourcc = DRMFormat::RG88;
135 case GST_VIDEO_FORMAT_RGB16:
136 case GST_VIDEO_FORMAT_BGR16:
137 return DRMFormat::RGB565;
139 case GST_VIDEO_FORMAT_RGB:
140 case GST_VIDEO_FORMAT_BGR:
143 case GST_VIDEO_FORMAT_BGRx:
144 case GST_VIDEO_FORMAT_BGRA:
147 case GST_VIDEO_FORMAT_AYUV:
148 if (singleEGLImage)
return DRMFormat::AYUV;
150 case GST_VIDEO_FORMAT_RGBx:
151 case GST_VIDEO_FORMAT_RGBA:
152 case GST_VIDEO_FORMAT_ARGB:
153 case GST_VIDEO_FORMAT_xRGB:
154 case GST_VIDEO_FORMAT_ABGR:
155 case GST_VIDEO_FORMAT_xBGR:
158 case GST_VIDEO_FORMAT_GRAY8:
159 return DRMFormat::R8;
161 case GST_VIDEO_FORMAT_YUY2:
162 return DRMFormat::YUYV;
164 case GST_VIDEO_FORMAT_UYVY:
165 return DRMFormat::UYVY;
167 case GST_VIDEO_FORMAT_GRAY16_LE:
168 case GST_VIDEO_FORMAT_GRAY16_BE:
169 if (singleEGLImage)
return DRMFormat::R16;
172 case GST_VIDEO_FORMAT_NV12:
173 if (singleEGLImage)
return DRMFormat::NV12;
175 case GST_VIDEO_FORMAT_NV21:
176 if (singleEGLImage)
return DRMFormat::NV21;
177 return plane == 0 ? DRMFormat::R8 : rg_fourcc;
179 case GST_VIDEO_FORMAT_I420:
180 if (singleEGLImage)
return DRMFormat::YUV420;
182 case GST_VIDEO_FORMAT_YV12:
183 if (singleEGLImage)
return DRMFormat::YVU420;
185 case GST_VIDEO_FORMAT_Y41B:
186 if (singleEGLImage)
return DRMFormat::YUV411;
188 case GST_VIDEO_FORMAT_Y42B:
189 if (singleEGLImage)
return DRMFormat::YUV422;
191 case GST_VIDEO_FORMAT_Y444:
192 if (singleEGLImage)
return DRMFormat::YUV444;
193 return DRMFormat::R8;
195 case GST_VIDEO_FORMAT_BGR10A2_LE:
196 return DRMFormat::BGRA1010102;
198 case GST_VIDEO_FORMAT_P010_10LE:
199 case GST_VIDEO_FORMAT_P010_10BE:
200 if (singleEGLImage)
return DRMFormat::P010;
201 return plane == 0 ? DRMFormat::R16 : DRMFormat::RG1616;
208static void logGlAndEglErrors(
const char *context)
210 if (!qLcGstVideoBuffer().isDebugEnabled())
213 const GLError glError = GLError(glGetError());
214 const EGLError eglError = EGLError(eglGetError());
215 if (glError == GLError::NoError && eglError == EGLError::Success)
218 qCDebug(qLcGstVideoBuffer).nospace()
219 << context <<
": GL error " << glError <<
", EGL error " << eglError;
223#if QT_CONFIG(gstreamer_gl)
228 std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{};
231class QGstQVideoFrameTextures :
public QVideoFrameTextures
234 QGstQVideoFrameTextures(QRhi *rhi,
236 QVideoFrameFormat::PixelFormat format,
237 GlTextures &textures,
238 QGstCaps::MemoryFormat memoryFormat)
240 , m_glTextures(textures)
242 QRhiTexture::Flags textureFlags = {};
243 if (QVideoTextureHelper::forceGlTextureExternalOesIsSet()
244 && m_rhi && rhi->backend() == QRhi::OpenGLES2)
245 textureFlags = {QRhiTexture::ExternalOES};
247 bool isDmaBuf = memoryFormat == QGstCaps::DMABuf;
248 auto fallbackPolicy = isDmaBuf
249 ? QVideoTextureHelper::TextureDescription::FallbackPolicy::Disable
250 : QVideoTextureHelper::TextureDescription::FallbackPolicy::Enable;
252 auto desc = QVideoTextureHelper::textureDescription(format);
253 for (uint i = 0; i < textures.count; ++i) {
255 QSize planeSize = desc->rhiPlaneSize(size, i, isDmaBuf ?
nullptr : m_rhi);
256 QRhiTexture::Format format = desc->rhiTextureFormat(i, m_rhi, fallbackPolicy);
257 m_textures[i].reset(rhi->newTexture(format, planeSize, 1, textureFlags));
258 m_textures[i]->createFrom({textures.names[i], 0});
262 ~QGstQVideoFrameTextures() override
264 m_rhi->makeThreadLocalNativeContextCurrent();
265 auto ctx = QOpenGLContext::currentContext();
266 if (m_glTextures.owned && ctx)
267 ctx->functions()->glDeleteTextures(
int(m_glTextures.count), m_glTextures.names.data());
270 QRhiTexture *texture(uint plane)
const override
272 return plane < m_glTextures.count ? m_textures[plane].get() :
nullptr;
276 QRhi *m_rhi =
nullptr;
277 GlTextures m_glTextures;
278 std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes];
281static GlTextures mapFromGlTexture(
const QGstBufferHandle &bufferHandle, GstVideoFrame &frame,
282 GstVideoInfo &videoInfo)
284 qCDebug(qLcGstVideoBuffer) <<
"Mapping textures from GL memory";
286 GstBuffer *buffer = bufferHandle.get();
287 auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0));
291 if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) {
292 qWarning() <<
"Could not map GL textures";
296 auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer);
297 GstBuffer *sync_buffer =
nullptr;
299 sync_buffer = gst_buffer_new();
300 sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer);
302 gst_gl_sync_meta_set_sync_point (sync_meta, mem->context);
303 gst_gl_sync_meta_wait (sync_meta, mem->context);
305 gst_buffer_unref(sync_buffer);
308 textures.count = frame.info.finfo->n_planes;
310 for (uint i = 0; i < textures.count; ++i)
311 textures.names[i] = *(guint32 *)frame.data[i];
313 gst_video_frame_unmap(&frame);
318# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
319static GlTextures mapFromDmaBuffer(QRhi *rhi,
const QGstBufferHandle &bufferHandle,
320 const QGstVideoInfo &videoInfo, EGLDisplay eglDisplay)
322 qCDebug(qLcGstVideoBuffer) <<
"Importing textures from DMA buffer";
323 logGlAndEglErrors(
"mapFromDmaBuffer");
325 GstBuffer *buffer = bufferHandle.get();
327 Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0)));
328 Q_ASSERT(eglDisplay);
330 Q_ASSERT(rhi->backend() == QRhi::OpenGLES2);
332 auto *nativeHandles =
static_cast<
const QRhiGles2NativeHandles *>(rhi->nativeHandles());
333 auto glContext = nativeHandles->context;
335 qCWarning(qLcGstVideoBuffer) <<
"no GL context";
339 const GstVideoInfo &gstVideoInfo = videoInfo.gstVideoInfo;
340 if (!gstVideoInfo.finfo) {
341 qCWarning(qLcGstVideoBuffer) <<
"Missing valid GstVideoInfo for DMABuf GstBuffer";
345 if (videoInfo.dmaDrmModifier && *videoInfo.dmaDrmModifier != 0) {
346 qCWarning(qLcGstVideoBuffer) <<
"Unsupported non-linear DMABuf modifier:"
347 << Qt::hex << *videoInfo.dmaDrmModifier;
351 const GstVideoMeta *videoMeta = gst_buffer_get_video_meta(buffer);
352 const GstVideoFormat videoInfoFormat = GST_VIDEO_INFO_FORMAT(&gstVideoInfo);
353 GstVideoFormat format = videoMeta ? videoMeta->format : videoInfoFormat;
354 if (format == GST_VIDEO_FORMAT_UNKNOWN)
355 format = videoInfoFormat;
357 const int nPlanes = videoMeta ? videoMeta->n_planes : GST_VIDEO_INFO_N_PLANES(&gstVideoInfo);
358 const int nMemoryBlocks = gst_buffer_n_memory(buffer);
359 static const bool externalOes = QVideoTextureHelper::forceGlTextureExternalOesIsSet();
360 static const bool singleEGLImage =
361 externalOes || qEnvironmentVariableIntValue(
"QT_GSTREAMER_FORCE_SINGLE_EGLIMAGE") != 0;
363 qCDebug(qLcGstVideoBuffer) <<
"format:" << gst_video_format_to_string(format)
364 <<
"nPlanes:" << nPlanes
365 <<
"nMemoryBlocks:" << nMemoryBlocks
366 <<
"externalOes:" << externalOes
367 <<
"singleEGLImage:" << singleEGLImage;
369 constexpr int maxPlanes = 4;
370 Q_ASSERT(nPlanes >= 1
371 && nPlanes <= maxPlanes
372 && (nMemoryBlocks == 1 || nMemoryBlocks == nPlanes));
374 const int nEGLImages = singleEGLImage ? 1 : nPlanes;
375 std::array<EGLAttrib, maxPlanes> planeFourcc{};
376 for (
int plane = 0; plane < nEGLImages; ++plane) {
377 const std::optional<DRMFormat> fourcc = fourccFromGstVideoFormat(format, plane, singleEGLImage);
379 qCWarning(qLcGstVideoBuffer) <<
"Unsupported format for DMABuf:"
380 << gst_video_format_to_string(format) <<
"plane:" << plane
381 <<
"singleEGLImage" << singleEGLImage;
384 planeFourcc[plane] = EGLAttrib(quint32(*fourcc));
387 GlTextures textures = {};
388 textures.owned =
true;
389 textures.count = nEGLImages;
391 QOpenGLFunctions functions(glContext);
392 functions.glGenTextures(
int(textures.count), textures.names.data());
393 logGlAndEglErrors(
"glGenTextures");
395 std::array<
int, maxPlanes> fds{-1, -1, -1, -1};
396 for (
int i = 0; i < nMemoryBlocks && i < maxPlanes; ++i) {
397 fds[i] = gst_dmabuf_memory_get_fd(gst_buffer_peek_memory(buffer, i));
400 auto fdForPlane = [&](
int plane) -> EGLAttrib {
401 if (plane < 0 || plane >= maxPlanes || plane >= nMemoryBlocks)
403 return (fds[plane] >= 0) ? fds[plane] : fds[0];
406 auto compWidth = [&](
int plane) -> EGLAttrib {
407 return singleEGLImage ? GST_VIDEO_INFO_WIDTH(&gstVideoInfo)
408 : GST_VIDEO_INFO_COMP_WIDTH(&gstVideoInfo, plane);
411 auto compHeight = [&](
int plane) -> EGLAttrib {
412 return singleEGLImage ? GST_VIDEO_INFO_HEIGHT(&gstVideoInfo)
413 : GST_VIDEO_INFO_COMP_HEIGHT(&gstVideoInfo, plane);
416 auto planeOffset = [&](
int plane) -> EGLAttrib {
418 if (nPlanes == nMemoryBlocks)
421 return videoMeta->offset[plane];
422 return GST_VIDEO_INFO_PLANE_OFFSET(&gstVideoInfo, plane);
425 auto planeStride = [&](
int plane) -> EGLAttrib {
427 return videoMeta->stride[plane];
428 return GST_VIDEO_INFO_PLANE_STRIDE(&gstVideoInfo, plane);
431 for (
int plane = 0; plane < nEGLImages; ++plane) {
432 constexpr int maxAttrCount = 31;
433 std::array<EGLAttrib, maxAttrCount> attr;
436 const int width = compWidth(plane);
437 const int height = compHeight(plane);
439 attr[i++] = EGL_WIDTH;
441 attr[i++] = EGL_HEIGHT;
443 attr[i++] = EGL_LINUX_DRM_FOURCC_EXT;
444 attr[i++] = planeFourcc[plane];
446 attr[i++] = EGL_DMA_BUF_PLANE0_FD_EXT;
447 attr[i++] = fdForPlane(plane);
448 attr[i++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT;
449 attr[i++] = planeOffset(plane);
450 attr[i++] = EGL_DMA_BUF_PLANE0_PITCH_EXT;
451 attr[i++] = planeStride(plane);
453 if (singleEGLImage && nPlanes > 1) {
454 attr[i++] = EGL_DMA_BUF_PLANE1_FD_EXT;
455 attr[i++] = fdForPlane(1);
456 attr[i++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT;
457 attr[i++] = planeOffset(1);
458 attr[i++] = EGL_DMA_BUF_PLANE1_PITCH_EXT;
459 attr[i++] = planeStride(1);
462 if (singleEGLImage && nPlanes > 2) {
463 attr[i++] = EGL_DMA_BUF_PLANE2_FD_EXT;
464 attr[i++] = fdForPlane(2);
465 attr[i++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT;
466 attr[i++] = planeOffset(2);
467 attr[i++] = EGL_DMA_BUF_PLANE2_PITCH_EXT;
468 attr[i++] = planeStride(2);
471 if (singleEGLImage && nPlanes > 3) {
472 attr[i++] = EGL_DMA_BUF_PLANE3_FD_EXT;
473 attr[i++] = fdForPlane(3);
474 attr[i++] = EGL_DMA_BUF_PLANE3_OFFSET_EXT;
475 attr[i++] = planeOffset(3);
476 attr[i++] = EGL_DMA_BUF_PLANE3_PITCH_EXT;
477 attr[i++] = planeStride(3);
480 attr[i++] = EGL_NONE;
481 Q_ASSERT(i <= maxAttrCount);
483 using namespace QtMultimediaPrivate;
484 const auto &elgImageFunctions = QEglImageFunctions::instance();
486 EGLImageHandle image = elgImageFunctions.eglCreateImage(
487 eglDisplay, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT,
nullptr,
488 QSpan<
const EGLAttrib>(attr.data(), i));
490 qCWarning(qLcGstVideoBuffer) <<
"could not create EGL image for plane" << plane
491 <<
", EGL error" << EGLError(eglGetError());
494 logGlAndEglErrors(
"eglCreateImage");
496 #ifdef GL_OES_EGL_image_external
497 GLenum target = externalOes ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
499 GLenum target = GL_TEXTURE_2D;
501 functions.glBindTexture(target, textures.names[plane]);
503 elgImageFunctions.glEGLImageTargetTexture2DOES(target, image.get());
504 logGlAndEglErrors(
"glEGLImageTargetTexture2DOES");
514 using namespace QtMultimediaPrivate;
516#if QT_CONFIG(gstreamer_gl)
517 GlTextures textures = {};
518 if (m_memoryFormat == QGstCaps::GLTexture)
519 textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo.gstVideoInfo);
521# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
522 else if (m_memoryFormat == QGstCaps::DMABuf && qGstEglCanMapDmaBuf(&rhi)
523 && rhi.backend() == QRhi::OpenGLES2)
524 textures = mapFromDmaBuffer(&rhi, m_buffer, m_videoInfo, resolveEglDisplay(rhi));
526 if (textures.count > 0)
527 return std::make_unique<QGstQVideoFrameTextures>(
528 &rhi, QSize{ m_videoInfo.gstVideoInfo.width, m_videoInfo.gstVideoInfo.height },
529 m_frameFormat.pixelFormat(), textures, m_memoryFormat);
QGstVideoBuffer(QGstBufferHandle buffer, const QGstVideoInfo &videoInfo, const QVideoFrameFormat &frameFormat)
~QGstVideoBuffer() override
bool isDmaBuf() const override
QVideoFrameTexturesUPtr mapTextures(QRhi &, QVideoFrameTexturesUPtr &) override
void unmap() override
Releases the memory mapped by the map() function.
MapData map(QVideoFrame::MapMode mode) override
Maps the planes of a video buffer to memory.
Combined button and popup list for selecting options.