5#include <private/qvideotexturehelper_p.h>
6#include <qpa/qplatformnativeinterface.h>
7#include <qguiapplication.h>
8#include <QtCore/qapplicationstatic.h>
9#include <QtCore/qloggingcategory.h>
11#include <gst/video/video.h>
12#include <gst/video/video-frame.h>
13#include <gst/video/gstvideometa.h>
14#include <gst/pbutils/gstpluginsbaseversion.h>
16#include <common/qgstutils_p.h>
18#if QT_CONFIG(gstreamer_gl)
19# include <QtGui/rhi/qrhi.h>
20# include <QtGui/qopenglcontext.h>
21# include <QtGui/qopenglfunctions.h>
22# include <QtGui/qopengl.h>
24# include <gst/gl/gstglconfig.h>
25# include <gst/gl/gstglmemory.h>
26# include <gst/gl/gstglsyncmeta.h>
28# if QT_CONFIG(gstreamer_gl_egl)
30# include <EGL/eglext.h>
33# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
34# include <gst/allocators/gstdmabuf.h>
40Q_STATIC_LOGGING_CATEGORY(qLcGstVideoBuffer,
"qt.multimedia.gstreamer.videobuffer");
42#if QT_CONFIG(gstreamer_gl) && QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
43Qt::HANDLE getEglDisplay() {
44 using namespace Qt::StringLiterals;
46 ? qGuiApp->platformNativeInterface()->nativeResourceForIntegration(
"egldisplay"_ba)
50Q_APPLICATION_STATIC(Qt::HANDLE, s_eglDisplay, getEglDisplay());
52Q_GLOBAL_STATIC(QFunctionPointer, g_eglImageTargetTexture2D,
53 eglGetProcAddress(
"glEGLImageTargetTexture2DOES"));
57#define fourcc_code(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8
) |
58 ((uint32_t)(c) << 16
) | ((uint32_t)(d) << 24
))
86 const QVideoFrameFormat &frameFormat)
94 m_type = m_memoryFormat != QGstCaps::CpuMemory ? QVideoFrame::RhiTextureHandle
95 : QVideoFrame::NoHandle;
96#if QT_CONFIG(gstreamer_gl) && QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
97 m_eglDisplay = *s_eglDisplay();
99 m_eglImageTargetTexture2D = *g_eglImageTargetTexture2D();
105 Q_ASSERT(m_mode == QVideoFrame::NotMapped);
110 const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0)
111 | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0));
114 if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped)
117 const GstVideoInfo &gstVideoInfo = m_videoInfo.gstVideoInfo;
118 if (!gstVideoInfo.finfo || gstVideoInfo.finfo->n_planes == 0) {
119 if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) {
120 mapData.planeCount = 1;
121 mapData.bytesPerLine[0] = -1;
122 mapData.dataSize[0] = m_frame.map[0].size;
123 mapData.data[0] =
static_cast<uchar *>(m_frame.map[0].data);
127 }
else if (gst_video_frame_map(&m_frame, &gstVideoInfo, m_buffer.get(), flags)) {
128 mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame);
130 for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) {
131 mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i);
132 mapData.data[i] =
static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i));
133 mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i);
143 if (m_mode != QVideoFrame::NotMapped) {
144 if (!m_videoInfo.gstVideoInfo.finfo || m_videoInfo.gstVideoInfo.finfo->n_planes == 0)
145 gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]);
147 gst_video_frame_unmap(&m_frame);
149 m_mode = QVideoFrame::NotMapped;
157#if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
160fourccFromGstVideoFormat(
const GstVideoFormat format,
int plane,
bool singleEGLImage)
162#if G_BYTE_ORDER == G_LITTLE_ENDIAN
163 const gint argb_fourcc = DRM_FORMAT_ARGB8888;
164 const gint rgba_fourcc = DRM_FORMAT_ABGR8888;
165 const gint rgb_fourcc = DRM_FORMAT_BGR888;
166 const gint rg_fourcc = DRM_FORMAT_GR88;
168 const gint argb_fourcc = DRM_FORMAT_BGRA8888;
169 const gint rgba_fourcc = DRM_FORMAT_RGBA8888;
170 const gint rgb_fourcc = DRM_FORMAT_RGB888;
171 const gint rg_fourcc = DRM_FORMAT_RG88;
175 case GST_VIDEO_FORMAT_RGB16:
176 case GST_VIDEO_FORMAT_BGR16:
177 return DRM_FORMAT_RGB565;
179 case GST_VIDEO_FORMAT_RGB:
180 case GST_VIDEO_FORMAT_BGR:
183 case GST_VIDEO_FORMAT_BGRx:
184 case GST_VIDEO_FORMAT_BGRA:
187 case GST_VIDEO_FORMAT_AYUV:
188 if (singleEGLImage)
return DRM_FORMAT_AYUV;
190 case GST_VIDEO_FORMAT_RGBx:
191 case GST_VIDEO_FORMAT_RGBA:
192 case GST_VIDEO_FORMAT_ARGB:
193 case GST_VIDEO_FORMAT_xRGB:
194 case GST_VIDEO_FORMAT_ABGR:
195 case GST_VIDEO_FORMAT_xBGR:
198 case GST_VIDEO_FORMAT_GRAY8:
199 return DRM_FORMAT_R8;
201 case GST_VIDEO_FORMAT_YUY2:
202 return DRM_FORMAT_YUYV;
204 case GST_VIDEO_FORMAT_UYVY:
205 return DRM_FORMAT_UYVY;
207 case GST_VIDEO_FORMAT_GRAY16_LE:
208 case GST_VIDEO_FORMAT_GRAY16_BE:
209 if (singleEGLImage)
return DRM_FORMAT_R16;
212 case GST_VIDEO_FORMAT_NV12:
213 if (singleEGLImage)
return DRM_FORMAT_NV12;
215 case GST_VIDEO_FORMAT_NV21:
216 if (singleEGLImage)
return DRM_FORMAT_NV21;
217 return plane == 0 ? DRM_FORMAT_R8 : rg_fourcc;
219 case GST_VIDEO_FORMAT_I420:
220 if (singleEGLImage)
return DRM_FORMAT_YUV420;
222 case GST_VIDEO_FORMAT_YV12:
223 if (singleEGLImage)
return DRM_FORMAT_YVU420;
225 case GST_VIDEO_FORMAT_Y41B:
226 if (singleEGLImage)
return DRM_FORMAT_YUV411;
228 case GST_VIDEO_FORMAT_Y42B:
229 if (singleEGLImage)
return DRM_FORMAT_YUV422;
231 case GST_VIDEO_FORMAT_Y444:
232 if (singleEGLImage)
return DRM_FORMAT_YUV444;
233 return DRM_FORMAT_R8;
235 case GST_VIDEO_FORMAT_BGR10A2_LE:
236 return DRM_FORMAT_BGRA1010102;
238 case GST_VIDEO_FORMAT_P010_10LE:
239 case GST_VIDEO_FORMAT_P010_10BE:
240 if (singleEGLImage)
return DRM_FORMAT_P010;
241 return plane == 0 ? DRM_FORMAT_R16 : DRM_FORMAT_RG1616;
248static void logGlAndEglErrors(
const char *context)
250 if (!qLcGstVideoBuffer().isDebugEnabled())
253 const GLenum glError = glGetError();
254 const EGLint eglError = eglGetError();
255 if (glError == GL_NO_ERROR && eglError == EGL_SUCCESS)
258 qCDebug(qLcGstVideoBuffer).nospace()
259 << context <<
": GL error 0x" << Qt::hex << glError
260 <<
", EGL error 0x" << eglError;
264#if QT_CONFIG(gstreamer_gl)
269 std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{};
272class QGstQVideoFrameTextures :
public QVideoFrameTextures
275 QGstQVideoFrameTextures(QRhi *rhi,
277 QVideoFrameFormat::PixelFormat format,
278 GlTextures &textures,
279 QGstCaps::MemoryFormat memoryFormat)
281 , m_glTextures(textures)
283 QRhiTexture::Flags textureFlags = {};
284 if (QVideoTextureHelper::forceGlTextureExternalOesIsSet()
285 && m_rhi && rhi->backend() == QRhi::OpenGLES2)
286 textureFlags = {QRhiTexture::ExternalOES};
288 bool isDmaBuf = memoryFormat == QGstCaps::DMABuf;
289 auto fallbackPolicy = isDmaBuf
290 ? QVideoTextureHelper::TextureDescription::FallbackPolicy::Disable
291 : QVideoTextureHelper::TextureDescription::FallbackPolicy::Enable;
293 auto desc = QVideoTextureHelper::textureDescription(format);
294 for (uint i = 0; i < textures.count; ++i) {
296 QSize planeSize = desc->rhiPlaneSize(size, i, isDmaBuf ?
nullptr : m_rhi);
297 QRhiTexture::Format format = desc->rhiTextureFormat(i, m_rhi, fallbackPolicy);
298 m_textures[i].reset(rhi->newTexture(format, planeSize, 1, textureFlags));
299 m_textures[i]->createFrom({textures.names[i], 0});
303 ~QGstQVideoFrameTextures() override
305 m_rhi->makeThreadLocalNativeContextCurrent();
306 auto ctx = QOpenGLContext::currentContext();
307 if (m_glTextures.owned && ctx)
308 ctx->functions()->glDeleteTextures(
int(m_glTextures.count), m_glTextures.names.data());
311 QRhiTexture *texture(uint plane)
const override
313 return plane < m_glTextures.count ? m_textures[plane].get() :
nullptr;
317 QRhi *m_rhi =
nullptr;
318 GlTextures m_glTextures;
319 std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes];
322static GlTextures mapFromGlTexture(
const QGstBufferHandle &bufferHandle, GstVideoFrame &frame,
323 GstVideoInfo &videoInfo)
325 qCDebug(qLcGstVideoBuffer) <<
"Mapping textures from GL memory";
327 GstBuffer *buffer = bufferHandle.get();
328 auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0));
332 if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) {
333 qWarning() <<
"Could not map GL textures";
337 auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer);
338 GstBuffer *sync_buffer =
nullptr;
340 sync_buffer = gst_buffer_new();
341 sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer);
343 gst_gl_sync_meta_set_sync_point (sync_meta, mem->context);
344 gst_gl_sync_meta_wait (sync_meta, mem->context);
346 gst_buffer_unref(sync_buffer);
349 textures.count = frame.info.finfo->n_planes;
351 for (uint i = 0; i < textures.count; ++i)
352 textures.names[i] = *(guint32 *)frame.data[i];
354 gst_video_frame_unmap(&frame);
359# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
360static GlTextures mapFromDmaBuffer(QRhi *rhi,
const QGstBufferHandle &bufferHandle,
361 const QGstVideoInfo &videoInfo, Qt::HANDLE eglDisplay,
362 QFunctionPointer eglImageTargetTexture2D)
364 qCDebug(qLcGstVideoBuffer) <<
"Importing textures from DMA buffer";
365 logGlAndEglErrors(
"mapFromDmaBuffer");
367 GstBuffer *buffer = bufferHandle.get();
369 Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0)));
370 Q_ASSERT(eglDisplay);
371 Q_ASSERT(eglImageTargetTexture2D);
373 Q_ASSERT(rhi->backend() == QRhi::OpenGLES2);
375 auto *nativeHandles =
static_cast<
const QRhiGles2NativeHandles *>(rhi->nativeHandles());
376 auto glContext = nativeHandles->context;
378 qCWarning(qLcGstVideoBuffer) <<
"no GL context";
382 const GstVideoInfo &gstVideoInfo = videoInfo.gstVideoInfo;
383 if (!gstVideoInfo.finfo) {
384 qCWarning(qLcGstVideoBuffer) <<
"Missing valid GstVideoInfo for DMABuf GstBuffer";
388 if (videoInfo.dmaDrmModifier && *videoInfo.dmaDrmModifier != 0) {
389 qCWarning(qLcGstVideoBuffer) <<
"Unsupported non-linear DMABuf modifier:"
390 << Qt::hex << *videoInfo.dmaDrmModifier;
394 const GstVideoMeta *videoMeta = gst_buffer_get_video_meta(buffer);
395 const GstVideoFormat videoInfoFormat = GST_VIDEO_INFO_FORMAT(&gstVideoInfo);
396 GstVideoFormat format = videoMeta ? videoMeta->format : videoInfoFormat;
397 if (format == GST_VIDEO_FORMAT_UNKNOWN)
398 format = videoInfoFormat;
400 const int nPlanes = videoMeta ? videoMeta->n_planes : GST_VIDEO_INFO_N_PLANES(&gstVideoInfo);
401 const int nMemoryBlocks = gst_buffer_n_memory(buffer);
402 static const bool externalOes = QVideoTextureHelper::forceGlTextureExternalOesIsSet();
403 static const bool singleEGLImage =
404 externalOes || qEnvironmentVariableIsSet(
"QT_GSTREAMER_FORCE_SINGLE_EGLIMAGE");
406 qCDebug(qLcGstVideoBuffer) <<
"format:" << gst_video_format_to_string(format)
407 <<
"nPlanes:" << nPlanes
408 <<
"nMemoryBlocks:" << nMemoryBlocks
409 <<
"externalOes:" << externalOes
410 <<
"singleEGLImage:" << singleEGLImage;
412 constexpr int maxPlanes = 4;
413 Q_ASSERT(nPlanes >= 1
414 && nPlanes <= maxPlanes
415 && (nMemoryBlocks == 1 || nMemoryBlocks == nPlanes));
417 const int nEGLImages = singleEGLImage ? 1 : nPlanes;
418 std::array<EGLAttrib, maxPlanes> planeFourcc{};
419 for (
int plane = 0; plane < nEGLImages; ++plane) {
420 const int fourcc = fourccFromGstVideoFormat(format, plane, singleEGLImage);
422 qCWarning(qLcGstVideoBuffer) <<
"Unsupported format for DMABuf:"
423 << gst_video_format_to_string(format) <<
"plane:" << plane
424 <<
"singleEGLImage" << singleEGLImage;
427 planeFourcc[plane] = EGLAttrib(fourcc);
430 GlTextures textures = {};
431 textures.owned =
true;
432 textures.count = nEGLImages;
434 QOpenGLFunctions functions(glContext);
435 functions.glGenTextures(
int(textures.count), textures.names.data());
436 logGlAndEglErrors(
"glGenTextures");
438 std::array<
int, maxPlanes> fds{-1, -1, -1, -1};
439 for (
int i = 0; i < nMemoryBlocks && i < maxPlanes; ++i) {
440 fds[i] = gst_dmabuf_memory_get_fd(gst_buffer_peek_memory(buffer, i));
443 auto fdForPlane = [&](
int plane) -> EGLAttrib {
444 if (plane < 0 || plane >= maxPlanes || plane >= nMemoryBlocks)
446 return (fds[plane] >= 0) ? fds[plane] : fds[0];
449 auto compWidth = [&](
int plane) -> EGLAttrib {
450 return singleEGLImage ? GST_VIDEO_INFO_WIDTH(&gstVideoInfo)
451 : GST_VIDEO_INFO_COMP_WIDTH(&gstVideoInfo, plane);
454 auto compHeight = [&](
int plane) -> EGLAttrib {
455 return singleEGLImage ? GST_VIDEO_INFO_HEIGHT(&gstVideoInfo)
456 : GST_VIDEO_INFO_COMP_HEIGHT(&gstVideoInfo, plane);
459 auto planeOffset = [&](
int plane) -> EGLAttrib {
461 if (nPlanes == nMemoryBlocks)
464 return videoMeta->offset[plane];
465 return GST_VIDEO_INFO_PLANE_OFFSET(&gstVideoInfo, plane);
468 auto planeStride = [&](
int plane) -> EGLAttrib {
470 return videoMeta->stride[plane];
471 return GST_VIDEO_INFO_PLANE_STRIDE(&gstVideoInfo, plane);
474 for (
int plane = 0; plane < nEGLImages; ++plane) {
475 constexpr int maxAttrCount = 31;
476 std::array<EGLAttrib, maxAttrCount> attr;
479 const int width = compWidth(plane);
480 const int height = compHeight(plane);
482 attr[i++] = EGL_WIDTH;
484 attr[i++] = EGL_HEIGHT;
486 attr[i++] = EGL_LINUX_DRM_FOURCC_EXT;
487 attr[i++] = planeFourcc[plane];
489 attr[i++] = EGL_DMA_BUF_PLANE0_FD_EXT;
490 attr[i++] = fdForPlane(plane);
491 attr[i++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT;
492 attr[i++] = planeOffset(plane);
493 attr[i++] = EGL_DMA_BUF_PLANE0_PITCH_EXT;
494 attr[i++] = planeStride(plane);
496 if (singleEGLImage && nPlanes > 1) {
497 attr[i++] = EGL_DMA_BUF_PLANE1_FD_EXT;
498 attr[i++] = fdForPlane(1);
499 attr[i++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT;
500 attr[i++] = planeOffset(1);
501 attr[i++] = EGL_DMA_BUF_PLANE1_PITCH_EXT;
502 attr[i++] = planeStride(1);
505 if (singleEGLImage && nPlanes > 2) {
506 attr[i++] = EGL_DMA_BUF_PLANE2_FD_EXT;
507 attr[i++] = fdForPlane(2);
508 attr[i++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT;
509 attr[i++] = planeOffset(2);
510 attr[i++] = EGL_DMA_BUF_PLANE2_PITCH_EXT;
511 attr[i++] = planeStride(2);
514 if (singleEGLImage && nPlanes > 3) {
515 attr[i++] = EGL_DMA_BUF_PLANE3_FD_EXT;
516 attr[i++] = fdForPlane(3);
517 attr[i++] = EGL_DMA_BUF_PLANE3_OFFSET_EXT;
518 attr[i++] = planeOffset(3);
519 attr[i++] = EGL_DMA_BUF_PLANE3_PITCH_EXT;
520 attr[i++] = planeStride(3);
523 attr[i++] = EGL_NONE;
524 Q_ASSERT(i <= maxAttrCount);
526 EGLImage image = eglCreateImage(eglDisplay,
528 EGL_LINUX_DMA_BUF_EXT,
531 if (image == EGL_NO_IMAGE_KHR) {
532 qCWarning(qLcGstVideoBuffer) <<
"could not create EGL image for plane" << plane
533 <<
", EGL error 0x" << Qt::hex << eglGetError();
536 logGlAndEglErrors(
"eglCreateImage");
538 #ifdef GL_OES_EGL_image_external
539 GLenum target = externalOes ? GL_TEXTURE_EXTERNAL_OES : GL_TEXTURE_2D;
541 GLenum target = GL_TEXTURE_2D;
543 functions.glBindTexture(target, textures.names[plane]);
545 auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D;
546 EGLImageTargetTexture2D(target, image);
547 logGlAndEglErrors(
"glEGLImageTargetTexture2DOES");
549 eglDestroyImage(eglDisplay, image);
559#if QT_CONFIG(gstreamer_gl)
560 GlTextures textures = {};
561 if (m_memoryFormat == QGstCaps::GLTexture)
562 textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo.gstVideoInfo);
564# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(linux_dmabuf)
565 else if (m_memoryFormat == QGstCaps::DMABuf && m_eglDisplay && m_eglImageTargetTexture2D
566 && rhi.backend() == QRhi::OpenGLES2)
567 textures = mapFromDmaBuffer(&rhi, m_buffer, m_videoInfo, m_eglDisplay,
568 m_eglImageTargetTexture2D);
571 if (textures.count > 0)
572 return std::make_unique<QGstQVideoFrameTextures>(
573 &rhi, QSize{ m_videoInfo.gstVideoInfo.width, m_videoInfo.gstVideoInfo.height },
574 m_frameFormat.pixelFormat(), textures, m_memoryFormat);
QGstVideoBuffer(QGstBufferHandle buffer, const QGstVideoInfo &videoInfo, const QVideoFrameFormat &frameFormat)
~QGstVideoBuffer() override
bool isDmaBuf() const override
QVideoFrameTexturesUPtr mapTextures(QRhi &, QVideoFrameTexturesUPtr &) override
void unmap() override
Releases the memory mapped by the map() function.
MapData map(QVideoFrame::MapMode mode) override
Maps the planes of a video buffer to memory.
Combined button and popup list for selecting options.
#define fourcc_code(a, b, c, d)