Qt
Internal/Contributor docs for the Qt SDK. Note: These are NOT official API docs; those are found at https://doc.qt.io/
Loading...
Searching...
No Matches
qgstreamervideosink.cpp
Go to the documentation of this file.
1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
3
4#include <common/qgstreamervideosink_p.h>
5#include <common/qgstvideorenderersink_p.h>
6#include <common/qgst_debug_p.h>
7#include <common/qgstutils_p.h>
8#include <rhi/qrhi.h>
9
10#include <QtCore/qdebug.h>
11#include <QtCore/qloggingcategory.h>
12
13#include <optional>
14#include <variant>
15
16#if QT_CONFIG(gstreamer_gl)
17# include <QtGui/qguiapplication.h>
18# include <QtGui/qguiapplication_platform.h>
19# include <QtGui/qopenglcontext.h>
20# include <QtGui/qwindow.h>
21# include <gst/gl/gstglconfig.h>
22# include <gst/gl/gstgldisplay.h>
23
24# if QT_CONFIG(gstreamer_gl_x11)
25# include <gst/gl/x11/gstgldisplay_x11.h>
26# endif
27# if QT_CONFIG(gstreamer_gl_egl)
28# include <common/qgstreameregldisplay_p.h>
29# include <QtMultimedia/private/qeglimagefunctions_p.h>
30# include <QtMultimedia/private/qmultimedia_gl_support_p.h>
31# include <gst/gl/egl/gstgldisplay_egl.h>
32# include <EGL/egl.h>
33# include <EGL/eglext.h>
34# endif
35# if QT_CONFIG(gstreamer_gl_wayland)
36# include <gst/gl/wayland/gstgldisplay_wayland.h>
37# endif
38#endif // #if QT_CONFIG(gstreamer_gl)
39
40QT_BEGIN_NAMESPACE
41
42Q_STATIC_LOGGING_CATEGORY(qLcGstVideoSink, "qt.multimedia.gstvideosink");
43
44QGstreamerPluggableVideoSink::QGstreamerPluggableVideoSink(QVideoSink *parent)
45 : QPlatformVideoSink{ parent }
46{
47}
48
49void QGstreamerPluggableVideoSink::setRhi(QRhi *rhi)
50{
51 if (m_rhi == rhi)
52 return;
53
54 m_rhi = rhi;
55 emit rhiChanged();
56}
57
58QRhi* QGstreamerPluggableVideoSink::rhi() const
59{
60 return m_rhi;
61}
62
63QGstreamerRelayVideoSink::QGstreamerRelayVideoSink(QObject *parent)
64 : QObject{
65 parent,
66 },
67 m_sinkBin{
68 QGstBin::create("videoSinkBin"),
69 }
70{
71 // This is a hack for some iMX and NVidia platforms. These require the use of a special video
72 // conversion element in the pipeline before the video sink, as they unfortunately
73 // output some proprietary format from the decoder even though it's sometimes marked as
74 // a regular supported video/x-raw format.
75 //
76 // To fix this, simply insert the element into the pipeline if it's available. Otherwise
77 // we simply use an identity element.
78 QGstElementFactoryHandle factory;
79
80 // QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT allows users to override the
81 // conversion element. Ideally we construct the element programatically, though.
82 QByteArray preprocessOverride = qgetenv("QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT");
83 if (!preprocessOverride.isEmpty()) {
84 qCDebug(qLcGstVideoSink) << "requesting conversion element from environment:"
85 << preprocessOverride;
86
87 m_gstPreprocess = QGstBin::createFromPipelineDescription(preprocessOverride, nullptr,
88 /*ghostUnlinkedPads=*/true);
89 if (!m_gstPreprocess)
90 qCWarning(qLcGstVideoSink) << "Cannot create conversion element:" << preprocessOverride;
91 }
92
93 if (!m_gstPreprocess) {
94 // This is a hack for some iMX and NVidia platforms. These require the use of a special
95 // video conversion element in the pipeline before the video sink, as they unfortunately
96 // output some proprietary format from the decoder even though it's sometimes marked as
97 // a regular supported video/x-raw format.
98 static constexpr auto decodersToTest = {
99 "imxvideoconvert_g2d",
100 "nvvidconv",
101 };
102
103 for (const char *decoder : decodersToTest) {
104 factory = QGstElement::findFactory(decoder);
105 if (factory)
106 break;
107 }
108
109 if (factory) {
110 qCDebug(qLcGstVideoSink)
111 << "instantiating conversion element:"
112 << g_type_name(gst_element_factory_get_element_type(factory.get()));
113
114 m_gstPreprocess = QGstElement::createFromFactory(factory, "preprocess");
115 }
116 }
117
118 bool disablePixelAspectRatio =
119 qEnvironmentVariableIntValue("QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO") != 0;
120 if (disablePixelAspectRatio) {
121 // Enabling the pixel aspect ratio may expose a gstreamer bug on cameras that don't expose a
122 // pixel-aspect-ratio via `VIDIOC_CROPCAP`. This can cause the caps negotiation to fail.
123 // Using the QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO environment variable, one can disable
124 // pixel-aspect-ratio handling
125 //
126 // compare: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6242
127 m_gstCapsFilter =
128 QGstElement::createFromFactory("identity", "nullPixelAspectRatioCapsFilter");
129 } else {
130 m_gstCapsFilter =
131 QGstElement::createFromFactory("capsfilter", "pixelAspectRatioCapsFilter");
132 QGstCaps capsFilterCaps{
133 gst_caps_new_simple("video/x-raw", "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL),
134 QGstCaps::HasRef,
135 };
136 g_object_set(m_gstCapsFilter.element(), "caps", capsFilterCaps.caps(), NULL);
137 }
138
139 if (m_gstPreprocess) {
140 m_sinkBin.add(m_gstPreprocess, m_gstCapsFilter);
141 qLinkGstElements(m_gstPreprocess, m_gstCapsFilter);
142 m_sinkBin.addGhostPad(m_gstPreprocess, "sink");
143 } else {
144 m_sinkBin.add(m_gstCapsFilter);
145 m_sinkBin.addGhostPad(m_gstCapsFilter, "sink");
146 }
147
148 updateGstContexts();
149}
150
151QGstreamerRelayVideoSink::~QGstreamerRelayVideoSink()
152{
153 emit aboutToBeDestroyed();
154
155 unrefGstContexts();
156}
157
158QGstElement QGstreamerRelayVideoSink::gstSink()
159{
160 if (!m_gstVideoSink) {
161 if (!m_gstQtSink)
162 createQtSink();
163
164 updateSinkElement(m_gstQtSink);
165 }
166
167 return m_sinkBin;
168}
169
170void QGstreamerRelayVideoSink::setActive(bool isActive)
171{
172 if (m_isActive == isActive)
173 return;
174 m_isActive = isActive;
175
176 if (m_gstQtSink)
177 m_gstQtSink.setActive(isActive);
178}
179
180void QGstreamerRelayVideoSink::setAsync(bool isAsync)
181{
182 m_sinkIsAsync = isAsync;
183}
184
185void QGstreamerRelayVideoSink::connectPluggableVideoSink(QGstreamerPluggableVideoSink *pluggableSink)
186{
187 Q_ASSERT(pluggableSink);
188 m_pluggableVideoSink = pluggableSink;
189 m_pluggableVideoSink->setVideoFrame(m_currentVideoFrame);
190 m_pluggableVideoSink->setSubtitleText(m_currentSubtitleText);
191 m_pluggableVideoSink->setNativeSize(m_currentNativeSize);
192 connect(this, &QGstreamerRelayVideoSink::videoFrameChanged,
193 m_pluggableVideoSink, &QPlatformVideoSink::setVideoFrame);
194 connect(this, &QGstreamerRelayVideoSink::subtitleTextChanged,
195 m_pluggableVideoSink, &QPlatformVideoSink::setSubtitleText);
196 connect(this, &QGstreamerRelayVideoSink::nativeSizeChanged,
197 m_pluggableVideoSink, &QPlatformVideoSink::setNativeSize);
198
199 // Update pipeline contexts using the rendering rhi
200 renderingRhiChanged(m_pluggableVideoSink->rhi());
201 m_rhiConnection = connect(m_pluggableVideoSink, &QPlatformVideoSink::rhiChanged, this, [this](){
202 renderingRhiChanged(m_pluggableVideoSink->rhi());
203 });
204}
205
206void QGstreamerRelayVideoSink::disconnectPluggableVideoSink()
207{
208 if (m_pluggableVideoSink) {
209 disconnect(this, &QGstreamerRelayVideoSink::videoFrameChanged,
210 m_pluggableVideoSink, &QPlatformVideoSink::setVideoFrame);
211 disconnect(this, &QGstreamerRelayVideoSink::subtitleTextChanged,
212 m_pluggableVideoSink, &QPlatformVideoSink::setSubtitleText);
213 disconnect(this, &QGstreamerRelayVideoSink::nativeSizeChanged,
214 m_pluggableVideoSink, &QPlatformVideoSink::setNativeSize);
215 disconnect(m_rhiConnection);
216 m_pluggableVideoSink = nullptr;
217 }
218}
219
220void QGstreamerRelayVideoSink::setVideoFrame(const QVideoFrame &frame)
221{
222 emit videoFrameChanged(frame);
223 m_currentVideoFrame = frame;
224}
225
226void QGstreamerRelayVideoSink::setSubtitleText(const QString &subtitleText)
227{
228 emit subtitleTextChanged(subtitleText);
229 m_currentSubtitleText = subtitleText;
230}
231
232void QGstreamerRelayVideoSink::setNativeSize(QSize size)
233{
234 emit nativeSizeChanged(size);
235 m_currentNativeSize = size;
236}
237
238void QGstreamerRelayVideoSink::createQtSink()
239{
240 Q_ASSERT(!m_gstQtSink);
241
242 m_gstQtSink = QGstVideoRendererSink::createSink(this);
243 if (!m_sinkIsAsync)
244 m_gstQtSink.set("async", false);
245 m_gstQtSink.setActive(m_isActive);
246}
247
248void QGstreamerRelayVideoSink::updateSinkElement(QGstVideoRendererSinkElement newSink)
249{
250 if (newSink == m_gstVideoSink)
251 return;
252
253 m_gstCapsFilter.src().modifyPipelineInIdleProbe([&] {
254 if (m_gstVideoSink)
255 m_sinkBin.stopAndRemoveElements(m_gstVideoSink);
256
257 m_gstVideoSink = std::move(newSink);
258 m_sinkBin.add(m_gstVideoSink);
259 qLinkGstElements(m_gstCapsFilter, m_gstVideoSink);
260 GstEvent *event = gst_event_new_reconfigure();
261 gst_element_send_event(m_gstVideoSink.element(), event);
262 m_gstVideoSink.syncStateWithParent();
263 });
264
265 m_sinkBin.dumpPipelineGraph("updateSinkElement");
266}
267
268void QGstreamerRelayVideoSink::unrefGstContexts()
269{
270 m_gstGlDisplayContext.reset();
271 m_gstGlLocalContext.reset();
272}
273
274#if QT_CONFIG(gstreamer_gl)
275
276namespace {
277
278using AnyNativeGLContextType = std::variant<
279# if QT_CONFIG(gstreamer_gl_egl) && QT_CONFIG(gstreamer_gl_x11) && QT_CONFIG(xcb_glx_plugin)
280 EGLContext, GLXContext
281# elif QT_CONFIG(gstreamer_gl_egl)
282 EGLContext
283# elif QT_CONFIG(gstreamer_gl_x11) && QT_CONFIG(xcb_glx_plugin)
284 GLXContext
285# else
286 void *
287# endif
288 >;
289
290std::optional<AnyNativeGLContextType>
291resolveNativeGlContext([[maybe_unused]] QOpenGLContext *glContext, GstGLPlatform glPlatform)
292{
293 switch (glPlatform) {
294# if QT_CONFIG(gstreamer_gl_egl)
295 case GST_GL_PLATFORM_EGL:
296 if (auto *eglContext = glContext->nativeInterface<QNativeInterface::QEGLContext>())
297 return eglContext->nativeContext();
298 return std::nullopt;
299# endif
300# if QT_CONFIG(gstreamer_gl_x11) && QT_CONFIG(xcb_glx_plugin)
301 case GST_GL_PLATFORM_GLX:
302 if (auto *glxContext = glContext->nativeInterface<QNativeInterface::QGLXContext>())
303 return glxContext->nativeContext();
304 return std::nullopt;
305# endif
306 default:
307 return std::nullopt;
308 }
309}
310
311struct ResolveGstGlDisplayResult
312{
313 QGstGLDisplayHandle display;
314 GstGLPlatform platform;
315};
316
317std::optional<ResolveGstGlDisplayResult> resolveGstGlDisplay([[maybe_unused]] QRhi &rhi)
318{
319 // use the egl display if we have one
320# if QT_CONFIG(gstreamer_gl_egl)
321 if (EGLDisplay eglDisplay = QtMultimediaPrivate::resolveEglDisplay(rhi)) {
322 QGstGLDisplayHandle gstGlDisplay{
323 GST_GL_DISPLAY_CAST(gst_gl_display_egl_new_with_egl_display(eglDisplay)),
324 QGstGLDisplayHandle::HasRef,
325 };
326 if (gstGlDisplay)
327 return ResolveGstGlDisplayResult{
328 std::move(gstGlDisplay),
329 GST_GL_PLATFORM_EGL,
330 };
331 }
332# endif
333
334# if QT_CONFIG(gstreamer_gl_wayland)
335 if (auto *waylandApp = qGuiApp->nativeInterface<QNativeInterface::QWaylandApplication>()) {
336 QGstGLDisplayHandle gstGlDisplay{
337 GST_GL_DISPLAY_CAST(gst_gl_display_wayland_new_with_display(waylandApp->display())),
338 QGstGLDisplayHandle::HasRef,
339 };
340 if (gstGlDisplay)
341 return ResolveGstGlDisplayResult{
342 std::move(gstGlDisplay),
343 GST_GL_PLATFORM_EGL,
344 };
345 }
346# endif
347
348# if QT_CONFIG(gstreamer_gl_x11) && QT_CONFIG(xcb_glx_plugin)
349 if (auto *x11App = qGuiApp->nativeInterface<QNativeInterface::QX11Application>()) {
350 QGstGLDisplayHandle gstGlDisplay{
351 GST_GL_DISPLAY_CAST(gst_gl_display_x11_new_with_display(x11App->display())),
352 QGstGLDisplayHandle::HasRef,
353 };
354 if (gstGlDisplay)
355 return ResolveGstGlDisplayResult{
356 std::move(gstGlDisplay),
357 GST_GL_PLATFORM_GLX,
358 };
359 }
360# endif
361
362 return std::nullopt;
363}
364
365QGstGLContextHandle resolveGstGLContextHandle(const QGstGLDisplayHandle &gstGlDisplay,
366 AnyNativeGLContextType nativeContext,
367 GstGLPlatform glPlatform)
368{
369 GstGLAPI glApi = QOpenGLContext::openGLModuleType() == QOpenGLContext::LibGL ? GST_GL_API_OPENGL
370 : GST_GL_API_GLES2;
371
372 guintptr nativeContextArgument = std::visit([](auto nativeContext) {
373 return guintptr(nativeContext);
374 }, nativeContext);
375
376 QGstGLContextHandle appContext{
377 gst_gl_context_new_wrapped(gstGlDisplay.get(), nativeContextArgument, glPlatform, glApi),
378 QGstGLContextHandle::HasRef,
379 };
380
381 return appContext;
382}
383
384} // namespace
385
386# endif // #if QT_CONFIG(gstreamer_gl)
387
388void QGstreamerRelayVideoSink::updateGstContexts()
389{
390 QRhi *currentRhi = rhi();
391
392 unrefGstContexts();
393
394#if QT_CONFIG(gstreamer_gl)
395 if (!currentRhi || currentRhi->backend() != QRhi::OpenGLES2)
396 return;
397 QOpenGLContext *glContext =
398 static_cast<const QRhiGles2NativeHandles *>(currentRhi->nativeHandles())->context;
399 Q_ASSERT(glContext);
400
401 std::optional resolvedDisplay = resolveGstGlDisplay(*currentRhi);
402 if (!resolvedDisplay) {
403 qWarning() << "Could not resolve GstGLDisplay";
404 return;
405 }
406
407 std::optional nativeContext = resolveNativeGlContext(glContext, resolvedDisplay->platform);
408 if (!nativeContext) {
409 qWarning() << "Could not find native gl context";
410 return;
411 }
412
413 QGstGLContextHandle appContext = resolveGstGLContextHandle(
414 resolvedDisplay->display, *nativeContext, resolvedDisplay->platform);
415 if (!appContext) {
416 qWarning() << "Could not create wrappped context for platform:"
417 << resolvedDisplay->platform;
418 return;
419 }
420
421 gst_gl_context_activate(appContext.get(), true);
422
423 QUniqueGErrorHandle error;
424 gst_gl_context_fill_info(appContext.get(), &error);
425 if (error) {
426 qWarning() << "Could not fill context info:" << error;
427 error = {};
428 }
429
430 QGstGLContextHandle displayContext;
431 gst_gl_display_create_context(resolvedDisplay->display.get(), appContext.get(), &displayContext,
432 &error);
433 if (error)
434 qWarning() << "Could not create display context:" << error;
435
436 appContext.reset();
437
438 m_gstGlDisplayContext = QGstContextHandle{
439 gst_context_new(GST_GL_DISPLAY_CONTEXT_TYPE, false),
440 QGstContextHandle::HasRef,
441 };
442 gst_context_set_gl_display(m_gstGlDisplayContext.get(), resolvedDisplay->display.get());
443
444 m_gstGlLocalContext = QGstContextHandle{
445 gst_context_new("gst.gl.local_context", false),
446 QGstContextHandle::HasRef,
447 };
448
449 GstStructure *structure = gst_context_writable_structure(m_gstGlLocalContext.get());
450 gst_structure_set(structure, "context", GST_TYPE_GL_CONTEXT, displayContext.get(), nullptr);
451 displayContext.reset();
452
453 // Note: after updating the context, we switch the sink and send gst_event_new_reconfigure()
454 // upstream. this will cause the context to be queried again.
455#endif // #if QT_CONFIG(gstreamer_gl)
456}
457
458void QGstreamerRelayVideoSink::renderingRhiChanged(QRhi *rhi)
459{
460 m_rhi = rhi;
461 updateGstContexts();
462}
463
464QT_END_NAMESPACE
465
466#include "moc_qgstreamervideosink_p.cpp"
QGstreamerRelayVideoSink::QGstreamerRelayVideoSink
QGstreamerRelayVideoSink(QObject *parent=nullptr)
Definition qgstreamervideosink.cpp:63
QGstreamerRelayVideoSink::setSubtitleText
void setSubtitleText(const QString &subtitleText)
Definition qgstreamervideosink.cpp:226
QGstreamerRelayVideoSink::connectPluggableVideoSink
void connectPluggableVideoSink(QGstreamerPluggableVideoSink *pluggableSink)
Definition qgstreamervideosink.cpp:185
QGstreamerRelayVideoSink::nativeSizeChanged
void nativeSizeChanged(QSize size)
Q_STATIC_LOGGING_CATEGORY
QT_BEGIN_NAMESPACE Q_STATIC_LOGGING_CATEGORY(lcSynthesizedIterableAccess, "qt.iterable.synthesized", QtWarningMsg)