11#include "libavutil/display.h"
20std::unique_ptr<VideoFrameEncoder>
22 const QSize &sourceSize,
24 qreal sourceFrameRate,
25 AVPixelFormat sourceFormat,
26 AVPixelFormat sourceSWFormat,
27 AVFormatContext *formatContext)
34 result->m_settings = encoderSettings;
35 result->m_sourceSize = sourceSize;
37 result->m_sourceRotation = sourceRotation;
42 if (!
result->m_settings.videoResolution().isValid())
43 result->m_settings.setVideoResolution(sourceSize);
45 if (
result->m_settings.videoFrameRate() <= 0.)
46 result->m_settings.setVideoFrameRate(sourceFrameRate);
49 || !
result->initCodecContext(formatContext)) {
60 result->updateConversions();
65bool VideoFrameEncoder::initCodec()
67 const auto qVideoCodec = m_settings.
videoCodec();
71 std::tie(m_codec, m_accel) =
findHwEncoder(codecID, resolution);
77 qWarning() <<
"Could not find encoder for codecId" << codecID;
81 qCDebug(qLcVideoFrameEncoder) <<
"found encoder" << m_codec->name <<
"for id" << m_codec->id;
85 if (strcmp(m_codec->name,
"h264_mf") == 0) {
86 auto makeEven = [](
int size) {
return size & ~1; };
87 const QSize fixedResolution(makeEven(resolution.width()), makeEven(resolution.height()));
88 if (fixedResolution != resolution) {
89 qCDebug(qLcVideoFrameEncoder) <<
"Fix odd video resolution for codec" << m_codec->name
90 <<
":" << resolution <<
"->" << fixedResolution;
97 if (resolution != fixedResolution) {
98 qCDebug(qLcVideoFrameEncoder) <<
"Fix odd video resolution for codec" << m_codec->name
99 <<
":" << resolution <<
"->" << fixedResolution;
105 for (
auto rate = m_codec->supported_framerates;
rate->num &&
rate->den; ++
rate)
106 qCDebug(qLcVideoFrameEncoder) <<
"supported frame rate:" << *
rate;
109 qCDebug(qLcVideoFrameEncoder) <<
"Adjusted frame rate:" << m_codecFrameRate;
114bool VideoFrameEncoder::initTargetFormats()
116 m_targetFormat =
findTargetFormat(m_sourceFormat, m_sourceSWFormat, m_codec, m_accel.get());
118 if (m_targetFormat == AV_PIX_FMT_NONE) {
119 qWarning() <<
"Could not find target format for codecId" << m_codec->id;
128 if (m_targetSWFormat == AV_PIX_FMT_NONE) {
129 qWarning() <<
"Cannot find software target format. sourceSWFormat:" << m_sourceSWFormat
130 <<
"targetFormat:" << m_targetFormat;
134 m_accel->createFramesContext(m_targetSWFormat, m_settings.
videoResolution());
135 if (!m_accel->hwFramesContextAsBuffer())
138 m_targetSWFormat = m_targetFormat;
146bool QFFmpeg::VideoFrameEncoder::initCodecContext(AVFormatContext *formatContext)
148 m_stream = avformat_new_stream(formatContext,
nullptr);
149 m_stream->id = formatContext->nb_streams - 1;
151 m_stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
152 m_stream->codecpar->codec_id = m_codec->id;
155 if (m_codec->id == AV_CODEC_ID_HEVC)
156 m_stream->codecpar->codec_tag = MKTAG(
'h',
'v',
'c',
'1');
158 const auto resolution = m_settings.videoResolution();
161 m_stream->codecpar->format = m_targetFormat;
162 m_stream->codecpar->width = resolution.width();
163 m_stream->codecpar->height = resolution.height();
164 m_stream->codecpar->sample_aspect_ratio = AVRational{ 1, 1 };
167 constexpr auto displayMatrixSize =
sizeof(int32_t) * 9;
168 AVPacketSideData sideData = {
reinterpret_cast<uint8_t *
>(av_malloc(displayMatrixSize)),
169 displayMatrixSize, AV_PKT_DATA_DISPLAYMATRIX };
170 av_display_rotation_set(
reinterpret_cast<int32_t *
>(sideData.data),
171 static_cast<double>(m_sourceRotation));
177 m_stream->time_base =
adjustFrameTimeBase(m_codec->supported_framerates, m_codecFrameRate);
178 m_codecContext.reset(avcodec_alloc_context3(m_codec));
179 if (!m_codecContext) {
180 qWarning() <<
"Could not allocate codec context";
184 avcodec_parameters_to_context(m_codecContext.get(), m_stream->codecpar);
185 m_codecContext->time_base = m_stream->time_base;
186 qCDebug(qLcVideoFrameEncoder) <<
"codecContext time base" << m_codecContext->time_base.num
187 << m_codecContext->time_base.den;
189 m_codecContext->framerate = m_codecFrameRate;
190 m_codecContext->pix_fmt = m_targetFormat;
191 m_codecContext->width = resolution.width();
192 m_codecContext->height = resolution.height();
195 auto deviceContext =
m_accel->hwDeviceContextAsBuffer();
197 m_codecContext->hw_device_ctx = av_buffer_ref(deviceContext);
199 if (
auto framesContext =
m_accel->hwFramesContextAsBuffer())
200 m_codecContext->hw_frames_ctx = av_buffer_ref(framesContext);
215 int res = avcodec_open2(m_codecContext.get(), m_codec, opts);
217 m_codecContext.reset();
221 qCDebug(qLcVideoFrameEncoder) <<
"video codec opened" <<
res <<
"time base"
222 << m_codecContext->time_base;
228 qint64 div = 1'000'000 * m_stream->time_base.num;
229 return div != 0 ? (us * m_stream->time_base.den + div / 2) / div : 0;
234 return m_stream->time_base;
239 if (!m_codecContext) {
240 qWarning() <<
"codec context is not initialized!";
241 return AVERROR(EINVAL);
245 return avcodec_send_frame(m_codecContext.get(),
frame.get());
247 if (
frame->format != m_sourceFormat) {
248 qWarning() <<
"Frame format has changed:" << m_sourceFormat <<
"->" <<
frame->format;
249 return AVERROR(EINVAL);
254 qCDebug(qLcVideoFrameEncoder) <<
"Update conversions on the fly. Source size"
261 AVRational timeBase = {};
264 if (m_downloadFromHW) {
267 int err = av_hwframe_transfer_data(
f.get(),
frame.get(), 0);
269 qCDebug(qLcVideoFrameEncoder) <<
"Error transferring frame data to surface." <<
err2str(err);
279 f->format = m_targetSWFormat;
283 av_frame_get_buffer(
f.get(), 0);
284 const auto scaledHeight = sws_scale(m_converter.get(),
frame->data,
frame->linesize, 0,
287 if (scaledHeight !=
f->height)
288 qCWarning(qLcVideoFrameEncoder) <<
"Scaled height" << scaledHeight <<
"!=" <<
f->height;
299 return AVERROR(ENOMEM);
302 qCDebug(qLcVideoFrameEncoder) <<
"Error getting HW buffer" <<
err2str(err);
305 qCDebug(qLcVideoFrameEncoder) <<
"got HW buffer";
307 if (!
f->hw_frames_ctx) {
308 qCDebug(qLcVideoFrameEncoder) <<
"no hw frames context";
309 return AVERROR(ENOMEM);
311 err = av_hwframe_transfer_data(
f.get(),
frame.get(), 0);
313 qCDebug(qLcVideoFrameEncoder) <<
"Error transferring frame data to surface." <<
err2str(err);
319 qCDebug(qLcVideoFrameEncoder) <<
"sending frame" << pts <<
"*" << timeBase;
322 return avcodec_send_frame(m_codecContext.get(),
frame.get());
330 auto getPacket = [&]() {
332 const int ret = avcodec_receive_packet(m_codecContext.get(), packet.get());
334 if (
ret != AVERROR(EOF) &&
ret != AVERROR(EAGAIN) &&
ret != AVERROR_EOF)
338 auto ts =
timeStampMs(packet->pts, m_stream->time_base);
341 <<
"got a packet" << packet->pts << packet->dts << (ts ? *ts : 0);
343 packet->stream_index = m_stream->id;
347 auto fixPacketDts = [&](AVPacket &packet) {
351 if (packet.dts == AV_NOPTS_VALUE)
354 packet.dts -= m_packetDtsOffset;
356 if (packet.pts != AV_NOPTS_VALUE && packet.pts < packet.dts) {
357 m_packetDtsOffset += packet.dts - packet.pts;
358 packet.dts = packet.pts;
360 if (m_prevPacketDts != AV_NOPTS_VALUE && packet.dts < m_prevPacketDts) {
362 <<
"Skip packet; failed to fix dts:" << packet.dts << m_prevPacketDts;
367 m_prevPacketDts = packet.dts;
372 while (
auto packet = getPacket()) {
373 if (fixPacketDts(*packet))
380void VideoFrameEncoder::updateConversions()
383 const bool zeroCopy = m_sourceFormat == m_targetFormat && !needToScale;
388 m_downloadFromHW =
false;
389 m_uploadToHW =
false;
391 qCDebug(qLcVideoFrameEncoder) <<
"zero copy encoding, format" << m_targetFormat;
396 m_downloadFromHW = m_sourceFormat != m_sourceSWFormat;
397 m_uploadToHW = m_targetFormat != m_targetSWFormat;
399 if (m_sourceSWFormat != m_targetSWFormat || needToScale) {
402 <<
"video source and encoder use different formats:" << m_sourceSWFormat
403 << m_targetSWFormat <<
"or sizes:" << m_sourceSize << targetSize;
405 m_converter.reset(sws_getContext(m_sourceSize.
width(), m_sourceSize.
height(),
406 m_sourceSWFormat, targetSize.width(), targetSize.height(),
407 m_targetSWFormat, SWS_FAST_BILINEAR,
nullptr,
nullptr,
411 qCDebug(qLcVideoFrameEncoder) <<
"VideoFrameEncoder conversions initialized:"
412 <<
"sourceFormat:" << m_sourceFormat
414 <<
"targetFormat:" << m_targetFormat
416 <<
"sourceSWFormat:" << m_sourceSWFormat
417 <<
"targetSWFormat:" << m_targetSWFormat
418 <<
"converter:" << m_converter.get();
const AVRational & getTimeBase() const
qint64 getPts(qint64 ms) const
AVPixelFormat sourceFormat() const
int sendFrame(AVFrameUPtr frame)
AVPacketUPtr retrievePacket()
static std::unique_ptr< VideoFrameEncoder > create(const QMediaEncoderSettings &encoderSettings, const QSize &sourceSize, QtVideo::Rotation sourceRotation, qreal sourceFrameRate, AVPixelFormat sourceFormat, AVPixelFormat sourceSWFormat, AVFormatContext *formatContext)
constexpr int height() const noexcept
Returns the height.
constexpr int width() const noexcept
Returns the width.
AVFrameUPtr makeAVFrame()
void getAVFrameTime(const AVFrame &frame, int64_t &pts, AVRational &timeBase)
AVPixelFormat findTargetFormat(AVPixelFormat sourceFormat, AVPixelFormat sourceSWFormat, const AVCodec *codec, const HWAccel *accel)
void applyVideoEncoderOptions(const QMediaEncoderSettings &settings, const QByteArray &codecName, AVCodecContext *codec, AVDictionary **opts)
bool isHwPixelFormat(AVPixelFormat format)
QSize adjustVideoResolution(const AVCodec *codec, QSize requestedResolution)
std::pair< const AVCodec *, std::unique_ptr< HWAccel > > findHwEncoder(AVCodecID codecID, const QSize &resolution)
AVPacketSideData * addStreamSideData(AVStream *stream, AVPacketSideData sideData)
QString err2str(int errnum)
void setAVFrameTime(AVFrame &frame, int64_t pts, const AVRational &timeBase)
bool isSwPixelFormat(AVPixelFormat format)
std::unique_ptr< AVFrame, AVDeleter< decltype(&av_frame_free), &av_frame_free > > AVFrameUPtr
const AVCodec * findSwEncoder(AVCodecID codecID, AVPixelFormat sourceSWFormat)
void applyExperimentalCodecOptions(const AVCodec *codec, AVDictionary **opts)
std::optional< qint64 > timeStampMs(qint64 ts, AVRational base)
AVRational adjustFrameRate(const AVRational *supportedRates, qreal requestedRate)
adjustFrameRate get a rational frame rate be requested qreal rate. If the codec supports fixed frame ...
AVPixelFormat findTargetSWFormat(AVPixelFormat sourceSWFormat, const AVCodec *codec, const HWAccel &accel)
std::unique_ptr< AVPacket, AVDeleter< decltype(&av_packet_free), &av_packet_free > > AVPacketUPtr
AVRational adjustFrameTimeBase(const AVRational *supportedRates, AVRational frameRate)
adjustFrameTimeBase gets adjusted timebase by a list of supported frame rates and an already adjusted...
Combined button and popup list for selecting options.
AVBufferRef * hwFramesContext
std::unique_ptr< QFFmpeg::HWAccel > m_accel
#define Q_LOGGING_CATEGORY(name,...)
#define qCWarning(category,...)
#define qCDebug(category,...)
GLenum GLuint GLintptr GLsizeiptr size
[1]
static constexpr QSize frameSize(const T &frame)