6#include <QtCore/qiodevice.h>
7#include <QtCore/qcoreapplication.h>
13# include <brotli/decode.h>
32 {
"zstd", QDecompressHelper::Zstandard },
35 {
"br", QDecompressHelper::Brotli },
37 {
"gzip", QDecompressHelper::GZip },
38 {
"deflate", QDecompressHelper::Deflate },
43 for (
const auto &mapping : contentEncodingMapping) {
44 if (ce.compare(mapping.name, Qt::CaseInsensitive) == 0)
45 return mapping.encoding;
47 return QDecompressHelper::None;
52 return static_cast<z_stream_s *>(ptr);
56BrotliDecoderState *toBrotliPointer(
void *ptr)
58 return static_cast<BrotliDecoderState *>(ptr);
63ZSTD_DStream *toZstandardPointer(
void *ptr)
65 return static_cast<ZSTD_DStream *>(ptr);
70bool QDecompressHelper::isSupportedEncoding(QByteArrayView encoding)
72 return encodingFromByteArray(encoding) != QDecompressHelper::None;
75QByteArrayList QDecompressHelper::acceptedEncoding()
78 list.reserve(std::size(contentEncodingMapping));
79 for (
const auto &mapping : contentEncodingMapping) {
80 list << mapping.name.toByteArray();
85QDecompressHelper::~QDecompressHelper()
90bool QDecompressHelper::setEncoding(QByteArrayView encoding)
92 Q_ASSERT(contentEncoding == QDecompressHelper::None);
93 if (contentEncoding != QDecompressHelper::None) {
94 qWarning(
"Encoding is already set.");
98 ContentEncoding ce = encodingFromByteArray(encoding);
100 errorStr = QCoreApplication::translate(
"QHttp",
"Unsupported content encoding: %1")
101 .arg(QLatin1String(encoding));
104 errorStr = QString();
105 return setEncoding(ce);
108bool QDecompressHelper::setEncoding(ContentEncoding ce)
110 Q_ASSERT(contentEncoding == None);
111 contentEncoding = ce;
112 switch (contentEncoding) {
118 z_stream *inflateStream =
new z_stream;
119 memset(inflateStream, 0,
sizeof(z_stream));
123 if (inflateInit2(inflateStream, MAX_WBITS + 32) != Z_OK) {
124 delete inflateStream;
125 inflateStream =
nullptr;
127 decoderPointer = inflateStream;
132 decoderPointer = BrotliDecoderCreateInstance(
nullptr,
nullptr,
nullptr);
139 decoderPointer = ZSTD_createDStream();
145 if (!decoderPointer) {
146 errorStr = QCoreApplication::translate(
"QHttp",
147 "Failed to initialize the compression decoder.");
148 contentEncoding = QDecompressHelper::None;
155
156
157
158
159
160
161
162bool QDecompressHelper::isCountingBytes()
const
164 return countDecompressed;
168
169
170
171
172
173
174
175
176
177
178
179
180void QDecompressHelper::setCountingBytesEnabled(
bool shouldCount)
184 Q_ASSERT(compressedDataBuffer.byteAmount() == 0);
185 Q_ASSERT(contentEncoding == None);
186 countDecompressed = shouldCount;
190
191
192
193
194
195
196
197
198
199
200
201
202qint64 QDecompressHelper::uncompressedSize()
const
204 Q_ASSERT(countDecompressed);
207 auto totalUncompressed =
208 countHelper && countHelper->totalUncompressedBytes > totalUncompressedBytes
209 ? countHelper->totalUncompressedBytes
210 : totalUncompressedBytes;
211 return totalUncompressed - totalBytesRead;
215
216
217
218void QDecompressHelper::feed(
const QByteArray &data)
220 return feed(QByteArray(data));
224
225
226
227
228
229
230
231
232void QDecompressHelper::feed(QByteArray &&data)
234 Q_ASSERT(contentEncoding != None);
235 totalCompressedBytes += data.size();
236 compressedDataBuffer.append(std::move(data));
237 if (!countInternal(compressedDataBuffer[compressedDataBuffer.bufferCount() - 1])) {
238 if (errorStr.isEmpty() && countHelper)
239 errorStr = countHelper->errorString();
245
246
247
248void QDecompressHelper::feed(
const QByteDataBuffer &buffer)
250 Q_ASSERT(contentEncoding != None);
251 totalCompressedBytes += buffer.byteAmount();
252 compressedDataBuffer.append(buffer);
253 if (!countInternal(buffer)) {
254 if (errorStr.isEmpty() && countHelper)
255 errorStr = countHelper->errorString();
261
262
263
264void QDecompressHelper::feed(QByteDataBuffer &&buffer)
266 Q_ASSERT(contentEncoding != None);
267 totalCompressedBytes += buffer.byteAmount();
268 const QByteDataBuffer copy(buffer);
269 compressedDataBuffer.append(std::move(buffer));
270 if (!countInternal(copy)) {
271 if (errorStr.isEmpty() && countHelper)
272 errorStr = countHelper->errorString();
278
279
280
281
282
283
284
285
286
287
288
289
290
291bool QDecompressHelper::countInternal()
293 Q_ASSERT(countDecompressed);
294 while (hasDataInternal()
295 && decompressedDataBuffer.byteAmount() < MaxDecompressedDataBufferSize) {
296 const qsizetype toRead = 256 * 1024;
297 QByteArray buffer(toRead, Qt::Uninitialized);
298 qsizetype bytesRead = readInternal(buffer.data(), buffer.size());
301 buffer.truncate(bytesRead);
302 decompressedDataBuffer.append(std::move(buffer));
304 if (!hasDataInternal())
307 while (countHelper->hasData()) {
308 std::array<
char, 1024> temp;
309 qsizetype bytesRead = countHelper->read(temp.data(), temp.size());
317
318
319
320bool QDecompressHelper::countInternal(
const QByteArray &data)
322 if (countDecompressed) {
324 countHelper = std::make_unique<QDecompressHelper>();
325 countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold);
326 countHelper->setEncoding(contentEncoding);
328 countHelper->feed(data);
329 return countInternal();
335
336
337
338bool QDecompressHelper::countInternal(
const QByteDataBuffer &buffer)
340 if (countDecompressed) {
342 countHelper = std::make_unique<QDecompressHelper>();
343 countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold);
344 countHelper->setEncoding(contentEncoding);
346 countHelper->feed(buffer);
347 return countInternal();
352qsizetype QDecompressHelper::read(
char *data, qsizetype maxSize)
363 qsizetype cachedRead = 0;
364 if (!decompressedDataBuffer.isEmpty()) {
365 cachedRead = decompressedDataBuffer.read(data, maxSize);
367 maxSize -= cachedRead;
370 qsizetype bytesRead = readInternal(data, maxSize);
373 totalBytesRead += bytesRead + cachedRead;
374 return bytesRead + cachedRead;
378
379
380
381
382qsizetype QDecompressHelper::readInternal(
char *data, qsizetype maxSize)
389 if (!hasDataInternal())
392 qsizetype bytesRead = -1;
393 switch (contentEncoding) {
399 bytesRead = readZLib(data, maxSize);
402 bytesRead = readBrotli(data, maxSize);
405 bytesRead = readZstandard(data, maxSize);
411 totalUncompressedBytes += bytesRead;
412 if (isPotentialArchiveBomb()) {
413 errorStr = QCoreApplication::translate(
415 "The decompressed output exceeds the limits specified by "
416 "QNetworkRequest::decompressedSafetyCheckThreshold()");
424
425
426
427
428
429void QDecompressHelper::setDecompressedSafetyCheckThreshold(qint64 threshold)
432 threshold = std::numeric_limits<qint64>::max();
433 archiveBombCheckThreshold = threshold;
436bool QDecompressHelper::isPotentialArchiveBomb()
const
438 if (totalCompressedBytes == 0)
441 if (totalUncompressedBytes <= archiveBombCheckThreshold)
446 double ratio =
double(totalUncompressedBytes) /
double(totalCompressedBytes);
447 switch (contentEncoding) {
472
473
474
475
476
477
478
479bool QDecompressHelper::hasData()
const
481 return hasDataInternal() || !decompressedDataBuffer.isEmpty();
485
486
487
488
489bool QDecompressHelper::hasDataInternal()
const
491 return encodedBytesAvailable() || decoderHasData;
494qint64 QDecompressHelper::encodedBytesAvailable()
const
496 return compressedDataBuffer.byteAmount();
500
501
502
503
504
505
506bool QDecompressHelper::isValid()
const
508 return contentEncoding != None;
512
513
514
515
516
517QString QDecompressHelper::errorString()
const
522void QDecompressHelper::clear()
524 switch (contentEncoding) {
529 z_stream *inflateStream = toZlibPointer(decoderPointer);
531 inflateEnd(inflateStream);
532 delete inflateStream;
537 BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
538 if (brotliDecoderState)
539 BrotliDecoderDestroyInstance(brotliDecoderState);
545 ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
547 ZSTD_freeDStream(zstdStream);
552 decoderPointer =
nullptr;
553 contentEncoding = None;
555 compressedDataBuffer.clear();
556 decompressedDataBuffer.clear();
557 decoderHasData =
false;
559 countDecompressed =
false;
562 totalUncompressedBytes = 0;
563 totalCompressedBytes = 0;
566qsizetype QDecompressHelper::readZLib(
char *data,
const qsizetype maxSize)
568 bool triedRawDeflate =
false;
570 z_stream *inflateStream = toZlibPointer(decoderPointer);
571 static const size_t zlibMaxSize =
572 size_t(std::numeric_limits<
decltype(inflateStream->avail_in)>::max());
574 QByteArrayView input = compressedDataBuffer.readPointer();
575 if (size_t(input.size()) > zlibMaxSize)
576 input = input.sliced(zlibMaxSize);
578 inflateStream->avail_in = input.size();
579 inflateStream->next_in =
reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
581 bool bigMaxSize = (zlibMaxSize < size_t(maxSize));
582 qsizetype adjustedAvailableOut = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize;
583 inflateStream->avail_out = adjustedAvailableOut;
584 inflateStream->next_out =
reinterpret_cast<Bytef *>(data);
586 qsizetype bytesDecoded = 0;
588 auto previous_avail_out = inflateStream->avail_out;
589 int ret = inflate(inflateStream, Z_NO_FLUSH);
594 if (ret == Z_DATA_ERROR && !triedRawDeflate) Q_UNLIKELY_BRANCH {
595 inflateEnd(inflateStream);
596 triedRawDeflate =
true;
597 inflateStream->zalloc = Z_NULL;
598 inflateStream->zfree = Z_NULL;
599 inflateStream->opaque = Z_NULL;
600 inflateStream->avail_in = 0;
601 inflateStream->next_in = Z_NULL;
602 int ret = inflateInit2(inflateStream, -MAX_WBITS);
606 inflateStream->avail_in = input.size();
607 inflateStream->next_in =
608 reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
611 }
else if (ret < 0 || ret == Z_NEED_DICT) {
614 bytesDecoded += qsizetype(previous_avail_out - inflateStream->avail_out);
615 if (ret == Z_STREAM_END) {
619 if (inflateStream->avail_in != 0) {
620 inflateEnd(inflateStream);
621 Bytef *next_in = inflateStream->next_in;
622 uInt avail_in = inflateStream->avail_in;
623 inflateStream->zalloc = Z_NULL;
624 inflateStream->zfree = Z_NULL;
625 inflateStream->opaque = Z_NULL;
626 if (inflateInit2(inflateStream, MAX_WBITS + 32) != Z_OK) {
627 delete inflateStream;
628 decoderPointer =
nullptr;
630 compressedDataBuffer.advanceReadPointer(input.size() - avail_in);
633 inflateStream->next_in = next_in;
634 inflateStream->avail_in = avail_in;
639 compressedDataBuffer.advanceReadPointer(input.size());
644 if (bigMaxSize && inflateStream->avail_out == 0) {
647 bigMaxSize = (zlibMaxSize < size_t(maxSize - bytesDecoded));
648 inflateStream->avail_out = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize - bytesDecoded;
649 inflateStream->next_out =
reinterpret_cast<Bytef *>(data + bytesDecoded);
652 if (inflateStream->avail_in == 0 && inflateStream->avail_out > 0) {
654 compressedDataBuffer.advanceReadPointer(input.size());
655 input = compressedDataBuffer.readPointer();
656 if (size_t(input.size()) > zlibMaxSize)
657 input = input.sliced(zlibMaxSize);
658 inflateStream->avail_in = input.size();
659 inflateStream->next_in =
660 reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
662 }
while (inflateStream->avail_out > 0 && inflateStream->avail_in > 0);
664 compressedDataBuffer.advanceReadPointer(input.size() - inflateStream->avail_in);
669qsizetype QDecompressHelper::readBrotli(
char *data,
const qsizetype maxSize)
671#if !QT_CONFIG(brotli)
676 qint64 bytesDecoded = 0;
678 BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
680 while (decoderHasData && bytesDecoded < maxSize) {
681 Q_ASSERT(brotliUnconsumedDataPtr || BrotliDecoderHasMoreOutput(brotliDecoderState));
682 if (brotliUnconsumedDataPtr) {
683 Q_ASSERT(brotliUnconsumedAmount);
684 size_t toRead = std::min(size_t(maxSize - bytesDecoded), brotliUnconsumedAmount);
685 memcpy(data + bytesDecoded, brotliUnconsumedDataPtr, toRead);
686 bytesDecoded += toRead;
687 brotliUnconsumedAmount -= toRead;
688 brotliUnconsumedDataPtr += toRead;
689 if (brotliUnconsumedAmount == 0) {
690 brotliUnconsumedDataPtr =
nullptr;
691 decoderHasData =
false;
694 if (BrotliDecoderHasMoreOutput(brotliDecoderState) == BROTLI_TRUE) {
695 brotliUnconsumedDataPtr =
696 BrotliDecoderTakeOutput(brotliDecoderState, &brotliUnconsumedAmount);
697 decoderHasData =
true;
700 if (bytesDecoded == maxSize)
702 Q_ASSERT(bytesDecoded < maxSize);
704 QByteArrayView input = compressedDataBuffer.readPointer();
705 const uint8_t *encodedPtr =
reinterpret_cast<
const uint8_t *>(input.data());
706 size_t encodedBytesRemaining = input.size();
708 uint8_t *decodedPtr =
reinterpret_cast<uint8_t *>(data + bytesDecoded);
709 size_t unusedDecodedSize = size_t(maxSize - bytesDecoded);
710 while (unusedDecodedSize > 0) {
711 auto previousUnusedDecodedSize = unusedDecodedSize;
712 BrotliDecoderResult result = BrotliDecoderDecompressStream(
713 brotliDecoderState, &encodedBytesRemaining, &encodedPtr, &unusedDecodedSize,
714 &decodedPtr,
nullptr);
715 bytesDecoded += previousUnusedDecodedSize - unusedDecodedSize;
719 case BROTLI_DECODER_RESULT_ERROR:
720 errorStr = QLatin1String(
"Brotli error: %1")
721 .arg(QString::fromUtf8(BrotliDecoderErrorString(
722 BrotliDecoderGetErrorCode(brotliDecoderState))));
724 case BROTLI_DECODER_RESULT_SUCCESS:
725 BrotliDecoderDestroyInstance(brotliDecoderState);
726 decoderPointer =
nullptr;
727 compressedDataBuffer.clear();
729 case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
730 compressedDataBuffer.advanceReadPointer(input.size());
731 input = compressedDataBuffer.readPointer();
732 if (!input.isEmpty()) {
733 encodedPtr =
reinterpret_cast<
const uint8_t *>(input.constData());
734 encodedBytesRemaining = input.size();
738 case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
740 decoderHasData = BrotliDecoderHasMoreOutput(brotliDecoderState);
741 Q_ASSERT(unusedDecodedSize == 0);
745 compressedDataBuffer.advanceReadPointer(input.size() - encodedBytesRemaining);
750qsizetype QDecompressHelper::readZstandard(
char *data,
const qsizetype maxSize)
757 ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
759 QByteArrayView input = compressedDataBuffer.readPointer();
760 ZSTD_inBuffer inBuf { input.data(), size_t(input.size()), 0 };
762 ZSTD_outBuffer outBuf { data, size_t(maxSize), 0 };
764 qsizetype bytesDecoded = 0;
765 while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) {
766 size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf);
767 if (ZSTD_isError(retValue)) Q_UNLIKELY_BRANCH {
768 errorStr = QLatin1String(
"ZStandard error: %1")
769 .arg(QString::fromUtf8(ZSTD_getErrorName(retValue)));
772 decoderHasData =
false;
773 bytesDecoded = outBuf.pos;
775 if (outBuf.pos == outBuf.size) {
776 decoderHasData =
true;
777 }
else if (inBuf.pos == inBuf.size) {
778 compressedDataBuffer.advanceReadPointer(input.size());
779 input = compressedDataBuffer.readPointer();
780 inBuf = { input.constData(), size_t(input.size()), 0 };
783 compressedDataBuffer.advanceReadPointer(inBuf.pos);
constexpr ContentEncodingMapping contentEncodingMapping[]
QDecompressHelper::ContentEncoding encodingFromByteArray(QByteArrayView ce) noexcept
z_stream * toZlibPointer(void *ptr)