7#include <QtCore/qiodevice.h>
8#include <QtCore/qcoreapplication.h>
14# include <brotli/decode.h>
25using namespace Qt::StringLiterals;
28struct ContentEncodingMapping
34constexpr ContentEncodingMapping contentEncodingMapping[] {
36 {
"zstd", QDecompressHelper::Zstandard },
39 {
"br", QDecompressHelper::Brotli },
47 for (
const auto &mapping : contentEncodingMapping) {
48 if (ce.compare(mapping.name, Qt::CaseInsensitive) == 0)
49 return mapping.encoding;
54z_stream *toZlibPointer(
void *ptr)
56 return static_cast<z_stream_s *>(ptr);
60BrotliDecoderState *toBrotliPointer(
void *ptr)
62 return static_cast<BrotliDecoderState *>(ptr);
67ZSTD_DStream *toZstandardPointer(
void *ptr)
69 return static_cast<ZSTD_DStream *>(ptr);
76 return encodingFromByteArray(encoding) != QDecompressHelper::None;
82 list.reserve(std::size(contentEncodingMapping));
83 for (
const auto &mapping : contentEncodingMapping) {
84 list << mapping.name.toByteArray();
98 qWarning(
"Encoding is already set.");
104 errorStr = QCoreApplication::translate(
"QHttp",
"Unsupported content encoding: %1")
105 .arg(QLatin1String(encoding));
108 errorStr = QString();
109 return setEncoding(ce);
114 Q_ASSERT(contentEncoding ==
None);
115 contentEncoding = ce;
116 switch (contentEncoding) {
122 z_stream *inflateStream =
new z_stream;
123 memset(inflateStream, 0,
sizeof(z_stream));
127 if (inflateInit2(inflateStream, MAX_WBITS + 32) != Z_OK) {
128 delete inflateStream;
129 inflateStream =
nullptr;
131 decoderPointer = inflateStream;
136 decoderPointer = BrotliDecoderCreateInstance(
nullptr,
nullptr,
nullptr);
143 decoderPointer = ZSTD_createDStream();
149 if (!decoderPointer) {
150 errorStr = QCoreApplication::translate(
"QHttp",
151 "Failed to initialize the compression decoder.");
159
160
161
162
163
164
165
168 return countDecompressed;
172
173
174
175
176
177
178
179
180
181
182
183
188 Q_ASSERT(compressedDataBuffer.byteAmount() == 0);
189 Q_ASSERT(contentEncoding ==
None);
190 countDecompressed = shouldCount;
194
195
196
197
198
199
200
201
202
203
204
205
208 Q_ASSERT(countDecompressed);
211 auto totalUncompressed =
212 countHelper && countHelper->totalUncompressedBytes > totalUncompressedBytes
213 ? countHelper->totalUncompressedBytes
214 : totalUncompressedBytes;
215 return totalUncompressed - totalBytesRead;
219
220
221
224 return feed(QByteArray(data));
228
229
230
231
232
233
234
235
238 Q_ASSERT(contentEncoding !=
None);
239 totalCompressedBytes += data.size();
240 compressedDataBuffer.append(std::move(data));
241 if (!countInternal(compressedDataBuffer[compressedDataBuffer.bufferCount() - 1])) {
242 if (errorStr.isEmpty() && countHelper)
243 errorStr = countHelper->errorString();
249
250
251
254 Q_ASSERT(contentEncoding !=
None);
255 totalCompressedBytes += buffer.byteAmount();
256 compressedDataBuffer.append(buffer);
257 if (!countInternal(buffer)) {
258 if (errorStr.isEmpty() && countHelper)
259 errorStr = countHelper->errorString();
265
266
267
270 Q_ASSERT(contentEncoding !=
None);
271 totalCompressedBytes += buffer.byteAmount();
272 const QByteDataBuffer copy(buffer);
273 compressedDataBuffer.append(std::move(buffer));
274 if (!countInternal(copy)) {
275 if (errorStr.isEmpty() && countHelper)
276 errorStr = countHelper->errorString();
282
283
284
285
286
287
288
289
290
291
292
293
294
297 Q_ASSERT(countDecompressed);
298 while (hasDataInternal()
299 && decompressedDataBuffer.byteAmount() < MaxDecompressedDataBufferSize) {
300 const qsizetype toRead = 256 * 1024;
301 QByteArray buffer(toRead, Qt::Uninitialized);
302 qsizetype bytesRead = readInternal(buffer.data(), buffer.size());
305 buffer.truncate(bytesRead);
306 decompressedDataBuffer.append(std::move(buffer));
308 if (!hasDataInternal())
311 while (countHelper->hasData()) {
312 std::array<
char, 1024> temp;
313 qsizetype bytesRead = countHelper->read(temp.data(), temp.size());
321
322
323
326 if (countDecompressed) {
328 countHelper = std::make_unique<QDecompressHelper>();
329 countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold);
330 countHelper->setEncoding(contentEncoding);
332 countHelper->feed(data);
333 return countInternal();
339
340
341
344 if (countDecompressed) {
346 countHelper = std::make_unique<QDecompressHelper>();
347 countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold);
348 countHelper->setEncoding(contentEncoding);
350 countHelper->feed(buffer);
351 return countInternal();
367 qsizetype cachedRead = 0;
368 if (!decompressedDataBuffer.isEmpty()) {
369 cachedRead = decompressedDataBuffer.read(data, maxSize);
371 maxSize -= cachedRead;
374 qsizetype bytesRead = readInternal(data, maxSize);
377 totalBytesRead += bytesRead + cachedRead;
378 return bytesRead + cachedRead;
382
383
384
385
393 if (!hasDataInternal())
396 qsizetype bytesRead = -1;
397 switch (contentEncoding) {
403 bytesRead = readZLib(data, maxSize);
406 bytesRead = readBrotli(data, maxSize);
409 bytesRead = readZstandard(data, maxSize);
415 totalUncompressedBytes += bytesRead;
416 if (isPotentialArchiveBomb()) {
417 errorStr = QCoreApplication::translate(
419 "The decompressed output exceeds the limits specified by "
420 "QNetworkRequest::decompressedSafetyCheckThreshold()");
428
429
430
431
432
436 threshold = std::numeric_limits<qint64>::max();
437 archiveBombCheckThreshold = threshold;
442 if (totalCompressedBytes == 0)
445 if (totalUncompressedBytes <= archiveBombCheckThreshold)
450 double ratio =
double(totalUncompressedBytes) /
double(totalCompressedBytes);
451 switch (contentEncoding) {
476
477
478
479
480
481
482
485 return hasDataInternal() || !decompressedDataBuffer.isEmpty();
489
490
491
492
495 return encodedBytesAvailable() || decoderHasData;
500 return compressedDataBuffer.byteAmount();
504
505
506
507
508
509
512 return contentEncoding !=
None;
516
517
518
519
520
528 switch (contentEncoding) {
533 z_stream *inflateStream = toZlibPointer(decoderPointer);
535 inflateEnd(inflateStream);
536 delete inflateStream;
541 BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
542 if (brotliDecoderState)
543 BrotliDecoderDestroyInstance(brotliDecoderState);
549 ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
551 ZSTD_freeDStream(zstdStream);
556 decoderPointer =
nullptr;
557 contentEncoding =
None;
559 compressedDataBuffer.clear();
560 decompressedDataBuffer.clear();
561 decoderHasData =
false;
563 countDecompressed =
false;
566 totalUncompressedBytes = 0;
567 totalCompressedBytes = 0;
572 bool triedRawDeflate =
false;
574 z_stream *inflateStream = toZlibPointer(decoderPointer);
575 static const size_t zlibMaxSize =
576 size_t(
std::numeric_limits<
decltype(inflateStream->avail_in)>::max());
578 QByteArrayView input = compressedDataBuffer.readPointer();
579 if (size_t(input.size()) > zlibMaxSize)
580 input = input.sliced(zlibMaxSize);
582 inflateStream->avail_in = input.size();
583 inflateStream->next_in =
reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
585 bool bigMaxSize = (zlibMaxSize < size_t(maxSize));
586 qsizetype adjustedAvailableOut = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize;
587 inflateStream->avail_out = adjustedAvailableOut;
588 inflateStream->next_out =
reinterpret_cast<Bytef *>(data);
590 qsizetype bytesDecoded = 0;
592 auto previous_avail_out = inflateStream->avail_out;
593 int ret = inflate(inflateStream, Z_NO_FLUSH);
598 if (ret == Z_DATA_ERROR && !triedRawDeflate) { Q_UNLIKELY_BRANCH
599 inflateEnd(inflateStream);
600 triedRawDeflate =
true;
601 inflateStream->zalloc = Z_NULL;
602 inflateStream->zfree = Z_NULL;
603 inflateStream->opaque = Z_NULL;
604 inflateStream->avail_in = 0;
605 inflateStream->next_in = Z_NULL;
606 int ret = inflateInit2(inflateStream, -MAX_WBITS);
610 inflateStream->avail_in = input.size();
611 inflateStream->next_in =
612 reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
615 }
else if (ret < 0 || ret == Z_NEED_DICT) {
618 bytesDecoded += qsizetype(previous_avail_out - inflateStream->avail_out);
619 if (ret == Z_STREAM_END) {
623 if (inflateStream->avail_in != 0) {
624 inflateEnd(inflateStream);
625 Bytef *next_in = inflateStream->next_in;
626 uInt avail_in = inflateStream->avail_in;
627 inflateStream->zalloc = Z_NULL;
628 inflateStream->zfree = Z_NULL;
629 inflateStream->opaque = Z_NULL;
630 if (inflateInit2(inflateStream, MAX_WBITS + 32) != Z_OK) {
631 delete inflateStream;
632 decoderPointer =
nullptr;
634 compressedDataBuffer.advanceReadPointer(input.size() - avail_in);
637 inflateStream->next_in = next_in;
638 inflateStream->avail_in = avail_in;
643 compressedDataBuffer.advanceReadPointer(input.size());
648 if (bigMaxSize && inflateStream->avail_out == 0) {
651 bigMaxSize = (zlibMaxSize < size_t(maxSize - bytesDecoded));
652 inflateStream->avail_out = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize - bytesDecoded;
653 inflateStream->next_out =
reinterpret_cast<Bytef *>(data + bytesDecoded);
656 if (inflateStream->avail_in == 0 && inflateStream->avail_out > 0) {
658 compressedDataBuffer.advanceReadPointer(input.size());
659 input = compressedDataBuffer.readPointer();
660 if (size_t(input.size()) > zlibMaxSize)
661 input = input.sliced(zlibMaxSize);
662 inflateStream->avail_in = input.size();
663 inflateStream->next_in =
664 reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
666 }
while (inflateStream->avail_out > 0 && inflateStream->avail_in > 0);
668 compressedDataBuffer.advanceReadPointer(input.size() - inflateStream->avail_in);
675#if !QT_CONFIG(brotli)
680 qint64 bytesDecoded = 0;
682 BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
684 while (decoderHasData && bytesDecoded < maxSize) {
685 Q_ASSERT(brotliUnconsumedDataPtr || BrotliDecoderHasMoreOutput(brotliDecoderState));
686 if (brotliUnconsumedDataPtr) {
687 Q_ASSERT(brotliUnconsumedAmount);
688 size_t toRead = std::min(size_t(maxSize - bytesDecoded), brotliUnconsumedAmount);
689 memcpy(data + bytesDecoded, brotliUnconsumedDataPtr, toRead);
690 bytesDecoded += toRead;
691 brotliUnconsumedAmount -= toRead;
692 brotliUnconsumedDataPtr += toRead;
693 if (brotliUnconsumedAmount == 0) {
694 brotliUnconsumedDataPtr =
nullptr;
695 decoderHasData =
false;
698 if (BrotliDecoderHasMoreOutput(brotliDecoderState) == BROTLI_TRUE) {
699 brotliUnconsumedDataPtr =
700 BrotliDecoderTakeOutput(brotliDecoderState, &brotliUnconsumedAmount);
701 decoderHasData =
true;
704 if (bytesDecoded == maxSize)
706 Q_ASSERT(bytesDecoded < maxSize);
708 QByteArrayView input = compressedDataBuffer.readPointer();
709 const uint8_t *encodedPtr =
reinterpret_cast<
const uint8_t *>(input.data());
710 size_t encodedBytesRemaining = input.size();
712 uint8_t *decodedPtr =
reinterpret_cast<uint8_t *>(data + bytesDecoded);
713 size_t unusedDecodedSize = size_t(maxSize - bytesDecoded);
714 while (unusedDecodedSize > 0) {
715 auto previousUnusedDecodedSize = unusedDecodedSize;
716 BrotliDecoderResult result = BrotliDecoderDecompressStream(
717 brotliDecoderState, &encodedBytesRemaining, &encodedPtr, &unusedDecodedSize,
718 &decodedPtr,
nullptr);
719 bytesDecoded += previousUnusedDecodedSize - unusedDecodedSize;
723 case BROTLI_DECODER_RESULT_ERROR:
725 errorStr = QCoreApplication::translate(
"QHttp",
"Brotli error: %1")
726 .arg(QUtf8StringView{BrotliDecoderErrorString(
727 BrotliDecoderGetErrorCode(brotliDecoderState))});
729 case BROTLI_DECODER_RESULT_SUCCESS:
730 BrotliDecoderDestroyInstance(brotliDecoderState);
731 decoderPointer =
nullptr;
732 compressedDataBuffer.clear();
734 case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
735 compressedDataBuffer.advanceReadPointer(input.size());
736 input = compressedDataBuffer.readPointer();
737 if (!input.isEmpty()) {
738 encodedPtr =
reinterpret_cast<
const uint8_t *>(input.constData());
739 encodedBytesRemaining = input.size();
743 case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
745 decoderHasData = BrotliDecoderHasMoreOutput(brotliDecoderState);
746 Q_ASSERT(unusedDecodedSize == 0);
750 compressedDataBuffer.advanceReadPointer(input.size() - encodedBytesRemaining);
762 ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
764 QByteArrayView input = compressedDataBuffer.readPointer();
765 ZSTD_inBuffer inBuf { input.data(), size_t(input.size()), 0 };
767 ZSTD_outBuffer outBuf { data, size_t(maxSize), 0 };
769 qsizetype bytesDecoded = 0;
770 while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) {
771 size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf);
772 if (ZSTD_isError(retValue)) { Q_UNLIKELY_BRANCH
773 errorStr = QCoreApplication::translate(
"QHttp",
"ZStandard error: %1")
774 .arg(QUtf8StringView{ZSTD_getErrorName(retValue)});
777 decoderHasData =
false;
778 bytesDecoded = outBuf.pos;
780 if (outBuf.pos == outBuf.size) {
781 decoderHasData =
true;
782 }
else if (inBuf.pos == inBuf.size) {
783 compressedDataBuffer.advanceReadPointer(input.size());
784 input = compressedDataBuffer.readPointer();
785 inBuf = { input.constData(), size_t(input.size()), 0 };
788 compressedDataBuffer.advanceReadPointer(inBuf.pos);
Q_NETWORK_EXPORT void setCountingBytesEnabled(bool shouldCount)
Q_NETWORK_EXPORT bool setEncoding(QByteArrayView contentEncoding)
Q_NETWORK_EXPORT void feed(const QByteArray &data)
Q_NETWORK_EXPORT bool isValid() const
Q_NETWORK_EXPORT bool hasData() const
Q_NETWORK_EXPORT void setDecompressedSafetyCheckThreshold(qint64 threshold)
Q_NETWORK_EXPORT void feed(QByteArray &&data)
Q_NETWORK_EXPORT bool isCountingBytes() const
Q_NETWORK_EXPORT ~QDecompressHelper()
Q_NETWORK_EXPORT void clear()