7#include <QtCore/qiodevice.h>
8#include <QtCore/qcoreapplication.h>
14# include <brotli/decode.h>
25using namespace Qt::StringLiterals;
28struct ContentEncodingMapping
34constexpr ContentEncodingMapping contentEncodingMapping[] {
36 {
"zstd", QDecompressHelper::Zstandard },
39 {
"br", QDecompressHelper::Brotli },
47 for (
const auto &mapping : contentEncodingMapping) {
48 if (ce.compare(mapping.name, Qt::CaseInsensitive) == 0)
49 return mapping.encoding;
54z_stream *toZlibPointer(
void *ptr)
56 return static_cast<z_stream_s *>(ptr);
60BrotliDecoderState *toBrotliPointer(
void *ptr)
62 return static_cast<BrotliDecoderState *>(ptr);
67ZSTD_DStream *toZstandardPointer(
void *ptr)
69 return static_cast<ZSTD_DStream *>(ptr);
76 return encodingFromByteArray(encoding) != QDecompressHelper::None;
82 list.reserve(std::size(contentEncodingMapping));
83 for (
const auto &mapping : contentEncodingMapping) {
84 list << mapping.name.toByteArray();
98 qWarning(
"Encoding is already set.");
104 errorStr = QCoreApplication::translate(
"QHttp",
"Unsupported content encoding: %1")
105 .arg(QLatin1String(encoding));
108 errorStr = QString();
109 return setEncoding(ce);
114 Q_ASSERT(contentEncoding ==
None);
115 contentEncoding = ce;
116 switch (contentEncoding) {
122 z_stream *inflateStream =
new z_stream;
123 memset(inflateStream, 0,
sizeof(z_stream));
127 if (inflateInit2(inflateStream, MAX_WBITS + 32) != Z_OK) {
128 delete inflateStream;
129 inflateStream =
nullptr;
131 decoderPointer = inflateStream;
136 decoderPointer = BrotliDecoderCreateInstance(
nullptr,
nullptr,
nullptr);
143 decoderPointer = ZSTD_createDStream();
149 if (!decoderPointer) {
150 errorStr = QCoreApplication::translate(
"QHttp",
151 "Failed to initialize the compression decoder.");
159
160
161
162
163
164
165
168 return countDecompressed;
172
173
174
175
176
177
178
179
180
181
182
183
188 Q_ASSERT(compressedDataBuffer.byteAmount() == 0);
189 Q_ASSERT(contentEncoding ==
None);
190 countDecompressed = shouldCount;
194
195
196
197
198
199
200
201
202
203
204
205
208 Q_ASSERT(countDecompressed);
211 auto totalUncompressed =
212 countHelper && countHelper->totalUncompressedBytes > totalUncompressedBytes
213 ? countHelper->totalUncompressedBytes
214 : totalUncompressedBytes;
215 return totalUncompressed - totalBytesRead;
219
220
221
224 return feed(QByteArray(data));
228
229
230
231
232
233
234
235
238 Q_ASSERT(contentEncoding !=
None);
239 totalCompressedBytes += data.size();
240 compressedDataBuffer.append(std::move(data));
241 if (!countInternal(compressedDataBuffer[compressedDataBuffer.bufferCount() - 1])) {
242 if (errorStr.isEmpty() && countHelper)
243 errorStr = countHelper->errorString();
249
250
251
254 Q_ASSERT(contentEncoding !=
None);
255 totalCompressedBytes += buffer.byteAmount();
256 compressedDataBuffer.append(buffer);
257 if (!countInternal(buffer)) {
258 if (errorStr.isEmpty() && countHelper)
259 errorStr = countHelper->errorString();
265
266
267
270 Q_ASSERT(contentEncoding !=
None);
271 totalCompressedBytes += buffer.byteAmount();
272 const QByteDataBuffer copy(buffer);
273 compressedDataBuffer.append(std::move(buffer));
274 if (!countInternal(copy)) {
275 if (errorStr.isEmpty() && countHelper)
276 errorStr = countHelper->errorString();
282
283
284
285
286
287
288
289
290
291
292
293
294
297 Q_ASSERT(countDecompressed);
298 while (hasDataInternal()
299 && decompressedDataBuffer.byteAmount() < MaxDecompressedDataBufferSize) {
300 const qsizetype toRead = 256 * 1024;
301 QByteArray buffer(toRead, Qt::Uninitialized);
302 qsizetype bytesRead = readInternal(buffer.data(), buffer.size());
305 buffer.truncate(bytesRead);
306 decompressedDataBuffer.append(std::move(buffer));
308 if (!hasDataInternal())
311 while (countHelper->hasData()) {
312 std::array<
char, 1024> temp;
313 qsizetype bytesRead = countHelper->read(temp.data(), temp.size());
321
322
323
326 if (countDecompressed) {
328 countHelper = std::make_unique<QDecompressHelper>();
329 countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold);
330 countHelper->setEncoding(contentEncoding);
332 countHelper->feed(data);
333 return countInternal();
339
340
341
344 if (countDecompressed) {
346 countHelper = std::make_unique<QDecompressHelper>();
347 countHelper->setDecompressedSafetyCheckThreshold(archiveBombCheckThreshold);
348 countHelper->setEncoding(contentEncoding);
350 countHelper->feed(buffer);
351 return countInternal();
367 qsizetype cachedRead = 0;
368 if (!decompressedDataBuffer.isEmpty()) {
369 cachedRead = decompressedDataBuffer.read(data, maxSize);
371 maxSize -= cachedRead;
374 qsizetype bytesRead = readInternal(data, maxSize);
377 totalBytesRead += bytesRead + cachedRead;
378 return bytesRead + cachedRead;
382
383
384
385
393 if (!hasDataInternal())
396 qsizetype bytesRead = -1;
397 switch (contentEncoding) {
403 bytesRead = readZLib(data, maxSize);
406 bytesRead = readBrotli(data, maxSize);
409 bytesRead = readZstandard(data, maxSize);
415 totalUncompressedBytes += bytesRead;
416 if (isPotentialArchiveBomb()) {
417 errorStr = QCoreApplication::translate(
419 "The decompressed output exceeds the limits specified by "
420 "QNetworkRequest::decompressedSafetyCheckThreshold()");
428
429
430
431
432
436 threshold = std::numeric_limits<qint64>::max();
437 archiveBombCheckThreshold = threshold;
442 if (totalCompressedBytes == 0)
445 if (totalUncompressedBytes <= archiveBombCheckThreshold)
450 double ratio =
double(totalUncompressedBytes) /
double(totalCompressedBytes);
451 switch (contentEncoding) {
476
477
478
479
480
481
482
485 return hasDataInternal() || !decompressedDataBuffer.isEmpty();
489
490
491
492
495 return encodedBytesAvailable() || decoderHasData;
500 return compressedDataBuffer.byteAmount();
504
505
506
507
508
509
512 return contentEncoding !=
None;
516
517
518
519
520
528 switch (contentEncoding) {
533 z_stream *inflateStream = toZlibPointer(decoderPointer);
535 inflateEnd(inflateStream);
536 delete inflateStream;
541 BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
542 if (brotliDecoderState)
543 BrotliDecoderDestroyInstance(brotliDecoderState);
549 ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
551 ZSTD_freeDStream(zstdStream);
556 decoderPointer =
nullptr;
557 contentEncoding =
None;
559 compressedDataBuffer.clear();
560 decompressedDataBuffer.clear();
561 decoderHasData =
false;
564 brotliUnconsumedDataPtr =
nullptr;
565 brotliUnconsumedAmount = 0;
568 countDecompressed =
false;
571 totalUncompressedBytes = 0;
572 totalCompressedBytes = 0;
577 bool triedRawDeflate =
false;
579 z_stream *inflateStream = toZlibPointer(decoderPointer);
580 static const size_t zlibMaxSize =
581 size_t(
std::numeric_limits<
decltype(inflateStream->avail_in)>::max());
583 QByteArrayView input = compressedDataBuffer.readPointer();
584 if (size_t(input.size()) > zlibMaxSize)
585 input = input.sliced(zlibMaxSize);
587 inflateStream->avail_in = input.size();
588 inflateStream->next_in =
reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
590 bool bigMaxSize = (zlibMaxSize < size_t(maxSize));
591 qsizetype adjustedAvailableOut = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize;
592 inflateStream->avail_out = adjustedAvailableOut;
593 inflateStream->next_out =
reinterpret_cast<Bytef *>(data);
595 qsizetype bytesDecoded = 0;
597 auto previous_avail_out = inflateStream->avail_out;
598 int ret = inflate(inflateStream, Z_NO_FLUSH);
603 if (ret == Z_DATA_ERROR && !triedRawDeflate) { Q_UNLIKELY_BRANCH
604 inflateEnd(inflateStream);
605 triedRawDeflate =
true;
606 inflateStream->zalloc = Z_NULL;
607 inflateStream->zfree = Z_NULL;
608 inflateStream->opaque = Z_NULL;
609 inflateStream->avail_in = 0;
610 inflateStream->next_in = Z_NULL;
611 int ret = inflateInit2(inflateStream, -MAX_WBITS);
615 inflateStream->avail_in = input.size();
616 inflateStream->next_in =
617 reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
620 }
else if (ret < 0 || ret == Z_NEED_DICT) {
623 bytesDecoded += qsizetype(previous_avail_out - inflateStream->avail_out);
624 if (ret == Z_STREAM_END) {
628 if (inflateStream->avail_in != 0) {
629 inflateEnd(inflateStream);
630 Bytef *next_in = inflateStream->next_in;
631 uInt avail_in = inflateStream->avail_in;
632 inflateStream->zalloc = Z_NULL;
633 inflateStream->zfree = Z_NULL;
634 inflateStream->opaque = Z_NULL;
635 if (inflateInit2(inflateStream, MAX_WBITS + 32) != Z_OK) {
639 errorStr = QCoreApplication::translate(
640 "QHttp",
"Failed to initialize the compression decoder.");
643 inflateStream->next_in = next_in;
644 inflateStream->avail_in = avail_in;
649 compressedDataBuffer.advanceReadPointer(input.size());
654 if (bigMaxSize && inflateStream->avail_out == 0) {
657 bigMaxSize = (zlibMaxSize < size_t(maxSize - bytesDecoded));
658 inflateStream->avail_out = bigMaxSize ? qsizetype(zlibMaxSize) : maxSize - bytesDecoded;
659 inflateStream->next_out =
reinterpret_cast<Bytef *>(data + bytesDecoded);
662 if (inflateStream->avail_in == 0 && inflateStream->avail_out > 0) {
664 compressedDataBuffer.advanceReadPointer(input.size());
665 input = compressedDataBuffer.readPointer();
666 if (size_t(input.size()) > zlibMaxSize)
667 input = input.sliced(zlibMaxSize);
668 inflateStream->avail_in = input.size();
669 inflateStream->next_in =
670 reinterpret_cast<Bytef *>(
const_cast<
char *>(input.data()));
672 }
while (inflateStream->avail_out > 0 && inflateStream->avail_in > 0);
674 compressedDataBuffer.advanceReadPointer(input.size() - inflateStream->avail_in);
681#if !QT_CONFIG(brotli)
686 qint64 bytesDecoded = 0;
688 BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
689 if (!brotliDecoderState) {
692 compressedDataBuffer.clear();
696 while (decoderHasData && bytesDecoded < maxSize) {
697 Q_ASSERT(brotliUnconsumedDataPtr || BrotliDecoderHasMoreOutput(brotliDecoderState));
698 if (brotliUnconsumedDataPtr) {
699 Q_ASSERT(brotliUnconsumedAmount);
700 size_t toRead =
std::min(size_t(maxSize - bytesDecoded), brotliUnconsumedAmount);
701 memcpy(data + bytesDecoded, brotliUnconsumedDataPtr, toRead);
702 bytesDecoded += toRead;
703 brotliUnconsumedAmount -= toRead;
704 brotliUnconsumedDataPtr += toRead;
705 if (brotliUnconsumedAmount == 0) {
706 brotliUnconsumedDataPtr =
nullptr;
707 decoderHasData =
false;
710 if (BrotliDecoderHasMoreOutput(brotliDecoderState) == BROTLI_TRUE) {
711 brotliUnconsumedDataPtr =
712 BrotliDecoderTakeOutput(brotliDecoderState, &brotliUnconsumedAmount);
713 decoderHasData =
true;
716 if (bytesDecoded == maxSize)
718 Q_ASSERT(bytesDecoded < maxSize);
720 QByteArrayView input = compressedDataBuffer.readPointer();
721 const uint8_t *encodedPtr =
reinterpret_cast<
const uint8_t *>(input.data());
722 size_t encodedBytesRemaining = input.size();
724 uint8_t *decodedPtr =
reinterpret_cast<uint8_t *>(data + bytesDecoded);
725 size_t unusedDecodedSize = size_t(maxSize - bytesDecoded);
726 while (unusedDecodedSize > 0) {
727 auto previousUnusedDecodedSize = unusedDecodedSize;
728 BrotliDecoderResult result = BrotliDecoderDecompressStream(
729 brotliDecoderState, &encodedBytesRemaining, &encodedPtr, &unusedDecodedSize,
730 &decodedPtr,
nullptr);
731 bytesDecoded += previousUnusedDecodedSize - unusedDecodedSize;
735 case BROTLI_DECODER_RESULT_ERROR:
737 errorStr = QCoreApplication::translate(
"QHttp",
"Brotli error: %1")
738 .arg(QUtf8StringView{BrotliDecoderErrorString(
739 BrotliDecoderGetErrorCode(brotliDecoderState))});
741 case BROTLI_DECODER_RESULT_SUCCESS:
742 BrotliDecoderDestroyInstance(brotliDecoderState);
743 decoderPointer =
nullptr;
744 compressedDataBuffer.clear();
746 case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
747 compressedDataBuffer.advanceReadPointer(input.size());
748 input = compressedDataBuffer.readPointer();
749 if (!input.isEmpty()) {
750 encodedPtr =
reinterpret_cast<
const uint8_t *>(input.constData());
751 encodedBytesRemaining = input.size();
755 case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
757 decoderHasData = BrotliDecoderHasMoreOutput(brotliDecoderState);
758 Q_ASSERT(unusedDecodedSize == 0);
762 compressedDataBuffer.advanceReadPointer(input.size() - encodedBytesRemaining);
774 ZSTD_DStream *zstdStream = toZstandardPointer(decoderPointer);
776 QByteArrayView input = compressedDataBuffer.readPointer();
777 ZSTD_inBuffer inBuf { input.data(), size_t(input.size()), 0 };
779 ZSTD_outBuffer outBuf { data, size_t(maxSize), 0 };
781 qsizetype bytesDecoded = 0;
782 while (outBuf.pos < outBuf.size && (inBuf.pos < inBuf.size || decoderHasData)) {
783 size_t retValue = ZSTD_decompressStream(zstdStream, &outBuf, &inBuf);
784 if (ZSTD_isError(retValue)) { Q_UNLIKELY_BRANCH
785 errorStr = QCoreApplication::translate(
"QHttp",
"ZStandard error: %1")
786 .arg(QUtf8StringView{ZSTD_getErrorName(retValue)});
789 decoderHasData =
false;
790 bytesDecoded = outBuf.pos;
792 if (outBuf.pos == outBuf.size) {
793 decoderHasData =
true;
794 }
else if (inBuf.pos == inBuf.size) {
795 compressedDataBuffer.advanceReadPointer(input.size());
796 input = compressedDataBuffer.readPointer();
797 inBuf = { input.constData(), size_t(input.size()), 0 };
800 compressedDataBuffer.advanceReadPointer(inBuf.pos);
Q_NETWORK_EXPORT void setCountingBytesEnabled(bool shouldCount)
Q_NETWORK_EXPORT bool setEncoding(QByteArrayView contentEncoding)
Q_NETWORK_EXPORT void feed(const QByteArray &data)
Q_NETWORK_EXPORT bool isValid() const
Q_NETWORK_EXPORT bool hasData() const
Q_NETWORK_EXPORT void setDecompressedSafetyCheckThreshold(qint64 threshold)
Q_NETWORK_EXPORT void feed(QByteArray &&data)
Q_NETWORK_EXPORT bool isCountingBytes() const
Q_NETWORK_EXPORT ~QDecompressHelper()
Q_NETWORK_EXPORT void clear()