Sara/behaviour-tree-test
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Packages Wiki Activity Actions

Merge pull request #38944 from Wavesonics/http-gzip

Browse source 
HttpRequest now handles gzipping response bodies
This commit is contained in:
Fabio Alessandrelli 2020-09-07 17:03:19 +02:00 • committed by GitHub
commit 2cb6b2ac6f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 272 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

87
core/io/compression.cpp
View file

@ -180,8 +180,95 @@ int Compression::decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p
ERR_FAIL_V(-1);
}
/**
This will handle both Gzip and Deflat streams. It will automatically allocate the output buffer into the provided p_dst_vect Vector.
This is required for compressed data who's final uncompressed size is unknown, as is the case for HTTP response bodies.
This is much slower however than using Compression::decompress because it may result in multiple full copies of the output buffer.
*/
int Compression::decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode) {
int ret;
uint8_t *dst = nullptr;
int out_mark = 0;
z_stream strm;
ERR_FAIL_COND_V(p_src_size <= 0, Z_DATA_ERROR);
// This function only supports GZip and Deflate
int window_bits = p_mode == MODE_DEFLATE ? 15 : 15 + 16;
ERR_FAIL_COND_V(p_mode != MODE_DEFLATE && p_mode != MODE_GZIP, Z_ERRNO);
// Initialize the stream
strm.zalloc = Z_NULL;
strm.zfree = Z_NULL;
strm.opaque = Z_NULL;
strm.avail_in = 0;
strm.next_in = Z_NULL;
int err = inflateInit2(&strm, window_bits);
ERR_FAIL_COND_V(err != Z_OK, -1);
// Setup the stream inputs
strm.next_in = (Bytef *)p_src;
strm.avail_in = p_src_size;
// Ensure the destination buffer is empty
p_dst_vect->resize(0);
// decompress until deflate stream ends or end of file
do {
// Add another chunk size to the output buffer
// This forces a copy of the whole buffer
p_dst_vect->resize(p_dst_vect->size() + gzip_chunk);
// Get pointer to the actual output buffer
dst = p_dst_vect->ptrw();
// Set the stream to the new output stream
// Since it was copied, we need to reset the stream to the new buffer
strm.next_out = &(dst[out_mark]);
strm.avail_out = gzip_chunk;
// run inflate() on input until output buffer is full and needs to be resized
// or input runs out
do {
ret = inflate(&strm, Z_SYNC_FLUSH);
switch (ret) {
case Z_NEED_DICT:
ret = Z_DATA_ERROR;
[[fallthrough]];
case Z_DATA_ERROR:
case Z_MEM_ERROR:
case Z_STREAM_ERROR:
WARN_PRINT(strm.msg);
(void)inflateEnd(&strm);
p_dst_vect->resize(0);
return ret;
}
} while (strm.avail_out > 0 && strm.avail_in > 0);
out_mark += gzip_chunk;
// Encorce max output size
if (p_max_dst_size > -1 && strm.total_out > (uint64_t)p_max_dst_size) {
(void)inflateEnd(&strm);
p_dst_vect->resize(0);
return Z_BUF_ERROR;
}
} while (ret != Z_STREAM_END);
// If all done successfully, resize the output if it's larger than the actual output
if (ret == Z_STREAM_END && (unsigned long)p_dst_vect->size() > strm.total_out) {
p_dst_vect->resize(strm.total_out);
}
// clean up and return
(void)inflateEnd(&strm);
return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
}
int Compression::zlib_level = Z_DEFAULT_COMPRESSION;
int Compression::gzip_level = Z_DEFAULT_COMPRESSION;
int Compression::zstd_level = 3;
bool Compression::zstd_long_distance_matching = false;
int Compression::zstd_window_log_size = 27; // ZSTD_WINDOWLOG_LIMIT_DEFAULT
int Compression::gzip_chunk = 16384;

3
core/io/compression.h
View file

@ -32,6 +32,7 @@
#define COMPRESSION_H
#include "core/typedefs.h"
#include "core/vector.h"
class Compression {
public:
@ -40,6 +41,7 @@ public:
static int zstd_level;
static bool zstd_long_distance_matching;
static int zstd_window_log_size;
static int gzip_chunk;
enum Mode {
MODE_FASTLZ,
@ -51,6 +53,7 @@ public:
static int compress(uint8_t *p_dst, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
static int get_max_compressed_buffer_size(int p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress(uint8_t *p_dst, int p_dst_max_size, const uint8_t *p_src, int p_src_size, Mode p_mode = MODE_ZSTD);
static int decompress_dynamic(Vector<uint8_t> *p_dst_vect, int p_max_dst_size, const uint8_t *p_src, int p_src_size, Mode p_mode);
Compression() {}
};

18
core/variant_call.cpp
View file

@ -712,6 +712,23 @@ struct _VariantCall {
r_ret = decompressed;
}
static void _call_PackedByteArray_decompress_dynamic(Variant &r_ret, Variant &p_self, const Variant **p_args) {
Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
PackedByteArray decompressed;
int max_output_size = (int)(*p_args[0]);
Compression::Mode mode = (Compression::Mode)(int)(*p_args[1]);
int result = Compression::decompress_dynamic(&decompressed, max_output_size, ba->array.ptr(), ba->array.size(), mode);
if (result == OK) {
r_ret = decompressed;
} else {
decompressed.clear();
r_ret = decompressed;
ERR_FAIL_MSG("Decompression failed.");
}
}
static void _call_PackedByteArray_hex_encode(Variant &r_ret, Variant &p_self, const Variant **p_args) {
Variant::PackedArrayRef<uint8_t> *ba = reinterpret_cast<Variant::PackedArrayRef<uint8_t> *>(p_self._data.packed_array);
if (ba->array.size() == 0) {
@ -2177,6 +2194,7 @@ void register_variant_methods() {
ADDFUNC0R(PACKED_BYTE_ARRAY, STRING, PackedByteArray, hex_encode, varray());
ADDFUNC1R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, compress, INT, "compression_mode", varray(0));
ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress, INT, "buffer_size", INT, "compression_mode", varray(0));
ADDFUNC2R(PACKED_BYTE_ARRAY, PACKED_BYTE_ARRAY, PackedByteArray, decompress_dynamic, INT, "max_output_size", INT, "compression_mode", varray(0));
ADDFUNC0R(PACKED_INT32_ARRAY, INT, PackedInt32Array, size, varray());
ADDFUNC0R(PACKED_INT32_ARRAY, BOOL, PackedInt32Array, empty, varray());