fix overflow after 2Gb of output in ui/vnc-enc-tight.c When amount of compressed data is more than 2Gb we will hit integer overflow (or when it's unsigned, the border is 4Gb). We don't use zstream->total_out for anything, so instead of remembering its previois value and compare with hext one, we can just reset it on entry and use its resulting value as the amount of bytes to deal with. diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c index 2522936..e1843cb 100644 --- a/ui/vnc-enc-tight.c +++ b/ui/vnc-enc-tight.c @@ -849,7 +849,6 @@ static int tight_compress_data(VncState *vs, int stream_id, size_t bytes, int level, int strategy) { z_streamp zstream = &vs->tight.stream[stream_id]; - int previous_out; if (bytes < VNC_TIGHT_MIN_TO_COMPRESS) { vnc_write(vs, vs->tight.tight.buffer, vs->tight.tight.offset); @@ -869,7 +868,7 @@ static int tight_compress_data(VncState *vs, int stream_id, size_t bytes, zstream->next_out = vs->tight.zlib.buffer + vs->tight.zlib.offset; zstream->avail_out = vs->tight.zlib.capacity - vs->tight.zlib.offset; zstream->data_type = Z_BINARY; - previous_out = zstream->total_out; + zstream->total_out = 0; /* start encoding */ if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) { @@ -878,7 +877,7 @@ static int tight_compress_data(VncState *vs, int stream_id, size_t bytes, } vs->tight.zlib.offset = vs->tight.zlib.capacity - zstream->avail_out; - bytes = zstream->total_out - previous_out; + bytes = zstream->total_out; tight_send_compact_size(vs, bytes); vnc_write(vs, vs->tight.zlib.buffer, bytes);