diff --git a/quicklz.pyx b/quicklz.pyx index d19ecad..8ed4ab7 100644 --- a/quicklz.pyx +++ b/quicklz.pyx @@ -1,5 +1,5 @@ __author__ = "davies " -__version__ = "1.4.1" +__version__ = "1.5.0" __copyright__ = "Copyright (C) 2010 douban.com" __license__ = "Apache License 2.0" @@ -8,9 +8,7 @@ from libc.stdlib cimport malloc, free from cpython cimport PyBytes_AsStringAndSize, PyBytes_FromStringAndSize cdef extern from "src/quicklz.h": - cdef enum: - QLZ_SCRATCH_COMPRESS - QLZ_SCRATCH_DECOMPRESS + int qlz_get_setting(int setting) nogil size_t qlz_size_decompressed(char *source) nogil size_t qlz_size_compressed(char *source) nogil size_t qlz_decompress(char *source, void *destination, char *scratch_decompress) nogil @@ -27,7 +25,7 @@ def compress(bytes val): PyBytes_AsStringAndSize(val, &src, &vlen) dst = malloc(vlen + 400) with nogil: - wbuf = malloc(QLZ_SCRATCH_COMPRESS) + wbuf = malloc(qlz_get_setting(1)) csize = qlz_compress(src, dst, vlen, wbuf) free(wbuf) @@ -36,7 +34,7 @@ def compress(bytes val): return val def decompress(bytes val): - cdef char wbuf[QLZ_SCRATCH_DECOMPRESS] + cdef char *wbuf cdef char *src cdef char *dst cdef Py_ssize_t slen, dlen @@ -47,7 +45,9 @@ def decompress(bytes val): raise ValueError('compressed length not match %d!=%d' % (slen, qlz_size_compressed(src))) dst = malloc(qlz_size_decompressed(src)) with nogil: + wbuf = malloc(qlz_get_setting(2)) dlen = qlz_decompress(src, dst, wbuf) + free(wbuf) val = PyBytes_FromStringAndSize(dst, dlen); free(dst) diff --git a/setup.py b/setup.py index 158cdb6..d2b4e8b 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, Extension setup( name='pyquicklz', - version='1.4.1', + version='1.5.0', description='python binding of quicklz', setup_requires=['Cython >= 0.18'], ext_modules=[ diff --git a/src/quicklz.c b/src/quicklz.c index 06fb068..839a28a 100644 --- a/src/quicklz.c +++ b/src/quicklz.c @@ -1,18 +1,18 @@ // Fast data compression library -// Copyright (C) 2006-2010 Lasse Mikkel Reinhold +// Copyright (C) 2006-2011 Lasse Mikkel Reinhold // lar@quicklz.com // -// QuickLZ can be used for free under the GPL-1, -2 or -3 license (where anything +// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything // released into public must be open source) or under a commercial license if such // has been acquired (see http://www.quicklz.com/order.html). The commercial license // does not cover derived or ported versions created by third parties under GPL. -// Version 1.4.1 final - april 2010 +// 1.5.0 final #include "quicklz.h" -#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 4 || QLZ_VERSION_REVISION != 1 -#error quicklz.c and quicklz.h have different versions +#if QLZ_VERSION_MAJOR != 1 || QLZ_VERSION_MINOR != 5 || QLZ_VERSION_REVISION != 0 + #error quicklz.c and quicklz.h have different versions #endif #if (defined(__X86__) || defined(__i386__) || defined(i386) || defined(_M_IX86) || defined(__386__) || defined(__x86_64__) || defined(_M_X64)) @@ -24,16 +24,21 @@ #define UNCOMPRESSED_END 4 #define CWORD_LEN 4 -typedef unsigned int ui32; -typedef unsigned short int ui16; +#if QLZ_COMPRESSION_LEVEL == 1 && defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 + #define OFFSET_BASE source + #define CAST (ui32)(size_t) +#else + #define OFFSET_BASE 0 + #define CAST +#endif int qlz_get_setting(int setting) { switch (setting) { case 0: return QLZ_COMPRESSION_LEVEL; - case 1: return QLZ_SCRATCH_COMPRESS; - case 2: return QLZ_SCRATCH_DECOMPRESS; + case 1: return sizeof(qlz_state_compress); + case 2: return sizeof(qlz_state_decompress); case 3: return QLZ_STREAMING_BUFFER; #ifdef QLZ_MEMORY_SAFE case 6: return 1; @@ -47,9 +52,39 @@ int qlz_get_setting(int setting) return -1; } -static void reset_state(unsigned char hash_counter[QLZ_HASH_VALUES]) +#if QLZ_COMPRESSION_LEVEL == 1 +static int same(const unsigned char *src, size_t n) +{ + while(n > 0 && *(src + n) == *src) + n--; + return n == 0 ? 1 : 0; +} +#endif + +static void reset_table_compress(qlz_state_compress *state) { - memset(hash_counter, 0, QLZ_HASH_VALUES); + int i; + for(i = 0; i < QLZ_HASH_VALUES; i++) + { +#if QLZ_COMPRESSION_LEVEL == 1 + state->hash[i].offset = 0; +#else + state->hash_counter[i] = 0; +#endif + } +} + +static void reset_table_decompress(qlz_state_decompress *state) +{ + int i; + (void)state; + (void)i; +#if QLZ_COMPRESSION_LEVEL == 2 + for(i = 0; i < QLZ_HASH_VALUES; i++) + { + state->hash_counter[i] = 0; + } +#endif } static __inline ui32 hash_func(ui32 i) @@ -85,6 +120,14 @@ static __inline ui32 fast_read(void const *src, ui32 bytes) #endif } +static __inline ui32 hashat(const unsigned char *src) +{ + ui32 fetch, hash; + fetch = fast_read(src, 3); + hash = hash_func(fetch); + return hash; +} + static __inline void fast_write(ui32 f, void *dst, size_t bytes) { #ifndef X86X64 @@ -130,6 +173,32 @@ static __inline void fast_write(ui32 f, void *dst, size_t bytes) #endif } + +size_t qlz_size_decompressed(const char *source) +{ + ui32 n, r; + n = (((*source) & 2) == 2) ? 4 : 1; + r = fast_read(source + 1 + n, n); + r = r & (0xffffffff >> ((4 - n)*8)); + return r; +} + +size_t qlz_size_compressed(const char *source) +{ + ui32 n, r; + n = (((*source) & 2) == 2) ? 4 : 1; + r = fast_read(source + 1, n); + r = r & (0xffffffff >> ((4 - n)*8)); + return r; +} + +size_t qlz_size_header(const char *source) +{ + size_t n = 2*((((*source) & 2) == 2) ? 4 : 1) + 1; + return n; +} + + static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui32 n) { // Caution if modifying memcpy_up! Overlap of dst and src must be special handled. @@ -150,41 +219,40 @@ static __inline void memcpy_up(unsigned char *dst, const unsigned char *src, ui3 } while (f < n); #endif - } - +} -#if QLZ_COMPRESSION_LEVEL <= 2 -static __inline void update_hash(qlz_hash_decompress h[QLZ_HASH_VALUES], unsigned char counter[QLZ_HASH_VALUES], const unsigned char *s) +static __inline void update_hash(qlz_state_decompress *state, const unsigned char *s) { #if QLZ_COMPRESSION_LEVEL == 1 - ui32 hash, fetch; - fetch = fast_read(s, 3); - hash = hash_func(fetch); - h[hash].offset[0] = s; - counter[hash] = 1; + ui32 hash; + hash = hashat(s); + state->hash[hash].offset = s; + state->hash_counter[hash] = 1; #elif QLZ_COMPRESSION_LEVEL == 2 - ui32 hash, fetch; + ui32 hash; unsigned char c; - fetch = fast_read(s, 3); - hash = hash_func(fetch); - c = counter[hash]; - h[hash].offset[c & (QLZ_POINTERS - 1)] = s; + hash = hashat(s); + c = state->hash_counter[hash]; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = s; c++; - counter[hash] = c; + state->hash_counter[hash] = c; #endif + (void)state; + (void)s; } -static void update_hash_upto(qlz_hash_decompress h[QLZ_HASH_VALUES], unsigned char counter[QLZ_HASH_VALUES], unsigned char **lh, const unsigned char *max) +#if QLZ_COMPRESSION_LEVEL <= 2 +static void update_hash_upto(qlz_state_decompress *state, unsigned char **lh, const unsigned char *max) { while(*lh < max) { (*lh)++; - update_hash(h, counter, *lh); + update_hash(state, *lh); } } #endif -static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_hash_compress hashtable[QLZ_HASH_VALUES], unsigned char hash_counter[QLZ_HASH_VALUES]) +static size_t qlz_compress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_compress *state) { const unsigned char *last_byte = source + size - 1; const unsigned char *src = source; @@ -205,7 +273,7 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest if ((cword_val & 1) == 1) { // store uncompressed if compression ratio is too low - if (src > source + 3*(size >> 2) && dst - destination > src - source - ((src - source) >> 5)) + if (src > source + (size >> 1) && dst - destination > src - source - ((src - source) >> 5)) return 0; fast_write((cword_val >> 1) | (1U << 31), cword_ptr, CWORD_LEN); @@ -221,22 +289,26 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest ui32 hash, cached; hash = hash_func(fetch); + cached = fetch ^ state->hash[hash].cache; + state->hash[hash].cache = fetch; - cached = fetch ^ hashtable[hash].cache[0]; - hashtable[hash].cache[0] = fetch; + o = state->hash[hash].offset + OFFSET_BASE; + state->hash[hash].offset = CAST(src - OFFSET_BASE); - o = hashtable[hash].offset[0]; - hashtable[hash].offset[0] = src; #ifdef X86X64 - if ((cached & 0xffffff) == 0 && hash_counter[hash] != 0 && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && *src == *(src - 3) && *src == *(src - 2) && *src == *(src - 1) && *src == *(src + 1) && *src == *(src + 2)))) + if ((cached & 0xffffff) == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) + { + if(cached != 0) + { #else - if (cached == 0 && hash_counter[hash] != 0 && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && *src == *(src - 3) && *src == *(src - 2) && *src == *(src - 1) && *src == *(src + 1) && *src == *(src + 2)))) -#endif + if (cached == 0 && o != OFFSET_BASE && (src - o > MINOFFSET || (src == o + 1 && lits >= 3 && src > source + 3 && same(src - 3, 6)))) { if (*(o + 3) != *(src + 3)) { +#endif + hash <<= 4; cword_val = (cword_val >> 1) | (1U << 31); - fast_write((3 - 2) | (hash << 4), dst, 2); + fast_write((3 - 2) | hash, dst, 2); src += 3; dst += 2; } @@ -244,6 +316,7 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest { const unsigned char *old_src = src; size_t matchlen; + hash <<= 4; cword_val = (cword_val >> 1) | (1U << 31); src += 4; @@ -262,7 +335,6 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest } matchlen = src - old_src; - hash <<= 4; if (matchlen < 18) { fast_write((ui32)(matchlen - 2) | hash, dst, 2); @@ -280,7 +352,6 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest else { lits++; - hash_counter[hash] = 1; *dst = *src; src++; dst++; @@ -288,7 +359,7 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest #ifdef X86X64 fetch = fast_read(src, 3); #else - fetch = fetch >> 8 & 0xffff | *(src + 2) << 16; + fetch = (fetch >> 8 & 0xffff) | (*(src + 2) << 16); #endif } } @@ -299,13 +370,15 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest unsigned char c; size_t remaining = (last_byte - UNCOMPRESSED_END - src + 1) > 255 ? 255 : (last_byte - UNCOMPRESSED_END - src + 1); (void)best_k; + + //hash = hashat(src); fetch = fast_read(src, 3); hash = hash_func(fetch); - c = hash_counter[hash]; + c = state->hash_counter[hash]; - offset2 = hashtable[hash].offset[0]; + offset2 = state->hash[hash].offset[0]; if(offset2 < src - MINOFFSET && c > 0 && ((fast_read(offset2, 3) ^ fetch) & 0xffffff) == 0) { matchlen = 3; @@ -320,7 +393,7 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest matchlen = 0; for(k = 1; k < QLZ_POINTERS && c > k; k++) { - o = hashtable[hash].offset[k]; + o = state->hash[hash].offset[k]; #if QLZ_COMPRESSION_LEVEL == 3 if(((fast_read(o, 3) ^ fetch) & 0xffffff) == 0 && o < src - MINOFFSET) #elif QLZ_COMPRESSION_LEVEL == 2 @@ -343,9 +416,9 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest } } o = offset2; - hashtable[hash].offset[c & (QLZ_POINTERS - 1)] = src; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; c++; - hash_counter[hash] = c; + state->hash_counter[hash] = c; #if QLZ_COMPRESSION_LEVEL == 3 if(matchlen > 2 && src - o < 131071) @@ -355,10 +428,9 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest for(u = 1; u < matchlen; u++) { - fetch = fast_read(src + u, 3); - hash = hash_func(fetch); - c = hash_counter[hash]++; - hashtable[hash].offset[c & (QLZ_POINTERS - 1)] = src + u; + hash = hashat(src + u); + c = state->hash_counter[hash]++; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src + u; } cword_val = (cword_val >> 1) | (1U << 31); @@ -377,20 +449,20 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest } else if (matchlen <= 18 && offset <= 1023) { - ui32 f = ((matchlen - 3) << 2) | (offset << 6) | 2; + ui32 f = ((matchlen - 3) << 2) | ((ui32)offset << 6) | 2; fast_write(f, dst, 2); dst += 2; } else if(matchlen <= 33) { - ui32 f = ((matchlen - 2) << 2) | (offset << 7) | 3; + ui32 f = ((matchlen - 2) << 2) | ((ui32)offset << 7) | 3; fast_write(f, dst, 3); dst += 3; } else { - ui32 f = ((matchlen - 3) << 7) | (offset << 15) | 3; + ui32 f = ((matchlen - 3) << 7) | ((ui32)offset << 15) | 3; fast_write(f, dst, 4); dst += 4; } @@ -402,7 +474,6 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest dst++; cword_val = (cword_val >> 1); } - #elif QLZ_COMPRESSION_LEVEL == 2 if(matchlen > 2) @@ -433,9 +504,7 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest #endif } #endif - } - while (src <= last_byte) { if ((cword_val & 1) == 1) @@ -449,28 +518,25 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest if (src <= last_byte - 3) { #if QLZ_COMPRESSION_LEVEL == 1 - ui32 hash; + ui32 hash, fetch; fetch = fast_read(src, 3); hash = hash_func(fetch); - hashtable[hash].offset[0] = src; - hashtable[hash].cache[0] = fetch; - hash_counter[hash] = 1; + state->hash[hash].offset = CAST(src - OFFSET_BASE); + state->hash[hash].cache = fetch; #elif QLZ_COMPRESSION_LEVEL == 2 ui32 hash; unsigned char c; - fetch = fast_read(src, 3); - hash = hash_func(fetch); - c = hash_counter[hash]; - hashtable[hash].offset[c & (QLZ_POINTERS - 1)] = src; + hash = hashat(src); + c = state->hash_counter[hash]; + state->hash[hash].offset[c & (QLZ_POINTERS - 1)] = src; c++; - hash_counter[hash] = c; + state->hash_counter[hash] = c; #endif } #endif *dst = *src; src++; dst++; - cword_val = (cword_val >> 1); } @@ -483,22 +549,21 @@ static size_t qlz_compress_core(const unsigned char *source, unsigned char *dest return dst - destination < 9 ? 9 : dst - destination; } -static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_hash_decompress hashtable[QLZ_HASH_VALUES], unsigned char hash_counter[QLZ_HASH_VALUES], unsigned char *history, const char *source_2) +static size_t qlz_decompress_core(const unsigned char *source, unsigned char *destination, size_t size, qlz_state_decompress *state, const unsigned char *history) { - const unsigned char *src = source; + const unsigned char *src = source + qlz_size_header((const char *)source); unsigned char *dst = destination; const unsigned char *last_destination_byte = destination + size - 1; ui32 cword_val = 1; - const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; const unsigned char *last_matchstart = last_destination_byte - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END; unsigned char *last_hashed = destination - 1; - const unsigned char *last_source_byte = (const unsigned char *)source_2 + qlz_size_compressed(source_2) - 1; + const unsigned char *last_source_byte = source + qlz_size_compressed((const char *)source) - 1; + static const ui32 bitlut[16] = {4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; (void) last_source_byte; - (void) history; (void) last_hashed; - (void) hash_counter; - (void) hashtable; + (void) state; + (void) history; for(;;) { @@ -522,7 +587,6 @@ static size_t qlz_decompress_core(const unsigned char *source, unsigned char *de fetch = fast_read(src, 4); if ((cword_val & 1) == 1) - { ui32 matchlen; const unsigned char *offset2; @@ -531,7 +595,7 @@ static size_t qlz_decompress_core(const unsigned char *source, unsigned char *de ui32 hash; cword_val = cword_val >> 1; hash = (fetch >> 4) & 0xfff; - offset2 = hashtable[hash].offset[0]; + offset2 = (const unsigned char *)(size_t)state->hash[hash].offset; if((fetch & 0xf) != 0) { @@ -550,12 +614,12 @@ static size_t qlz_decompress_core(const unsigned char *source, unsigned char *de cword_val = cword_val >> 1; hash = (fetch >> 5) & 0x7ff; c = (unsigned char)(fetch & 0x3); - offset2 = hashtable[hash].offset[c]; + offset2 = state->hash[hash].offset[c]; if((fetch & (28)) != 0) { matchlen = ((fetch >> 2) & 0x7) + 2; - src += 2; + src += 2; } else { @@ -607,29 +671,30 @@ static size_t qlz_decompress_core(const unsigned char *source, unsigned char *de if(matchlen > (ui32)(last_destination_byte - dst - UNCOMPRESSED_END + 1)) return 0; #endif + memcpy_up(dst, offset2, matchlen); dst += matchlen; #if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(hashtable, hash_counter, &last_hashed, dst - matchlen); + update_hash_upto(state, &last_hashed, dst - matchlen); last_hashed = dst - 1; #endif } - else { if (dst < last_matchstart) { + unsigned int n = bitlut[cword_val & 0xf]; #ifdef X86X64 *(ui32 *)dst = *(ui32 *)src; #else memcpy_up(dst, src, 4); #endif - dst += bitlut[cword_val & 0xf]; - src += bitlut[cword_val & 0xf]; - cword_val = cword_val >> (bitlut[cword_val & 0xf]); + cword_val = cword_val >> n; + dst += n; + src += n; #if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(hashtable, hash_counter, &last_hashed, dst - 3); + update_hash_upto(state, &last_hashed, dst - 3); #endif } else @@ -652,7 +717,7 @@ static size_t qlz_decompress_core(const unsigned char *source, unsigned char *de } #if QLZ_COMPRESSION_LEVEL <= 2 - update_hash_upto(hashtable, hash_counter, &last_hashed, last_destination_byte - 3); // todo, use constant + update_hash_upto(state, &last_hashed, last_destination_byte - 3); // todo, use constant #endif return size; } @@ -661,33 +726,8 @@ static size_t qlz_decompress_core(const unsigned char *source, unsigned char *de } } -size_t qlz_size_decompressed(const char *source) +size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state) { - ui32 n, r; - n = (((*source) & 2) == 2) ? 4 : 1; - r = fast_read(source + 1 + n, n); - r = r & (0xffffffff >> ((4 - n)*8)); - return r; -} - -size_t qlz_size_compressed(const char *source) -{ - ui32 n, r; - n = (((*source) & 2) == 2) ? 4 : 1; - r = fast_read(source + 1, n); - r = r & (0xffffffff >> ((4 - n)*8)); - return r; -} - -size_t qlz_compress(const void *source, char *destination, size_t size, char *scratch_compress) -{ - unsigned char *scratch_aligned = (unsigned char *)scratch_compress + QLZ_ALIGNMENT_PADD - (((size_t)scratch_compress) % QLZ_ALIGNMENT_PADD); - size_t *buffersize = (size_t *)scratch_aligned; - qlz_hash_compress *hashtable = (qlz_hash_compress *)(scratch_aligned + QLZ_BUFFER_COUNTER); - unsigned char *hash_counter = (unsigned char*)hashtable + sizeof(qlz_hash_compress[QLZ_HASH_VALUES]); -#if QLZ_STREAMING_BUFFER > 0 - unsigned char *streambuffer = hash_counter + QLZ_HASH_VALUES; -#endif size_t r; ui32 compressed; size_t base; @@ -701,13 +741,13 @@ size_t qlz_compress(const void *source, char *destination, size_t size, char *sc base = 9; #if QLZ_STREAMING_BUFFER > 0 - if (*buffersize + size - 1 >= QLZ_STREAMING_BUFFER) + if (state->stream_counter + size - 1 >= QLZ_STREAMING_BUFFER) #endif { - reset_state(hash_counter); - r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, hashtable, hash_counter); + reset_table_compress(state); + r = base + qlz_compress_core((const unsigned char *)source, (unsigned char*)destination + base, size, state); #if QLZ_STREAMING_BUFFER > 0 - reset_state(hash_counter); + reset_table_compress(state); #endif if(r == base) { @@ -719,26 +759,28 @@ size_t qlz_compress(const void *source, char *destination, size_t size, char *sc { compressed = 1; } - *buffersize = 0; + state->stream_counter = 0; } #if QLZ_STREAMING_BUFFER > 0 else { - memcpy(streambuffer + *buffersize, source, size); - r = base + qlz_compress_core((const unsigned char *)streambuffer + *buffersize, (unsigned char*)destination + base, size, hashtable, hash_counter); + unsigned char *src = state->stream_buffer + state->stream_counter; - if(r == base) + memcpy(src, source, size); + r = base + qlz_compress_core(src, (unsigned char*)destination + base, size, state); + + if(r == base) { - memcpy(destination + base, streambuffer + *buffersize, size); + memcpy(destination + base, src, size); r = size + base; compressed = 0; - reset_state(hash_counter); + reset_table_compress(state); } else { compressed = 1; } - *buffersize += size; + state->stream_counter += size; } #endif if(base == 3) @@ -764,62 +806,41 @@ size_t qlz_compress(const void *source, char *destination, size_t size, char *sc return r; } -size_t qlz_decompress(const char *source, void *destination, char *scratch_compress) +size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state) { - unsigned char *scratch_aligned = (unsigned char *)scratch_compress + QLZ_ALIGNMENT_PADD - (((size_t)scratch_compress) % QLZ_ALIGNMENT_PADD); - size_t *buffersize = (size_t *)scratch_aligned; -#if QLZ_COMPRESSION_LEVEL == 3 -#if QLZ_STREAMING_BUFFER > 0 - unsigned char *streambuffer = scratch_aligned + QLZ_BUFFER_COUNTER; -#endif - unsigned char *hash_counter = 0; - qlz_hash_decompress *hashtable = 0; -#elif QLZ_COMPRESSION_LEVEL <= 2 - qlz_hash_decompress *hashtable = (qlz_hash_decompress *)(scratch_aligned + QLZ_BUFFER_COUNTER); - unsigned char *hash_counter = (unsigned char*)hashtable + sizeof(qlz_hash_decompress[QLZ_HASH_VALUES]); -#if QLZ_STREAMING_BUFFER > 0 - unsigned char *streambuffer = hash_counter + QLZ_HASH_VALUES; -#endif -#endif - ui32 headerlen = 2*((((*source) & 2) == 2) ? 4 : 1) + 1; size_t dsiz = qlz_size_decompressed(source); #if QLZ_STREAMING_BUFFER > 0 - if (*buffersize + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) + if (state->stream_counter + qlz_size_decompressed(source) - 1 >= QLZ_STREAMING_BUFFER) #endif - { + { if((*source & 1) == 1) { -#if QLZ_COMPRESSION_LEVEL != 3 - reset_state(hash_counter); -#endif - dsiz = qlz_decompress_core((const unsigned char *)source + headerlen, (unsigned char *)destination, dsiz, hashtable, hash_counter, (unsigned char *)destination, source); + reset_table_decompress(state); + dsiz = qlz_decompress_core((const unsigned char *)source, (unsigned char *)destination, dsiz, state, (const unsigned char *)destination); } else { - memcpy(destination, source + headerlen, dsiz); + memcpy(destination, source + qlz_size_header(source), dsiz); } - *buffersize = 0; -#if QLZ_COMPRESSION_LEVEL != 3 - reset_state(hash_counter); -#endif + state->stream_counter = 0; + reset_table_decompress(state); } #if QLZ_STREAMING_BUFFER > 0 else { + unsigned char *dst = state->stream_buffer + state->stream_counter; if((*source & 1) == 1) { - dsiz = qlz_decompress_core((const unsigned char *)source + headerlen, streambuffer + *buffersize, dsiz, hashtable, hash_counter, streambuffer, source); + dsiz = qlz_decompress_core((const unsigned char *)source, dst, dsiz, state, (const unsigned char *)state->stream_buffer); } else { - memcpy(streambuffer + *buffersize, source + headerlen, dsiz); -#if QLZ_COMPRESSION_LEVEL != 3 - reset_state(hash_counter); -#endif + memcpy(dst, source + qlz_size_header(source), dsiz); + reset_table_decompress(state); } - memcpy(destination, streambuffer + *buffersize, dsiz); - *buffersize += dsiz; + memcpy(destination, dst, dsiz); + state->stream_counter += dsiz; } #endif return dsiz; diff --git a/src/quicklz.h b/src/quicklz.h index fc5ee76..c33a87f 100644 --- a/src/quicklz.h +++ b/src/quicklz.h @@ -2,54 +2,57 @@ #define QLZ_HEADER // Fast data compression library -// Copyright (C) 2006-2010 Lasse Mikkel Reinhold +// Copyright (C) 2006-2011 Lasse Mikkel Reinhold // lar@quicklz.com // -// QuickLZ can be used for free under the GPL-1, -2 or -3 license (where anything +// QuickLZ can be used for free under the GPL 1, 2 or 3 license (where anything // released into public must be open source) or under a commercial license if such // has been acquired (see http://www.quicklz.com/order.html). The commercial license // does not cover derived or ported versions created by third parties under GPL. -// Version 1.4.1 final - april 2010 - // You can edit following user settings. Data must be decompressed with the same // setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed // (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially // zeroed out (see manual). First #ifndef makes it possible to define settings from -// the outside like the compiler command line or from higher level code. +// the outside like the compiler command line. + +// 1.5.0 final #ifndef QLZ_COMPRESSION_LEVEL + + // 1 gives fastest compression speed. 3 gives fastest decompression speed and best + // compression ratio. //#define QLZ_COMPRESSION_LEVEL 1 //#define QLZ_COMPRESSION_LEVEL 2 #define QLZ_COMPRESSION_LEVEL 3 + // If > 0, zero out both states prior to first call to qlz_compress() or qlz_decompress() + // and decompress packets in the same order as they were compressed #define QLZ_STREAMING_BUFFER 0 //#define QLZ_STREAMING_BUFFER 100000 //#define QLZ_STREAMING_BUFFER 1000000 + // Guarantees that decompression of corrupted data cannot crash. Decreases decompression + // speed 10-20%. Compression speed not affected. //#define QLZ_MEMORY_SAFE #endif #define QLZ_VERSION_MAJOR 1 -#define QLZ_VERSION_MINOR 4 -#define QLZ_VERSION_REVISION 1 +#define QLZ_VERSION_MINOR 5 +#define QLZ_VERSION_REVISION 0 // Using size_t, memset() and memcpy() #include -// Public functions of QuickLZ -size_t qlz_size_decompressed(const char *source); -size_t qlz_size_compressed(const char *source); -size_t qlz_decompress(const char *source, void *destination, char *scratch_decompress); -size_t qlz_compress(const void *source, char *destination, size_t size, char *scratch_compress); -int qlz_get_setting(int setting); - // Verify compression level #if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 #error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 #endif -// Compute QLZ_SCRATCH_COMPRESS and QLZ_SCRATCH_DECOMPRESS +typedef unsigned int ui32; +typedef unsigned short int ui16; + +// Decrease QLZ_POINTERS for level 3 to increase compression speed. Do not touch any other values! #if QLZ_COMPRESSION_LEVEL == 1 #define QLZ_POINTERS 1 #define QLZ_HASH_VALUES 4096 @@ -61,34 +64,86 @@ int qlz_get_setting(int setting); #define QLZ_HASH_VALUES 4096 #endif +// Detect if pointer size is 64-bit. It's not fatal if some 64-bit target is not detected because this is only for adding an optional 64-bit optimization. +#if defined _LP64 || defined __LP64__ || defined __64BIT__ || _ADDR64 || defined _WIN64 || defined __arch64__ || __WORDSIZE == 64 || (defined __sparc && defined __sparcv9) || defined __x86_64 || defined __amd64 || defined __x86_64__ || defined _M_X64 || defined _M_IA64 || defined __ia64 || defined __IA64__ + #define QLZ_PTR_64 +#endif + +// hash entry typedef struct { #if QLZ_COMPRESSION_LEVEL == 1 - unsigned int cache[QLZ_POINTERS]; + ui32 cache; +#if defined QLZ_PTR_64 && QLZ_STREAMING_BUFFER == 0 + unsigned int offset; +#else + const unsigned char *offset; #endif +#else const unsigned char *offset[QLZ_POINTERS]; +#endif + } qlz_hash_compress; typedef struct { +#if QLZ_COMPRESSION_LEVEL == 1 + const unsigned char *offset; +#else const unsigned char *offset[QLZ_POINTERS]; +#endif } qlz_hash_decompress; -#define QLZ_ALIGNMENT_PADD 8 -#define QLZ_BUFFER_COUNTER 8 - -#define QLZ_SCRATCH_COMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER + sizeof(qlz_hash_compress[QLZ_HASH_VALUES]) + QLZ_HASH_VALUES) - -#if QLZ_COMPRESSION_LEVEL < 3 - #define QLZ_SCRATCH_DECOMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER + sizeof(qlz_hash_decompress[QLZ_HASH_VALUES]) + QLZ_HASH_VALUES) -#else - #define QLZ_SCRATCH_DECOMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER) +// states +typedef struct +{ + #if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; + #endif + size_t stream_counter; + qlz_hash_compress hash[QLZ_HASH_VALUES]; + unsigned char hash_counter[QLZ_HASH_VALUES]; +} qlz_state_compress; + + +#if QLZ_COMPRESSION_LEVEL == 1 || QLZ_COMPRESSION_LEVEL == 2 + typedef struct + { +#if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; #endif - + qlz_hash_decompress hash[QLZ_HASH_VALUES]; + unsigned char hash_counter[QLZ_HASH_VALUES]; + size_t stream_counter; + } qlz_state_decompress; +#elif QLZ_COMPRESSION_LEVEL == 3 + typedef struct + { +#if QLZ_STREAMING_BUFFER > 0 + unsigned char stream_buffer[QLZ_STREAMING_BUFFER]; +#endif +#if QLZ_COMPRESSION_LEVEL <= 2 + qlz_hash_decompress hash[QLZ_HASH_VALUES]; +#endif + size_t stream_counter; + } qlz_state_decompress; #endif +#if defined (__cplusplus) +extern "C" { +#endif +// Public functions of QuickLZ +size_t qlz_size_decompressed(const char *source); +size_t qlz_size_compressed(const char *source); +size_t qlz_compress(const void *source, char *destination, size_t size, qlz_state_compress *state); +size_t qlz_decompress(const char *source, void *destination, qlz_state_decompress *state); +int qlz_get_setting(int setting); +#if defined (__cplusplus) +} +#endif +#endif