diff --git a/video/compression_szip.h b/video/compression_szip.h index ce596ac..dc46921 100644 --- a/video/compression_szip.h +++ b/video/compression_szip.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include "szip/port.h" @@ -18,6 +19,9 @@ extern "C" { #define MAJOR_VERSION 1 #define MINOR_VERSION 12 +// Define an output-chunk size for dynamic buffer expansion +#define OUTPUT_CHUNK_SIZE 256 + #pragma pack(push, 1) typedef struct { uint8_t marker[3]; // "SZ\n" @@ -42,18 +46,48 @@ typedef struct { uint8_t recordsize; } SzipConfig; -// Reads a byte from the stream -static inline int szip_read_byte(SzipBufferStream *stream) { - return (stream->pos >= stream->size) ? EOF : stream->data[stream->pos++]; +// Dynamic output buffer for decompression +typedef struct { + uint8_t *data; + size_t used; // bytes written so far + size_t allocated; // total allocated size +} DynamicBuffer; + +// Helper: Ensure that the dynamic buffer has at least (current used + extra) bytes. +// If not, expand it by OUTPUT_CHUNK_SIZE increments. +static int ensure_dynamic_buffer_capacity(DynamicBuffer *db, size_t extra) { + if (db->used + extra > db->allocated) { + size_t new_allocated = db->allocated; + while (db->used + extra > new_allocated) { + new_allocated += OUTPUT_CHUNK_SIZE; + } + uint8_t *new_data = (uint8_t *) ps_malloc(new_allocated); + if (!new_data) { + printf("ERROR: Unable to allocate %zu bytes for dynamic buffer.\n", new_allocated); + return -1; + } + if (db->data) { + memcpy(new_data, db->data, db->used); + heap_caps_free(db->data); + } + db->data = new_data; + db->allocated = new_allocated; + } + return 0; } -// Writes a byte to the stream +// Writes a byte to a stream static inline void szip_write_byte(SzipBufferStream *stream, uint8_t byte) { if (stream->pos < stream->size) { stream->data[stream->pos++] = byte; } } +// Reads a byte from the stream +static inline int szip_read_byte(SzipBufferStream *stream) { + return (stream->pos >= stream->size) ? EOF : stream->data[stream->pos++]; +} + // Reads a 3-byte big-endian integer static inline uint32_t szip_read_uint3(SzipBufferStream *stream) { uint32_t value = szip_read_byte(stream); @@ -69,115 +103,165 @@ static inline void szip_write_uint3(SzipBufferStream *stream, uint32_t value) { szip_write_byte(stream, value & 0xFF); } -// Writes the global SZIP header -static inline void szip_write_global_header(SzipBufferStream *stream) { - szip_write_byte(stream, 0x53); // 'S' - szip_write_byte(stream, 0x5A); // 'Z' - szip_write_byte(stream, 0x0A); // Newline - szip_write_byte(stream, 0x04); // Type - szip_write_byte(stream, MAJOR_VERSION); - szip_write_byte(stream, MINOR_VERSION); -} - // Reads and validates the SZIP global header static inline int szip_read_global_header(SzipBufferStream *stream) { if (szip_read_byte(stream) != 0x53 || szip_read_byte(stream) != 0x5A || szip_read_byte(stream) != 0x0A || szip_read_byte(stream) != 0x04) return -1; - int vmay = szip_read_byte(stream); int vmin = szip_read_byte(stream); return (vmay == EOF || vmin == EOF) ? -1 : 0; } -// Writes a block header, including the extra 0x00 byte -static inline void szip_write_block_header(SzipBufferStream *stream, uint32_t uncompressed_size) { - szip_write_byte(stream, 0x42); // 'B' - szip_write_byte(stream, 0x48); // 'H' - szip_write_uint3(stream, uncompressed_size); - szip_write_byte(stream, 0x00); // Extra empty 'filename' byte -} - -// Reads a block header, correctly skipping the orphaned byte +// Reads a block header, skipping the orphaned byte, and returns the block type. +// Expected on-disk layout: +// 0x42, 0x48, 3-byte block size, 0x00, block type (0x01 for compressed) static inline int szip_read_block_header(SzipBufferStream *stream, uint32_t *uncompressed_size, uint8_t *block_type) { if (szip_read_byte(stream) != 0x42 || szip_read_byte(stream) != 0x48) return -1; - *uncompressed_size = szip_read_uint3(stream); - - // Skip the extra orphaned byte (0x00) if (szip_read_byte(stream) != 0x00) { printf("ERROR: Expected orphaned byte 0x00 but found something else.\n"); return -1; } - - // Read the block type (should be 0x01 for compressed) *block_type = szip_read_byte(stream); return 0; } -// Reads a compressed block -static void szip_read_szip_block(SzipBufferStream *stream, uint8_t *buffer, uint32_t buflen, SzipConfig *config) { - uint32_t index_last = szip_read_uint3(stream); // Read 3-byte index_last - uint8_t order = szip_read_byte(stream); // Read order byte +// Reads a compressed block from the stream. +// Block layout (after block header): +// 3 bytes: index_last (big-endian; expected to be < block_size) +// 1 byte: order +// ... followed by compressed data runs +// This function writes exactly 'buflen' bytes into the provided destination buffer. + +static void szip_read_szip_block(SzipBufferStream *stream, uint8_t *dest, uint32_t buflen, SzipConfig *config) { + uint32_t index_last = szip_read_uint3(stream); + uint8_t order = szip_read_byte(stream); uint32_t bytes_left = buflen; - SzipModel m; - - initmodel(&m, -1, &(config->recordsize)); + + printf("szip_read_szip_block: Initializing model (index_last=%u, order=%u, buflen=%u)\n", + index_last, order, buflen); + + // ✅ Allocate model on heap + SzipModel *m = (SzipModel *) ps_malloc(sizeof(SzipModel)); + if (!m) { + printf("ERROR: Failed to allocate memory for SzipModel.\n"); + return; + } + + printf("szip_read_szip_block: Allocated model at %p\n", m); + + // ✅ Initialize model + initmodel(m, -1, &(config->recordsize)); + printf("szip_read_szip_block: Model initialized successfully.\n"); uint32_t runlength; uint32_t ch; - sz_decode(&m, &ch, &runlength); - if (runlength > bytes_left) return; - bytes_left -= runlength; - memset(buffer, ch, runlength); - fixafterfirst(&m); - buffer += runlength; + // ✅ Decode first run + printf("szip_read_szip_block: Decoding first run...\n"); + sz_decode(m, &ch, &runlength); + printf("szip_read_szip_block: First run decoded: ch=%u, runlength=%u\n", ch, runlength); + + if (runlength > bytes_left) { + printf("ERROR: runlength (%u) exceeds block size (%u).\n", runlength, bytes_left); + heap_caps_free(m); + return; + } + bytes_left -= runlength; + memset(dest, ch, runlength); + fixafterfirst(m); + dest += runlength; + // ✅ Decode remaining runs while (bytes_left) { - sz_decode(&m, &ch, &runlength); - if (runlength > bytes_left) return; + printf("szip_read_szip_block: Decoding next run (bytes_left=%u)...\n", bytes_left); + sz_decode(m, &ch, &runlength); + printf("szip_read_szip_block: Next run decoded: ch=%u, runlength=%u\n", ch, runlength); + + if (runlength > bytes_left) { + printf("ERROR: runlength (%u) exceeds remaining bytes (%u).\n", runlength, bytes_left); + heap_caps_free(m); + return; + } bytes_left -= runlength; - memset(buffer, ch, runlength); - buffer += runlength; + memset(dest, ch, runlength); + dest += runlength; } - - deletemodel(&m); + + printf("szip_read_szip_block: All runs decoded. Cleaning up model...\n"); + deletemodel(m); + heap_caps_free(m); // ✅ Free model memory + + printf("szip_read_szip_block: Block decompression complete.\n"); } -// Decompress function for ESP32 -void szip_decompress(uint8_t *input, uint32_t input_size, uint8_t *output, uint32_t *output_size, SzipConfig *config) { - SzipBufferStream in_stream = {input, input_size, 0}; - SzipBufferStream out_stream = {output, *output_size, 0}; +// DECOMPRESSION FUNCTION WITH DYNAMIC OUTPUT BUFFER +// +// This function processes the entire input buffer (which begins with the global header) +// and then one or more blocks. For each block, it reads the block header (including the +// uncompressed size for that block), ensures that the dynamic output buffer is expanded +// as needed, calls szip_read_szip_block to decompress the block directly into the dynamic +// output buffer at the appropriate offset, and updates the buffer usage. +// +// Upon completion, *output will point to the decompressed data and *output_size will hold its size. +// (The caller is responsible for freeing the output buffer with heap_caps_free.) +uint8_t* szip_decompress_dynamic(uint8_t *input, uint32_t input_size, uint32_t *output_size, SzipConfig *config) { + SzipBufferStream in_stream = { input, input_size, 0 }; + if (szip_read_global_header(&in_stream) < 0) { printf("ERROR: Invalid SZIP global header!\n"); - return; + return NULL; + } + + // Dynamically allocate DynamicBuffer struct to avoid stack usage. + DynamicBuffer *db = (DynamicBuffer *) ps_malloc(sizeof(DynamicBuffer)); + if (!db) { + printf("ERROR: Unable to allocate memory for DynamicBuffer struct.\n"); + return NULL; + } + db->allocated = OUTPUT_CHUNK_SIZE; + db->used = 0; + db->data = (uint8_t *) ps_malloc(db->allocated); + if (!db->data) { + printf("ERROR: Unable to allocate initial output buffer.\n"); + heap_caps_free(db); + return NULL; } uint32_t uncompressed_size; uint8_t block_type; - + // Process each block in the input stream. while (in_stream.pos < in_stream.size) { - // Read block header correctly if (szip_read_block_header(&in_stream, &uncompressed_size, &block_type) < 0) { printf("ERROR: Failed to read block header!\n"); - return; + heap_caps_free(db->data); + heap_caps_free(db); + return NULL; } - - // Ensure block type is valid before proceeding if (block_type != 0x01) { printf("ERROR: Unexpected block type %02X! Expected 0x01 for compressed block.\n", block_type); - return; + heap_caps_free(db->data); + heap_caps_free(db); + return NULL; } - // Read and decompress block data - szip_read_szip_block(&in_stream, output + out_stream.pos, uncompressed_size, config); - out_stream.pos += uncompressed_size; - } + // Ensure dynamic buffer has space for this block. + if (ensure_dynamic_buffer_capacity(db, uncompressed_size) < 0) { + heap_caps_free(db->data); + heap_caps_free(db); + return NULL; + } - *output_size = out_stream.pos; + // Decompress into heap-allocated buffer (not stack). + printf("Decompressing block with size %u\n", uncompressed_size); + szip_read_szip_block(&in_stream, db->data + db->used, uncompressed_size, config); + db->used += uncompressed_size; + } + *output_size = db->used; + return db->data; } #ifdef __cplusplus diff --git a/video/szip/sz_mod4.c b/video/szip/sz_mod4.c index ca6e49c..bd58770 100755 --- a/video/szip/sz_mod4.c +++ b/video/szip/sz_mod4.c @@ -28,21 +28,27 @@ SzipModel mod; void initmodel(SzipModel *m, int headersize, uint8_t *first) { int i; - /* Initialize the arithmetic coder */ + printf("initmodel: Starting initialization (compress=%d)\n", headersize >= 0); + + // Initialize arithmetic coder m->compress = (headersize >= 0); if (m->compress) { + printf("initmodel: Starting encoding initialization...\n"); start_encoding(&(m->ac), *first, headersize); } else { + printf("initmodel: Starting decoding initialization...\n"); *first = start_decoding(&(m->ac)); } - /* Initialize full model */ + printf("initmodel: Initializing bitmodel...\n"); init_bitmodel(&(m->full), ALPHABETSIZE, 40 * ALPHABETSIZE, 10 * ALPHABETSIZE, NULL); + + printf("initmodel: Clearing lastseen array...\n"); for (i = 0; i < ALPHABETSIZE; i++) { m->lastseen[i] = FULLFLAG; } - /* Initialize cache with symbols */ + printf("initmodel: Initializing cache...\n"); CachePtr tmp = m->cache; for (i = 0; i < CACHESIZE - 1; i++) { tmp->next = tmp + 1; @@ -55,41 +61,49 @@ void initmodel(SzipModel *m, int headersize, uint8_t *first) { tmp->what = 0; tmp++; } + + printf("initmodel: Completing cache initialization...\n"); m->cache[0].prev = m->cache + (CACHESIZE - 1); tmp->next = m->cache; tmp->prev = tmp - 1; tmp->sy_f = 0; - m->newest = m->cache + (CACHESIZE - 2); m->lastnew = m->cache + (CACHESIZE - 7); m->cachetotf = CACHESIZE; - /* Initialize whatmodel */ + printf("initmodel: Initializing whatmodel and mtf models...\n"); m->whatmod[0] = 41; m->whatmod[1] = 8; m->whatmod[2] = 15; - /* Initialize MTF models */ m->mtfhist[0].next = MTFHISTSIZE - 1; m->mtfhist[0].sym = CACHESIZE; for (i = 1; i < MTFSIZE << 1; i++) { m->mtfhist[i].next = i - 1; m->mtfhist[i].sym = CACHESIZE + i; } + for (; i < MTFHISTSIZE; i++) { m->mtfhist[i].next = 0xFFFF; } + + printf("initmodel: Initializing mtfsize values...\n"); m->mtfsize = MTFSIZE << 1; m->mtfsizeact = 0; m->mtffirst = (MTFSIZE << 1) - 1; + + printf("initmodel: Initializing qsmodel...\n"); init_qsmodel(&(m->mtfmod), MTFSIZE, MTFSHIFT, 400, NULL, m->compress); - /* Initialize run-length models */ + printf("initmodel: Initializing run-length models...\n"); for (i = 0; i < 5; i++) { init_qsmodel(m->rlemod + i, 7, RLSHIFT, 150, NULL, m->compress); } + + printf("initmodel: Completed successfully.\n"); } + /* Call after encoding/decoding first run */ void fixafterfirst(SzipModel *m) { m->cachetotf--; diff --git a/video/vdu_buffered.h b/video/vdu_buffered.h index 3291fa5..6744b11 100644 --- a/video/vdu_buffered.h +++ b/video/vdu_buffered.h @@ -2522,85 +2522,54 @@ void VDUStreamProcessor::bufferDecompressSzip(uint16_t bufferId, uint16_t source return; } auto &sourceBuffer = sourceBufferIter->second; - + if (sourceBuffer.empty() || sourceBuffer[0]->size() < sizeof(SzipFileHeader)) { printf("bufferDecompressSzip: buffer too small for header\n"); return; } - - // Initialize stream reader - SzipBufferStream stream = { sourceBuffer[0]->getBuffer(), sourceBuffer[0]->size(), 0 }; - - // Read and validate global SZIP header - if (szip_read_global_header(&stream) < 0) { - printf("bufferDecompressSzip: invalid SZIP header\n"); - return; - } - printf("SZIP global header validated.\n"); - - // Read block header, correctly extracting block type - uint32_t block_size; - uint8_t block_type; - if (szip_read_block_header(&stream, &block_size, &block_type) < 0) { - printf("bufferDecompressSzip: invalid block header\n"); - return; - } - printf("Block size extracted: %u\n", block_size); - printf("Block type read: %02X (should be 01 for compressed)\n", block_type); - - if (block_type != 0x01) { - printf("ERROR: Unexpected block type %02X! Expected 0x01 for compressed block.\n", block_type); - return; - } - - // Read index_last - uint32_t index_last = szip_read_uint3(&stream); - printf("Next 3 bytes (index_last): %02X %02X %02X\n", - (index_last >> 16) & 0xFF, (index_last >> 8) & 0xFF, index_last & 0xFF); - printf("Index Last: %u (expected < block_size)\n", index_last); - - // Read the order byte - uint8_t order = szip_read_byte(&stream); - printf("Next byte (order): %02X\n", order); - printf("Order extracted: %u\n", order); - - // Validate index_last - if (index_last >= block_size) { - printf("ERROR: index_last (%u) is greater than block_size (%u)! Possible misalignment.\n", - index_last, block_size); + + // Retrieve the compressed input buffer and its size. + uint8_t* compressedData = sourceBuffer[0]->getBuffer(); + uint32_t compressedSize = sourceBuffer[0]->size(); + + // Set up decompression configuration. + SzipConfig config; + // Set config.block_size to the maximum block size expected (if known) + // Otherwise, it is used only for compression; decompression reads each block's size. + config.block_size = 0; // not used during decompression + config.order = 0; // will be read from the block header + config.verbosity = 0; + config.recordsize = 1; + + printf("Starting dynamic decompression for buffer %u...\n", bufferId); + + uint32_t decompressedSize = 0; + uint8_t* decompressedData = szip_decompress_dynamic(compressedData, compressedSize, &decompressedSize, &config); + if (!decompressedData) { + printf("ERROR: Decompression failed for buffer %u.\n", bufferId); return; } - - // Create output buffer - auto bufferStream = make_shared_psram(block_size); + + printf("Decompressed %u bytes from %u compressed bytes.\n", decompressedSize, compressedSize); + + // Store the decompressed buffer. + bufferClear(bufferId); + // Here, assume that make_shared_psram accepts an externally allocated buffer. + // Otherwise, wrap decompressedData into a BufferStream as appropriate. + auto bufferStream = make_shared_psram(decompressedSize); if (!bufferStream || !bufferStream->getBuffer()) { - printf("bufferDecompressSzip: failed to create buffer %d\n", bufferId); + printf("bufferDecompressSzip: failed to create output buffer for buffer %d\n", bufferId); + heap_caps_free(decompressedData); return; } - - uint8_t *outBuffer = bufferStream->getBuffer(); - size_t outSize = block_size; - - // Configure decompression parameters - SzipConfig config = { block_size, order, 0, 1 }; // Use extracted values - - printf("Starting decompression for buffer %u using block size %u, order %u...\n", - bufferId, block_size, order); - - // Perform decompression - szip_decompress(sourceBuffer[0]->getBuffer(), block_size, outBuffer, &outSize, &config); - - // Store decompressed buffer - bufferClear(bufferId); + memcpy(bufferStream->getBuffer(), decompressedData, decompressedSize); buffers[bufferId].push_back(bufferStream); - - uint32_t compressionRatio = (outSize * 100) / block_size; - printf("Decompressed %u bytes to %u bytes (%u%%)\n", block_size, outSize, compressionRatio); - - if (outSize != block_size) { - printf("Warning: decompressed size %u does not match expected %u\n", outSize, block_size); - } - + + uint32_t compressionRatio = (decompressedSize * 100) / compressedSize; + printf("Decompressed %u bytes to %u bytes (%u%%)\n", compressedSize, decompressedSize, compressionRatio); + + heap_caps_free(decompressedData); + #ifdef DEBUG printf("Decompression took %u ms\n", millis() - start); #endif