diff --git a/src/ESP32-targz-lib.hpp b/src/ESP32-targz-lib.hpp index 8a01fce..85ed5a4 100644 --- a/src/ESP32-targz-lib.hpp +++ b/src/ESP32-targz-lib.hpp @@ -139,7 +139,7 @@ #elif defined ARDUINO_ARCH_RP2040 - #pragma message "Experimental RP2040 support" + #pragma message "Experimental RP2040 support, compression is disabled" #undef DEST_FS_USES_SD_MMC // unsupported #undef DEST_FS_USES_FFAT // unsupported @@ -156,7 +156,7 @@ #define FS_NAME "LITTLEFS (picolib)" #endif - static FSInfo fsinfo; + [[maybe_unused]] static FSInfo fsinfo; #else diff --git a/src/ESP32-targz-log.hpp b/src/ESP32-targz-log.hpp index dc4cc85..e92eb4b 100644 --- a/src/ESP32-targz-log.hpp +++ b/src/ESP32-targz-log.hpp @@ -33,7 +33,7 @@ #pragma once -inline void NullLoggerCallback( [[maybe_unused]] const char* format, ...) { yield(); } +inline void NullLoggerCallback( [[maybe_unused]] const char* format, ...) { } #if defined ESP8266 || defined ESP32 // those have OTA and common device API @@ -86,6 +86,9 @@ inline void NullLoggerCallback( [[maybe_unused]] const char* format, ...) { yiel #define DEVICE_RESTART() rp2040.restart() #define HEAP_AVAILABLE() rp2040.getFreeHeap() + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wformat" + // ESP like log functions turned to macros to allow gathering of file name, log level, etc #define log_v(format, ...) TGZ::LOG(__FILE__, __LINE__, TGZ::LogLevelVerbose, format, ##__VA_ARGS__) #define log_d(format, ...) TGZ::LOG(__FILE__, __LINE__, TGZ::LogLevelDebug, format, ##__VA_ARGS__) @@ -119,7 +122,8 @@ inline void NullLoggerCallback( [[maybe_unused]] const char* format, ...) { yiel #endif #if !defined TGZ_DEFAULT_LOG_LEVEL - #define TGZ_DEFAULT_LOG_LEVEL LogLevelWarning + //#define TGZ_DEFAULT_LOG_LEVEL LogLevelWarning + #define TGZ_DEFAULT_LOG_LEVEL LogLevelDebug #endif namespace TGZ @@ -189,6 +193,7 @@ inline void NullLoggerCallback( [[maybe_unused]] const char* format, ...) { yiel vsnprintf(log_buffer, LOG_MAXLENGTH, fmr, arg); va_end(arg); if( log_buffer[0] != '\0' ) { + printf("log_level %d", loglevel); switch( loglevel ) { case LogLevelVerbose: LOG_PRINTF("[V][%d][%s:%d] %s\r\n", HEAP_AVAILABLE(), TGZ_PATHNAME(path), line, log_buffer); break; case LogLevelDebug: LOG_PRINTF("[D][%d][%s:%d] %s\r\n", HEAP_AVAILABLE(), TGZ_PATHNAME(path), line, log_buffer); break; diff --git a/src/libpacker/LibPacker.cpp b/src/libpacker/LibPacker.cpp index a30906e..750e34f 100644 --- a/src/libpacker/LibPacker.cpp +++ b/src/libpacker/LibPacker.cpp @@ -36,49 +36,6 @@ -namespace TarPacker -{ - uint32_t readbuf_size = 1024; - uint8_t *readbuf = NULL; - uint32_t readbuf_numblocks = 0; - - uint32_t writebuf_size = 4096; - uint8_t *writebuf = NULL; - uint32_t writebuf_numblocks = 0; - - fs::File fileRO; // file handle for input files to read - fs::File fileRW; // file handle for output tar to write - - - namespace io - { - void * open(void *_fs, const char *filename, const char *mode); - int close(void *fs, void *file); - int stat(void *_fs, const char *path, void *_stat); - ssize_t read(void *_fs, void *_stream, void * buf, size_t count); - ssize_t write_buffered(void *_fs, void *_stream, void * buf, size_t count); - ssize_t write_stream(void *_fs, void *_stream, void * buf, size_t count); - ssize_t write_finalize(void*_fs, void*_stream); - - - tar_callback_t TarIOFunctions = { - .openfunc = io::open, - .closefunc = io::close, - .readfunc = io::read, - .writefunc = io::write_buffered, - .closewritefunc = io::write_finalize, - .statfunc = io::stat - }; - - }; - - using io::TarIOFunctions; - -}; - - - - namespace LZPacker { // LZPacker uses buffered streams @@ -91,7 +48,7 @@ namespace LZPacker void (*progressCb)( size_t progress, size_t total ) = nullptr; // write LZ77 header - size_t lzHeader(uint8_t* buf, bool gzip_header=true) + size_t lzHeader(uint8_t* buf, bool gzip_header) { assert(buf); size_t len = 0; @@ -123,7 +80,7 @@ namespace LZPacker // write LZ77 footer - size_t lzFooter(uint8_t* buf, size_t outlen, unsigned crc, bool terminate=false) + size_t lzFooter(uint8_t* buf, size_t outlen, unsigned crc, bool terminate) { assert(buf); size_t len = 0; @@ -267,6 +224,7 @@ namespace LZPacker // stream to stream size_t compress( Stream* srcStream, size_t srcLen, Stream* dstStream ) { + log_d("LZPacker::compress(stream, size=%d, stream)", srcLen); assert(srcStream); assert(srcLen>0); assert(dstStream); @@ -302,12 +260,14 @@ namespace LZPacker int input_bytes = 0; // for do..while state size_t total_bytes = 0; // for progress meter + //size_t total_read_bytes = 0; int defl_mode = Z_BLOCK; uint8_t footer[8]; size_t footer_len = 0; size_t write_size = 0; + size_t written_bytes = 0; do { if(uzstream.out.avail > 0){ @@ -321,11 +281,13 @@ namespace LZPacker log_e("No more input bytes, srcStream->readBytes() miss?"); //prev_state = Z_STREAM_END; break; - } - if(input_bytes == -1) { + } else if(input_bytes == -1) { log_e("srcStream->readBytes() failed"); return -1; + } else { + // log_d("srcStream->readBytes() returned %d bytes", input_bytes); } + uzstream.in.next = inputBuffer; uzstream.in.avail = input_bytes; @@ -342,7 +304,21 @@ namespace LZPacker write_size = uzstream.out.next - outputBuffer; - dstLen += dstStream->write(outputBuffer, write_size); // write to output stream + written_bytes = dstStream->write(outputBuffer, write_size); // write to output stream + + if( written_bytes == 0 ) { + log_e("Write failed at offset %d", input_bytes ); + break; + } else { + log_v("Wrote %d/%d bytes", written_bytes, write_size ); + } + + dstLen += written_bytes; + + if( total_bytes > srcLen ) { + log_e("Read more bytes (%d) than source contains (%d), something is wrong", total_bytes, srcLen); + break; + } } while(prev_state==Z_OK); @@ -352,15 +328,17 @@ namespace LZPacker success = false; } + if( c->progress_cb ) + c->progress_cb(srcLen, srcLen); // send progress end signal, whatever the outcome + if( total_bytes != srcLen ) { success = false; int diff = srcLen - total_bytes; if( diff>0 ) { - log_e("Bad input stream size: could not read all of %d requested bytes, missed %d bytes.", srcLen, diff); + log_e("Bad input stream size: could not read every %d bytes, missed %d bytes.", srcLen, diff); } else { - log_e("Bad input stream size: read more than %d requested bytes, got %d extra bytes.", srcLen, -diff); + log_e("Bad input stream size: read %d further than %d requested bytes.", -diff, srcLen); } - srcLen = total_bytes; } @@ -420,72 +398,104 @@ namespace LZPacker - - - - - namespace TarPacker { - using namespace TAR; + uint8_t block_buf[512]; + + static bool use_lock = false; + static bool targzlock = false; - void deallocReadBuffer() + void (*progressCb)( size_t progress, size_t total ) = nullptr; + + // progress callback setter + void setProgressCallBack(totalProgressCallback cb) { - free(TarPacker::readbuf); - TarPacker::readbuf = NULL; + TarPacker::progressCb = cb; } - void deallocWriteBuffer() + size_t readBytesAsync(tar_params_t *params, uint8_t* buf, size_t len); + + void takeLock(); + void setLock(bool set=true); + void releaseLock(); + + namespace io { - free(TarPacker::writebuf); - TarPacker::writebuf = NULL; - } + fs::File fileRO; + fs::File fileRW; + + void * open(void *_fs, const char *filename, const char *mode); + int close(void *fs, void *file); + int stat(void *_fs, const char *path, void *_stat); + ssize_t read(void *_fs, void *_stream, void * buf, size_t count); + ssize_t write_finalize(void*_fs, void*_stream); + ssize_t write_stream(void *_fs, void *_stream, void * buf, size_t count); + }; + - void deallocBuffers() + tar_callback_t TarIOFunctions = { + .src_fs = nullptr, + .dst_fs = nullptr, + .openfunc = io::open, // r/w + .closefunc = io::close, // r/w + .readfunc = io::read, // ro + .writefunc = io::write_stream, // wo + .closewritefunc = io::write_finalize, + .statfunc = io::stat + }; + + std::vector _tarEntities; + size_t _tar_estimated_filesize = 0; + TAR::TAR* _tar; + + size_t readBytesAsync(tar_params_t *params, uint8_t* buf, size_t len) { - deallocReadBuffer(); - deallocWriteBuffer(); + if(len%512!=0) { // not a multiple of 512 + log_e("%d is not a multiple of 512!", len); + return 0; + } + int idx = 0; + do { + while(!targzlock) + vTaskDelay(1); + memcpy(&buf[idx], block_buf, 512); + releaseLock(); + idx += 512; + if( idx == len ) + return len; + } while(params->ret == 1); + + return idx; } - bool allocReadBuffer(uint32_t readbuf_size) + void takeLock() { - TarPacker::readbuf = (uint8_t*)calloc(1, readbuf_size); - if( TarPacker::readbuf==NULL ) { - log_e("Failed to alloc %lu bytes for read buffer", readbuf_size); - return false; + if( use_lock ) { + while(targzlock) + vTaskDelay(1); } - return true; } - bool allocWriteBuffer(uint32_t writebuf_size) + void setLock(bool set) { - TarPacker::writebuf = (uint8_t*)calloc(1, writebuf_size); - if( TarPacker::writebuf==NULL ) { - log_e("Failed to alloc %lu bytes for read buffer", writebuf_size); - return false; + if( use_lock ) { + targzlock = set; } - return true; } - bool allocBuffers() + void releaseLock() { - if( !allocReadBuffer(TarPacker::readbuf_size) ) { - return false; + if( use_lock ) { + setLock( false ); } - if( !allocWriteBuffer(TarPacker::writebuf_size) ) { - deallocReadBuffer(); - return false; - } - return true; } - // i/o functions namespace io { @@ -496,51 +506,42 @@ namespace TarPacker fs::FS* fs = (fs::FS*)_fs; String flagStr; void* retPtr; - - log_d("io::open(%s, mode=%s)", filename, mode); - + log_v("io::open(%s, mode=%s)", filename, mode); if( String(mode) == "r" ) { - readbuf_numblocks = 0; // reset read buffer flagStr = "r"; - TarPacker::fileRO = fs->open(filename, "r"); - if(!TarPacker::fileRO) { + fileRO = fs->open(filename, "r"); + if(!fileRO) { log_e("Unable to open %s for reading", filename); return (void*)-1; } - retPtr = &TarPacker::fileRO; + retPtr = &fileRO; } else { - writebuf_numblocks = 0; // reset write buffer flagStr = "w"; - TarPacker::fileRW = fs->open(filename, "w"); - if(!TarPacker::fileRW) { + fileRW = fs->open(filename, "w"); + if(!fileRW) { log_e("Unable to open %s for writing", filename); return (void*)-1; } - retPtr = &TarPacker::fileRW; + retPtr = &fileRW; } - return retPtr; } - int close(void *fs, void *file) { assert(file); fs::File* filePtr = (fs::File*)file; - - log_d("io::close(fs, %s)", filePtr->name() ); - + log_v("io::close(fs, %s)", filePtr->name() ); filePtr->close(); return 0; } - int stat(void *_fs, const char *path, void *_stat) { - assert(_fs); - assert(_stat); + if(!_fs || !_stat) + return -1; fs::FS* fs = (fs::FS*)_fs; struct stat *s = (struct stat *)_stat; static int inode_num = 0; @@ -549,9 +550,8 @@ namespace TarPacker log_e("Path %s does not exist", path); return -1; } - log_v("stat_func: stating %s", path ); - fs::File f = fs->open(path, "r"); + fs::File f = fs->open(path, "r"); if(!f) { log_e("Unable to open %s for stat", path); return -1; @@ -567,7 +567,7 @@ namespace TarPacker s->st_mtime = strcmp(path, "/") == 0 ? 0 : f.getLastWrite(); f.close(); - + log_v("stat_func: [%s] %s %d bytes", is_dir?"dir":"file", path, s->st_size ); return 0; } @@ -576,40 +576,15 @@ namespace TarPacker { assert(_stream); Stream* streamPtr = (Stream*)_stream; - - uint32_t buf_idx = (readbuf_numblocks*T_BLOCKSIZE)%readbuf_size; - ssize_t readbytes = count; - if( buf_idx == 0 ) { - readbytes = streamPtr->readBytes(TarPacker::readbuf, readbuf_size); - if( readbytes readbuf_size ) { - log_e("Aborting read to prevent buffer overflow\n"); - return -1; - } - memcpy(buf, &TarPacker::readbuf[buf_idx], count ); - - readbuf_numblocks++; - - return count > 0 ? count : -1; + return streamPtr->readBytes(block_buf, count); } ssize_t write_finalize(void*_fs, void*_stream) { assert(_stream); - Stream* streamPtr = (Stream*)_stream; - - uint32_t leftover_bytes = (writebuf_numblocks*T_BLOCKSIZE)%writebuf_size; - if( leftover_bytes == 0 ) { - log_d("writefunc::commit_finalize(no leftover bytes at block %d)", writebuf_numblocks); - return 0; // no data leftover in buffer - } - log_d("writefunc::commit_finalize(%lu leftover bytes)", leftover_bytes ); - return streamPtr->write(TarPacker::writebuf, leftover_bytes); + // Stream* streamPtr = (Stream*)_stream; + return 0; } @@ -617,455 +592,329 @@ namespace TarPacker ssize_t write_stream(void *_fs, void *_stream, void * buf, size_t count) { Stream* streamPtr = (Stream*)_stream; - return streamPtr->write(TarPacker::writebuf, writebuf_size); - } - - - // buffered writes - ssize_t write_buffered(void *_fs, void *_stream, void * buf, size_t count) - { - assert(_stream); - Stream* streamPtr = (Stream*)_stream; - // printf("write: Got output tar file handle: %p at position %lu\n", _stream ); - uint32_t buf_idx = (writebuf_numblocks*T_BLOCKSIZE)%writebuf_size; - - if( buf_idx + count > TarPacker::writebuf_size ) { - log_e("Aborting write to prevent buffer overflow"); - return -1; - } - - if( writebuf_numblocks>0 && buf_idx==0 ) { // commit buffer write - log_d("writefunc::commit(%lu bytes, block=%lu, idx=%lu)", writebuf_size, writebuf_numblocks, buf_idx); - ssize_t written = streamPtr->write(TarPacker::writebuf, writebuf_size); - if( written != writebuf_size ) { - log_e("Buffer write fail (req %u bytes, wrote %d/%lu bytes)", count, written, writebuf_size ); - return -1; - } - } else { - log_d("writefunc::buffer(req %u/%lu bytes, block=%lu, idx=%lu)", count, writebuf_size, writebuf_numblocks, buf_idx); - } - - memcpy(&TarPacker::writebuf[buf_idx], buf, count ); - - writebuf_numblocks++; - - return count; + return streamPtr->write((uint8_t*)buf, count); } }; // end namespace io - bool pack_files_step(tar_files_packer_t *packer); - // private: batch append files - int append_files(tar_files_packer_t*p) + int add_header(TAR::TAR* tar, tar_entity_t tarEntity) { + takeLock(); - assert(p); - do { - if(!pack_files_step(p)) { - p->status = false; - break; - } - } while( p->step != PACK_FILES_END ); - - // free(p->tar); + struct stat entity_stat; - return p->status ? 0 : 1; + if( tar->io->statfunc(tar->io->src_fs, tarEntity.realpath.c_str(), &entity_stat) != 0) { + // file or dir not found + log_e("File not found: %s", tarEntity.realpath.c_str() ); + return -1; + } + memset(&(tar->th_buf), 0, sizeof(struct tar_header)); // clear header buffer + th_set_from_stat(tar, &entity_stat); // set header block + th_set_path(tar, tarEntity.savepath.c_str()); // set the header path + ssize_t written_bytes = 0; + if (th_write(tar, &written_bytes) != 0) { // header write failed? + return -1; + } + if( written_bytes != 512 ) { // th_write() made a boo boo! + log_e("header write failed for: %s", tarEntity.realpath.c_str() ); + return -1; + } + setLock(); + return 512; } - // private: state machine stepper when appending files to a tar archive - int append_files_step(tar_files_packer_t*p) + int add_body_chunk(TAR::TAR* tar) { - if( p == nullptr || p->dirIterator == nullptr ) { // prevent crashing - log_e("Malformed tar_files_packer, aborting"); + takeLock(); + memset(block_buf, 0, 512); + ssize_t read_bytes = tar->io->readfunc(tar->io->src_fs, tar->src_file, block_buf, 512); + if( read_bytes <= 0 ) { + log_e("ReadBytes Failed"); return -1; } + if( read_bytes != 512 ) { // block was zero padded anyway + log_v("only got %d bytes of %d requested", read_bytes, 512 ); + } + auto ret = tar->io->writefunc(tar->io->dst_fs, tar->dst_file, block_buf, 512); + setLock(); + return ret; + } - dir_iterator_t* dirIterator = p->dirIterator; // local alias to make code easier to read - - enum steps_t { init, loop, eof, end }; - - static steps_t step = init; - - if(!p->tar) - step = init; - if(step!=loop) { - log_d("Step: %d p->tar = %p", step, p->tar); - } - switch(step) - { - case init: - p->tar = (TAR::TAR*)calloc(1, sizeof(TAR::TAR)); - if( p->tar == NULL ) { - log_e("Failed to alloc %d bytes for p->tar", sizeof(TAR::TAR) ); - } - p->status = tar_open(p->tar, p->tar_path, p->io, "w", p->fs) == 0 ? true : false; - log_d("INIT p->tar = %p", p->tar); - if(!p->status) { - log_e("Failed to open input tar for writing"); - return -1; - } - // basic health check, first entity in tar should be the root directory - if( !dirIterator->available() ) { - log_e("Error: first item should be a directory e.g. '/'"); - return -1; - } - step = loop; // begin loop - if( !dirIterator->next() ) { // append root directory entity - return -1; - } - log_d("ROOTDIR (%s -> %s)", dirIterator->dirEntity.path.c_str(), dirIterator->SavePath.c_str()); - // process TAR header - do { - if( tar_append_entity_step(&dirIterator->entity) != 0 ) - return -1; - } while( ! dirIterator->stepComplete() ); - // fallthrough - case loop: - if( dirIterator->iterator > 1 && !dirIterator->stepComplete() ) { // in loop doing entity step - log_v("-> [ x ] ENTITY STEP (%s)", dirIterator->dirEntity.path.c_str()); - return tar_append_entity_step(&dirIterator->entity); - } else { - if( dirIterator->stepComplete() ) { - log_d("Finished adding %s", dirIterator->dirEntity.path.c_str()); - } - } - // check for main loop end - if( dirIterator->complete() ) { - if( dirIterator->stepComplete() ) { - log_d("No more files to add"); - step = eof; - } - return 0; - } - // in loop assigning next entity, or skipping if entity is filtered - if( ! dirIterator->next() ) { - log_d("Skipping entity #%d (%s)", dirIterator->iterator, dirIterator->dirEntity.path.c_str()); - dirIterator->entity.step = ENTITY_END; - } - return 0; - case eof: log_d("Appending EOF to TAR archive (p->tar = %p)", p->tar); - if( tar_append_eof(p->tar, p->written_bytes) != 0 ) - return -1; - step = end; - // return 0; - // fallthrough - case end: log_d("Calling io::close(p->tar = %p)", p->tar); - p->entity_step = PACK_FILES_END; - if( tar_close(p->tar) !=0 ) { - log_e("Failed to close archive"); - return -1; - } - free( p->tar ); - p->tar = NULL; - p->status = true; - step = init; - break; - } - return p->status ? 0 : -1; + int add_eof_chunk(TAR::TAR* tar) + { + takeLock(); + memset(block_buf, 0, 512); + size_t written_bytes = tar->io->writefunc(tar->io->dst_fs, tar->dst_file, block_buf, 512); + setLock(); + return written_bytes; } - // private: state machine stepper when packing files - bool pack_files_step(tar_files_packer_t *packer) + int pack_tar_init(tar_callback_t *io, fs::FS *srcFS, std::vector dirEntities, fs::FS *dstFS, const char*output_file_path, const char* tar_prefix=nullptr) { - static bool ret = false; + _tarEntities.clear(); - switch(packer->step) - { - case PACK_FILES_SETUP: - log_d("Pack files setup"); - ret = false; - if( !allocBuffers()) - return false; - packer->step = PACK_FILES_STEP; - // fallthrough - case PACK_FILES_STEP : - log_v("Pack step"); - if(TarPacker::append_files_step(packer) != 0 ) - return false; - if(packer->entity_step == PACK_FILES_END ) { - packer->status = true; // success! - // TODO: trigger last write? - packer->step = PACK_FILES_END; - } else - return true; - // fallthrough - case PACK_FILES_END : - log_v("PACK_FILES_END"); - deallocBuffers(); - ret = packer->status; - break; - } - - return ret; - } + _tar_estimated_filesize = 0; + for(int i=0;i0) { + _tar_estimated_filesize += d.size + (512 - (d.size%512)); // align to 512 bytes + } - // public - size_t pack_files(fs::FS *fs, std::vector *dirEntities, const char*tar_path, const char*dst_path) - { - assert(fs); - assert(dirEntities); + auto realpath = d.path; + auto savepath = tar_prefix ? String(tar_prefix)+realpath : realpath; - TarGzStream tarStream(fs, dirEntities, tar_path, NULL, dst_path); + if( !tar_prefix && savepath.startsWith("/") ) // tar paths can't be slash-prepended, add a dot + savepath = "." + savepath; - if( tarStream.size() <= 0 ) { - log_e("Nothing to archive!"); - return -1; + _tarEntities.push_back( { realpath, savepath, d.is_dir, d.size } ); + log_w("Add entity( [%4s]\t%-32s\t%d bytes -> %s", d.is_dir?"DIR":"FILE", realpath.c_str(), d.size, savepath.c_str() ); } - ssize_t bytes_ready = 0; - uint8_t buf[4096]; + _tar_estimated_filesize += 1024; //tar footer - fs::File out = fs->open(tar_path, "w"); - if(!out) - return -1; + log_i("TAR estimated file size: %d", _tar_estimated_filesize ); - do { - ssize_t bytes_read = tarStream.readBytes(buf, 4096); - if( bytes_read<=0 ) {// EOF - log_v("EOF"); - break; - } + _tar = (TAR::TAR*)calloc(1, sizeof(TAR::TAR)); + if( _tar == NULL ) { + log_e("Failed to alloc %d bytes for tar", sizeof(TAR::TAR) ); + } - ssize_t bytes_written = out.write(buf, bytes_read); - if( bytes_written != bytes_read ) { - log_e("Write error, got %d bytes to write but wrote %d (total %d/%d)", bytes_read, bytes_written, bytes_ready, tarStream.size() ); - out.close(); - return -1; - } + io->src_fs = srcFS; + io->dst_fs = dstFS; - bytes_ready += bytes_read; - log_v("Bytes ready: %d", bytes_ready); - } while( bytes_ready0 ) { + chunks = ceil(float(current_entity.size)/512.0); + _tar->src_file = _tar->io->openfunc( _tar->io->src_fs, current_entity.realpath.c_str(), "r" ); + if( _tar->src_file == (void*)-1 ) { + log_e("Open failed for: %s", current_entity.realpath.c_str() ); + return -1; + } + // log_d("got %d chunks in %d bytes for %s", chunks, current_entity.size, current_entity.realpath.c_str() ); + for(int c=0;cio->closefunc( _tar->io->src_fs, _tar->src_file ); + } } + total_bytes += add_eof_chunk(_tar); + total_bytes += add_eof_chunk(_tar); - tarGzStreamPtr = NULL; + if( TarPacker::progressCb ) + TarPacker::progressCb(_tar_estimated_filesize,_tar_estimated_filesize); // send end signal, whatever the outcome - using namespace TarPacker; + tar_close(_tar); - // restore the original function before lambda is destroyed - IOFunctions = TarIOFunctions; + return total_bytes; } - void TarGzStream::_init() + + void pack_tar_task(void* params) { - // init tar files packer - assert(srcFS); - assert(dirEntities); - assert(tar_path); - assert(bufSize>=4096); + tar_params_t *p = (tar_params_t*)params; + p->ret = pack_tar_impl(); + vTaskDelete(NULL); + } - using namespace TarPacker; - buffer = (uint8_t*)malloc(bufSize); // create tar<->gz buffer - if( buffer == NULL ) { - log_e("Failed to allocate %d bytes for tar to gz buffer, halting", bufSize); - while(1) yield(); - } - log_d("Stream to Stream mode enabled"); - - // for lambda capture - tarGzStreamPtr = this; - - IOFunctions = { - // Override the default open() callback triggered from tar_open() to optionally return this stream: - // - opening mode = read only -> return (void*) file descriptor (legacy behaviour) - // - opening mode = write only -> return (Stream*)[this] - .openfunc = [](void *_fs, const char *filename, const char *mode) -> void* - { - if(String(mode)=="r") { // open input file to archive on behalf of tar - readbuf_numblocks = 0; // reset gz read buffer - return TarIOFunctions.openfunc(_fs, filename, mode); // return file descriptor - } else { // opening gz output on behalf of LZPacker - writebuf_numblocks = 0; // reset gz write buffer - return tarGzStreamPtr; // return stream - } - }, - .closefunc = [](void *fs, void *file) -> int { - if( file == tarGzStreamPtr ) { - log_d("GZ: Closing stream"); - return 0; - } - log_d("TAR: Closing input file %s", ((fs::File*)file)->name() ); - return TarIOFunctions.closefunc(fs, file); - }, - .readfunc = io::read, - .writefunc = io::write_buffered, - .closewritefunc = io::write_finalize, - .statfunc = io::stat + int pack_files(fs::FS *srcFS, std::vector dirEntities, Stream* dstStream, const char* tar_prefix) + { + auto TarStreamFunctions = TarIOFunctions; + TarStreamFunctions.src_fs = srcFS; + TarStreamFunctions.dst_fs = nullptr; + + tar_params_t params = + { + .srcFS = srcFS, + .dirEntities = dirEntities, + .dstFS = nullptr, + .output_file_path = nullptr, + .tar_prefix = tar_prefix, + .io = &TarStreamFunctions, + .ret = 1 }; - // estimate the full tar size so that LZPacker can write the gz header correctly - total_bytes = 0; + ssize_t tar_estimated_filesize = pack_tar_init(params.io, params.srcFS, params.dirEntities, params.dstFS, params.output_file_path, params.tar_prefix); + if( tar_estimated_filesize <=0 ) + return -1; - log_i("[%4s] %-32s %-8s\t%-8s", "Type", "Path", "FileSize", "TarSize" ); - log_i("[%4s] %-32s %-8s\t%-8s", "----", "--------------------------------", "--------", "--------" ); + _tar->dst_file = dstStream; + use_lock = false; - for(int i=1; isize();i++) { - auto dirEntity = dirEntities->at(i); - if( dirEntity.path == String(tar_path) // skip root dir - || dirEntity.path == String(tgz_path)) // skip tgz file - continue; - size_t tar_size = 512; // tar entity header - if(! dirEntity.is_dir ) - tar_size += dirEntity.size + (512 - (dirEntity.size%512)); // add file size aligned to 512 bytes - log_i("[%4s] %-32s %8d\t%8d", dirEntity.is_dir?"dir":"file", dirEntity.path.c_str(), dirEntity.size, tar_size ); - total_bytes += tar_size; - } - - total_bytes += 1024; // tar eof + TaskHandle_t tarTaskHandle = NULL; + xTaskCreate(pack_tar_task, "pack_tar_task", 4096, ¶ms, tskIDLE_PRIORITY, &tarTaskHandle ); - log_i("Tar estimated file size for %d elements: %d", dirEntities->size(), total_bytes); + while(params.ret == 1) { + vTaskDelay(1); + } - dirIterator = { &tar_files_packer, dirEntities }; + vTaskDelay(1); - tar_files_packer = TAR::tar_files_packer_t{ - .tar = NULL, - .fs = srcFS, - .tar_path = tar_path, - .tgz_path = tgz_path, - .dst_path = dst_path, - .written_bytes = &written_bytes, - .io = &IOFunctions, - .status = false, - .step = PACK_FILES_SETUP, - .entity_step = PACK_FILES_SETUP, - .dirIterator = &dirIterator - }; + free(_tar); + return params.ret; } - // write to gz input buffer - size_t TarGzStream::write(const uint8_t* data, size_t len) + int pack_files(fs::FS *srcFS, std::vector dirEntities, fs::FS *dstFS, const char*tar_output_file_path, const char* tar_prefix) { - if( len > bufSize ) { - log_w("Bad Write request (%d bytes) exceeds targz buffer size (total %d), aborting", len, bufSize); + auto tar = dstFS->open(tar_output_file_path, "w"); + if(!tar) return -1; - } - log_d("writestream: store(%d bytes)", len); - bytes_ready = len; - memcpy(buffer, data, len); - return len; + auto ret = pack_files(srcFS, dirEntities, &tar, tar_prefix); + tar.close(); + return ret; } +}; // end namespace TarPacker - // read bytes from input tarPacker - size_t TarGzStream::readBytes(uint8_t* dest, size_t len) - { - using namespace TAR; - - if( len > bufSize ) { - log_e("Bad Read request (%d bytes) exceeds targz buffer size (%d bytes), aborting", len, bufSize); - return -1; - } - - bytes_ready = 0; - - do { - if(!TarPacker::pack_files_step(&tar_files_packer)) { - log_w("pack_files_step failed"); - tar_files_packer.status = false; - break; - } - // until buffer full or end of tar - } while( bytes_ready==0 && tar_files_packer.step != PACK_FILES_END ); - if( bytes_ready != len ) { // not using the entire buffer ? - log_d("Zerofilling buffer (len=%d, ready=%d)", len, bytes_ready); - memset(dest, 0, len); // zerofill buffer - } - memcpy(dest, buffer, bytes_ready ); +namespace TarGzPacker +{ + using namespace TAR; + using namespace TarPacker; - return bytes_ready; - } + class TarGzStreamReader : public Stream + { + private: + TAR::tar_params_t* tar_params; + public: + TarGzStreamReader(TAR::tar_params_t* tar_params) : tar_params(tar_params) { }; + ~TarGzStreamReader() { }; + // read bytes from input tarPacker + virtual size_t readBytes(uint8_t* dest, size_t count) { return TarPacker::readBytesAsync(tar_params, dest, count); } + // all other methods are unused + virtual size_t write(const uint8_t* data, size_t len) { return 0; }; + virtual int available() { return 0; }; + virtual int read() { return 0; }; + virtual int peek() { return 0; }; + virtual void flush() { }; + virtual void end() { }; + virtual size_t write(uint8_t c) { return c?0:0; }; + }; - size_t compress(fs::FS *srcFS, const char* src_path, const char* tgz_path, const char* dst_path) + int compress(fs::FS *srcFS, std::vector dirEntities, Stream* dstStream, const char* tar_prefix) { + auto TarGzStreamFunctions = TarIOFunctions; - log_d("compress(fs::FS*, src_path=%s, tgz_path=%s, dst_path=%s)", src_path, tgz_path?tgz_path:"null", dst_path?dst_path:"null"); + TarGzStreamFunctions.src_fs = srcFS; + TarGzStreamFunctions.dst_fs = nullptr; - std::vector dirEntities; + TarGzStreamFunctions.writefunc = [](void *_fs, void *_stream, void * buf, size_t count)->ssize_t { + if(count!=512) { + log_e("BAD block write (req=%d, avail=512)", count ); + return -1; + } + memcpy(block_buf, buf, count); + return 512; + }; - collectDirEntities(&dirEntities, srcFS, src_path, 3); + tar_params_t tp = + { + .srcFS = srcFS, + .dirEntities = dirEntities, + .dstFS = nullptr, // none when streaming + .output_file_path = nullptr, // none when streaming + .tar_prefix = tar_prefix, + .io = &TarGzStreamFunctions, + .ret = 1 + }; - TarGzStream tarStream(srcFS, &dirEntities, src_path, tgz_path, dst_path, LZPacker::inputBufferSize); + ssize_t srcLen = pack_tar_init(tp.io, tp.srcFS, tp.dirEntities, tp.dstFS, tp.output_file_path, tp.tar_prefix); + if( srcLen <=0 ) + return -1; - if( tarStream.size() <= 0 ) { - log_e("Nothing to compress, aborting"); - return 0; - } + TarGzStreamReader tarStream(&tp); - auto tarGzFile = srcFS->open(tgz_path, "w"); + log_d("Tar estimated data size: %d bytes", srcLen); - if(!tarGzFile) { - log_e("Can't open %s for writing, aborting", tgz_path); - return 0; - } + // set async + use_lock = true; + releaseLock(); + + TaskHandle_t tarGzTaskHandle = NULL; + xTaskCreate(pack_tar_task, "pack_tar_task", 4096, &tp, tskIDLE_PRIORITY, &tarGzTaskHandle ); - auto compressedSize = TarGzPacker::compress(&tarStream, &tarGzFile); + auto ret = LZPacker::compress(&tarStream, srcLen, dstStream); - tarGzFile.close(); + vTaskDelay(1); - return compressedSize; + free(_tar); + return ret; } - size_t compress(TarGzStream *tarStream, Stream *dstStream) + int compress(fs::FS *srcFS, std::vector dirEntities, fs::FS *dstFS, const char* tgz_name, const char* tar_prefix) { - log_d("compress(TarGzStream*, Stream *)"); - assert(tarStream); - assert(dstStream); - - if( !tarStream->ready() ) { - log_e("TarGz creation failed, aborting"); - return 0; + auto dstFile = dstFS->open(tgz_name, "w"); + if(!dstFile) { + log_e("Can't open %s for writing", tgz_name); + return -1; } + auto ret = compress(srcFS, dirEntities, &dstFile, tar_prefix); + dstFile.close(); + return ret; + } - log_d("Input size: %d bytes", tarStream->size()); - return LZPacker::compress( tarStream, tarStream->size(), dstStream ); - } }; // end namespace TarGzPacker diff --git a/src/libpacker/LibPacker.hpp b/src/libpacker/LibPacker.hpp index f8d29e1..a7c41d2 100644 --- a/src/libpacker/LibPacker.hpp +++ b/src/libpacker/LibPacker.hpp @@ -36,6 +36,54 @@ #include "./common.hpp" + +// struct BasePacker +// { +// BasePacker(); +// void haltOnError( bool halt ); +// }; +// +// struct TarPacker : virtual public BasePacker +// { +// TarPacker(); +// +// std::vector *dirEntities = nullptr; +// fs::FS* fs = nullptr; +// +// // input data +// std::vector dirEntities getEntities(fs::FS* fs, const char* tar_inputdir); +// void setEntities(fs::FS* fs, std::vector dirEntities); +// +// // tar archive property +// void setRoot(const char* tar_rootdir); +// +// // output data +// void setOutput(fs::FS*, const char* tar_output_filename); +// void setOutputStream(Stream* stream); +// void setOutputFile(Stream* stream); +// +// // batch process dirEntities +// void pack_files(); +// +// void open(); +// void add_entity(); +// void close(); +// +// }; +// +// struct GzPacker : virtual public BasePacker +// { +// +// }; +// +// struct TarGzPacker : public TarPacker, public GzPacker +// { +// +// }; + + + + // .gz compressor (LZ77/deflate) namespace LZPacker { @@ -50,67 +98,36 @@ namespace LZPacker // progress callback setter [](size_t bytes_read, size_t total_bytes) void setProgressCallBack(totalProgressCallback cb); void defaultProgressCallback( size_t progress, size_t total ); + + size_t lzHeader(uint8_t* buf, bool gzip_header=true); + size_t lzFooter(uint8_t* buf, size_t outlen, unsigned crc, bool terminate=false); + struct GZ::uzlib_comp* lzInit(); + }; -// .tar packager namespace TarPacker { using namespace TAR; - // TAR pack from directory to a tar file on same filesystem - size_t pack_files(fs::FS *srcFS, std::vector *dirEntities, const char*tar_path, const char*dst_path); - // TAR pack from directory entities to output stream - //size_t pack_files(tar_files_packer_t *packer); -}; + int pack_files(fs::FS *srcFS, std::vector dirEntities, Stream* dstStream, const char* tar_prefix=nullptr); + int pack_files(fs::FS *srcFS, std::vector dirEntities, fs::FS *dstFS, const char*tar_output_file_path, const char* tar_prefix=nullptr); + void setProgressCallBack(totalProgressCallback cb); + void defaultProgressCallback( size_t progress, size_t total ); + +} -// .tar.gz packager+compressor namespace TarGzPacker { - // tar to gz stream helper - class TarGzStream : public Stream - { - private: - fs::FS *srcFS = nullptr; - std::vector *dirEntities = nullptr; - const char* tar_path = nullptr; // e.g. "/test.tar", or "/" if streaming to gz - const char* tgz_path = nullptr; // e.g "/text.tar.gz" - const char* dst_path = nullptr; - ssize_t written_bytes = 0; - ssize_t total_bytes = 0; - ssize_t bytes_ready = 0; - uint8_t *buffer = nullptr; - size_t bufSize = 4096; - TAR::tar_files_packer_t tar_files_packer; - TAR::dir_iterator_t dirIterator; - void _init(); - public: - TarGzStream(fs::FS *srcFS, std::vector *dirEntities, const char* tar_path, const char* tgz_path=NULL, const char* dst_path=NULL, size_t bufSize=4096) - : srcFS(srcFS), dirEntities(dirEntities), tar_path(tar_path), dst_path(dst_path), bufSize(bufSize) { _init(); }; - ~TarGzStream(); - // write to gz input buffer - virtual size_t write(const uint8_t* data, size_t len); - // read bytes from input tarPacker - virtual size_t readBytes(uint8_t* dest, size_t count); - // return evaluated tar size - ssize_t size() { return total_bytes; } - inline bool ready() { return size()>0; }; - // Stream methods (not used in this implementation) - virtual int available() { return 0; }; - virtual int read() { return 0; }; - virtual int peek() { return 0; }; - virtual void flush() { }; - // Print methods (not used in this implementation) - virtual void end() { }; - virtual size_t write(uint8_t c) { return c?0:0; }; - }; - - // tar gz methods - size_t compress(fs::FS *srcFS, const char* src_path, const char* tgz_path=NULL, const char* dst_path=NULL); - size_t compress(TarGzStream *tarStream, Stream *dstStream); + using namespace TAR; + + int compress(fs::FS *srcFS, std::vector dirEntities, Stream* dstStream, const char* tar_prefix=nullptr); + int compress(fs::FS *srcFS, std::vector dirEntities, fs::FS *dstFS, const char* tgz_name, const char* tar_prefix=nullptr); + }; -using TarGzPacker::TarGzStream; + +// using TarGzPacker::TarGzStreamReader; diff --git a/src/libpacker/common.hpp b/src/libpacker/common.hpp index b5a4b7d..b3ad4da 100644 --- a/src/libpacker/common.hpp +++ b/src/libpacker/common.hpp @@ -13,15 +13,17 @@ namespace GZ } +#if defined ARDUINO_ARCH_RP2040 + #include + #include +#endif + + namespace TAR { [[maybe_unused]] inline static int max_path_len=255; - - struct tar_files_packer_t; // forward declaration - struct dir_iterator_t; // forward declaration - struct dir_entity_t { String path{""}; @@ -30,81 +32,27 @@ namespace TAR }; - enum tar_files_packer_step_t + struct tar_entity_t { - PACK_FILES_SETUP, - PACK_FILES_STEP, - PACK_FILES_END - }; - - struct tar_files_packer_t - { - TAR *tar{nullptr}; - fs::FS*fs{nullptr}; - const char*tar_path{nullptr}; - const char* tgz_path{nullptr}; - const char*dst_path{nullptr}; - ssize_t *written_bytes{nullptr}; - tar_callback_t *io{nullptr}; - bool status{false}; - tar_files_packer_step_t step{PACK_FILES_SETUP}; - tar_files_packer_step_t entity_step{PACK_FILES_SETUP}; - dir_iterator_t *dirIterator{nullptr}; + String realpath{""}; // source path on filesystem + String savepath{""}; // dst path in tar archive + bool is_dir{false}; + size_t size{0}; }; - struct dir_iterator_t + struct tar_params_t { - - tar_files_packer_t *p{nullptr};; - std::vector *dirEntities{nullptr}; - size_t iterator{0}; - dir_entity_t dirEntity{"",false,0}; // last parsed entity - entity_t entity{ nullptr, nullptr, nullptr, nullptr, ENTITY_STAT, 0 }; - String SavePath{""}; - - inline bool available() { - if( !p || !dirEntities ) { - log_e("No packer!"); - return false; - } - - if( !dirEntities || dirEntities->size() == 0 ) { - log_e("No root entity!"); - return false; - } - dirEntity = dirEntities->at(0); - return dirEntity.is_dir; - } - - inline bool complete() { - assert(p); - assert( dirEntities ); - return iterator >= dirEntities->size(); - } - - inline bool stepComplete() { - return entity.step == ENTITY_END; - } - - inline bool next() { - assert(p); - assert( dirEntities ); - dirEntity = dirEntities->at(iterator); - SavePath = p->dst_path ? String(p->dst_path)+dirEntity.path : ""; - if(iterator==0) // root dir - entity = { p->tar, dirEntity.path.c_str(), SavePath.c_str(), p->written_bytes, ENTITY_STAT, -1 }; - else // child entity (dir or file) - entity = { p->tar, dirEntity.path.c_str(), (p->dst_path ? SavePath.c_str() : NULL), p->written_bytes, ENTITY_STAT, -1 }; - iterator++; - // recursion prevention: don't include target output file in the archive ;) - return ! (p->tgz_path != NULL && dirEntity.path==String(p->tgz_path)); // true = process, false = filter - } - + fs::FS *srcFS; + std::vector dirEntities; + fs::FS *dstFS; + const char* output_file_path; // may be .tar or .tar.gz + const char* tar_prefix; + tar_callback_t *io; + int ret; }; - // helper function to create dirEntities from a given folder inline void collectDirEntities(std::vector *dirEntities, fs::FS *fs, const char *dirname, uint8_t levels) { @@ -122,7 +70,8 @@ namespace TAR } //log_v(" DIR: %-16s", dirname); - dirEntities->push_back( { String(dirname), true, 0 } ); + if( String(dirname) != "/" ) + dirEntities->push_back( { String(dirname), true, 0 } ); File file = root.openNextFile(); diff --git a/src/libunpacker/LibUnpacker.hpp b/src/libunpacker/LibUnpacker.hpp index 72891ce..3230403 100644 --- a/src/libunpacker/LibUnpacker.hpp +++ b/src/libunpacker/LibUnpacker.hpp @@ -137,9 +137,12 @@ struct BaseUnpacker void tarGzClearError(); void haltOnError( bool halt ); void initFSCallbacks(); + + // TODO: move these debug tools outside the library void tarGzListDir( fs::FS &fs, const char * dirName, uint8_t levels, bool hexDump = false); void hexDumpData( const char* buff, size_t buffsize, uint32_t output_size = 32 ); void hexDumpFile( fs::FS &fs, const char* filename, uint32_t output_size = 32 ); + void setLoggerCallback( genericLoggerCallback cb ); void setupFSCallbacks( fsTotalBytesCb cbt, fsFreeBytesCb cbf ); // setup abstract filesystem helpers (totalBytes, freeBytes) #ifdef ESP8266 diff --git a/src/tar/libtar.c b/src/tar/libtar.c index b4937a8..55614cc 100644 --- a/src/tar/libtar.c +++ b/src/tar/libtar.c @@ -41,7 +41,6 @@ #include // for isprint() #include // for calloc(), because RP2040 wants it - // #include "../ESP32-targz-log.hpp" // import log_e(), log_w(), log_d() and log_i(), all behaving like printf() #define UID "root" @@ -51,7 +50,7 @@ char * tar_block; // init tar for archive creation -int tar_init(TAR *t, const char *pathname, tar_callback_t *io, void *opaque) +int tar_init(TAR *t, const char *pathname, tar_callback_t *io) { if (t == NULL) { //log_e("Failed to alloc %d files for tar", sizeof(TAR)); @@ -60,7 +59,15 @@ int tar_init(TAR *t, const char *pathname, tar_callback_t *io, void *opaque) t->pathname = (char *)pathname; t->io = io; - t->opaque = opaque; + + if( t->io->src_fs == NULL ) { + //log_e("Missing io->src_fs"); + return -1; + } + + if( t->io->dst_fs == NULL ) { + //log_d("No dst_fs provided, output may be a stream"); + } tar_block = (char*)calloc(1, T_BLOCKSIZE); if( tar_block==NULL ) { @@ -78,15 +85,19 @@ int tar_init(TAR *t, const char *pathname, tar_callback_t *io, void *opaque) return 0; } -// open a new tarfile handle -int tar_open(TAR *t, const char *pathname, tar_callback_t *io, const char *mode, void *opaque) + +int tar_open(TAR *t, const char *pathname, tar_callback_t *io) { - if (tar_init(t, pathname, io, opaque) == -1) + if (tar_init(t, pathname, io) == -1) return -1; - //log_d("Tar open %s with '%s' flag", pathname, mode); - t->fd = (*(t->io->openfunc))(t->opaque, pathname, mode); - if (t->fd == (void *)-1) { - //log_e("Failed to open file %s with %s flag", pathname, mode); + + if(t->io->dst_fs == NULL) // not using fs for output + return 0; + + //log_d("Tar open %s with 'w' flag", pathname); + t->dst_file = (*(t->io->openfunc))(t->io->dst_fs, pathname, "w"); + if (t->dst_file == (void *)-1) { + //log_e("Failed to open file %s with 'w' flag", pathname); return -1; } return 0; @@ -94,116 +105,17 @@ int tar_open(TAR *t, const char *pathname, tar_callback_t *io, const char *mode, -// close tarfile handle int tar_close(TAR *t) { int i = -1; - if(t->io->closefunc) - i = t->io->closefunc(t->opaque, t->fd); + if(t->io->dst_fs && t->io->closefunc) + i = t->io->closefunc(t->io->dst_fs, t->dst_file); free(tar_block); tar_block = NULL; return i; } - -typedef enum { - OPEN_FILE, - WRITE_BLOCK, - WRITE_LAST_BLOCK, - CLOSE_FILE -} regfile_steps_t; - - -regfile_steps_t regfile_step = OPEN_FILE; -int regfile_iterator = 0; -ssize_t regfile_read_bytes = 0; -ssize_t regfile_size = 0; -void * regfile_fd = NULL; -int regfile_rv = -1; - -// forward declaration -int tar_append_regfile_step(TAR *t, const char *realname, ssize_t *written_bytes); - - - -typedef enum { - TREE_OPEN, - TREE_APPEND_STEP, - TREE_APPEND_EOF, - TREE_CLOSE -} tree_packer_step_t; - - -static struct stat entity_stat; - - -int tar_append_entity_step(entity_t *e) -{ - assert(e); - switch( e->step ) - { - case ENTITY_STAT: - e->step_rv = -1; - if(e->tar == NULL || e->tar->io == NULL || e->tar->io->statfunc == NULL) { - // malformed entity - return -1; - } - if( e->tar->io->statfunc(e->tar->opaque, e->realname, &entity_stat) != 0) { - // file not found - return -1; - } - memset(&(e->tar->th_buf), 0, sizeof(struct tar_header)); // clear header buffer - th_set_from_stat(e->tar, &entity_stat); // set header block - th_set_path(e->tar, (e->savename ? e->savename : e->realname)); // set the header path - e->step = ENTITY_HEADER; - // fallthrough - case ENTITY_HEADER: //log_d("Entity header(%s)", e->realname); - if (th_write(e->tar, e->written_bytes) != 0) { // header write failed? - return -1; - } - e->step = th_is_regfile(e->tar) ? ENTITY_REGFILE : ENTITY_END; - return 0; - case ENTITY_REGFILE: - regfile_step = OPEN_FILE; - e->step = ENTITY_REGFILE_STEP; - // fallthrough - case ENTITY_REGFILE_STEP: - if( tar_append_regfile_step(e->tar, e->realname, e->written_bytes) == -1 ) - return -1; - if(regfile_step==CLOSE_FILE) { - e->step = ENTITY_END; - e->step_rv = 0; - } - return 0; - case ENTITY_END: - return e->step_rv; - } - - return 0; - -} - - - - - -// appends a dir entity (file or folder) to the tar archive -int tar_append_entity(entity_t* entity) -{ - assert(entity); - int ret_value = -1; - entity->step = ENTITY_STAT; - do { - ret_value = tar_append_entity_step(entity); - if( ret_value == -1 ) - return -1; - } while(entity->step!=ENTITY_END); - - return ret_value; -} - - // write EOF indicator int tar_append_eof(TAR *t, ssize_t *written_bytes) { @@ -211,7 +123,7 @@ int tar_append_eof(TAR *t, ssize_t *written_bytes) memset(tar_block, 0, T_BLOCKSIZE); for (j = 0; j < 2; j++) { - i = t->io->writefunc(t->opaque, t->fd, tar_block, T_BLOCKSIZE); + i = t->io->writefunc(t->io->dst_fs, t->dst_file, tar_block, T_BLOCKSIZE); *written_bytes += i; if (i != T_BLOCKSIZE) { if (i != -1) @@ -220,119 +132,22 @@ int tar_append_eof(TAR *t, ssize_t *written_bytes) return -1; } } + //log_v("EOF done"); // trigger last write signal (reminder: writefunc may be async) - if( t->io->closewritefunc(t->opaque, t->fd) == -1 ) + if( t->io->closewritefunc(t->io->dst_fs, t->dst_file) == -1 ) return -1; - //log_v("Wrote %d leftover bytes", leftover_bytes); - return 0; -} - - - - - - - -int tar_append_regfile_step(TAR *t, const char *realname, ssize_t *written_bytes) -{ - ssize_t block_size = 0; - - switch(regfile_step) - { - case OPEN_FILE: - regfile_iterator = 0; - regfile_rv = -1; - //log_d("Adding %s", realname); - regfile_fd = t->io->openfunc(t->opaque, realname, "r"); - if (regfile_fd == (void *)-1) - return -1; - regfile_size = (unsigned int)oct_to_int((t)->th_buf.size); - memset(tar_block, 0, T_BLOCKSIZE); - regfile_iterator = regfile_size; - //log_v("Assigning size (%d)", regfile_size); - regfile_step = WRITE_BLOCK; - // fallthrough - case WRITE_BLOCK: - { - //log_v("WRITE_BLOCK"); - if(regfile_iterator <= T_BLOCKSIZE ) { - regfile_step = WRITE_LAST_BLOCK; - return 0; - } - regfile_read_bytes = t->io->readfunc(t->opaque, regfile_fd, tar_block, T_BLOCKSIZE); - if (regfile_read_bytes != T_BLOCKSIZE) { - if (regfile_read_bytes != -1) { - errno = EINVAL; - } - regfile_step = CLOSE_FILE; - return 0; - } - block_size = t->io->writefunc(t->opaque, t->fd, tar_block, T_BLOCKSIZE); - if (block_size == -1) { - regfile_step = CLOSE_FILE; - return 0; - } - regfile_iterator -= block_size; - //log_v("Wrote %lu bytes, regfile_iterator=%d", T_BLOCKSIZE, regfile_iterator); - *written_bytes += block_size; - } - break; - case WRITE_LAST_BLOCK: - { - //log_v("case WRITE_LAST_BLOCK"); - regfile_step = CLOSE_FILE; - if (regfile_iterator > 0) { - regfile_read_bytes = t->io->readfunc(t->opaque, regfile_fd, tar_block, regfile_iterator); - if (regfile_read_bytes == -1) { - return 0; - } - memset(&tar_block[regfile_iterator], 0, T_BLOCKSIZE - regfile_iterator); - block_size = t->io->writefunc(t->opaque, (t)->fd, tar_block, T_BLOCKSIZE); - if ( block_size == -1) { - return 0; - } - //log_v("Wrote %lu last bytes", T_BLOCKSIZE); - *written_bytes += block_size; - } - } - regfile_rv = 0; - // fallthrough - case CLOSE_FILE: - //log_v("case CLOSE_FILE"); - t->io->closefunc(t->opaque, regfile_fd); - regfile_fd = NULL; - return regfile_rv; - break; - } - + //log_v("io::closewritefunc done"); return 0; } - -// add file contents to a tarchive -int tar_append_regfile(TAR *t, const char *realname, ssize_t *written_bytes) -{ - - regfile_step = OPEN_FILE; - int ret_value; - do { - ret_value = tar_append_regfile_step(t, realname, written_bytes); - if( ret_value == -1 ) - return -1; - } while(regfile_step!=CLOSE_FILE); - - return ret_value; -} - - // write a header block int th_write(TAR *t, ssize_t *written_bytes) { int i;//, j; th_set_ustar(t); - i = t->io->writefunc(t->opaque, (t)->fd, &(t->th_buf), T_BLOCKSIZE); + i = t->io->writefunc(t->io->dst_fs, t->dst_file, &(t->th_buf), T_BLOCKSIZE); if (i != T_BLOCKSIZE) { //log_e("ERROR in th_write, returned block size %d didn't match expexted size %d", i, (int)T_BLOCKSIZE); if (i != -1) { @@ -349,8 +164,11 @@ int th_write(TAR *t, ssize_t *written_bytes) // magic, version, and checksum void th_set_ustar(TAR *t) { + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstringop-truncation" strncpy(t->th_buf.version, TVERSION, TVERSLEN); strncpy(t->th_buf.magic, TMAGIC, TMAGLEN); + #pragma GCC diagnostic pop int_to_oct(th_crc_calc(t), t->th_buf.chksum, 8); } @@ -437,9 +255,6 @@ void th_set_from_stat(TAR *t, struct stat *s) } -// utils - - // check if entity is a file int th_is_regfile(TAR *t) { diff --git a/src/tar/libtar.h b/src/tar/libtar.h index ac279cf..23db74c 100644 --- a/src/tar/libtar.h +++ b/src/tar/libtar.h @@ -48,11 +48,13 @@ extern "C" /* useful constants */ -const uint32_t T_BLOCKSIZE = 512; +const ssize_t T_BLOCKSIZE = 512; #define T_NAMELEN 100 #define T_PREFIXLEN 155 #define T_MAXPATHLEN (T_NAMELEN + T_PREFIXLEN) -#define MAXPATHLEN T_MAXPATHLEN // mock #include but reduce from 1024 to 255 +#if !defined MAXPATHLEN + #define MAXPATHLEN T_MAXPATHLEN // mock #include but reduce from 1024 to 255 +#endif /* our version of the tar header structure */ struct tar_header @@ -87,6 +89,8 @@ typedef ssize_t (*closewritefunc_t)(void *fs, void *file); // i/o callbacks typedef struct { + void *src_fs; + void *dst_fs; openfunc_t openfunc; closefunc_t closefunc; readfunc_t readfunc; @@ -100,42 +104,12 @@ typedef struct { typedef struct { tar_callback_t *io; char *pathname; - void *fd; + void *dst_file; + void *src_file; struct tar_header th_buf; - void *opaque; } TAR; -// state machine for main loop -typedef enum { - TAR_MAIN_HEADER, - TAR_ENT_HEADER, - TAR_ENT_BLOCK, - TAR_ENT_FOOTER, - TAR_MAIN_FOOTER, - FINISHED -} tar_step_t; - -// state machine for dir entities -typedef enum { - ENTITY_STAT, - ENTITY_HEADER, - ENTITY_REGFILE, - ENTITY_REGFILE_STEP, - ENTITY_END -} entity_steps_t; - -// state holder for dir entities state machine -typedef struct entity_t -{ - TAR *tar; - const char *realname; - const char *savename; - ssize_t *written_bytes; - entity_steps_t step; - int step_rv; // return value -} entity_t; - // helpers @@ -151,19 +125,14 @@ int oct_to_int(char *oct); // integer to string-octal conversion, no NULL void int_to_oct_nonull(int num, char *oct, size_t octlen); - // open new tar instance -int tar_open(TAR *t, const char *pathname, tar_callback_t *io, const char *mode, void *opaque); +int tar_open(TAR *t, const char *pathname, tar_callback_t *io); // set tar info and allocate block memory -int tar_init(TAR *t, const char *pathname, tar_callback_t *io, void *opaque); +int tar_init(TAR *t, const char *pathname, tar_callback_t *io); // close tar handle and release block memory int tar_close(TAR *t); -// Appends a dir entity (file or folder) to the tar archive -int tar_append_entity(entity_t *entity); -int tar_append_entity_step(entity_t *entity); // state machine stepper - // encode entity header block (minus the path) void th_set_from_stat(TAR *t, struct stat *s); @@ -173,9 +142,6 @@ void th_set_path(TAR *t, const char *pathname); // write tar header int th_write(TAR *t, ssize_t *written_bytes); -// add file contents to a tar archive -int tar_append_regfile(TAR *t, const char *realname, ssize_t *written_bytes); - // write EOF indicator int tar_append_eof(TAR *t, ssize_t *written_bytes); diff --git a/src/uzlib/genlz77.c b/src/uzlib/genlz77.c index 725768d..da9070a 100644 --- a/src/uzlib/genlz77.c +++ b/src/uzlib/genlz77.c @@ -38,12 +38,10 @@ #include #include "uzlib.h" -#if 0 -#define HASH_BITS 12 -#else -#define HASH_BITS data->hash_bits -#endif +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wsign-compare" +#define HASH_BITS data->hash_bits #define HASH_SIZE (1< d->dest - d->destStart) { #if UZLIB_CONF_DEBUG_LOG >= 2 printf("tinf_inflate_block_data: bad dictionnary offset, %d oustide max range %d])\n", offs, d->dest - d->destStart ); @@ -518,6 +522,8 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) return TINF_DATA_ERROR; } d->lzOff = -offs; + + #pragma GCC diagnostic pop } }