diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index c40b9bd9a..6316512ec 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -373,22 +373,104 @@ jobs: - name: Run unit tests with miri run: "cargo +nightly miri nextest run -j4 -p libbzip2-rs-sys" - no_stdio: - name: "test no_stdio compile + run" + link-c-dynamic-library: + name: vanilla dynamic library + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + features: + - '' runs-on: ubuntu-latest steps: - name: Checkout sources uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 with: persist-credentials: false - submodules: true - - name: Install toolchain + - name: Install rust toolchain uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248 with: - toolchain: "stable" - - name: build static library - run: "cargo build --release -p libbzip2-rs-sys --no-default-features --features='c-allocator,std'" - - name: build C binary - run: "cc test-libbzip2-rs-sys/no_stdio.c -o no_stdio target/release/liblibbzip2_rs_sys.a -I ." - - name: run C binary - run: "./no_stdio | grep 'hello world!'" + toolchain: stable + targets: ${{matrix.target}} + - name: "cdylib: default settings" + working-directory: libbzip2-rs-sys-cdylib + env: + LD_LIBRARY_PATH: "target/${{matrix.target}}/release/deps" + run: | + cargo build --release --target ${{matrix.target}} + cc -o bzpipe bzpipe.c target/${{matrix.target}}/release/deps/libbz2_rs.so -I ../ + ./bzpipe < Cargo.toml | ./bzpipe -d > out.txt + cmp -s Cargo.toml out.txt + - name: "cdylib: rust-allocator" + env: + LD_LIBRARY_PATH: "target/${{matrix.target}}/release/deps" + working-directory: libbzip2-rs-sys-cdylib + run: | + cargo build --release --target ${{matrix.target}} --no-default-features --features="rust-allocator" + cc -o bzpipe bzpipe.c target/${{matrix.target}}/release/deps/libbz2_rs.so -I ../ + ./bzpipe < Cargo.toml | ./bzpipe -d > out.txt + cmp -s Cargo.toml out.txt + - name: "cdylib: no_std" + env: + LD_LIBRARY_PATH: "target/${{matrix.target}}/release/deps" + working-directory: libbzip2-rs-sys-cdylib + run: | + cargo build --release --target ${{matrix.target}} --no-default-features + cc -DNO_STD -o bzpipe bzpipe.c target/${{matrix.target}}/release/deps/libbz2_rs.so -I ../ + ./bzpipe < Cargo.toml | ./bzpipe -d > out.txt + cmp -s Cargo.toml out.txt + - name: "staticlib: no stdio" + env: + LD_LIBRARY_PATH: "target/${{matrix.target}}/release/deps" + working-directory: libbzip2-rs-sys-cdylib + run: | + cargo build --release --target ${{matrix.target}} --no-default-features --features="c-allocator" + cc -o bzpipe bzpipe.c target/${{matrix.target}}/release/deps/libbz2_rs.so -I ../ + ./bzpipe < Cargo.toml | ./bzpipe -d > out.txt + cmp -s Cargo.toml out.txt + - name: "cdylib: custom-prefix" + working-directory: libbzip2-rs-sys-cdylib + env: + LIBBZIP2_RS_SYS_PREFIX: "MY_CUSTOM_PREFIX_" + run: | + cargo build --release --target ${{matrix.target}} --features=custom-prefix + objdump -tT target/${{matrix.target}}/release/deps/libbz2_rs.so | grep -q "MY_CUSTOM_PREFIX_BZ2_bzCompressInit" || (echo "symbol not found!" && exit 1) + + cargo-c-dynamic-library: + name: cargo-c dynamic library + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + features: + - '' + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + with: + persist-credentials: false + - name: Install rust toolchain + uses: dtolnay/rust-toolchain@be73d7920c329f220ce78e0234b8f96b7ae60248 + with: + toolchain: stable + targets: ${{matrix.target}} + - name: Install cargo-c + env: + LINK: https://github.com/lu-zero/cargo-c/releases/download/v0.10.5 + run: | + curl -L "$LINK/cargo-c-x86_64-unknown-linux-musl.tar.gz" | + tar xz -C $HOME/.cargo/bin + - name: build with and test the result of cargo-c + working-directory: libbzip2-rs-sys-cdylib + run: | + # build using cargo-c this time + cargo cinstall --release --destdir=/tmp/cargo-cbuild-libbzip2-rs # somehow --offline does not work here + tree /tmp/cargo-cbuild-libbzip2-rs + # verify that the SONAME is set and includes a version + objdump -p target/x86_64-unknown-linux-gnu/release/libbz2_rs.so | awk '/SONAME/{print $2}' | grep -E 'libbz2_rs\.so\.1' + # build bzpipe with our library + cc -o bzpipe bzpipe.c -L/tmp/cargo-cbuild-libbzip2-rs/usr/local/lib/x86_64-linux-gnu -lbz2_rs -I ../ + export LD_LIBRARY_PATH=/tmp/cargo-cbuild-libbzip2-rs/usr/local/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH + ./bzpipe < Cargo.toml | ./bzpipe -d > out.txt + cmp -s Cargo.toml out.txt diff --git a/Cargo.toml b/Cargo.toml index d11ff5906..f37ca7644 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,9 @@ members = [ "libbzip2-rs-sys", "test-libbzip2-rs-sys", ] +exclude = [ + "libbzip2-rs-sys-cdylib", +] package.edition = "2021" diff --git a/libbzip2-rs-sys-cdylib/Cargo.lock b/libbzip2-rs-sys-cdylib/Cargo.lock new file mode 100644 index 000000000..2bf3797f9 --- /dev/null +++ b/libbzip2-rs-sys-cdylib/Cargo.lock @@ -0,0 +1,23 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "libbz2-rs-sys-cdylib" +version = "0.0.0" +dependencies = [ + "libbzip2-rs-sys", +] + +[[package]] +name = "libbzip2-rs-sys" +version = "0.0.0" +dependencies = [ + "libc", +] + +[[package]] +name = "libc" +version = "0.2.167" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" diff --git a/libbzip2-rs-sys-cdylib/Cargo.toml b/libbzip2-rs-sys-cdylib/Cargo.toml new file mode 100644 index 000000000..af115967e --- /dev/null +++ b/libbzip2-rs-sys-cdylib/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "libbz2-rs-sys-cdylib" +version = "0.0.0" +edition = "2021" +readme = "README.md" +license = "bzip2-1.0.6" +repository = "https://github.com/trifectatechfoundation/libbzip2-rs" +homepage = "https://github.com/trifectatechfoundation/libbzip2-rs" +publish = true +description = "A memory-safe bzip2 implementation written in rust" +rust-version = "1.82" # MSRV + +[lib] +name = "bz2_rs" # turns into e.g. `libbz2_rs.so` +crate-type=["cdylib"] + +[features] +default = ["c-allocator", "libbzip2-rs-sys/stdio", "libbzip2-rs-sys/std"] # when used as a cdylib crate, use the c allocator +c-allocator = ["libbzip2-rs-sys/c-allocator"] # by default, use malloc/free for memory allocation +rust-allocator = ["libbzip2-rs-sys/rust-allocator", "libbzip2-rs-sys/std"] # by default, use the rust global alloctor for memory allocation +custom-prefix = ["libbzip2-rs-sys/custom-prefix"] # use the LIBBZIP2_RS_SYS_PREFIX to prefix all exported symbols +capi = [] + +[dependencies] +libbzip2-rs-sys = { version = "0.0.0", path = "../libbzip2-rs-sys", default-features = false } + +[package.metadata.capi.library] +version = "1.0.9" # the bzip2 api version we match +name = "bz2_rs" + +[package.metadata.capi.header] +enabled = false + +[package.metadata.capi.pkg_config] +name = "libbz2_rs" +filename = "libbz2_rs" diff --git a/libbzip2-rs-sys-cdylib/README.md b/libbzip2-rs-sys-cdylib/README.md new file mode 100644 index 000000000..f1e393413 --- /dev/null +++ b/libbzip2-rs-sys-cdylib/README.md @@ -0,0 +1,114 @@ +# `libbzip2-rs-sys-cdylib` + +A drop-in replacement for the `libbz2` dynamic library + +```sh +# build the cdylib +# using `cargo build` will work but has limitations, see below +cargo build --release + +# the extension of a cdylib varies per platform +cc bzpipe.c -o bzpipe target/release/libbz2_rs.so -I ../ + +# verify the implementation can compress and decompress our Cargo.toml +./bzpipe < Cargo.toml | ./bzpipe -d +``` + +By default this build uses libc `malloc`/`free` to (de)allocate memory, and only depends on the rust `core` library. +See below for the available feature flags. + +## Feature Flags + +### Allocators + +We provide three options for the default allocator + +**`c-allocator`** + +```sh +cargo build --release --no-default-features --features "c-allocator" +``` + +Uses the libc `malloc` and `free` functions for memory allocation. + +**`rust-allocator`** + +```sh +cargo build --release --no-default-features --features "std,rust-allocator" +``` +Uses the rust standard library global allocator for memory allocation. + +**no allocator** + +```sh +cargo build --release --no-default-features +``` + +No allocator is configured automatically. This means that, before [`BZ2_bzCompressInit`] or [`BZ2_bzDecompressInit`] are called, +the user must set the `bzalloc` and `bzfree` fields of the `bz_stream` to valid allocation and deallocation functions, +and the `opaque` field to either `NULL` or a pointer expected by the (de)allocation functions. + +If no allocator is configured, the initialization functions will return `BZ_PARAM_ERROR`. + +### Symbol Prefix + +Symbols in C programs all live in the same namespace. A common solution to prevent names from clashing is to prefix +all of a library's symbols with a prefix. We support prefixing the name at build time with the `custom-prefix` feature +flag. When enabled, the value of the `LIBBZIP2_RS_SYS_PREFIX` is used as a prefix for all exported symbols. For example: + +```ignore +> LIBBZIP2_RS_SYS_PREFIX="MY_CUSTOM_PREFIX_" cargo build --release --features=custom-prefix + + Compiling libbzip2-rs-sys v0.0.0 (libbzip2-rs/libbzip2-rs-sys) + Compiling libz-rs-sys-cdylib v0.0.0 (libbzip2-rs/libbzip2-rs-sys-cdylib) + Finished `release` profile [optimized] target(s) in 0.97s +> objdump -tT target/release/libbz2_rs.so | grep "BZ2_bzCompressInit" +000000000002f300 l F .text 0000000000000441 .hidden _ZN15libbzip2_rs_sys5bzlib22BZ2_bzCompressInitHelp17hac60bda3d983fe05E +000000000002f2e0 g F .text 000000000000001a MY_CUSTOM_PREFIX_BZ2_bzCompressInit +000000000002f2e0 g DF .text 000000000000001a Base MY_CUSTOM_PREFIX_BZ2_bzCompressInit +``` + +### `#![no_std]` + +The dynamic library can be built without the rust `std` crate, e.g. for embedded devices that don't support it. Disabling +the standard library has the following limitations: + +- The `rust-allocator` should not be used. It internally enables the standard library, causing issues. Using `c-allocator` + or not providing an allocator at build time is still supported.On embedded it is most common to provide a custom allocator + that "allocates" into a custom array. + +## Build for Distribution + +A `cargo build` currently does not set some fields that are required or useful when using a dynamic library from C. +For instance, the soname and version are not set by a standard `cargo build`. + +To build a proper, installable dynamic library, we recommend [`cargo-c`](https://github.com/lu-zero/cargo-c): + +``` +cargo install cargo-c +``` + +This tool deals with setting fields (soname, version) that a normal `cargo build` does not set (today). +It's configuration is in the `Cargo.toml`, where e.g. the library name or version can be changed. + +``` +> cargo cbuild --release + Compiling libc v0.2.167 + Compiling libbzip2-rs-sys v0.0.0 (libbzip2-rs/libbzip2-rs-sys) + Compiling libz-rs-sys-cdylib v0.0.0 (libbzip2-rs/libbzip2-rs-sys-cdylib) + Finished `release` profile [optimized] target(s) in 1.63s + Building pkg-config files +> tree target +target +├── CACHEDIR.TAG +└── x86_64-unknown-linux-gnu + ├── CACHEDIR.TAG + └── release + ├── examples + ├── incremental + ├── libbz2_rs.a + ├── libbz2_rs.d + ├── libbz2_rs.pc + ├── libbz2_rs.so + └── libbz2_rs-uninstalled.pc +``` diff --git a/libbzip2-rs-sys-cdylib/bzpipe.c b/libbzip2-rs-sys-cdylib/bzpipe.c new file mode 100644 index 000000000..33103904d --- /dev/null +++ b/libbzip2-rs-sys-cdylib/bzpipe.c @@ -0,0 +1,222 @@ +#include +#include +#include +#include +#include "bzlib.h" + +extern void bz_internal_error(int errcode) { + fprintf(stderr, "bzip2 hit internal error code: %d\n", errcode); +} + +#ifdef NO_STD +void *custom_bzalloc(void *opaque, int items, int size) { + return malloc(items * size); +} + +void custom_bzfree(void *opaque, void *address) { + free(address); +} +#endif + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include +# include +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define CHUNK 256 + +/* Compress from file source to file dest until EOF on source. + def() returns BZ_OK on success, BZ_MEM_ERROR if memory could not be + allocated for processing, or BZ_IO_ERROR if there is + an error reading or writing the files. */ +int def(FILE *source, FILE *dest) +{ + int ret; + unsigned have; + bz_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate deflate state */ +#ifdef NO_STD + strm.bzalloc = custom_bzalloc; + strm.bzfree = custom_bzfree; +#else + strm.bzalloc = NULL; + strm.bzfree = NULL; +#endif + strm.opaque = NULL; + ret = BZ2_bzCompressInit(&strm, 9, 0, 0); + if (ret != BZ_OK) + return ret; + + /* compress until end of file */ + int done = 0; + while (!done) { + strm.avail_in = fread(in, 1, CHUNK, source); + done = feof(source); + + if (ferror(source)) { + (void)BZ2_bzCompressEnd(&strm); + return BZ_IO_ERROR; + } + strm.next_in = in; + + /* run deflate() on input until output buffer not full, finish + compression if all of source has been read in */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = BZ2_bzCompress(&strm, BZ_FLUSH); /* no bad return value */ + assert(ret != BZ_PARAM_ERROR); /* state not clobbered */ + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)BZ2_bzCompressEnd(&strm); + return BZ_IO_ERROR; + } + } while (strm.avail_out == 0); + assert(strm.avail_in == 0); /* all input will be used */ + + /* done when last data in file processed */ + } + + strm.avail_out = CHUNK; + strm.next_out = out; + ret = BZ2_bzCompress(&strm, BZ_FINISH); /* no bad return value */ + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)BZ2_bzCompressEnd(&strm); + return BZ_IO_ERROR; + } + + assert(ret == BZ_STREAM_END); /* stream will be complete */ + + /* clean up and return */ + (void)BZ2_bzCompressEnd(&strm); + return BZ_OK; +} + +/* Decompress from file source to file dest until stream ends or EOF. + inf() returns BZ_OK on success, BZ_MEM_ERROR if memory could not be + allocated for processing, BZ_DATA_ERROR if the deflate data is + invalid or incomplete, or BZ_IO_ERROR if there + is an error reading or writing the files. */ +int inf(FILE *source, FILE *dest) +{ + int ret; + unsigned have; + bz_stream strm; + unsigned char in[CHUNK]; + unsigned char out[CHUNK]; + + /* allocate inflate state */ +#ifdef NO_STD + strm.bzalloc = custom_bzalloc; + strm.bzfree = custom_bzfree; +#else + strm.bzalloc = NULL; + strm.bzfree = NULL; +#endif + strm.opaque = NULL; + strm.avail_in = 0; + strm.next_in = NULL; + ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + return ret; + + /* decompress until deflate stream ends or end of file */ + do { + strm.avail_in = fread(in, 1, CHUNK, source); + if (ferror(source)) { + (void)BZ2_bzDecompressEnd(&strm); + return BZ_IO_ERROR; + } + if (strm.avail_in == 0) + break; + strm.next_in = in; + + /* run inflate() on input until output buffer not full */ + do { + strm.avail_out = CHUNK; + strm.next_out = out; + ret = BZ2_bzDecompress(&strm); + assert(ret != BZ_PARAM_ERROR); /* state not clobbered */ + switch (ret) { + case BZ_DATA_ERROR: + case BZ_MEM_ERROR: + (void)BZ2_bzDecompressEnd(&strm); + return ret; + } + have = CHUNK - strm.avail_out; + if (fwrite(out, 1, have, dest) != have || ferror(dest)) { + (void)BZ2_bzDecompressEnd(&strm); + return BZ_IO_ERROR; + } + } while (strm.avail_out == 0); + + /* done when BZ2_bzDecompress() says it's done */ + } while (ret != BZ_STREAM_END); + + /* clean up and return */ + (void)BZ2_bzDecompressEnd(&strm); + return ret == BZ_STREAM_END ? BZ_OK : BZ_DATA_ERROR; +} + +/* report a bzip2 or i/o error */ +void zerr(int ret) +{ + fputs("bzpipe: ", stderr); + switch (ret) { + case BZ_IO_ERROR: + if (ferror(stdin)) + fputs("error reading stdin\n", stderr); + if (ferror(stdout)) + fputs("error writing stdout\n", stderr); + break; + case BZ_PARAM_ERROR: + fputs("invalid block size\n", stderr); + break; + case BZ_DATA_ERROR: + case BZ_DATA_ERROR_MAGIC: + fputs("invalid or incomplete data\n", stderr); + break; + case BZ_MEM_ERROR: + fputs("out of memory\n", stderr); + break; + } +} + +/* compress or decompress from stdin to stdout */ +int main(int argc, char **argv) +{ + int ret; + + /* avoid end-of-line conversions */ + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + + /* do compression if no arguments */ + if (argc == 1) { + ret = def(stdin, stdout); + if (ret != BZ_OK) + zerr(ret); + return ret; + } + + /* do decompression if -d specified */ + else if (argc == 2 && strcmp(argv[1], "-d") == 0) { + ret = inf(stdin, stdout); + if (ret != BZ_OK) + zerr(ret); + return ret; + } + + /* otherwise, report usage */ + else { + fputs("bzpipe usage: bzpipe [-d] < source > dest\n", stderr); + return 1; + } +} diff --git a/libbzip2-rs-sys-cdylib/src/lib.rs b/libbzip2-rs-sys-cdylib/src/lib.rs new file mode 100644 index 000000000..4c833f413 --- /dev/null +++ b/libbzip2-rs-sys-cdylib/src/lib.rs @@ -0,0 +1,3 @@ +extern crate libbzip2_rs_sys; + +pub use libbzip2_rs_sys::*; diff --git a/libbzip2-rs-sys/Cargo.toml b/libbzip2-rs-sys/Cargo.toml index 48d67dc12..ad15cd317 100644 --- a/libbzip2-rs-sys/Cargo.toml +++ b/libbzip2-rs-sys/Cargo.toml @@ -3,11 +3,6 @@ name = "libbzip2-rs-sys" readme = "README.md" edition.workspace = true -[lib] -name = "libbzip2_rs_sys" -crate-type = ["staticlib", "rlib"] - - [features] default = ["std", "c-allocator", "stdio"] c-allocator = ["dep:libc"] # use a malloc-based C allocator (rust is picked over c if both are configured) diff --git a/test-libbzip2-rs-sys/no_stdio.c b/test-libbzip2-rs-sys/no_stdio.c deleted file mode 100644 index 25d7cdc2a..000000000 --- a/test-libbzip2-rs-sys/no_stdio.c +++ /dev/null @@ -1,41 +0,0 @@ -#include -#include -#include -#include "bzlib.h" - -int main() { - unsigned char compressed_data[] = { - 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xa2, 0x9d, - 0x5a, 0x47, 0x00, 0x00, 0x02, 0xd1, 0x80, 0x00, 0x10, 0x60, 0x00, 0x06, - 0x44, 0x90, 0x80, 0x20, 0x00, 0x31, 0x00, 0x30, 0x20, 0x34, 0x62, 0x59, - 0x04, 0xea, 0x42, 0x19, 0x7e, 0x2e, 0xe4, 0x8a, 0x70, 0xa1, 0x21, 0x45, - 0x3a, 0xb4, 0x8e - }; - unsigned int compressed_data_len = 51; - - // Allocate memory for the decompressed data - int destLen = 1024; // Adjust this as needed - char *decompressed_data = (char *)malloc(destLen); - - // Perform decompression - int result = BZ2_bzBuffToBuffDecompress(decompressed_data, &destLen, - compressed_data, compressed_data_len, - 0, 0); - - if (result == BZ_OK) { - printf("Decompression successful!\n"); - printf("Decompressed data: %s\n", decompressed_data); - } else { - printf("Decompression failed with error code: %d\n", result); - } - - free(decompressed_data); - return 0; -} - -extern void bz_internal_error(int errcode); - -void bz_internal_error(int errcode) { - fprintf(stderr, "bzip2 internal error: %d\n", errcode); - exit(EXIT_FAILURE); -}