From 6c55df34678b888b40c6b488803897c44eaef905 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Fri, 20 Dec 2024 12:35:57 +0100 Subject: [PATCH] gzip: istream --- Makefile | 5 +- frontends/ast/simplify.cc | 14 ++-- kernel/gzip.cc | 132 ++++++++++++++++++++++++++++++++++++++ kernel/gzip.h | 79 +++++++++++++++++++++++ kernel/io.cc | 96 --------------------------- kernel/io.h | 36 +---------- kernel/register.cc | 1 + 7 files changed, 225 insertions(+), 138 deletions(-) create mode 100644 kernel/gzip.cc create mode 100644 kernel/gzip.h diff --git a/Makefile b/Makefile index 1506d08b95a..cf7465f90ea 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ YOSYS_SRC := $(dir $(firstword $(MAKEFILE_LIST))) VPATH := $(YOSYS_SRC) CXXSTD ?= c++17 -CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -Werror=implicit-function-declaration -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include +CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include LIBS := $(LIBS) -lstdc++ -lm PLUGIN_LINKFLAGS := PLUGIN_LIBS := @@ -578,6 +578,7 @@ $(eval $(call add_include_file,kernel/fmt.h)) ifeq ($(ENABLE_ZLIB),1) $(eval $(call add_include_file,kernel/fstdata.h)) endif +$(eval $(call add_include_file,kernel/gzip.h)) $(eval $(call add_include_file,kernel/hashlib.h)) $(eval $(call add_include_file,kernel/io.h)) $(eval $(call add_include_file,kernel/json.h)) @@ -610,7 +611,7 @@ $(eval $(call add_include_file,frontends/ast/ast_binding.h)) $(eval $(call add_include_file,frontends/blif/blifparse.h)) $(eval $(call add_include_file,backends/rtlil/rtlil_backend.h)) -OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o kernel/io.o +OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o kernel/io.o kernel/gzip.o OBJS += kernel/binding.o kernel/tclapi.o OBJS += kernel/cellaigs.o kernel/celledges.o kernel/cost.o kernel/satgen.o kernel/scopeinfo.o kernel/qcsat.o kernel/mem.o kernel/ffmerge.o kernel/ff.o kernel/yw.o kernel/json.o kernel/fmt.o kernel/sexpr.o OBJS += kernel/drivertools.o kernel/functional.o diff --git a/frontends/ast/simplify.cc b/frontends/ast/simplify.cc index 475f27cdfbf..b00cde28e2c 100644 --- a/frontends/ast/simplify.cc +++ b/frontends/ast/simplify.cc @@ -2777,13 +2777,13 @@ bool AstNode::simplify(bool const_fold, int stage, int width_hint, bool sign_hin mux_input = new AstNode(AST_BIT_NOT, mux_input); } AstNode *node = new AstNode(AST_TERNARY, children_list.at(2)); - // if (str == "bufif0") { - // node->children.push_back(AstNode::mkconst_bits(z_const, false)); - // node->children.push_back(mux_input); - // } else { - // node->children.push_back(mux_input); - // node->children.push_back(AstNode::mkconst_bits(z_const, false)); - // } + if (str == "bufif0") { + node->children.push_back(AstNode::mkconst_bits(z_const, false)); + node->children.push_back(mux_input); + } else { + node->children.push_back(mux_input); + node->children.push_back(AstNode::mkconst_bits(z_const, false)); + } str.clear(); type = AST_ASSIGN; diff --git a/kernel/gzip.cc b/kernel/gzip.cc new file mode 100644 index 00000000000..900e33bf49d --- /dev/null +++ b/kernel/gzip.cc @@ -0,0 +1,132 @@ +#include "kernel/yosys_common.h" +#include "kernel/log.h" +#include "kernel/gzip.h" +#include +#include +#include +#include +#include + +YOSYS_NAMESPACE_BEGIN + + +#ifdef YOSYS_ENABLE_ZLIB + +PRIVATE_NAMESPACE_BEGIN + +using namespace Zlib; + +PRIVATE_NAMESPACE_END + +gzip_ostream::obuf::obuf() { + setp(buffer, buffer + buffer_size - 1); +} + +bool gzip_ostream::obuf::open(const std::string &filename) { + gzf = gzopen(filename.c_str(), "wb"); + return gzf != nullptr; +} + +int gzip_ostream::obuf::sync() { + int num = pptr() - pbase(); + if (num > 0) { + if (gzwrite(gzf, reinterpret_cast(pbase()), num) != num) { + return -1; + } + pbump(-num); + } + return 0; +} + +gzip_ostream::obuf::~obuf() { + if (gzf) { + sync(); + gzclose(gzf); + } +} + +bool gzip_istream::ibuf::open(const std::string& filename) { + if (gzf) { + Zlib::gzclose(gzf); + } + gzf = Zlib::gzopen(filename.c_str(), "rb"); + if (!gzf) { + return false; + } + // Empty and point to start + setg(buffer, buffer, buffer); + return true; +} + +// Called when the buffer is empty and more input is needed +std::istream::int_type gzip_istream::ibuf::underflow() { + log_assert(gzf && "No gzfile opened\n"); + int bytes_read = Zlib::gzread(gzf, buffer, buffer_size); + if (bytes_read <= 0) { + if (Zlib::gzeof(gzf)) + return traits_type::eof(); + + int err; + const char* error_msg = Zlib::gzerror(gzf, &err); + if (err != Z_OK) + log_error("%s", error_msg); + else + log_error("Decompression logic failure: "\ + "read <=0 bytes but neither EOF nor error\n"); + } + + // Keep size and point to start + setg(buffer, buffer, buffer + bytes_read); + return traits_type::to_int_type(buffer[0]); +} + +gzip_istream::ibuf::~ibuf() { + if (gzf) { + int err = Zlib::gzclose(gzf); + if (err != Z_OK) { + // OK to overwrite rr it, it doesn't change + const char* error_msg = Zlib::gzerror(gzf, &err); + log_error("%s", error_msg); + } + } +} + +#endif // YOSYS_ENABLE_ZLIB + + +// Takes a successfully opened ifstream. If it's gzipped, returns an istream. Otherwise, +// returns the original ifstream, rewound to the start. +std::istream* uncompressed(std::ifstream* f, const std::string filename) { + if (!f) + return nullptr; + // Check for gzip magic + unsigned char magic[3]; + int n = 0; + while (n < 3) + { + int c = f->get(); + if (c != EOF) { + magic[n] = (unsigned char) c; + } + n++; + } + if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) { +#ifdef YOSYS_ENABLE_ZLIB + log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str()); + if (magic[2] != 8) + log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n", + filename.c_str(), unsigned(magic[2])); + delete f; + gzip_istream* s = new gzip_istream(); + return s->open(filename.c_str()) ? s : nullptr; +#else + log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str()); +#endif // YOSYS_ENABLE_ZLIB + } else { + f->clear(); + f->seekg(0, std::ios::beg); + return f; + } +} + +YOSYS_NAMESPACE_END diff --git a/kernel/gzip.h b/kernel/gzip.h new file mode 100644 index 00000000000..22f5093a0bf --- /dev/null +++ b/kernel/gzip.h @@ -0,0 +1,79 @@ +#include +#include +#include "kernel/yosys_common.h" + +#ifndef YOSYS_GZIP_H +#define YOSYS_GZIP_H + +YOSYS_NAMESPACE_BEGIN + +#ifdef YOSYS_ENABLE_ZLIB + +namespace Zlib { +#include +} + +/* +An output stream that uses a stringbuf to buffer data internally, +using zlib to write gzip-compressed data every time the stream is flushed. +*/ +class gzip_ostream : public std::ostream { +public: + gzip_ostream(): std::ostream(nullptr) { + rdbuf(&outbuf); + } + bool open(const std::string &filename) { + return outbuf.open(filename); + } +private: + class obuf : public std::stringbuf { + public: + obuf(); + bool open(const std::string &filename); + virtual int sync() override; + virtual ~obuf(); + private: + static const int buffer_size = 4096; + char buffer[buffer_size]; // Internal buffer for compressed data + Zlib::gzFile gzf = nullptr; // Handle to the gzip file + }; + + obuf outbuf; // The stream buffer instance +}; + +/* +An input stream that uses zlib to read gzip-compressed data from a file, +buffering the decompressed data internally using its own buffer. +*/ +class gzip_istream final : public std::istream { +public: + gzip_istream() : std::istream(&inbuf) {} + bool open(const std::string& filename) { + return inbuf.open(filename); + } +private: + class ibuf final : public std::streambuf { + public: + ibuf() : gzf(nullptr) {} + bool open(const std::string& filename); + virtual ~ibuf(); + + protected: + // Called when the buffer is empty and more input is needed + virtual int_type underflow() override; + private: + static const int buffer_size = 8192; + char buffer[buffer_size]; + Zlib::gzFile gzf; + }; + + ibuf inbuf; // The stream buffer instance +}; + +#endif // YOSYS_ENABLE_ZLIB + +std::istream* uncompressed(std::ifstream* f, const std::string filename); + +YOSYS_NAMESPACE_END + +#endif // YOSYS_GZIP_H diff --git a/kernel/io.cc b/kernel/io.cc index a91827c697b..0fdbd3c3791 100644 --- a/kernel/io.cc +++ b/kernel/io.cc @@ -3,8 +3,6 @@ #include #include #include -#include -#include YOSYS_NAMESPACE_BEGIN @@ -360,98 +358,4 @@ std::string escape_filename_spaces(const std::string& filename) return out; } -#ifdef YOSYS_ENABLE_ZLIB - -PRIVATE_NAMESPACE_BEGIN - -static const size_t GZ_BUFFER_SIZE = 8192; -static void decompress_gzip(const std::string &filename, std::stringstream &out) -{ - char buffer[GZ_BUFFER_SIZE]; - int bytes_read; - gzFile gzf = gzopen(filename.c_str(), "rb"); - while(!gzeof(gzf)) { - bytes_read = gzread(gzf, reinterpret_cast(buffer), GZ_BUFFER_SIZE); - out.write(buffer, bytes_read); - } - gzclose(gzf); -} - -PRIVATE_NAMESPACE_END - -gzip_ostream::gzip_ostream() : std::ostream(nullptr) { - rdbuf(&outbuf); -} - -bool gzip_ostream::open(const std::string &filename) { - return outbuf.open(filename); -} - -gzip_ostream::gzip_streambuf::gzip_streambuf() { - setp(buffer, buffer + buffer_size - 1); -} - -bool gzip_ostream::gzip_streambuf::open(const std::string &filename) { - gzf = gzopen(filename.c_str(), "wb"); - return gzf != nullptr; -} - -int gzip_ostream::gzip_streambuf::sync() { - int num = pptr() - pbase(); - if (num > 0) { - if (gzwrite(gzf, reinterpret_cast(pbase()), num) != num) { - return -1; - } - pbump(-num); - } - return 0; -} - -gzip_ostream::gzip_streambuf::~gzip_streambuf() { - if (gzf) { - sync(); - gzclose(gzf); - } -} - -#endif // YOSYS_ENABLE_ZLIB - - -// Takes a successfully opened ifstream. If it's gzipped, returns an istream -// over a buffer of the file fully decompressed in memory. Otherwise, -// returns the original ifstream, rewound to the start. -std::istream* uncompressed(std::ifstream* f, const std::string filename) { - if (!f) - return nullptr; - // Check for gzip magic - unsigned char magic[3]; - int n = 0; - while (n < 3) - { - int c = f->get(); - if (c != EOF) { - magic[n] = (unsigned char) c; - } - n++; - } - if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) { -#ifdef YOSYS_ENABLE_ZLIB - log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str()); - if (magic[2] != 8) - log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n", - filename.c_str(), unsigned(magic[2])); - delete f; - std::stringstream *df = new std::stringstream(); - decompress_gzip(filename, *df); - return df; -#else - log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str()); -#endif // YOSYS_ENABLE_ZLIB - } else { - f->clear(); - f->seekg(0, std::ios::beg); - return f; - } -} - YOSYS_NAMESPACE_END diff --git a/kernel/io.h b/kernel/io.h index 99b2dc62ac0..1eb3a0fc100 100644 --- a/kernel/io.h +++ b/kernel/io.h @@ -1,13 +1,11 @@ -#include "kernel/yosys_common.h" #include +#include +#include +#include "kernel/yosys_common.h" #ifndef YOSYS_IO_H #define YOSYS_IO_H -#ifdef YOSYS_ENABLE_ZLIB -#include -#endif - YOSYS_NAMESPACE_BEGIN inline std::string vstringf(const char *fmt, va_list ap) @@ -67,34 +65,6 @@ inline std::string stringf(const char *fmt, ...) return string; } -#ifdef YOSYS_ENABLE_ZLIB -/* -An output stream that uses a stringbuf to buffer data internally, -using zlib to write gzip-compressed data every time the stream is flushed. -*/ -class gzip_ostream : public std::ostream { -public: - gzip_ostream(); - bool open(const std::string &filename); -private: - class gzip_streambuf : public std::stringbuf { - public: - gzip_streambuf(); - bool open(const std::string &filename); - virtual int sync() override; - virtual ~gzip_streambuf(); - private: - static const int buffer_size = 4096; // Size of the internal buffer - char buffer[buffer_size]; // Internal buffer for compressed data - gzFile gzf = nullptr; // Handle to the gzip file - }; - - gzip_streambuf outbuf; // The stream buffer instance -}; -#endif // YOSYS_ENABLE_ZLIB - -std::istream* uncompressed(std::ifstream* f, const std::string filename); - YOSYS_NAMESPACE_END #endif // YOSYS_IO_H diff --git a/kernel/register.cc b/kernel/register.cc index 00e0c29fda8..a2e7cb5772a 100644 --- a/kernel/register.cc +++ b/kernel/register.cc @@ -20,6 +20,7 @@ #include "kernel/yosys.h" #include "kernel/satgen.h" #include "kernel/json.h" +#include "kernel/gzip.h" #include #include