binary incompat, add sorted offset table for iterating files, limit n…

…um_objects to 64k
jkent · Feb 24, 2021 · 8405dfa · 8405dfa
1 parent 52b17b0
commit 8405dfa
Show file tree

Hide file tree

Showing 10 changed files with 254 additions and 96 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# About libespfs
+# About Libespfs
 
 Libespfs is a read-only filesystem component designed for
 [ESP-IDF](https://github.com/espressif/esp-idf) and
@@ -13,7 +13,7 @@ Libespfs can be used in other projects though, and works fine on Linux. There
 is a test Linux program in `tools/test` to read files from an espfs image.
 
 
-# Getting started
+# Getting Started
 
 To use this component, make a components directory in your project's root
 directory and within that directory run:
@@ -25,17 +25,26 @@ You can generate a filesystem using `tools/mkespfsiage.py`. The tool takes two
 arguments, ROOT, the directory containing the files to generate from, and
 IMAGE, the output file for the image. The script references an espfs.yaml file
 in the image ROOT, with the default settings to not add it to the image. The
-yaml file the various preprocessors and compressors to run while building the
-image. Example:
+yaml file defines filters for the various preprocessors and compressors to run
+while building the image. The espfs.yaml file overrides any settings in the
+espfs_defaults.yaml file included with libespfs. Example:
 
 ```yaml
-paths:
+filters:
     '*.html': ['html-minifier', 'gzip']
+    '*.zip': no-compress
     '*': heatshrink
 ```
 
+There are 5 preprocessors (babel-convert, babel-minifiy, html-minifier,
+uglifycss, and uglifyjs) and there are two compressors (gzip and heatshrink).
+These can be prefixed with 'no-' to disable them in more specific filters.
+There is also the commands 'discard' to prevent files from being added to the
+image, 'skip' to cancel all processing, 'no-preprocessing', and
+'no-compression'.
+
 You can add your own preprocessors as well. Look at the espfs_default.yaml
-within the component as an example.
+within libespfs for an example.
 
 
 ## Building an espfs image
@@ -118,6 +127,7 @@ fclose) functions to access files.
 ## Raw interface
 
 ```C
+const char *espfs_get_path(espfs_fs_t *fs, uint16_t index);
 bool espfs_stat(espfs_fs_t *fs, const char *path, espfs_stat_t *s);
 espfs_file_t *espfs_fopen(espfs_fs_t *fs, const char *path);
 void espfs_fclose(espfs_file_t *f);

diff --git a/cmake/include.cmake b/cmake/include.cmake
@@ -50,10 +50,18 @@ function(define_target_espfs target dir output)
     get_filename_component(output_dir ${output} DIRECTORY)
 
     add_custom_target(${target}
-        BYPRODUCTS ${output}
+        BYPRODUCTS ${dir}/espfs.paths
         DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/libespfs.dir/requirements.stamp
         COMMAND ${CMAKE_COMMAND} -E make_directory ${output_dir}
+        COMMAND ${python} ${libespfs_DIR}/tools/pathlist.py ${dir}
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMANT "Updating espfs.paths for ${target}"
+        VERBATIM
+    )
+
+    add_custom_command(OUTPUT ${output}
         COMMAND ${python} ${libespfs_DIR}/tools/mkespfsimage.py ${dir} ${output}
+        DEPENDS ${dir}/espfs.paths
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
         COMMENT "Building espfs binary ${output}"
         USES_TERMINAL
@@ -72,30 +80,38 @@ function(target_add_espfs target name)
         file(RELATIVE_PATH dir ${PROJECT_SOURCE_DIR} ${dir})
     endif()
 
-    set(output ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${target}.dir/${name}.bin)
+    set(output ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${target}.dir/${name})
     file(RELATIVE_PATH rel_output ${CMAKE_CURRENT_BINARY_DIR} ${output})
 
     get_filename_component(output_dir ${output} DIRECTORY)
 
-    add_custom_target(espfs_image_${name} ALL
-        BYPRODUCTS ${output}
+    add_custom_target(espfs_image_${name}
+        BYPRODUCTS ${dir}/espfs.paths
         DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/libespfs.dir/requirements.stamp
         COMMAND ${CMAKE_COMMAND} -E make_directory ${output_dir}
-        COMMAND ${python} ${libespfs_DIR}/tools/mkespfsimage.py ${dir} ${output}
+        COMMAND ${python} ${libespfs_DIR}/tools/pathlist.py ${dir}
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+        COMMENT "Updating pathlist for espfs_image_${name}"
+        VERBATIM
+    )
+    add_dependencies(${target} espfs_image_${name})
+
+    add_custom_command(OUTPUT ${output}.bin
+        COMMAND ${python} ${libespfs_DIR}/tools/mkespfsimage.py ${dir} ${output}.bin
+        DEPENDS ${dir}/espfs.paths
         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
         COMMENT "Building espfs binary ${rel_output}"
         USES_TERMINAL
         VERBATIM
     )
-    add_dependencies(${target} espfs_image_${name})
 
-    add_custom_command(OUTPUT ${output}.c
-        COMMAND ${python} ${libespfs_DIR}/tools/bin2c.py ${output} ${output}.c
-        DEPENDS ${output}
+    add_custom_command(OUTPUT ${output}.bin.c
+        COMMAND ${python} ${libespfs_DIR}/tools/bin2c.py ${output}.bin ${output}.bin.c
+        DEPENDS ${output}.bin
         COMMENT "Building source file ${rel_output}.c"
         VERBATIM
     )
-    target_sources(${target} PRIVATE ${output}.c)
+    target_sources(${target} PRIVATE ${output}.bin.c)
 endfunction()
 
 function(target_config_vars)

diff --git a/docs/api-reference/fs.rst b/docs/api-reference/fs.rst
@@ -8,6 +8,7 @@ Functions
 
 .. doxygenfunction:: espfs_init
 .. doxygenfunction:: espfs_deinit
+.. doxygenfunction:: espfs_get_path
 .. doxygenfunction:: espfs_stat
 
 Structures

diff --git a/espfs_defaults.yaml b/espfs_defaults.yaml
@@ -1,7 +1,4 @@
 preprocessors:
-    gzip:
-        level: 9
-
     babel-convert:
         npm:
             - '@babel/core'
@@ -26,13 +23,19 @@ preprocessors:
         command: npx uglifyjs
 
 compressors:
+    gzip:
+        level: 9
+
     heatshrink:
         window_sz2: 11
         lookahead_sz2: 4
 
-paths:
+filters:
+    'espfs.paths': discard
     'espfs.yaml': discard
     '*.css': uglifycss
     '*.html': html-minifier
     '*.js': [babel-convert, uglifyjs]
-    '*': heatshrink
+    '*.woff': no-compression
+    '*.woff2': no-compression
+    '*': [cache, heatshrink]
diff --git a/include/libespfs/espfs.h b/include/libespfs/espfs.h
@@ -35,7 +35,8 @@ enum espfs_stat_type_t {
  * \brief Object flags
  */
 enum espfs_flags_t {
-    ESPFS_FLAG_GZIP = (1 << 0),
+    ESPFS_FLAG_GZIP  = (1 << 0),
+    ESPFS_FLAG_CACHE = (1 << 1),
 };
 
 /**
@@ -61,6 +62,7 @@ struct espfs_config_t {
  * \brief Structure filled by \a espfs_stat and \a espfs_fstat functions
  */
 struct espfs_stat_t {
+    uint16_t index; /**< file index */
     espfs_stat_type_t type; /**< file type */
     espfs_flags_t flags; /**< file flags */
     espfs_compression_type_t compression; /**< compression type */
@@ -83,6 +85,16 @@ void espfs_deinit(
     espfs_fs_t *fs /** [in] espfs fs pointer */
 );
 
+/**
+ * \brief Get path for sorted espfs object index
+ *
+ * \return path or NULL if the index is invalid
+ */
+const char *espfs_get_path(
+    espfs_fs_t *fs, /** [in] espfs fs pointer */
+    uint16_t index /** [in] espfs file index */
+);
+
 /**
  * \brief Get information about an espfs object
  *
@@ -116,7 +128,6 @@ void espfs_fclose(
  */
 void espfs_fstat(
     espfs_file_t *f, /** [in] espfs file */
-    const char *path, /** [in] espfs path */
     espfs_stat_t *s /** [out] stat structure */
 );
 

diff --git a/include/libespfs/espfs_format.h b/include/libespfs/espfs_format.h
@@ -11,13 +11,13 @@
  * \brief Magic number used in the espfs file header
  */
 #define ESPFS_MAGIC 0x2B534645 /** EFS+ */
-#define ESPFS_VERSION_MAJOR 0
-#define ESPFS_VERSION_MINOR 1
+#define ESPFS_VERSION_MAJOR 1
+#define ESPFS_VERSION_MINOR 0
 
 typedef struct espfs_fs_header_t espfs_fs_header_t;
 typedef struct espfs_hashtable_entry_t espfs_hashtable_entry_t;
+typedef struct espfs_sorttable_entry_t espfs_sorttable_entry_t;
 typedef struct espfs_object_header_t espfs_object_header_t;
-typedef struct espfs_dir_header_t espfs_dir_header_t;
 typedef struct espfs_file_header_t espfs_file_header_t;
 typedef struct espfs_heatshrink_header_t espfs_heatshrink_header_t;
 typedef struct espfs_crc32_footer_t espfs_crc32_footer_t;
@@ -27,23 +27,26 @@ struct espfs_fs_header_t {
     uint8_t len;
     uint8_t version_major;
     uint16_t version_minor;
-    uint32_t num_objects;
     uint32_t binary_len;
+    uint16_t num_objects;
+    uint16_t reserved;
 } __attribute__((packed));
 
 struct espfs_hashtable_entry_t {
     uint32_t hash;
     uint32_t offset;
 } __attribute__((packed));
 
+struct espfs_sorttable_entry_t {
+    uint32_t offset;
+} __attribute__((packed));
+
 struct espfs_object_header_t {
     uint8_t type;
     uint8_t len;
+    uint16_t index;
     uint16_t path_len;
-} __attribute__((packed));
-
-struct espfs_dir_header_t {
-    espfs_object_header_t object;
+    uint16_t reserved;
 } __attribute__((packed));
 
 struct espfs_file_header_t {
@@ -63,4 +66,4 @@ struct espfs_heatshrink_header_t {
 
 struct espfs_crc32_footer_t {
     uint32_t crc32;
-} __attribute__((packed));
+} __attribute__((packed));
diff --git a/src/espfs.c b/src/espfs.c
@@ -78,7 +78,9 @@ espfs_fs_t *espfs_init(espfs_config_t *conf)
         goto err_out;
     }
 
-    fs->hashtable = (void *) fs->header + fs->header->len;
+    fs->hashtable = (const void *) fs->header + fs->header->len;
+    fs->sorttable = (const void *) fs->hashtable +
+            (sizeof(espfs_hashtable_entry_t) * fs->header->num_objects);
 
     return fs;
 
@@ -99,6 +101,20 @@ void espfs_deinit(espfs_fs_t *fs)
     free(fs);
 }
 
+const char *espfs_get_path(espfs_fs_t *fs, uint16_t index)
+{
+    assert(fs != NULL);
+
+    if (index >= fs->header->num_objects) {
+        return NULL;
+    }
+
+    const espfs_sorttable_entry_t *entry = fs->sorttable + index;
+    const espfs_object_header_t *object = (const void *) fs->header +
+            entry->offset;
+    return (const char *) object + object->len;
+}
+
 static uint32_t hash_path(const char *path)
 {
     uint32_t hash = 5381;
@@ -113,7 +129,7 @@ static uint32_t hash_path(const char *path)
     return hash;
 }
 
-static void *find_object(espfs_fs_t *fs, const char *path)
+static const void *find_object(espfs_fs_t *fs, const char *path)
 {
     assert(fs != NULL);
 
@@ -169,7 +185,7 @@ static void *find_object(espfs_fs_t *fs, const char *path)
     do {
         if (middle != skip) {
             object = (void *) fs->header + entry->offset;
-            if (strcmp(path, (char *) object + object->len) == 0) {
+            if (strcmp(path, (const char *) object + object->len) == 0) {
                 ESPFS_LOGV(__func__, "object %d", middle);
                 return object;
             }
@@ -186,17 +202,17 @@ bool espfs_stat(espfs_fs_t *fs, const char *path, espfs_stat_t *stat)
 {
     assert(fs != NULL);
 
-    espfs_object_header_t *object = find_object(fs, path);
+    const espfs_object_header_t *object = find_object(fs, path);
     if (object == NULL) {
         ESPFS_LOGD(__func__, "object not found: %s", path);
         return false;
     }
 
     memset(stat, 0, sizeof(espfs_stat_t));
     stat->type = object->type;
+    stat->index = object->index;
     if (object->type == ESPFS_TYPE_FILE) {
-        espfs_file_header_t *fh = (espfs_file_header_t *) object;
-
+        const espfs_file_header_t *fh = (const espfs_file_header_t *) object;
         stat->flags = fh->flags;
         stat->compression = fh->compression;
         stat->size = fh->file_len;
@@ -210,13 +226,13 @@ espfs_file_t *espfs_fopen(espfs_fs_t *fs, const char *path)
 {
     assert(fs != NULL);
 
-    espfs_object_header_t *object = find_object(fs, path);
+    const espfs_object_header_t *object = find_object(fs, path);
     if ((object == NULL) || (object->type != ESPFS_TYPE_FILE)) {
         ESPFS_LOGD(__func__, "file not found: %s", path);
         return NULL;
     }
 
-    espfs_file_header_t *fh = (espfs_file_header_t *) object;
+    const espfs_file_header_t *fh = (const espfs_file_header_t *) object;
 
     espfs_file_t *f = malloc(sizeof(espfs_file_t));
     if (f == NULL) {
@@ -276,12 +292,13 @@ void espfs_fclose(espfs_file_t *f)
     free(f);
 }
 
-void espfs_fstat(espfs_file_t *f, const char *path, espfs_stat_t *stat)
+void espfs_fstat(espfs_file_t *f, espfs_stat_t *stat)
 {
     assert(f != NULL);
 
     memset(stat, 0, sizeof(espfs_stat_t));
     stat->type = f->fh->object.type;
+    stat->index = f->fh->object.index;
     stat->flags = f->fh->flags;
     stat->compression = f->fh->compression;
     stat->size = f->fh->file_len;
@@ -355,12 +372,12 @@ ssize_t espfs_fread(espfs_file_t *f, void *buf, size_t len)
 
             if (remain == 0) {
                 if (f->file_pos == f->fh->file_len) {
-                    ESPFS_LOGD(__func__, "heatshrink decoder finished");
                     HSD_finish_res res = heatshrink_decoder_finish(hsd);
                     if (res < 0) {
                         ESPFS_LOGE(__func__, "heatshrink_decoder_finish");
                         return -1;
                     }
+                    ESPFS_LOGV(__func__, "heatshrink_decoder_finish");
                 }
                 return decoded;
             }

diff --git a/src/espfs_priv.h b/src/espfs_priv.h
@@ -22,6 +22,7 @@ struct espfs_fs_t {
 #endif
     const espfs_fs_header_t *header;
     const espfs_hashtable_entry_t *hashtable;
+    const espfs_sorttable_entry_t *sorttable;
 };
 
 struct espfs_file_t {