improved filesystem design (EFS+), not compatible with old versions

README.md

@@ -1 +1,91 @@
-# About libespfs
+# About libespfs
+
+Libespfs is a read-only filesystem component designed for
+[ESP-IDF](https://github.com/espressif/esp-idf) and
+[ESP8266_RTOS_SDK](https://github.com/espressif/ESP8266_RTOS_SDK) that uses a
+sorted hash table to locate file and directory entries. It works with a
+monolithic binary that is generated using the mkespfsimage.py tool. It
+currently supports [heatshrink](https://github.com/atomicobject/heatshrink)
+for compression. It was originally written to use with Sprite_tm's esphttpd,
+but has been separated to be used for other uses.
+
+Libespfs can be used in other projects though, and works fine on Linux. There
+is a test Linux program in `tools/test` to read files from an espfs image.
+
+# Getting started
+
+To use this component, make a components directory in your project's root
+directory and within that directory run:
+
+`git clone --recursive https://github.com/jkent/libespfs`
+
+
+You can generate a filesystem using `tools/mkespfsiage.py`. The tool takes two
+arguments, ROOT, the directory containing the files to generate from, and
+IMAGE, the output file for the image. The script references an espfs.yaml file
+in the image ROOT, with the default settings to not add it to the image. The
+yaml file the various preprocessors and compressors to run while building the
+image. Example:
+
+```yaml
+paths:
+    '*.html': ['html-minifier', 'gzip']
+    '*': heatshrink
+```
+
+You can add your own preprocessors as well. Look at the espfs_default.yaml
+within the component as an example.
+
+# Usage
+
+There are two different ways you can use libespfs. There is the low level
+interface and there is a vfs interface for IDF. The vfs interface is the
+normal way to use libespfs. But you can use both at the same time if you wish.
+
+## Common initialization
+
+Example initialization:
+
+```C
+extern const uint8_t espfs_bin[];
+
+espfs_config_t espfs_config = {
+    .addr = espfs_bin,
+};
+espfs_fs_t *fs = espfs_init(&espfs_config);
+assert(fs != NULL);
+```
+
+```C
+espfs_deinit(fs);
+```
+
+You can also mount a filesystem from a flash partition. Instead of specifying
+**addr**, you'd specify a **part_label** string.
+
+## VFS interface
+
+```C
+esp_vfs_espfs_conf_t vfs_espfs_conf = {
+    .base_path = "/espfs",
+    .fs = fs,
+    .max_files = 5,
+};
+esp_vfs_espfs_register(&vfs_espfs_conf);
+```
+
+You can then use the system (open, read, close) or file stream (fopen, fread,
+fclose) functions to access files.
+
+## Raw interface
+
+```C
+bool espfs_stat(espfs_fs_t *fs, const char *path, espfs_stat_t *s);
+espfs_file_t *espfs_fopen(espfs_fs_t *fs, const char *path);
+void espfs_fclose(espfs_file_t *f);
+void espfs_fstat(espfs_file_t *f, const char *path, espfs_stat_t *s);
+ssize_t espfs_fread(espfs_file_t *f, void *buf, size_t len);
+ssize_t espfs_fseek(espfs_file_t *f, long offset, int mode);
+size_t espfs_ftell(espfs_file_t *f);
+ssize_t espfs_faccess(espfs_file_t *f, void **buf);
+```

cmake/include.cmake

@@ -2,22 +2,27 @@ get_filename_component(libespfs_DIR ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE)
 
 set(libespfs_SRCS
     ${libespfs_DIR}/src/espfs.c
-    ${libespfs_DIR}/third-party/heatshrink/heatshrink_decoder.c
 )
 
 set(libespfs_INCLUDE_DIRS
     ${libespfs_DIR}/include
 )
 
-set(libespfs_PRIV_INCLUDE_DIRS
-    ${libespfs_DIR}/third-party/heatshrink
-)
+if(CONFIG_ESPFS_USE_HEATSHRINK)
+    set(libespfs_SRCS ${libespfs_SRCS}
+        ${libespfs_DIR}/third-party/heatshrink/heatshrink_decoder.c
+    )
+    set(libespfs_PRIV_INCLUDE_DIRS
+        ${libespfs_DIR}/third-party/heatshrink
+    )
+endif()
 
 if(CONFIG_IDF_TARGET_ESP8266 OR ESP_PLATFORM)
     set(libespfs_SRCS ${libespfs_SRCS}
         ${libespfs_DIR}/src/vfs.c
     )
     set(libespfs_PRIV_REQUIRES
+        spi_flash
         vfs
     )
 endif()
@@ -31,7 +36,7 @@ endif()
 if(NOT CMAKE_BUILD_EARLY_EXPANSION)
     add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/libespfs.dir/requirements.stamp
         DEPENDS ${libespfs_DIR}/requirements.txt
-        COMMAND ${CMAKE_COMMAND} -E make_directory ${libespfs_DIR}/requirements.txtectory ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/libespfs.dir
+        COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/libespfs.dir
         COMMAND ${python} -m pip install -r ${libespfs_DIR}/requirements.txt
         COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/libespfs.dir/requirements.stamp
     )

docs/Doxyfile

@@ -38,8 +38,8 @@ ENABLE_PREPROCESSING   = YES
 MACRO_EXPANSION        = YES
 EXPAND_ONLY_PREDEF     = YES
 PREDEFINED             = \
-    __attribute__(x)= \
-    CONFIG_ENABLE_FLASH_MMAP
+    CONFIG_ENABLE_FLASH_MMAP=y \
+    CONFIG_ESPFS_USE_HEATSHRINK=y
 
 ## Do not complain about not having dot
 ##

docs/Makefile

@@ -8,6 +8,8 @@ SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
 
+html:
+
 # Put it first so that "make" without argument is like "make help".
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/api-reference/fh.rst

@@ -6,15 +6,13 @@ Files
 Functions
 ^^^^^^^^^
 
-.. doxygenfunction:: espfs_open
-.. doxygenfunction:: espfs_stat
-.. doxygenfunction:: espfs_flags
-.. doxygenfunction:: espfs_read
-.. doxygenfunction:: espfs_seek
-.. doxygenfunction:: espfs_is_compressed
-.. doxygenfunction:: espfs_access
-.. doxygenfunction:: espfs_filesize
-.. doxygenfunction:: espfs_close
+.. doxygenfunction:: espfs_fopen
+.. doxygenfunction:: espfs_fclose
+.. doxygenfunction:: espfs_fstat
+.. doxygenfunction:: espfs_fread
+.. doxygenfunction:: espfs_fseek
+.. doxygenfunction:: espfs_ftell
+.. doxygenfunction:: espfs_faccess
 
 Structures
 ^^^^^^^^^^
@@ -30,6 +28,6 @@ Type Definitions
 Enumerations
 ^^^^^^^^^^^^
 
-.. doxygenenum:: espfs_compression_type_t
-.. doxygenenum:: espfs_flags_t
 .. doxygenenum:: espfs_stat_type_t
+.. doxygenenum:: espfs_flags_t
+.. doxygenenum:: espfs_compression_type_t

docs/api-reference/fs.rst

@@ -8,6 +8,7 @@ Functions
 
 .. doxygenfunction:: espfs_init
 .. doxygenfunction:: espfs_deinit
+.. doxygenfunction:: espfs_stat
 
 Structures
 ^^^^^^^^^^

docs/index.rst

@@ -7,5 +7,4 @@ libespfs Documentation
 
    readme/index
    api-reference/index
-   examples
    genindex

espfs.yaml.example

@@ -1,6 +1,3 @@
-process:
-  '*':
-    - heatshrink
-  '*.js':
-    - babel-convert
-    - uglifyjs
+paths:
+    '*.html': gzip
+    '*':

espfs_defaults.yaml

@@ -1,23 +1,38 @@
-tools:
-  heatshrink:
-    level: 9
-  gzip:
-    level: 9
-  babel-convert:
-    npm:
-    - '@babel/core'
-    - '@babel/cli'
-    - '@babel/preset-env'
-    command: npx babel --no-babelrc --presets @babel/preset-env
-  babel-minify:
-    npm: babel-preset-minify
-    command: npx babel --no-babelrc --presets minify
-  html-minifier:
-    npm: html-minifier
-    command: npx html-minifier
-  uglifycss:
-    npm: uglifycss
-    command: npx uglifycss
-  uglifyjs:
-    npm: uglify-js
-    command: npx uglifyjs
+preprocessors:
+    gzip:
+        level: 9
+
+    babel-convert:
+        npm:
+            - '@babel/core'
+            - '@babel/cli'
+            - '@babel/preset-env'
+        command: npx babel --no-babelrc --presets @babel/preset-env
+
+    babel-minify:
+        npm: babel-preset-minify
+        command: npx babel --no-babelrc --presets minify
+
+    html-minifier:
+        npm: html-minifier
+        command: npx html-minifier
+
+    uglifycss:
+        npm: uglifycss
+        command: npx uglifycss
+
+    uglifyjs:
+        npm: uglify-js
+        command: npx uglifyjs
+
+compressors:
+    heatshrink:
+        window_sz2: 11
+        lookahead_sz2: 4
+
+paths:
+    'espfs.yaml': discard
+    '*.css': uglifycss
+    '*.html': html-minifier
+    '*.js': [babel-convert, uglifyjs]
+    '*': heatshrink