Compression feature

.github/workflows/clang-format-check.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'skip-ci')"
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run clang-format style check for C programs.
       uses: DoozyX/[email protected]
       with:

.github/workflows/clang-format-fix.yml

@@ -1,25 +1,27 @@
-name: clang-format Check
+name: clang-format Fix Format
 on: 
   workflow_dispatch:
   push:
 jobs:
   formatting-check:
-    name: Formatting Check
+    name: Commit Format Changes
     runs-on: ubuntu-latest
     if: "!contains(github.event.head_commit.message, 'skip-ci')"
+    permissions:
+      contents: write
     steps:
-    - uses: actions/checkout@v2
-    - name: Run clang-format style check for C programs.
-      uses: DoozyX/clang-format-lint-action@v0.11
+    - uses: actions/checkout@v3
+    - name: Fix C formatting issues detected by clang-format
+      uses: DoozyX/clang-format-lint-action@v0.13
       with:
         source: '.'
         extensions: 'c,h,cpp,hpp'
         clangFormatVersion: 10
         inplace: True
         style: file
         # exclude: './config '
-    - uses: EndBug/add-and-commit@v7
+    - uses: EndBug/add-and-commit@v9
       with:
           author_name: github-actions
           author_email: github-actions[bot]@users.noreply.github.com
-          message: 'Committing clang-format changes'
+          message: 'committing clang-format changes'

.github/workflows/h5bench-hdf5-1.10.4.yml

@@ -1,5 +1,8 @@
 name: h5bench (HDF5 1.10.4)
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 on:
   pull_request:
 
@@ -20,7 +23,7 @@ jobs:
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -223,7 +226,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: test
           path: build/storage/**/std*

.github/workflows/h5bench-hdf5-1.10.7.yml

@@ -1,5 +1,8 @@
 name: h5bench (HDF5 1.10.7)
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 on:
   pull_request:
 
@@ -20,7 +23,7 @@ jobs:
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -223,7 +226,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: test
           path: build/h5bench_e3sm-prefix/src/h5bench_e3sm-stamp/*

.github/workflows/h5bench-hdf5-1.10.8.yml

@@ -1,5 +1,8 @@
 name: h5bench (HDF5 1.10.8)
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 on:
   pull_request:
 
@@ -20,7 +23,7 @@ jobs:
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -223,7 +226,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: test
           path: build/storage/**/std*

.github/workflows/h5bench-hdf5-1.12.0.yml

@@ -20,7 +20,7 @@ jobs:
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -258,7 +258,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: test
           path: build/storage/**/std*

.github/workflows/h5bench-hdf5-1.14.0.yml

@@ -20,7 +20,7 @@ jobs:
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -513,7 +513,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: test
           path: build*/storage/**/std*

.github/workflows/h5bench-hdf5-1.14.1-compress.yml

@@ -0,0 +1,125 @@
+name: h5bench (HDF5 1.14.1 + compression) 
+
+on:
+  pull_request:
+
+  workflow_dispatch:
+
+jobs:
+  h5bench:
+    runs-on: ubuntu-20.04
+    container:
+      image: hpcio/hdf5-1.14.1-compression
+    timeout-minutes: 60
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Build h5bench SYNC
+        run: |
+          current="$PWD"
+
+          export HDF5_HOME=/opt/hdf5
+
+          mkdir build-sync
+          cd build-sync
+
+          cmake .. \
+            -DCMAKE_C_COMPILER=$HDF5_HOME/bin/h5pcc
+          make
+
+      - name: Test h5bench SYNC write/read
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-contig-1d-small.json
+
+      - name: Test h5bench SYNC write/read 1D N_BIT filter
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-1d-N_BIT.json
+
+      - name: Test h5bench SYNC write/read 1D SZIP filter
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-1d-SZIP.json
+
+      - name: Test h5bench SYNC write/read 1D GZIP filter
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-1d-GZIP.json
+
+      - name: Test h5bench SYNC write/read 1D SZ3 filter
+        run: |
+          export LD_LIBRARY_PATH=/opt/SZ3/lib:$LD_LIBRARY_PATH
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-1d-SZ3.json
+
+      - name: Test h5bench SYNC write/read 1D ZFP filter
+        run: |
+          export LD_LIBRARY_PATH=/opt/zfp/lib:$LD_LIBRARY_PATH
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-1d-ZFP.json
+
+      - name: Test h5bench SYNC write/read 2D N_BIT filter 
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-2d-N_BIT.json
+
+      - name: Test h5bench SYNC write/read 2D SZIP filter
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-2d-SZIP.json
+
+      - name: Test h5bench SYNC write/read 2D GZIP filter
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-2d-GZIP.json
+
+      - name: Test h5bench SYNC write/read 2D SZ3 filter
+        run: |
+          export LD_LIBRARY_PATH=/opt/SZ3/lib:$LD_LIBRARY_PATH
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-2d-SZ3.json
+
+      - name: Test h5bench SYNC write/read 2D ZFP filter
+        run: |
+          export LD_LIBRARY_PATH=/opt/zfp/lib:$LD_LIBRARY_PATH
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-2d-ZFP.json
+
+      - name: Test h5bench SYNC write/read 3D N_BIT filter
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-3d-N_BIT.json
+
+      - name: Test h5bench SYNC write/read 3D SZIP filter
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-3d-SZIP.json
+
+      - name: Test h5bench SYNC write/read 3D GZIP filter
+        run: |
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-3d-GZIP.json
+
+      - name: Test h5bench SYNC write/read 3D SZ3 filter
+        run: |
+          export LD_LIBRARY_PATH=/opt/SZ3/lib:$LD_LIBRARY_PATH
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-3d-SZ3.json
+
+      - name: Test h5bench SYNC write/read 3D ZFP filter
+        run: |
+          export LD_LIBRARY_PATH=/opt/zfp/lib:$LD_LIBRARY_PATH
+          cd build-sync
+          ./h5bench --debug --abort-on-failure --validate-mode ../samples/sync-write-read-chunked-3d-ZFP.json
+
+      - name: Upload artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test
+          path: build*/storage/**/std*
+          retention-days: 1

.github/workflows/h5bench-hdf5-1.14.1.yml

@@ -20,7 +20,7 @@ jobs:
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -513,7 +513,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: test
           path: build*/storage/**/std*

.github/workflows/h5bench-hdf5-develop-test.yml

@@ -15,7 +15,7 @@ jobs:
       OMPI_MCA_rmaps_base_oversubscribe: "yes"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -131,6 +131,9 @@ jobs:
           export ABT_DIR=/opt/argobots
           export ASYNC_DIR=/opt/vol-async
 
+          export LD_LIBRARY_PATH=/opt/SZ3/lib:$LD_LIBRARY_PATH
+          export LD_LIBRARY_PATH=/opt/zfp/lib:$LD_LIBRARY_PATH
+
           cd build-sync
 
           ctest --verbose .
@@ -152,7 +155,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: test
           path: build*/storage/**/std*

.github/workflows/h5bench-hdf5-develop.yml

@@ -15,7 +15,7 @@ jobs:
       OMPI_MCA_rmaps_base_oversubscribe: "yes"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
@@ -645,7 +645,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: test
           path: build*/storage/**/std*

.gitignore

@@ -101,4 +101,7 @@ Temporary Items
 # iCloud generated files
 *.icloud
 
+# Build and installer directories
+/build
+/installer
 # End of https://www.toptal.com/developers/gitignore/api/macos,c++,cmake,linux

.gitmodules

@@ -1,14 +1,14 @@
 [submodule "amrex"]
 	path = amrex
-	url = https://github.com/AMReX-Codes/amrex
+	url = https://github.com/AMReX-Codes/amrex.git
 
 [submodule "openpmd"]
 	path = openpmd
-	url = https://github.com/openPMD/openPMD-api
+	url = https://github.com/openPMD/openPMD-api.git
 
 [submodule "e3sm"]
 	path = e3sm
-	url = https://github.com/Parallel-NetCDF/E3SM-IO
+	url = https://github.com/Parallel-NetCDF/E3SM-IO.git
 [submodule "macsio"]
 	path = macsio
-	url = https://github.com/LLNL/MACSio
+	url = https://github.com/LLNL/MACSio.git

CMakeLists.txt

@@ -233,7 +233,7 @@ if(H5BENCH_E3SM)
     ExternalProject_Add(h5bench_e3sm
         SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/e3sm
         CONFIGURE_COMMAND autoreconf -i COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/e3sm/configure --prefix=${CMAKE_BINARY_DIR} --with-hdf5=${HDF5_HOME}
-        BUILD_COMMAND make -j 1
+        BUILD_COMMAND make
         INSTALL_COMMAND ${CMAKE_COMMAND} -E copy src/e3sm_io ${CMAKE_BINARY_DIR}/h5bench_e3sm 
         BUILD_IN_SOURCE 1
         LOG_CONFIGURE 1