updating for 1.0.0 (#85)

LLNL · Aug 18, 2022 · 59b7f38 · 59b7f38
1 parent 3802c0f
commit 59b7f38
Show file tree

Hide file tree

Showing 17 changed files with 690 additions and 208 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -53,7 +53,7 @@ jobs:
         git clone https://github.com/LLNL/zfp.git
         export HOME_DIR=$(echo ~)
         cd zfp
-        git checkout 0.5.5
+        git checkout 1.0.0
         mkdir build;cd build
         cmake -D ZFP_BIT_STREAM_WORD_SIZE=8 -D BUILD_CFP=ON -D CMAKE_INSTALL_PREFIX=$PWD/zfp -D BUILD_TESTING=OFF -D BUILD_UTILITIES=OFF ..
         make install
@@ -107,13 +107,15 @@ jobs:
         run: |
           git clone https://github.com/LLNL/zfp.git
           cd zfp
-          git checkout 0.5.5
+          git checkout 1.0.0
           mkdir -p build
           cd build
           cmake .. \
             -DCMAKE_BUILD_TYPE=Release \
             -DBUILD_UTILITIES=OFF \
-            -DBUILD_TESTING=OFF
+            -DBUILD_TESTING=OFF \
+            -DBUILD_CFP=ON \
+            -DZFP_BIT_STREAM_WORD_SIZE=8
           cmake --build . -j 3
           cmake --build . --target install
       - name: update PATH

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -8,13 +8,13 @@ if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
 endif ()
 
 #------------------------------------------------------------------------------#
-# Parse version number from H5Zzfp_plugin.h
+# Parse version number from H5Zzfp_version.h
 #------------------------------------------------------------------------------#
-file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_plugin.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_MAJOR")
+file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_version.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_MAJOR")
 string(REGEX REPLACE "^.*MAJOR " "" H5Z_ZFP_VERSION_MAJOR "${H5Z_ZFP_H}")
-file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_plugin.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_MINOR")
+file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_version.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_MINOR")
 string(REGEX REPLACE "^.*MINOR " "" H5Z_ZFP_VERSION_MINOR "${H5Z_ZFP_H}")
-file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_plugin.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_PATCH")
+file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/src/H5Zzfp_version.h" H5Z_ZFP_H REGEX "^\#define H5Z_FILTER_ZFP_VERSION_PATCH")
 string(REGEX REPLACE "^.*PATCH " "" H5Z_ZFP_VERSION_PATCH "${H5Z_ZFP_H}")
 set(H5Z_ZFP_VERSION "${H5Z_ZFP_VERSION_MAJOR}.${H5Z_ZFP_VERSION_MINOR}.${H5Z_ZFP_VERSION_PATCH}")
 

diff --git a/config.make b/config.make
@@ -22,7 +22,19 @@ H5Z_ZFP_PLUGIN := $(H5Z_ZFP_BASE)/plugin
 H5Z_ZFP_VERSINFO := $(shell grep '^\#define H5Z_FILTER_ZFP_VERSION_[MP]' $(H5Z_ZFP_BASE)/H5Zzfp_plugin.h | cut -d' ' -f3 | tr '\n' '.' | cut -d'.' -f-3 2>/dev/null)
 ZFP_HAS_REVERSIBLE :=
 ifneq ($(ZFP_HOME),)
-ZFP_HAS_REVERSIBLE := $(shell grep zfp_stream_set_reversible $(ZFP_HOME)/include/zfp.h)
+    ZFP_HAS_REVERSIBLE := $(shell grep zfp_stream_set_reversible $(ZFP_HOME)/include/zfp.h 2>/dev/null)
+endif
+
+# Construct make-time knowledge of ZFP library version
+ZFP_LIB_VERSION := $(shell grep '^\#define ZFP_VERSION_[MPT]' $(ZFP_HOME)/include/zfp/version.h 2>/dev/null | tr ' ' '\n' | grep '[0-9]' | tr -d '\n')
+ifeq ($(ZFP_LIB_VERSION),)
+    ZFP_LIB_VERSION := $(shell grep '^\#define ZFP_VERSION_[MRPT]' $(ZFP_HOME)/include/zfp.h 2>/dev/null | tr ' ' '\n' | grep '[0-9]' | tr -d '\n' 2>/dev/null)
+endif
+ifeq ($(ZFP_LIB_VERSION),)
+    ZFP_LIB_VERSION := $(shell grep '^\#define ZFP_VERSION_[MRPT]' $(ZFP_HOME)/inc/zfp.h 2>/dev/null | tr ' ' '\n' | grep '[0-9]' | tr -d '\n' 2>/dev/null)
+endif
+ifeq ($(ZFP_LIB_VERSION),)
+    $(warning WARNING: ZFP lib version not detected by make -- some tests may run)
 endif
 
 # Detect system type
@@ -131,8 +143,9 @@ ifneq ($(wildcard $(ZFP_HOME)/include),)
 ZFP_INC = $(ZFP_HOME)/include
 else ifneq ($(wildcard $(ZFP_HOME)/inc),)
 ZFP_INC = $(ZFP_HOME)/inc
-else
-$(error "cannot find ZFP include dir")
+endif
+ifeq ($(wildcard $(ZFP_INC)/zfp.h),) # no header file
+$(error "zfp.h not found")
 endif
 
 ifeq ($(wildcard $(ZFP_HOME)/lib),)
@@ -142,10 +155,18 @@ ZFP_LIB = $(ZFP_HOME)/lib
 endif
 
 # Check if ZFP has CFP
-ifeq ($(wildcard $(ZFP_LIB)/libcfp.*),)
-ZFP_HAS_CFP = 0
+ifeq ($(wildcard $(ZFP_LIB)/libcfp.*),) # no cfp lib file
+  ZFP_HAS_CFP = 0
 else
-ZFP_HAS_CFP = 1
+  ifeq ($(wildcard $(ZFP_INC)/zfp/array.h),) # no 1.0.0 header file
+    ifeq ($(wildcard $(ZFP_INC)/cfparrays.h),) # no 0.5.5 header file
+        ZFP_HAS_CFP = 0
+    else
+        ZFP_HAS_CFP = 1
+    endif
+  else
+    ZFP_HAS_CFP = 1
+  endif
 endif
 
 # Check if specified individually the HDF5 include directory,

diff --git a/src/H5Zzfp.c b/src/H5Zzfp.c
@@ -39,7 +39,6 @@ and calls to bitstream methods with 'B ' as in
 #include "H5PLextern.h"
 #include "H5Spublic.h"
 #include "zfp.h"
-#include "bitstream.h"
 #define Z
 #define B 
 #endif /* ] AS_SILO_BUILTIN */
@@ -48,9 +47,29 @@ and calls to bitstream methods with 'B ' as in
 #include "H5Zzfp_props_private.h"
 
 /* Convenient CPP logic to capture ZFP lib version numbers as compile time hex number */
-#define ZFP_VERSION_NO__(Maj,Min,Rel)  (0x0 ## Maj ## Min ## Rel)
-#define ZFP_VERSION_NO_(Maj,Min,Rel)   ZFP_VERSION_NO__(Maj,Min,Rel)
-#define ZFP_VERSION_NO                 ZFP_VERSION_NO_(ZFP_VERSION_MAJOR,ZFP_VERSION_MINOR,ZFP_VERSION_RELEASE)
+#define ZFP_VERSION_NO__(Maj,Min,Pat,Twk)  (0x ## Maj ## Min ## Pat ## Twk)
+#define ZFP_VERSION_NO_(Maj,Min,Pat,Twk)   ZFP_VERSION_NO__(Maj,Min,Pat,Twk)
+#if defined(ZFP_VERSION_TWEAK)
+#define ZFP_VERSION_NO                 ZFP_VERSION_NO_(ZFP_VERSION_MAJOR,ZFP_VERSION_MINOR,ZFP_VERSION_PATCH,ZFP_VERSION_TWEAK)
+#elif defined(ZFP_VERSION_RELEASE)
+#define ZFP_VERSION_NO                 ZFP_VERSION_NO_(ZFP_VERSION_MAJOR,ZFP_VERSION_MINOR,ZFP_VERSION_RELEASE,0)
+#elif defined(ZFP_VERSION_PATCH)
+#define ZFP_VERSION_NO                 ZFP_VERSION_NO_(ZFP_VERSION_MAJOR,ZFP_VERSION_MINOR,ZFP_VERSION_PATCH,0)
+#else
+#error ZFP LIBRARY VERSION NOT DETECTED
+#endif
+
+/* Older versions of ZFP don't define this */
+#ifndef ZFP_VERSION_STRING
+#define ZFP_VERSION_STR__(Maj,Min,Rel) #Maj "." #Min "." #Rel
+#define ZFP_VERSION_STR_(Maj,Min,Rel)  ZFP_VERSION_STR__(Maj,Min,Rel)
+#define ZFP_VERSION_STRING             ZFP_VERSION_STR_(ZFP_VERSION_MAJOR,ZFP_VERSION_MINOR,ZFP_VERSION_RELEASE)
+#endif
+
+/* Older versions of ZFP don't define this publicly */
+#ifndef ZFP_CODEC
+#define ZFP_CODEC ZFP_VERSION_MINOR
+#endif
 
 /* Convenient CPP logic to capture H5Z_ZFP Filter version numbers as string and hex number */
 #define H5Z_FILTER_ZFP_VERSION_STR__(Maj,Min,Pat) #Maj "." #Min "." #Pat
@@ -150,7 +169,7 @@ H5Z_zfp_can_apply(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id)
         H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, -1, "bad chunk data space");
 
     /* confirm ZFP library can handle this data */
-#if ZFP_VERSION_NO < 0x0051
+#if ZFP_VERSION_NO < 0x0510
     if (!(dclass == H5T_FLOAT))
         H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_BADTYPE, 0,
             "requires datatype class of H5T_FLOAT");
@@ -173,7 +192,7 @@ H5Z_zfp_can_apply(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id)
 
     if (ndims_used == 0 || ndims_used > max_ndims)
         H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0,
-#if ZFP_VERSION_NO < 0x0053
+#if ZFP_VERSION_NO < 0x0530
             "chunk must have only 1...3 non-unity dimensions");
 #else
             "chunk must have only 1...4 non-unity dimensions");
@@ -261,11 +280,11 @@ H5Z_zfp_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id)
         case 1: dummy_field = Z zfp_field_1d(0, zt, dims_used[0]); break;
         case 2: dummy_field = Z zfp_field_2d(0, zt, dims_used[1], dims_used[0]); break;
         case 3: dummy_field = Z zfp_field_3d(0, zt, dims_used[2], dims_used[1], dims_used[0]); break;
-#if ZFP_VERSION_NO >= 0x0054
+#if ZFP_VERSION_NO >= 0x0540
         case 4: dummy_field = Z zfp_field_4d(0, zt, dims_used[3], dims_used[2], dims_used[1], dims_used[0]); break;
 #endif
         default: H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_BADVALUE, 0,
-#if ZFP_VERSION_NO < 0x0053
+#if ZFP_VERSION_NO < 0x0530
                      "chunks may have only 1...3 non-unity dims");
 #else
                      "chunks may have only 1...4 non-unity dims");
@@ -297,7 +316,7 @@ H5Z_zfp_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id)
 
     /* Into hdr_cd_values, we encode ZFP library and H5Z-ZFP plugin version info at
        entry 0 and use remaining entries as a tiny buffer to write ZFP native header. */
-    hdr_cd_values[0] = (unsigned int) ((ZFP_VERSION_NO<<16) | H5Z_FILTER_ZFP_VERSION_NO);
+    hdr_cd_values[0] = (unsigned int) ((ZFP_VERSION_NO<<16) | (ZFP_CODEC<<12) | H5Z_FILTER_ZFP_VERSION_NO);
     if (0 == (dummy_bstr = B stream_open(&hdr_cd_values[1], sizeof(hdr_cd_values))))
         H5Z_ZFP_PUSH_AND_GOTO(H5E_RESOURCE, H5E_NOSPACE, 0, "stream_open() failed");
 
@@ -313,14 +332,14 @@ H5Z_zfp_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id)
                 Z zfp_stream_set_rate(dummy_zstr, ctrls.details.rate, zt, ndims_used, 0);
                 break;
             case H5Z_ZFP_MODE_PRECISION:
-#if ZFP_VERSION_NO < 0x0051
+#if ZFP_VERSION_NO < 0x0510
                 Z zfp_stream_set_precision(dummy_zstr, ctrls.details.prec, zt);
 #else
                 Z zfp_stream_set_precision(dummy_zstr, ctrls.details.prec);
 #endif
                 break;
             case H5Z_ZFP_MODE_ACCURACY:
-#if ZFP_VERSION_NO < 0x0051
+#if ZFP_VERSION_NO < 0x0510
                 Z zfp_stream_set_accuracy(dummy_zstr, ctrls.details.acc, zt);
 #else
                 Z zfp_stream_set_accuracy(dummy_zstr, ctrls.details.acc);
@@ -331,7 +350,7 @@ H5Z_zfp_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id)
                     ctrls.details.expert.maxbits, ctrls.details.expert.maxprec,
                     ctrls.details.expert.minexp);
                 break;
-#if ZFP_VERSION_NO >= 0x0055
+#if ZFP_VERSION_NO >= 0x0550
             case H5Z_ZFP_MODE_REVERSIBLE:
                 Z zfp_stream_set_reversible(dummy_zstr);
                 break;
@@ -348,14 +367,14 @@ H5Z_zfp_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id)
                 Z zfp_stream_set_rate(dummy_zstr, *((double*) &mem_cd_values[2]), zt, ndims_used, 0);
                 break;
             case H5Z_ZFP_MODE_PRECISION:
-#if ZFP_VERSION_NO < 0x0051
+#if ZFP_VERSION_NO < 0x0510
                 Z zfp_stream_set_precision(dummy_zstr, mem_cd_values[2], zt);
 #else
                 Z zfp_stream_set_precision(dummy_zstr, mem_cd_values[2]);
 #endif
                 break;
             case H5Z_ZFP_MODE_ACCURACY:
-#if ZFP_VERSION_NO < 0x0051
+#if ZFP_VERSION_NO < 0x0510
                 Z zfp_stream_set_accuracy(dummy_zstr, *((double*) &mem_cd_values[2]), zt);
 #else
                 Z zfp_stream_set_accuracy(dummy_zstr, *((double*) &mem_cd_values[2]));
@@ -365,7 +384,7 @@ H5Z_zfp_set_local(hid_t dcpl_id, hid_t type_id, hid_t chunk_space_id)
                 Z zfp_stream_set_params(dummy_zstr, mem_cd_values[2], mem_cd_values[3],
                     mem_cd_values[4], (int) mem_cd_values[5]);
                 break;
-#if ZFP_VERSION_NO >= 0x0055
+#if ZFP_VERSION_NO >= 0x0550
             case H5Z_ZFP_MODE_REVERSIBLE:
                 Z zfp_stream_set_reversible(dummy_zstr);
                 break;
@@ -480,6 +499,58 @@ get_zfp_info_from_cd_values(size_t cd_nelmts, unsigned int const *cd_values,
     return retval;
 }
 
+/*
+Compare ZFP codec version used when data was written to what is
+currently being used to read the data. There is a challenge here
+in that earlier versions of this filter recorded only the ZFP
+library version, not the codec version. Although ZFP codec version
+was encoded as minor digit of ZFP library version, that convention
+ended with ZFP version 1.0.0. So, if an old version of this filter
+is used with newer ZFP libraries, we won't know the codec version
+used to write the data with certainty. The best we can do is guess
+it. If there becomes a version of the ZFP library for which that guess
+(currently 5) is wrong, the logic here will need to be updated to
+capture knowledge of the ZFP library version for which the codec
+version was incrimented.
+*/
+
+static int
+zfp_codec_version_mismatch(
+    unsigned int h5zfpver_from_cd_val_data_in_file,
+    unsigned int zfpver_from_cd_val_data_in_file,
+    unsigned int zfpcodec_from_cd_val_data_in_file)
+{
+    int writer_codec;
+    int reader_codec;
+
+    if (h5zfpver_from_cd_val_data_in_file < 0x0110)
+    {
+        /* for data written with older versions of the filter,
+           we infer codec from ZFP library version stored in the file. */
+        zfpver_from_cd_val_data_in_file <<= 4;
+        if (zfpver_from_cd_val_data_in_file < 0x0500)
+            writer_codec = 4;
+        else if (zfpver_from_cd_val_data_in_file < 0x1000)
+            writer_codec = (zfpver_from_cd_val_data_in_file & 0x0F00)>>8;
+        else if (zfpver_from_cd_val_data_in_file == 0x1000)
+            writer_codec = 5;
+        else
+            writer_codec = 5; /* can only guess */
+    }
+    else
+        writer_codec = zfpcodec_from_cd_val_data_in_file;
+
+#if ZFP_VERSION_NO < 0x0500
+    reader_codec = 4;
+#elif ZFP_VERSION_NO < 0x1000
+    reader_codec = 5;
+#else
+    reader_codec = ZFP_CODEC;
+#endif
+
+    return writer_codec > reader_codec;
+}
+
 static size_t
 H5Z_filter_zfp(unsigned int flags, size_t cd_nelmts,
     const unsigned int cd_values[], size_t nbytes,
@@ -488,7 +559,9 @@ H5Z_filter_zfp(unsigned int flags, size_t cd_nelmts,
     static char const *_funcname_ = "H5Z_filter_zfp";
     void *newbuf = 0;
     size_t retval = 0;
-    int cd_vals_zfpver = (cd_values[0]>>16)&0x0000FFFF;
+    unsigned int cd_vals_h5zzfpver = cd_values[0]&0x00000FFF;
+    unsigned int cd_vals_zfpcodec = (cd_values[0]>>12)&0x0000000F;
+    unsigned int cd_vals_zfpver = (cd_values[0]>>16)&0x0000FFFF;
     H5T_order_t swap = H5T_ORDER_NONE;
     uint64 zfp_mode, zfp_meta;
     bitstream *bstr = 0;
@@ -504,10 +577,9 @@ H5Z_filter_zfp(unsigned int flags, size_t cd_nelmts,
         int status;
         size_t bsize, dsize;
 
-        /* Worry about zfp version and endian mismatch only for decompression */
-        if (cd_vals_zfpver > ZFP_VERSION)
-            H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_NOSPACE, 0, "ZFP lib version, "
-                ZFP_VERSION_STRING ", too old to decompress this data");
+        /* Worry about zfp version mismatch only for decompression */
+        if (zfp_codec_version_mismatch(cd_vals_h5zzfpver, cd_vals_zfpver, cd_vals_zfpcodec))
+            H5Z_ZFP_PUSH_AND_GOTO(H5E_PLINE, H5E_READERROR, 0, "ZFP codec version mismatch");
 
         /* Set up the ZFP field object */
         if (0 == (zfld = Z zfp_field_alloc()))

diff --git a/src/H5Zzfp_plugin.h b/src/H5Zzfp_plugin.h
@@ -1,23 +1,14 @@
 #ifndef H5Z_ZFP_PLUGIN_H
 #define H5Z_ZFP_PLUGIN_H
 
-/* Filter ID number registered with The HDF Group */
-#define H5Z_FILTER_ZFP 32013
+#include "H5Zzfp_version.h"
 
-#define H5Z_FILTER_ZFP_VERSION_MAJOR 1
-#define H5Z_FILTER_ZFP_VERSION_MINOR 0
-#define H5Z_FILTER_ZFP_VERSION_PATCH 1
+/* HDF5 generic cd_vals[] memory layout (6 unsigned ints) for
+   controlling H5Z-ZFP behavior as a plugin. NOTE: These cd_vals
+   used to pass properties in-memory from caller to filter via HDF5
+   generic interface are NOT THE SAME AS the cd_vals[] that
+   ultimately get stored to the file for the filter "header" data. 
 
-#define H5Z_ZFP_MODE_RATE      1
-#define H5Z_ZFP_MODE_PRECISION 2
-#define H5Z_ZFP_MODE_ACCURACY  3
-#define H5Z_ZFP_MODE_EXPERT    4
-#define H5Z_ZFP_MODE_REVERSIBLE 5
-
-#define H5Z_ZFP_CD_NELMTS_MEM ((size_t) 6) /* used in public API to filter */
-#define H5Z_ZFP_CD_NELMTS_MAX ((size_t) 6) /* max, over all versions, used in dataset header */
-
-/* HDF5 filter cd_vals[] layout (6 unsigned ints)
 cd_vals    0       1        2         3         4         5    
 ----------------------------------------------------------------
 rate:      1    unused    rateA     rateB     unused    unused
@@ -26,10 +17,6 @@ accuracy:  3    unused    accA      accB      unused    unused
 expert:    4    unused    minbits   maxbits   maxprec   minexp
 
 A/B are high/low words of a double.
-
-Note: This is *NOT* the same layout that is ultimately stored
-to the file. A wholly different, cd_vals is stored in the file
-using zfp_write_header.
 */
 
 #define H5Pset_zfp_rate_cdata(R, N, CD)          \
@@ -67,10 +54,10 @@ do {                                                    \
     if ((N>=6)&&(CD[0] == H5Z_ZFP_MODE_EXPERT))         \
     {                                                   \
         unsigned int *p; int *q;                        \
-        p = MiB; *p = CD[2];                            \
-        p = MaB; *p = CD[3];                            \
-        p = MaP; *p = CD[4];                            \
-        q = MiE; *q = (int) CD[5];                      \
+        p = &MiB; *p = CD[2];                           \
+        p = &MaB; *p = CD[3];                           \
+        p = &MaP; *p = CD[4];                           \
+        q = &MiE; *q = (int) CD[5];                     \
     }                                                   \
 } while(0)