Skip to content

Commit

Permalink
Added support for Float16
Browse files Browse the repository at this point in the history
 * Saving and loading Float16 values is now supported
 * Float16 is no longer an "opaque" type.
 * Added some more wrapper functions relating to setting and getting
   floating-point datatype fields
 * `H5T_FLOAT16` needs to be created dynamically. Attempts at making the
   list of natively supported types dynamically modifiable by the user
   is  blocked by the fact that `HDF5BitsKind` is used for dispatch
   everywhere...
 * Unit test
  • Loading branch information
Andy Ferris authored and OmriTreidel committed Oct 26, 2016
1 parent 9b57f05 commit 814521e
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 11 deletions.
70 changes: 59 additions & 11 deletions src/plain.jl
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,16 @@ const H5T_NATIVE_DOUBLE = read_const(:H5T_NATIVE_DOUBLE_g)
# Library versions
const H5F_LIBVER_EARLIEST = 0
const H5F_LIBVER_LATEST = 1
# Constructed types (occurs at runtime)
function make_float16()
FLOAT16 = h5t_copy(H5T_NATIVE_FLOAT)
h5t_set_fields(FLOAT16, 15, 10, 5, 0, 10)
h5t_set_size(FLOAT16, 2)
h5t_set_ebias(FLOAT16, 15)
h5t_lock(FLOAT16)
return FLOAT16
end
# const H5T_FLOAT16 = make_float16() (in `__init__()`)

## Conversion between Julia types and HDF5 atomic types
hdf5_type_id(::Type{Int8}) = H5T_NATIVE_INT8
Expand All @@ -236,10 +246,11 @@ hdf5_type_id(::Type{Int32}) = H5T_NATIVE_INT32
hdf5_type_id(::Type{UInt32}) = H5T_NATIVE_UINT32
hdf5_type_id(::Type{Int64}) = H5T_NATIVE_INT64
hdf5_type_id(::Type{UInt64}) = H5T_NATIVE_UINT64
#hdf5_type_id(::Type{Float16}) = H5T_FLOAT16 (in `__init__()`)
hdf5_type_id(::Type{Float32}) = H5T_NATIVE_FLOAT
hdf5_type_id(::Type{Float64}) = H5T_NATIVE_DOUBLE

@compat typealias HDF5BitsKind Union{Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Float32, Float64}
@compat typealias HDF5BitsKind Union{Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Float16, Float32, Float64}
@compat typealias BitsKindOrString Union{HDF5BitsKind, String}

# It's not safe to use particular id codes because these can change, so we use characteristics of the type.
Expand All @@ -252,6 +263,7 @@ const hdf5_type_map = @compat Dict(
(H5T_INTEGER, H5T_SGN_NONE, convert(Csize_t, 2)) => UInt16,
(H5T_INTEGER, H5T_SGN_NONE, convert(Csize_t, 4)) => UInt32,
(H5T_INTEGER, H5T_SGN_NONE, convert(Csize_t, 8)) => UInt64,
(H5T_FLOAT, nothing, convert(Csize_t, 2)) => Float16,
(H5T_FLOAT, nothing, convert(Csize_t, 4)) => Float32,
(H5T_FLOAT, nothing, convert(Csize_t, 8)) => Float64,
)
Expand Down Expand Up @@ -1797,17 +1809,30 @@ function hdf5_to_julia_eltype(objtype)
error("character set ", cset, " not recognized")
end
elseif class_id == H5T_INTEGER || class_id == H5T_FLOAT
native_type = h5t_get_native_type(objtype.id)
try
native_size = h5t_get_size(native_type)
if class_id == H5T_INTEGER
is_signed = h5t_get_sign(native_type)
else
is_signed = nothing
# First look in the type last for a match
# otherwise fall back to a native datatype
# Allows for users to dynamically add types to the typemap
t_size = h5t_get_size(objtype)
if class_id == H5T_INTEGER
is_signed = h5t_get_sign(objtype)
else
is_signed = nothing # probably should include the mantissa size, etc...
end
if haskey(hdf5_type_map, (class_id, is_signed, t_size))
T = hdf5_type_map[(class_id, is_signed, t_size)]
else
native_type = h5t_get_native_type(objtype.id)
try
native_size = h5t_get_size(native_type)
if class_id == H5T_INTEGER
is_signed = h5t_get_sign(native_type)
else
is_signed = nothing
end
T = hdf5_type_map[(class_id, is_signed, native_size)]
finally
h5t_close(native_type)
end
T = hdf5_type_map[(class_id, is_signed, native_size)]
finally
h5t_close(native_type)
end
elseif class_id == H5T_ENUM
super_type = h5t_get_super(objtype.id)
Expand Down Expand Up @@ -1990,7 +2015,10 @@ for (jlname, h5name, outtype, argtypes, argsyms, msg) in
(:h5s_select_hyperslab, :H5Sselect_hyperslab, Herr, (Hid, Cint, Ptr{Hsize}, Ptr{Hsize}, Ptr{Hsize}, Ptr{Hsize}), (:dspace_id, :seloper, :start, :stride, :count, :block), "Error selecting hyperslab"),
(:h5t_commit, :H5Tcommit2, Herr, (Hid, Ptr{UInt8}, Hid, Hid, Hid, Hid), (:loc_id, :name, :dtype_id, :lcpl_id, :tcpl_id, :tapl_id), "Error committing type"),
(:h5t_close, :H5Tclose, Herr, (Hid,), (:dtype_id,), "Error closing datatype"),
(:h5t_lock, :H5Tlock, Herr, (Hid,), (:dtype_id,), "Error locking datatype"),
(:h5t_set_cset, :H5Tset_cset, Herr, (Hid, Cint), (:dtype_id, :cset), "Error setting character set in datatype"),
(:h5t_set_ebias, :H5Tset_ebias, Herr, (Hid, Csize_t), (:dtype_id, :ebias), "Error setting exponential bias of floating-point type"),
(:h5t_set_fields, :H5Tset_fields, Herr, (Hid, Csize_t, Csize_t, Csize_t, Csize_t, Csize_t), (:dtype_id, :spos, :epos, :esize, :mpos, :msize), "Error setting floating-point type fields"),
(:h5t_set_size, :H5Tset_size, Herr, (Hid, Csize_t), (:dtype_id, :sz), "Error setting size of datatype"),
)

Expand Down Expand Up @@ -2076,6 +2104,7 @@ for (jlname, h5name, outtype, argtypes, argsyms, ex_error) in
(:h5t_get_array_ndims, :H5Tget_array_ndims, Cint, (Hid,), (:dtype_id,), :(error("Error getting ndims of array"))),
(:h5t_get_class, :H5Tget_class, Cint, (Hid,), (:dtype_id,), :(error("Error getting class"))),
(:h5t_get_cset, :H5Tget_cset, Cint, (Hid,), (:dtype_id,), :(error("Error getting character set encoding"))),
(:h5t_get_ebias, :H5Tget_ebias, Csize_t, (Hid,), (:dtype_id,), :(error("Error getting exponential bias"))),
(:h5t_get_member_class, :H5Tget_member_class, Cint, (Hid, Cuint), (:dtype_id, :index), :(error("Error getting class of compound datatype member #", index))),
(:h5t_get_member_index, :H5Tget_member_index, Cint, (Hid, Ptr{UInt8}), (:dtype_id, :membername), :(error("Error getting index of compound datatype member \"", membername, "\""))),
(:h5t_get_member_offset, :H5Tget_member_offset, Csize_t, (Hid, Cuint), (:dtype_id, :index), :(error("Error getting offset of compound datatype member #", index))),
Expand Down Expand Up @@ -2166,6 +2195,21 @@ function h5s_get_simple_extent_dims(space_id::Hid)
h5s_get_simple_extent_dims(space_id, dims, maxdims)
return tuple(reverse!(dims)...), tuple(reverse!(maxdims)...)
end
function h5t_get_fields(type_id::Hid)
spos = Ref{Csize_t}()
epos = Ref{Csize_t}()
esize = Ref{Csize_t}()
mpos = Ref{Csize_t}()
msize = Ref{Csize_t}()
herr = ccall((:H5Tget_fields, libhdf5),
Herr,
(Hid, Ptr{Csize_t}, Ptr{Csize_t}, Ptr{Csize_t}, Ptr{Csize_t}, Ptr{Csize_t}),
type_id, spos, epos, esize, mpos, msize)
if herr < 0
error("Error getting fields of floating-point datatype")
end
return (spos[], epos[], esize[], mpos[], msize[])
end
function h5t_get_member_name(type_id::Hid, index::Integer)
pn = ccall((:H5Tget_member_name, libhdf5),
Ptr{UInt8},
Expand Down Expand Up @@ -2423,6 +2467,10 @@ function __init__()
UTF8_ATTRIBUTE_PROPERTIES[] = p_create(H5P_ATTRIBUTE_CREATE)
h5p_set_char_encoding(UTF8_ATTRIBUTE_PROPERTIES[].id, H5T_CSET_UTF8)

# Set up Float16 (must occur at runtime)
eval(:(const H5T_FLOAT16 = make_float16()))
eval(:(hdf5_type_id(::Type{Float16}) = H5T_FLOAT16))

rehash!(hdf5_type_map, length(hdf5_type_map.keys))
rehash!(hdf5_prop_get_set, length(hdf5_prop_get_set.keys))

Expand Down
7 changes: 7 additions & 0 deletions test/plain.jl
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,7 @@ h5open(fn, "r", "libver_bounds",
@test intarray == [1,2,3]
end


fd = h5open(joinpath(test_path, "datasets.h5"), "w")
fd["level_0"] = [1,2,3]
grp = g_create(fd, "mygroup")
Expand All @@ -324,3 +325,9 @@ fd["mygroup/deep_group/level_2"] = [6.0, 7.0]
datasets = get_datasets(fd)
@assert sort(map(name, datasets)) == ["/level_0", "/mygroup/deep_group/level_2", "/mygroup/level_1"]
close(fd)

arr_float16 = Float16[1.0, 0.25, 0.5, 8.0]
h5write("test_float16.h5", "x", arr_float16)
arr_float16_2 = h5read("test_float16.h5", "x")
@test isa(arr_float16_2, Vector{Float16})
@test arr_float16_2 == arr_float16

0 comments on commit 814521e

Please sign in to comment.