Merge pull request #76 from xiaodaigh/inlinestring-support

Inlinestring support

Project.toml

@@ -9,6 +9,7 @@ BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
 DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+InlineStrings = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
 Missings = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
 PooledArrays = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

src/column_loader.jl

@@ -1,13 +1,13 @@
 """
 Load data from file using metadata
 """
-column_loader(t::Type{T}, io, metadata) where {T} = begin
+function column_loader(t::Type{T}, io, metadata) where {T}
     buffer = Vector{UInt8}(undef, metadata.len)
     column_loader!(buffer, t, io, metadata)
 end
 
-# load bytes bytes from io decompress into type
-column_loader!(buffer, ::Type{T}, io, metadata) where {T} = begin
+# load bytes from io decompress into type
+function column_loader!(buffer, ::Type{T}, io, metadata) where {T}
     readbytes!(io, buffer, metadata.len)
     return Blosc.decompress(T, buffer)
 end

src/compress_then_write.jl

@@ -1,10 +1,9 @@
-compress_then_write(b::AbstractVector{T}, io) where {T} = begin
-    #return eltype(b), Vector(b)
+function compress_then_write(b::AbstractVector{T}, io) where {T}
     compress_then_write(Vector(b), io)
 end
 
 # the generic dispatch for T where is isbits
-compress_then_write(b::Vector{T}, io) where {T} = begin
+function compress_then_write(b::Vector{T}, io) where {T}
     bbc = Blosc.compress(b)
     res = length(bbc)
     write(io, bbc)

test/test-inlinestrings.jl

@@ -0,0 +1,30 @@
+# Based on the Julia implementation
+# InlineStringN are all `isbitstype` so they work with JDF automatically
+# However for other languages, we may still need to explicitly support them and conver them
+# to string is not available in those languages
+using JDF
+using InlineStrings, DataFrames, CSV
+
+using Random:randstring
+
+@testset "Test InlineStrings get loaded and saved properly" begin
+    a = DataFrames.DataFrame(a = [randstring(254) |> String255 for i in 1:100])
+
+    path = tempdir()
+
+    CSV.write(joinpath(path, "tmp.csv"), a)
+
+    a1 = CSV.read(joinpath(path, "tmp.csv"), DataFrame)
+
+    a1.a = a1.a .|> String255
+
+    JDF.save(joinpath(path, "tmp.jdf"), a1)
+
+    a2 = JDF.load(joinpath(path, "tmp.jdf")) |> DataFrame
+
+    @test eltype(a2.a) == String255
+
+    # clean up
+    rm("tmp.csv"; force=true)
+    rm("tmp.jdf"; force=true, recursive=true)
+end