Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add SQLite compatibility with new module #190

Merged
merged 30 commits into from
Dec 8, 2023
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
e2ce592
Add initial functions and tests
pedroripper Nov 27, 2023
924911b
Update
pedroripper Nov 28, 2023
b592004
Remove schema [no ci]
pedroripper Nov 28, 2023
8448908
Update tests and handle bin files for SQL databases
pedroripper Nov 28, 2023
8003e92
Update tests
pedroripper Nov 28, 2023
e465c80
Format
pedroripper Nov 28, 2023
1359a00
Update tests for Time Series
pedroripper Nov 29, 2023
735aa5d
Format and add type to function parameter
pedroripper Nov 29, 2023
808fde6
Fix
pedroripper Nov 29, 2023
ab8f264
Update bin hdr interface
pedroripper Nov 29, 2023
39675fe
move some things around
guilhermebodin Dec 5, 2023
a7dca19
Update schema [no ci]
pedroripper Dec 5, 2023
f61ef8a
Fix tests
pedroripper Dec 5, 2023
99a56f6
Add database validation
pedroripper Dec 6, 2023
c3209f9
Remove comment
pedroripper Dec 6, 2023
be854e3
Add `set_vector_related` and tests
pedroripper Dec 6, 2023
d0bb131
Update interface
pedroripper Dec 6, 2023
4b06d44
Aqua quick fix
pedroripper Dec 6, 2023
7b7a658
Fix interface [no ci]
pedroripper Dec 6, 2023
40a8a2b
Update interface and regex [no ci]
pedroripper Dec 6, 2023
b9b14aa
leave the most complicate dbase regexes as a comment [no ci]
guilhermebodin Dec 6, 2023
587245f
Update readme [no ci]
pedroripper Dec 6, 2023
9f5af4f
Fix time series interface [no ci]
pedroripper Dec 6, 2023
af0d065
Fix docs
pedroripper Dec 7, 2023
e124c3a
Ignore time series test for `OpenSQL`
pedroripper Dec 7, 2023
093f6fd
Format
pedroripper Dec 7, 2023
fe149d5
Update interface
pedroripper Dec 7, 2023
cd39f1a
Fix
pedroripper Dec 7, 2023
090d137
Fix
Dec 8, 2023
37be1c2
update README.md
guilhermebodin Dec 8, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,5 @@ Manifest.toml

*.out
*.ok
debug_psrclasses
debug_psrclasses
*.sqlite
5 changes: 4 additions & 1 deletion Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@ uuid = "1eab49e5-27d8-4905-b9f6-327b6ea666c4"
version = "0.11.1"

[deps]
DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965"
DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
Encodings = "8275c4fe-57c3-4fbf-b39c-271e6148849a"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9"
Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"

[compat]
Dates = "1.6.7"
Encodings = "0.1.1"
JSON = "0.21"
Tables = "1.10"
julia = "1.6"
Dates = "1.6.7"
22 changes: 22 additions & 0 deletions src/OpenSQL/OpenSQL.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
module OpenSQL

using SQLite
using DBInterface
using Tables
# TODO talvez a gente nem precise dos DataFrames, da pra fazer com o Tables mesmo
using DataFrames

const DB = SQLite.DB

"""
SQLInterface
"""
struct SQLInterface end

include("utils.jl")
include("create.jl")
include("read.jl")
include("update.jl")
include("delete.jl")

end # module OpenSQL
100 changes: 100 additions & 0 deletions src/OpenSQL/create.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
function create_parameters!(
db::SQLite.DB,
table::String,
parameters,
)
columns = string.(keys(parameters))
sanity_check(db, table, columns)

cols = join(keys(parameters), ", ")
vals = join(values(parameters), "', '")
DBInterface.execute(db, "INSERT INTO $table ($cols) VALUES ('$vals')")
return nothing
end

function create_vector!(
db::SQLite.DB,
table::String,
id::String,
vector_name::String,
values::V,
) where {V <: AbstractVector}
table_name = "_" * table * "_" * vector_name
sanity_check(db, table_name, vector_name)
num_values = length(values)
ids = fill(id, num_values)
idx = collect(1:num_values)
tbl = Tables.table([ids idx values]; header = [:id, :idx, vector_name])
SQLite.load!(tbl, db, table_name)
return nothing
end

function create_vectors!(db::SQLite.DB, table::String, id::String, vectors)
for (vector_name, values) in vectors
create_vector!(db, table, id, string(vector_name), values)
end
return nothing
end

function create_element!(
db::SQLite.DB,
table::String;
kwargs...,
)
@assert !isempty(kwargs)
dict_parameters = Dict()
dict_vectors = Dict()

for (key, value) in kwargs
if isa(value, AbstractVector)
dict_vectors[key] = value
else
dict_parameters[key] = value
end
end

if !haskey(dict_parameters, :id)
error("A new object requires an \"id\".")

Check warning on line 57 in src/OpenSQL/create.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/create.jl#L57

Added line #L57 was not covered by tests
end
id = dict_parameters[:id]

# TODO a gente deveria ter algum esquema de transactions aqui
# se um for bem sucedido e o outro não, deveriamos dar rollback para
# antes de começar a salvar esse cara.
create_parameters!(db, table, dict_parameters)
create_vectors!(db, table, id, dict_vectors)

return nothing
end

function create_related_time_series!(

Check warning on line 70 in src/OpenSQL/create.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/create.jl#L70

Added line #L70 was not covered by tests
db::SQLite.DB,
table::String;
kwargs...,
)
table_name = "_" * table * "_TimeSeries"
dict_time_series = Dict()
for (key, value) in kwargs
@assert isa(value, String)

Check warning on line 78 in src/OpenSQL/create.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/create.jl#L75-L78

Added lines #L75 - L78 were not covered by tests
# TODO we could validate if the path exists
dict_time_series[key] = [value]
end
df = DataFrame(dict_time_series)
SQLite.load!(df, db, table_name)
return nothing

Check warning on line 84 in src/OpenSQL/create.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/create.jl#L80-L84

Added lines #L80 - L84 were not covered by tests
end

function set_related!(
db::SQLite.DB,
table1::String,
table2::String,
id_1::String,
id_2::String,
)
id_parameter_on_table_1 = lowercase(table2) * "_id"
SQLite.execute(
db,
"UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'",
)
return nothing
end
36 changes: 36 additions & 0 deletions src/OpenSQL/delete.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
function delete!(
db::SQLite.DB,
table::String,
id::String,
)
sanity_check(db, table, "id")
id_exist_in_table(db, table, id)

DBInterface.execute(db, "DELETE FROM $table WHERE id = '$id'")

# TODO We might want to delete corresponding entries in the vector tables too
return nothing
end

function delete_relation!(
guilhermebodin marked this conversation as resolved.
Show resolved Hide resolved
db::SQLite.DB,
table_1::String,
table_2::String,
table_1_id::String,
table_2_id::String,
)
if !are_related(db, table_1, table_2, table_1_id, table_2_id)
error(

Check warning on line 23 in src/OpenSQL/delete.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/delete.jl#L23

Added line #L23 was not covered by tests
"Element with id $table_1_id from table $table_1 is not related to element with id $table_2_id from table $table_2.",
)
end

id_parameter_on_table_1 = lowercase(table_2) * "_id"

DBInterface.execute(
db,
"UPDATE $table_1 SET $id_parameter_on_table_1 = '' WHERE id = '$table_1_id'",
)

return nothing
end
72 changes: 72 additions & 0 deletions src/OpenSQL/read.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
function read_parameter(
db::SQLite.DB,
table::String,
column::String,
)
if !column_exist_in_table(db, table, column) && is_vector_parameter(db, table, column)
error("column $column is a vector parameter, use `read_vector` instead.")

Check warning on line 7 in src/OpenSQL/read.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/read.jl#L7

Added line #L7 was not covered by tests
end

sanity_check(db, table, column)

query = "SELECT $column FROM $table"
df = DBInterface.execute(db, query) |> DataFrame
# TODO it can have missing values, we should decide what to do with this.
results = df[!, 1]
return results
end

function read_parameter(
db::SQLite.DB,
table::String,
column::String,
id::String,
)
if !column_exist_in_table(db, table, column) && is_vector_parameter(db, table, column)
error("column $column is a vector parameter, use `read_vector` instead.")

Check warning on line 26 in src/OpenSQL/read.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/read.jl#L26

Added line #L26 was not covered by tests
end

sanity_check(db, table, column)

query = "SELECT $column FROM $table WHERE id = '$id'"
df = DBInterface.execute(db, query) |> DataFrame
# This could be a missing value
if isempty(df)
error("id \"$id\" does not exist in table \"$table\".")

Check warning on line 35 in src/OpenSQL/read.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/read.jl#L35

Added line #L35 was not covered by tests
end
result = df[!, 1][1]
return result
end

function read_vector(
db::SQLite.DB,
table::String,
vector_name::String,
)
table_name = "_" * table * "_" * vector_name
sanity_check(db, table_name, vector_name)
ids_in_table = read_parameter(db, table, "id")

results = []
for id in ids_in_table
push!(results, read_vector(db, table, vector_name, id))
end

return results
end

function read_vector(
db::SQLite.DB,
table::String,
vector_name::String,
id::String,
)
table_name = "_" * table * "_" * vector_name
sanity_check(db, table_name, vector_name)

query = "SELECT $vector_name FROM $table_name WHERE id = '$id' ORDER BY idx"
df = DBInterface.execute(db, query) |> DataFrame
# This could be a missing value
result = df[!, 1]
return result
end
22 changes: 22 additions & 0 deletions src/OpenSQL/update.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
function update!(
db::SQLite.DB,
table::String,
column::String,
id::String,
val,
)
sanity_check(db, table, column)
DBInterface.execute(db, "UPDATE $table SET $column = '$val' WHERE id = '$id'")
return nothing
end

function update!(

Check warning on line 13 in src/OpenSQL/update.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/update.jl#L13

Added line #L13 was not covered by tests
db::SQLite.DB,
table::String,
columns::String,
id::String,
vals::V,
) where {V <: AbstractVector}
error("Updating vectors is not yet implemented.")
return nothing

Check warning on line 21 in src/OpenSQL/update.jl

View check run for this annotation

Codecov / codecov/patch

src/OpenSQL/update.jl#L20-L21

Added lines #L20 - L21 were not covered by tests
end
Loading
Loading