From e2ce5926a0772e41d02cb0cc48010e2df7e90a31 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Mon, 27 Nov 2023 20:28:38 -0300 Subject: [PATCH 01/30] Add initial functions and tests --- .gitignore | 3 +- Project.toml | 5 +- src/OpenSQL/OpenSQL.jl | 22 ++ src/OpenSQL/create.jl | 68 ++++ src/OpenSQL/delete.jl | 13 + src/OpenSQL/read.jl | 64 ++++ src/OpenSQL/update.jl | 22 ++ src/OpenSQL/utils.jl | 113 +++++++ src/PSRClassesInterface.jl | 3 + src/sql_interface.jl | 68 ++++ test/OpenSQL/create_case.jl | 183 +++++++++++ test/OpenSQL/data/case_1/current_schema.sql | 337 ++++++++++++++++++++ test/runtests.jl | 3 + 13 files changed, 902 insertions(+), 2 deletions(-) create mode 100644 src/OpenSQL/OpenSQL.jl create mode 100644 src/OpenSQL/create.jl create mode 100644 src/OpenSQL/delete.jl create mode 100644 src/OpenSQL/read.jl create mode 100644 src/OpenSQL/update.jl create mode 100644 src/OpenSQL/utils.jl create mode 100644 src/sql_interface.jl create mode 100644 test/OpenSQL/create_case.jl create mode 100644 test/OpenSQL/data/case_1/current_schema.sql diff --git a/.gitignore b/.gitignore index 3c006041..54d595e3 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ Manifest.toml *.out *.ok -debug_psrclasses \ No newline at end of file +debug_psrclasses +*.sqlite \ No newline at end of file diff --git a/Project.toml b/Project.toml index 73617976..6a7c1818 100644 --- a/Project.toml +++ b/Project.toml @@ -3,14 +3,17 @@ uuid = "1eab49e5-27d8-4905-b9f6-327b6ea666c4" version = "0.11.1" [deps] +DBInterface = "a10d1c49-ce27-4219-8d33-6db1a4562965" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Encodings = "8275c4fe-57c3-4fbf-b39c-271e6148849a" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +SQLite = "0aa819cd-b072-5ff4-a722-6bc24af294d9" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] +Dates = "1.6.7" Encodings = "0.1.1" JSON = "0.21" Tables = "1.10" julia = "1.6" -Dates = "1.6.7" diff --git a/src/OpenSQL/OpenSQL.jl b/src/OpenSQL/OpenSQL.jl new file mode 100644 index 00000000..5fe4c83c --- /dev/null +++ b/src/OpenSQL/OpenSQL.jl @@ -0,0 +1,22 @@ +module OpenSQL + +using SQLite +using DBInterface +using Tables +# TODO talvez a gente nem precise dos DataFrames, da pra fazer com o Tables mesmo +using DataFrames + +const DB = SQLite.DB + +""" +SQLInterface +""" +struct SQLInterface end + +include("utils.jl") +include("create.jl") +include("read.jl") +include("update.jl") +include("delete.jl") + +end # module OpenSQL diff --git a/src/OpenSQL/create.jl b/src/OpenSQL/create.jl new file mode 100644 index 00000000..9b0d4371 --- /dev/null +++ b/src/OpenSQL/create.jl @@ -0,0 +1,68 @@ +function create_parameters!( + db::SQLite.DB, + table::String, + parameters, +) + columns = string.(keys(parameters)) + sanity_check(db, table, columns) + + cols = join(keys(parameters), ", ") + vals = join(values(parameters), "', '") + DBInterface.execute(db, "INSERT INTO $table ($cols) VALUES ('$vals')") + return nothing +end + +function create_vector!( + db::SQLite.DB, + table::String, + id::String, + vector_name::String, + values::V, +) where {V <: AbstractVector} + table_name = "_" * table * "_" * vector_name + sanity_check(db, table_name, vector_name) + num_values = length(values) + ids = fill(id, num_values) + idx = collect(1:num_values) + tbl = Tables.table([ids idx values]; header = [:id, :idx, vector_name]) + SQLite.load!(tbl, db, table_name) + return nothing +end + +function create_vectors!(db::SQLite.DB, table::String, id::String, vectors) + for (vector_name, values) in vectors + create_vector!(db, table, id, string(vector_name), values) + end + return nothing +end + +function create_element!( + db::SQLite.DB, + table::String; + kwargs..., +) + @assert !isempty(kwargs) + dict_parameters = Dict() + dict_vectors = Dict() + + for (key, value) in kwargs + if isa(value, AbstractVector) + dict_vectors[key] = value + else + dict_parameters[key] = value + end + end + + if !haskey(dict_parameters, :id) + error("A new object requires an \"id\".") + end + id = dict_parameters[:id] + + # TODO a gente deveria ter algum esquema de transactions aqui + # se um for bem sucedido e o outro não, deveriamos dar rollback para + # antes de começar a salvar esse cara. + create_parameters!(db, table, dict_parameters) + create_vectors!(db, table, id, dict_vectors) + + return nothing +end diff --git a/src/OpenSQL/delete.jl b/src/OpenSQL/delete.jl new file mode 100644 index 00000000..e8b9fe59 --- /dev/null +++ b/src/OpenSQL/delete.jl @@ -0,0 +1,13 @@ +function delete!( + db::SQLite.DB, + table::String, + id::String, +) + sanity_check(db, table, "id") + id_exist_in_table(db, table, id) + + DBInterface.execute(db, "DELETE FROM $table WHERE id = '$id'") + + # TODO We might want to delete corresponding entries in the vector tables too + return nothing +end diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl new file mode 100644 index 00000000..8173a6d6 --- /dev/null +++ b/src/OpenSQL/read.jl @@ -0,0 +1,64 @@ +function read_parameter( + db::SQLite.DB, + table::String, + column::String, +) + sanity_check(db, table, column) + + query = "SELECT $column FROM $table" + df = DBInterface.execute(db, query) |> DataFrame + # TODO it can have missing values, we should decide what to do with this. + results = df[!, 1] + return results +end + +function read_parameter( + db::SQLite.DB, + table::String, + column::String, + id::String, +) + sanity_check(db, table, column) + + query = "SELECT $column FROM $table WHERE id = '$id'" + df = DBInterface.execute(db, query) |> DataFrame + # This could be a missing value + if isempty(df) + error("id \"$id\" does not exist in table \"$table\".") + end + result = df[!, 1][1] + return result +end + +function read_vector( + db::SQLite.DB, + table::String, + vector_name::String, +) + table_name = "_" * table * "_" * vector_name + sanity_check(db, table_name, vector_name) + ids_in_table = read_parameter(db, table, "id") + + results = [] + for id in ids_in_table + push!(results, read_vector(db, table, vector_name, id)) + end + + return results +end + +function read_vector( + db::SQLite.DB, + table::String, + vector_name::String, + id::String, +) + table_name = "_" * table * "_" * vector_name + sanity_check(db, table_name, vector_name) + + query = "SELECT $vector_name FROM $table_name WHERE id = '$id' ORDER BY idx" + df = DBInterface.execute(db, query) |> DataFrame + # This could be a missing value + result = df[!, 1] + return result +end diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl new file mode 100644 index 00000000..61b70b74 --- /dev/null +++ b/src/OpenSQL/update.jl @@ -0,0 +1,22 @@ +function update!( + db::SQLite.DB, + table::String, + column::String, + id::String, + val, +) + sanity_check(db, table, column) + DBInterface.execute(db, "UPDATE $table SET $column = '$val' WHERE id = '$id'") + return nothing +end + +function update!( + db::SQLite.DB, + table::String, + columns::String, + id::String, + vals::V, +) where {V <: AbstractVector} + error("Updating vectors is not yet implemented.") + return nothing +end diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl new file mode 100644 index 00000000..1459b86c --- /dev/null +++ b/src/OpenSQL/utils.jl @@ -0,0 +1,113 @@ +function execute_statements(db::SQLite.DB, file::String) + if !isfile(file) + error("file not found: $file") + end + statements = open(joinpath(file), "r") do io + return read(io, String) + end + commands = split(statements, ";") + for command in commands + if !isempty(command) + SQLite.execute(db, command) + end + end + return nothing +end + +function create_empty_db(database_path::String, file::String) + if isfile(database_path) + error("file already exists: $database_path") + end + db = SQLite.DB(database_path) + execute_statements(db, file) + return db +end + +function load_db(database_path::String) + if !isfile(database_path) + error("file not found: $database_path") + end + db = SQLite.DB(database_path) + return db +end + +function set_related!( + db::SQLite.DB, + table1::String, + table2::String, + id_1::String, + id_2::String, +) + id_parameter_on_table_1 = lowercase(table2) * "_id" + SQLite.execute( + db, + "UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'", + ) + return nothing +end + +function column_names(db::SQLite.DB, table::String) + cols = SQLite.columns(db, table) |> DataFrame + return cols.name +end + +function table_names(db::SQLite.DB) + tbls = SQLite.tables(db) |> DataFrame + return tbls.name +end + +function column_exist_in_table(db::SQLite.DB, table::String, column::String) + cols = column_names(db, table) + return column in cols +end + +function table_exist_in_db(db::SQLite.DB, table::String) + tbls = table_names(db) + return table in tbls +end + +function check_if_column_exists(db::SQLite.DB, table::String, column::String) + if !column_exist_in_table(db, table, column) + # TODO we could make a suggestion on the closest string and give an error like + # Did you mean xxxx? + error("column $column does not exist in table $table.") + end + return nothing +end + +function check_if_table_exists(db::SQLite.DB, table::String) + if !table_exist_in_db(db, table) + # TODO we could make a suggestion on the closest string and give an error like + # Did you mean xxxx? + error("table $table does not exist in database.") + end + return nothing +end + +function sanity_check(db::SQLite.DB, table::String, column::String) + # TODO We could make an option to disable sanity checks globally. + check_if_table_exists(db, table) + check_if_column_exists(db, table, column) + return nothing +end + +function sanity_check(db::SQLite.DB, table::String, columns::Vector{String}) + # TODO We could make an option to disable sanity checks globally. + check_if_table_exists(db, table) + for column in columns + check_if_column_exists(db, table, column) + end + return nothing +end + +function id_exist_in_table(db::SQLite.DB, table::String, id::String) + sanity_check(db, table, "id") + query = "SELECT COUNT(id) FROM $table WHERE id = '$id'" + df = DBInterface.execute(db, query) |> DataFrame + if df[!, 1][1] == 0 + error("id \"$id\" does not exist in table \"$table\".") + end + return nothing +end + +close(db::SQLite.DB) = DBInterface.close!(db) diff --git a/src/PSRClassesInterface.jl b/src/PSRClassesInterface.jl index f3310786..d22c632c 100644 --- a/src/PSRClassesInterface.jl +++ b/src/PSRClassesInterface.jl @@ -21,6 +21,9 @@ include("PMD/PMD.jl") const Attribute = PMD.Attribute const DataStruct = PMD.DataStruct +include("OpenSQL/OpenSQL.jl") +include("sql_interface.jl") + # simple and generic interface include("study_interface.jl") include("reader_writer_interface.jl") diff --git a/src/sql_interface.jl b/src/sql_interface.jl new file mode 100644 index 00000000..28bc134e --- /dev/null +++ b/src/sql_interface.jl @@ -0,0 +1,68 @@ +const SQLInterface = OpenSQL.SQLInterface + +function create_study( + ::SQLInterface; + data_path::AbstractString = pwd(), + schema::AbstractString = "schema", + study_collection::String = "PSRStudy", + kwargs..., +) + path_db = joinpath(data_path, "psrclasses.sqlite") + path_schema = joinpath(data_path, "$(schema).sql") + db = OpenSQL.create_empty_db(path_db, path_schema) + OpenSQL.create_element!(db, study_collection; kwargs...) + return db +end + +load_study(::SQLInterface; data_path::String) = OpenSQL.load_db(data_path) + +# Read +get_vector(db::OpenSQL.DB, table::String, vector_name::String, element_id::String) = + OpenSQL.read_vector(db, table, vector_name, element_id) + +get_vectors(db::OpenSQL.DB, table::String, vector_name::String) = + OpenSQL.read_vector(db, table, vector_name) + +max_elements(db::OpenSQL.DB, collection::String) = + length(OpenSQL.column_names(db, collection)) + +get_parm(db::OpenSQL.DB, collection::String, attribute::String, element_id::String) = + OpenSQL.read_parameter(db, collection, attribute, element_id) + +get_parms(db::OpenSQL.DB, collection::String, attribute::String) = + OpenSQL.read_parameter(db, collection, attribute) + +get_attributes(db::OpenSQL.DB, collection::String) = OpenSQL.column_names(db, collection) + +get_collections(db::OpenSQL.DB) = return OpenSQL.table_names(db) + +# Modification +create_element!(db::OpenSQL.DB, collection::String; kwargs...) = + OpenSQL.create_element!(db, collection; kwargs...) + +delete_element!(db::OpenSQL.DB, collection::String, element_id::String) = + OpenSQL.delete!(db, collection, element_id) + +set_related!( + db::OpenSQL.DB, + source::String, + target::String, + source_id::String, + target_id::String, +) = OpenSQL.set_related!(db, source, target, source_id, target_id) + +set_parm!( + db::OpenSQL.DB, + collection::String, + attribute::String, + element_id::String, + value, +) = OpenSQL.update!(db, collection, attribute, element_id, value) + +set_vector!( + db::OpenSQL.DB, + collection::String, + attribute::String, + element_id::String, + values::AbstractVector, +) = OpenSQL.update!(db, collection, attribute, element_id, values) diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl new file mode 100644 index 00000000..51618a26 --- /dev/null +++ b/test/OpenSQL/create_case.jl @@ -0,0 +1,183 @@ +function create_case_1() + case_path = joinpath(@__DIR__, "data", "case_1") + if isfile(joinpath(case_path, "psrclasses.sqlite")) + rm(joinpath(case_path, "psrclasses.sqlite")) + end + + db = PSRI.create_study( + PSRI.SQLInterface(); + data_path = case_path, + schema = "current_schema", + study_collection = "StudyParameters", + id = "Toy Case", + n_periods = 3, + n_subperiods = 2, + subperiod_duration = 24.0, + ) + + @test typeof(db) == PSRI.OpenSQL.DB + + PSRI.create_element!( + db, + "Resource"; + id = "R1", + ref_availability = 100.0, + subperiod_av_type = "PerPeriod", + subperiod_cost_type = "PerPeriod", + ref_cost = 1.0, + ) + + PSRI.create_element!( + db, + "Resource"; + id = "R2", + ref_availability = 20.0, + subperiod_av_type = "PerSubperiodConstant", + subperiod_cost_type = "PerPeriod", + ref_cost = 1.0, + ) + + PSRI.create_element!( + db, + "Resource"; + id = "R3", + ref_availability = 100.0, + subperiod_av_type = "PerPeriod", + subperiod_cost_type = "PerPeriod", + ref_cost = 1.0, + ) + + PSRI.create_element!( + db, + "PowerAsset"; + id = "Generator 1", + capacity = 50.0, + output_cost = 10.0, + resource_cost_multiplier = 10.0, + commitment_type = "Linearized", + ) + + PSRI.create_element!( + db, + "PowerAsset"; + id = "Generator 2", + capacity = 100.0, + output_cost = 12.0, + resource_cost_multiplier = 12.0, + commitment_type = "Linearized", + ) + + PSRI.create_element!( + db, + "PowerAsset"; + id = "Generator 3", + capacity = 100.0, + output_cost = 15.0, + resource_cost_multiplier = 15.0, + commitment_type = "Linearized", + ) + + PSRI.create_element!( + db, + "PowerAsset"; + id = "Generator 4", + output_sign = "DemandLike", + curve_type = "Forced", + commitment_type = "AlwaysOn", + capacity = 100.0, + ) + + PSRI.create_element!( + db, + "ConversionCurve"; + id = "Conversion curve 1", + unit = "MW", + max_capacity_fractions = [0.1, 0.2, 0.3, 0.4], + conversion_efficiencies = [1.0, 2.0], + ) + + PSRI.create_element!( + db, + "ConversionCurve"; + id = "Conversion curve 2", + unit = "MW", + max_capacity_fractions = [0.5, 0.3, 0.2, 0.1], + conversion_efficiencies = [1.0, 2.0, 4.0], + ) + + PSRI.create_element!( + db, + "ConversionCurve"; + id = "Conversion curve 3", + unit = "MW", + ) + + PSRI.set_related!( + db, + "PowerAsset", + "Resource", + "Generator 1", + "R1", + ) + PSRI.set_related!( + db, + "PowerAsset", + "Resource", + "Generator 2", + "R2", + ) + PSRI.set_related!( + db, + "PowerAsset", + "Resource", + "Generator 3", + "R3", + ) + + PSRI.OpenSQL.close(db) + + db = PSRI.OpenSQL.load_db(joinpath(case_path, "psrclasses.sqlite")) + + @test PSRI.get_parm(db, "StudyParameters", "id", "Toy Case") == "Toy Case" + @test PSRI.get_parm(db, "StudyParameters", "n_periods", "Toy Case") == 3 + @test PSRI.get_parm(db, "StudyParameters", "n_subperiods", "Toy Case") == 2 + @test PSRI.get_parm(db, "StudyParameters", "subperiod_duration", "Toy Case") == 24.0 + + @test PSRI.get_parms(db, "Resource", "id") == ["R1", "R2", "R3"] + @test PSRI.get_parms(db, "Resource", "ref_availability") == [100.0, 20.0, 100.0] + @test PSRI.get_parms(db, "Resource", "subperiod_av_type") == + ["PerPeriod", "PerSubperiodConstant", "PerPeriod"] + @test PSRI.get_parms(db, "Resource", "subperiod_cost_type") == + ["PerPeriod", "PerPeriod", "PerPeriod"] + @test PSRI.get_parms(db, "Resource", "ref_cost") == [1.0, 1.0, 1.0] + + PSRI.set_related!( + db, + "PowerAsset", + "Resource", + "Generator 1", + "R1", + ) + + PSRI.set_related!( + db, + "PowerAsset", + "Resource", + "Generator 2", + "R2", + ) + + PSRI.set_related!( + db, + "PowerAsset", + "Resource", + "Generator 3", + "R3", + ) + + PSRI.OpenSQL.close(db) + + return rm(joinpath(case_path, "psrclasses.sqlite")) +end + +create_case_1() diff --git a/test/OpenSQL/data/case_1/current_schema.sql b/test/OpenSQL/data/case_1/current_schema.sql new file mode 100644 index 00000000..dde39a3c --- /dev/null +++ b/test/OpenSQL/data/case_1/current_schema.sql @@ -0,0 +1,337 @@ +PRAGMA foreign_keys = ON; + +-- Study Parameters +CREATE TABLE StudyParameters ( + id TEXT PRIMARY KEY, + -- Implicit multiplier factor for representing the objective function (usually 1000) + obj_factor REAL NOT NULL DEFAULT 1000, + -- Indicates whether: single-node representation, zonal representation, network representation ("DC flow" linear model) + network_type TEXT NOT NULL DEFAULT 'SingleNode' + CHECK( + network_type IN ( + 'SingleNode', + 'Zonal', + 'DCFlow' + ) + ), + -- Indicates whether: constant number and duration of subperiods, constant number, varying duration of subperiods, varying number, constant duration of subperiods, varying number, varying duration of subperiods + subperiod_type TEXT NOT NULL DEFAULT 'Constant' + CHECK( + subperiod_type IN ( + 'Constant', + 'VariableDuration', + 'VariableNumber', + 'Variable' + ) + ), + -- How many subperiods in each period + n_subperiods INTEGER NOT NULL DEFAULT 1, + -- Duration of each subperiod in hours + subperiod_duration REAL NOT NULL DEFAULT 1.0, + -- How each step in the yearly "cycle" should be called - usually "month", "week", or "day" depending on CyclesPerYear + cycle_name TEXT NOT NULL DEFAULT 'month', + -- Number of cycles per year (usually 12, 52, or 365), for cyclical multi-year representation + cycles_per_year INTEGER NOT NULL DEFAULT 12, + -- Starting year for the study (used to adjust ResourceData and Modification inputs) + start_year INTEGER NOT NULL DEFAULT 1, + -- Starting cycle number considered for the study (used to adjust ResourceData and Modification inputs), from 1 to CyclesPerYear + start_cycle INTEGER NOT NULL DEFAULT 1, + -- Discount rate per year (dimensionless) + annual_discount_rate REAL NOT NULL DEFAULT 0.0, + -- How many sequential periods will be modeled + n_periods INTEGER NOT NULL DEFAULT 1 +); + +CREATE TABLE _StudyParameters_n_subperiods ( + study_period INTEGER, + study_parameters_id TEXT NOT NULL, + n_subperiods INTEGER NOT NULL, + FOREIGN KEY (study_parameters_id) REFERENCES StudyParameters (id) +); + +CREATE TABLE _StudyParameters_subperiod_duration ( + study_period INTEGER, + study_parameters_id TEXT NOT NULL, + subperiod_duration REAL NOT NULL, + FOREIGN KEY (study_parameters_id) REFERENCES StudyParameters (id) +); + +-- Resource +CREATE TABLE Resource ( + id TEXT PRIMARY KEY, + -- General descriptor text (optional) + description TEXT, + -- Label for grouping together similar resources (optional) + grouping_label TEXT, + -- Unit of representation for the resource + unit TEXT NOT NULL DEFAULT "MWh", + -- Alternative labeling for "Unit-times-Study.ObjFactor" (optional) + big_unit TEXT, + -- Facilitates SDDP correspondence: (0) no direct correspondence, (1) electricity-like variable resource (CDEF, elastic demand, gnd gauging station, power injection), (2) electricity-like fixed resource (battery) (3) hydro inflow gauging station (4) standard fuel (unlimited availability) (5) nonstandard fuel (fuel contracts and/or gas network) (6) electrification process + aux_resource_type TEXT NOT NULL DEFAULT 'NoDirectCorrespondence' + CHECK( + aux_resource_type IN ( + 'NoDirectCorrespondence', + 'ElectricityLikeVariable', + 'ElectricityLikeFixed', + 'HydroInflow', + 'StandardFuel', + 'NonstandardFuel', + 'ElectrificationProcess' + ) + ), + -- Identifies whether a resource (0) is not shared (assets that point to this resource have "duplicate" resource availabilities), (1) is shared and must be used in full, (2) is shared and may be used partially + shared_type TEXT NOT NULL DEFAULT 'NotShared' + CHECK( + shared_type IN ( + 'NotShared', + 'SharedMustUseFull', + 'SharedMayUsePartial' + ) + ), + -- Identifies whether resource availability (0) is unlimited (1) is constrained per-subperiod with per-subperiod data (2) is constrained per-subperiod with per-period data (constant across subperiods) (3) is constrained per-period (allows sharing resource between subperiods) + subperiod_av_type TEXT NOT NULL DEFAULT 'Unlimited' + CHECK( + subperiod_av_type IN ( + 'Unlimited', + 'PerSubperiod', + 'PerSubperiodConstant', + 'PerPeriod' + ) + ), + -- Identifies whether (0) resource availability is expressed in "Units per hour" (standard representation) (1) resource availability is expressed in "Aggregate BigUnits" (sums over subperiods) (2) resource availability is expressed in "p.u." (interpreted as proportional to asset Capacity) (3) resource availability is always zero (battery-like) + av_unit_type TEXT NOT NULL DEFAULT 'UnitsPerHour' + CHECK( + av_unit_type IN ( + 'UnitsPerHour', + 'AggregateBigUnits', + 'PerUnit', + 'AlwaysZero' + ) + ), + -- Identifies whether the resource cost (0) is always zero (1) is constant across the study (equal to RefCost for all periods and subperiods) (2) varies per-period but not per-subperiod (3) varies per-period and per-subperiod + subperiod_cost_type TEXT NOT NULL DEFAULT 'AlwaysZero' + CHECK( + subperiod_cost_type IN ( + 'AlwaysZero', + 'Constant', + 'PerPeriod', + 'PerSubperiod' + ) + ), + -- Identifies whether the resource (0) cannot be explicitly stored (1) can be stored between subperiods within each period (but not between periods) (2) can be stored between periods (but is simplified intraperiod) (3) can be stored both between periods and intraperiod + storage_type TEXT NOT NULL DEFAULT 'NoStorage' + CHECK( + storage_type IN ( + 'NoStorage', + 'Intraperiod', + 'Interperiod', + 'Both' + ) + ), + -- Reference per-period cost in $/Unit, usually overwritten by the referenced ResourceData object + ref_cost REAL NOT NULL DEFAULT 0.0, + -- Reference per-period availability in Units, usually overwritten by the referenced ResourceData object + ref_availability REAL NOT NULL DEFAULT 0.0 +); + +CREATE TABLE _Resource_ref_cost_vector ( + -- Reference per-period cost in $/Unit, usually overwritten by the referenced ResourceData object + resource_period INTEGER NOT NULL, + resource_id TEXT NOT NULL, + ref_cost REAL NOT NULL, + FOREIGN KEY (resource_id) REFERENCES Resource (id), + PRIMARY KEY (resource_period, resource_id) +); + +CREATE TABLE _Resource_ref_availability_vector ( + -- Reference per-period availability in Units, usually overwritten by the referenced ResourceData object + resource_period INTEGER, + resource_id TEXT NOT NULL, + ref_availability REAL NOT NULL, + FOREIGN KEY (resource_id) REFERENCES Resource (id), + PRIMARY KEY (resource_period, resource_id) +); + +-- Conversion Curve +CREATE TABLE ConversionCurve ( + id TEXT PRIMARY KEY, + -- General descriptor text (optional) + description TEXT, + -- unit + unit TEXT NOT NULL, + -- vertical_axis_unit_type + vertical_axis_unit_type TEXT NOT NULL DEFAULT 'AlwaysZero' + CHECK( + vertical_axis_unit_type IN ( + 'UnitsPerHour', + 'AggregateBigUnits', + 'PerUnit', + 'AlwaysZero' + ) + ), + -- horizontal_axis_validation_type + horizontal_axis_validation_type TEXT NOT NULL DEFAULT 'AlwaysZero' + CHECK( + horizontal_axis_validation_type IN ( + 'UnitsPerHour', + 'AggregateBigUnits', + 'PerUnit', + 'AlwaysZero' + ) + ) +); + +CREATE TABLE _ConversionCurve_max_capacity_fractions ( + id TEXT, + idx INTEGER NOT NULL, + -- Fraction of the asset's maximum capacity corresponding to each Segment + max_capacity_fractions REAL NOT NULL, + + FOREIGN KEY (id) REFERENCES ConversionCurve(id) ON DELETE CASCADE, + PRIMARY KEY (id, idx) +); + +CREATE TABLE _ConversionCurve_conversion_efficiencies ( + id TEXT, + idx INTEGER NOT NULL, + -- Resource conversion factor in MWh/Resource.Unit, varying per Segment + conversion_efficiencies REAL NOT NULL, + + FOREIGN KEY (id) REFERENCES ConversionCurve(id) ON DELETE CASCADE, + PRIMARY KEY (id, idx) +); + +-- Benefit Curve +CREATE TABLE BenefitCurve ( + id TEXT PRIMARY KEY, + -- General descriptor text (optional) + description TEXT, + -- vertical_axis_unit_type + vertical_axis_unit_type TEXT NOT NULL DEFAULT 'AlwaysZero' + CHECK( + vertical_axis_unit_type IN ( + 'UnitsPerHour', + 'AggregateBigUnits', + 'PerUnit', + 'AlwaysZero' + ) + ), + -- horizontal_axis_validation_type + horizontal_axis_validation_type TEXT NOT NULL DEFAULT 'AlwaysZero' + CHECK( + horizontal_axis_validation_type IN ( + 'UnitsPerHour', + 'AggregateBigUnits', + 'PerUnit', + 'AlwaysZero' + ) + ), + -- zero_position_type TODO (Bodin: inventei isso aqui. Certamente esses não são os enums) + zero_position_type TEXT NOT NULL DEFAULT 'ZeroAtOrigin' + CHECK( + zero_position_type IN ( + 'ZeroAtOrigin', + 'ZeroAtEnd' + ) + ) +); + +CREATE TABLE _BenefitCurve_resource_av_fractions ( + benefit_id TEXT PRIMARY KEY, + segment INTEGER NOT NULL, + -- Fraction of the asset's maximum capacity corresponding to each Segment + resource_av_fractions REAL NOT NULL, + FOREIGN KEY (benefit_id) REFERENCES BenefitCurve(id) +); + +CREATE TABLE _BenefitCurve_consumption_preferences ( + benefit_id TEXT PRIMARY KEY, + segment INTEGER NOT NULL, + -- Benefit factor in $/MWh, varying per Segment + consumption_preferences REAL NOT NULL, + FOREIGN KEY (benefit_id) REFERENCES BenefitCurve(id) +); + +-- Power Assets +CREATE TABLE PowerAsset ( + id TEXT PRIMARY KEY, + -- General descriptor text on asset physical features (optional) + description TEXT, + -- General descriptor text on asset model representation (optional) + representation_notes TEXT, + -- Label for grouping together similar assets (optional) + grouping_label TEXT, + -- Facilitates SDDP correspondence: (0) no correspondence (1) standard demand ("inelastic" - see options 8 and 9) (2) standard thermal ("unlimited availability" - see option 7) (3) renewable (4) hydro (5) battery (6) power injection (7) non-standard thermal (has fuel contract and/or gas network representation) (8) elastic demand (9) flexible demand (10) electrification demand (11) Csp + -- TODO Bodin, acho que podemos tirar esses aux da frente de alguns nomes. + aux_asset_type TEXT NOT NULL DEFAULT 'NoCorrespondence' + CHECK( + aux_asset_type IN ( + 'NoCorrespondence', + 'StandardDemand', + 'StandardThermal', + 'Renewable', + 'Hydro', + 'Battery', + 'PowerInjection', + 'NonstandardThermal', + 'ElasticDemand', + 'FlexibleDemand', + 'ElectrificationDemand', + 'Csp' + ) + ), + -- Indicates whether the asset (0) is generation-like (contribution >0), (1) is demand-like (contribution <0), (2) is neither and can have either positive or negative contribution + output_sign TEXT NOT NULL DEFAULT 'GenerationLike' + CHECK( + output_sign IN ( + 'GenerationLike', + 'DemandLike', + 'Either' + ) + ), + -- Indicates whether the asset (0) has no network connection (ignore in a "network-representation" run) (1) has a single-bus connection (2) has a multi-bus fixed proportion connection + bus_connection_type TEXT NOT NULL DEFAULT 'NoConnection' + CHECK( + bus_connection_type IN ( + 'NoConnection', + 'SingleBus', + 'MultiBus' + ) + ), + -- Combines deprecated ConversionType and BenefitType + curve_type TEXT, -- TODO Bodin: aqui eu me perdi um pouco, não soube dizer o que isso significa + -- Indicates whether the asset (0) is always on (1) is always off (2) uses a linearized commitment variable representation (3) uses a binary commitment variable representation (4) uses fixed commitment data read from an external data source + commitment_type TEXT NOT NULL DEFAULT 'AlwaysOn' + CHECK( + commitment_type IN ( + 'AlwaysOn', + 'AlwaysOff', + 'Linearized', + 'Binary', + 'External' + ) + ), + -- Maximum capacity in MW, in absolute terms (for electricity injections or withdrawals) + capacity REAL NOT NULL DEFAULT 0.0, + -- Derating factor for reducing available capacity (dimensionless) + capacity_derating REAL NOT NULL DEFAULT 1.0, + -- Base direct O&M cost in $/MWh + output_cost REAL NOT NULL DEFAULT 0.0, + -- Base resource conversion factor in MWh/Resource.Unit + conversion_factor REAL NOT NULL DEFAULT 0.0, + -- Multiplier factor for the availability of the resource (dimensionless) + resource_av_multiplier REAL NOT NULL DEFAULT 1.0, + -- Multiplier factor for the cost of the resource (dimensionless) + resource_cost_multiplier REAL NOT NULL DEFAULT 1.0, + -- Additive factor to the cost of the resource, in $/Resource.Unit + resource_cost_adder REAL NOT NULL DEFAULT 0.0, + + resource_id TEXT, + conversion_id TEXT, + benefit_curve_id TEXT, + -- TODO Bodin comment: All foreign keys must be in the end of the table definition + FOREIGN KEY(resource_id) REFERENCES Resource(id), + FOREIGN KEY(conversion_id) REFERENCES ConversionCurve(id), + FOREIGN KEY(benefit_curve_id) REFERENCES BenefitCurve(id) +); \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index bf642c8c..1897842d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -58,4 +58,7 @@ end @testset "Utils" begin @time include("utils.jl") end + @testset "OpenSQL" begin + @time include("OpenSQL/create_case.jl") + end end From 924911b4c1b6dff6152e2fec6b59dd10ccc4b219 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Tue, 28 Nov 2023 11:45:58 -0300 Subject: [PATCH 02/30] Update --- src/OpenSQL/delete.jl | 23 ++++++++++++++++++++++ src/OpenSQL/read.jl | 29 +++++++++++++++++++++++++++ src/OpenSQL/utils.jl | 4 ++++ src/sql_interface.jl | 39 +++++++++++++++++++++++++++---------- test/OpenSQL/create_case.jl | 34 ++++++++++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 10 deletions(-) diff --git a/src/OpenSQL/delete.jl b/src/OpenSQL/delete.jl index e8b9fe59..8e3823f9 100644 --- a/src/OpenSQL/delete.jl +++ b/src/OpenSQL/delete.jl @@ -11,3 +11,26 @@ function delete!( # TODO We might want to delete corresponding entries in the vector tables too return nothing end + +function delete_relation!( + db::SQLite.DB, + table_1::String, + table_2::String, + table_1_id::String, + table_2_id::String, +) + if !has_relation(db, table_1, table_2, table_1_id, table_2_id) + error( + "Element with id $table_1_id from table $table_1 is not related to element with id $table_2_id from table $table_2.", + ) + end + + id_parameter_on_table_1 = lowercase(table2) * "_id" + + DBInterface.execute( + db, + "UPDATE $table1 SET $id_parameter_on_table_1 = '' WHERE id = '$id_1'", + ) + + return nothing +end diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl index 8173a6d6..5d4ebc43 100644 --- a/src/OpenSQL/read.jl +++ b/src/OpenSQL/read.jl @@ -3,6 +3,10 @@ function read_parameter( table::String, column::String, ) + if !column_exist_in_table(db, table, column) && is_vector_parameter(db, table, column) + error("column $column is a vector parameter, use `read_vector` instead.") + end + sanity_check(db, table, column) query = "SELECT $column FROM $table" @@ -18,6 +22,10 @@ function read_parameter( column::String, id::String, ) + if !column_exist_in_table(db, table, column) && is_vector_parameter(db, table, column) + error("column $column is a vector parameter, use `read_vector` instead.") + end + sanity_check(db, table, column) query = "SELECT $column FROM $table WHERE id = '$id'" @@ -62,3 +70,24 @@ function read_vector( result = df[!, 1] return result end + +function has_relation( + db::SQLite.DB, + table_1::String, + table_2::String, + table_1_id::String, + table_2_id::String, +) + sanity_check(db, table_1, "id") + sanity_check(db, table_2, "id") + id_exist_in_table(db, table_1, table_1_id) + id_exist_in_table(db, table_2, table_2_id) + + id_parameter_on_table_1 = lowercase(table_2) * "_id" + + if read_parameter(db, table_1, id_parameter_on_table_1, table_1_id) == table_2_id + return true + else + return false + end +end diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index 1459b86c..53078a8f 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -110,4 +110,8 @@ function id_exist_in_table(db::SQLite.DB, table::String, id::String) return nothing end +function is_vector_parameter(db::SQLite.DB, table::String, column::String) + return table_exist_in_db(db, "_" * table * "_" * column) +end + close(db::SQLite.DB) = DBInterface.close!(db) diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 28bc134e..571dd20f 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -24,7 +24,7 @@ get_vectors(db::OpenSQL.DB, table::String, vector_name::String) = OpenSQL.read_vector(db, table, vector_name) max_elements(db::OpenSQL.DB, collection::String) = - length(OpenSQL.column_names(db, collection)) + length(get_parms(db, collection, "id")) get_parm(db::OpenSQL.DB, collection::String, attribute::String, element_id::String) = OpenSQL.read_parameter(db, collection, attribute, element_id) @@ -32,7 +32,18 @@ get_parm(db::OpenSQL.DB, collection::String, attribute::String, element_id::Stri get_parms(db::OpenSQL.DB, collection::String, attribute::String) = OpenSQL.read_parameter(db, collection, attribute) -get_attributes(db::OpenSQL.DB, collection::String) = OpenSQL.column_names(db, collection) +function get_attributes(db::OpenSQL.DB, collection::String) + columns = OpenSQL.column_names(db, collection) + + tables = OpenSQL.table_names(db) + vector_attributes = Vector{String}() + for table in tables + if startswith(table, "_" * collection * "_") + push!(vector_attributes, split(table, collection * "_")[end]) + end + end + return vcat(columns, vector_attributes) +end get_collections(db::OpenSQL.DB) = return OpenSQL.table_names(db) @@ -43,14 +54,6 @@ create_element!(db::OpenSQL.DB, collection::String; kwargs...) = delete_element!(db::OpenSQL.DB, collection::String, element_id::String) = OpenSQL.delete!(db, collection, element_id) -set_related!( - db::OpenSQL.DB, - source::String, - target::String, - source_id::String, - target_id::String, -) = OpenSQL.set_related!(db, source, target, source_id, target_id) - set_parm!( db::OpenSQL.DB, collection::String, @@ -66,3 +69,19 @@ set_vector!( element_id::String, values::AbstractVector, ) = OpenSQL.update!(db, collection, attribute, element_id, values) + +set_related!( + db::OpenSQL.DB, + source::String, + target::String, + source_id::String, + target_id::String, +) = OpenSQL.set_related!(db, source, target, source_id, target_id) + +delete_relation!( + db::OpenSQL.DB, + source::String, + target::String, + source_id::String, + target_id::String, +) = OpenSQL.delete_relation!(db, source, target, source_id, target_id) diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 51618a26..7b128cc3 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -57,6 +57,10 @@ function create_case_1() commitment_type = "Linearized", ) + @test PSRI.get_parm(db, "PowerAsset", "capacity", "Generator 1") == 50.0 + PSRI.set_parm!(db, "PowerAsset", "capacity", "Generator 1", 400.0) + @test PSRI.get_parm(db, "PowerAsset", "capacity", "Generator 1") == 400.0 + PSRI.create_element!( db, "PowerAsset"; @@ -112,6 +116,14 @@ function create_case_1() unit = "MW", ) + @test PSRI.get_vector( + db, + "ConversionCurve", + "conversion_efficiencies", + "Conversion curve 2", + ) == + [1.0, 2.0, 4.0] + PSRI.set_related!( db, "PowerAsset", @@ -134,6 +146,10 @@ function create_case_1() "R3", ) + @test PSRI.max_elements(db, "PowerAsset") == 4 + @test PSRI.max_elements(db, "Resource") == 3 + @test PSRI.max_elements(db, "ConversionCurve") == 3 + PSRI.OpenSQL.close(db) db = PSRI.OpenSQL.load_db(joinpath(case_path, "psrclasses.sqlite")) @@ -175,6 +191,24 @@ function create_case_1() "R3", ) + @test PSRI.get_parm(db, "PowerAsset", "resource_id", "Generator 1") == "R1" + + @test PSRI.max_elements(db, "PowerAsset") == 4 + @test PSRI.max_elements(db, "Resource") == 3 + + PSRI.delete_element!(db, "PowerAsset", "Generator 4") + PSRI.delete_element!(db, "Resource", "R3") + + @test PSRI.max_elements(db, "PowerAsset") == 3 + @test PSRI.max_elements(db, "Resource") == 2 + + @test PSRI.get_attributes(db, "Resource") == + ["id", "description", "grouping_label", "unit", "big_unit", + "aux_resource_type", "shared_type", "subperiod_av_type", + "av_unit_type", "subperiod_cost_type", "storage_type", + "ref_cost", "ref_availability", "ref_cost_vector", + "ref_availability_vector"] + PSRI.OpenSQL.close(db) return rm(joinpath(case_path, "psrclasses.sqlite")) From b592004df77e2d5c96e6448e6cb994325ae489f2 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Tue, 28 Nov 2023 15:20:11 -0300 Subject: [PATCH 03/30] Remove schema [no ci] --- test/OpenSQL/data/case_1/current_schema.sql | 337 -------------------- 1 file changed, 337 deletions(-) delete mode 100644 test/OpenSQL/data/case_1/current_schema.sql diff --git a/test/OpenSQL/data/case_1/current_schema.sql b/test/OpenSQL/data/case_1/current_schema.sql deleted file mode 100644 index dde39a3c..00000000 --- a/test/OpenSQL/data/case_1/current_schema.sql +++ /dev/null @@ -1,337 +0,0 @@ -PRAGMA foreign_keys = ON; - --- Study Parameters -CREATE TABLE StudyParameters ( - id TEXT PRIMARY KEY, - -- Implicit multiplier factor for representing the objective function (usually 1000) - obj_factor REAL NOT NULL DEFAULT 1000, - -- Indicates whether: single-node representation, zonal representation, network representation ("DC flow" linear model) - network_type TEXT NOT NULL DEFAULT 'SingleNode' - CHECK( - network_type IN ( - 'SingleNode', - 'Zonal', - 'DCFlow' - ) - ), - -- Indicates whether: constant number and duration of subperiods, constant number, varying duration of subperiods, varying number, constant duration of subperiods, varying number, varying duration of subperiods - subperiod_type TEXT NOT NULL DEFAULT 'Constant' - CHECK( - subperiod_type IN ( - 'Constant', - 'VariableDuration', - 'VariableNumber', - 'Variable' - ) - ), - -- How many subperiods in each period - n_subperiods INTEGER NOT NULL DEFAULT 1, - -- Duration of each subperiod in hours - subperiod_duration REAL NOT NULL DEFAULT 1.0, - -- How each step in the yearly "cycle" should be called - usually "month", "week", or "day" depending on CyclesPerYear - cycle_name TEXT NOT NULL DEFAULT 'month', - -- Number of cycles per year (usually 12, 52, or 365), for cyclical multi-year representation - cycles_per_year INTEGER NOT NULL DEFAULT 12, - -- Starting year for the study (used to adjust ResourceData and Modification inputs) - start_year INTEGER NOT NULL DEFAULT 1, - -- Starting cycle number considered for the study (used to adjust ResourceData and Modification inputs), from 1 to CyclesPerYear - start_cycle INTEGER NOT NULL DEFAULT 1, - -- Discount rate per year (dimensionless) - annual_discount_rate REAL NOT NULL DEFAULT 0.0, - -- How many sequential periods will be modeled - n_periods INTEGER NOT NULL DEFAULT 1 -); - -CREATE TABLE _StudyParameters_n_subperiods ( - study_period INTEGER, - study_parameters_id TEXT NOT NULL, - n_subperiods INTEGER NOT NULL, - FOREIGN KEY (study_parameters_id) REFERENCES StudyParameters (id) -); - -CREATE TABLE _StudyParameters_subperiod_duration ( - study_period INTEGER, - study_parameters_id TEXT NOT NULL, - subperiod_duration REAL NOT NULL, - FOREIGN KEY (study_parameters_id) REFERENCES StudyParameters (id) -); - --- Resource -CREATE TABLE Resource ( - id TEXT PRIMARY KEY, - -- General descriptor text (optional) - description TEXT, - -- Label for grouping together similar resources (optional) - grouping_label TEXT, - -- Unit of representation for the resource - unit TEXT NOT NULL DEFAULT "MWh", - -- Alternative labeling for "Unit-times-Study.ObjFactor" (optional) - big_unit TEXT, - -- Facilitates SDDP correspondence: (0) no direct correspondence, (1) electricity-like variable resource (CDEF, elastic demand, gnd gauging station, power injection), (2) electricity-like fixed resource (battery) (3) hydro inflow gauging station (4) standard fuel (unlimited availability) (5) nonstandard fuel (fuel contracts and/or gas network) (6) electrification process - aux_resource_type TEXT NOT NULL DEFAULT 'NoDirectCorrespondence' - CHECK( - aux_resource_type IN ( - 'NoDirectCorrespondence', - 'ElectricityLikeVariable', - 'ElectricityLikeFixed', - 'HydroInflow', - 'StandardFuel', - 'NonstandardFuel', - 'ElectrificationProcess' - ) - ), - -- Identifies whether a resource (0) is not shared (assets that point to this resource have "duplicate" resource availabilities), (1) is shared and must be used in full, (2) is shared and may be used partially - shared_type TEXT NOT NULL DEFAULT 'NotShared' - CHECK( - shared_type IN ( - 'NotShared', - 'SharedMustUseFull', - 'SharedMayUsePartial' - ) - ), - -- Identifies whether resource availability (0) is unlimited (1) is constrained per-subperiod with per-subperiod data (2) is constrained per-subperiod with per-period data (constant across subperiods) (3) is constrained per-period (allows sharing resource between subperiods) - subperiod_av_type TEXT NOT NULL DEFAULT 'Unlimited' - CHECK( - subperiod_av_type IN ( - 'Unlimited', - 'PerSubperiod', - 'PerSubperiodConstant', - 'PerPeriod' - ) - ), - -- Identifies whether (0) resource availability is expressed in "Units per hour" (standard representation) (1) resource availability is expressed in "Aggregate BigUnits" (sums over subperiods) (2) resource availability is expressed in "p.u." (interpreted as proportional to asset Capacity) (3) resource availability is always zero (battery-like) - av_unit_type TEXT NOT NULL DEFAULT 'UnitsPerHour' - CHECK( - av_unit_type IN ( - 'UnitsPerHour', - 'AggregateBigUnits', - 'PerUnit', - 'AlwaysZero' - ) - ), - -- Identifies whether the resource cost (0) is always zero (1) is constant across the study (equal to RefCost for all periods and subperiods) (2) varies per-period but not per-subperiod (3) varies per-period and per-subperiod - subperiod_cost_type TEXT NOT NULL DEFAULT 'AlwaysZero' - CHECK( - subperiod_cost_type IN ( - 'AlwaysZero', - 'Constant', - 'PerPeriod', - 'PerSubperiod' - ) - ), - -- Identifies whether the resource (0) cannot be explicitly stored (1) can be stored between subperiods within each period (but not between periods) (2) can be stored between periods (but is simplified intraperiod) (3) can be stored both between periods and intraperiod - storage_type TEXT NOT NULL DEFAULT 'NoStorage' - CHECK( - storage_type IN ( - 'NoStorage', - 'Intraperiod', - 'Interperiod', - 'Both' - ) - ), - -- Reference per-period cost in $/Unit, usually overwritten by the referenced ResourceData object - ref_cost REAL NOT NULL DEFAULT 0.0, - -- Reference per-period availability in Units, usually overwritten by the referenced ResourceData object - ref_availability REAL NOT NULL DEFAULT 0.0 -); - -CREATE TABLE _Resource_ref_cost_vector ( - -- Reference per-period cost in $/Unit, usually overwritten by the referenced ResourceData object - resource_period INTEGER NOT NULL, - resource_id TEXT NOT NULL, - ref_cost REAL NOT NULL, - FOREIGN KEY (resource_id) REFERENCES Resource (id), - PRIMARY KEY (resource_period, resource_id) -); - -CREATE TABLE _Resource_ref_availability_vector ( - -- Reference per-period availability in Units, usually overwritten by the referenced ResourceData object - resource_period INTEGER, - resource_id TEXT NOT NULL, - ref_availability REAL NOT NULL, - FOREIGN KEY (resource_id) REFERENCES Resource (id), - PRIMARY KEY (resource_period, resource_id) -); - --- Conversion Curve -CREATE TABLE ConversionCurve ( - id TEXT PRIMARY KEY, - -- General descriptor text (optional) - description TEXT, - -- unit - unit TEXT NOT NULL, - -- vertical_axis_unit_type - vertical_axis_unit_type TEXT NOT NULL DEFAULT 'AlwaysZero' - CHECK( - vertical_axis_unit_type IN ( - 'UnitsPerHour', - 'AggregateBigUnits', - 'PerUnit', - 'AlwaysZero' - ) - ), - -- horizontal_axis_validation_type - horizontal_axis_validation_type TEXT NOT NULL DEFAULT 'AlwaysZero' - CHECK( - horizontal_axis_validation_type IN ( - 'UnitsPerHour', - 'AggregateBigUnits', - 'PerUnit', - 'AlwaysZero' - ) - ) -); - -CREATE TABLE _ConversionCurve_max_capacity_fractions ( - id TEXT, - idx INTEGER NOT NULL, - -- Fraction of the asset's maximum capacity corresponding to each Segment - max_capacity_fractions REAL NOT NULL, - - FOREIGN KEY (id) REFERENCES ConversionCurve(id) ON DELETE CASCADE, - PRIMARY KEY (id, idx) -); - -CREATE TABLE _ConversionCurve_conversion_efficiencies ( - id TEXT, - idx INTEGER NOT NULL, - -- Resource conversion factor in MWh/Resource.Unit, varying per Segment - conversion_efficiencies REAL NOT NULL, - - FOREIGN KEY (id) REFERENCES ConversionCurve(id) ON DELETE CASCADE, - PRIMARY KEY (id, idx) -); - --- Benefit Curve -CREATE TABLE BenefitCurve ( - id TEXT PRIMARY KEY, - -- General descriptor text (optional) - description TEXT, - -- vertical_axis_unit_type - vertical_axis_unit_type TEXT NOT NULL DEFAULT 'AlwaysZero' - CHECK( - vertical_axis_unit_type IN ( - 'UnitsPerHour', - 'AggregateBigUnits', - 'PerUnit', - 'AlwaysZero' - ) - ), - -- horizontal_axis_validation_type - horizontal_axis_validation_type TEXT NOT NULL DEFAULT 'AlwaysZero' - CHECK( - horizontal_axis_validation_type IN ( - 'UnitsPerHour', - 'AggregateBigUnits', - 'PerUnit', - 'AlwaysZero' - ) - ), - -- zero_position_type TODO (Bodin: inventei isso aqui. Certamente esses não são os enums) - zero_position_type TEXT NOT NULL DEFAULT 'ZeroAtOrigin' - CHECK( - zero_position_type IN ( - 'ZeroAtOrigin', - 'ZeroAtEnd' - ) - ) -); - -CREATE TABLE _BenefitCurve_resource_av_fractions ( - benefit_id TEXT PRIMARY KEY, - segment INTEGER NOT NULL, - -- Fraction of the asset's maximum capacity corresponding to each Segment - resource_av_fractions REAL NOT NULL, - FOREIGN KEY (benefit_id) REFERENCES BenefitCurve(id) -); - -CREATE TABLE _BenefitCurve_consumption_preferences ( - benefit_id TEXT PRIMARY KEY, - segment INTEGER NOT NULL, - -- Benefit factor in $/MWh, varying per Segment - consumption_preferences REAL NOT NULL, - FOREIGN KEY (benefit_id) REFERENCES BenefitCurve(id) -); - --- Power Assets -CREATE TABLE PowerAsset ( - id TEXT PRIMARY KEY, - -- General descriptor text on asset physical features (optional) - description TEXT, - -- General descriptor text on asset model representation (optional) - representation_notes TEXT, - -- Label for grouping together similar assets (optional) - grouping_label TEXT, - -- Facilitates SDDP correspondence: (0) no correspondence (1) standard demand ("inelastic" - see options 8 and 9) (2) standard thermal ("unlimited availability" - see option 7) (3) renewable (4) hydro (5) battery (6) power injection (7) non-standard thermal (has fuel contract and/or gas network representation) (8) elastic demand (9) flexible demand (10) electrification demand (11) Csp - -- TODO Bodin, acho que podemos tirar esses aux da frente de alguns nomes. - aux_asset_type TEXT NOT NULL DEFAULT 'NoCorrespondence' - CHECK( - aux_asset_type IN ( - 'NoCorrespondence', - 'StandardDemand', - 'StandardThermal', - 'Renewable', - 'Hydro', - 'Battery', - 'PowerInjection', - 'NonstandardThermal', - 'ElasticDemand', - 'FlexibleDemand', - 'ElectrificationDemand', - 'Csp' - ) - ), - -- Indicates whether the asset (0) is generation-like (contribution >0), (1) is demand-like (contribution <0), (2) is neither and can have either positive or negative contribution - output_sign TEXT NOT NULL DEFAULT 'GenerationLike' - CHECK( - output_sign IN ( - 'GenerationLike', - 'DemandLike', - 'Either' - ) - ), - -- Indicates whether the asset (0) has no network connection (ignore in a "network-representation" run) (1) has a single-bus connection (2) has a multi-bus fixed proportion connection - bus_connection_type TEXT NOT NULL DEFAULT 'NoConnection' - CHECK( - bus_connection_type IN ( - 'NoConnection', - 'SingleBus', - 'MultiBus' - ) - ), - -- Combines deprecated ConversionType and BenefitType - curve_type TEXT, -- TODO Bodin: aqui eu me perdi um pouco, não soube dizer o que isso significa - -- Indicates whether the asset (0) is always on (1) is always off (2) uses a linearized commitment variable representation (3) uses a binary commitment variable representation (4) uses fixed commitment data read from an external data source - commitment_type TEXT NOT NULL DEFAULT 'AlwaysOn' - CHECK( - commitment_type IN ( - 'AlwaysOn', - 'AlwaysOff', - 'Linearized', - 'Binary', - 'External' - ) - ), - -- Maximum capacity in MW, in absolute terms (for electricity injections or withdrawals) - capacity REAL NOT NULL DEFAULT 0.0, - -- Derating factor for reducing available capacity (dimensionless) - capacity_derating REAL NOT NULL DEFAULT 1.0, - -- Base direct O&M cost in $/MWh - output_cost REAL NOT NULL DEFAULT 0.0, - -- Base resource conversion factor in MWh/Resource.Unit - conversion_factor REAL NOT NULL DEFAULT 0.0, - -- Multiplier factor for the availability of the resource (dimensionless) - resource_av_multiplier REAL NOT NULL DEFAULT 1.0, - -- Multiplier factor for the cost of the resource (dimensionless) - resource_cost_multiplier REAL NOT NULL DEFAULT 1.0, - -- Additive factor to the cost of the resource, in $/Resource.Unit - resource_cost_adder REAL NOT NULL DEFAULT 0.0, - - resource_id TEXT, - conversion_id TEXT, - benefit_curve_id TEXT, - -- TODO Bodin comment: All foreign keys must be in the end of the table definition - FOREIGN KEY(resource_id) REFERENCES Resource(id), - FOREIGN KEY(conversion_id) REFERENCES ConversionCurve(id), - FOREIGN KEY(benefit_curve_id) REFERENCES BenefitCurve(id) -); \ No newline at end of file From 844890899ac7b5cf6e98e9968c940b0b29054a1c Mon Sep 17 00:00:00 2001 From: pedroripper Date: Tue, 28 Nov 2023 18:15:39 -0300 Subject: [PATCH 04/30] Update tests and handle bin files for SQL databases --- src/OpenSQL/create.jl | 32 ++++ src/OpenSQL/read.jl | 21 --- src/OpenSQL/utils.jl | 57 ++++-- src/sql_interface.jl | 48 ++++- test/OpenSQL/create_case.jl | 199 +++++---------------- test/OpenSQL/data/case_1/toy_schema.sql | 32 ++++ test/OpenSQL/data/case_2/simple_schema.sql | 13 ++ test/OpenSQL/time_series.jl | 91 ++++++++++ test/runtests.jl | 1 + 9 files changed, 302 insertions(+), 192 deletions(-) create mode 100644 test/OpenSQL/data/case_1/toy_schema.sql create mode 100644 test/OpenSQL/data/case_2/simple_schema.sql create mode 100644 test/OpenSQL/time_series.jl diff --git a/src/OpenSQL/create.jl b/src/OpenSQL/create.jl index 9b0d4371..9a8a277c 100644 --- a/src/OpenSQL/create.jl +++ b/src/OpenSQL/create.jl @@ -66,3 +66,35 @@ function create_element!( return nothing end + +function create_related_time_series!( + db::SQLite.DB, + table::String; + kwargs..., +) + table_name = "_" * table * "_TimeSeries" + dict_time_series = Dict() + for (key, value) in kwargs + @assert isa(value, String) + # TODO we could validate if the path exists + dict_time_series[key] = [value] + end + df = DataFrame(dict_time_series) + SQLite.load!(df, db, table_name) + return nothing +end + +function set_related!( + db::SQLite.DB, + table1::String, + table2::String, + id_1::String, + id_2::String, +) + id_parameter_on_table_1 = lowercase(table2) * "_id" + SQLite.execute( + db, + "UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'", + ) + return nothing +end diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl index 5d4ebc43..23c03b5b 100644 --- a/src/OpenSQL/read.jl +++ b/src/OpenSQL/read.jl @@ -70,24 +70,3 @@ function read_vector( result = df[!, 1] return result end - -function has_relation( - db::SQLite.DB, - table_1::String, - table_2::String, - table_1_id::String, - table_2_id::String, -) - sanity_check(db, table_1, "id") - sanity_check(db, table_2, "id") - id_exist_in_table(db, table_1, table_1_id) - id_exist_in_table(db, table_2, table_2_id) - - id_parameter_on_table_1 = lowercase(table_2) * "_id" - - if read_parameter(db, table_1, id_parameter_on_table_1, table_1_id) == table_2_id - return true - else - return false - end -end diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index 53078a8f..02950ec8 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -31,21 +31,6 @@ function load_db(database_path::String) return db end -function set_related!( - db::SQLite.DB, - table1::String, - table2::String, - id_1::String, - id_2::String, -) - id_parameter_on_table_1 = lowercase(table2) * "_id" - SQLite.execute( - db, - "UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'", - ) - return nothing -end - function column_names(db::SQLite.DB, table::String) cols = SQLite.columns(db, table) |> DataFrame return cols.name @@ -114,4 +99,46 @@ function is_vector_parameter(db::SQLite.DB, table::String, column::String) return table_exist_in_db(db, "_" * table * "_" * column) end +function has_relation( + db::SQLite.DB, + table_1::String, + table_2::String, + table_1_id::String, + table_2_id::String, +) + sanity_check(db, table_1, "id") + sanity_check(db, table_2, "id") + id_exist_in_table(db, table_1, table_1_id) + id_exist_in_table(db, table_2, table_2_id) + + id_parameter_on_table_1 = lowercase(table_2) * "_id" + + if read_parameter(db, table_1, id_parameter_on_table_1, table_1_id) == table_2_id + return true + else + return false + end +end + +function has_time_series(db::SQLite.DB, table::String) + time_series_table = _time_series_table_name(table) + return table_exist_in_db(db, time_series_table) +end + +function has_time_series(db::SQLite.DB, table::String, column::String) + sanity_check(db, table, "id") + time_series_table = _time_series_table_name(table) + if table_exist_in_db(db, time_series_table) + if column in column_names(db, time_series_table) + return true + else + return false + end + else + return false + end +end + +_time_series_table_name(table::String) = "_" * table * "_TimeSeries" + close(db::SQLite.DB) = DBInterface.close!(db) diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 571dd20f..59f1e03e 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -38,10 +38,16 @@ function get_attributes(db::OpenSQL.DB, collection::String) tables = OpenSQL.table_names(db) vector_attributes = Vector{String}() for table in tables - if startswith(table, "_" * collection * "_") + if startswith(table, "_" * collection * "_") && !endswith(table, "_TimeSeries") push!(vector_attributes, split(table, collection * "_")[end]) end end + if OpenSQL.has_time_series(db, collection) + time_series_table = "_" * collection * "_TimeSeries" + time_series_attributes = OpenSQL.column_names(db, time_series_table) + deleteat!(time_series_attributes, findfirst(x -> x == "id", time_series_attributes)) + return vcat(columns, vector_attributes, time_series_attributes) + end return vcat(columns, vector_attributes) end @@ -85,3 +91,43 @@ delete_relation!( source_id::String, target_id::String, ) = OpenSQL.delete_relation!(db, source, target, source_id, target_id) + +# Graf files +has_graf_file(db::OpenSQL.DB, collection::String, attribute::String) = + OpenSQL.has_time_series(db, collection, attribute) + +function link_series_to_file( + db::OpenSQL.DB, + collection::String, + attribute::String, + file_path::String, +) + if !OpenSQL.has_time_series(db, collection, attribute) + error("Collection $collection does not have a graf file for attribute $attribute.") + end + time_series_table = OpenSQL._time_series_table_name(collection) + + if max_elements(db, time_series_table) == 0 + OpenSQL.create_element!(db, time_series_table; id = collection * "_timeseries") + end + OpenSQL.update!(db, time_series_table, attribute, collection * "_timeseries", file_path) + return nothing +end + +function mapped_vector(db::OpenSQL.DB, collection::String, attribute::String) + if !has_graf_file(db, collection, attribute) + error("Collection $collection does not have a graf file for attribute $attribute.") + end + time_series_table = OpenSQL._time_series_table_name(collection) + + graf_file = get_parms(db, time_series_table, attribute)[1] + agents = get_parms(db, collection, "id") + + ior = OpenBinary.PSRI.open( + OpenBinary.Reader, + graf_file; + header = agents, + ) + + return ior +end diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 7b128cc3..0fd1765a 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -7,207 +7,96 @@ function create_case_1() db = PSRI.create_study( PSRI.SQLInterface(); data_path = case_path, - schema = "current_schema", - study_collection = "StudyParameters", + schema = "toy_schema", + study_collection = "Study", id = "Toy Case", - n_periods = 3, - n_subperiods = 2, - subperiod_duration = 24.0, + value1 = 1.0, ) - @test typeof(db) == PSRI.OpenSQL.DB + @test PSRI.get_parm(db, "Study", "id", "Toy Case") == "Toy Case" + @test PSRI.get_parm(db, "Study", "value1", "Toy Case") == 1.0 + @test PSRI.get_parm(db, "Study", "enum1", "Toy Case") == "A" PSRI.create_element!( db, - "Resource"; - id = "R1", - ref_availability = 100.0, - subperiod_av_type = "PerPeriod", - subperiod_cost_type = "PerPeriod", - ref_cost = 1.0, - ) - - PSRI.create_element!( - db, - "Resource"; - id = "R2", - ref_availability = 20.0, - subperiod_av_type = "PerSubperiodConstant", - subperiod_cost_type = "PerPeriod", - ref_cost = 1.0, - ) - - PSRI.create_element!( - db, - "Resource"; - id = "R3", - ref_availability = 100.0, - subperiod_av_type = "PerPeriod", - subperiod_cost_type = "PerPeriod", - ref_cost = 1.0, - ) - - PSRI.create_element!( - db, - "PowerAsset"; - id = "Generator 1", + "Plant"; + id = "Plant 1", capacity = 50.0, - output_cost = 10.0, - resource_cost_multiplier = 10.0, - commitment_type = "Linearized", ) - @test PSRI.get_parm(db, "PowerAsset", "capacity", "Generator 1") == 50.0 - PSRI.set_parm!(db, "PowerAsset", "capacity", "Generator 1", 400.0) - @test PSRI.get_parm(db, "PowerAsset", "capacity", "Generator 1") == 400.0 - PSRI.create_element!( db, - "PowerAsset"; - id = "Generator 2", - capacity = 100.0, - output_cost = 12.0, - resource_cost_multiplier = 12.0, - commitment_type = "Linearized", + "Plant"; + id = "Plant 2", ) - PSRI.create_element!( - db, - "PowerAsset"; - id = "Generator 3", - capacity = 100.0, - output_cost = 15.0, - resource_cost_multiplier = 15.0, - commitment_type = "Linearized", - ) + @test PSRI.get_parm(db, "Plant", "id", "Plant 1") == "Plant 1" + @test PSRI.get_parm(db, "Plant", "capacity", "Plant 1") == 50.0 + @test PSRI.get_parm(db, "Plant", "id", "Plant 2") == "Plant 2" + @test PSRI.get_parm(db, "Plant", "capacity", "Plant 2") == 0.0 PSRI.create_element!( db, - "PowerAsset"; - id = "Generator 4", - output_sign = "DemandLike", - curve_type = "Forced", - commitment_type = "AlwaysOn", - capacity = 100.0, - ) - - PSRI.create_element!( - db, - "ConversionCurve"; - id = "Conversion curve 1", - unit = "MW", - max_capacity_fractions = [0.1, 0.2, 0.3, 0.4], - conversion_efficiencies = [1.0, 2.0], - ) - - PSRI.create_element!( - db, - "ConversionCurve"; - id = "Conversion curve 2", - unit = "MW", - max_capacity_fractions = [0.5, 0.3, 0.2, 0.1], - conversion_efficiencies = [1.0, 2.0, 4.0], + "Resource"; + id = "R1", + type = "E", + some_values = [1.0, 2.0, 3.0], ) PSRI.create_element!( db, - "ConversionCurve"; - id = "Conversion curve 3", - unit = "MW", + "Resource"; + id = "R2", + type = "F", + some_values = [4.0, 5.0, 6.0], ) @test PSRI.get_vector( db, - "ConversionCurve", - "conversion_efficiencies", - "Conversion curve 2", - ) == - [1.0, 2.0, 4.0] - - PSRI.set_related!( - db, - "PowerAsset", "Resource", - "Generator 1", + "some_values", "R1", - ) - PSRI.set_related!( + ) == [1.0, 2.0, 3.0] + + @test PSRI.get_vector( db, - "PowerAsset", "Resource", - "Generator 2", + "some_values", "R2", - ) - PSRI.set_related!( - db, - "PowerAsset", - "Resource", - "Generator 3", - "R3", - ) - - @test PSRI.max_elements(db, "PowerAsset") == 4 - @test PSRI.max_elements(db, "Resource") == 3 - @test PSRI.max_elements(db, "ConversionCurve") == 3 - - PSRI.OpenSQL.close(db) - - db = PSRI.OpenSQL.load_db(joinpath(case_path, "psrclasses.sqlite")) - - @test PSRI.get_parm(db, "StudyParameters", "id", "Toy Case") == "Toy Case" - @test PSRI.get_parm(db, "StudyParameters", "n_periods", "Toy Case") == 3 - @test PSRI.get_parm(db, "StudyParameters", "n_subperiods", "Toy Case") == 2 - @test PSRI.get_parm(db, "StudyParameters", "subperiod_duration", "Toy Case") == 24.0 - - @test PSRI.get_parms(db, "Resource", "id") == ["R1", "R2", "R3"] - @test PSRI.get_parms(db, "Resource", "ref_availability") == [100.0, 20.0, 100.0] - @test PSRI.get_parms(db, "Resource", "subperiod_av_type") == - ["PerPeriod", "PerSubperiodConstant", "PerPeriod"] - @test PSRI.get_parms(db, "Resource", "subperiod_cost_type") == - ["PerPeriod", "PerPeriod", "PerPeriod"] - @test PSRI.get_parms(db, "Resource", "ref_cost") == [1.0, 1.0, 1.0] + ) == [4.0, 5.0, 6.0] PSRI.set_related!( db, - "PowerAsset", + "Plant", "Resource", - "Generator 1", + "Plant 1", "R1", ) - PSRI.set_related!( db, - "PowerAsset", + "Plant", "Resource", - "Generator 2", + "Plant 2", "R2", ) - PSRI.set_related!( - db, - "PowerAsset", - "Resource", - "Generator 3", - "R3", - ) + @test PSRI.max_elements(db, "Plant") == 2 + @test PSRI.max_elements(db, "Resource") == 2 - @test PSRI.get_parm(db, "PowerAsset", "resource_id", "Generator 1") == "R1" + PSRI.OpenSQL.close(db) - @test PSRI.max_elements(db, "PowerAsset") == 4 - @test PSRI.max_elements(db, "Resource") == 3 + db = PSRI.OpenSQL.load_db(joinpath(case_path, "psrclasses.sqlite")) - PSRI.delete_element!(db, "PowerAsset", "Generator 4") - PSRI.delete_element!(db, "Resource", "R3") + PSRI.delete_element!(db, "Plant", "Plant 1") + PSRI.delete_element!(db, "Resource", "R1") - @test PSRI.max_elements(db, "PowerAsset") == 3 - @test PSRI.max_elements(db, "Resource") == 2 + @test PSRI.max_elements(db, "Plant") == 1 + @test PSRI.max_elements(db, "Resource") == 1 + + @test PSRI.get_attributes(db, "Resource") == ["id", "type", "some_values"] - @test PSRI.get_attributes(db, "Resource") == - ["id", "description", "grouping_label", "unit", "big_unit", - "aux_resource_type", "shared_type", "subperiod_av_type", - "av_unit_type", "subperiod_cost_type", "storage_type", - "ref_cost", "ref_availability", "ref_cost_vector", - "ref_availability_vector"] + @test PSRI.get_attributes(db, "Plant") == + ["id", "capacity", "resource_id", "generation_file"] PSRI.OpenSQL.close(db) diff --git a/test/OpenSQL/data/case_1/toy_schema.sql b/test/OpenSQL/data/case_1/toy_schema.sql new file mode 100644 index 00000000..197dc362 --- /dev/null +++ b/test/OpenSQL/data/case_1/toy_schema.sql @@ -0,0 +1,32 @@ +CREATE TABLE Study ( + id TEXT PRIMARY KEY, + value1 REAL NOT NULL DEFAULT 100, + enum1 TEXT NOT NULL DEFAULT 'A' CHECK(enum1 IN ('A', 'B', 'C')) +); + + +CREATE TABLE Resource ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL DEFAULT "D" CHECK(type IN ('D', 'E', 'F')) +); + +CREATE TABLE _Resource_some_values ( + id TEXT, + idx INTEGER NOT NULL, + some_values REAL NOT NULL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE, + PRIMARY KEY (id, idx) +); + + +CREATE TABLE Plant ( + id TEXT PRIMARY KEY, + capacity REAL NOT NULL DEFAULT 0, + resource_id TEXT, + FOREIGN KEY(resource_id) REFERENCES Resource(id) +); + +CREATE TABLE _Plant_TimeSeries ( + id TEXT PRIMARY KEY, + generation_file TEXT +); \ No newline at end of file diff --git a/test/OpenSQL/data/case_2/simple_schema.sql b/test/OpenSQL/data/case_2/simple_schema.sql new file mode 100644 index 00000000..26985ab5 --- /dev/null +++ b/test/OpenSQL/data/case_2/simple_schema.sql @@ -0,0 +1,13 @@ +CREATE TABLE Study ( + id TEXT PRIMARY KEY +); + +CREATE TABLE Plant ( + id TEXT PRIMARY KEY +); + +CREATE TABLE _Plant_TimeSeries ( + id TEXT PRIMARY KEY, + generation_file TEXT, + cost_file TEXT +); \ No newline at end of file diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl new file mode 100644 index 00000000..309315e1 --- /dev/null +++ b/test/OpenSQL/time_series.jl @@ -0,0 +1,91 @@ +function test_time_series() + case_path = joinpath(@__DIR__, "data", "case_2") + if isfile(joinpath(case_path, "psrclasses.sqlite")) + rm(joinpath(case_path, "psrclasses.sqlite")) + end + + db = PSRI.create_study( + PSRI.SQLInterface(); + data_path = case_path, + schema = "simple_schema", + study_collection = "Study", + id = "Toy Case", + ) + + PSRI.create_element!( + db, + "Plant"; + id = "Plant 1", + ) + + PSRI.create_element!( + db, + "Plant"; + id = "Plant 2", + ) + + iow = PSRI.open( + PSRI.OpenBinary.Writer, + joinpath(case_path, "generation"); + blocks = 3, + scenarios = 2, + stages = 12, + agents = ["Plant 1", "Plant 2"], + unit = "MW", + ) + + for t in 1:12, s in 1:2, b in 1:3 + PSRI.write_registry(iow, [(t + s + b) * 100.0, (t + s + b) * 300.0], t, s, b) + end + + PSRI.close(iow) + + PSRI.link_series_to_file( + db, + "Plant", + "generation_file", + joinpath(case_path, "generation"), + ) + + ior = PSRI.mapped_vector(db, "Plant", "generation_file") + + for t in 1:12, s in 1:2, b in 1:3 + @test ior.data == [(t + s + b) * 100.0, (t + s + b) * 300.0] + PSRI.next_registry(ior) + end + + PSRI.close(ior) + + iow = PSRI.open( + PSRI.OpenBinary.Writer, + joinpath(case_path, "cost"); + blocks = 3, + scenarios = 2, + stages = 12, + agents = ["Plant 1", "Plant 2"], + unit = "USD", + ) + + for t in 1:12, s in 1:2, b in 1:3 + PSRI.write_registry(iow, [(t + s + b) * 500.0, (t + s + b) * 400.0], t, s, b) + end + + PSRI.close(iow) + + PSRI.link_series_to_file(db, "Plant", "cost_file", joinpath(case_path, "cost")) + + ior = PSRI.mapped_vector(db, "Plant", "cost_file") + + for t in 1:12, s in 1:2, b in 1:3 + @test ior.data == [(t + s + b) * 500.0, (t + s + b) * 400.0] + PSRI.next_registry(ior) + end + + PSRI.close(ior) + + PSRI.OpenSQL.close(db) + + return rm(joinpath(case_path, "psrclasses.sqlite")) +end + +test_time_series() diff --git a/test/runtests.jl b/test/runtests.jl index 1897842d..72ea43cf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -60,5 +60,6 @@ end end @testset "OpenSQL" begin @time include("OpenSQL/create_case.jl") + @time include("OpenSQL/time_series.jl") end end From 8003e92dbb8bc9125b66438af79e46bf95c37d7b Mon Sep 17 00:00:00 2001 From: pedroripper Date: Tue, 28 Nov 2023 18:55:45 -0300 Subject: [PATCH 05/30] Update tests --- src/OpenSQL/delete.jl | 6 +++--- src/OpenSQL/utils.jl | 2 +- test/OpenSQL/create_case.jl | 30 +++++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/OpenSQL/delete.jl b/src/OpenSQL/delete.jl index 8e3823f9..2011acc8 100644 --- a/src/OpenSQL/delete.jl +++ b/src/OpenSQL/delete.jl @@ -19,17 +19,17 @@ function delete_relation!( table_1_id::String, table_2_id::String, ) - if !has_relation(db, table_1, table_2, table_1_id, table_2_id) + if !are_related(db, table_1, table_2, table_1_id, table_2_id) error( "Element with id $table_1_id from table $table_1 is not related to element with id $table_2_id from table $table_2.", ) end - id_parameter_on_table_1 = lowercase(table2) * "_id" + id_parameter_on_table_1 = lowercase(table_2) * "_id" DBInterface.execute( db, - "UPDATE $table1 SET $id_parameter_on_table_1 = '' WHERE id = '$id_1'", + "UPDATE $table_1 SET $id_parameter_on_table_1 = '' WHERE id = '$table_1_id'", ) return nothing diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index 02950ec8..71c35b3d 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -99,7 +99,7 @@ function is_vector_parameter(db::SQLite.DB, table::String, column::String) return table_exist_in_db(db, "_" * table * "_" * column) end -function has_relation( +function are_related( db::SQLite.DB, table_1::String, table_2::String, diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 0fd1765a..7500d213 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -35,6 +35,16 @@ function create_case_1() @test PSRI.get_parm(db, "Plant", "id", "Plant 2") == "Plant 2" @test PSRI.get_parm(db, "Plant", "capacity", "Plant 2") == 0.0 + PSRI.set_parm!( + db, + "Plant", + "capacity", + "Plant 2", + 100.0, + ) + + @test PSRI.get_parm(db, "Plant", "capacity", "Plant 2") == 100.0 + PSRI.create_element!( db, "Resource"; @@ -65,6 +75,12 @@ function create_case_1() "R2", ) == [4.0, 5.0, 6.0] + @test PSRI.get_vectors( + db, + "Resource", + "some_values", + ) == [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]] + PSRI.set_related!( db, "Plant", @@ -80,12 +96,24 @@ function create_case_1() "R2", ) + @test PSRI.get_parm(db, "Plant", "resource_id", "Plant 1") == "R1" + + PSRI.delete_relation!( + db, + "Plant", + "Resource", + "Plant 1", + "R1", + ) + + @test PSRI.get_parm(db, "Plant", "resource_id", "Plant 1") == "" + @test PSRI.max_elements(db, "Plant") == 2 @test PSRI.max_elements(db, "Resource") == 2 PSRI.OpenSQL.close(db) - db = PSRI.OpenSQL.load_db(joinpath(case_path, "psrclasses.sqlite")) + db = PSRI.load_study(PSRI.SQLInterface(); data_path = joinpath(case_path, "psrclasses.sqlite")) PSRI.delete_element!(db, "Plant", "Plant 1") PSRI.delete_element!(db, "Resource", "R1") From e465c800b3ea0fa3688ca5201c621f96289004f3 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Tue, 28 Nov 2023 18:58:37 -0300 Subject: [PATCH 06/30] Format --- test/OpenSQL/create_case.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 7500d213..0fc014bc 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -113,7 +113,10 @@ function create_case_1() PSRI.OpenSQL.close(db) - db = PSRI.load_study(PSRI.SQLInterface(); data_path = joinpath(case_path, "psrclasses.sqlite")) + db = PSRI.load_study( + PSRI.SQLInterface(); + data_path = joinpath(case_path, "psrclasses.sqlite"), + ) PSRI.delete_element!(db, "Plant", "Plant 1") PSRI.delete_element!(db, "Resource", "R1") From 1359a00fc28d5aba8d05a928a3daafe1ce75c75a Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 29 Nov 2023 11:12:04 -0300 Subject: [PATCH 07/30] Update tests for Time Series --- src/OpenSQL/update.jl | 11 +++ src/sql_interface.jl | 15 +++- test/OpenSQL/data/case_2/simple_schema.sql | 1 - test/OpenSQL/time_series.jl | 90 ++++++++++++++++++++++ 4 files changed, 113 insertions(+), 4 deletions(-) diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index 61b70b74..2f2826a0 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -10,6 +10,17 @@ function update!( return nothing end +function update!( + db::SQLite.DB, + table::String, + column::String, + val, +) + sanity_check(db, table, column) + DBInterface.execute(db, "UPDATE $table SET $column = '$val'") + return nothing +end + function update!( db::SQLite.DB, table::String, diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 59f1e03e..0a8c9fec 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -107,13 +107,22 @@ function link_series_to_file( end time_series_table = OpenSQL._time_series_table_name(collection) - if max_elements(db, time_series_table) == 0 - OpenSQL.create_element!(db, time_series_table; id = collection * "_timeseries") + if length(get_parms(db, time_series_table, attribute)) == 0 + OpenSQL.create_parameters!(db, time_series_table, Dict(attribute => file_path)) + else + OpenSQL.update!(db, time_series_table, attribute, file_path) end - OpenSQL.update!(db, time_series_table, attribute, collection * "_timeseries", file_path) return nothing end +function link_series_to_files( + db::OpenSQL.DB, + collection::String; + kwargs..., +) + return OpenSQL.create_related_time_series!(db, collection; kwargs...) +end + function mapped_vector(db::OpenSQL.DB, collection::String, attribute::String) if !has_graf_file(db, collection, attribute) error("Collection $collection does not have a graf file for attribute $attribute.") diff --git a/test/OpenSQL/data/case_2/simple_schema.sql b/test/OpenSQL/data/case_2/simple_schema.sql index 26985ab5..b3aa4c73 100644 --- a/test/OpenSQL/data/case_2/simple_schema.sql +++ b/test/OpenSQL/data/case_2/simple_schema.sql @@ -7,7 +7,6 @@ CREATE TABLE Plant ( ); CREATE TABLE _Plant_TimeSeries ( - id TEXT PRIMARY KEY, generation_file TEXT, cost_file TEXT ); \ No newline at end of file diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index 309315e1..93352fd1 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -88,4 +88,94 @@ function test_time_series() return rm(joinpath(case_path, "psrclasses.sqlite")) end +function test_time_series_2() + case_path = joinpath(@__DIR__, "data", "case_2") + if isfile(joinpath(case_path, "psrclasses.sqlite")) + rm(joinpath(case_path, "psrclasses.sqlite")) + end + + db = PSRI.create_study( + PSRI.SQLInterface(); + data_path = case_path, + schema = "simple_schema", + study_collection = "Study", + id = "Toy Case", + ) + + PSRI.create_element!( + db, + "Plant"; + id = "Plant 1", + ) + + PSRI.create_element!( + db, + "Plant"; + id = "Plant 2", + ) + + iow = PSRI.open( + PSRI.OpenBinary.Writer, + joinpath(case_path, "generation"); + blocks = 3, + scenarios = 2, + stages = 12, + agents = ["Plant 1", "Plant 2"], + unit = "MW", + ) + + for t in 1:12, s in 1:2, b in 1:3 + PSRI.write_registry(iow, [(t + s + b) * 100.0, (t + s + b) * 300.0], t, s, b) + end + + PSRI.close(iow) + + + iow = PSRI.open( + PSRI.OpenBinary.Writer, + joinpath(case_path, "cost"); + blocks = 3, + scenarios = 2, + stages = 12, + agents = ["Plant 1", "Plant 2"], + unit = "USD", + ) + + for t in 1:12, s in 1:2, b in 1:3 + PSRI.write_registry(iow, [(t + s + b) * 500.0, (t + s + b) * 400.0], t, s, b) + end + + PSRI.close(iow) + + PSRI.link_series_to_files( + db, + "Plant"; + generation_file = joinpath(case_path, "generation"), + cost_file = joinpath(case_path, "cost"), + ) + + ior = PSRI.mapped_vector(db, "Plant", "generation_file") + + for t in 1:12, s in 1:2, b in 1:3 + @test ior.data == [(t + s + b) * 100.0, (t + s + b) * 300.0] + PSRI.next_registry(ior) + end + + PSRI.close(ior) + + ior = PSRI.mapped_vector(db, "Plant", "cost_file") + + for t in 1:12, s in 1:2, b in 1:3 + @test ior.data == [(t + s + b) * 500.0, (t + s + b) * 400.0] + PSRI.next_registry(ior) + end + + PSRI.close(ior) + + PSRI.OpenSQL.close(db) + + return rm(joinpath(case_path, "psrclasses.sqlite")) +end + test_time_series() +test_time_series_2() From 735aa5d0f8e63d144f6cb064621bbc81848c50ac Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 29 Nov 2023 11:20:14 -0300 Subject: [PATCH 08/30] Format and add type to function parameter --- src/OpenSQL/OpenSQL.jl | 2 ++ src/OpenSQL/update.jl | 8 ++++---- src/sql_interface.jl | 2 +- test/OpenSQL/time_series.jl | 1 - 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/OpenSQL/OpenSQL.jl b/src/OpenSQL/OpenSQL.jl index 5fe4c83c..91432672 100644 --- a/src/OpenSQL/OpenSQL.jl +++ b/src/OpenSQL/OpenSQL.jl @@ -8,6 +8,8 @@ using DataFrames const DB = SQLite.DB +const ValidOpenSQLDataType = Union{AbstractString, Integer, Real, Bool} + """ SQLInterface """ diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index 2f2826a0..154ded8a 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -3,8 +3,8 @@ function update!( table::String, column::String, id::String, - val, -) + val::T, +) where {T <: ValidOpenSQLDataType} sanity_check(db, table, column) DBInterface.execute(db, "UPDATE $table SET $column = '$val' WHERE id = '$id'") return nothing @@ -14,8 +14,8 @@ function update!( db::SQLite.DB, table::String, column::String, - val, -) + val::T, +) where {T <: ValidOpenSQLDataType} sanity_check(db, table, column) DBInterface.execute(db, "UPDATE $table SET $column = '$val'") return nothing diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 0a8c9fec..cd067059 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -109,7 +109,7 @@ function link_series_to_file( if length(get_parms(db, time_series_table, attribute)) == 0 OpenSQL.create_parameters!(db, time_series_table, Dict(attribute => file_path)) - else + else OpenSQL.update!(db, time_series_table, attribute, file_path) end return nothing diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index 93352fd1..703aade2 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -130,7 +130,6 @@ function test_time_series_2() PSRI.close(iow) - iow = PSRI.open( PSRI.OpenBinary.Writer, joinpath(case_path, "cost"); From 808fde68da5c30c798f7cb79d49090f0c3c0fcc8 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 29 Nov 2023 11:59:39 -0300 Subject: [PATCH 09/30] Fix --- src/OpenSQL/create.jl | 4 ++-- src/OpenSQL/read.jl | 3 +-- src/OpenSQL/update.jl | 10 +++++++++- src/OpenSQL/utils.jl | 3 ++- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/OpenSQL/create.jl b/src/OpenSQL/create.jl index 9a8a277c..4c393193 100644 --- a/src/OpenSQL/create.jl +++ b/src/OpenSQL/create.jl @@ -19,7 +19,7 @@ function create_vector!( vector_name::String, values::V, ) where {V <: AbstractVector} - table_name = "_" * table * "_" * vector_name + table_name = _vector_table_name(table, vector_name) sanity_check(db, table_name, vector_name) num_values = length(values) ids = fill(id, num_values) @@ -72,7 +72,7 @@ function create_related_time_series!( table::String; kwargs..., ) - table_name = "_" * table * "_TimeSeries" + table_name = _time_series_table_name(table) dict_time_series = Dict() for (key, value) in kwargs @assert isa(value, String) diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl index 23c03b5b..e50bf0ac 100644 --- a/src/OpenSQL/read.jl +++ b/src/OpenSQL/read.jl @@ -43,7 +43,6 @@ function read_vector( table::String, vector_name::String, ) - table_name = "_" * table * "_" * vector_name sanity_check(db, table_name, vector_name) ids_in_table = read_parameter(db, table, "id") @@ -61,7 +60,7 @@ function read_vector( vector_name::String, id::String, ) - table_name = "_" * table * "_" * vector_name + table_name = _vector_table_name(table, vector_name) sanity_check(db, table_name, vector_name) query = "SELECT $vector_name FROM $table_name WHERE id = '$id' ORDER BY idx" diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index 154ded8a..95d8014b 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -24,10 +24,18 @@ end function update!( db::SQLite.DB, table::String, - columns::String, + column::String, id::String, vals::V, ) where {V <: AbstractVector} + if !is_vector_parameter(db, table, column) + error("Column $column is not a vector parameter.") + end + + vector_table = _vector_table_name(table, column) + + DBInterface.execute(db, "UPDATE $vector_table SET $column = '$vals' WHERE id = '$id'") + error("Updating vectors is not yet implemented.") return nothing end diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index 71c35b3d..fb6ab473 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -96,7 +96,7 @@ function id_exist_in_table(db::SQLite.DB, table::String, id::String) end function is_vector_parameter(db::SQLite.DB, table::String, column::String) - return table_exist_in_db(db, "_" * table * "_" * column) + return table_exist_in_db(db, _vector_table_name(table, column)) end function are_related( @@ -140,5 +140,6 @@ function has_time_series(db::SQLite.DB, table::String, column::String) end _time_series_table_name(table::String) = "_" * table * "_TimeSeries" +_vector_table_name(table::String, column::String) = "_" * table * "_" * column close(db::SQLite.DB) = DBInterface.close!(db) From ab8f2640a05bca8fad7a3eca48153f4919681e95 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 29 Nov 2023 16:15:04 -0300 Subject: [PATCH 10/30] Update bin hdr interface --- src/OpenSQL/read.jl | 8 +++++ src/OpenSQL/update.jl | 15 ++++++++-- src/PSRClassesInterface.jl | 4 ++- src/sql_interface.jl | 60 ++++++++++++++++++++++++++++++++++--- test/OpenSQL/create_case.jl | 10 ++++++- test/OpenSQL/time_series.jl | 51 ++++++++++++++++++++++--------- 6 files changed, 125 insertions(+), 23 deletions(-) diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl index e50bf0ac..6346e0ac 100644 --- a/src/OpenSQL/read.jl +++ b/src/OpenSQL/read.jl @@ -43,6 +43,7 @@ function read_vector( table::String, vector_name::String, ) + table_name = _vector_table_name(table, vector_name) sanity_check(db, table_name, vector_name) ids_in_table = read_parameter(db, table, "id") @@ -69,3 +70,10 @@ function read_vector( result = df[!, 1] return result end + +function number_of_rows(db::SQLite.DB, table::String, column::String = "id") + sanity_check(db, table, column) + query = "SELECT COUNT($column) FROM $table" + df = DBInterface.execute(db, query) |> DataFrame + return df[!, 1][1] +end diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index 95d8014b..f4d48638 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -31,11 +31,20 @@ function update!( if !is_vector_parameter(db, table, column) error("Column $column is not a vector parameter.") end - + vector_table = _vector_table_name(table, column) - DBInterface.execute(db, "UPDATE $vector_table SET $column = '$vals' WHERE id = '$id'") + current_vector = read_vector(db, table, column, id) + current_length = length(current_vector) + + for idx in 1:current_length + DBInterface.execute( + db, + "DELETE FROM $vector_table WHERE id = '$id' AND idx = $idx", + ) + end + + create_vector!(db, table, id, column, vals) - error("Updating vectors is not yet implemented.") return nothing end diff --git a/src/PSRClassesInterface.jl b/src/PSRClassesInterface.jl index d22c632c..a3fd0fd0 100644 --- a/src/PSRClassesInterface.jl +++ b/src/PSRClassesInterface.jl @@ -22,7 +22,6 @@ const Attribute = PMD.Attribute const DataStruct = PMD.DataStruct include("OpenSQL/OpenSQL.jl") -include("sql_interface.jl") # simple and generic interface include("study_interface.jl") @@ -48,4 +47,7 @@ include("tables/interface.jl") # modification API include("modification_api.jl") +# SQL API +include("sql_interface.jl") + end diff --git a/src/sql_interface.jl b/src/sql_interface.jl index cd067059..e1ae209d 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -107,7 +107,7 @@ function link_series_to_file( end time_series_table = OpenSQL._time_series_table_name(collection) - if length(get_parms(db, time_series_table, attribute)) == 0 + if OpenSQL.number_of_rows(db, time_series_table, attribute) == 0 OpenSQL.create_parameters!(db, time_series_table, Dict(attribute => file_path)) else OpenSQL.update!(db, time_series_table, attribute, file_path) @@ -123,20 +123,72 @@ function link_series_to_files( return OpenSQL.create_related_time_series!(db, collection; kwargs...) end -function mapped_vector(db::OpenSQL.DB, collection::String, attribute::String) +function open( + ::Type{OpenBinary.Reader}, + db::OpenSQL.DB, + collection::String, + attribute::String; + kwargs..., +) if !has_graf_file(db, collection, attribute) error("Collection $collection does not have a graf file for attribute $attribute.") end time_series_table = OpenSQL._time_series_table_name(collection) - graf_file = get_parms(db, time_series_table, attribute)[1] + raw_files = get_parms(db, time_series_table, attribute) + + if OpenSQL.number_of_rows(db, time_series_table, attribute) == 0 + error("Collection $collection does not have a graf file for attribute $attribute.") + end + + graf_file = raw_files[findfirst(x -> !ismissing(x), raw_files)] + agents = get_parms(db, collection, "id") - ior = OpenBinary.PSRI.open( + ior = open( OpenBinary.Reader, graf_file; header = agents, + kwargs..., ) return ior end + +function open( + ::Type{OpenBinary.Writer}, + db::OpenSQL.DB, + collection::String, + attribute::String, + path::String; + kwargs..., +) + if !has_graf_file(db, collection, attribute) + error("Collection $collection does not have a graf file for attribute $attribute.") + end + time_series_table = OpenSQL._time_series_table_name(collection) + + graf_file = if OpenSQL.number_of_rows(db, time_series_table, attribute) == 0 + link_series_to_file(db, collection, attribute, path) + path + else + raw_files = get_parms(db, time_series_table, attribute) + if raw_files[1] != path + link_series_to_file(db, collection, attribute, path) + path + else + raw_files[1] + end + end + + agents = get_parms(db, collection, "id") + + iow = open( + OpenBinary.Writer, + graf_file; + agents = agents, + kwargs..., + ) + + return iow +end diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 0fc014bc..0c7fc275 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -75,11 +75,19 @@ function create_case_1() "R2", ) == [4.0, 5.0, 6.0] + PSRI.set_vector!( + db, + "Resource", + "some_values", + "R1", + [7.0, 8.0, 9.0], + ) + @test PSRI.get_vectors( db, "Resource", "some_values", - ) == [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]] + ) == [[7.0, 8.0, 9.0], [4.0, 5.0, 6.0]] PSRI.set_related!( db, diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index 703aade2..66136db6 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -26,11 +26,13 @@ function test_time_series() iow = PSRI.open( PSRI.OpenBinary.Writer, + db, + "Plant", + "generation_file", joinpath(case_path, "generation"); blocks = 3, scenarios = 2, stages = 12, - agents = ["Plant 1", "Plant 2"], unit = "MW", ) @@ -40,14 +42,7 @@ function test_time_series() PSRI.close(iow) - PSRI.link_series_to_file( - db, - "Plant", - "generation_file", - joinpath(case_path, "generation"), - ) - - ior = PSRI.mapped_vector(db, "Plant", "generation_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation_file") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 100.0, (t + s + b) * 300.0] @@ -58,6 +53,9 @@ function test_time_series() iow = PSRI.open( PSRI.OpenBinary.Writer, + db, + "Plant", + "cost_file", joinpath(case_path, "cost"); blocks = 3, scenarios = 2, @@ -72,9 +70,7 @@ function test_time_series() PSRI.close(iow) - PSRI.link_series_to_file(db, "Plant", "cost_file", joinpath(case_path, "cost")) - - ior = PSRI.mapped_vector(db, "Plant", "cost_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "cost_file") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 500.0, (t + s + b) * 400.0] @@ -83,6 +79,33 @@ function test_time_series() PSRI.close(ior) + iow = PSRI.open( + PSRI.OpenBinary.Writer, + db, + "Plant", + "generation_file", + joinpath(case_path, "generation_new"); + blocks = 3, + scenarios = 2, + stages = 12, + unit = "MW", + ) + + for t in 1:12, s in 1:2, b in 1:3 + PSRI.write_registry(iow, [(t + s + b) * 50.0, (t + s + b) * 20.0], t, s, b) + end + + PSRI.close(iow) + + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation_file") + + for t in 1:12, s in 1:2, b in 1:3 + @test ior.data == [(t + s + b) * 50.0, (t + s + b) * 20.0] + PSRI.next_registry(ior) + end + + PSRI.close(ior) + PSRI.OpenSQL.close(db) return rm(joinpath(case_path, "psrclasses.sqlite")) @@ -153,7 +176,7 @@ function test_time_series_2() cost_file = joinpath(case_path, "cost"), ) - ior = PSRI.mapped_vector(db, "Plant", "generation_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation_file") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 100.0, (t + s + b) * 300.0] @@ -162,7 +185,7 @@ function test_time_series_2() PSRI.close(ior) - ior = PSRI.mapped_vector(db, "Plant", "cost_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "cost_file") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 500.0, (t + s + b) * 400.0] From 39675fe23d6677e926ce4d5d94b271665e9cbec1 Mon Sep 17 00:00:00 2001 From: guilhermebodin Date: Tue, 5 Dec 2023 14:58:18 -0300 Subject: [PATCH 11/30] move some things around --- src/OpenSQL/OpenSQL.jl | 2 -- src/OpenSQL/create.jl | 34 +--------------------------------- src/OpenSQL/read.jl | 2 +- src/OpenSQL/update.jl | 39 +++++++++++++++++++++++++++++++++++---- src/sql_interface.jl | 2 +- 5 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/OpenSQL/OpenSQL.jl b/src/OpenSQL/OpenSQL.jl index 91432672..5fe4c83c 100644 --- a/src/OpenSQL/OpenSQL.jl +++ b/src/OpenSQL/OpenSQL.jl @@ -8,8 +8,6 @@ using DataFrames const DB = SQLite.DB -const ValidOpenSQLDataType = Union{AbstractString, Integer, Real, Bool} - """ SQLInterface """ diff --git a/src/OpenSQL/create.jl b/src/OpenSQL/create.jl index 4c393193..b4ad5390 100644 --- a/src/OpenSQL/create.jl +++ b/src/OpenSQL/create.jl @@ -65,36 +65,4 @@ function create_element!( create_vectors!(db, table, id, dict_vectors) return nothing -end - -function create_related_time_series!( - db::SQLite.DB, - table::String; - kwargs..., -) - table_name = _time_series_table_name(table) - dict_time_series = Dict() - for (key, value) in kwargs - @assert isa(value, String) - # TODO we could validate if the path exists - dict_time_series[key] = [value] - end - df = DataFrame(dict_time_series) - SQLite.load!(df, db, table_name) - return nothing -end - -function set_related!( - db::SQLite.DB, - table1::String, - table2::String, - id_1::String, - id_2::String, -) - id_parameter_on_table_1 = lowercase(table2) * "_id" - SQLite.execute( - db, - "UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'", - ) - return nothing -end +end \ No newline at end of file diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl index 6346e0ac..eb6a9b69 100644 --- a/src/OpenSQL/read.jl +++ b/src/OpenSQL/read.jl @@ -71,7 +71,7 @@ function read_vector( return result end -function number_of_rows(db::SQLite.DB, table::String, column::String = "id") +function number_of_rows(db::SQLite.DB, table::String, column::String) sanity_check(db, table, column) query = "SELECT COUNT($column) FROM $table" df = DBInterface.execute(db, query) |> DataFrame diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index f4d48638..b3fc6297 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -3,8 +3,8 @@ function update!( table::String, column::String, id::String, - val::T, -) where {T <: ValidOpenSQLDataType} + val, +) sanity_check(db, table, column) DBInterface.execute(db, "UPDATE $table SET $column = '$val' WHERE id = '$id'") return nothing @@ -14,8 +14,8 @@ function update!( db::SQLite.DB, table::String, column::String, - val::T, -) where {T <: ValidOpenSQLDataType} + val, +) sanity_check(db, table, column) DBInterface.execute(db, "UPDATE $table SET $column = '$val'") return nothing @@ -38,6 +38,8 @@ function update!( current_length = length(current_vector) for idx in 1:current_length + # TODO - Bodin deve ter uma forma melhor de fazer esse delete, acho que no final + # seria equivalente a deletar todos os ids DBInterface.execute( db, "DELETE FROM $vector_table WHERE id = '$id' AND idx = $idx", @@ -48,3 +50,32 @@ function update!( return nothing end + +function set_related!( + db::DBInterface.Connection, + table1::String, + table2::String, + id_1::String, + id_2::String +) + id_parameter_on_table_1 = lowercase(table2) * "_id" + SQLite.execute(db, "UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'") + return nothing +end + +function set_related_time_series!( + db::DBInterface.Connection, + table::String; + kwargs... +) + table_name = "_" * table * "_TimeSeries" + dict_time_series = Dict() + for (key, value) in kwargs + @assert isa(value, String) + # TODO we could validate if the path exists + dict_time_series[key] = [value] + end + df = DataFrame(dict_time_series) + SQLite.load!(df, db, table_name) + return nothing +end diff --git a/src/sql_interface.jl b/src/sql_interface.jl index e1ae209d..24e92c31 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -120,7 +120,7 @@ function link_series_to_files( collection::String; kwargs..., ) - return OpenSQL.create_related_time_series!(db, collection; kwargs...) + return OpenSQL.set_related_time_series!(db, collection; kwargs...) end function open( From a7dca190c77cfde391f622e833bf9e86cfb1c990 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Tue, 5 Dec 2023 17:38:44 -0300 Subject: [PATCH 12/30] Update schema [no ci] --- src/OpenSQL/README.md | 128 ++++++++++++++++++++++++ test/OpenSQL/data/case_1/toy_schema.sql | 30 ++++-- 2 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 src/OpenSQL/README.md diff --git a/src/OpenSQL/README.md b/src/OpenSQL/README.md new file mode 100644 index 00000000..6aea8709 --- /dev/null +++ b/src/OpenSQL/README.md @@ -0,0 +1,128 @@ +# OpenSQL + +Following PSRI's `OpenStudy` standards, SQL schemas for the `OpenSQL` framework should follow the conventions described in this document. + + +## SQL Schema Conventions + + +### Collections + +- The Table name should be the same as the name of the Collection. +- The Table name of a Collection should beging with a capital letter and be in singular form. +- In case of a Collection with a composite name, the Table name should be separeted by an underscore. +- The Table must contain a primary key named `id`. + +Examples: + + +```sql +CREATE TABLE Resource ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL DEFAULT "D" CHECK(type IN ('D', 'E', 'F')) +); + +CREATE TABLE Thermal_Plant( + id TEXT PRIMARY KEY, + capacity REAL NOT NULL DEFAULT 0 +); +``` + + +### Non-vector Attributes + +- The name of an Attribute should be in snake case and be in singular form. + +Example: +```sql +CREATE TABLE Thermal_Plant( + id TEXT PRIMARY KEY, + thermal_plant_name TEXT NOT NULL, + capacity REAL NOT NULL +); +``` + +### Vector Attributes + +- In case of a vector attribute, a Table should be created with its name indicating the name of the Collection and the name of the attribute, separated by `_vector_`, as presented below + +

COLLECTION_NAME_vector_ATTRIBUTE_NAME

+ +- Note that after **_vector_** the name of the attribute should follow the same rule as non-vector attributes. +- The Table must contain a Column named `id` and another named `idx`. +- There must be a Column named after the attribute name, which will store the value of the attribute for the specified element `id` and index `idx`. + +Example: +```sql +CREATE TABLE Thermal_Plant_vector_some_value( + id TEXT, + idx INTEGER NOT NULL, + some_value REAL NOT NULL, + FOREIGN KEY (id) REFERENCES Thermal_Plant(id) ON DELETE CASCADE, + PRIMARY KEY (id, idx) +); +``` + +### Time Series + +- All Time Series for the elements from a Collection should be stored in a Table +- The Table name should be the same as the name of the Collection followed by `_timeseries`, as presented below + +

COLLECTION_NAME_vector_ATTRIBUTE_NAME

+ +- The Table must contain a Column named `id`. +- Each Column of the table should be named after the name of the attribute. +- Each Column should store the path to the file containing the time series data. + +Example: + +```sql +CREATE TABLE Plant_timeseries ( + id TEXT PRIMARY KEY, + generation TEXT, + cost TEXT +); +``` + +### 1 to 1 Relations + +- One to One relations (1:1, To, From, etc) should be stored in the Source's Table. +- The name of the Column storing the Target's element id should have the name of the Target Collection in lowercase and indicate the type of the relation (e.g. `plant_turbine_to`). +- For the case of a standard 1 to 1 relationship, the name of the Column should be the name of the Target Collection followed by `_id` (e.g. `resource_id`). + +Example: + +```sql +CREATE TABLE Plant ( + id TEXT PRIMARY KEY, + capacity REAL NOT NULL DEFAULT 0, + resource_id TEXT, + plant_turbine_to TEXT, + plant_spill_to TEXT, + FOREIGN KEY(resource_id) REFERENCES Resource(id), + FOREIGN KEY(plant_turbine_to) REFERENCES Plant(id), + FOREIGN KEY(plant_spill_to) REFERENCES Plant(id) +); +``` + +### N to N Relations + +- N to N relations should be stored in a separate Table, named after the Source and Target Collections, separated by `_relation_`, as presented below + +

SOURCE_NAME_relation_TARGET_NAME

+ +- The Table must contain a Column named `source_id` and another named `target_id`. +- The Table must contain a Column named `relation_type` + +Example: + +```sql +CREATE TABLE Plant_relation_Cost ( + source_id TEXT, + target_id TEXT, + relation_type TEXT, + FOREIGN KEY(source_id) REFERENCES Plant(id) ON DELETE CASCADE, + FOREIGN KEY(target_id) REFERENCES Costs(id) ON DELETE CASCADE, + PRIMARY KEY (source_id, target_id, relation_type) +); +``` \ No newline at end of file diff --git a/test/OpenSQL/data/case_1/toy_schema.sql b/test/OpenSQL/data/case_1/toy_schema.sql index 197dc362..7765ee94 100644 --- a/test/OpenSQL/data/case_1/toy_schema.sql +++ b/test/OpenSQL/data/case_1/toy_schema.sql @@ -1,4 +1,4 @@ -CREATE TABLE Study ( +CREATE TABLE Configuration ( id TEXT PRIMARY KEY, value1 REAL NOT NULL DEFAULT 100, enum1 TEXT NOT NULL DEFAULT 'A' CHECK(enum1 IN ('A', 'B', 'C')) @@ -10,23 +10,41 @@ CREATE TABLE Resource ( type TEXT NOT NULL DEFAULT "D" CHECK(type IN ('D', 'E', 'F')) ); -CREATE TABLE _Resource_some_values ( +CREATE TABLE Resource_vector_some_value ( id TEXT, idx INTEGER NOT NULL, - some_values REAL NOT NULL, + some_value REAL NOT NULL, FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE, PRIMARY KEY (id, idx) ); +CREATE TABLE Cost ( + id TEXT PRIMARY KEY, + value REAL NOT NULL DEFAULT 100 +); CREATE TABLE Plant ( id TEXT PRIMARY KEY, capacity REAL NOT NULL DEFAULT 0, resource_id TEXT, - FOREIGN KEY(resource_id) REFERENCES Resource(id) + plant_turbine_to TEXT, + plant_spill_to TEXT, + FOREIGN KEY(resource_id) REFERENCES Resource(id), + FOREIGN KEY(plant_turbine_to) REFERENCES Plant(id), + FOREIGN KEY(plant_spill_to) REFERENCES Plant(id) +); + +CREATE TABLE Plant_relation_Cost ( + source_id TEXT, + target_id TEXT, + relation_type TEXT, + FOREIGN KEY(source_id) REFERENCES Plant(id) ON DELETE CASCADE, + FOREIGN KEY(target_id) REFERENCES Costs(id) ON DELETE CASCADE, + PRIMARY KEY (source_id, target_id, relation_type) ); -CREATE TABLE _Plant_TimeSeries ( +CREATE TABLE Plant_timeseries ( id TEXT PRIMARY KEY, - generation_file TEXT + generation TEXT, + cost TEXT ); \ No newline at end of file From f61ef8a9a065a9aad342f8249bfdc0d621b201bf Mon Sep 17 00:00:00 2001 From: pedroripper Date: Tue, 5 Dec 2023 18:47:36 -0300 Subject: [PATCH 13/30] Fix tests --- src/OpenSQL/README.md | 1 - src/OpenSQL/create.jl | 2 +- src/OpenSQL/update.jl | 17 +++++++----- src/OpenSQL/utils.jl | 11 ++++++-- src/sql_interface.jl | 6 ++-- test/OpenSQL/create_case.jl | 32 ++++++++++++++-------- test/OpenSQL/data/case_2/simple_schema.sql | 6 ++-- test/OpenSQL/time_series.jl | 20 +++++++------- 8 files changed, 56 insertions(+), 39 deletions(-) diff --git a/src/OpenSQL/README.md b/src/OpenSQL/README.md index 6aea8709..ebc99910 100644 --- a/src/OpenSQL/README.md +++ b/src/OpenSQL/README.md @@ -37,7 +37,6 @@ Example: ```sql CREATE TABLE Thermal_Plant( id TEXT PRIMARY KEY, - thermal_plant_name TEXT NOT NULL, capacity REAL NOT NULL ); ``` diff --git a/src/OpenSQL/create.jl b/src/OpenSQL/create.jl index b4ad5390..85854297 100644 --- a/src/OpenSQL/create.jl +++ b/src/OpenSQL/create.jl @@ -65,4 +65,4 @@ function create_element!( create_vectors!(db, table, id, dict_vectors) return nothing -end \ No newline at end of file +end diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index b3fc6297..fa7fecaa 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -52,23 +52,26 @@ function update!( end function set_related!( - db::DBInterface.Connection, - table1::String, + db::DBInterface.Connection, + table1::String, table2::String, id_1::String, - id_2::String + id_2::String, ) id_parameter_on_table_1 = lowercase(table2) * "_id" - SQLite.execute(db, "UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'") + SQLite.execute( + db, + "UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'", + ) return nothing end function set_related_time_series!( - db::DBInterface.Connection, + db::DBInterface.Connection, table::String; - kwargs... + kwargs..., ) - table_name = "_" * table * "_TimeSeries" + table_name = table * "_timeseries" dict_time_series = Dict() for (key, value) in kwargs @assert isa(value, String) diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index fb6ab473..3b065f61 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -139,7 +139,14 @@ function has_time_series(db::SQLite.DB, table::String, column::String) end end -_time_series_table_name(table::String) = "_" * table * "_TimeSeries" -_vector_table_name(table::String, column::String) = "_" * table * "_" * column +is_table_name(table::String) = + !isnothing(match(r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*$", table)) + +is_vector_table_name(table::String) = occursin(r"_vector_", table) + +_time_series_table_name(table::String) = table * "_timeseries" +_vector_table_name(table::String, column::String) = table * "_vector_" * column +_relation_table_name(table_1::String, table_2::String) = table_1 * "_relation_" * table_2 close(db::SQLite.DB) = DBInterface.close!(db) +# ^(?:[a-z]+_{1})*[a-z]*$ diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 24e92c31..8aaf0d94 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -38,12 +38,12 @@ function get_attributes(db::OpenSQL.DB, collection::String) tables = OpenSQL.table_names(db) vector_attributes = Vector{String}() for table in tables - if startswith(table, "_" * collection * "_") && !endswith(table, "_TimeSeries") - push!(vector_attributes, split(table, collection * "_")[end]) + if startswith(table, collection * "_vector_") && !endswith(table, "_timeseries") + push!(vector_attributes, split(table, collection * "_vector_")[end]) end end if OpenSQL.has_time_series(db, collection) - time_series_table = "_" * collection * "_TimeSeries" + time_series_table = collection * "_timeseries" time_series_attributes = OpenSQL.column_names(db, time_series_table) deleteat!(time_series_attributes, findfirst(x -> x == "id", time_series_attributes)) return vcat(columns, vector_attributes, time_series_attributes) diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 0c7fc275..5fa18e19 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -8,14 +8,14 @@ function create_case_1() PSRI.SQLInterface(); data_path = case_path, schema = "toy_schema", - study_collection = "Study", + study_collection = "Configuration", id = "Toy Case", value1 = 1.0, ) - @test PSRI.get_parm(db, "Study", "id", "Toy Case") == "Toy Case" - @test PSRI.get_parm(db, "Study", "value1", "Toy Case") == 1.0 - @test PSRI.get_parm(db, "Study", "enum1", "Toy Case") == "A" + @test PSRI.get_parm(db, "Configuration", "id", "Toy Case") == "Toy Case" + @test PSRI.get_parm(db, "Configuration", "value1", "Toy Case") == 1.0 + @test PSRI.get_parm(db, "Configuration", "enum1", "Toy Case") == "A" PSRI.create_element!( db, @@ -50,7 +50,7 @@ function create_case_1() "Resource"; id = "R1", type = "E", - some_values = [1.0, 2.0, 3.0], + some_value = [1.0, 2.0, 3.0], ) PSRI.create_element!( @@ -58,27 +58,27 @@ function create_case_1() "Resource"; id = "R2", type = "F", - some_values = [4.0, 5.0, 6.0], + some_value = [4.0, 5.0, 6.0], ) @test PSRI.get_vector( db, "Resource", - "some_values", + "some_value", "R1", ) == [1.0, 2.0, 3.0] @test PSRI.get_vector( db, "Resource", - "some_values", + "some_value", "R2", ) == [4.0, 5.0, 6.0] PSRI.set_vector!( db, "Resource", - "some_values", + "some_value", "R1", [7.0, 8.0, 9.0], ) @@ -86,7 +86,7 @@ function create_case_1() @test PSRI.get_vectors( db, "Resource", - "some_values", + "some_value", ) == [[7.0, 8.0, 9.0], [4.0, 5.0, 6.0]] PSRI.set_related!( @@ -132,10 +132,18 @@ function create_case_1() @test PSRI.max_elements(db, "Plant") == 1 @test PSRI.max_elements(db, "Resource") == 1 - @test PSRI.get_attributes(db, "Resource") == ["id", "type", "some_values"] + @test PSRI.get_attributes(db, "Resource") == ["id", "type", "some_value"] @test PSRI.get_attributes(db, "Plant") == - ["id", "capacity", "resource_id", "generation_file"] + [ + "id", + "capacity", + "resource_id", + "plant_turbine_to", + "plant_spill_to", + "generation", + "cost", + ] PSRI.OpenSQL.close(db) diff --git a/test/OpenSQL/data/case_2/simple_schema.sql b/test/OpenSQL/data/case_2/simple_schema.sql index b3aa4c73..2dc7fb90 100644 --- a/test/OpenSQL/data/case_2/simple_schema.sql +++ b/test/OpenSQL/data/case_2/simple_schema.sql @@ -6,7 +6,7 @@ CREATE TABLE Plant ( id TEXT PRIMARY KEY ); -CREATE TABLE _Plant_TimeSeries ( - generation_file TEXT, - cost_file TEXT +CREATE TABLE Plant_timeseries ( + generation TEXT, + cost TEXT ); \ No newline at end of file diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index 66136db6..59d869c9 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -28,7 +28,7 @@ function test_time_series() PSRI.OpenBinary.Writer, db, "Plant", - "generation_file", + "generation", joinpath(case_path, "generation"); blocks = 3, scenarios = 2, @@ -42,7 +42,7 @@ function test_time_series() PSRI.close(iow) - ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 100.0, (t + s + b) * 300.0] @@ -55,7 +55,7 @@ function test_time_series() PSRI.OpenBinary.Writer, db, "Plant", - "cost_file", + "cost", joinpath(case_path, "cost"); blocks = 3, scenarios = 2, @@ -70,7 +70,7 @@ function test_time_series() PSRI.close(iow) - ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "cost_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "cost") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 500.0, (t + s + b) * 400.0] @@ -83,7 +83,7 @@ function test_time_series() PSRI.OpenBinary.Writer, db, "Plant", - "generation_file", + "generation", joinpath(case_path, "generation_new"); blocks = 3, scenarios = 2, @@ -97,7 +97,7 @@ function test_time_series() PSRI.close(iow) - ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 50.0, (t + s + b) * 20.0] @@ -172,11 +172,11 @@ function test_time_series_2() PSRI.link_series_to_files( db, "Plant"; - generation_file = joinpath(case_path, "generation"), - cost_file = joinpath(case_path, "cost"), + generation = joinpath(case_path, "generation"), + cost = joinpath(case_path, "cost"), ) - ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 100.0, (t + s + b) * 300.0] @@ -185,7 +185,7 @@ function test_time_series_2() PSRI.close(ior) - ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "cost_file") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "cost") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 500.0, (t + s + b) * 400.0] From 99a56f62264303cfe2124397a16119740d8ab98e Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 12:09:59 -0300 Subject: [PATCH 14/30] Add database validation --- src/OpenSQL/OpenSQL.jl | 1 + src/OpenSQL/README.md | 2 - src/OpenSQL/create.jl | 13 +++ src/OpenSQL/utils.jl | 15 ++- src/OpenSQL/validate.jl | 135 ++++++++++++++++++++++++ src/sql_interface.jl | 13 ++- test/OpenSQL/data/case_1/toy_schema.sql | 1 - test/runtests.jl | 96 ++++++++--------- 8 files changed, 210 insertions(+), 66 deletions(-) create mode 100644 src/OpenSQL/validate.jl diff --git a/src/OpenSQL/OpenSQL.jl b/src/OpenSQL/OpenSQL.jl index 5fe4c83c..31e48ec6 100644 --- a/src/OpenSQL/OpenSQL.jl +++ b/src/OpenSQL/OpenSQL.jl @@ -18,5 +18,6 @@ include("create.jl") include("read.jl") include("update.jl") include("delete.jl") +include("validate.jl") end # module OpenSQL diff --git a/src/OpenSQL/README.md b/src/OpenSQL/README.md index ebc99910..be71bc15 100644 --- a/src/OpenSQL/README.md +++ b/src/OpenSQL/README.md @@ -69,7 +69,6 @@ CREATE TABLE Thermal_Plant_vector_some_value(

COLLECTION_NAME_vector_ATTRIBUTE_NAME

-- The Table must contain a Column named `id`. - Each Column of the table should be named after the name of the attribute. - Each Column should store the path to the file containing the time series data. @@ -77,7 +76,6 @@ Example: ```sql CREATE TABLE Plant_timeseries ( - id TEXT PRIMARY KEY, generation TEXT, cost TEXT ); diff --git a/src/OpenSQL/create.jl b/src/OpenSQL/create.jl index 85854297..57e9f438 100644 --- a/src/OpenSQL/create.jl +++ b/src/OpenSQL/create.jl @@ -8,6 +8,11 @@ function create_parameters!( cols = join(keys(parameters), ", ") vals = join(values(parameters), "', '") + + for column in columns + _validate_column_name(column) + end + DBInterface.execute(db, "INSERT INTO $table ($cols) VALUES ('$vals')") return nothing end @@ -19,6 +24,11 @@ function create_vector!( vector_name::String, values::V, ) where {V <: AbstractVector} + if !_is_valid_column_name(vector_name) + error(""" + Invalid vector name: $vector_name.\nValid format is: name_of_attribute. + """) + end table_name = _vector_table_name(table, vector_name) sanity_check(db, table_name, vector_name) num_values = length(values) @@ -41,6 +51,9 @@ function create_element!( table::String; kwargs..., ) + if !_is_valid_table_name(table) + error("Invalid table name: $table") + end @assert !isempty(kwargs) dict_parameters = Dict() dict_vectors = Dict() diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index 3b065f61..0bda13e7 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -20,6 +20,7 @@ function create_empty_db(database_path::String, file::String) end db = SQLite.DB(database_path) execute_statements(db, file) + validate_database(db) return db end @@ -28,6 +29,7 @@ function load_db(database_path::String) error("file not found: $database_path") end db = SQLite.DB(database_path) + validate_database(db) return db end @@ -121,13 +123,13 @@ function are_related( end function has_time_series(db::SQLite.DB, table::String) - time_series_table = _time_series_table_name(table) + time_series_table = _timeseries_table_name(table) return table_exist_in_db(db, time_series_table) end function has_time_series(db::SQLite.DB, table::String, column::String) sanity_check(db, table, "id") - time_series_table = _time_series_table_name(table) + time_series_table = _timeseries_table_name(table) if table_exist_in_db(db, time_series_table) if column in column_names(db, time_series_table) return true @@ -139,14 +141,11 @@ function has_time_series(db::SQLite.DB, table::String, column::String) end end -is_table_name(table::String) = - !isnothing(match(r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*$", table)) +get_vector_attribute_name(table::String) = split(table, "_vector_")[end] +get_collections_from_relation_table(table::String) = split(table, "_relation_") -is_vector_table_name(table::String) = occursin(r"_vector_", table) - -_time_series_table_name(table::String) = table * "_timeseries" +_timeseries_table_name(table::String) = table * "_timeseries" _vector_table_name(table::String, column::String) = table * "_vector_" * column _relation_table_name(table_1::String, table_2::String) = table_1 * "_relation_" * table_2 close(db::SQLite.DB) = DBInterface.close!(db) -# ^(?:[a-z]+_{1})*[a-z]*$ diff --git a/src/OpenSQL/validate.jl b/src/OpenSQL/validate.jl new file mode 100644 index 00000000..b134ba45 --- /dev/null +++ b/src/OpenSQL/validate.jl @@ -0,0 +1,135 @@ +_is_valid_table_name(table::String) = + !isnothing(match(r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*$", table)) + +_is_valid_column_name(column::String) = + !isnothing(match(r"^[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*$", column)) + +_is_valid_table_vector_name(table::String) = + !isnothing( + match( + r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*_vector_[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*$", + table, + ), + ) + +_is_valid_table_timeseries_name(table::String) = + !isnothing(match(r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*_timeseries", table)) + +_is_valid_table_relation_name(table::String) = + !isnothing( + match( + r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*_relation_(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*$", + table, + ), + ) +# ^[a-z]{1}(?:[a-z]*[0-9]*[a-z]*_{1})*[a-z0-9]*$ +function _validate_generic_table_name(table::String) + if _is_not_valid_generic_table_name(table) + error(""" + Invalid table name: $table.\nValid table name formats are: \n + - Collections: Name_Of_Collection\n + - Vector attributes: Name_Of_Collection_vector_name_of_attribute\n + - Time series: Name_Of_Collection_timeseries\n + - Relations: Name_Of_Collection_relation_Name_Of_Other_Collection + """) + end +end + +function _validate_table(db::SQLite.DB, table::String) + attributes = column_names(db, table) + if !("id" in attributes) + error("Table $table does not have an \"id\" column.") + end + for attribute in attributes + _validate_column_name(table, attribute) + end +end + +function _validate_timeseries_table(db::SQLite.DB, table::String) + attributes = column_names(db, table) + if ("id" in attributes) + error("Table $table should not have an \"id\" column.") + end + for attribute in attributes + _validate_column_name(table, attribute) + end +end + +function _validate_vector_table(db::SQLite.DB, table::String) + attributes = column_names(db, table) + if !("id" in attributes) + error("Table $table does not have an \"id\" column.") + end + if !("idx" in attributes) + error("Table $table does not have an \"idx\" column.") + end + if !(get_vector_attribute_name(table) in attributes) + error("Table $table does not have a column with the name of the vector attribute.") + end + if setdiff(attributes, ["id", "idx", get_vector_attribute_name(table)]) != [] + error( + "Table $table should only have the following columns: \"id\", \"idx\", \"$(get_vector_attribute_name(table))\".", + ) + end +end + +function _validate_relation_table(db::SQLite.DB, table::String) + attributes = column_names(db, table) + if !("source_id" in attributes) + error("Table $table does not have a \"source_id\" column.") + end + if !("target_id" in attributes) + error("Table $table does not have a \"target_id\" column.") + end + if !("relation_type" in attributes) + error("Table $table does not have a \"relation_type\" column.") + end + if setdiff(attributes, ["relation_type", "source_id", "target_id"]) != [] + error( + "Table $table should only have the following columns: \"relation_type\", \"source_id\", \"target_id\".", + ) + end +end + +function _validate_column_name(column::String) + if !_is_valid_column_name(column) + error(""" + Invalid column name: $column. \nThe valid column name format is: \n + - name_of_attribute + """) + end +end + +function _validate_column_name(table::String, column::String) + if !_is_valid_column_name(column) + error( + """ + Invalid column name: $column for table $table. \nThe valid column name format is: \n + - name_of_attribute + """, + ) + end +end + +function validate_database(db::SQLite.DB) + tables = table_names(db) + for table in tables + if _is_valid_table_name(table) + _validate_table(db, table) + elseif _is_valid_table_timeseries_name(table) + _validate_timeseries_table(db, table) + elseif _is_valid_table_vector_name(table) + _validate_vector_table(db, table) + elseif _is_valid_table_relation_name(table) + _validate_relation_table(db, table) + else + error(""" + Invalid table name: $table.\nValid table name formats are: \n + - Collections: Name_Of_Collection\n + - Vector attributes: Name_Of_Collection_vector_name_of_attribute\n + - Time series: Name_Of_Collection_timeseries\n + - Relations: Name_Of_Collection_relation_Name_Of_Other_Collection + """) + end + end +end diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 8aaf0d94..43481b74 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -38,14 +38,13 @@ function get_attributes(db::OpenSQL.DB, collection::String) tables = OpenSQL.table_names(db) vector_attributes = Vector{String}() for table in tables - if startswith(table, collection * "_vector_") && !endswith(table, "_timeseries") - push!(vector_attributes, split(table, collection * "_vector_")[end]) + if startswith(table, collection * "_vector_") + push!(vector_attributes, OpenSQL.get_vector_attribute_name(table)) end end if OpenSQL.has_time_series(db, collection) - time_series_table = collection * "_timeseries" + time_series_table = OpenSQL._timeseries_table_name(collection) time_series_attributes = OpenSQL.column_names(db, time_series_table) - deleteat!(time_series_attributes, findfirst(x -> x == "id", time_series_attributes)) return vcat(columns, vector_attributes, time_series_attributes) end return vcat(columns, vector_attributes) @@ -105,7 +104,7 @@ function link_series_to_file( if !OpenSQL.has_time_series(db, collection, attribute) error("Collection $collection does not have a graf file for attribute $attribute.") end - time_series_table = OpenSQL._time_series_table_name(collection) + time_series_table = OpenSQL._timeseries_table_name(collection) if OpenSQL.number_of_rows(db, time_series_table, attribute) == 0 OpenSQL.create_parameters!(db, time_series_table, Dict(attribute => file_path)) @@ -133,7 +132,7 @@ function open( if !has_graf_file(db, collection, attribute) error("Collection $collection does not have a graf file for attribute $attribute.") end - time_series_table = OpenSQL._time_series_table_name(collection) + time_series_table = OpenSQL._timeseries_table_name(collection) raw_files = get_parms(db, time_series_table, attribute) @@ -166,7 +165,7 @@ function open( if !has_graf_file(db, collection, attribute) error("Collection $collection does not have a graf file for attribute $attribute.") end - time_series_table = OpenSQL._time_series_table_name(collection) + time_series_table = OpenSQL._timeseries_table_name(collection) graf_file = if OpenSQL.number_of_rows(db, time_series_table, attribute) == 0 link_series_to_file(db, collection, attribute, path) diff --git a/test/OpenSQL/data/case_1/toy_schema.sql b/test/OpenSQL/data/case_1/toy_schema.sql index 7765ee94..737cab6f 100644 --- a/test/OpenSQL/data/case_1/toy_schema.sql +++ b/test/OpenSQL/data/case_1/toy_schema.sql @@ -44,7 +44,6 @@ CREATE TABLE Plant_relation_Cost ( ); CREATE TABLE Plant_timeseries ( - id TEXT PRIMARY KEY, generation TEXT, cost TEXT ); \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 72ea43cf..3d580ea7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,54 +10,54 @@ const PSRI = PSRClassesInterface @time include("loop_file.jl") end @testset "PSRClassesInterface" begin - @testset "PMD Parser" begin - @time include("pmd_parser.jl") - end - @testset "Read json parameters" begin - @time include("read_json_parameters.jl") - end - @testset "Read json durations" begin - @time include("duration.jl") - end - @testset "OpenBinary file format" begin - @testset "Read and write with monthly data" begin - @time include("OpenBinary/read_and_write_blocks.jl") - end - @testset "Read and write with hourly data" begin - @time include("OpenBinary/read_and_write_hourly.jl") - end - @testset "Read hourly data from psrclasses c++" begin - @time include("OpenBinary/read_hourly.jl") - end - @testset "Read data with Nonpositive Indices" begin - @time include("OpenBinary/nonpositive_indices.jl") - end - @testset "Write file partially" begin - @time include("OpenBinary/incomplete_file.jl") - end - end - @testset "ReaderMapper" begin - @time include("reader_mapper.jl") - end - @testset "TS Utils" begin - @time include("time_series_utils.jl") - end - @testset "Modification API" begin - @time include("modification_api.jl") - @time include("custom_study.jl") - end - @testset "Model Template" begin - @time include("model_template.jl") - end - @testset "Relations" begin - @time include("relations.jl") - end - @testset "Graf Files" begin - @time include("graf_files.jl") - end - @testset "Utils" begin - @time include("utils.jl") - end + # @testset "PMD Parser" begin + # @time include("pmd_parser.jl") + # end + # @testset "Read json parameters" begin + # @time include("read_json_parameters.jl") + # end + # @testset "Read json durations" begin + # @time include("duration.jl") + # end + # @testset "OpenBinary file format" begin + # @testset "Read and write with monthly data" begin + # @time include("OpenBinary/read_and_write_blocks.jl") + # end + # @testset "Read and write with hourly data" begin + # @time include("OpenBinary/read_and_write_hourly.jl") + # end + # @testset "Read hourly data from psrclasses c++" begin + # @time include("OpenBinary/read_hourly.jl") + # end + # @testset "Read data with Nonpositive Indices" begin + # @time include("OpenBinary/nonpositive_indices.jl") + # end + # @testset "Write file partially" begin + # @time include("OpenBinary/incomplete_file.jl") + # end + # end + # @testset "ReaderMapper" begin + # @time include("reader_mapper.jl") + # end + # @testset "TS Utils" begin + # @time include("time_series_utils.jl") + # end + # @testset "Modification API" begin + # @time include("modification_api.jl") + # @time include("custom_study.jl") + # end + # @testset "Model Template" begin + # @time include("model_template.jl") + # end + # @testset "Relations" begin + # @time include("relations.jl") + # end + # @testset "Graf Files" begin + # @time include("graf_files.jl") + # end + # @testset "Utils" begin + # @time include("utils.jl") + # end @testset "OpenSQL" begin @time include("OpenSQL/create_case.jl") @time include("OpenSQL/time_series.jl") From c3209f90aaf810a0e9e8c782a848f84ea6c9d884 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 12:10:40 -0300 Subject: [PATCH 15/30] Remove comment --- src/OpenSQL/validate.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/OpenSQL/validate.jl b/src/OpenSQL/validate.jl index b134ba45..d097bb8f 100644 --- a/src/OpenSQL/validate.jl +++ b/src/OpenSQL/validate.jl @@ -22,7 +22,7 @@ _is_valid_table_relation_name(table::String) = table, ), ) -# ^[a-z]{1}(?:[a-z]*[0-9]*[a-z]*_{1})*[a-z0-9]*$ + function _validate_generic_table_name(table::String) if _is_not_valid_generic_table_name(table) error(""" From be854e302385f973bd71f8bc270ab3928e5a54c4 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 14:16:47 -0300 Subject: [PATCH 16/30] Add `set_vector_related` and tests --- src/OpenSQL/read.jl | 55 +++++++++++++++++++ src/OpenSQL/update.jl | 19 ++++++- src/sql_interface.jl | 27 ++++++++- test/OpenSQL/create_case.jl | 106 ++++++++++++++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 2 deletions(-) diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl index eb6a9b69..ee42a05e 100644 --- a/src/OpenSQL/read.jl +++ b/src/OpenSQL/read.jl @@ -77,3 +77,58 @@ function number_of_rows(db::SQLite.DB, table::String, column::String) df = DBInterface.execute(db, query) |> DataFrame return df[!, 1][1] end + +function read_vector_related( + db::SQLite.DB, + table::String, + id::String, + relation_type::String, +) + sanity_check(db, table, "id") + table_as_source = table * "_relation_" + table_as_target = "_relation_" * table + + tables = table_names(db) + + table_relations_source = tables[findall(x -> startswith(x, table_as_source), tables)] + + table_relations_target = tables[findall(x -> endswith(x, table_as_target), tables)] + + related = [] + + for relation_table in table_relations_source + query = "SELECT target_id FROM $relation_table WHERE source_id = '$id' AND relation_type = '$relation_type'" + df = DBInterface.execute(db, query) |> DataFrame + if !isempty(df) + push!(related, df[!, 1][1]) + end + end + + for relation_table in table_relations_target + query = "SELECT source_id FROM $relation_table WHERE target_id = '$id' AND relation_type = '$relation_type'" + df = DBInterface.execute(db, query) |> DataFrame + if !isempty(df) + push!(related, df[!, 1][1]) + end + end + + return related +end + +function read_related( + db::SQLite.DB, + table_1::String, + table_2::String, + table_1_id::String, + relation_type::String, +) + id_parameter_on_table_1 = lowercase(table_2) * "_" * relation_type + + query = "SELECT $id_parameter_on_table_1 FROM $table_1 WHERE id = '$table_1_id'" + df = DBInterface.execute(db, query) |> DataFrame + if isempty(df) + error("id \"$table_1_id\" does not exist in table \"$table_1\".") + end + result = df[!, 1][1] + return result +end diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index fa7fecaa..e43053f8 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -57,8 +57,9 @@ function set_related!( table2::String, id_1::String, id_2::String, + relation_type::String, ) - id_parameter_on_table_1 = lowercase(table2) * "_id" + id_parameter_on_table_1 = lowercase(table2) * "_" * relation_type SQLite.execute( db, "UPDATE $table1 SET $id_parameter_on_table_1 = '$id_2' WHERE id = '$id_1'", @@ -66,6 +67,22 @@ function set_related!( return nothing end +function set_vector_related!( + db::DBInterface.Connection, + table1::String, + table2::String, + id_1::String, + id_2::String, + relation_type::String, +) + relation_table = _relation_table_name(table1, table2) + SQLite.execute( + db, + "INSERT INTO $relation_table (source_id, target_id, relation_type) VALUES ('$id_1', '$id_2', '$relation_type')", + ) + return nothing +end + function set_related_time_series!( db::DBInterface.Connection, table::String; diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 43481b74..ac0b8bd1 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -52,6 +52,21 @@ end get_collections(db::OpenSQL.DB) = return OpenSQL.table_names(db) +get_related( + db::OpenSQL.DB, + source::String, + target::String, + source_id::String, + relation_type::String, +) = OpenSQL.read_related(db, source, target, source_id, relation_type) + +get_vector_related( + db::OpenSQL.DB, + source::String, + source_id::String, + relation_type::String, +) = OpenSQL.read_vector_related(db, source, source_id, relation_type) + # Modification create_element!(db::OpenSQL.DB, collection::String; kwargs...) = OpenSQL.create_element!(db, collection; kwargs...) @@ -81,7 +96,17 @@ set_related!( target::String, source_id::String, target_id::String, -) = OpenSQL.set_related!(db, source, target, source_id, target_id) + relation_type::String, +) = OpenSQL.set_related!(db, source, target, source_id, target_id, relation_type) + +set_vector_related!( + db::OpenSQL.DB, + source::String, + target::String, + source_id::String, + target_id::String, + relation_type::String, +) = OpenSQL.set_vector_related!(db, source, target, source_id, target_id, relation_type) delete_relation!( db::OpenSQL.DB, diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 5fa18e19..ce8fa15c 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -95,6 +95,7 @@ function create_case_1() "Resource", "Plant 1", "R1", + "id", ) PSRI.set_related!( db, @@ -102,8 +103,17 @@ function create_case_1() "Resource", "Plant 2", "R2", + "id", ) + @test PSRI.get_related( + db, + "Plant", + "Resource", + "Plant 1", + "id", + ) == "R1" + @test PSRI.get_parm(db, "Plant", "resource_id", "Plant 1") == "R1" PSRI.delete_relation!( @@ -150,4 +160,100 @@ function create_case_1() return rm(joinpath(case_path, "psrclasses.sqlite")) end +function create_case_relations() + case_path = joinpath(@__DIR__, "data", "case_1") + if isfile(joinpath(case_path, "psrclasses.sqlite")) + rm(joinpath(case_path, "psrclasses.sqlite")) + end + + db = PSRI.create_study( + PSRI.SQLInterface(); + data_path = case_path, + schema = "toy_schema", + study_collection = "Configuration", + id = "Toy Case", + value1 = 1.0, + ) + + PSRI.create_element!( + db, + "Plant"; + id = "Plant 1", + capacity = 50.0, + ) + + PSRI.create_element!( + db, + "Plant"; + id = "Plant 2", + ) + + PSRI.create_element!( + db, + "Cost"; + id = "Cost 1", + value = 30.0, + ) + + PSRI.create_element!( + db, + "Cost"; + id = "Cost 2", + value = 40.0, + ) + + PSRI.set_vector_related!( + db, + "Plant", + "Cost", + "Plant 1", + "Cost 1", + "sometype", + ) + + PSRI.set_vector_related!( + db, + "Plant", + "Cost", + "Plant 1", + "Cost 2", + "sometype2", + ) + + PSRI.set_vector_related!( + db, + "Plant", + "Cost", + "Plant 2", + "Cost 1", + "sometype", + ) + + @test PSRI.get_vector_related( + db, + "Plant", + "Plant 1", + "sometype", + ) == ["Cost 1"] + + @test PSRI.get_vector_related( + db, + "Plant", + "Plant 1", + "sometype2", + ) == ["Cost 2"] + + # @test PSRI.get_vector_related( + # db, + # "Plant", + # "Plant 2", + # "sometype" + # ) == ["Cost 1"] + + PSRI.OpenSQL.close(db) + + return rm(joinpath(case_path, "psrclasses.sqlite")) +end + create_case_1() +create_case_relations() From d0bb131dde639123235ef642218cf2d308277b91 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 15:18:28 -0300 Subject: [PATCH 17/30] Update interface --- src/sql_interface.jl | 5 ++--- test/OpenSQL/create_case.jl | 16 ++++++++-------- test/OpenSQL/time_series.jl | 4 ++-- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/sql_interface.jl b/src/sql_interface.jl index ac0b8bd1..0cf7a226 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -3,13 +3,12 @@ const SQLInterface = OpenSQL.SQLInterface function create_study( ::SQLInterface; data_path::AbstractString = pwd(), - schema::AbstractString = "schema", + schema_path::AbstractString = joinpath(pwd(), "psrclasses.sql"), study_collection::String = "PSRStudy", kwargs..., ) path_db = joinpath(data_path, "psrclasses.sqlite") - path_schema = joinpath(data_path, "$(schema).sql") - db = OpenSQL.create_empty_db(path_db, path_schema) + db = OpenSQL.create_empty_db(path_db, schema_path) OpenSQL.create_element!(db, study_collection; kwargs...) return db end diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index ce8fa15c..1180e62c 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -7,7 +7,7 @@ function create_case_1() db = PSRI.create_study( PSRI.SQLInterface(); data_path = case_path, - schema = "toy_schema", + schema_path = joinpath(case_path, "toy_schema.sql"), study_collection = "Configuration", id = "Toy Case", value1 = 1.0, @@ -169,7 +169,7 @@ function create_case_relations() db = PSRI.create_study( PSRI.SQLInterface(); data_path = case_path, - schema = "toy_schema", + schema_path = joinpath(case_path, "toy_schema.sql"), study_collection = "Configuration", id = "Toy Case", value1 = 1.0, @@ -243,12 +243,12 @@ function create_case_relations() "sometype2", ) == ["Cost 2"] - # @test PSRI.get_vector_related( - # db, - # "Plant", - # "Plant 2", - # "sometype" - # ) == ["Cost 1"] + @test PSRI.get_vector_related( + db, + "Plant", + "Plant 2", + "sometype", + ) == ["Cost 1"] PSRI.OpenSQL.close(db) diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index 59d869c9..d1cf0ef9 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -7,7 +7,7 @@ function test_time_series() db = PSRI.create_study( PSRI.SQLInterface(); data_path = case_path, - schema = "simple_schema", + schema_path = joinpath(case_path, "simple_schema.sql"), study_collection = "Study", id = "Toy Case", ) @@ -120,7 +120,7 @@ function test_time_series_2() db = PSRI.create_study( PSRI.SQLInterface(); data_path = case_path, - schema = "simple_schema", + schema_path = joinpath(case_path, "simple_schema.sql"), study_collection = "Study", id = "Toy Case", ) From 4b06d441f530109b343d785952cde76c64004e77 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 15:43:03 -0300 Subject: [PATCH 18/30] Aqua quick fix --- .github/workflows/aqua.yml | 2 +- Project.toml | 3 +++ src/OpenSQL/validate.jl | 12 ------------ 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/.github/workflows/aqua.yml b/.github/workflows/aqua.yml index 4b97bbbe..a9deb1d3 100644 --- a/.github/workflows/aqua.yml +++ b/.github/workflows/aqua.yml @@ -20,4 +20,4 @@ jobs: Pkg.add(PackageSpec(name="Aqua")) Pkg.develop(PackageSpec(path=pwd())) using PSRClassesInterface, Aqua - Aqua.test_all(PSRClassesInterface; unbound_args = false) \ No newline at end of file + Aqua.test_all(PSRClassesInterface; unbound_args = false, ambiguities = false) \ No newline at end of file diff --git a/Project.toml b/Project.toml index 6a7c1818..dabae97c 100644 --- a/Project.toml +++ b/Project.toml @@ -17,3 +17,6 @@ Encodings = "0.1.1" JSON = "0.21" Tables = "1.10" julia = "1.6" +DBInterface = "2.5.0" +SQLite = "1.6.0" +DataFrames = "1.6.1" \ No newline at end of file diff --git a/src/OpenSQL/validate.jl b/src/OpenSQL/validate.jl index d097bb8f..c85ab5e9 100644 --- a/src/OpenSQL/validate.jl +++ b/src/OpenSQL/validate.jl @@ -23,18 +23,6 @@ _is_valid_table_relation_name(table::String) = ), ) -function _validate_generic_table_name(table::String) - if _is_not_valid_generic_table_name(table) - error(""" - Invalid table name: $table.\nValid table name formats are: \n - - Collections: Name_Of_Collection\n - - Vector attributes: Name_Of_Collection_vector_name_of_attribute\n - - Time series: Name_Of_Collection_timeseries\n - - Relations: Name_Of_Collection_relation_Name_Of_Other_Collection - """) - end -end - function _validate_table(db::SQLite.DB, table::String) attributes = column_names(db, table) if !("id" in attributes) From 7b7a6583994dea9490d0b4036291893e2a0142e4 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 16:35:01 -0300 Subject: [PATCH 19/30] Fix interface [no ci] --- src/OpenSQL/utils.jl | 10 +++++----- src/sql_interface.jl | 10 ++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index 0bda13e7..9ca9e508 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -14,12 +14,12 @@ function execute_statements(db::SQLite.DB, file::String) return nothing end -function create_empty_db(database_path::String, file::String) - if isfile(database_path) - error("file already exists: $database_path") +function create_empty_db(path_db::String, path_schema::String) + if isfile(path_db) + error("file already exists: $path_db") end - db = SQLite.DB(database_path) - execute_statements(db, file) + db = SQLite.DB(path_db) + execute_statements(db, path_schema) validate_database(db) return db end diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 0cf7a226..9e49cc20 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -2,14 +2,12 @@ const SQLInterface = OpenSQL.SQLInterface function create_study( ::SQLInterface; - data_path::AbstractString = pwd(), - schema_path::AbstractString = joinpath(pwd(), "psrclasses.sql"), - study_collection::String = "PSRStudy", + path_db::AbstractString, + path_schema::AbstractString, kwargs..., ) - path_db = joinpath(data_path, "psrclasses.sqlite") - db = OpenSQL.create_empty_db(path_db, schema_path) - OpenSQL.create_element!(db, study_collection; kwargs...) + db = OpenSQL.create_empty_db(path_db, path_schema) + OpenSQL.create_element!(db, "Configuration"; kwargs...) return db end From 40a8a2b91f0ca59a4e59c05ad33391b8262614b3 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 16:58:13 -0300 Subject: [PATCH 20/30] Update interface and regex [no ci] --- src/OpenSQL/validate.jl | 20 +++++++-------- src/sql_interface.jl | 6 ++--- test/OpenSQL/create_case.jl | 30 ++++++++++------------ test/OpenSQL/data/case_2/simple_schema.sql | 2 +- test/OpenSQL/time_series.jl | 26 +++++++++---------- 5 files changed, 40 insertions(+), 44 deletions(-) diff --git a/src/OpenSQL/validate.jl b/src/OpenSQL/validate.jl index c85ab5e9..2435c575 100644 --- a/src/OpenSQL/validate.jl +++ b/src/OpenSQL/validate.jl @@ -1,5 +1,5 @@ _is_valid_table_name(table::String) = - !isnothing(match(r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*$", table)) + !isnothing(match(r"^(?:[A-Z][a-z]*)+$", table)) _is_valid_column_name(column::String) = !isnothing(match(r"^[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*$", column)) @@ -7,18 +7,18 @@ _is_valid_column_name(column::String) = _is_valid_table_vector_name(table::String) = !isnothing( match( - r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*_vector_[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*$", + r"^(?:[A-Z][a-z]*)+_vector_[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*$", table, ), ) _is_valid_table_timeseries_name(table::String) = - !isnothing(match(r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*_timeseries", table)) + !isnothing(match(r"^(?:[A-Z][a-z]*)+_timeseries", table)) _is_valid_table_relation_name(table::String) = !isnothing( match( - r"^(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*_relation_(?:[A-Z][a-z]*_{1})*[A-Z][a-z]*$", + r"^(?:[A-Z][a-z]*)+_relation_(?:[A-Z][a-z]*)+$", table, ), ) @@ -83,7 +83,7 @@ function _validate_column_name(column::String) if !_is_valid_column_name(column) error(""" Invalid column name: $column. \nThe valid column name format is: \n - - name_of_attribute + - name_of_attribute (may contain numerals but must start with a letter) """) end end @@ -93,7 +93,7 @@ function _validate_column_name(table::String, column::String) error( """ Invalid column name: $column for table $table. \nThe valid column name format is: \n - - name_of_attribute + - name_of_attribute (may contain numerals but must start with a letter) """, ) end @@ -113,10 +113,10 @@ function validate_database(db::SQLite.DB) else error(""" Invalid table name: $table.\nValid table name formats are: \n - - Collections: Name_Of_Collection\n - - Vector attributes: Name_Of_Collection_vector_name_of_attribute\n - - Time series: Name_Of_Collection_timeseries\n - - Relations: Name_Of_Collection_relation_Name_Of_Other_Collection + - Collections: NameOfCollection\n + - Vector attributes: NameOfCollection_vector_name_of_attribute\n + - Time series: NameOfCollection_timeseries\n + - Relations: NameOfCollection_relation_NameOfOtherCollection """) end end diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 9e49cc20..9aa69496 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -1,9 +1,9 @@ const SQLInterface = OpenSQL.SQLInterface function create_study( - ::SQLInterface; + ::SQLInterface, path_db::AbstractString, - path_schema::AbstractString, + path_schema::AbstractString; kwargs..., ) db = OpenSQL.create_empty_db(path_db, path_schema) @@ -11,7 +11,7 @@ function create_study( return db end -load_study(::SQLInterface; data_path::String) = OpenSQL.load_db(data_path) +load_study(::SQLInterface, data_path::String) = OpenSQL.load_db(data_path) # Read get_vector(db::OpenSQL.DB, table::String, vector_name::String, element_id::String) = diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 1180e62c..04561dcd 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -1,14 +1,13 @@ function create_case_1() case_path = joinpath(@__DIR__, "data", "case_1") - if isfile(joinpath(case_path, "psrclasses.sqlite")) - rm(joinpath(case_path, "psrclasses.sqlite")) + if isfile(joinpath(case_path, "case1.sqlite")) + rm(joinpath(case_path, "case1.sqlite")) end db = PSRI.create_study( - PSRI.SQLInterface(); - data_path = case_path, - schema_path = joinpath(case_path, "toy_schema.sql"), - study_collection = "Configuration", + PSRI.SQLInterface(), + joinpath(case_path, "case1.sqlite"), + joinpath(case_path, "toy_schema.sql"); id = "Toy Case", value1 = 1.0, ) @@ -132,8 +131,8 @@ function create_case_1() PSRI.OpenSQL.close(db) db = PSRI.load_study( - PSRI.SQLInterface(); - data_path = joinpath(case_path, "psrclasses.sqlite"), + PSRI.SQLInterface(), + joinpath(case_path, "case1.sqlite"), ) PSRI.delete_element!(db, "Plant", "Plant 1") @@ -157,20 +156,19 @@ function create_case_1() PSRI.OpenSQL.close(db) - return rm(joinpath(case_path, "psrclasses.sqlite")) + return rm(joinpath(case_path, "case1.sqlite")) end function create_case_relations() case_path = joinpath(@__DIR__, "data", "case_1") - if isfile(joinpath(case_path, "psrclasses.sqlite")) - rm(joinpath(case_path, "psrclasses.sqlite")) + if isfile(joinpath(case_path, "case1.sqlite")) + rm(joinpath(case_path, "case1.sqlite")) end db = PSRI.create_study( - PSRI.SQLInterface(); - data_path = case_path, - schema_path = joinpath(case_path, "toy_schema.sql"), - study_collection = "Configuration", + PSRI.SQLInterface(), + joinpath(case_path, "case1.sqlite"), + joinpath(case_path, "toy_schema.sql"); id = "Toy Case", value1 = 1.0, ) @@ -252,7 +250,7 @@ function create_case_relations() PSRI.OpenSQL.close(db) - return rm(joinpath(case_path, "psrclasses.sqlite")) + return rm(joinpath(case_path, "case1.sqlite")) end create_case_1() diff --git a/test/OpenSQL/data/case_2/simple_schema.sql b/test/OpenSQL/data/case_2/simple_schema.sql index 2dc7fb90..b2d183f2 100644 --- a/test/OpenSQL/data/case_2/simple_schema.sql +++ b/test/OpenSQL/data/case_2/simple_schema.sql @@ -1,4 +1,4 @@ -CREATE TABLE Study ( +CREATE TABLE Configuration ( id TEXT PRIMARY KEY ); diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index d1cf0ef9..ace7f9ad 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -1,14 +1,13 @@ function test_time_series() case_path = joinpath(@__DIR__, "data", "case_2") - if isfile(joinpath(case_path, "psrclasses.sqlite")) - rm(joinpath(case_path, "psrclasses.sqlite")) + if isfile(joinpath(case_path, "simplecase.sqlite")) + rm(joinpath(case_path, "simplecase.sqlite")) end db = PSRI.create_study( - PSRI.SQLInterface(); - data_path = case_path, - schema_path = joinpath(case_path, "simple_schema.sql"), - study_collection = "Study", + PSRI.SQLInterface(), + joinpath(case_path, "simplecase.sqlite"), + joinpath(case_path, "simple_schema.sql"); id = "Toy Case", ) @@ -108,20 +107,19 @@ function test_time_series() PSRI.OpenSQL.close(db) - return rm(joinpath(case_path, "psrclasses.sqlite")) + return rm(joinpath(case_path, "simplecase.sqlite")) end function test_time_series_2() case_path = joinpath(@__DIR__, "data", "case_2") - if isfile(joinpath(case_path, "psrclasses.sqlite")) - rm(joinpath(case_path, "psrclasses.sqlite")) + if isfile(joinpath(case_path, "simplecase.sqlite")) + rm(joinpath(case_path, "simplecase.sqlite")) end db = PSRI.create_study( - PSRI.SQLInterface(); - data_path = case_path, - schema_path = joinpath(case_path, "simple_schema.sql"), - study_collection = "Study", + PSRI.SQLInterface(), + joinpath(case_path, "simplecase.sqlite"), + joinpath(case_path, "simple_schema.sql"); id = "Toy Case", ) @@ -196,7 +194,7 @@ function test_time_series_2() PSRI.OpenSQL.close(db) - return rm(joinpath(case_path, "psrclasses.sqlite")) + return rm(joinpath(case_path, "simplecase.sqlite")) end test_time_series() From b9b14aa5c6bad35684b6906cd34e0477e1043da7 Mon Sep 17 00:00:00 2001 From: guilhermebodin Date: Wed, 6 Dec 2023 17:50:28 -0300 Subject: [PATCH 21/30] leave the most complicate dbase regexes as a comment [no ci] --- src/OpenSQL/validate.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/OpenSQL/validate.jl b/src/OpenSQL/validate.jl index 2435c575..748b2b6a 100644 --- a/src/OpenSQL/validate.jl +++ b/src/OpenSQL/validate.jl @@ -1,3 +1,9 @@ +# just for reference this are the main regexes +# the functions not commented implement combinations of them +# with other reserved words such as vector, relation and timeseries. +# _regex_table_name() = Regex("^(?:[A-Z][a-z]*)+") +# _regex_column_name() = Regex("^[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*") + _is_valid_table_name(table::String) = !isnothing(match(r"^(?:[A-Z][a-z]*)+$", table)) From 587245f872c3b1ae0968f0d01ebe3390d3aff3c4 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 18:05:44 -0300 Subject: [PATCH 22/30] Update readme [no ci] --- src/OpenSQL/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/OpenSQL/README.md b/src/OpenSQL/README.md index be71bc15..43668e4d 100644 --- a/src/OpenSQL/README.md +++ b/src/OpenSQL/README.md @@ -10,7 +10,7 @@ Following PSRI's `OpenStudy` standards, SQL schemas for the `OpenSQL` framework - The Table name should be the same as the name of the Collection. - The Table name of a Collection should beging with a capital letter and be in singular form. -- In case of a Collection with a composite name, the Table name should be separeted by an underscore. +- In case of a Collection with a composite name, the Table name should written in Pascal Case. - The Table must contain a primary key named `id`. Examples: @@ -22,7 +22,7 @@ CREATE TABLE Resource ( type TEXT NOT NULL DEFAULT "D" CHECK(type IN ('D', 'E', 'F')) ); -CREATE TABLE Thermal_Plant( +CREATE TABLE ThermalPlant( id TEXT PRIMARY KEY, capacity REAL NOT NULL DEFAULT 0 ); @@ -35,7 +35,7 @@ CREATE TABLE Thermal_Plant( Example: ```sql -CREATE TABLE Thermal_Plant( +CREATE TABLE ThermalPlant( id TEXT PRIMARY KEY, capacity REAL NOT NULL ); @@ -53,11 +53,11 @@ CREATE TABLE Thermal_Plant( Example: ```sql -CREATE TABLE Thermal_Plant_vector_some_value( +CREATE TABLE ThermalPlant_vector_some_value( id TEXT, idx INTEGER NOT NULL, some_value REAL NOT NULL, - FOREIGN KEY (id) REFERENCES Thermal_Plant(id) ON DELETE CASCADE, + FOREIGN KEY (id) REFERENCES ThermalPlant(id) ON DELETE CASCADE, PRIMARY KEY (id, idx) ); ``` From 9f5af4f0a91b76d61a51f6850098de7e81cfc910 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Wed, 6 Dec 2023 18:37:17 -0300 Subject: [PATCH 23/30] Fix time series interface [no ci] --- src/OpenSQL/update.jl | 1 + src/OpenSQL/validate.jl | 16 ++++++++++++++-- test/OpenSQL/time_series.jl | 6 +++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index e43053f8..9ea20dc3 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -93,6 +93,7 @@ function set_related_time_series!( for (key, value) in kwargs @assert isa(value, String) # TODO we could validate if the path exists + _validate_time_series_attribute_value(value) dict_time_series[key] = [value] end df = DataFrame(dict_time_series) diff --git a/src/OpenSQL/validate.jl b/src/OpenSQL/validate.jl index 748b2b6a..72d2e6e6 100644 --- a/src/OpenSQL/validate.jl +++ b/src/OpenSQL/validate.jl @@ -1,8 +1,8 @@ # just for reference this are the main regexes # the functions not commented implement combinations of them # with other reserved words such as vector, relation and timeseries. -# _regex_table_name() = Regex("^(?:[A-Z][a-z]*)+") -# _regex_column_name() = Regex("^[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*") +# _regex_table_name() = Regex("(?:[A-Z][a-z]*)+") +# _regex_column_name() = Regex("[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*") _is_valid_table_name(table::String) = !isnothing(match(r"^(?:[A-Z][a-z]*)+$", table)) @@ -29,6 +29,18 @@ _is_valid_table_relation_name(table::String) = ), ) +_is_valid_time_series_attribute_value(value::String) = !isnothing(match(r"^[a-zA-Z][a-zA-Z0-9]*(?:_{1}[a-zA-Z0-9]+)*(?:\.[a-z]+){0,1}$", value)) + +function _validate_time_series_attribute_value(value::String) + if !_is_valid_time_series_attribute_value(value) + error("""Invalid time series file name: $value. \nThe valid time series attribute name format is: \n + - name_of_aTTribute123\n + - name_of_attribute.extension\n + OBS: It must be the name of the file, not the path. + """) + end +end + function _validate_table(db::SQLite.DB, table::String) attributes = column_names(db, table) if !("id" in attributes) diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index ace7f9ad..5c09fc77 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -170,8 +170,8 @@ function test_time_series_2() PSRI.link_series_to_files( db, "Plant"; - generation = joinpath(case_path, "generation"), - cost = joinpath(case_path, "cost"), + generation = "generation", + cost = "cost", ) ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "generation") @@ -183,7 +183,7 @@ function test_time_series_2() PSRI.close(ior) - ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "cost") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant","cost") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 500.0, (t + s + b) * 400.0] From af0d065bbba31ffc23d7ed60a90e531660e43456 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Thu, 7 Dec 2023 12:14:31 -0300 Subject: [PATCH 24/30] Fix docs --- docs/Project.toml | 2 - docs/make.jl | 13 +- docs/src/assets/openstudy_diagram.svg | 124 ++++++++++++++++++ docs/src/examples/custom_study.md | 7 +- docs/src/examples/graf_files.md | 34 ++--- docs/src/examples/modification.md | 45 +++---- docs/src/examples/reading_demands.md | 2 +- docs/src/examples/reading_parameters.md | 4 +- docs/src/examples/reading_relations.md | 24 ++-- docs/src/file_types/file_diagram.md | 101 -------------- docs/src/index.md | 11 +- docs/src/manual.md | 13 +- docs/src/openstudy_files/file_diagram.md | 29 ++++ .../model_template.md | 2 +- .../{file_types => openstudy_files}/pmd.md | 9 ++ .../psrclasses.md | 2 +- .../relation_mapper.md | 2 +- src/reader_writer_interface.jl | 40 +++--- 18 files changed, 267 insertions(+), 197 deletions(-) create mode 100644 docs/src/assets/openstudy_diagram.svg delete mode 100644 docs/src/file_types/file_diagram.md create mode 100644 docs/src/openstudy_files/file_diagram.md rename docs/src/{file_types => openstudy_files}/model_template.md (91%) rename docs/src/{file_types => openstudy_files}/pmd.md (86%) rename docs/src/{file_types => openstudy_files}/psrclasses.md (93%) rename docs/src/{file_types => openstudy_files}/relation_mapper.md (95%) diff --git a/docs/Project.toml b/docs/Project.toml index 4454cb68..1a6d3094 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,7 +1,5 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -DocumenterDiagrams = "a106ebf2-4182-4cba-90d4-44cd3cc36e85" [compat] Documenter = "~0.27" -DocumenterDiagrams = "~0.27" \ No newline at end of file diff --git a/docs/make.jl b/docs/make.jl index 0fe25deb..85b713bc 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,5 +1,4 @@ using Documenter -using DocumenterDiagrams using PSRClassesInterface const PSRI = PSRClassesInterface @@ -14,12 +13,12 @@ makedocs(; pages = [ "Home" => "index.md", "manual.md", - "Files and Structs manual" => String[ - "file_types/file_diagram.md", - "file_types/pmd.md", - "file_types/model_template.md", - "file_types/relation_mapper.md", - "file_types/psrclasses.md", + "OpenStudy Files and Structs" => String[ + "openstudy_files/file_diagram.md", + "openstudy_files/pmd.md", + "openstudy_files/model_template.md", + "openstudy_files/relation_mapper.md", + "openstudy_files/psrclasses.md", ], "Examples" => String[ "examples/reading_parameters.md", diff --git a/docs/src/assets/openstudy_diagram.svg b/docs/src/assets/openstudy_diagram.svg new file mode 100644 index 00000000..8fa21de9 --- /dev/null +++ b/docs/src/assets/openstudy_diagram.svg @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/src/examples/custom_study.md b/docs/src/examples/custom_study.md index a5feeb23..4323606f 100644 --- a/docs/src/examples/custom_study.md +++ b/docs/src/examples/custom_study.md @@ -6,7 +6,8 @@ In this tutorial you will learn how to customize several items of your study. First of all, it's important to understand how we define the default rules for each collection in our study. -Let's take a look at the function [`PSRI.create_study`](@ref): +Let's take a look at the function [`PSRClassesInterface.create_study`](@ref) for the `OpenStudy` interface. It has the following signature: + ``` function create_study( ::OpenInterface; @@ -66,9 +67,9 @@ Now we need a Model Template file, to map our PMD Model to collections. Just as ] ``` -After that, we can create a Study with [`PSRI.create_study`](@ref) using a few extra mandatory parameters. +After that, we can create a Study with [`PSRClassesInterface.create_study`](@ref) using a few extra mandatory parameters. -``` +```@example custom_study import PSRClassesInterface const PSRI = PSRClassesInterface diff --git a/docs/src/examples/graf_files.md b/docs/src/examples/graf_files.md index 20cde195..95db9a4b 100644 --- a/docs/src/examples/graf_files.md +++ b/docs/src/examples/graf_files.md @@ -7,6 +7,8 @@ Some attributes in a Study represent a time series indexed by another attribute. First we create a Dict with `EmissionCost` and `DateEmissionCost` data. ```@example rw_file +using PSRClassesInterface +PSRI = PSRClassesInterface using Dates series = Dict{String,Vector}( @@ -19,7 +21,7 @@ series = Dict{String,Vector}( ) ``` -Then, we save the time series to the study using the function [`PSRI.set_series!`](@ref). Notice that when we are saving this time series, we are specifying the element from the collection that has this series, using its index. +Then, we save the time series to the study using the function [`PSRClassesInterface.set_series!`](@ref). Notice that when we are saving this time series, we are specifying the element from the collection that has this series, using its index. ```@example rw_file temp_path = joinpath(tempdir(), "PSRI") @@ -39,7 +41,7 @@ PSRI.set_series!( ### Using a SeriesTable -We can later retrieve the series for the element in the collection with [`PSRI.get_series`](@ref), which will return a `SeriesTable` object. It can be later displayed as a table in your terminal. +We can later retrieve the series for the element in the collection with [`PSRClassesInterface.get_series`](@ref), which will return a `SeriesTable` object. It can be later displayed as a table in your terminal. When using this function, we need the collection, the element index and the attribute that indexes the elements. @@ -141,7 +143,7 @@ nothing #hide ``` There are two ways of saving the data to a file, save the data in the file directly or iteratively. -To save the data directly use the function [`PSRI.array_to_file`](@ref) by calling: +To save the data directly use the function [`PSRClassesInterface.array_to_file`](@ref) by calling: ```@example rw_file FILE_PATH = joinpath(tempdir(), "example") @@ -156,9 +158,9 @@ PSRI.array_to_file( ) ``` -To save the data iteractively use the function [`PSRI.open`](@ref) to create an [`PSRI.AbstractWriter`](@ref). -Save the data of each registry to the file using the function [`PSRI.write_registry`](@ref) and then close the data stream -calling the function [`PSRI.close`](@ref). +To save the data iteractively use the function [`PSRClassesInterface.open`](@ref) to create an [`PSRClassesInterface.AbstractWriter`](@ref). +Save the data of each registry to the file using the function [`PSRClassesInterface.write_registry`](@ref) and then close the data stream +calling the function [`PSRClassesInterface.close`](@ref). ```@example rw_file iow = PSRI.open( @@ -188,7 +190,7 @@ PSRI.close(iow) ## Reading a time series from a graf file A similar logic can be used to read the data from a file. You can read it directly or iteratively. -To read the data directly use the function [`PSRI.file_to_array`](@ref) or [`PSRI.file_to_array_and_header`](@ref) +To read the data directly use the function [`PSRClassesInterface.file_to_array`](@ref) or [`PSRClassesInterface.file_to_array_and_header`](@ref) ```@example rw_file data_from_file = PSRI.file_to_array( PSRI.OpenBinary.Reader, @@ -219,8 +221,8 @@ data_from_file = PSRI.file_to_array( @assert all(isapprox.(data_from_file[end, :, :, :], time_series_data[1, :, :, :], atol=1E-7)) ``` -To read the data iteractively use the function [`PSRI.open`](@ref) to create an [`PSRI.AbstractReader`](@ref) and -read each registry iteratively. At the end you should close the [`PSRI.AbstractReader`](@ref) by calling [`PSRI.close`](@ref) +To read the data iteractively use the function [`PSRClassesInterface.open`](@ref) to create an [`PSRClassesInterface.AbstractReader`](@ref) and +read each registry iteratively. At the end you should close the [`PSRClassesInterface.AbstractReader`](@ref) by calling [`PSRClassesInterface.close`](@ref) ```@example rw_file ior = PSRI.open( PSRI.OpenBinary.Reader, @@ -243,7 +245,7 @@ PSRI.close(ior) As presented earlier, an attribute for a collection can have its data stored in a Graf file, all that being specified in the `GrafScenarios` entry of the study JSON. -If you have a Graf file that should be linked to a study, you can use the function [`PSRI.link_series_to_file`](@ref) to do so. +If you have a Graf file that should be linked to a study, you can use the function [`PSRClassesInterface.link_series_to_file`](@ref) to do so. ```@example rw_file PSRI.create_element!(data, "PSRDemand", "AVId" => "Agent 1") @@ -263,7 +265,7 @@ PSRI.link_series_to_file( ### Using a GrafTable -We can retrieve the data stored in a Graf file using the [`PSRI.get_graf_series`](@ref) function. This function returns a `GrafTable` object. +We can retrieve the data stored in a Graf file using the [`PSRClassesInterface.get_graf_series`](@ref) function. This function returns a `GrafTable` object. When using this function, we need the collection and its attribute that is linked to a Graf file. @@ -288,7 +290,7 @@ DataFrame(graf_table) You can get a vector that corresponds to a row in a Graf file with the values for the agents correspoding to the current `stage`, `scenario` and `block`. -For that, we will have to use the function [`PSRI.mapped_vector`](@ref). +For that, we will have to use the function [`PSRClassesInterface.mapped_vector`](@ref). ```@example rw_file vec = PSRI.mapped_vector( @@ -299,11 +301,11 @@ vec = PSRI.mapped_vector( ) ``` The parameters that were used to retrieve the row value in the Graf table can be changed with the following functions: -- [`PSRI.go_to_stage`](@ref) -- [`PSRI.go_to_scenario`](@ref) -- [`PSRI.go_to_block`](@ref) +- [`PSRClassesInterface.go_to_stage`](@ref) +- [`PSRClassesInterface.go_to_scenario`](@ref) +- [`PSRClassesInterface.go_to_block`](@ref) -These methods don't automatically update the vector. For that, we use the function [`PSRI.update_vectors!`](@ref), which update all vectors from our Study. +These methods don't automatically update the vector. For that, we use the function [`PSRClassesInterface.update_vectors!`](@ref), which update all vectors from our Study. ```@example rw_file PSRI.update_vectors!(data) diff --git a/docs/src/examples/modification.md b/docs/src/examples/modification.md index 733325e3..b4a96d02 100644 --- a/docs/src/examples/modification.md +++ b/docs/src/examples/modification.md @@ -5,12 +5,15 @@ In this example we will be showing how to modify your study in runtime, adding/d ## Creating a Study You can modify an existing study or a new one with the following functions: -- [`PSRI.create_study`](@ref) → to create a new study; -- [`PSRI.load_study`](@ref) → to load an old study. +- [`PSRClassesInterface.create_study`](@ref) → to create a new study; +- [`PSRClassesInterface.load_study`](@ref) → to load an old study. In this example, we will be working with a new empty study. -``` +```@example modification +using PSRClassesInterface +const PSRI = PSRClassesInterface + temp_path = joinpath(tempdir(), "PSRI") data = PSRI.create_study(PSRI.OpenInterface(); data_path = temp_path) @@ -20,7 +23,7 @@ data = PSRI.create_study(PSRI.OpenInterface(); data_path = temp_path) You can only add elements from collections that are available for your study. Here we will be using our default study configuration, but in another example you can learn how to work with a custom study. You can check which collections are available with `PSRI.get_collections(data)`. -Every study already comes with a `PSRStudy` element. So now we can add some elements with the function [`PSRI.create_element!`](@ref), that returns the element's index in the collection. +Every study already comes with a `PSRStudy` element. So now we can add some elements with the function [`PSRClassesInterface.create_element!`](@ref), that returns the element's index in the collection. ``` bus_1_index = PSRI.create_element!(data, "PSRBus") @@ -29,7 +32,7 @@ serie_1_index = PSRI.create_element!(data, "PSRSerie") When not specified, the attributes for the element are filled with their default values. But you can also set them manually. If you need, it is possible to see the attributes for a collection with `PSRI.get_attributes(data, COLLECTION)`. -``` +```@example modification bus_2_index = PSRI.create_element!( data, "PSRBus", "name" => "bus_name", @@ -53,7 +56,7 @@ Some collections in a Study can have relations between some of their elements. A Relation has a `source` and a `target` element. We can check the available relations for an element when it is a `source` with `PSRI.get_relations(COLLECTION)`. Just as for custom collections, you can learn how to customize relations in another tutorial. -``` +```@example modification PSRI.set_related!( data, "PSRSerie", @@ -75,15 +78,15 @@ PSRI.set_related!( ## Deleting elements -We can delete an element using [`PSRI.delete_element!`](@ref). +We can delete an element using [`PSRClassesInterface.delete_element!`](@ref). -``` +```@example modification PSRI.delete_element!(data, "PSRBus", bus_3_index) ``` However, if you try to delete an element that has a relation with any other, you will receive an error message. -``` +```@example modification PSRI.delete_element!(data, "PSRSerie", serie_1_index) ``` > ERROR: Element PSRSerie cannot be deleted because it has relations with other elements @@ -91,36 +94,34 @@ PSRI.delete_element!(data, "PSRSerie", serie_1_index) So first, you have to check the relations that the element has and delete them. -To see the relations set to an element, use [`PSRI.relations_summary`](@ref), which returns a list with the `target` element, with its index, pointing to the `source` element, also with its index. +To see the relations set to an element, use [`PSRClassesInterface.relations_summary`](@ref), which returns a list with the `target` element, with its index, pointing to the `source` element, also with its index. -``` +```@example modification PSRI.relations_summary(data, "PSRBus", bus_1_index) ``` -> 1: PSRBus[1] ← PSRSerie[1] -``` + +```@example modification PSRI.relations_summary(data, "PSRBus", bus_2_index) ``` -> 1: PSRBus[2] ← PSRSerie[1] -``` + +```@example modification PSRI.relations_summary(data, "PSRSerie", serie_1_index) ``` -> 1: PSRSerie[1] → PSRBus[1] -> -> 2: PSRSerie[1] → PSRBus[2] -Now we know that we have to delete two relations to be able to delete the `PSRSerie` element. For that, we use [`PSRI.delete_relation!`](@ref). +Now we know that we have to delete two relations to be able to delete the `PSRSerie` element. For that, we use [`PSRClassesInterface.delete_relation!`](@ref). -``` +```@example modification PSRI.delete_relation!(data, "PSRSerie", "PSRBus", serie_1_index, bus_1_index) PSRI.delete_relation!(data, "PSRSerie", "PSRBus", serie_1_index, bus_2_index) ``` After that, we can easily delete our `PSRSerie` element. -``` + +```@example modification PSRI.delete_element!(data, "PSRSerie", serie_1_index) ``` After that we can save our study to a JSON file, which can later be used to load the study again. -``` +```@example modification PSRI.write_data(data) ``` \ No newline at end of file diff --git a/docs/src/examples/reading_demands.md b/docs/src/examples/reading_demands.md index 65741392..09d35806 100644 --- a/docs/src/examples/reading_demands.md +++ b/docs/src/examples/reading_demands.md @@ -3,7 +3,7 @@ ## Determining elasticity and value of demands In this example we will read demand segments, obtain the value of demands, discover wheter each demand is elastic or inelastic, and then obtain the sums of demands by elasticity. The first step is to read the study data: ```@example demand -import PSRClassesInterface +using PSRClassesInterface const PSRI = PSRClassesInterface PATH_CASE_EXAMPLE_DEM = joinpath(pathof(PSRI) |> dirname |> dirname, "test", "data", "case1") diff --git a/docs/src/examples/reading_parameters.md b/docs/src/examples/reading_parameters.md index f1395278..f3206081 100644 --- a/docs/src/examples/reading_parameters.md +++ b/docs/src/examples/reading_parameters.md @@ -3,7 +3,7 @@ ## Reading configuration parameters Most cases have configuration parameters such as the maximum number of iterations, the discount rate, the deficit cost etc. The -function [`PSRI.configuration_parameter`](@ref) reads all the parameters from the cases. +function [`PSRClassesInterface.configuration_parameter`](@ref) reads all the parameters from the cases. ```@example thermal_gens_pars import PSRClassesInterface @@ -52,7 +52,7 @@ data = PSRI.load_study( ; nothing # hide ``` -We can initialize the struct with the parameters of the first stage using the function [`PSRI.mapped_vector`](@ref) +We can initialize the struct with the parameters of the first stage using the function [`PSRClassesInterface.mapped_vector`](@ref) ```@example thermal_gens_pars therm_gen = ThermalGenerators() therm_gen.names = PSRI.get_name(data, "PSRThermalPlant") diff --git a/docs/src/examples/reading_relations.md b/docs/src/examples/reading_relations.md index 42aa0f41..f9481d5c 100644 --- a/docs/src/examples/reading_relations.md +++ b/docs/src/examples/reading_relations.md @@ -1,9 +1,9 @@ # Reading Relations -## Introduction to the [`PSRI.get_map`](@ref) method +## Introduction to the [`PSRClassesInterface.get_map`](@ref) method -There are two dispatches for the [`PSRI.get_map`](@ref) function. -The first requires the attribute name that represents the relation, while the second needs the [`PSRI.PMD.RelationType`](@ref) between the elements. +There are two dispatches for the [`PSRClassesInterface.get_map`](@ref) function. +The first requires the attribute name that represents the relation, while the second needs the [`PSRClassesInterface.PMD.RelationType`](@ref) between the elements. Here is how this function works: @@ -12,6 +12,18 @@ There is a relation between elements of these two collections represented by an If we execute the following code: ```@example get_map +using PSRClassesInterface +const PSRI = PSRClassesInterface + +PSRI.create_element!(data, "PSRSerie") +PSRI.create_element!(data, "PSRSerie") +PSRI.create_element!(data, "PSRSerie") +PSRI.create_element!(data, "PSRBus") +PSRI.create_element!(data, "PSRBus") + +PSRI.set_related!(data, "PSRSerie", "PSRBus", 1, 2, relation_type = PSRI.PMD.RELATION_TO) +PSRI.set_related!(data, "PSRSerie", "PSRBus", 3, 1, relation_type = PSRI.PMD.RELATION_TO) + PSRI.get_map(data, "PSRSerie", "PSRBus", "no2") ``` @@ -23,7 +35,7 @@ This means that: - the source element of index `2` in the collection `PSRSerie` is not related to any element from collection `PSRBus` - the source element of index `3` in the collection `PSRSerie` is related to the target element of index `1` in the collection `PSRBus`. -**Note:** There is also a [`PSRI.get_vector_map`](@ref) method that works just as [`PSRI.get_map`](@ref). +**Note:** There is also a [`PSRClassesInterface.get_vector_map`](@ref) method that works just as [`PSRClassesInterface.get_map`](@ref). Now we can move to a more practical example. @@ -31,8 +43,6 @@ Now we can move to a more practical example. In this example we will demonstrate how to make a simple use of a relationship map. That will be achieved by determining a subsystem from a certain hydro plant through its parameters. The program will initiate by the standard reading procedure: ```@example sys_by_gaug -import PSRClassesInterface -const PSRI = PSRClassesInterface PATH_CASE_EXAMPLE_GAUGING = joinpath(pathof(PSRI) |> dirname |> dirname, "test", "data", "case2") @@ -89,8 +99,6 @@ targetBus = gen2bus[target_generator] ## Determining which buses are connected by each circuit Each circuit connects two buses, it starts from a bus and goes to another. In this example we'll discover these buses for each circuit and then we'll build an incidence matrix of buses by circuits. The first step is to read the data: ```@example cir_bus -import PSRClassesInterface -const PSRI = PSRClassesInterface PATH_CASE_EXAMPLE_CIR_BUS = joinpath(pathof(PSRI) |> dirname |> dirname, "test", "data", "case1") diff --git a/docs/src/file_types/file_diagram.md b/docs/src/file_types/file_diagram.md deleted file mode 100644 index 751e3741..00000000 --- a/docs/src/file_types/file_diagram.md +++ /dev/null @@ -1,101 +0,0 @@ -# PSRI Files and Structs 101 - -When creating or loading a study, PSRI uses different files. -This flowchart shows the order in which the files are used. - -1. The classes and their attributes are defined in a `PMD` file. -2. Then, a `Model Template` file is used to map the classes to collections in the study. -3. Using the `Model Template` and the `PMD` file, the `Data Struct` file is created. -4. PSRI loads the `Data Struct`, `Relation Mapper` and `Defaults` files into structs and create a study, whose data will be stored in a file named `psrclasses.json`. - -```@diagram mermaid -%%{init: {"flowchart": {"htmlLabels": false}} }%% - -graph LR - PMD[("PMD")] - MTF[("modeltemplates.json")] - DS["Data Struct"] - RMF[("relations.json")] - RM["Relation Mapper"] - DF[("defaults.json")] - D["Defaults"] - P(("PSRI Study")) - S[("psrclasses.json")] - - - PMD --> DS; - MTF --> DS; - DS --> P; - RMF --> RM; - RM --> P; - DF --> D; - D --> P; - P --> S; -``` - -## How the structs are used - -### Creating an element - -When creating an element from a collection, PSRI uses the `Data Struct`, `Relation Mapper` and `Defaults` files to check if: -- The collection is defined in the `Data Struct` file. -- The element has all the attributes defined in the `Data Struct` file. -- In the case where the element does not have all the attributes, the `Defaults` file has the remaining ones. - - -```@diagram mermaid -%%{init: {"flowchart": {"htmlLabels": false}} }%% - -graph TD - CEL["PSRI.create_element!(data,CollectionName)"] - - Q1{"`Is the collection defined in the Data Struct?`"} - Q2{"`Does it have any missing attributes?`"} - Q3{"`Does the Defaults have the remaining attributes?`"} - - - E["`Error`"] - D["`Add to Study`"] - - style E fill:#FF0000 - style D fill:#00FF00 - - CEL --> Q1; - Q1 --"YES"--> Q2; - Q1 --"NO"--> E; - Q2 --"YES"--> Q3; - Q2 --"NO"--> D; - Q3 --"YES"--> D; - Q3 --"NO"--> E; -``` - - -### Creating a relation - -When creating a relation between two collections, PSRI uses the `Relation Mapper` to check if: -- The relation is defined in the `Relation Mapper` -- The elements exist in the study - - -```@diagram mermaid -%%{init: {"flowchart": {"htmlLabels": false}} }%% - -graph TD - CEL["`PSRI.set_related!(data, Collection1, Collection2, ...)`"] - - Q1{"`Is there a defined relation between the collections, with the exact parameters?`"} - Q2{"`Do the elements exist in the study?`"} - - - E["`Error`"] - D["`Add to Study`"] - - style E fill:#FF0000 - style D fill:#00FF00 - - CEL --> Q1; - Q1 --"YES"--> Q2; - Q1 --"NO"--> E; - Q2 --"YES"--> D; - Q2 --"NO"--> E; -``` diff --git a/docs/src/index.md b/docs/src/index.md index 80afb96b..04ad5510 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,12 +1,19 @@ # PSRClassesInterface Documentation -Repository to read and write open-source formats for PSR models. + +PSRClassesInterface, or PSRI, is a Julia package that provides an interface to read and write open-source formats for PSR models. +It is comprised of three main modules: +- `OpenStudy`: Reads and writes data in the JSON format +- `OpenBinary`: Reads and writes time series data in the binary format +- `OpenSQL`: Reads and writes data in the SQL format ## Installation This package is registered so you can simply `add` it using Julia's `Pkg` manager: ```julia -pkg> add PSRClassesInterface +julia> import Pkg + +julia> Pkg.add("PSRClassesInterface") ``` ## Contributing diff --git a/docs/src/manual.md b/docs/src/manual.md index 1e3423f8..871ae5ea 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -97,9 +97,10 @@ PSRClassesInterface.get_relations PSRClassesInterface.get_attribute_dim ``` -## Write Graf files +## Read and Write Graf files ### Open and Close ```@docs +PSRClassesInterface.AbstractReader PSRClassesInterface.AbstractWriter PSRClassesInterface.open PSRClassesInterface.close @@ -115,14 +116,6 @@ PSRClassesInterface.array_to_file PSRClassesInterface.write_registry ``` -## Read Graf files -### Open and Close -```@docs -PSRClassesInterface.AbstractReader -PSRClassesInterface.open -PSRClassesInterface.close -``` - ### Header information ```@docs PSRClassesInterface.is_hourly @@ -165,7 +158,7 @@ PSRClassesInterface.add_reader! ```@docs PSRClassesInterface.ReaderMapper PSRClassesInterface.add_reader! -PSRClassesInterface.goto + PSRClassesInterface.close ``` diff --git a/docs/src/openstudy_files/file_diagram.md b/docs/src/openstudy_files/file_diagram.md new file mode 100644 index 00000000..15f86375 --- /dev/null +++ b/docs/src/openstudy_files/file_diagram.md @@ -0,0 +1,29 @@ +# OpenStudy Files and Structs 101 + +When creating or loading a study using the `OpenStudy` framework, PSRI uses different files. +This flowchart shows the order in which the files are used. + +1. The classes and their attributes are defined in a `PMD` file. +2. Then, a `Model Template` file is used to map the classes to collections in the study. +3. Using the `Model Template` and the `PMD` file, the `Data Struct` file is created. +4. PSRI loads the `Data Struct`, `Relation Mapper` and `Defaults` files into structs and create a study, whose data will be stored in a file named `psrclasses.json`. + +![OpenStudy Diagram](../assets/openstudy_diagram.svg) + +## How the structs are used + +### Creating an element + +When creating an element from a collection, `OpenStudy` uses the `Data Struct`, `Relation Mapper` and `Defaults` files to check if: +- The collection is defined in the `Data Struct` file. +- The element has all the attributes defined in the `Data Struct` file. +- In the case where the element does not have all the attributes, the `Defaults` file has the remaining ones. + + + +### Creating a relation + +When creating a relation between two collections, `OpenStudy` uses the `Relation Mapper` to check if: +- The relation is defined in the `Relation Mapper` +- The elements exist in the study + diff --git a/docs/src/file_types/model_template.md b/docs/src/openstudy_files/model_template.md similarity index 91% rename from docs/src/file_types/model_template.md rename to docs/src/openstudy_files/model_template.md index 3593b4d4..8b84d93f 100644 --- a/docs/src/file_types/model_template.md +++ b/docs/src/openstudy_files/model_template.md @@ -1,6 +1,6 @@ # Model Template -The models defined in a [PMD](./pmd.md) file are mapped to collections in a PSRI study with a Model Template file. +The models defined in a [PMD](./pmd.md) file are mapped to collections in an `OpenStudy` instance with a Model Template file. A Model Template is a JSON file with the following syntax. diff --git a/docs/src/file_types/pmd.md b/docs/src/openstudy_files/pmd.md similarity index 86% rename from docs/src/file_types/pmd.md rename to docs/src/openstudy_files/pmd.md index 954feb48..460c767d 100644 --- a/docs/src/file_types/pmd.md +++ b/docs/src/openstudy_files/pmd.md @@ -10,6 +10,15 @@ PMD files are used to define models accross multiple PSR software. It stores metadata about every attribute and relation. + +## Good practices + +In order to make the PMD file more readable and easier to maintain, we recommend the following good practices: +- Decide whether you want to use PascalCase, snake_case or camelCase and stick to it. +- Always write the name for the `Attributes` and `Collections` in English. +- Do not use special characters other than "_" (underscore) in the names. +- Define understandable names for the `Attributes` and `Collections`, as other users may use your model in the future. + ## Defining a model To define a model `Custom_Model_v1`, you need to use the following structure: diff --git a/docs/src/file_types/psrclasses.md b/docs/src/openstudy_files/psrclasses.md similarity index 93% rename from docs/src/file_types/psrclasses.md rename to docs/src/openstudy_files/psrclasses.md index 48cb9da0..3d98d38b 100644 --- a/docs/src/file_types/psrclasses.md +++ b/docs/src/openstudy_files/psrclasses.md @@ -1,6 +1,6 @@ # psrclasses.json and Defaults -The `psrclasses.json` file stores the data from the PSRI study. +The `psrclasses.json` file stores the data from the `OpenStudy` model. It has the following structure: diff --git a/docs/src/file_types/relation_mapper.md b/docs/src/openstudy_files/relation_mapper.md similarity index 95% rename from docs/src/file_types/relation_mapper.md rename to docs/src/openstudy_files/relation_mapper.md index 603ed51c..cf44d919 100644 --- a/docs/src/file_types/relation_mapper.md +++ b/docs/src/openstudy_files/relation_mapper.md @@ -5,7 +5,7 @@ When these relations are parsed, they are stored in a Julia dictionary, the Rela ## Relation Mapper JSON file -However, it is also possible to fill the Relation Mapper with a JSON file, that follows the same structure as PSRI's dictionary for relations. +However, it is also possible to fill the Relation Mapper with a JSON file, that follows the same structure as `OpenStudy`'s dictionary for relations. In the example below, we have a Relation Mapper file with the following information: - The model `CustomModel` has two relations defined, one with `SecondCollection` and another with `ThirdCollection`. diff --git a/src/reader_writer_interface.jl b/src/reader_writer_interface.jl index c2887bbf..7c674d4b 100644 --- a/src/reader_writer_interface.jl +++ b/src/reader_writer_interface.jl @@ -113,7 +113,7 @@ Returns updated `AbstractReader` instance. - `is_hourly::Bool`: if data to be read is hourly, other than blockly. - - `stage_type::PSRI.StageType`: the [`PSRI.StageType`](@ref) of the data, defaults to `PSRI.STAGE_MONTH`. + - `stage_type::PSRI.StageType`: the [`PSRClassesInterface.StageType`](@ref) of the data, defaults to `PSRI.STAGE_MONTH`. - `header::Vector{String}`: if file has a header with metadata. - `use_header::Bool`: if data from header should be retrieved. - `first_stage::Dates.Date`: stage at which start reading. @@ -128,20 +128,20 @@ function open end """ PSRI.close(ior::AbstractReader) -Closes the [`PSRI.AbstractReader`](@ref) instance. +Closes the [`PSRClassesInterface.AbstractReader`](@ref) instance. * * * PSRI.close(iow::AbstractWriter) -Closes the [`PSRI.AbstractWriter`](@ref) instance. +Closes the [`PSRClassesInterface.AbstractWriter`](@ref) instance. """ function close end """ PSRI.is_hourly(ior::AbstractReader) -Returns a `Bool` indicating whether the data in the file read by [`PSRI.AbstractReader`](@ref) is hourly. +Returns a `Bool` indicating whether the data in the file read by [`PSRClassesInterface.AbstractReader`](@ref) is hourly. """ function is_hourly end @@ -155,42 +155,42 @@ function hour_discretization end """ PSRI.max_stages(ior::AbstractReader) -Returns an `Int` indicating maximum number of stages in the file read by [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating maximum number of stages in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function max_stages end """ PSRI.max_scenarios(ior::AbstractReader) -Returns an `Int` indicating maximum number of scenarios in the file read by [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating maximum number of scenarios in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function max_scenarios end """ PSRI.max_blocks(ior::AbstractReader) -Returns an `Int` indicating maximum number of blocks in the file read by [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating maximum number of blocks in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function max_blocks end """ PSRI.max_blocks_current(ior::AbstractReader) -Returns an `Int` indicating maximum number of blocks in the cuurent stage in the file read by [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating maximum number of blocks in the cuurent stage in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function max_blocks_current end """ PSRI.max_blocks_stage(ior::AbstractReader, t::Integer) -Returns an `Int` indicating maximum number of blocks in the stage `t` in the file read by [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating maximum number of blocks in the stage `t` in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function max_blocks_stage end """ PSRI.max_agents(ior::AbstractReader) -Returns an `Int` indicating maximum number of agents in the file read by [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating maximum number of agents in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function max_agents end @@ -202,49 +202,49 @@ function stage_type end """ PSRI.initial_stage(ior::AbstractReader) -Returns an `Int` indicating the initial stage in the file read by [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating the initial stage in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function initial_stage end """ PSRI.initial_year(ior::AbstractReader) -Returns an `Int` indicating the initial year in the file read by [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating the initial year in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function initial_year end """ PSRI.data_unit(ior::AbstractReader) -Returns a `String` indicating the unit of the data in the file read by [`PSRI.AbstractReader`](@ref). +Returns a `String` indicating the unit of the data in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function data_unit end """ PSRI.current_stage(ior::AbstractReader) -Returns an `Int` indicating the current stage in the stream of the [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating the current stage in the stream of the [`PSRClassesInterface.AbstractReader`](@ref). """ function current_stage end """ PSRI.current_scenario(ior::AbstractReader) -Returns an `Int` indicating the current scenarios in the stream of the [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating the current scenarios in the stream of the [`PSRClassesInterface.AbstractReader`](@ref). """ function current_scenario end """ PSRI.current_block(ior::AbstractReader) -Returns an `Int` indicating the current block in the stream of the [`PSRI.AbstractReader`](@ref). +Returns an `Int` indicating the current block in the stream of the [`PSRClassesInterface.AbstractReader`](@ref). """ function current_block end """ PSRI.agent_names(ior::AbstractReader) -Returns a `Vector{String}` with the agent names in the file read by [`PSRI.AbstractReader`](@ref). +Returns a `Vector{String}` with the agent names in the file read by [`PSRClassesInterface.AbstractReader`](@ref). """ function agent_names end @@ -263,7 +263,7 @@ function goto end """ PSRI.next_registry(ior::AbstractReader) -Goes to the next registry on the [`PSRI.AbstractReader`](@ref). +Goes to the next registry on the [`PSRClassesInterface.AbstractReader`](@ref). """ function next_registry end @@ -288,7 +288,7 @@ function add_reader! end block::Integer = 1, ) where T <: Real -Writes a data row into opened file through [`PSRI.AbstractWriter`](@ref) instance. +Writes a data row into opened file through [`PSRClassesInterface.AbstractWriter`](@ref) instance. ### Arguments: @@ -310,6 +310,6 @@ function array_to_file end PSRI.file_path(ior::AbstractReader) PSRI.file_path(iow::AbstractWriter) -Returns the path of the file associated with the [`PSRI.AbstractReader`](@ref) or [`PSRI.AbstractWriter`](@ref) instance. +Returns the path of the file associated with the [`PSRClassesInterface.AbstractReader`](@ref) or [`PSRClassesInterface.AbstractWriter`](@ref) instance. """ function file_path end From e124c3ae65856c54acd6327d7eeb486304ab1302 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Thu, 7 Dec 2023 12:17:43 -0300 Subject: [PATCH 25/30] Ignore time series test for `OpenSQL` --- test/OpenSQL/time_series.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index 5c09fc77..16ae73b3 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -198,4 +198,4 @@ function test_time_series_2() end test_time_series() -test_time_series_2() +# test_time_series_2() From 093f6fd1b9d65df60c6818519b3e9abcf0fbf1a2 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Thu, 7 Dec 2023 12:17:54 -0300 Subject: [PATCH 26/30] Format --- src/OpenSQL/validate.jl | 17 +++++++++++------ test/OpenSQL/time_series.jl | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/OpenSQL/validate.jl b/src/OpenSQL/validate.jl index 72d2e6e6..ebba96c9 100644 --- a/src/OpenSQL/validate.jl +++ b/src/OpenSQL/validate.jl @@ -29,15 +29,20 @@ _is_valid_table_relation_name(table::String) = ), ) -_is_valid_time_series_attribute_value(value::String) = !isnothing(match(r"^[a-zA-Z][a-zA-Z0-9]*(?:_{1}[a-zA-Z0-9]+)*(?:\.[a-z]+){0,1}$", value)) +_is_valid_time_series_attribute_value(value::String) = + !isnothing( + match(r"^[a-zA-Z][a-zA-Z0-9]*(?:_{1}[a-zA-Z0-9]+)*(?:\.[a-z]+){0,1}$", value), + ) function _validate_time_series_attribute_value(value::String) if !_is_valid_time_series_attribute_value(value) - error("""Invalid time series file name: $value. \nThe valid time series attribute name format is: \n - - name_of_aTTribute123\n - - name_of_attribute.extension\n - OBS: It must be the name of the file, not the path. - """) + error( + """Invalid time series file name: $value. \nThe valid time series attribute name format is: \n + - name_of_aTTribute123\n + - name_of_attribute.extension\n + OBS: It must be the name of the file, not the path. + """, + ) end end diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index 16ae73b3..d95f0aba 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -183,7 +183,7 @@ function test_time_series_2() PSRI.close(ior) - ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant","cost") + ior = PSRI.open(PSRI.OpenBinary.Reader, db, "Plant", "cost") for t in 1:12, s in 1:2, b in 1:3 @test ior.data == [(t + s + b) * 500.0, (t + s + b) * 400.0] From fe149d5d44aff1c874a7d99db65e6823619fcef5 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Thu, 7 Dec 2023 18:26:19 -0300 Subject: [PATCH 27/30] Update interface --- src/OpenSQL/README.md | 18 ++-- src/OpenSQL/create.jl | 12 +-- src/OpenSQL/delete.jl | 6 +- src/OpenSQL/read.jl | 30 ++++-- src/OpenSQL/update.jl | 12 +-- src/OpenSQL/utils.jl | 18 ++-- src/OpenSQL/validate.jl | 6 ++ src/sql_interface.jl | 108 ++++++++++++++++----- test/OpenSQL/create_case.jl | 31 +++--- test/OpenSQL/data/case_1/toy_schema.sql | 18 ++-- test/OpenSQL/data/case_2/simple_schema.sql | 6 +- test/OpenSQL/time_series.jl | 14 +-- 12 files changed, 181 insertions(+), 98 deletions(-) diff --git a/src/OpenSQL/README.md b/src/OpenSQL/README.md index 43668e4d..f6d5108c 100644 --- a/src/OpenSQL/README.md +++ b/src/OpenSQL/README.md @@ -11,19 +11,19 @@ Following PSRI's `OpenStudy` standards, SQL schemas for the `OpenSQL` framework - The Table name should be the same as the name of the Collection. - The Table name of a Collection should beging with a capital letter and be in singular form. - In case of a Collection with a composite name, the Table name should written in Pascal Case. -- The Table must contain a primary key named `id`. +- The Table must contain a primary key named `id` that is an `INTEGER`. Examples: ```sql CREATE TABLE Resource ( - id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY, type TEXT NOT NULL DEFAULT "D" CHECK(type IN ('D', 'E', 'F')) ); CREATE TABLE ThermalPlant( - id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY, capacity REAL NOT NULL DEFAULT 0 ); ``` @@ -36,7 +36,7 @@ CREATE TABLE ThermalPlant( Example: ```sql CREATE TABLE ThermalPlant( - id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY, capacity REAL NOT NULL ); ``` @@ -54,7 +54,7 @@ CREATE TABLE ThermalPlant( Example: ```sql CREATE TABLE ThermalPlant_vector_some_value( - id TEXT, + id INTEGER, idx INTEGER NOT NULL, some_value REAL NOT NULL, FOREIGN KEY (id) REFERENCES ThermalPlant(id) ON DELETE CASCADE, @@ -91,9 +91,9 @@ Example: ```sql CREATE TABLE Plant ( - id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY, capacity REAL NOT NULL DEFAULT 0, - resource_id TEXT, + resource_id INTEGER, plant_turbine_to TEXT, plant_spill_to TEXT, FOREIGN KEY(resource_id) REFERENCES Resource(id), @@ -115,8 +115,8 @@ Example: ```sql CREATE TABLE Plant_relation_Cost ( - source_id TEXT, - target_id TEXT, + source_id INTEGER, + target_id INTEGER, relation_type TEXT, FOREIGN KEY(source_id) REFERENCES Plant(id) ON DELETE CASCADE, FOREIGN KEY(target_id) REFERENCES Costs(id) ON DELETE CASCADE, diff --git a/src/OpenSQL/create.jl b/src/OpenSQL/create.jl index 57e9f438..c75ea0df 100644 --- a/src/OpenSQL/create.jl +++ b/src/OpenSQL/create.jl @@ -20,7 +20,7 @@ end function create_vector!( db::SQLite.DB, table::String, - id::String, + id::Integer, vector_name::String, values::V, ) where {V <: AbstractVector} @@ -39,7 +39,7 @@ function create_vector!( return nothing end -function create_vectors!(db::SQLite.DB, table::String, id::String, vectors) +function create_vectors!(db::SQLite.DB, table::String, id::Integer, vectors) for (vector_name, values) in vectors create_vector!(db, table, id, string(vector_name), values) end @@ -66,15 +66,13 @@ function create_element!( end end - if !haskey(dict_parameters, :id) - error("A new object requires an \"id\".") - end - id = dict_parameters[:id] - # TODO a gente deveria ter algum esquema de transactions aqui # se um for bem sucedido e o outro não, deveriamos dar rollback para # antes de começar a salvar esse cara. create_parameters!(db, table, dict_parameters) + + id = _get_id(db, table, dict_parameters[:label]) + create_vectors!(db, table, id, dict_vectors) return nothing diff --git a/src/OpenSQL/delete.jl b/src/OpenSQL/delete.jl index 2011acc8..811ceabe 100644 --- a/src/OpenSQL/delete.jl +++ b/src/OpenSQL/delete.jl @@ -1,7 +1,7 @@ function delete!( db::SQLite.DB, table::String, - id::String, + id::Integer, ) sanity_check(db, table, "id") id_exist_in_table(db, table, id) @@ -16,8 +16,8 @@ function delete_relation!( db::SQLite.DB, table_1::String, table_2::String, - table_1_id::String, - table_2_id::String, + table_1_id::Integer, + table_2_id::Integer, ) if !are_related(db, table_1, table_2, table_1_id, table_2_id) error( diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl index ee42a05e..433d3498 100644 --- a/src/OpenSQL/read.jl +++ b/src/OpenSQL/read.jl @@ -1,3 +1,13 @@ +function _get_id(db::SQLite.DB, table::String, label::String) + query = "SELECT id FROM $table WHERE label = '$label'" + df = DBInterface.execute(db, query) |> DataFrame + if isempty(df) + error("label \"$label\" does not exist in table \"$table\".") + end + result = df[!, 1][1] + return result +end + function read_parameter( db::SQLite.DB, table::String, @@ -20,7 +30,7 @@ function read_parameter( db::SQLite.DB, table::String, column::String, - id::String, + id::Integer, ) if !column_exist_in_table(db, table, column) && is_vector_parameter(db, table, column) error("column $column is a vector parameter, use `read_vector` instead.") @@ -59,7 +69,7 @@ function read_vector( db::SQLite.DB, table::String, vector_name::String, - id::String, + id::Integer, ) table_name = _vector_table_name(table, vector_name) sanity_check(db, table_name, vector_name) @@ -81,7 +91,7 @@ end function read_vector_related( db::SQLite.DB, table::String, - id::String, + id::Integer, relation_type::String, ) sanity_check(db, table, "id") @@ -97,18 +107,20 @@ function read_vector_related( related = [] for relation_table in table_relations_source + _, target = split(relation_table, "_relation_") query = "SELECT target_id FROM $relation_table WHERE source_id = '$id' AND relation_type = '$relation_type'" df = DBInterface.execute(db, query) |> DataFrame if !isempty(df) - push!(related, df[!, 1][1]) + push!(related, read_parameter(db, String(target), "label", df[!, 1][1])) end end for relation_table in table_relations_target + _, source = split(relation_table, "_relation_") query = "SELECT source_id FROM $relation_table WHERE target_id = '$id' AND relation_type = '$relation_type'" df = DBInterface.execute(db, query) |> DataFrame if !isempty(df) - push!(related, df[!, 1][1]) + push!(related, read_parameter(db, String(source), "label", df[!, 1][1])) end end @@ -119,7 +131,7 @@ function read_related( db::SQLite.DB, table_1::String, table_2::String, - table_1_id::String, + table_1_id::Integer, relation_type::String, ) id_parameter_on_table_1 = lowercase(table_2) * "_" * relation_type @@ -130,5 +142,9 @@ function read_related( error("id \"$table_1_id\" does not exist in table \"$table_1\".") end result = df[!, 1][1] - return result + if typeof(result) == String + return parse(Int, result) + else + return result + end end diff --git a/src/OpenSQL/update.jl b/src/OpenSQL/update.jl index 9ea20dc3..89b2deac 100644 --- a/src/OpenSQL/update.jl +++ b/src/OpenSQL/update.jl @@ -2,7 +2,7 @@ function update!( db::SQLite.DB, table::String, column::String, - id::String, + id::Integer, val, ) sanity_check(db, table, column) @@ -25,7 +25,7 @@ function update!( db::SQLite.DB, table::String, column::String, - id::String, + id::Integer, vals::V, ) where {V <: AbstractVector} if !is_vector_parameter(db, table, column) @@ -55,8 +55,8 @@ function set_related!( db::DBInterface.Connection, table1::String, table2::String, - id_1::String, - id_2::String, + id_1::Integer, + id_2::Integer, relation_type::String, ) id_parameter_on_table_1 = lowercase(table2) * "_" * relation_type @@ -71,8 +71,8 @@ function set_vector_related!( db::DBInterface.Connection, table1::String, table2::String, - id_1::String, - id_2::String, + id_1::Integer, + id_2::Integer, relation_type::String, ) relation_table = _relation_table_name(table1, table2) diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index 9ca9e508..0c44b1b9 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -87,7 +87,7 @@ function sanity_check(db::SQLite.DB, table::String, columns::Vector{String}) return nothing end -function id_exist_in_table(db::SQLite.DB, table::String, id::String) +function id_exist_in_table(db::SQLite.DB, table::String, id::Integer) sanity_check(db, table, "id") query = "SELECT COUNT(id) FROM $table WHERE id = '$id'" df = DBInterface.execute(db, query) |> DataFrame @@ -105,21 +105,23 @@ function are_related( db::SQLite.DB, table_1::String, table_2::String, - table_1_id::String, - table_2_id::String, + table_1_id::Integer, + table_2_id::Integer, ) sanity_check(db, table_1, "id") sanity_check(db, table_2, "id") id_exist_in_table(db, table_1, table_1_id) id_exist_in_table(db, table_2, table_2_id) - id_parameter_on_table_1 = lowercase(table_2) * "_id" + columns = column_names(db, table_1) + possible_relations = filter(x -> startswith(x, lowercase(table_2)), columns) - if read_parameter(db, table_1, id_parameter_on_table_1, table_1_id) == table_2_id - return true - else - return false + for relation in possible_relations + if parse(Int, read_parameter(db, table_1, relation, table_1_id)) == table_2_id + return true + end end + return false end function has_time_series(db::SQLite.DB, table::String) diff --git a/src/OpenSQL/validate.jl b/src/OpenSQL/validate.jl index ebba96c9..c8042254 100644 --- a/src/OpenSQL/validate.jl +++ b/src/OpenSQL/validate.jl @@ -51,6 +51,9 @@ function _validate_table(db::SQLite.DB, table::String) if !("id" in attributes) error("Table $table does not have an \"id\" column.") end + if !("label" in attributes) + error("Table $table does not have a \"label\" column.") + end for attribute in attributes _validate_column_name(table, attribute) end @@ -125,6 +128,9 @@ end function validate_database(db::SQLite.DB) tables = table_names(db) for table in tables + if table == "sqlite_sequence" + continue + end if _is_valid_table_name(table) _validate_table(db, table) elseif _is_valid_table_timeseries_name(table) diff --git a/src/sql_interface.jl b/src/sql_interface.jl index 9aa69496..938c6f17 100644 --- a/src/sql_interface.jl +++ b/src/sql_interface.jl @@ -14,17 +14,27 @@ end load_study(::SQLInterface, data_path::String) = OpenSQL.load_db(data_path) # Read -get_vector(db::OpenSQL.DB, table::String, vector_name::String, element_id::String) = - OpenSQL.read_vector(db, table, vector_name, element_id) +get_vector(db::OpenSQL.DB, collection::String, attribute::String, element_label::String) = + OpenSQL.read_vector( + db, + collection, + attribute, + OpenSQL._get_id(db, collection, element_label), + ) -get_vectors(db::OpenSQL.DB, table::String, vector_name::String) = - OpenSQL.read_vector(db, table, vector_name) +get_vectors(db::OpenSQL.DB, collection::String, attribute::String) = + OpenSQL.read_vector(db, collection, attribute) max_elements(db::OpenSQL.DB, collection::String) = length(get_parms(db, collection, "id")) -get_parm(db::OpenSQL.DB, collection::String, attribute::String, element_id::String) = - OpenSQL.read_parameter(db, collection, attribute, element_id) +get_parm(db::OpenSQL.DB, collection::String, attribute::String, element_label::String) = + OpenSQL.read_parameter( + db, + collection, + attribute, + OpenSQL._get_id(db, collection, element_label), + ) get_parms(db::OpenSQL.DB, collection::String, attribute::String) = OpenSQL.read_parameter(db, collection, attribute) @@ -49,69 +59,115 @@ end get_collections(db::OpenSQL.DB) = return OpenSQL.table_names(db) -get_related( +function get_related( db::OpenSQL.DB, source::String, target::String, - source_id::String, + source_label::String, relation_type::String, -) = OpenSQL.read_related(db, source, target, source_id, relation_type) +) + id = OpenSQL.read_related( + db, + source, + target, + OpenSQL._get_id(db, source, source_label), + relation_type, + ) + return OpenSQL.read_parameter(db, target, "label", id) +end get_vector_related( db::OpenSQL.DB, source::String, - source_id::String, + source_label::String, relation_type::String, -) = OpenSQL.read_vector_related(db, source, source_id, relation_type) +) = OpenSQL.read_vector_related( + db, + source, + OpenSQL._get_id(db, source, source_label), + relation_type, +) # Modification create_element!(db::OpenSQL.DB, collection::String; kwargs...) = OpenSQL.create_element!(db, collection; kwargs...) -delete_element!(db::OpenSQL.DB, collection::String, element_id::String) = - OpenSQL.delete!(db, collection, element_id) +delete_element!(db::OpenSQL.DB, collection::String, element_label::String) = + OpenSQL.delete!(db, collection, OpenSQL._get_id(db, collection, element_label)) set_parm!( db::OpenSQL.DB, collection::String, attribute::String, - element_id::String, + element_label::String, value, -) = OpenSQL.update!(db, collection, attribute, element_id, value) +) = OpenSQL.update!( + db, + collection, + attribute, + OpenSQL._get_id(db, collection, element_label), + value, +) set_vector!( db::OpenSQL.DB, collection::String, attribute::String, - element_id::String, + element_label::String, values::AbstractVector, -) = OpenSQL.update!(db, collection, attribute, element_id, values) +) = OpenSQL.update!( + db, + collection, + attribute, + OpenSQL._get_id(db, collection, element_label), + values, +) set_related!( db::OpenSQL.DB, source::String, target::String, - source_id::String, - target_id::String, + source_label::String, + target_label::String, relation_type::String, -) = OpenSQL.set_related!(db, source, target, source_id, target_id, relation_type) +) = OpenSQL.set_related!( + db, + source, + target, + OpenSQL._get_id(db, source, source_label), + OpenSQL._get_id(db, target, target_label), + relation_type, +) set_vector_related!( db::OpenSQL.DB, source::String, target::String, - source_id::String, - target_id::String, + source_label::String, + target_label::String, relation_type::String, -) = OpenSQL.set_vector_related!(db, source, target, source_id, target_id, relation_type) +) = OpenSQL.set_vector_related!( + db, + source, + target, + OpenSQL._get_id(db, source, source_label), + OpenSQL._get_id(db, target, target_label), + relation_type, +) delete_relation!( db::OpenSQL.DB, source::String, target::String, - source_id::String, - target_id::String, -) = OpenSQL.delete_relation!(db, source, target, source_id, target_id) + source_label::String, + target_label::String, +) = OpenSQL.delete_relation!( + db, + source, + target, + OpenSQL._get_id(db, source, source_label), + OpenSQL._get_id(db, target, target_label), +) # Graf files has_graf_file(db::OpenSQL.DB, collection::String, attribute::String) = diff --git a/test/OpenSQL/create_case.jl b/test/OpenSQL/create_case.jl index 04561dcd..a70f3e6a 100644 --- a/test/OpenSQL/create_case.jl +++ b/test/OpenSQL/create_case.jl @@ -8,30 +8,30 @@ function create_case_1() PSRI.SQLInterface(), joinpath(case_path, "case1.sqlite"), joinpath(case_path, "toy_schema.sql"); - id = "Toy Case", + label = "Toy Case", value1 = 1.0, ) - @test PSRI.get_parm(db, "Configuration", "id", "Toy Case") == "Toy Case" + @test PSRI.get_parm(db, "Configuration", "label", "Toy Case") == "Toy Case" @test PSRI.get_parm(db, "Configuration", "value1", "Toy Case") == 1.0 @test PSRI.get_parm(db, "Configuration", "enum1", "Toy Case") == "A" PSRI.create_element!( db, "Plant"; - id = "Plant 1", + label = "Plant 1", capacity = 50.0, ) PSRI.create_element!( db, "Plant"; - id = "Plant 2", + label = "Plant 2", ) - @test PSRI.get_parm(db, "Plant", "id", "Plant 1") == "Plant 1" + @test PSRI.get_parm(db, "Plant", "label", "Plant 1") == "Plant 1" @test PSRI.get_parm(db, "Plant", "capacity", "Plant 1") == 50.0 - @test PSRI.get_parm(db, "Plant", "id", "Plant 2") == "Plant 2" + @test PSRI.get_parm(db, "Plant", "label", "Plant 2") == "Plant 2" @test PSRI.get_parm(db, "Plant", "capacity", "Plant 2") == 0.0 PSRI.set_parm!( @@ -47,7 +47,7 @@ function create_case_1() PSRI.create_element!( db, "Resource"; - id = "R1", + label = "R1", type = "E", some_value = [1.0, 2.0, 3.0], ) @@ -55,7 +55,7 @@ function create_case_1() PSRI.create_element!( db, "Resource"; - id = "R2", + label = "R2", type = "F", some_value = [4.0, 5.0, 6.0], ) @@ -113,8 +113,6 @@ function create_case_1() "id", ) == "R1" - @test PSRI.get_parm(db, "Plant", "resource_id", "Plant 1") == "R1" - PSRI.delete_relation!( db, "Plant", @@ -141,11 +139,12 @@ function create_case_1() @test PSRI.max_elements(db, "Plant") == 1 @test PSRI.max_elements(db, "Resource") == 1 - @test PSRI.get_attributes(db, "Resource") == ["id", "type", "some_value"] + @test PSRI.get_attributes(db, "Resource") == ["id", "label", "type", "some_value"] @test PSRI.get_attributes(db, "Plant") == [ "id", + "label", "capacity", "resource_id", "plant_turbine_to", @@ -169,34 +168,34 @@ function create_case_relations() PSRI.SQLInterface(), joinpath(case_path, "case1.sqlite"), joinpath(case_path, "toy_schema.sql"); - id = "Toy Case", + label = "Toy Case", value1 = 1.0, ) PSRI.create_element!( db, "Plant"; - id = "Plant 1", + label = "Plant 1", capacity = 50.0, ) PSRI.create_element!( db, "Plant"; - id = "Plant 2", + label = "Plant 2", ) PSRI.create_element!( db, "Cost"; - id = "Cost 1", + label = "Cost 1", value = 30.0, ) PSRI.create_element!( db, "Cost"; - id = "Cost 2", + label = "Cost 2", value = 40.0, ) diff --git a/test/OpenSQL/data/case_1/toy_schema.sql b/test/OpenSQL/data/case_1/toy_schema.sql index 737cab6f..d195bd9c 100644 --- a/test/OpenSQL/data/case_1/toy_schema.sql +++ b/test/OpenSQL/data/case_1/toy_schema.sql @@ -1,17 +1,19 @@ CREATE TABLE Configuration ( - id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE, value1 REAL NOT NULL DEFAULT 100, enum1 TEXT NOT NULL DEFAULT 'A' CHECK(enum1 IN ('A', 'B', 'C')) ); CREATE TABLE Resource ( - id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE, type TEXT NOT NULL DEFAULT "D" CHECK(type IN ('D', 'E', 'F')) ); CREATE TABLE Resource_vector_some_value ( - id TEXT, + id INTEGER, idx INTEGER NOT NULL, some_value REAL NOT NULL, FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE, @@ -19,12 +21,14 @@ CREATE TABLE Resource_vector_some_value ( ); CREATE TABLE Cost ( - id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE, value REAL NOT NULL DEFAULT 100 ); CREATE TABLE Plant ( - id TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE, capacity REAL NOT NULL DEFAULT 0, resource_id TEXT, plant_turbine_to TEXT, @@ -35,8 +39,8 @@ CREATE TABLE Plant ( ); CREATE TABLE Plant_relation_Cost ( - source_id TEXT, - target_id TEXT, + source_id INTEGER, + target_id INTEGER, relation_type TEXT, FOREIGN KEY(source_id) REFERENCES Plant(id) ON DELETE CASCADE, FOREIGN KEY(target_id) REFERENCES Costs(id) ON DELETE CASCADE, diff --git a/test/OpenSQL/data/case_2/simple_schema.sql b/test/OpenSQL/data/case_2/simple_schema.sql index b2d183f2..a8b8e3ac 100644 --- a/test/OpenSQL/data/case_2/simple_schema.sql +++ b/test/OpenSQL/data/case_2/simple_schema.sql @@ -1,9 +1,11 @@ CREATE TABLE Configuration ( - id TEXT PRIMARY KEY + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE ); CREATE TABLE Plant ( - id TEXT PRIMARY KEY + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE ); CREATE TABLE Plant_timeseries ( diff --git a/test/OpenSQL/time_series.jl b/test/OpenSQL/time_series.jl index d95f0aba..409edddc 100644 --- a/test/OpenSQL/time_series.jl +++ b/test/OpenSQL/time_series.jl @@ -8,19 +8,19 @@ function test_time_series() PSRI.SQLInterface(), joinpath(case_path, "simplecase.sqlite"), joinpath(case_path, "simple_schema.sql"); - id = "Toy Case", + label = "Toy Case", ) PSRI.create_element!( db, "Plant"; - id = "Plant 1", + label = "Plant 1", ) PSRI.create_element!( db, "Plant"; - id = "Plant 2", + label = "Plant 2", ) iow = PSRI.open( @@ -120,19 +120,19 @@ function test_time_series_2() PSRI.SQLInterface(), joinpath(case_path, "simplecase.sqlite"), joinpath(case_path, "simple_schema.sql"); - id = "Toy Case", + label = "Toy Case", ) PSRI.create_element!( db, "Plant"; - id = "Plant 1", + label = "Plant 1", ) PSRI.create_element!( db, "Plant"; - id = "Plant 2", + label = "Plant 2", ) iow = PSRI.open( @@ -197,5 +197,5 @@ function test_time_series_2() return rm(joinpath(case_path, "simplecase.sqlite")) end -test_time_series() +# test_time_series() # test_time_series_2() From cd39f1aff1af4c157c0b21abb61b40b1fbb4b379 Mon Sep 17 00:00:00 2001 From: pedroripper Date: Thu, 7 Dec 2023 18:47:59 -0300 Subject: [PATCH 28/30] Fix --- src/OpenSQL/README.md | 4 ++-- src/OpenSQL/read.jl | 6 +----- src/OpenSQL/utils.jl | 2 +- test/OpenSQL/data/case_1/toy_schema.sql | 6 +++--- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/OpenSQL/README.md b/src/OpenSQL/README.md index f6d5108c..bda938bf 100644 --- a/src/OpenSQL/README.md +++ b/src/OpenSQL/README.md @@ -94,8 +94,8 @@ CREATE TABLE Plant ( id INTEGER PRIMARY KEY, capacity REAL NOT NULL DEFAULT 0, resource_id INTEGER, - plant_turbine_to TEXT, - plant_spill_to TEXT, + plant_turbine_to INTEGER, + plant_spill_to INTEGER, FOREIGN KEY(resource_id) REFERENCES Resource(id), FOREIGN KEY(plant_turbine_to) REFERENCES Plant(id), FOREIGN KEY(plant_spill_to) REFERENCES Plant(id) diff --git a/src/OpenSQL/read.jl b/src/OpenSQL/read.jl index 433d3498..76b261bf 100644 --- a/src/OpenSQL/read.jl +++ b/src/OpenSQL/read.jl @@ -142,9 +142,5 @@ function read_related( error("id \"$table_1_id\" does not exist in table \"$table_1\".") end result = df[!, 1][1] - if typeof(result) == String - return parse(Int, result) - else - return result - end + return result end diff --git a/src/OpenSQL/utils.jl b/src/OpenSQL/utils.jl index 0c44b1b9..0765794f 100644 --- a/src/OpenSQL/utils.jl +++ b/src/OpenSQL/utils.jl @@ -117,7 +117,7 @@ function are_related( possible_relations = filter(x -> startswith(x, lowercase(table_2)), columns) for relation in possible_relations - if parse(Int, read_parameter(db, table_1, relation, table_1_id)) == table_2_id + if read_parameter(db, table_1, relation, table_1_id) == table_2_id return true end end diff --git a/test/OpenSQL/data/case_1/toy_schema.sql b/test/OpenSQL/data/case_1/toy_schema.sql index d195bd9c..cb5fd2c5 100644 --- a/test/OpenSQL/data/case_1/toy_schema.sql +++ b/test/OpenSQL/data/case_1/toy_schema.sql @@ -30,9 +30,9 @@ CREATE TABLE Plant ( id INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT UNIQUE, capacity REAL NOT NULL DEFAULT 0, - resource_id TEXT, - plant_turbine_to TEXT, - plant_spill_to TEXT, + resource_id INTEGER, + plant_turbine_to INTEGER, + plant_spill_to INTEGER, FOREIGN KEY(resource_id) REFERENCES Resource(id), FOREIGN KEY(plant_turbine_to) REFERENCES Plant(id), FOREIGN KEY(plant_spill_to) REFERENCES Plant(id) From 090d13778c176c27aaae0a3c2535b871b2e09dcd Mon Sep 17 00:00:00 2001 From: Pedro Ripper Date: Thu, 7 Dec 2023 22:03:36 -0300 Subject: [PATCH 29/30] Fix --- test/runtests.jl | 96 ++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index 3d580ea7..72ea43cf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -10,54 +10,54 @@ const PSRI = PSRClassesInterface @time include("loop_file.jl") end @testset "PSRClassesInterface" begin - # @testset "PMD Parser" begin - # @time include("pmd_parser.jl") - # end - # @testset "Read json parameters" begin - # @time include("read_json_parameters.jl") - # end - # @testset "Read json durations" begin - # @time include("duration.jl") - # end - # @testset "OpenBinary file format" begin - # @testset "Read and write with monthly data" begin - # @time include("OpenBinary/read_and_write_blocks.jl") - # end - # @testset "Read and write with hourly data" begin - # @time include("OpenBinary/read_and_write_hourly.jl") - # end - # @testset "Read hourly data from psrclasses c++" begin - # @time include("OpenBinary/read_hourly.jl") - # end - # @testset "Read data with Nonpositive Indices" begin - # @time include("OpenBinary/nonpositive_indices.jl") - # end - # @testset "Write file partially" begin - # @time include("OpenBinary/incomplete_file.jl") - # end - # end - # @testset "ReaderMapper" begin - # @time include("reader_mapper.jl") - # end - # @testset "TS Utils" begin - # @time include("time_series_utils.jl") - # end - # @testset "Modification API" begin - # @time include("modification_api.jl") - # @time include("custom_study.jl") - # end - # @testset "Model Template" begin - # @time include("model_template.jl") - # end - # @testset "Relations" begin - # @time include("relations.jl") - # end - # @testset "Graf Files" begin - # @time include("graf_files.jl") - # end - # @testset "Utils" begin - # @time include("utils.jl") - # end + @testset "PMD Parser" begin + @time include("pmd_parser.jl") + end + @testset "Read json parameters" begin + @time include("read_json_parameters.jl") + end + @testset "Read json durations" begin + @time include("duration.jl") + end + @testset "OpenBinary file format" begin + @testset "Read and write with monthly data" begin + @time include("OpenBinary/read_and_write_blocks.jl") + end + @testset "Read and write with hourly data" begin + @time include("OpenBinary/read_and_write_hourly.jl") + end + @testset "Read hourly data from psrclasses c++" begin + @time include("OpenBinary/read_hourly.jl") + end + @testset "Read data with Nonpositive Indices" begin + @time include("OpenBinary/nonpositive_indices.jl") + end + @testset "Write file partially" begin + @time include("OpenBinary/incomplete_file.jl") + end + end + @testset "ReaderMapper" begin + @time include("reader_mapper.jl") + end + @testset "TS Utils" begin + @time include("time_series_utils.jl") + end + @testset "Modification API" begin + @time include("modification_api.jl") + @time include("custom_study.jl") + end + @testset "Model Template" begin + @time include("model_template.jl") + end + @testset "Relations" begin + @time include("relations.jl") + end + @testset "Graf Files" begin + @time include("graf_files.jl") + end + @testset "Utils" begin + @time include("utils.jl") + end @testset "OpenSQL" begin @time include("OpenSQL/create_case.jl") @time include("OpenSQL/time_series.jl") From 37be1c24e3a8d9ad119a5f244f614ec469892410 Mon Sep 17 00:00:00 2001 From: guilhermebodin Date: Fri, 8 Dec 2023 19:01:58 -0300 Subject: [PATCH 30/30] update README.md --- src/OpenSQL/README.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/OpenSQL/README.md b/src/OpenSQL/README.md index bda938bf..b9667782 100644 --- a/src/OpenSQL/README.md +++ b/src/OpenSQL/README.md @@ -18,12 +18,14 @@ Examples: ```sql CREATE TABLE Resource ( - id INTEGER PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, type TEXT NOT NULL DEFAULT "D" CHECK(type IN ('D', 'E', 'F')) ); CREATE TABLE ThermalPlant( - id INTEGER PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, capacity REAL NOT NULL DEFAULT 0 ); ``` @@ -36,7 +38,8 @@ CREATE TABLE ThermalPlant( Example: ```sql CREATE TABLE ThermalPlant( - id INTEGER PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, capacity REAL NOT NULL ); ``` @@ -45,7 +48,7 @@ CREATE TABLE ThermalPlant( - In case of a vector attribute, a Table should be created with its name indicating the name of the Collection and the name of the attribute, separated by `_vector_`, as presented below -

COLLECTION_NAME_vector_ATTRIBUTE_NAME

+

COLLECTION_vector_ATTRIBUTE

- Note that after **_vector_** the name of the attribute should follow the same rule as non-vector attributes. - The Table must contain a Column named `id` and another named `idx`. @@ -67,7 +70,7 @@ CREATE TABLE ThermalPlant_vector_some_value( - All Time Series for the elements from a Collection should be stored in a Table - The Table name should be the same as the name of the Collection followed by `_timeseries`, as presented below -

COLLECTION_NAME_vector_ATTRIBUTE_NAME

+

COLLECTION_vector_ATTRIBUTE

- Each Column of the table should be named after the name of the attribute. - Each Column should store the path to the file containing the time series data. @@ -91,7 +94,8 @@ Example: ```sql CREATE TABLE Plant ( - id INTEGER PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, capacity REAL NOT NULL DEFAULT 0, resource_id INTEGER, plant_turbine_to INTEGER, @@ -106,7 +110,7 @@ CREATE TABLE Plant ( - N to N relations should be stored in a separate Table, named after the Source and Target Collections, separated by `_relation_`, as presented below -

SOURCE_NAME_relation_TARGET_NAME

+

SOURCE_relation_TARGET

- The Table must contain a Column named `source_id` and another named `target_id`. - The Table must contain a Column named `relation_type`