diff --git a/.gitignore b/.gitignore index 54d595e3..177a3129 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ Manifest.toml *.out *.ok debug_psrclasses +*.gz *.sqlite \ No newline at end of file diff --git a/docs/make.jl b/docs/make.jl index 93577b7c..7ce61e5f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -23,6 +23,7 @@ makedocs(; "PSRDatabaseSQLite Overview" => String[ "psrdatabasesqlite/introduction.md", "psrdatabasesqlite/rules.md", + "psrdatabasesqlite/time_series.md", ], "OpenStudy and OpenBinary Examples" => String[ "examples/reading_parameters.md", diff --git a/docs/src/psrdatabasesqlite/rules.md b/docs/src/psrdatabasesqlite/rules.md index 3c241b4e..91a5d939 100644 --- a/docs/src/psrdatabasesqlite/rules.md +++ b/docs/src/psrdatabasesqlite/rules.md @@ -132,10 +132,10 @@ CREATE TABLE HydroPlant_vector_GaugingStation( ``` -### Time Series +### Time Series Files -- All Time Series for the elements from a Collection should be stored in a Table -- The Table name should be the same as the name of the Collection followed by `_timeseriesfiles`, as presented below +- All Time Series files for the elements from a Collection should be stored in a Table +- The Table name should be the same as the name of the Collection followed by `_time_series_files`, as presented below

COLLECTION_vector_ATTRIBUTE

@@ -145,12 +145,36 @@ CREATE TABLE HydroPlant_vector_GaugingStation( Example: ```sql -CREATE TABLE Plant_timeseriesfiles ( +CREATE TABLE Plant_time_series_files ( generation TEXT, cost TEXT ) STRICT; ``` +### Time Series +- Time Series stored in the database should be stored in a table with the name of the Collection followed by `_time_series_` and the name of the attribute group, as presented below. + +

COLLECTION_time_series_GROUP_OF_ATTRIBUTES

+ +Notice that it is quite similar to the vector attributes, but without the `vector_index` column. +Instead, a mandatory column named `date_time` should be created to store the date of the time series data. + +Example: + +```sql +CREATE TABLE Resource_time_series_group1 ( + id INTEGER, + date_time TEXT NOT NULL, + some_vector1 REAL, + some_vector2 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time) +) STRICT; +``` + +!!! tip +For more information on how to handle time series data, please refer to the [Time Series](./time_series.md) section. + ## Migrations Migrations are an important part of the `DatabaseSQLite` framework. They are used to update the database schema to a new version without the need to delete the database and create a new one from scratch. Migrations are defined by two separate `.sql` files that are stored in the `migrations` directory of the model. The first file is the `up` migration and it is used to update the database schema to a new version. The second file is the `down` migration and it is used to revert the changes made by the `up` migration. Migrations are stored in directories in the model and they have a specific naming convention. The name of the migration folder should be the number of the version (e.g. `/migrations/1/`). diff --git a/docs/src/psrdatabasesqlite/time_series.md b/docs/src/psrdatabasesqlite/time_series.md new file mode 100644 index 00000000..9013898e --- /dev/null +++ b/docs/src/psrdatabasesqlite/time_series.md @@ -0,0 +1,254 @@ +# Time Series + +It is possible to store time series data in your database. Time series in `PSRDatabaseSQLite` are very flexible. You can have missing values, and you can have sparse data. + +There is a specific table format that must be followed. Consider the following example: + +```sql +CREATE TABLE Resource ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL +) STRICT; + +CREATE TABLE Resource_time_series_group1 ( + id INTEGER, + date_time TEXT NOT NULL, + some_vector1 REAL, + some_vector2 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time) +) STRICT; +``` + +It is mandatory for a time series to be indexed by a `date_time` column with the following format: `YYYY-MM-DD HH:MM:SS`. You can use the `Dates.jl` package for handling this format. + +```julia +using Dates +date = DateTime(2024, 3, 1) # 2024-03-01T00:00:00 (March 1st, 2024) +``` + +Notice that in this example, there are two value columns `some_vector1` and `some_vector2`. You can have as many value columns as you want. You can also separate the time series data into different tables, by creating a table `Resource_time_series_group2` for example. + +It is also possible to add more dimensions to your time series, such as `block` and `scenario`. + +```sql +CREATE TABLE Resource_time_series_group2 ( + id INTEGER, + date_time TEXT NOT NULL, + block INTEGER NOT NULL, + some_vector3 REAL, + some_vector4 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time, block) +) STRICT; +``` + +## Rules + +Time series in `PSRDatabaseSQLite` are very flexible. You can have missing values, and you can have sparse data. + +If you are querying for a time series row entry that has a missing value, it first checks if there is a data with a `date_time` earlier than the queried `date_time`. If there is, it returns the value of the previous data. If there is no data earlier than the queried `date_time`, it returns a specified value according to the type of data you are querying. + +- For `Float64`, it returns `NaN`. +- For `Int64`, it returns `typemin(Int)`. +- For `String`, it returns `""` (empty String). +- For `DateTime`, it returns `typemin(DateTime)`. + +For example, if you have the following data: + +| **Date** | **some_vector1(Float64)** | **some_vector2(Float64)** | +|:--------:|:-----------:|:-----------:| +| 2020 | 1.0 | missing | +| 2021 | missing | 1.0 | +| 2022 | 3.0 | missing | + +1. If you query for `some_vector1` at `2020`, it returns `1.0`. +2. If you query for `some_vector2` at `2020`, it returns `NaN`. +3. If you query for `some_vector1` at `2021`, it returns `1.0`. +4. If you query for `some_vector2` at `2021`, it returns `1.0`. +5. If you query for `some_vector1` at `2022`, it returns `3.0`. +6. If you query for `some_vector2` at `2022`, it returns `1.0`. + + +## Inserting data + +When creating a new element that has a time series, you can pass this information via a `DataFrame`. Consider the collection `Resource` with the two time series tables `Resource_time_series_group1` and `Resource_time_series_group2`. + +```julia +using DataFrames +using Dates +using PSRClassesInterface +PSRDatabaseSQLite = PSRClassesInterface.PSRDatabaseSQLite + +db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + +PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + +df_group1 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001), DateTime(2002)], + some_vector1 = [missing, 1.0, 2.0], + some_vector2 = [1.0, missing, 5.0], + ) + +df_group2 = DataFrame(; + date_time = [ + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2001), + DateTime(2001), + DateTime(2001), + DateTime(2009), + ], + block = [1, 1, 1, 1, 2, 2, 2, 2], + some_vector3 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4], + some_vector4 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4], + ) + + +PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df_group1, + group2 = df_group2, +) +``` + +It is also possible to insert a single row of a time series. This is useful when you want to insert a specific dimension entry. This way of inserting time series is less efficient than inserting a whole `DataFrame`. + +```julia +using DataFrames +using Dates +using PSRClassesInterface +PSRDatabaseSQLite = PSRClassesInterface.PSRDatabaseSQLite + +db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + +PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + +PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1" +) + +PSRDatabaseSQLite.add_time_series_row!( + db, + "Resource", + "some_vector1", + "Resource 1", + 10.0; # new value + date_time = DateTime(2000) +) + +PSRDatabaseSQLite.add_time_series_row!( + db, + "Resource", + "some_vector1", + "Resource 1", + 11.0; # new value + date_time = DateTime(2001) +) +``` + +## Reading data + +You can read the information from the time series in two different ways. + +### Reading as a table +First, you can read the whole time series table for a given value, as a `DataFrame`. + +```julia +df = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + "some_vector1", + "Resource 1", +) +``` + +### Reading a single row + +It is also possible to read a single row of the time series in the form of an array. This is useful when you want to query a specific dimension entry. +For this function, there are performance improvements when reading the data via caching the previous and next non-missing values. + +```julia +values = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1", + Float64; + date_time = DateTime(2020) +) +``` + +When querying a row, all values should non-missing. However, if there is a missing value, the function will return the previous non-missing value. And if even the previous value is missing, it will return a specified value according to the type of data you are querying. + + +- For `Float64`, it returns `NaN`. +- For `Int64`, it returns `typemin(Int)`. +- For `String`, it returns `""` (empty String). +- For `DateTime`, it returns `typemin(DateTime)`. + +For example, if you have the following data for the time series `some_vector1`: + +| **Date** | **Resource 1** | **Resource 2** | +|:--------:|:-----------:|:-----------:| +| 2020 | 1.0 | missing | +| 2021 | missing | 1.0 | +| 2022 | 3.0 | missing | + +1. If you query at `2020`, it returns `[1.0, NaN]`. +3. If you query at `2021`, it returns `[1.0, 1.0]`. +5. If you query at `2022`, it returns `[3.0, 1.0]`. + + +## Updating data + +When updating one of the entries of a time series for a given element and attribute, you need to specify the exact dimension values of the row you want to update. + + +For example, consider a time series that has `block` and `data_time` dimensions. + +```julia +PSRDatabaseSQLite.update_time_series_row!( + db, + "Resource", + "some_vector3", + "Resource 1", + 10.0; # new value + date_time = DateTime(2000), + block = 1 +) +``` + +## Deleting data + +You can delete the whole time series of an element for a given time series group. +Consider the following table: + +```sql +CREATE TABLE Resource_time_series_group1 ( + id INTEGER, + date_time TEXT NOT NULL, + some_vector1 REAL, + some_vector2 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time) +) STRICT; +``` + +This table represents a "group" that stores two time series `some_vector1` and `some_vector2`. You can delete all the data from this group by calling the following function: + +```julia +PSRDatabaseSQLite.delete_time_series!( + db, + "Resource", + "group1", + "Resource 1", +) +``` + +When trying to read a time series that has been deleted, the function will return an empty `DataFrame`. diff --git a/profiling/Project.toml b/profiling/Project.toml new file mode 100644 index 00000000..31ced8d1 --- /dev/null +++ b/profiling/Project.toml @@ -0,0 +1,3 @@ +[deps] +PProf = "e4faabce-9ead-11e9-39d9-4379958e3056" +Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" diff --git a/profiling/create_profile.jl b/profiling/create_profile.jl new file mode 100644 index 00000000..782c6406 --- /dev/null +++ b/profiling/create_profile.jl @@ -0,0 +1,12 @@ +# You should run the script from the profiling directory + +using Profile +using PProf +import Pkg +root_path = dirname(@__DIR__) +Pkg.activate(root_path) +using PSRClassesInterface + +include("../script_time_controller.jl") +@profile include("../script_time_controller.jl") +pprof() diff --git a/profiling/open_profile.jl b/profiling/open_profile.jl new file mode 100644 index 00000000..7199bfd7 --- /dev/null +++ b/profiling/open_profile.jl @@ -0,0 +1,8 @@ +# You should run the script from the profiling directory + +using Profile +using PProf + +file_name = "profile.pb.gz" + +PProf.refresh(; file = file_name, webport = 57998) diff --git a/src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl b/src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl index 284f1b8a..7bf589bf 100644 --- a/src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl +++ b/src/PSRDatabaseSQLite/PSRDatabaseSQLite.jl @@ -20,6 +20,7 @@ include("exceptions.jl") include("utils.jl") include("attribute.jl") include("collection.jl") +include("time_controller.jl") include("database_sqlite.jl") include("create.jl") include("read.jl") diff --git a/src/PSRDatabaseSQLite/attribute.jl b/src/PSRDatabaseSQLite/attribute.jl index 6cf2c1f9..41e92127 100644 --- a/src/PSRDatabaseSQLite/attribute.jl +++ b/src/PSRDatabaseSQLite/attribute.jl @@ -99,6 +99,18 @@ mutable struct VectorRelation{T} <: VectorAttribute end end +mutable struct TimeSeries{T} <: VectorAttribute + id::String + type::Type{T} + default_value::Union{Missing, T} + not_null::Bool + group_id::String + parent_collection::String + table_where_is_located::String + dimension_names::Vector{String} + num_dimensions::Int +end + mutable struct TimeSeriesFile{T} <: ReferenceToFileAttribute id::String type::Type{T} diff --git a/src/PSRDatabaseSQLite/collection.jl b/src/PSRDatabaseSQLite/collection.jl index 080275f8..5f092f6d 100644 --- a/src/PSRDatabaseSQLite/collection.jl +++ b/src/PSRDatabaseSQLite/collection.jl @@ -10,6 +10,7 @@ mutable struct Collection scalar_relations::OrderedDict{String, ScalarRelation} vector_parameters::OrderedDict{String, VectorParameter} vector_relations::OrderedDict{String, VectorRelation} + time_series::OrderedDict{String, TimeSeries} time_series_files::OrderedDict{String, TimeSeriesFile} end @@ -27,7 +28,8 @@ function _create_collections_map!( scalar_relations = _create_collection_scalar_relations(db, collection_id) vector_parameters = _create_collection_vector_parameters(db, collection_id) vector_relations = _create_collection_vector_relations(db, collection_id) - time_series = _get_collection_time_series(db, collection_id) + time_series = _create_collection_time_series(db, collection_id) + time_series_files = _create_collection_time_series_files(db, collection_id) collection = Collection( collection_id, scalar_parameters, @@ -35,6 +37,7 @@ function _create_collections_map!( vector_parameters, vector_relations, time_series, + time_series_files, ) collections_map[collection_id] = collection end @@ -159,7 +162,7 @@ function _create_collection_vector_parameters(db::SQLite.DB, collection_id::Stri not_null = Bool(vector_attribute.notnull) if haskey(vector_parameters, id) psr_database_sqlite_error( - "Duplicated vector parameter \"$name\" in collection \"$collection_id\"", + "Duplicated vector parameter \"$id\" in collection \"$collection_id\"", ) end vector_parameters[id] = VectorParameter( @@ -237,8 +240,77 @@ function _create_collection_vector_relations(db::SQLite.DB, collection_id::Strin return vector_relations end -function _get_collection_time_series(db::SQLite.DB, collection_id::String) - time_series_table = _get_collection_time_series_tables(db, collection_id) +function _get_time_series_dimension_names(df_table_infos::DataFrame) + dimension_names = Vector{String}(undef, 0) + for time_series_attribute in eachrow(df_table_infos) + if time_series_attribute.name == "id" + continue + end + if time_series_attribute.pk != 0 + push!(dimension_names, time_series_attribute.name) + end + end + return dimension_names +end + +function _create_collection_time_series(db::SQLite.DB, collection_id::String) + time_series_tables = _get_collection_time_series_tables(db, collection_id) + time_series = OrderedDict{String, TimeSeries}() + parent_collection = collection_id + for table_name in time_series_tables + group_id = _id_of_time_series_group(table_name) + table_where_is_located = table_name + df_table_infos = table_info(db, table_name) + dimension_names = _get_time_series_dimension_names(df_table_infos) + for time_series_attribute in eachrow(df_table_infos) + id = time_series_attribute.name + if id == "id" || id == "date_time" + # These are obligatory for every vector table + # and have no point in being stored in the database definition. + if time_series_attribute.pk == 0 + psr_database_sqlite_error( + "Invalid table \"$(table_name)\" of time_series attributes of collection \"$(collection_id)\". " * + "The column \"$(time_series_attribute.name)\" is not a primary key but it should.", + ) + end + continue + end + # There is no point in storing the other primary keys of these tables + if time_series_attribute.pk != 0 + if _sql_type_to_julia_type(id, time_series_attribute.type) != Int64 + psr_database_sqlite_error( + "Invalid table \"$(table_name)\" of time_series attributes of collection \"$(collection_id)\". " * + "The column \"$(time_series_attribute.name)\" is not an integer primary key but it should.", + ) + end + continue + end + type = _sql_type_to_julia_type(id, time_series_attribute.type) + default_value = _get_default_value(type, time_series_attribute.dflt_value) + not_null = Bool(time_series_attribute.notnull) + if haskey(time_series, id) + psr_database_sqlite_error( + "Duplicated time_series attribute \"$id\" in collection \"$collection_id\"", + ) + end + time_series[id] = TimeSeries( + id, + type, + default_value, + not_null, + group_id, + parent_collection, + table_where_is_located, + dimension_names, + length(dimension_names), + ) + end + end + return time_series +end + +function _create_collection_time_series_files(db::SQLite.DB, collection_id::String) + time_series_table = _get_collection_time_series_files_tables(db, collection_id) time_series = OrderedDict{String, TimeSeriesFile}() df_table_infos = table_info(db, time_series_table) for time_series_id in eachrow(df_table_infos) @@ -332,8 +404,28 @@ function _id_of_vector_group(table_name::String) return string(matches.captures[1]) end -function _get_collection_time_series_tables(::SQLite.DB, collection_id::String) - return string(collection_id, "_timeseriesfiles") +function _id_of_time_series_group(table_name::String) + matches = match(r"_time_series_(.*)", table_name) + return string(matches.captures[1]) +end + +function _get_collection_time_series_tables( + sqlite_db::SQLite.DB, + collection_id::String, +) + tables = SQLite.tables(sqlite_db) + time_series_tables = Vector{String}(undef, 0) + for table in tables + table_name = table.name + if _is_collection_time_series_table_name(table_name, collection_id) + push!(time_series_tables, table_name) + end + end + return time_series_tables +end + +function _get_collection_time_series_files_tables(::SQLite.DB, collection_id::String) + return string(collection_id, "_time_series_files") end function _validate_actions_on_foreign_key( diff --git a/src/PSRDatabaseSQLite/create.jl b/src/PSRDatabaseSQLite/create.jl index 27bfdd7a..4ac58ac7 100644 --- a/src/PSRDatabaseSQLite/create.jl +++ b/src/PSRDatabaseSQLite/create.jl @@ -21,6 +21,32 @@ function _create_scalar_attributes!( return nothing end +function _insert_vectors_from_df( + db::DatabaseSQLite, + df::DataFrame, + table_name::String, +) + # Code to insert rows without using a transaction + cols = join(string.(names(df)), ", ") + num_cols = size(df, 2) + for row in eachrow(df) + query = "INSERT INTO $table_name ($cols) VALUES (" + for (i, value) in enumerate(row) + if ismissing(value) + query *= "NULL, " + else + query *= "\'$value\', " + end + if i == num_cols + query = query[1:end-2] + query *= ")" + end + end + DBInterface.execute(db.sqlite_db, query) + end + return nothing +end + function _create_vector_group!( db::DatabaseSQLite, collection_id::String, @@ -51,23 +77,7 @@ function _create_vector_group!( vector_index = collect(1:num_values) DataFrames.insertcols!(df, 1, :vector_index => vector_index) DataFrames.insertcols!(df, 1, :id => ids) - cols = join(string.(names(df)), ", ") - num_cols = size(df, 2) - for row in eachrow(df) - query = "INSERT INTO $vectors_group_table_name ($cols) VALUES (" - for (i, value) in enumerate(row) - if ismissing(value) - query *= "NULL, " - else - query *= "\'$value\', " - end - if i == num_cols - query = query[1:end-2] - query *= ")" - end - end - DBInterface.execute(db.sqlite_db, query) - end + _insert_vectors_from_df(db, df, vectors_group_table_name) return nothing end @@ -103,6 +113,27 @@ function _create_vectors!( return nothing end +function _create_time_series!( + db::DatabaseSQLite, + collection_id::String, + id::Integer, + dict_time_series_attributes, +) + for (group, df) in dict_time_series_attributes + time_series_group_table_name = _time_series_group_table_name(collection_id, string(group)) + ids = fill(id, nrow(df)) + DataFrames.insertcols!(df, 1, :id => ids) + # Convert datetime column to string + df[!, :date_time] = string.(df[!, :date_time]) + # Add missing columns + missing_names_in_df = setdiff(_attributes_in_time_series_group(db, collection_id, string(group)), string.(names(df))) + for missing_attribute in missing_names_in_df + df[!, Symbol(missing_attribute)] = fill(missing, nrow(df)) + end + _insert_vectors_from_df(db, df, time_series_group_table_name) + end +end + function _create_element!( db::DatabaseSQLite, collection_id::String; @@ -111,7 +142,9 @@ function _create_element!( _throw_if_collection_does_not_exist(db, collection_id) dict_scalar_attributes = Dict{Symbol, Any}() dict_vector_attributes = Dict{Symbol, Any}() + dict_time_series_attributes = Dict{Symbol, Any}() + # Validate that the arguments will be valid for (key, value) in kwargs if isa(value, AbstractVector) _throw_if_not_vector_attribute(db, collection_id, string(key)) @@ -121,6 +154,15 @@ function _create_element!( ) end dict_vector_attributes[key] = value + elseif isa(value, DataFrame) + _throw_if_not_time_series_group(db, collection_id, string(key)) + _throw_if_data_does_not_match_group(db, collection_id, string(key), value) + if isempty(value) + psr_database_sqlite_error( + "Cannot create the time series group \"$key\" with an empty DataFrame.", + ) + end + dict_time_series_attributes[key] = value else _throw_if_is_time_series_file(db, collection_id, string(key)) _throw_if_not_scalar_attribute(db, collection_id, string(key)) @@ -146,6 +188,15 @@ function _create_element!( _create_vectors!(db, collection_id, id, dict_vector_attributes) end + if !isempty(dict_time_series_attributes) + id = get( + dict_scalar_attributes, + :id, + _get_id(db, collection_id, dict_scalar_attributes[:label]), + ) + _create_time_series!(db, collection_id, id, dict_time_series_attributes) + end + return nothing end @@ -272,3 +323,57 @@ function _validate_attribute_types_on_creation!( ) return nothing end + +function _add_time_series_row!( + db::DatabaseSQLite, + attribute::Attribute, + id::Integer, + val, + dimensions, +) + # Adding a time series element column by column as it is implemented on this function + # is not the most efficient way to do it. In any case if the user wants to add a time + # series column by column, this function can only be implemented as an upsert statements + # for each column. This is because the user can add a value in a primary key that already + # exists in the time series. In that case the column should be updated instead of inserted. + dimensions_string = join(keys(dimensions), ", ") + values_string = "$id, " + for dim in dimensions + values_string *= "'$(dim[2])', " + end + values_string *= "'$val'" + query = """ + INSERT INTO $(attribute.table_where_is_located) (id, $dimensions_string, $(attribute.id)) + VALUES ($values_string) + ON CONFLICT(id, $dimensions_string) DO UPDATE SET $(attribute.id) = '$val' + """ + DBInterface.execute(db.sqlite_db, query) + return nothing +end + +function add_time_series_row!( + db::DatabaseSQLite, + collection_id::String, + attribute_id::String, + label::String, + val; + dimensions..., +) + if !_is_time_series(db, collection_id, attribute_id) + psr_database_sqlite_error( + "The attribute $attribute_id is not a time series.", + ) + end + attribute = _get_attribute(db, collection_id, attribute_id) + id = _get_id(db, collection_id, label) + _validate_time_series_dimensions(collection_id, attribute, dimensions) + + if length(dimensions) != length(attribute.dimension_names) + psr_database_sqlite_error( + "The number of dimensions in the time series does not match the number of dimensions in the attribute. " * + "The attribute has $(attribute.num_dimensions) dimensions: $(join(attribute.dimension_names, ", ")).", + ) + end + + return _add_time_series_row!(db, attribute, id, val, dimensions) +end diff --git a/src/PSRDatabaseSQLite/database_sqlite.jl b/src/PSRDatabaseSQLite/database_sqlite.jl index cc221d9d..731e34ab 100644 --- a/src/PSRDatabaseSQLite/database_sqlite.jl +++ b/src/PSRDatabaseSQLite/database_sqlite.jl @@ -1,6 +1,41 @@ -mutable struct DatabaseSQLite +Base.@kwdef mutable struct DatabaseSQLite sqlite_db::SQLite.DB + database_path::String = "" collections_map::OrderedDict{String, Collection} + read_only::Bool = false + # TimeController is a cache that allows PSRDatabaseSQLite to + # store information about the last time_series query. This is useful for avoiding to + # re-query the database when the same query is made multiple times. + # The TimeController is a private behaviour and whenever it is used + # it changes the database mode to read-only. + _time_controller::TimeController = TimeController() +end + +_is_read_only(db::DatabaseSQLite) = db.read_only +function database_path(db::DatabaseSQLite) + return db.database_path +end + +function _set_default_pragmas!(db::SQLite.DB) + _set_foreign_keys_on!(db) + _set_busy_timeout!(db, 5000) + return nothing +end + +function _set_foreign_keys_on!(db::SQLite.DB) + # https://www.sqlite.org/foreignkeys.html#fk_enable + # Foreign keys are enabled per connection, they are not something + # that can be stored in the database itself like user_version. + # This is needed to ensure that the foreign keys are enabled + # behaviours like cascade delete and update are enabled. + DBInterface.execute(db, "PRAGMA foreign_keys = ON;") + return nothing +end + +function _set_busy_timeout!(db::SQLite.DB, timeout::Int) + # https://www.sqlite.org/pragma.html#pragma_busy_timeout + DBInterface.execute(db, "PRAGMA busy_timeout = $timeout;") + return nothing end function DatabaseSQLite_from_schema( @@ -9,7 +44,7 @@ function DatabaseSQLite_from_schema( ) sqlite_db = SQLite.DB(database_path) - DBInterface.execute(sqlite_db, "PRAGMA busy_timeout = 5000;") + _set_default_pragmas!(sqlite_db) collections_map = try execute_statements(sqlite_db, path_schema) @@ -20,8 +55,9 @@ function DatabaseSQLite_from_schema( rethrow(e) end - db = DatabaseSQLite( + db = DatabaseSQLite(; sqlite_db, + database_path, collections_map, ) @@ -34,7 +70,7 @@ function DatabaseSQLite_from_migrations( ) sqlite_db = SQLite.DB(database_path) - DBInterface.execute(sqlite_db, "PRAGMA busy_timeout = 5000;") + _set_default_pragmas!(sqlite_db) collections_map = try current_version = get_user_version(sqlite_db) @@ -54,8 +90,9 @@ function DatabaseSQLite_from_migrations( rethrow(e) end - db = DatabaseSQLite( + db = DatabaseSQLite(; sqlite_db, + database_path, collections_map, ) @@ -67,10 +104,9 @@ function DatabaseSQLite( read_only::Bool = false, ) sqlite_db = - read_only ? SQLite.DB("file:" * database_path * "?mode=ro&immutable=1") : - SQLite.DB(database_path) + read_only ? SQLite.DB("file:" * database_path * "?mode=ro&immutable=1") : SQLite.DB(database_path) - DBInterface.execute(sqlite_db, "PRAGMA busy_timeout = 5000;") + _set_default_pragmas!(sqlite_db) collections_map = try _validate_database(sqlite_db) @@ -80,9 +116,11 @@ function DatabaseSQLite( rethrow(e) end - db = DatabaseSQLite( + db = DatabaseSQLite(; sqlite_db, + database_path, collections_map, + read_only, ) return db end @@ -123,6 +161,29 @@ function _is_vector_relation( return haskey(collection.vector_relations, attribute_id) end +function _is_time_series( + db::DatabaseSQLite, + collection_id::String, + attribute_id::String, +) + collection = _get_collection(db, collection_id) + return haskey(collection.time_series, attribute_id) +end + +function _is_time_series_group( + db::DatabaseSQLite, + collection_id::String, + group_id::String, +) + collection = _get_collection(db, collection_id) + for (_, attribute) in collection.time_series + if attribute.group_id == group_id + return true + end + end + return false +end + function _is_time_series_file( db::DatabaseSQLite, collection_id::String, @@ -157,6 +218,8 @@ function _get_attribute( return collection.scalar_relations[attribute_id] elseif _is_vector_relation(db, collection_id, attribute_id) return collection.vector_relations[attribute_id] + elseif _is_time_series(db, collection_id, attribute_id) + return collection.time_series[attribute_id] elseif _is_time_series_file(db, collection_id, attribute_id) return collection.time_series_files[attribute_id] else @@ -206,6 +269,7 @@ function _attribute_exists( _is_vector_parameter(db, collection_id, attribute_id) || _is_scalar_relation(db, collection_id, attribute_id) || _is_vector_relation(db, collection_id, attribute_id) || + _is_time_series(db, collection_id, attribute_id) || _is_time_series_file(db, collection_id, attribute_id) end @@ -240,10 +304,29 @@ function _map_of_groups_to_vector_attributes( return map_of_groups_to_vector_attributes end +function _attributes_in_time_series_group( + db::DatabaseSQLite, + collection_id::String, + group_id::String, +) + collection = _get_collection(db, collection_id) + attributes_in_time_series_group = Vector{String}(undef, 0) + for (_, attribute) in collection.time_series + if attribute.group_id == group_id + push!(attributes_in_time_series_group, attribute.id) + end + end + return attributes_in_time_series_group +end + function _vectors_group_table_name(collection_id::String, group::String) return string(collection_id, "_vector_", group) end +function _time_series_group_table_name(collection_id::String, group::String) + return string(collection_id, "_time_series_", group) +end + function _is_collection_id(name::String) # Collections don't have underscores in their names return !occursin("_", name) @@ -253,6 +336,10 @@ function _is_collection_vector_table_name(name::String, collection_id::String) return startswith(name, "$(collection_id)_vector_") end +function _is_collection_time_series_table_name(name::String, collection_id::String) + return startswith(name, "$(collection_id)_time_series_") && !endswith(name, "_time_series_files") +end + _get_collection_ids(db::DatabaseSQLite) = collect(keys(db.collections_map)) function _get_collection_ids(db::SQLite.DB) tables = SQLite.tables(db) diff --git a/src/PSRDatabaseSQLite/delete.jl b/src/PSRDatabaseSQLite/delete.jl index ef3a7539..46f59de0 100644 --- a/src/PSRDatabaseSQLite/delete.jl +++ b/src/PSRDatabaseSQLite/delete.jl @@ -24,3 +24,32 @@ function delete_element!( ) return nothing end + +function _delete_time_series!( + db::DatabaseSQLite, + collection_id::String, + group_id::String, + id::Integer, +) + time_series_table_name = "$(collection_id)_time_series_$(group_id)" + + DBInterface.execute( + db.sqlite_db, + "DELETE FROM $(time_series_table_name) WHERE id = '$id'", + ) + return nothing +end + +function delete_time_series!( + db::DatabaseSQLite, + collection_id::String, + group_id::String, + label::String, +) + _throw_if_collection_does_not_exist(db, collection_id) + id = _get_id(db, collection_id, label) + + _delete_time_series!(db, collection_id, group_id, id) + + return nothing +end diff --git a/src/PSRDatabaseSQLite/read.jl b/src/PSRDatabaseSQLite/read.jl index 29fbe163..116c5d65 100644 --- a/src/PSRDatabaseSQLite/read.jl +++ b/src/PSRDatabaseSQLite/read.jl @@ -3,9 +3,22 @@ const READ_METHODS_BY_CLASS_OF_ATTRIBUTE = Dict( ScalarRelation => "read_scalar_relations", VectorParameter => "read_vector_parameters", VectorRelation => "read_vector_relations", + TimeSeries => "read_time_series_row", TimeSeriesFile => "read_time_series_file", ) +function number_of_elements(db::DatabaseSQLite, collection_id::String)::Int + query = "SELECT COUNT(*) FROM $collection_id" + result = DBInterface.execute(db.sqlite_db, query) + for row in result + return row[1] + end +end + +function _collection_has_any_data(db::DatabaseSQLite, collection_id::String)::Bool + return number_of_elements(db, collection_id) > 0 +end + function _get_id( db::DatabaseSQLite, collection_id::String, @@ -41,7 +54,7 @@ function read_scalar_parameters( attribute = _get_attribute(db, collection_id, attribute_id) table = _table_where_is_located(attribute) - query = "SELECT $attribute_id FROM $table ORDER BY rowid" + query = "SELECT $attribute_id FROM $table ORDER BY id" df = DBInterface.execute(db.sqlite_db, query) |> DataFrame results = df[!, 1] results = _treat_query_result(results, attribute, default) @@ -62,9 +75,7 @@ function read_scalar_parameter( :read, ) - attribute = _get_attribute(db, collection_id, attribute_id) - table = _table_where_is_located(attribute) - id = _get_id(db, table, label) + id = _get_id(db, collection_id, label) return read_scalar_parameter(db, collection_id, attribute_id, id; default) end @@ -146,6 +157,27 @@ function _query_vector( return results end +function end_date_query(db::DatabaseSQLite, attribute::Attribute) + # First checks if the date or dimension value is within the range of the data. + # Then it queries the closest date before the provided date. + # If there is no date query the data with date 0 (which will probably return no data.) + end_date_query = "SELECT MAX(DATE(date_time)) FROM $(attribute.table_where_is_located)" + end_date = DBInterface.execute(db.sqlite_db, end_date_query) |> DataFrame + if isempty(end_date) + return DateTime(0) + end + return DateTime(end_date[!, 1][1]) +end + +function closest_date_query(db::DatabaseSQLite, attribute::Attribute, dim_value::DateTime) + closest_date_query_earlier = "SELECT DISTINCT date_time FROM $(attribute.table_where_is_located) WHERE $(attribute.id) IS NOT NULL AND DATE(date_time) <= DATE('$(dim_value)') ORDER BY DATE(date_time) DESC LIMIT 1" + closest_date = DBInterface.execute(db.sqlite_db, closest_date_query_earlier) |> DataFrame + if isempty(closest_date) + return DateTime(0) + end + return DateTime(closest_date[!, 1][1]) +end + """ TODO """ @@ -164,7 +196,7 @@ function read_scalar_relations( names_in_collection_to = read_scalar_parameters(db, collection_to, "label") num_elements = length(names_in_collection_to) replace_dict = Dict{Any, String}(zip(collect(1:num_elements), names_in_collection_to)) - push!(replace_dict, _opensql_default_value_for_type(Int) => "") + push!(replace_dict, _psrdatabasesqlite_null_value(Int) => "") return replace(map_of_elements, replace_dict...) end @@ -201,7 +233,7 @@ function _get_scalar_relation_map( ) attribute = _get_attribute(db, collection_from, attribute_on_collection_from) - query = "SELECT $(attribute.id) FROM $(attribute.table_where_is_located) ORDER BY rowid" + query = "SELECT $(attribute.id) FROM $(attribute.table_where_is_located)" df = DBInterface.execute(db.sqlite_db, query) |> DataFrame results = df[!, 1] num_results = length(results) @@ -209,7 +241,7 @@ function _get_scalar_relation_map( ids_in_collection_to = read_scalar_parameters(db, collection_to, "id") for i in 1:num_results if ismissing(results[i]) - map_of_indexes[i] = _opensql_default_value_for_type(Int) + map_of_indexes[i] = _psrdatabasesqlite_null_value(Int) else map_of_indexes[i] = findfirst(isequal(results[i]), ids_in_collection_to) end @@ -233,7 +265,7 @@ function read_vector_relations( names_in_collection_to = read_scalar_parameters(db, collection_to, "label") num_elements = length(names_in_collection_to) replace_dict = Dict{Any, String}(zip(collect(1:num_elements), names_in_collection_to)) - push!(replace_dict, _opensql_default_value_for_type(Int) => "") + push!(replace_dict, _psrdatabasesqlite_null_value(Int) => "") map_with_labels = Vector{Vector{String}}(undef, length(map_of_vector_with_indexes)) @@ -300,7 +332,7 @@ function _get_vector_relation_map( if isnothing(index_of_id_collection_to) push!( map_of_vector_with_indexes[index_of_id], - _opensql_default_value_for_type(Int), + _psrdatabasesqlite_null_value(Int), ) else push!(map_of_vector_with_indexes[index_of_id], index_of_id_collection_to) @@ -324,7 +356,7 @@ function read_time_series_file( attribute = _get_attribute(db, collection_id, attribute_id) table = attribute.table_where_is_located - query = "SELECT $(attribute.id) FROM $table ORDER BY rowid" + query = "SELECT $(attribute.id) FROM $table" df = DBInterface.execute(db.sqlite_db, query) |> DataFrame result = df[!, 1] if isempty(result) @@ -340,6 +372,74 @@ function read_time_series_file( end end +function read_time_series_row( + db, + collection_id::String, + attribute_id::String; + date_time::DateTime, +) + _throw_if_attribute_is_not_time_series( + db, + collection_id, + attribute_id, + :read, + ) + @assert _is_read_only(db) "Time series mapping only works in read only databases" + + collection_attribute = _collection_attribute(collection_id, attribute_id) + attribute = _get_attribute(db, collection_id, attribute_id) + + T = attribute.type + + if !(_collection_has_any_data(db, collection_id)) + return Vector{T}(undef, 0) + end + if !haskey(db._time_controller.cache, collection_attribute) + db._time_controller.cache[collection_attribute] = _start_time_controller_cache(db, attribute, date_time, T) + end + cache = db._time_controller.cache[collection_attribute] + # If we don`t need to update anything we just return the data + if _no_need_to_query_any_id(cache, date_time) + cache.last_date_requested = date_time + return cache.data + end + # If we need to update the cache we update the dates and the data + _update_time_controller_cache!(cache, db, attribute, date_time) + return cache.data +end + +function _read_time_series_table( + db::DatabaseSQLite, + attribute::Attribute, + id::Int, +) + query = string("SELECT ", join(attribute.dimension_names, ",", ", "), ", ", attribute.id) + query *= " FROM $(attribute.table_where_is_located) WHERE id = '$id'" + return DBInterface.execute(db.sqlite_db, query) |> DataFrame +end + +function read_time_series_table( + db::DatabaseSQLite, + collection_id::String, + attribute_id::String, + label::String, +) + _throw_if_attribute_is_not_time_series( + db, + collection_id, + attribute_id, + :read, + ) + attribute = _get_attribute(db, collection_id, attribute_id) + id = _get_id(db, collection_id, label) + + return _read_time_series_table( + db, + attribute, + id, + ) +end + function _treat_query_result( query_results::Vector{Missing}, attribute::Attribute, @@ -347,7 +447,7 @@ function _treat_query_result( ) type_of_attribute = _type(attribute) default = if isnothing(default) - _opensql_default_value_for_type(type_of_attribute) + _psrdatabasesqlite_null_value(type_of_attribute) else default end @@ -361,7 +461,7 @@ function _treat_query_result( ) where {T <: Union{Int64, Float64}} type_of_attribute = _type(attribute) default = if isnothing(default) - _opensql_default_value_for_type(type_of_attribute) + _psrdatabasesqlite_null_value(type_of_attribute) else if isa(default, type_of_attribute) default @@ -386,7 +486,7 @@ function _treat_query_result( ) type_of_attribute = _type(attribute) default = if isnothing(default) - _opensql_default_value_for_type(type_of_attribute) + _psrdatabasesqlite_null_value(type_of_attribute) else if isa(default, type_of_attribute) default @@ -414,10 +514,10 @@ _treat_query_result( ::Union{Nothing, Any}, ) where {T <: Union{Int64, Float64}} = results -_opensql_default_value_for_type(::Type{Float64}) = NaN -_opensql_default_value_for_type(::Type{Int64}) = typemin(Int64) -_opensql_default_value_for_type(::Type{String}) = "" -_opensql_default_value_for_type(::Type{DateTime}) = typemin(DateTime) +_psrdatabasesqlite_null_value(::Type{Float64}) = NaN +_psrdatabasesqlite_null_value(::Type{Int64}) = typemin(Int64) +_psrdatabasesqlite_null_value(::Type{String}) = "" +_psrdatabasesqlite_null_value(::Type{DateTime}) = typemin(DateTime) function _is_null_in_db(value::Float64) return isnan(value) diff --git a/src/PSRDatabaseSQLite/time_controller.jl b/src/PSRDatabaseSQLite/time_controller.jl new file mode 100644 index 00000000..19dc37a8 --- /dev/null +++ b/src/PSRDatabaseSQLite/time_controller.jl @@ -0,0 +1,148 @@ +abstract type TimeSeriesRequestStatus end + +struct TimeSeriesDidNotChange <: TimeSeriesRequestStatus end +struct TimeSeriesChanged <: TimeSeriesRequestStatus end + +const CollectionAttribute = Tuple{String, String} + +# Some comments +# TODO we can further optimize the time controller with a few strategies +# 1 - We can try to ask for the data in the same query that we ask for the dates. I just don`t know how to write the good query for that +# 2 - We can use prepared statements for the queries +# 3 - Avoid querying the data for every id in the attribute. Currently we fill the cache of dates before making the query and use it to inform which date each id should query. This is quite inneficient +# The best way of optimizing it would be to solve 1 and 2. + +mutable struct TimeControllerCache{T} + data::Vector{T} + # Control of dates requested per element in a given pair collection attribute + closest_previous_date_with_data::Vector{DateTime} + last_date_requested::DateTime + closest_next_date_with_data::Vector{DateTime} + + # Private caches with the closest previous and next dates + # _closest_previous_date_with_data = maximum(closest_previous_date_with_data) + # _closest_next_date_with_data = minimum(closest_next_date_with_data) + _closest_global_previous_date_with_data::DateTime + _closest_global_next_date_with_data::DateTime + + # Cache of collection_ids + _collection_ids::Vector{Int} +end + +Base.@kwdef mutable struct TimeController + cache::Dict{CollectionAttribute, TimeControllerCache} = Dict{CollectionAttribute, TimeControllerCache}() +end + +function _collection_attribute(collection_id::String, attribute_id::String)::CollectionAttribute + return (collection_id, attribute_id) +end + +function _update_time_controller_cache!( + cache::TimeControllerCache, + db, + attribute::Attribute, + date_time::DateTime, +) + _update_time_controller_cache_dates!(cache, db, attribute, date_time) + + for (i, id) in enumerate(cache._collection_ids) + cache.data[i] = + _request_time_series_data_for_time_controller_cache(db, attribute, id, cache.closest_previous_date_with_data[i]) + end + + return nothing +end + +function _request_time_series_data_for_time_controller_cache( + db, + attribute::Attribute, + id::Int, + date_time::DateTime, +) + query = """ + SELECT $(attribute.id) + FROM $(attribute.table_where_is_located) + WHERE id = $id AND DATETIME(date_time) = DATETIME('$date_time') + """ + result = DBInterface.execute(db.sqlite_db, query) + + T = attribute.type + + for row in result + return T(row[1]) + end + return _psrdatabasesqlite_null_value(T) +end + +function _update_time_controller_cache_dates!( + cache::TimeControllerCache, + db, + attribute::Attribute, + date_time::DateTime, +) + cache.last_date_requested = date_time + query = """ + SELECT + id, + MAX(CASE WHEN DATE(date_time) <= DATE('$date_time') AND $(attribute.id) IS NOT NULL THEN DATE(date_time) ELSE NULL END) AS closest_previous_date_with_data, + MIN(CASE WHEN DATE(date_time) > DATE('$date_time') AND $(attribute.id) IS NOT NULL THEN DATE(date_time) ELSE NULL END) AS closest_next_date_with_data + FROM $(attribute.table_where_is_located) + GROUP BY id + ORDER BY id + """ + result = DBInterface.execute(db.sqlite_db, query) + for (i, row) in enumerate(result) + id = row[1] + @assert id == cache._collection_ids[i] "The id in the database is different from the one in the cache" + closest_previous_date_with_data = row[2] + closest_next_date_with_data = row[3] + if ismissing(closest_previous_date_with_data) + cache.closest_previous_date_with_data[i] = typemin(DateTime) + else + cache.closest_previous_date_with_data[i] = DateTime(closest_previous_date_with_data) + end + if ismissing(closest_next_date_with_data) + cache.closest_next_date_with_data[i] = typemax(DateTime) + else + cache.closest_next_date_with_data[i] = DateTime(closest_next_date_with_data) + end + end + cache._closest_global_previous_date_with_data = maximum(cache.closest_previous_date_with_data) + cache._closest_global_next_date_with_data = minimum(cache.closest_next_date_with_data) + return cache +end + +function _no_need_to_query_any_id( + cache::TimeControllerCache, + date_time::DateTime, +)::Bool + return cache._closest_global_previous_date_with_data <= date_time < cache._closest_global_next_date_with_data +end + +function _start_time_controller_cache( + db, + attribute::Attribute, + date_time::DateTime, + ::Type{T}, +) where {T} + _collection_ids = read_scalar_parameters(db, attribute.parent_collection, "id") + data = fill(_psrdatabasesqlite_null_value(T), length(_collection_ids)) + closest_previous_date_with_data = fill(typemin(DateTime), length(_collection_ids)) + closest_next_date_with_data = fill(typemax(DateTime), length(_collection_ids)) + _closest_global_previous_date_with_data = maximum(closest_previous_date_with_data) + _closest_global_next_date_with_data = minimum(closest_next_date_with_data) + + cache = TimeControllerCache{T}( + data, + closest_previous_date_with_data, + date_time, + closest_next_date_with_data, + _closest_global_previous_date_with_data, + _closest_global_next_date_with_data, + _collection_ids, + ) + + _update_time_controller_cache!(cache, db, attribute, date_time) + + return cache +end diff --git a/src/PSRDatabaseSQLite/update.jl b/src/PSRDatabaseSQLite/update.jl index e71740fa..cb5e450b 100644 --- a/src/PSRDatabaseSQLite/update.jl +++ b/src/PSRDatabaseSQLite/update.jl @@ -3,6 +3,7 @@ const UPDATE_METHODS_BY_CLASS_OF_ATTRIBUTE = Dict( ScalarRelation => "set_scalar_relation!", VectorParameter => "update_vector_parameter!", VectorRelation => "set_vector_relation!", + TimeSeries => "update_time_series_row!", TimeSeriesFile => "set_time_series_file!", ) @@ -272,7 +273,7 @@ function set_time_series_file!( kwargs..., ) _throw_if_collection_does_not_exist(db, collection_id) - table_name = collection_id * "_timeseriesfiles" + table_name = collection_id * "_time_series_files" dict_time_series = Dict() for (key, value) in kwargs if !isa(value, AbstractString) @@ -327,3 +328,84 @@ function set_time_series_file!( end return nothing end + +function _dimension_value_exists( + db::DatabaseSQLite, + attribute::Attribute, + id::Integer, + dimensions..., +) + query = "SELECT $(attribute.id) FROM $(attribute.table_where_is_located) WHERE id = $id AND " + for (i, (key, value)) in enumerate(dimensions) + if key == "date_time" + query *= "$(key) = DATE('$(value)')" + else + query *= "$(key) = '$(value)'" + end + if i < length(dimensions) + query *= " AND " + end + end + results = DBInterface.execute(db.sqlite_db, query) |> DataFrame + if isempty(results) + return false + end + return true +end + +function _update_time_series_row!( + db::DatabaseSQLite, + attribute::Attribute, + id::Integer, + val, + dimensions, +) + query = "UPDATE $(attribute.table_where_is_located) SET $(attribute.id) = '$val'" + query *= " WHERE id = '$id' AND " + for (i, (key, value)) in enumerate(dimensions) + if key == "date_time" + query *= "$(key) = DATE('$(value)')" + else + query *= "$(key) = '$(value)'" + end + if i < length(dimensions) + query *= " AND " + end + end + DBInterface.execute(db.sqlite_db, query) + return nothing +end + +function update_time_series_row!( + db::DatabaseSQLite, + collection_id::String, + attribute_id::String, + label::String, + val; + dimensions..., +) + _throw_if_attribute_is_not_time_series( + db, + collection_id, + attribute_id, + :update, + ) + attribute = _get_attribute(db, collection_id, attribute_id) + id = _get_id(db, collection_id, label) + _validate_time_series_dimensions(collection_id, attribute, dimensions) + + if !_dimension_value_exists(db, attribute, id, dimensions...) + psr_database_sqlite_error( + "The chosen values for dimensions $(join(keys(dimensions), ", ")) do not exist in the time series for element $(label) in collection $(collection_id).", + ) + end + + if length(dimensions) != length(attribute.dimension_names) + psr_database_sqlite_error( + "The number of dimensions in the time series does not match the number of dimensions in the attribute. " * + "The attribute has $(attribute.num_dimensions) dimensions: $(join(attribute.dimension_names, ", ")).", + ) + end + + return _update_time_series_row!(db, attribute, id, val, dimensions) +end diff --git a/src/PSRDatabaseSQLite/utils.jl b/src/PSRDatabaseSQLite/utils.jl index 699c93d3..5aeffe47 100644 --- a/src/PSRDatabaseSQLite/utils.jl +++ b/src/PSRDatabaseSQLite/utils.jl @@ -127,6 +127,3 @@ function table_names(db::SQLite.DB) end return tables end - -_timeseries_table_name(table::String) = table * "_timeseriesfiles" -_relation_table_name(table_1::String, table_2::String) = table_1 * "_relation_" * table_2 diff --git a/src/PSRDatabaseSQLite/validate.jl b/src/PSRDatabaseSQLite/validate.jl index 81211ae5..644bfcd7 100644 --- a/src/PSRDatabaseSQLite/validate.jl +++ b/src/PSRDatabaseSQLite/validate.jl @@ -1,6 +1,6 @@ # just for reference this are the main regexes # the functions not commented implement combinations of them -# with other reserved words such as vector, relation and timeseries. +# with other reserved words such as vector, relation and time_series. # _regex_table_name() = Regex("(?:[A-Z][a-z]*)+") # _regex_column_name() = Regex("[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*") @@ -18,8 +18,15 @@ _is_valid_table_vector_name(table::String) = ), ) -_is_valid_table_timeseries_name(table::String) = - !isnothing(match(r"^(?:[A-Z][a-z]*)+_timeseriesfiles", table)) +_is_valid_time_series_name(table::String) = + !isnothing( + match( + r"^(?:[A-Z][a-z]*)+_time_series_(?!files$)[a-z][a-z0-9]*(?:_{1}[a-z0-9]+)*$", + table, + ), + ) + +_is_valid_table_time_series_files_name(table::String) = !isnothing(match(r"^(?:[A-Z][a-z]*)+_time_series_files", table)) _is_valid_time_series_attribute_value(value::String) = !isnothing( @@ -67,7 +74,23 @@ function _validate_table(db::SQLite.DB, table::String) return num_errors end -function _validate_timeseries_table(db::SQLite.DB, table::String) +function _validate_time_series_table(db::SQLite.DB, table::String) + attributes = column_names(db, table) + num_errors = 0 + if !("id" in attributes) + @error("Table $table is a time_series table and does not have an \"id\" column.") + num_errors += 1 + end + if !("date_time" in attributes) + @error( + "Table $table is a time_series table and does not have an \"date_time\" column.", + ) + num_errors += 1 + end + return num_errors +end + +function _validate_time_series_files_table(db::SQLite.DB, table::String) attributes = column_names(db, table) num_errors = 0 if ("id" in attributes) @@ -116,7 +139,6 @@ function _validate_database(db::SQLite.DB) psr_database_sqlite_error("Database does not have a \"Configuration\" table.") end _validate_database_pragmas(db) - _set_default_pragmas!(db) num_errors = 0 for table in tables if table == "sqlite_sequence" @@ -124,8 +146,10 @@ function _validate_database(db::SQLite.DB) end if _is_valid_table_name(table) num_errors += _validate_table(db, table) - elseif _is_valid_table_timeseries_name(table) - num_errors += _validate_timeseries_table(db, table) + elseif _is_valid_table_time_series_files_name(table) + num_errors += _validate_time_series_files_table(db, table) + elseif _is_valid_time_series_name(table) + num_errors += _validate_time_series_table(db, table) elseif _is_valid_table_vector_name(table) num_errors += _validate_vector_table(db, table) else @@ -134,7 +158,8 @@ function _validate_database(db::SQLite.DB) Valid table name formats are: - Collections: NameOfCollection - Vector attributes: NameOfCollection_vector_group_id - - Time series: NameOfCollection_timeseriesfiles + - Time series: NameOfCollection_time_series_group_id + - Time series files: NameOfCollection_time_series_files """) num_errors += 1 end @@ -245,6 +270,26 @@ function _throw_if_attribute_is_not_vector_relation( return nothing end +function _throw_if_attribute_is_not_time_series( + db::DatabaseSQLite, + collection::String, + attribute::String, + action::Symbol, +) + _throw_if_collection_or_attribute_do_not_exist(db, collection, attribute) + + if !_is_time_series(db, collection, attribute) + correct_composity_type = + _attribute_composite_type(db, collection, attribute) + string_of_composite_types = _string_for_composite_types(correct_composity_type) + correct_method_to_use = _get_correct_method_to_use(correct_composity_type, action) + psr_database_sqlite_error( + "Attribute \"$attribute\" is not a time series. It is a $string_of_composite_types. Use `$correct_method_to_use` instead.", + ) + end + return nothing +end + function _throw_if_attribute_is_not_time_series_file( db::DatabaseSQLite, collection::String, @@ -272,8 +317,8 @@ function _throw_if_not_scalar_attribute( ) _throw_if_collection_or_attribute_do_not_exist(db, collection, attribute) - if _is_vector_parameter(db, collection, attribute) || - _is_vector_relation(db, collection, attribute) + if !_is_scalar_parameter(db, collection, attribute) && + !_is_scalar_relation(db, collection, attribute) psr_database_sqlite_error( "Attribute \"$attribute\" is not a scalar attribute. You must input a vector for this attribute.", ) @@ -299,19 +344,58 @@ function _throw_if_not_vector_attribute( return nothing end -function _throw_if_relation_does_not_exist( - collection_from::String, - collection_to::String, - relation_type::String, +function _throw_if_not_time_series_group( + db::DatabaseSQLite, + collection::String, + group::String, ) - if !_scalar_relation_exists(collection_from, collection_to, relation_type) && - !_vector_relation_exists(collection_from, collection_to, relation_type) + if !_is_time_series_group(db, collection, group) psr_database_sqlite_error( - "relation `$relation_type` between $collection_from and $collection_to does not exist. \n" * - "This is the list of relations that exist: " * - "$(_show_existing_relation_types(_list_of_relation_types(collection_from, collection_to)))", + "Group \"$group\" is not a time series group. ", ) end + return nothing +end + +function _throw_if_data_does_not_match_group( + db::DatabaseSQLite, + collection_id::String, + group::String, + df::DataFrame, +) + collection = _get_collection(db, collection_id) + dimensions_in_df = [] + attributes_in_df = [] + + for column in names(df) + if column in keys(collection.time_series) + # should be an attribute + push!(attributes_in_df, column) + else + # should be a dimension + push!(dimensions_in_df, column) + end + end + + # validate if the attributes belong to the same group and if the dimensions are valid for this group + for attribute_id in attributes_in_df + attribute = _get_attribute(db, collection_id, attribute_id) + if attribute.group_id != group + psr_database_sqlite_error( + "Attribute \"$attribute_id\" is not in the time series group \"$group\".", + ) + end + end + + for dimension in dimensions_in_df + if !(dimension in collection.time_series[attributes_in_df[1]].dimension_names) + psr_database_sqlite_error( + "The dimension \"$dimension\" is not defined in the time series group \"$group\".", + ) + end + end + + return nothing end function _throw_if_is_time_series_file( @@ -343,8 +427,8 @@ function _validate_attribute_types!( db::DatabaseSQLite, collection_id::String, label_or_id::Union{Integer, String}, - dict_scalar_attributes, - dict_vector_attributes, + dict_scalar_attributes::AbstractDict, + dict_vector_attributes::AbstractDict, ) for (key, value) in dict_scalar_attributes attribute = _get_attribute(db, collection_id, string(key)) @@ -417,19 +501,19 @@ function _validate_vector_relation_type( end end -function _set_default_pragmas!(db::SQLite.DB) - _set_foreign_keys_on!(db) - return nothing -end - -function _set_foreign_keys_on!(db::SQLite.DB) - # https://www.sqlite.org/foreignkeys.html#fk_enable - # Foreign keys are enabled per connection, they are not something - # that can be stored in the database itself like user_version. - # This is needed to ensure that the foreign keys are enabled - # behaviours like cascade delete and update are enabled. - DBInterface.execute(db, "PRAGMA foreign_keys = ON;") - return nothing +function _validate_time_series_dimensions( + collection_id::String, + attribute::Attribute, + dimensions..., +) + for dim_name in keys(dimensions...) + if !(string(dim_name) in attribute.dimension_names) + psr_database_sqlite_error( + "The dimension \"$dim_name\" is not defined in the time series attribute \"$(attribute.id)\" of collection \"$collection_id\". " * + "The available dimensions are: $(attribute.dimension_names).", + ) + end + end end function _validate_database_pragmas(db::SQLite.DB) diff --git a/test/PSRDatabaseSQLite/test_create/test_create.jl b/test/PSRDatabaseSQLite/test_create/test_create.jl index 2bb8ddc8..ee5b7f10 100644 --- a/test/PSRDatabaseSQLite/test_create/test_create.jl +++ b/test/PSRDatabaseSQLite/test_create/test_create.jl @@ -3,6 +3,7 @@ module TestCreate using PSRClassesInterface.PSRDatabaseSQLite using SQLite using Dates +using DataFrames using Test function test_create_parameters() @@ -265,6 +266,71 @@ function test_create_vectors_with_relations() return nothing end +function test_create_time_series() + path_schema = joinpath(@__DIR__, "test_create_time_series.sql") + db_path = joinpath(@__DIR__, "test_create_time_series.sqlite") + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + for i in 1:3 + df_time_series_group1 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 2.0] .* i, + some_vector2 = [2.0, 3.0] .* i, + ) + df_time_series_group2 = DataFrame(; + date_time = [DateTime(2000), DateTime(2000), DateTime(2001), DateTime(2001)], + block = [1, 2, 1, 2], + some_vector3 = [1.0, missing, 3.0, 4.0] .* i, + ) + df_time_series_group3 = DataFrame(; + date_time = [ + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2001), + DateTime(2001), + DateTime(2001), + DateTime(2009), + ], + block = [1, 1, 1, 1, 2, 2, 2, 2], + segment = [1, 2, 3, 4, 1, 2, 3, 4], + some_vector5 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4] .* i, + some_vector6 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4] .* i, + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource $i", + group1 = df_time_series_group1, + group2 = df_time_series_group2, + group3 = df_time_series_group3, + ) + end + + df_time_series_group5 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 2.0], + some_vector2 = [2.0, 3.0], + ) + + @test_throws PSRDatabaseSQLite.DatabaseException PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 4", + group5 = df_time_series_group5, + ) + + PSRDatabaseSQLite.close!(db) + GC.gc() + GC.gc() + rm(db_path) + @test true + return nothing +end + function runtests() Base.GC.gc() Base.GC.gc() diff --git a/test/PSRDatabaseSQLite/test_create/test_create_parameters_and_vectors.sql b/test/PSRDatabaseSQLite/test_create/test_create_parameters_and_vectors.sql index a80781b3..a0106f2c 100644 --- a/test/PSRDatabaseSQLite/test_create/test_create_parameters_and_vectors.sql +++ b/test/PSRDatabaseSQLite/test_create/test_create_parameters_and_vectors.sql @@ -81,7 +81,7 @@ CREATE TABLE Process_vector_outputs ( PRIMARY KEY (id, vector_index) ) STRICT; -CREATE TABLE Plant_timeseriesfiles ( +CREATE TABLE Plant_time_series_files ( generation TEXT, prices TEXT ) STRICT; \ No newline at end of file diff --git a/test/PSRDatabaseSQLite/test_create/test_create_time_series.sql b/test/PSRDatabaseSQLite/test_create/test_create_time_series.sql new file mode 100644 index 00000000..67aeb17d --- /dev/null +++ b/test/PSRDatabaseSQLite/test_create/test_create_time_series.sql @@ -0,0 +1,58 @@ +PRAGMA user_version = 1; +PRAGMA foreign_keys = ON; + +CREATE TABLE Configuration ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, + value1 REAL NOT NULL DEFAULT 100, + enum1 TEXT NOT NULL DEFAULT 'A' CHECK(enum1 IN ('A', 'B', 'C')) +) STRICT; + + +CREATE TABLE Resource ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, + type TEXT NOT NULL DEFAULT "D" +) STRICT; + +CREATE TABLE Resource_time_series_group1 ( + id INTEGER, + date_time TEXT NOT NULL, + some_vector1 REAL, + some_vector2 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time) +) STRICT; + +CREATE TABLE Resource_time_series_group2 ( + id INTEGER, + date_time TEXT NOT NULL, + block INTEGER NOT NULL, + some_vector3 REAL, + some_vector4 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time, block) +) STRICT; + +CREATE TABLE Resource_time_series_group3 ( + id INTEGER, + date_time TEXT NOT NULL, + block INTEGER NOT NULL, + segment INTEGER NOT NULL, + some_vector5 REAL, + some_vector6 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time, block, segment) +) STRICT; + +CREATE TABLE Resource_time_series_group4 ( + id INTEGER, + date_time TEXT NOT NULL, + block INTEGER NOT NULL, + segment INTEGER NOT NULL, + some_other_dimension INTEGER NOT NULL, + some_vector7 REAL, + some_vector8 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time, block, segment, some_other_dimension) +) STRICT; \ No newline at end of file diff --git a/test/PSRDatabaseSQLite/test_psri_study_interface/toy_schema.sql b/test/PSRDatabaseSQLite/test_psri_study_interface/toy_schema.sql index eaa9be29..a391c41b 100644 --- a/test/PSRDatabaseSQLite/test_psri_study_interface/toy_schema.sql +++ b/test/PSRDatabaseSQLite/test_psri_study_interface/toy_schema.sql @@ -59,7 +59,7 @@ CREATE TABLE Plant_vector_cost_relation ( PRIMARY KEY (id, vector_index) ) STRICT; -CREATE TABLE Plant_timeseriesfiles ( +CREATE TABLE Plant_time_series_files ( generation TEXT, cost TEXT ) STRICT; \ No newline at end of file diff --git a/test/PSRDatabaseSQLite/test_read/test_read.jl b/test/PSRDatabaseSQLite/test_read/test_read.jl index a933bbc5..58a9bb22 100644 --- a/test/PSRDatabaseSQLite/test_read/test_read.jl +++ b/test/PSRDatabaseSQLite/test_read/test_read.jl @@ -3,6 +3,7 @@ module TestRead using PSRClassesInterface.PSRDatabaseSQLite using SQLite using Dates +using DataFrames using Test function test_read_parameters() diff --git a/test/PSRDatabaseSQLite/test_read/test_read.sql b/test/PSRDatabaseSQLite/test_read/test_read.sql index 18e9cfc7..29ec562b 100644 --- a/test/PSRDatabaseSQLite/test_read/test_read.sql +++ b/test/PSRDatabaseSQLite/test_read/test_read.sql @@ -54,7 +54,7 @@ CREATE TABLE Plant_vector_cost_relation ( PRIMARY KEY (id, vector_index) ) STRICT; -CREATE TABLE Plant_timeseriesfiles ( +CREATE TABLE Plant_time_series_files ( wind_speed TEXT, wind_direction TEXT ) STRICT; \ No newline at end of file diff --git a/test/PSRDatabaseSQLite/test_time_series/test_read_time_series.sql b/test/PSRDatabaseSQLite/test_time_series/test_read_time_series.sql new file mode 100644 index 00000000..67aeb17d --- /dev/null +++ b/test/PSRDatabaseSQLite/test_time_series/test_read_time_series.sql @@ -0,0 +1,58 @@ +PRAGMA user_version = 1; +PRAGMA foreign_keys = ON; + +CREATE TABLE Configuration ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, + value1 REAL NOT NULL DEFAULT 100, + enum1 TEXT NOT NULL DEFAULT 'A' CHECK(enum1 IN ('A', 'B', 'C')) +) STRICT; + + +CREATE TABLE Resource ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, + type TEXT NOT NULL DEFAULT "D" +) STRICT; + +CREATE TABLE Resource_time_series_group1 ( + id INTEGER, + date_time TEXT NOT NULL, + some_vector1 REAL, + some_vector2 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time) +) STRICT; + +CREATE TABLE Resource_time_series_group2 ( + id INTEGER, + date_time TEXT NOT NULL, + block INTEGER NOT NULL, + some_vector3 REAL, + some_vector4 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time, block) +) STRICT; + +CREATE TABLE Resource_time_series_group3 ( + id INTEGER, + date_time TEXT NOT NULL, + block INTEGER NOT NULL, + segment INTEGER NOT NULL, + some_vector5 REAL, + some_vector6 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time, block, segment) +) STRICT; + +CREATE TABLE Resource_time_series_group4 ( + id INTEGER, + date_time TEXT NOT NULL, + block INTEGER NOT NULL, + segment INTEGER NOT NULL, + some_other_dimension INTEGER NOT NULL, + some_vector7 REAL, + some_vector8 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time, block, segment, some_other_dimension) +) STRICT; \ No newline at end of file diff --git a/test/PSRDatabaseSQLite/test_time_series/test_time_controller.sql b/test/PSRDatabaseSQLite/test_time_series/test_time_controller.sql new file mode 100644 index 00000000..4104f865 --- /dev/null +++ b/test/PSRDatabaseSQLite/test_time_series/test_time_controller.sql @@ -0,0 +1,28 @@ +PRAGMA user_version = 1; +PRAGMA foreign_keys = ON; + +CREATE TABLE Configuration ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, + value1 REAL NOT NULL DEFAULT 100, + enum1 TEXT NOT NULL DEFAULT 'A' CHECK(enum1 IN ('A', 'B', 'C')) +) STRICT; + +CREATE TABLE Resource ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT UNIQUE NOT NULL, + type TEXT NOT NULL DEFAULT "D" +) STRICT; + +CREATE TABLE Resource_time_series_group1 ( + id INTEGER, + date_time TEXT NOT NULL, + some_vector1 REAL, + some_vector2 REAL, + some_vector3 REAL, + some_vector4 REAL, + some_vector5 REAL, + some_vector6 REAL, + FOREIGN KEY(id) REFERENCES Resource(id) ON DELETE CASCADE ON UPDATE CASCADE, + PRIMARY KEY (id, date_time) +) STRICT; \ No newline at end of file diff --git a/test/PSRDatabaseSQLite/test_time_series/test_time_series.jl b/test/PSRDatabaseSQLite/test_time_series/test_time_series.jl new file mode 100644 index 00000000..970e0359 --- /dev/null +++ b/test/PSRDatabaseSQLite/test_time_series/test_time_series.jl @@ -0,0 +1,988 @@ +module TestTimeController + +using PSRClassesInterface.PSRDatabaseSQLite +using SQLite +using Dates +using DataFrames +using Test + +function _test_row(cached_data, answer) + @test length(cached_data) == length(answer) + for i in eachindex(cached_data) + if isnan(answer[i]) + @test isnan(cached_data[i]) + else + @test cached_data[i] == answer[i] + end + end +end + +function _test_table(table, answer) + for (i, row) in enumerate(eachrow(table)) + for col in names(table) + if col == "date_time" + @test DateTime(row[col]) == answer[i, col] + continue + end + if ismissing(answer[i, col]) + @test ismissing(row[col]) + else + @test row[col] == answer[i, col] + end + end + end +end + +##################### +# Time Series Table # +##################### + +function test_read_time_series_single() + path_schema = joinpath(@__DIR__, "test_read_time_series.sql") + db_path = joinpath(@__DIR__, "test_read_time_series.sqlite") + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + for i in 1:3 + df_time_series_group1 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 2.0] .* i, + some_vector2 = [2.0, 3.0] .* i, + ) + df_time_series_group2 = DataFrame(; + date_time = [DateTime(2000), DateTime(2000), DateTime(2001), DateTime(2001)], + block = [1, 2, 1, 2], + some_vector3 = [1.0, missing, 3.0, 4.0] .* i, + ) + df_time_series_group3 = DataFrame(; + date_time = [ + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2001), + DateTime(2001), + DateTime(2001), + DateTime(2009), + ], + block = [1, 1, 1, 1, 2, 2, 2, 2], + segment = [1, 2, 3, 4, 1, 2, 3, 4], + some_vector5 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4] .* i, + some_vector6 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4] .* i, + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource $i", + group1 = df_time_series_group1, + group2 = df_time_series_group2, + group3 = df_time_series_group3, + ) + end + + # return single dataframe + + for i in 1:3 + df_group1_answer = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 2.0] .* i, + some_vector2 = [2.0, 3.0] .* i, + ) + df_group2_answer = DataFrame(; + date_time = [DateTime(2000), DateTime(2000), DateTime(2001), DateTime(2001)], + block = [1, 2, 1, 2], + some_vector3 = [1.0, missing, 3.0, 4.0] .* i, + ) + df_group3_answer = DataFrame(; + date_time = [ + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2001), + DateTime(2001), + DateTime(2001), + DateTime(2009), + ], + block = [1, 1, 1, 1, 2, 2, 2, 2], + segment = [1, 2, 3, 4, 1, 2, 3, 4], + some_vector5 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4] .* i, + some_vector6 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4] .* i, + ) + + all_answers = [df_group1_answer, df_group2_answer, df_group3_answer] + + # iterating over the three groups + + for df_answer in all_answers + for col in names(df_answer) + if startswith(col, "some_vector") + df = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + col, + "Resource $i", + ) + _test_table(df, df_answer) + end + end + end + end + + PSRDatabaseSQLite.close!(db) + GC.gc() + GC.gc() + rm(db_path) + @test true + return nothing +end + +# ################## +# # Time Controller# +# ################## + +# # For each date, test the returned value with the expected value +function test_time_controller_read() + path_schema = joinpath(@__DIR__, "test_time_controller.sql") + db_path = joinpath(@__DIR__, "test_time_controller_read.sqlite") + GC.gc() + GC.gc() + if isfile(db_path) + rm(db_path) + end + + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + df = DataFrame(; + date_time = [DateTime(2000), DateTime(2001), DateTime(2002)], + some_vector1 = [missing, 1.0, 2.0], + some_vector2 = [1.0, 2.0, 3.0], + some_vector3 = [3.0, 2.0, 1.0], + some_vector4 = [1.0, missing, 5.0], + some_vector5 = [missing, missing, missing], + some_vector6 = [6.0, missing, missing], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df, + ) + + PSRDatabaseSQLite.close!(db) + db = PSRDatabaseSQLite.load_db(db_path; read_only = true) + + some_vector1_answer = [[NaN], [1.0], [2.0]] + some_vector2_answer = [[1.0], [2.0], [3.0]] + some_vector3_answer = [[3.0], [2.0], [1.0]] + some_vector4_answer = [[1.0], [1.0], [5.0]] + some_vector5_answer = [[NaN], [NaN], [NaN]] + some_vector6_answer = [[6.0], [6.0], [6.0]] + + # test for dates in correct sequence + for d_i in eachindex(df.date_time) + cached_1 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_1, some_vector1_answer[d_i]) + + cached_2 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector2"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_2, some_vector2_answer[d_i]) + + cached_3 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector3"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_3, some_vector3_answer[d_i]) + + cached_4 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector4"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_4, some_vector4_answer[d_i]) + + cached_5 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector5"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_5, some_vector5_answer[d_i]) + + cached_6 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector6"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_6, some_vector6_answer[d_i]) + end + + # test for dates in reverse sequence + for d_i in reverse(eachindex(df.date_time)) + cached_1 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_1, some_vector1_answer[d_i]) + + cached_2 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector2"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_2, some_vector2_answer[d_i]) + + cached_3 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector3"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_3, some_vector3_answer[d_i]) + + cached_4 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector4"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_4, some_vector4_answer[d_i]) + + cached_5 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector5"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_5, some_vector5_answer[d_i]) + + cached_6 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector6"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_6, some_vector6_answer[d_i]) + end + + # test for dates in random sequence + for d_i in [2, 1, 3] + cached_1 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_1, some_vector1_answer[d_i]) + + cached_2 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector2"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_2, some_vector2_answer[d_i]) + + cached_3 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector3"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_3, some_vector3_answer[d_i]) + + cached_4 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector4"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_4, some_vector4_answer[d_i]) + + cached_5 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector5"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_5, some_vector5_answer[d_i]) + + cached_6 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector6"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_6, some_vector6_answer[d_i]) + end + + PSRDatabaseSQLite.close!(db) + return rm(db_path) +end + +function test_time_controller_read_more_agents() + path_schema = joinpath(@__DIR__, "test_time_controller.sql") + db_path = joinpath(@__DIR__, "test_time_controller_read_multiple.sqlite") + GC.gc() + GC.gc() + if isfile(db_path) + rm(db_path) + end + + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + df = DataFrame(; + date_time = [DateTime(2000), DateTime(2001), DateTime(2002)], + some_vector1 = [missing, 1.0, 2.0], + some_vector2 = [1.0, missing, 5.0], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df, + ) + + df2 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001), DateTime(2002)], + some_vector1 = [missing, 10.0, 20.0], + some_vector2 = [10.0, missing, 50.0], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 2", + group1 = df2, + ) + + PSRDatabaseSQLite.close!(db) + db = PSRDatabaseSQLite.load_db(db_path; read_only = true) + + some_vector1_answer = [[NaN, NaN], [1.0, 10.0], [2.0, 20.0]] + some_vector2_answer = [[1.0, 10.0], [1.0, 10.0], [5.0, 50.0]] + + # test for dates in correct sequence + for d_i in eachindex(df.date_time) + cached_1 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_1, some_vector1_answer[d_i]) + + cached_2 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector2"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_2, some_vector2_answer[d_i]) + end + + PSRDatabaseSQLite.close!(db) + return rm(db_path) +end + +function test_time_controller_read_more_agents_2() + path_schema = joinpath(@__DIR__, "test_time_controller.sql") + db_path = joinpath(@__DIR__, "test_time_controller_read_multiple_2.sqlite") + GC.gc() + GC.gc() + if isfile(db_path) + rm(db_path) + end + + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + df = DataFrame(; + date_time = [DateTime(2000)], + some_vector1 = [missing], + some_vector2 = [1.0], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df, + ) + + df2 = DataFrame(; + date_time = [DateTime(2000)], + some_vector1 = [1.0], + some_vector2 = [10.0], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 2", + group1 = df2, + ) + + PSRDatabaseSQLite.close!(db) + db = PSRDatabaseSQLite.load_db(db_path; read_only = true) + + some_vector1_answer = [[NaN, 1.0]] + some_vector2_answer = [[1.0, 10.0]] + + # test for dates in correct sequence + for d_i in eachindex(df.date_time) + cached_1 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_1, some_vector1_answer[d_i]) + + cached_2 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector2"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_2, some_vector2_answer[d_i]) + end + + PSRDatabaseSQLite.close!(db) + return rm(db_path) +end + +function test_time_controller_empty() + path_schema = joinpath(@__DIR__, "test_time_controller.sql") + db_path = joinpath(@__DIR__, "test_time_controller_read_empty.sqlite") + GC.gc() + GC.gc() + if isfile(db_path) + rm(db_path) + end + + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + PSRDatabaseSQLite.close!(db) + db = PSRDatabaseSQLite.load_db(db_path; read_only = true) + + empty_cache = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1"; + date_time = DateTime(2000), + ) + _test_row(empty_cache, []) + + PSRDatabaseSQLite.close!(db) + return rm(db_path) +end + +function test_time_controller_filled_then_empty() + path_schema = joinpath(@__DIR__, "test_time_controller.sql") + db_path = joinpath(@__DIR__, "test_time_controller_read_filled_then_empty.sqlite") + GC.gc() + GC.gc() + if isfile(db_path) + rm(db_path) + end + + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + df = DataFrame(; + date_time = [DateTime(2000), DateTime(2001), DateTime(2002)], + some_vector1 = [missing, 1.0, 2.0], + some_vector2 = [1.0, missing, 5.0], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df, + ) + + df2 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001), DateTime(2002)], + some_vector1 = [missing, 10.0, 20.0], + some_vector2 = [10.0, missing, 50.0], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 2", + group1 = df2, + ) + + PSRDatabaseSQLite.close!(db) + db = PSRDatabaseSQLite.load_db(db_path; read_only = true) + + some_vector1_answer = [[NaN, NaN], [1.0, 10.0], [2.0, 20.0]] + some_vector2_answer = [[1.0, 10.0], [1.0, 10.0], [5.0, 50.0]] + + # test for dates in correct sequence + for d_i in eachindex(df.date_time) + cached_1 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_1, some_vector1_answer[d_i]) + + cached_2 = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector2"; + date_time = DateTime(df.date_time[d_i]), + ) + _test_row(cached_2, some_vector2_answer[d_i]) + end + + PSRDatabaseSQLite.close!(db) + + db = PSRDatabaseSQLite.load_db(db_path; read_only = false) + + PSRDatabaseSQLite.delete_element!(db, "Resource", "Resource 1") + PSRDatabaseSQLite.delete_element!(db, "Resource", "Resource 2") + + PSRDatabaseSQLite.close!(db) + + db = PSRDatabaseSQLite.load_db(db_path; read_only = true) + + empty_cache = PSRDatabaseSQLite.read_time_series_row( + db, + "Resource", + "some_vector1"; + date_time = DateTime(2000), + ) + _test_row(empty_cache, []) + + PSRDatabaseSQLite.close!(db) + + return rm(db_path) +end + +function test_update_time_series() + path_schema = joinpath(@__DIR__, "test_read_time_series.sql") + db_path = joinpath(@__DIR__, "test_update_time_series.sqlite") + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + df_time_series_group1 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 2.0], + some_vector2 = [2.0, 3.0], + ) + + df_time_series_group3 = DataFrame(; + date_time = [ + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2001), + DateTime(2001), + DateTime(2001), + DateTime(2009), + ], + block = [1, 1, 1, 1, 2, 2, 2, 2], + segment = [1, 2, 3, 4, 1, 2, 3, 4], + some_vector5 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4], + some_vector6 = [1.0, 2.0, 3.0, 4.0, 1, 2, 3, 4], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df_time_series_group1, + group3 = df_time_series_group3, + ) + + PSRDatabaseSQLite.update_time_series_row!( + db, + "Resource", + "some_vector1", + "Resource 1", + 10.0; + date_time = DateTime(2001), + ) + + PSRDatabaseSQLite.update_time_series_row!( + db, + "Resource", + "some_vector2", + "Resource 1", + 50.0; + date_time = DateTime(2001), + ) + + PSRDatabaseSQLite.update_time_series_row!( + db, + "Resource", + "some_vector5", + "Resource 1", + 10.0; + date_time = DateTime(2000), + block = 1, + segment = 2, + ) + + PSRDatabaseSQLite.update_time_series_row!( + db, + "Resource", + "some_vector5", + "Resource 1", + 3.0; + date_time = DateTime(2000), + block = 1, + segment = 1, + ) + + PSRDatabaseSQLite.update_time_series_row!( + db, + "Resource", + "some_vector6", + "Resource 1", + 33.0; + date_time = DateTime(2000), + block = 1, + segment = 3, + ) + + @test_throws PSRDatabaseSQLite.DatabaseException PSRDatabaseSQLite.update_time_series_row!( + db, + "Resource", + "some_vector6", + "Resource 1", + 10.0; + date_time = DateTime(2000), + segment = 2, + ) + + @test_throws PSRDatabaseSQLite.DatabaseException PSRDatabaseSQLite.update_time_series_row!( + db, + "Resource", + "some_vector5", + "Resource 1", + 3.0; + date_time = DateTime(1890), + block = 999, + segment = 2, + ) + + df_group1_answer = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 10.0], + some_vector2 = [2.0, 50.0], + ) + df_group3_answer = DataFrame(; + date_time = [ + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2000), + DateTime(2001), + DateTime(2001), + DateTime(2001), + DateTime(2009), + ], + block = [1, 1, 1, 1, 2, 2, 2, 2], + segment = [1, 2, 3, 4, 1, 2, 3, 4], + some_vector5 = [3.0, 10.0, 3.0, 4.0, 1, 2, 3, 4], + some_vector6 = [1.0, 2.0, 33.0, 4.0, 1, 2, 3, 4], + ) + + all_answers = [df_group1_answer, df_group3_answer] + + # iterating over the three groups + + for df_answer in all_answers + for col in names(df_answer) + if startswith(col, "some_vector") + df = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + col, + "Resource 1", + ) + _test_table(df, df_answer) + end + end + end + + PSRDatabaseSQLite.close!(db) + GC.gc() + GC.gc() + rm(db_path) + @test true + return nothing +end + +function test_delete_time_series() + path_schema = joinpath(@__DIR__, "test_read_time_series.sql") + db_path = joinpath(@__DIR__, "test_delete_time_series.sqlite") + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + df_time_series_group1 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 2.0], + some_vector2 = [2.0, 3.0], + ) + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df_time_series_group1, + ) + + PSRDatabaseSQLite.delete_time_series!( + db, + "Resource", + "group1", + "Resource 1", + ) + + df = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + "some_vector1", + "Resource 1", + ) + + @test isempty(df) + + df = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + "some_vector2", + "Resource 1", + ) + + @test isempty(df) + + PSRDatabaseSQLite.close!(db) + GC.gc() + GC.gc() + rm(db_path) + @test true + return nothing +end + +function test_create_wrong_time_series() + path_schema = joinpath(@__DIR__, "test_read_time_series.sql") + db_path = joinpath(@__DIR__, "test_create_wrong_time_series.sqlite") + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + df_time_series_group1_wrong = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 2.0], + some_vector20 = [2.0, 3.0], + ) + + df_time_series_group1_wrong2 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + block = [1, 2], + some_vector1 = [1.0, 2.0], + some_vector2 = [2.0, 3.0], + ) + + df_time_series_group1_wrong3 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + something = [1, 2], + some_vector1 = [1.0, 2.0], + some_vector2 = [2.0, 3.0], + ) + + df_time_series_group1 = DataFrame(; + date_time = [DateTime(2000), DateTime(2001)], + some_vector1 = [1.0, 2.0], + some_vector2 = [2.0, 3.0], + ) + + @test_throws PSRDatabaseSQLite.DatabaseException PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df_time_series_group1_wrong, + ) + + @test_throws PSRDatabaseSQLite.DatabaseException PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df_time_series_group1_wrong2, + ) + + @test_throws PSRDatabaseSQLite.DatabaseException PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df_time_series_group1_wrong3, + ) + + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + group1 = df_time_series_group1, + ) + + PSRDatabaseSQLite.close!(db) + GC.gc() + GC.gc() + rm(db_path) + @test true + return nothing +end + +function test_add_time_series_row() + path_schema = joinpath(@__DIR__, "test_read_time_series.sql") + db_path = joinpath(@__DIR__, "test_add_time_series_row.sqlite") + db = PSRDatabaseSQLite.create_empty_db_from_schema(db_path, path_schema; force = true) + + PSRDatabaseSQLite.create_element!(db, "Configuration"; label = "Toy Case", value1 = 1.0) + + PSRDatabaseSQLite.create_element!( + db, + "Resource"; + label = "Resource 1", + ) + + PSRDatabaseSQLite.add_time_series_row!( + db, + "Resource", + "some_vector1", + "Resource 1", + 1.0; + date_time = DateTime(2000), + ) + + PSRDatabaseSQLite.add_time_series_row!( + db, + "Resource", + "some_vector2", + "Resource 1", + 2.0; + date_time = DateTime(2000), + ) + + PSRDatabaseSQLite.add_time_series_row!( + db, + "Resource", + "some_vector3", + "Resource 1", + 3.0; + date_time = DateTime(2001), + block = 1, + ) + + PSRDatabaseSQLite.add_time_series_row!( + db, + "Resource", + "some_vector4", + "Resource 1", + 4.0; + date_time = DateTime(2001), + block = 1, + ) + + # Attribute is not a time series + @test_throws PSRDatabaseSQLite.DatabaseException PSRDatabaseSQLite.add_time_series_row!( + db, + "Resource", + "label", + "Resource 1", + 4.0; + date_time = DateTime(2001), + ) + + # Wrong dimensions + @test_throws PSRDatabaseSQLite.DatabaseException PSRDatabaseSQLite.add_time_series_row!( + db, + "Resource", + "some_vector1", + "Resource 1", + 4.0; + date_time = DateTime(2001), + block = 1, + segment = 1, + ) + + df_some_vector1 = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + "some_vector1", + "Resource 1", + ) + + df_some_vector2 = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + "some_vector2", + "Resource 1", + ) + + df_some_vector3 = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + "some_vector3", + "Resource 1", + ) + + df_some_vector4 = PSRDatabaseSQLite.read_time_series_table( + db, + "Resource", + "some_vector4", + "Resource 1", + ) + + @test df_some_vector1[1, :some_vector1] == 1.0 + @test df_some_vector2[1, :some_vector2] == 2.0 + @test df_some_vector3[1, :some_vector3] == 3.0 + @test df_some_vector4[1, :some_vector4] == 4.0 + + PSRDatabaseSQLite.close!(db) + GC.gc() + GC.gc() + rm(db_path) + return nothing +end + +function runtests() + Base.GC.gc() + Base.GC.gc() + for name in names(@__MODULE__; all = true) + if startswith("$name", "test_") + @testset "$(name)" begin + getfield(@__MODULE__, name)() + end + end + end +end + +TestTimeController.runtests() + +end diff --git a/test/PSRDatabaseSQLite/test_update/test_create_time_series_files.sql b/test/PSRDatabaseSQLite/test_update/test_create_time_series_files.sql index f9d0588b..daed5bd4 100644 --- a/test/PSRDatabaseSQLite/test_update/test_create_time_series_files.sql +++ b/test/PSRDatabaseSQLite/test_update/test_create_time_series_files.sql @@ -14,7 +14,7 @@ CREATE TABLE Resource ( type TEXT NOT NULL DEFAULT "D" CHECK(type IN ('D', 'E', 'F')) ) STRICT; -CREATE TABLE Resource_timeseriesfiles ( +CREATE TABLE Resource_time_series_files ( wind_speed TEXT, wind_direction TEXT ) STRICT; \ No newline at end of file diff --git a/test/PSRDatabaseSQLite/test_update/test_update_time_series.sql b/test/PSRDatabaseSQLite/test_update/test_update_time_series.sql index 991ff13c..d4eb9048 100644 --- a/test/PSRDatabaseSQLite/test_update/test_update_time_series.sql +++ b/test/PSRDatabaseSQLite/test_update/test_update_time_series.sql @@ -11,7 +11,7 @@ CREATE TABLE Plant ( label TEXT UNIQUE NOT NULL ); -CREATE TABLE Plant_timeseriesfiles ( +CREATE TABLE Plant_time_series_files ( generation TEXT ); @@ -20,7 +20,7 @@ CREATE TABLE Resource ( label TEXT UNIQUE NOT NULL ); -CREATE TABLE Resource_timeseriesfiles ( +CREATE TABLE Resource_time_series_files ( generation TEXT, other_generation TEXT ); \ No newline at end of file diff --git a/test/PSRDatabaseSQLite/test_valid_database_definitions/test_valid_database.sql b/test/PSRDatabaseSQLite/test_valid_database_definitions/test_valid_database.sql index 6ee4811c..24393db8 100644 --- a/test/PSRDatabaseSQLite/test_valid_database_definitions/test_valid_database.sql +++ b/test/PSRDatabaseSQLite/test_valid_database_definitions/test_valid_database.sql @@ -60,7 +60,7 @@ CREATE TABLE Plant_vector_cost_relation ( PRIMARY KEY (id, vector_index) ) STRICT; -CREATE TABLE Plant_timeseriesfiles ( +CREATE TABLE Plant_time_series_files ( generation TEXT, cost TEXT ) STRICT; \ No newline at end of file