From 000b01e2377870b70413ede16f4ce1ed2c8d56fd Mon Sep 17 00:00:00 2001 From: Samuel Neumann Date: Tue, 20 Jun 2023 09:38:07 -0600 Subject: [PATCH 1/3] Save arbitrary dimension `AbstractArray`s to SQL Added the ability to save `AbstractArray`s of arbitrary dimensions to a SQL table. Added a return to the main experiment of `examples/experiment.jl` to demonstrate this. --- examples/configs/arg_iter_config_sql.toml | 8 ++-- examples/experiment.jl | 6 ++- src/save/sql_manager.jl | 47 +++++++++++++++++------ 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/examples/configs/arg_iter_config_sql.toml b/examples/configs/arg_iter_config_sql.toml index 044e6af..77de1b7 100644 --- a/examples/configs/arg_iter_config_sql.toml +++ b/examples/configs/arg_iter_config_sql.toml @@ -6,7 +6,7 @@ save_backend="mysql" # mysql only database backend supported database="test_iter" # for params and resutsl save_dir="test_iter_sql" # for exceptions, settings, and more! -exp_file="experiment.jl" +exp_file="examples/experiment.jl" exp_module_name = "TestExperiment" exp_func_name = "main_experiment" arg_iter_type = "iter" @@ -15,6 +15,6 @@ arg_iter_type = "iter" steps=102902 [sweep_args] -opt2 = [5,6,7,8,9] -opt1 = "1:50" -"opt3+opt4" = [["a", 1], ["b", 2], ["c", 3]] \ No newline at end of file +opt2 = [5, 6] +opt1 = "1:2" +"opt3+opt4" = [["a", 1], ["b", 2]] diff --git a/examples/experiment.jl b/examples/experiment.jl index 076159e..1e11a87 100644 --- a/examples/experiment.jl +++ b/examples/experiment.jl @@ -17,7 +17,7 @@ import Reproduce: - `opt1::Int`: The first argument. Experiment errors on `opt1 == 2` - `opt2::Int`: The second argument. - `opt2::String`: The Third argument. This is required to be a string - - `opt4::Int`: The fourth argument. + - `opt4::Int`: The fourth argument. """ opt1 => 1 opt2 => 2 @@ -44,7 +44,9 @@ function main_experiment(config::Dict, extra_arg = nothing; progress=false, test Dict("mean"=>0.1, "vec"=>rand(100), "mat"=>rand(10, 10), - "vec_vec"=>[rand(10) for _ in 1:10]) + "vec_vec"=>[rand(10) for _ in 1:10], + "3darr"=>reshape(collect(1:27), 3, 3, 3), + ) end end diff --git a/src/save/sql_manager.jl b/src/save/sql_manager.jl index b505786..682fa4c 100644 --- a/src/save/sql_manager.jl +++ b/src/save/sql_manager.jl @@ -51,7 +51,7 @@ end Create the tables to store the results. """ function create_results_tables(dbm::DBManager, results) - + tbl_name = get_results_table_name() if table_exists(dbm, tbl_name) return @@ -64,7 +64,7 @@ function create_results_tables(dbm::DBManager, results) # add Hash push!(names, HASH_KEY) push!(types, get_hash_type()) - + for k in keys(results) if results[k] isa AbstractArray # Do crazy things... @@ -78,22 +78,22 @@ function create_results_tables(dbm::DBManager, results) # push!(names, string(k)) # push!(types, "BOOLEAN NOT NULL DEFAULT 0") - + # create_results_subtable(dbm, k, eltype(results[k])) # elseif results[k] isa DataType && results[k] <: AbstractVector # push!(names, string(k)) # push!(types, "BOOLEAN NOT NULL DEFAULT 0") - + # create_results_subtable(dbm, k, results[k].parameters[1]) - + else # add to types nms, dtys = get_sql_schema(string(k), results[k]) append!(names, nms isa String ? [nms] : nms) append!(types, dtys isa String ? [dtys] : dtys) - + end end @@ -106,7 +106,7 @@ function create_results_subtable(dbm::DBManager, key, elt) if table_exists(dbm, tbl_name) return end - + create_array_table(dbm, tbl_name, elt) end @@ -135,12 +135,14 @@ function get_array_table_sql_statement(tbl_name, data::AbstractArray) sql *= "step_$(i) INT UNSIGNED, " end sql *= "INDEX (_HASH));" + @show sql + return sql end function create_array_table(dbm::DBManager, tbl_name, data) sql = get_array_table_sql_statement(tbl_name, data) - + try close!(execute(dbm, sql)) catch err @@ -163,8 +165,8 @@ end function save_params(dbm::DBManager, params; filter_keys = String[], use_git_info = true) # returns hash p_names, p_values = get_sql_names_values(params) - - + + # hash key pms_hash = hash_params(params; filter_keys=filter_keys) push!(p_names, HASH_KEY) @@ -196,7 +198,7 @@ function save_results(dbm::DBManager, pms_hash, results) # save to sub table save_sub_results(dbm, pms_hash, k, results[k]) - + names *= "$(k)" values *= "true" @@ -230,7 +232,7 @@ function save_results(dbm::DBManager, pms_hash, results) end append_row(dbm, get_results_table_name(), names, values) - + end @@ -255,6 +257,27 @@ function save_sub_results(dbm::DBManager, pms_hash, key, results::AbstractMatrix end +function save_sub_results(dbm::DBManager, pms_hash, key, results::AbstractArray) + tbl_name = get_results_subtable_name(key) + + indices = map(x -> range(1, x), size(results)) + + for inds in Iterators.product(indices...) + v = results[inds...] + + names = "(" * HASH_KEY * ", " + values = "($(pms_hash), " + for i in 1:length(inds) + names *= "step_$i, " + values *= "$(inds[i]), " + end + names *= "data)" + values *= "$(v))" + + append_row(dbm, tbl_name, names, values) + end +end + function save_sub_results(dbm::DBManager, pms_hash, key, results::AbstractVector{V}) where {V<:AbstractVector} tbl_name = get_results_subtable_name(key) for (i, vec) in enumerate(results) From 937fa14deaff1ce4cf300cba10546b0c92d9611a Mon Sep 17 00:00:00 2001 From: Samuel Neumann Date: Wed, 21 Jun 2023 15:51:13 -0600 Subject: [PATCH 2/3] Revert `examples/configs/arg_iter_config_sql.toml` --- examples/configs/arg_iter_config_sql.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/configs/arg_iter_config_sql.toml b/examples/configs/arg_iter_config_sql.toml index 77de1b7..b022732 100644 --- a/examples/configs/arg_iter_config_sql.toml +++ b/examples/configs/arg_iter_config_sql.toml @@ -6,7 +6,7 @@ save_backend="mysql" # mysql only database backend supported database="test_iter" # for params and resutsl save_dir="test_iter_sql" # for exceptions, settings, and more! -exp_file="examples/experiment.jl" +exp_file="experiment.jl" exp_module_name = "TestExperiment" exp_func_name = "main_experiment" arg_iter_type = "iter" From e269b504b8324037afe17d18b6b02a39f7d6133d Mon Sep 17 00:00:00 2001 From: Samuel Neumann Date: Thu, 22 Jun 2023 08:58:24 -0600 Subject: [PATCH 3/3] Revert changes to `arg_iter_config_sql.toml` --- examples/configs/arg_iter_config_sql.toml | 6 +++--- examples/experiment.jl | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/configs/arg_iter_config_sql.toml b/examples/configs/arg_iter_config_sql.toml index b022732..7fc06fe 100644 --- a/examples/configs/arg_iter_config_sql.toml +++ b/examples/configs/arg_iter_config_sql.toml @@ -15,6 +15,6 @@ arg_iter_type = "iter" steps=102902 [sweep_args] -opt2 = [5, 6] -opt1 = "1:2" -"opt3+opt4" = [["a", 1], ["b", 2]] +opt2 = [5,6,7,8,9] +opt1 = "1:50" +"opt3+opt4" = [["a", 1], ["b", 2], ["c", 3]] diff --git a/examples/experiment.jl b/examples/experiment.jl index 1e11a87..8123f3a 100644 --- a/examples/experiment.jl +++ b/examples/experiment.jl @@ -41,11 +41,12 @@ function main_experiment(config::Dict, extra_arg = nothing; progress=false, test throw("Oh No!!!") end - Dict("mean"=>0.1, + Dict( + "mean"=>0.1, "vec"=>rand(100), "mat"=>rand(10, 10), - "vec_vec"=>[rand(10) for _ in 1:10], "3darr"=>reshape(collect(1:27), 3, 3, 3), + "vec_vec"=>[rand(10) for _ in 1:10], ) end end