diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 70735da1..e0c95e93 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,9 +3,7 @@ steps: plugins: - JuliaCI/julia#v1: version: "1" - - JuliaCI/julia-test#v1: - - JuliaCI/julia-coverage#v1: - codecov: true + - JuliaCI/julia-test#v1: ~ agents: queue: "juliagpu" cuda: "*" diff --git a/Project.toml b/Project.toml index 6340de39..dc34ef05 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MLJFlux" uuid = "094fc8d1-fd35-5302-93ea-dabda2abf845" authors = ["Anthony D. Blaom ", "Ayush Shridhar "] -version = "0.2.5" +version = "0.2.6" [deps] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" diff --git a/README.md b/README.md index 09d3edcc..024dfff5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ An interface to the Flux deep learning models for the [MLJ](https://github.com/alan-turing-institute/MLJ.jl) machine -learning framework +learning framework. | Branch | Julia | CPU CI | GPU CI | Coverage | | -------- | ----- | ------ | ----- | -------- | @@ -23,7 +23,7 @@ learning framework MLJFlux makes it possible to apply the machine learning meta-algorithms provided by MLJ - such as out-of-sample performance -evaluation and hyper-parameter optimization - to some classes of +evaluation, hyper-parameter optimization, and iteration control - to some classes of **supervised deep learning models**. It does this by providing an interface to the [Flux](https://fluxml.ai/Flux.jl/stable/) framework. diff --git a/examples/boston/Manifest.toml b/examples/boston/Manifest.toml new file mode 100644 index 00000000..f45eecff --- /dev/null +++ b/examples/boston/Manifest.toml @@ -0,0 +1,2 @@ +# This file is machine-generated - editing it directly is not advised + diff --git a/examples/boston/Project.toml b/examples/boston/Project.toml new file mode 100644 index 00000000..e69de29b diff --git a/examples/boston/README.md b/examples/boston/README.md new file mode 100644 index 00000000..84a3ec6f --- /dev/null +++ b/examples/boston/README.md @@ -0,0 +1,4 @@ +The files notebook.* in this directory cannot be executed without the +accomanying env/ directory, which contains the package environment +used. + diff --git a/examples/boston/assets/learning_rate.png b/examples/boston/assets/learning_rate.png new file mode 100644 index 00000000..5244f79d Binary files /dev/null and b/examples/boston/assets/learning_rate.png differ diff --git a/examples/boston/assets/loss.png b/examples/boston/assets/loss.png new file mode 100644 index 00000000..81d39d60 Binary files /dev/null and b/examples/boston/assets/loss.png differ diff --git a/examples/boston/assets/performance_vs_learning_rate.png b/examples/boston/assets/performance_vs_learning_rate.png new file mode 100644 index 00000000..57bf388d Binary files /dev/null and b/examples/boston/assets/performance_vs_learning_rate.png differ diff --git a/examples/boston/boston.jl b/examples/boston/boston.jl new file mode 100644 index 00000000..b990ba6a --- /dev/null +++ b/examples/boston/boston.jl @@ -0,0 +1,250 @@ +# # Using training an MLJFlux regression model on the Boston house +# # price dataset + +using Pkg +Pkg.activate(@__DIR__) +Pkg.instantiate() + +# **Julia version** is assumed to be 1.6.* + +using MLJ +using MLJFlux +using Flux + +MLJ.color_off() + +using Plots + +# This tutorial uses some MLJ's `IteratedModel` wrapper wrappers, to +# transform the MLJFLux `NeuralNetworkRegressor` into a model that +# **automatically selects the number of epochs** required to optimize +# an out-of-sample loss. + +# We also show how to include the model in a **pipeline** to carry out +# standardization of the features and target. + + +# ## Loading data + +data = OpenML.load(531); # Loads from https://www.openml.org/d/531 + +# The target `y` is `:MEDV` and everything else except `:CHAS` goes +# into the features `X`: + +y, X = unpack(data, ==(:MEDV), !=(:CHAS)); + +# (The Charles River dummy variable, `:CHAS`, is not deemed to be +# relevant.) + +# Inspect the scientific types: + +scitype(y) + +#- + +schema(X) + +# We'll regard `:RAD` (index of accessibility to radial highways) as +# `Continuous` as Flux models don't handle ordered factors: + +X = coerce(X, :RAD => Continuous); + +# Let's split off a test set for final testing: + +X, Xtest = partition(X, 0.7); +y, ytest = partition(y, 0.7); + +# ## Defining a builder + +# In the macro call below, `n_in` is expected to represent the number +# of inputs features and `rng` a RNG (builders are generic, ie can be +# applied to data with any number of input features): + +builder = MLJFlux.@builder begin + init=Flux.glorot_uniform(rng) + Chain(Dense(n_in, 64, relu, init=init), + Dense(64, 32, relu, init=init), + Dense(32, 1, init=init)) +end + + +# ## Defining a MLJFlux model: + +NeuralNetworkRegressor = @load NeuralNetworkRegressor + model = NeuralNetworkRegressor(builder=builder, + rng=123, + epochs=20) + + +# ## Standardization + +# The following wraps our regressor in feature and target standardizations: + +pipe = @pipeline Standardizer model target=Standardizer + +# Notice that our original neural network model is now a +# hyper-parameter of the composite `pipe`, with the automatically +# generated name, `:neural_network_regressor`. + + +# ## Choosing an initial learning rate + +# Let's see how the training losses look for the default optimiser. For +# MLJFlux models, `fit!` will print these losses if we bump the +# verbosity level (default is always 1): + +mach = machine(pipe, X, y) +fit!(mach, verbosity=2) + +# They are also extractable from the training report (which includes +# the pre-train loss): + +report(mach).neural_network_regressor.training_losses + +# Next, let's visually compare a few learning rates: + +plt = plot() +rates = [5e-4, 0.001, 0.005, 0.01] + +# By default, changing only the optimiser will not trigger a +# cold-restart when we `fit!` (to allow for adaptive learning rate +# control). So we call `fit!` with the `force=true` option. We'll skip +# the first few losses to get a better vertical scale in our plot. + +foreach(rates) do η + pipe.neural_network_regressor.optimiser.eta = η + fit!(mach, force=true) + losses = report(mach).neural_network_regressor.training_losses[3:end] + plot!(:length(losses), losses, label=η) +end +plt #!md + +#- + +savefig("learning_rate.png") + +# ![](learing_rate.png) #md + +# We'll use a relatively conservative rate here: + +pipe.neural_network_regressor.optimiser.eta = 0.001 + + +# ## Wrapping in iteration control + +# We want a model that trains until an out-of-sample loss satisfies +# the `NumberSinceBest(4)` stopping criterion. We'll add some fallback +# stopping criterion `InvalidValue` and `TimeLimit(1/60)`, and +# controls to print traces of the losses. + +# For intializing or clearing the traces: + +clear() = begin + global losses = [] + global training_losses = [] + global epochs = [] + return nothing +end + +# And to update the traces: + +update_loss(loss) = push!(losses, loss) +update_training_loss(report) = + push!(training_losses, report.neural_network_regressor.training_losses[end]) +update_epochs(epoch) = push!(epochs, epoch) + +# The controls to apply: + +controls=[Step(1), + NumberSinceBest(4), + InvalidValue(), + TimeLimit(1/60), + WithLossDo(update_loss), + WithReportDo(update_training_loss), + WithIterationsDo(update_epochs)] + +# Next we create a "self-iterating" version of the pipeline. Note +# that the iteration parameter is a nested hyperparameter: + +iterated_pipe = + IteratedModel(model=pipe, + controls=controls, + resampling=Holdout(fraction_train=0.8), + iteration_parameter=:(neural_network_regressor.epochs), + measure = l2) + +# Training the wrapped model on all the train/validation data: + +clear() +mach = machine(iterated_pipe, X, y) +fit!(mach) + +# And plotting the traces: + +plot(epochs, losses, + xlab = "epoch", + ylab = "mean sum of sqaures error", + label="out-of-sample", + legend = :topleft); +scatter!(twinx(), epochs, training_losses, label="training", color=:red) #!md + +#- + +savefig("loss.png") + +# ![](losses.png) #md + +# **How `IteratedModel` works.** Training an `IteratedModel` means +# holding out some data (80% in this case) so an out-of-sample loss +# can be tracked and used in the specified stopping criterion, +# NumberSinceBest(4)`. However, once the stop is triggered, the model +# wrapped by `IteratedModel` (our pipeline model) is retrained on all +# data for the same number of iterations. Calling `predict(mach, +# Xnew)` on new data uses the updated learned parameters. + +# In other words, `iterated_model` is a "self-iterating" version of +# the original model, where `epochs` has been transformed from +# hyper-parameter to *learned* parameter. + + +# ## An evaluation the self-iterating model + +# Here's an estimate of performance of our "self-iterating" +# model: + +e = evaluate!(mach, + resampling=CV(nfolds=5), + measures=[l2, l1]) + +using Measurements +err = e.measurement[1] ± std(e.per_fold[1])/sqrt(4) +@show err + +# which we can see has substantial uncertainty. + + +# ## Comparing the model with other models + +# Although we cannot assign them statistical significance, here are +# comparisons of our fully automated model with a couple other models +# (trained using default hyperparameters): + +function performance(model) + mach = machine(model, X, y) |> fit! + yhat = predict(mach, Xtest) + l2(yhat, ytest) |> mean +end +performance(iterated_pipe) + +three_models = [(@load EvoTreeRegressor)(), # tree boosting model + (@load LinearRegressor pkg=MLJLinearModels)(), + tuned_iterated_pipe] + +errs = performance.(three_models) + +(models=typeof.(three_models), mean_square_errors=errs) |> pretty + +# So, apparently better than linear, but no match for the +# tree-booster. And our cv-estimate of performance on the +# train/validate dataset is wildly optimistic. + diff --git a/examples/boston/env/Manifest.toml b/examples/boston/env/Manifest.toml new file mode 100644 index 00000000..67c59570 --- /dev/null +++ b/examples/boston/env/Manifest.toml @@ -0,0 +1,1411 @@ +# This file is machine-generated - editing it directly is not advised + +[[ARFFFiles]] +deps = ["CategoricalArrays", "Dates", "Parsers", "Tables"] +git-tree-sha1 = "3556fa90c0bea9f965388c0e123418cb9f5ff2e3" +uuid = "da404889-ca92-49ff-9e8b-0aa6b4d38dc8" +version = "1.4.0" + +[[AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "485ee0867925449198280d4af84bdb46a2a404d0" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.0.1" + +[[AbstractTrees]] +git-tree-sha1 = "03e0550477d86222521d254b741d470ba17ea0b5" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.3.4" + +[[Adapt]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "84918055d15b3114ede17ac6a7182f68870c16f7" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.3.1" + +[[ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" + +[[ArrayInterface]] +deps = ["Compat", "IfElse", "LinearAlgebra", "Requires", "SparseArrays", "Static"] +git-tree-sha1 = "e527b258413e0c6d4f66ade574744c94edef81f8" +uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +version = "3.1.40" + +[[Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[BFloat16s]] +deps = ["LinearAlgebra", "Printf", "Random", "Test"] +git-tree-sha1 = "a598ecb0d717092b5539dbbe890c98bac842b072" +uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" +version = "0.2.0" + +[[BSON]] +git-tree-sha1 = "ebcd6e22d69f21249b7b8668351ebf42d6dc87a1" +uuid = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0" +version = "0.3.4" + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "19a35467a82e236ff51bc17a3a44b69ef35185a2" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.8+0" + +[[CEnum]] +git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.4.1" + +[[CUDA]] +deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CompilerSupportLibraries_jll", "ExprTools", "GPUArrays", "GPUCompiler", "LLVM", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "SpecialFunctions", "TimerOutputs"] +git-tree-sha1 = "2c8329f16addffd09e6ca84c556e2185a4933c64" +uuid = "052768ef-5323-5732-b1bb-66c8b64840ba" +version = "3.5.0" + +[[Cairo_jll]] +deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "f2202b55d816427cd385a9a4f3ffb226bee80f99" +uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" +version = "1.16.1+0" + +[[Calculus]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad" +uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" +version = "0.5.1" + +[[CategoricalArrays]] +deps = ["DataAPI", "Future", "Missings", "Printf", "Requires", "Statistics", "Unicode"] +git-tree-sha1 = "fbc5c413a005abdeeb50ad0e54d85d000a1ca667" +uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597" +version = "0.10.1" + +[[ChainRules]] +deps = ["ChainRulesCore", "Compat", "LinearAlgebra", "Random", "RealDot", "Statistics"] +git-tree-sha1 = "035ef8a5382a614b2d8e3091b6fdbb1c2b050e11" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.12.1" + +[[ChainRulesCore]] +deps = ["Compat", "LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "f885e7e7c124f8c92650d61b9477b9ac2ee607dd" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.11.1" + +[[CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "ded953804d019afa9a3f98981d99b33e3db7b6da" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.0" + +[[ColorSchemes]] +deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random"] +git-tree-sha1 = "a851fec56cb73cfdf43762999ec72eff5b86882a" +uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +version = "3.15.0" + +[[ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "024fe24d83e4a5bf5fc80501a314ce0d1aa35597" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.11.0" + +[[Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "417b0ed7b8b838aa6ca0a87aadf1bb9eb111ce40" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.8" + +[[CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "dce3e3fea680869eaa0b774b2e8343e9ff442313" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "3.40.0" + +[[CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" + +[[ComputationalResources]] +git-tree-sha1 = "52cb3ec90e8a8bea0e62e275ba577ad0f74821f7" +uuid = "ed09eef8-17a6-5b46-8889-db040fac31e3" +version = "0.3.2" + +[[Contour]] +deps = ["StaticArrays"] +git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7" +uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" +version = "0.5.7" + +[[Crayons]] +git-tree-sha1 = "3f71217b538d7aaee0b69ab47d9b7724ca8afa0d" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.0.4" + +[[DataAPI]] +git-tree-sha1 = "cc70b17275652eb47bc9e5f81635981f13cea5c8" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.9.0" + +[[DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "7d9d316f04214f7efdbb6398d545446e246eff02" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.10" + +[[DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[DiffResults]] +deps = ["StaticArrays"] +git-tree-sha1 = "c18e98cba888c6c25d1c3b048e4b3380ca956805" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.0.3" + +[[DiffRules]] +deps = ["LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "3287dacf67c3652d3fed09f4c12c187ae4dbb89a" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.4.0" + +[[Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "837c83e5574582e07662bbbba733964ff7c26b9d" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.6" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[Distributions]] +deps = ["ChainRulesCore", "FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"] +git-tree-sha1 = "72dcda9e19f88d09bf21b5f9507a0bb430bce2aa" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.24" + +[[DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.8.6" + +[[Downloads]] +deps = ["ArgTools", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" + +[[EarCut_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "3f3a2501fa7236e9b911e0f7a588c657e822bb6d" +uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5" +version = "2.2.3+0" + +[[EarlyStopping]] +deps = ["Dates", "Statistics"] +git-tree-sha1 = "ea0b56527cefce87419d4b7559811bd96974a6c8" +uuid = "792122b4-ca99-40de-a6bc-6742525f08b6" +version = "0.1.9" + +[[EvoTrees]] +deps = ["BSON", "CUDA", "CategoricalArrays", "Distributions", "MLJModelInterface", "NetworkLayout", "Random", "RecipesBase", "StaticArrays", "Statistics", "StatsBase"] +git-tree-sha1 = "556a81b4b1bd7a317c82ffc595ed0fe9fcd03d85" +uuid = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5" +version = "0.8.5" + +[[Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "b3bfd02e98aedfa5cf885665493c5598c350cd2f" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.2.10+0" + +[[ExprTools]] +git-tree-sha1 = "b7e3d17636b348f005f11040025ae8c6f645fe92" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.6" + +[[FFMPEG]] +deps = ["FFMPEG_jll"] +git-tree-sha1 = "b57e3acbe22f8484b4b5ff66a7499717fe1a9cc8" +uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" +version = "0.4.1" + +[[FFMPEG_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] +git-tree-sha1 = "d8a578692e3077ac998b50c0217dfd67f21d1e5f" +uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" +version = "4.4.0+0" + +[[FilePathsBase]] +deps = ["Dates", "Mmap", "Printf", "Test", "UUIDs"] +git-tree-sha1 = "d962b5a47b6d191dbcd8ae0db841bc70a05a3f5b" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.13" + +[[FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "Statistics"] +git-tree-sha1 = "8756f9935b7ccc9064c6eef0bff0ad643df733a3" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "0.12.7" + +[[FiniteDiff]] +deps = ["ArrayInterface", "LinearAlgebra", "Requires", "SparseArrays", "StaticArrays"] +git-tree-sha1 = "8b3c09b56acaf3c0e581c66638b85c8650ee9dca" +uuid = "6a86dc24-6348-571c-b903-95158fe2bd41" +version = "2.8.1" + +[[FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.4" + +[[Flux]] +deps = ["AbstractTrees", "Adapt", "ArrayInterface", "CUDA", "CodecZlib", "Colors", "DelimitedFiles", "Functors", "Juno", "LinearAlgebra", "MacroTools", "NNlib", "NNlibCUDA", "Pkg", "Printf", "Random", "Reexport", "SHA", "SparseArrays", "Statistics", "StatsBase", "Test", "ZipFile", "Zygote"] +git-tree-sha1 = "e8b37bb43c01eed0418821d1f9d20eca5ba6ab21" +uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" +version = "0.12.8" + +[[Fontconfig_jll]] +deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03" +uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" +version = "2.13.93+0" + +[[Formatting]] +deps = ["Printf"] +git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" +uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" +version = "0.4.2" + +[[ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions", "StaticArrays"] +git-tree-sha1 = "ef3fec65f9db26fa2cf8f4133c697c5b7ce63c1d" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.22" + +[[FreeType2_jll]] +deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "87eb71354d8ec1a96d4a7636bd57a7347dde3ef9" +uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" +version = "2.10.4+0" + +[[FriBidi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91" +uuid = "559328eb-81f9-559d-9380-de523a88c83c" +version = "1.0.10+0" + +[[Functors]] +git-tree-sha1 = "e4768c3b7f597d5a352afa09874d16e3c3f6ead2" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.2.7" + +[[Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[GLFW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libglvnd_jll", "Pkg", "Xorg_libXcursor_jll", "Xorg_libXi_jll", "Xorg_libXinerama_jll", "Xorg_libXrandr_jll"] +git-tree-sha1 = "0c603255764a1fa0b61752d2bec14cfbd18f7fe8" +uuid = "0656b61e-2033-5cc2-a64a-77c0f6c09b89" +version = "3.3.5+1" + +[[GPUArrays]] +deps = ["Adapt", "LinearAlgebra", "Printf", "Random", "Serialization", "Statistics"] +git-tree-sha1 = "7772508f17f1d482fe0df72cabc5b55bec06bbe0" +uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" +version = "8.1.2" + +[[GPUCompiler]] +deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "TimerOutputs", "UUIDs"] +git-tree-sha1 = "77d915a0af27d474f0aaf12fcd46c400a552e84c" +uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" +version = "0.13.7" + +[[GR]] +deps = ["Base64", "DelimitedFiles", "GR_jll", "HTTP", "JSON", "Libdl", "LinearAlgebra", "Pkg", "Printf", "Random", "Serialization", "Sockets", "Test", "UUIDs"] +git-tree-sha1 = "30f2b340c2fff8410d89bfcdc9c0a6dd661ac5f7" +uuid = "28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71" +version = "0.62.1" + +[[GR_jll]] +deps = ["Artifacts", "Bzip2_jll", "Cairo_jll", "FFMPEG_jll", "Fontconfig_jll", "GLFW_jll", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Libtiff_jll", "Pixman_jll", "Pkg", "Qt5Base_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "fd75fa3a2080109a2c0ec9864a6e14c60cca3866" +uuid = "d2c73de3-f751-5644-a686-071e5b155ba9" +version = "0.62.0+0" + +[[GeometryBasics]] +deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"] +git-tree-sha1 = "58bcdf5ebc057b085e58d95c138725628dd7453c" +uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326" +version = "0.4.1" + +[[Gettext_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" +uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" +version = "0.21.0+0" + +[[Glib_jll]] +deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "7bf67e9a481712b3dbe9cb3dac852dc4b1162e02" +uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" +version = "2.68.3+0" + +[[Graphite2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011" +uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" +version = "1.3.14+0" + +[[Grisu]] +git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" +uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" +version = "1.0.2" + +[[HTTP]] +deps = ["Base64", "Dates", "IniFile", "Logging", "MbedTLS", "NetworkOptions", "Sockets", "URIs"] +git-tree-sha1 = "14eece7a3308b4d8be910e265c724a6ba51a9798" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "0.9.16" + +[[HarfBuzz_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] +git-tree-sha1 = "8a954fed8ac097d5be04921d595f741115c1b2ad" +uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" +version = "2.8.1+0" + +[[IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.2" + +[[IRTools]] +deps = ["InteractiveUtils", "MacroTools", "Test"] +git-tree-sha1 = "95215cd0076a150ef46ff7928892bc341864c73c" +uuid = "7869d1d1-7146-5819-86e3-90919afe41df" +version = "0.4.3" + +[[IfElse]] +git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1" +uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" +version = "0.1.1" + +[[IniFile]] +deps = ["Test"] +git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" +uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" +version = "0.5.0" + +[[InlineStrings]] +deps = ["Parsers"] +git-tree-sha1 = "19cb49649f8c41de7fea32d089d37de917b553da" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.0.1" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[InverseFunctions]] +deps = ["Test"] +git-tree-sha1 = "f0c6489b12d28fb4c2103073ec7452f3423bd308" +uuid = "3587e190-3f89-42d0-90ee-14403ec27112" +version = "0.1.1" + +[[InvertedIndices]] +git-tree-sha1 = "bee5f1ef5bf65df56bdd2e40447590b272a5471f" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.1.0" + +[[IrrationalConstants]] +git-tree-sha1 = "7fd44fd4ff43fc60815f8e764c0f352b83c49151" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.1.1" + +[[IterTools]] +git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.3.0" + +[[IterationControl]] +deps = ["EarlyStopping", "InteractiveUtils"] +git-tree-sha1 = "f61d5d4d0e433b3fab03ca5a1bfa2d7dcbb8094c" +uuid = "b3c1a2ee-3fec-4384-bf48-272ea71de57c" +version = "0.4.0" + +[[IterativeSolvers]] +deps = ["LinearAlgebra", "Printf", "Random", "RecipesBase", "SparseArrays"] +git-tree-sha1 = "1169632f425f79429f245113b775a0e3d121457c" +uuid = "42fd0dbc-a981-5370-80f2-aaf504508153" +version = "0.9.2" + +[[IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[JLLWrappers]] +deps = ["Preferences"] +git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.3.0" + +[[JLSO]] +deps = ["BSON", "CodecZlib", "FilePathsBase", "Memento", "Pkg", "Serialization"] +git-tree-sha1 = "e00feb9d56e9e8518e0d60eef4d1040b282771e2" +uuid = "9da8a3cd-07a3-59c0-a743-3fdc52c30d11" +version = "2.6.0" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.2" + +[[JpegTurbo_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "d735490ac75c5cb9f1b00d8b5509c11984dc6943" +uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" +version = "2.1.0+0" + +[[Juno]] +deps = ["Base64", "Logging", "Media", "Profile"] +git-tree-sha1 = "07cb43290a840908a771552911a6274bc6c072c7" +uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d" +version = "0.8.4" + +[[LAME_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c" +uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" +version = "3.100.1+0" + +[[LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"] +git-tree-sha1 = "46092047ca4edc10720ecab437c42283cd7c44f3" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "4.6.0" + +[[LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6a2af408fe809c4f1a54d2b3f188fdd3698549d6" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.11+0" + +[[LZO_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6" +uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" +version = "2.10.1+0" + +[[LaTeXStrings]] +git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.3.0" + +[[Latexify]] +deps = ["Formatting", "InteractiveUtils", "LaTeXStrings", "MacroTools", "Markdown", "Printf", "Requires"] +git-tree-sha1 = "a8f4f279b6fa3c3c4f1adadd78a621b13a506bce" +uuid = "23fbe1c1-3f47-55db-b15f-69d7ec21a316" +version = "0.15.9" + +[[LatinHypercubeSampling]] +deps = ["Random", "StableRNGs", "StatsBase", "Test"] +git-tree-sha1 = "42938ab65e9ed3c3029a8d2c58382ca75bdab243" +uuid = "a5e1c1ea-c99a-51d3-a14d-a9a37257b02d" +version = "1.8.0" + +[[LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[LearnBase]] +git-tree-sha1 = "a0d90569edd490b82fdc4dc078ea54a5a800d30a" +uuid = "7f8f8fb0-2700-5f03-b4bd-41f8cfc144b6" +version = "0.4.1" + +[[LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" + +[[LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" + +[[LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[Libffi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "761a393aeccd6aa92ec3515e428c26bf99575b3b" +uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" +version = "3.2.2+0" + +[[Libgcrypt_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"] +git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae" +uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" +version = "1.8.7+0" + +[[Libglvnd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll", "Xorg_libXext_jll"] +git-tree-sha1 = "7739f837d6447403596a75d19ed01fd08d6f56bf" +uuid = "7e76a0d4-f3c7-5321-8279-8d96eeed0f29" +version = "1.3.0+3" + +[[Libgpg_error_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9" +uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" +version = "1.42.0+0" + +[[Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "42b62845d70a619f063a7da093d995ec8e15e778" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.16.1+1" + +[[Libmount_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73" +uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" +version = "2.35.0+0" + +[[Libtiff_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "Zlib_jll", "Zstd_jll"] +git-tree-sha1 = "340e257aada13f95f98ee352d316c3bed37c8ab9" +uuid = "89763e89-9b03-5906-acba-b20f662cd828" +version = "4.3.0+0" + +[[Libuuid_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066" +uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" +version = "2.36.0+0" + +[[LineSearches]] +deps = ["LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "Printf"] +git-tree-sha1 = "f27132e551e959b3667d8c93eae90973225032dd" +uuid = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" +version = "7.1.1" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[LinearMaps]] +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "dbb14c604fc47aa4f2e19d0ebb7b6416f3cfa5f5" +uuid = "7a12625a-238d-50fd-b39a-03d52299707e" +version = "3.5.1" + +[[Literate]] +deps = ["Base64", "IOCapture", "JSON", "REPL"] +git-tree-sha1 = "d3493acfb9e6aa0cff46b09773fc2342327b0feb" +uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +version = "2.9.4" + +[[LogExpFunctions]] +deps = ["ChainRulesCore", "DocStringExtensions", "InverseFunctions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "6193c3815f13ba1b78a51ce391db8be016ae9214" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.4" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[LossFunctions]] +deps = ["InteractiveUtils", "LearnBase", "Markdown", "RecipesBase", "StatsBase"] +git-tree-sha1 = "0f057f6ea90a84e73a8ef6eebb4dc7b5c330020f" +uuid = "30fc2ffe-d236-52d8-8643-a9d8f7c094a7" +version = "0.7.2" + +[[MLJ]] +deps = ["CategoricalArrays", "ComputationalResources", "Distributed", "Distributions", "LinearAlgebra", "MLJBase", "MLJEnsembles", "MLJIteration", "MLJModels", "MLJSerialization", "MLJTuning", "OpenML", "Pkg", "ProgressMeter", "Random", "ScientificTypes", "Statistics", "StatsBase", "Tables"] +git-tree-sha1 = "2a1ed07cdeeb238bc986235b303d3d73e02118f6" +uuid = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7" +version = "0.16.11" + +[[MLJBase]] +deps = ["CategoricalArrays", "ComputationalResources", "Dates", "DelimitedFiles", "Distributed", "Distributions", "InteractiveUtils", "InvertedIndices", "LinearAlgebra", "LossFunctions", "MLJModelInterface", "Missings", "OrderedCollections", "Parameters", "PrettyTables", "ProgressMeter", "Random", "ScientificTypes", "StatisticalTraits", "Statistics", "StatsBase", "Tables"] +git-tree-sha1 = "fb9e0429525ccbb0fb4528bff2dc3cdba1feab53" +uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d" +version = "0.18.24" + +[[MLJEnsembles]] +deps = ["CategoricalArrays", "ComputationalResources", "Distributed", "Distributions", "MLJBase", "MLJModelInterface", "ProgressMeter", "Random", "ScientificTypes", "StatsBase"] +git-tree-sha1 = "f8ca949d52432b81f621d9da641cf59829ad2c8c" +uuid = "50ed68f4-41fd-4504-931a-ed422449fee0" +version = "0.1.2" + +[[MLJFlux]] +deps = ["CategoricalArrays", "ColorTypes", "ComputationalResources", "Flux", "MLJModelInterface", "ProgressMeter", "Random", "Statistics", "Tables"] +git-tree-sha1 = "0e4846efe287018f87b89d8e922bbcee0d6bc39d" +uuid = "094fc8d1-fd35-5302-93ea-dabda2abf845" +version = "0.2.5" + +[[MLJIteration]] +deps = ["IterationControl", "MLJBase", "Random"] +git-tree-sha1 = "1c94830f8927b10a5653d6e1868c20faccf57be5" +uuid = "614be32b-d00c-4edb-bd02-1eb411ab5e55" +version = "0.3.3" + +[[MLJLinearModels]] +deps = ["DocStringExtensions", "IterativeSolvers", "LinearAlgebra", "LinearMaps", "MLJModelInterface", "Optim", "Parameters"] +git-tree-sha1 = "9eb4f07f23d44a898213eb4927869598442d90bb" +uuid = "6ee0df7b-362f-4a72-a706-9e79364fb692" +version = "0.5.6" + +[[MLJModelInterface]] +deps = ["Random", "ScientificTypesBase", "StatisticalTraits"] +git-tree-sha1 = "0174e9d180b0cae1f8fe7976350ad52f0e70e0d8" +uuid = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" +version = "1.3.3" + +[[MLJModels]] +deps = ["CategoricalArrays", "Dates", "Distances", "Distributions", "InteractiveUtils", "LinearAlgebra", "MLJBase", "MLJModelInterface", "OrderedCollections", "Parameters", "Pkg", "REPL", "Random", "Requires", "ScientificTypes", "Statistics", "StatsBase", "Tables"] +git-tree-sha1 = "25dfbfa33d1e7dabcdec63e00585974f85715851" +uuid = "d491faf4-2d78-11e9-2867-c94bc002c0b7" +version = "0.14.12" + +[[MLJSerialization]] +deps = ["IterationControl", "JLSO", "MLJBase", "MLJModelInterface"] +git-tree-sha1 = "cd6285f95948fe1047b7d6fd346c172e247c1188" +uuid = "17bed46d-0ab5-4cd4-b792-a5c4b8547c6d" +version = "1.1.2" + +[[MLJTuning]] +deps = ["ComputationalResources", "Distributed", "Distributions", "LatinHypercubeSampling", "MLJBase", "ProgressMeter", "Random", "RecipesBase"] +git-tree-sha1 = "8f3911fa3aef4299059f573cf75669d61f8bcef5" +uuid = "03970b2e-30c4-11ea-3135-d1576263f10f" +version = "0.6.14" + +[[MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "3d3e902b31198a27340d0bf00d6ac452866021cf" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.9" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"] +git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.0.3" + +[[MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" + +[[Measurements]] +deps = ["Calculus", "LinearAlgebra", "Printf", "RecipesBase", "Requires"] +git-tree-sha1 = "31c8c0569b914111c94dd31149265ed47c238c5b" +uuid = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" +version = "2.6.0" + +[[Measures]] +git-tree-sha1 = "e498ddeee6f9fdb4551ce855a46f54dbd900245f" +uuid = "442fdcdd-2543-5da2-b0f3-8c86c306513e" +version = "0.3.1" + +[[Media]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "75a54abd10709c01f1b86b84ec225d26e840ed58" +uuid = "e89f7d12-3494-54d1-8411-f7d8b9ae1f27" +version = "0.5.0" + +[[Memento]] +deps = ["Dates", "Distributed", "JSON", "Serialization", "Sockets", "Syslogs", "Test", "TimeZones", "UUIDs"] +git-tree-sha1 = "19650888f97362a2ae6c84f0f5f6cda84c30ac38" +uuid = "f28f55f0-a522-5efc-85c2-fe41dfb9b2d9" +version = "1.2.0" + +[[Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "bf210ce90b6c9eed32d25dbcae1ebc565df2687f" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.0.2" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[Mocking]] +deps = ["Compat", "ExprTools"] +git-tree-sha1 = "29714d0a7a8083bba8427a4fbfb00a540c681ce7" +uuid = "78c3b35d-d492-501b-9361-3d52fe80e533" +version = "0.7.3" + +[[MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" + +[[NLSolversBase]] +deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"] +git-tree-sha1 = "50310f934e55e5ca3912fb941dec199b49ca9b68" +uuid = "d41bc354-129a-5804-8e4c-c37616107c6c" +version = "7.8.2" + +[[NNlib]] +deps = ["Adapt", "ChainRulesCore", "Compat", "LinearAlgebra", "Pkg", "Requires", "Statistics"] +git-tree-sha1 = "5203a4532ad28c44f82c76634ad621d7c90abcbd" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.7.29" + +[[NNlibCUDA]] +deps = ["CUDA", "LinearAlgebra", "NNlib", "Random", "Statistics"] +git-tree-sha1 = "04490d5e7570c038b1cb0f5c3627597181cc15a9" +uuid = "a00861dc-f156-4864-bf3c-e6376f28a68d" +version = "0.1.9" + +[[NaNMath]] +git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "0.3.5" + +[[NetworkLayout]] +deps = ["GeometryBasics", "LinearAlgebra", "Random", "Requires", "SparseArrays"] +git-tree-sha1 = "24e10982e84dd35cd867102243454bf8a4581a76" +uuid = "46757867-2c16-5918-afeb-47bfcb05e46a" +version = "0.4.3" + +[[NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" + +[[Ogg_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "7937eda4681660b4d6aeeecc2f7e1c81c8ee4e2f" +uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" +version = "1.3.5+0" + +[[OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" + +[[OpenML]] +deps = ["ARFFFiles", "HTTP", "JSON", "Markdown", "Pkg", "ScientificTypes"] +git-tree-sha1 = "79ffa09cf7c730b36342699553feef3e1f169ec6" +uuid = "8b6db2d4-7670-4922-a472-f9537c81ab66" +version = "0.1.1" + +[[OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "15003dcb7d8db3c6c857fda14891a539a8f2705a" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "1.1.10+0" + +[[OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[Optim]] +deps = ["Compat", "FillArrays", "ForwardDiff", "LineSearches", "LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "PositiveFactorizations", "Printf", "SparseArrays", "StatsBase"] +git-tree-sha1 = "35d435b512fbab1d1a29138b5229279925eba369" +uuid = "429524aa-4258-5aef-a3af-852621145aeb" +version = "1.5.0" + +[[Opus_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720" +uuid = "91d4177d-7536-5919-b921-800302f37372" +version = "1.3.2+0" + +[[OrderedCollections]] +git-tree-sha1 = "85f8e6578bf1f9ee0d11e7bb1b1456435479d47c" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.4.1" + +[[PCRE_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "b2a7af664e098055a7529ad1a900ded962bca488" +uuid = "2f80f16e-611a-54ab-bc61-aa92de5b98fc" +version = "8.44.0+0" + +[[PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "c8b8775b2f242c80ea85c83714c64ecfa3c53355" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.3" + +[[Parameters]] +deps = ["OrderedCollections", "UnPack"] +git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" +uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" +version = "0.12.3" + +[[Parsers]] +deps = ["Dates"] +git-tree-sha1 = "ae4bbcadb2906ccc085cf52ac286dc1377dceccc" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.1.2" + +[[Pixman_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "b4f5d02549a10e20780a24fce72bea96b6329e29" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.40.1+0" + +[[Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" + +[[PlotThemes]] +deps = ["PlotUtils", "Requires", "Statistics"] +git-tree-sha1 = "a3a964ce9dc7898193536002a6dd892b1b5a6f1d" +uuid = "ccf2f8ad-2431-5c83-bf29-c5338b663b6a" +version = "2.0.1" + +[[PlotUtils]] +deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"] +git-tree-sha1 = "b084324b4af5a438cd63619fd006614b3b20b87b" +uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043" +version = "1.0.15" + +[[Plots]] +deps = ["Base64", "Contour", "Dates", "Downloads", "FFMPEG", "FixedPointNumbers", "GR", "GeometryBasics", "JSON", "Latexify", "LinearAlgebra", "Measures", "NaNMath", "PlotThemes", "PlotUtils", "Printf", "REPL", "Random", "RecipesBase", "RecipesPipeline", "Reexport", "Requires", "Scratch", "Showoff", "SparseArrays", "Statistics", "StatsBase", "UUIDs", "UnicodeFun"] +git-tree-sha1 = "7dc03c2b145168f5854085a16d054429d612b637" +uuid = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" +version = "1.23.5" + +[[PositiveFactorizations]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "17275485f373e6673f7e7f97051f703ed5b15b20" +uuid = "85a6dd25-e78a-55b7-8502-1745935b8125" +version = "0.2.4" + +[[Preferences]] +deps = ["TOML"] +git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.2.2" + +[[PrettyTables]] +deps = ["Crayons", "Formatting", "Markdown", "Reexport", "Tables"] +git-tree-sha1 = "d940010be611ee9d67064fe559edbb305f8cc0eb" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "1.2.3" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[Profile]] +deps = ["Printf"] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" + +[[ProgressMeter]] +deps = ["Distributed", "Printf"] +git-tree-sha1 = "afadeba63d90ff223a6a48d2009434ecee2ec9e8" +uuid = "92933f4c-e287-5a05-a399-4b506db050ca" +version = "1.7.1" + +[[Qt5Base_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Fontconfig_jll", "Glib_jll", "JLLWrappers", "Libdl", "Libglvnd_jll", "OpenSSL_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libxcb_jll", "Xorg_xcb_util_image_jll", "Xorg_xcb_util_keysyms_jll", "Xorg_xcb_util_renderutil_jll", "Xorg_xcb_util_wm_jll", "Zlib_jll", "xkbcommon_jll"] +git-tree-sha1 = "ad368663a5e20dbb8d6dc2fddeefe4dae0781ae8" +uuid = "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1" +version = "5.15.3+0" + +[[QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "78aadffb3efd2155af139781b8a8df1ef279ea39" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.4.2" + +[[REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Random123]] +deps = ["Libdl", "Random", "RandomNumbers"] +git-tree-sha1 = "0e8b146557ad1c6deb1367655e052276690e71a3" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.4.2" + +[[RandomNumbers]] +deps = ["Random", "Requires"] +git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.5.3" + +[[RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[RecipesBase]] +git-tree-sha1 = "44a75aa7a527910ee3d1751d1f0e4148698add9e" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.1.2" + +[[RecipesPipeline]] +deps = ["Dates", "NaNMath", "PlotUtils", "RecipesBase"] +git-tree-sha1 = "7ad0dfa8d03b7bcf8c597f59f5292801730c55b8" +uuid = "01d81517-befc-4cb6-b9ec-a95719d0359c" +version = "0.4.1" + +[[Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "4036a3bd08ac7e968e27c203d45f5fff15020621" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.1.3" + +[[Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "bf3188feca147ce108c76ad82c2792c57abe7b1f" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.0" + +[[Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "68db32dff12bb6127bac73c209881191bf0efbb7" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.3.0+0" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[ScientificTypes]] +deps = ["CategoricalArrays", "ColorTypes", "Dates", "Distributions", "PrettyTables", "Reexport", "ScientificTypesBase", "StatisticalTraits", "Tables"] +git-tree-sha1 = "7a3efcacd212801a8cf2f961e8238ffb2109b30d" +uuid = "321657f4-b219-11e9-178b-2701a2544e81" +version = "2.3.3" + +[[ScientificTypesBase]] +git-tree-sha1 = "185e373beaf6b381c1e7151ce2c2a722351d6637" +uuid = "30f210dd-8aff-4c5f-94ba-8e64358c1161" +version = "2.3.0" + +[[Scratch]] +deps = ["Dates"] +git-tree-sha1 = "0b4b7f1393cff97c33891da2a0bf69c6ed241fda" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.1.0" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Showoff]] +deps = ["Dates", "Grisu"] +git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de" +uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" +version = "1.0.3" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "b3363d7460f7d098ca0912c69b082f75625d7508" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.0.1" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[SpecialFunctions]] +deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "f0bccf98e16759818ffc5d97ac3ebf87eb950150" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "1.8.1" + +[[StableRNGs]] +deps = ["Random", "Test"] +git-tree-sha1 = "3be7d49667040add7ee151fefaf1f8c04c8c8276" +uuid = "860ef19b-820b-49d6-a774-d7a799459cd3" +version = "1.0.0" + +[[Static]] +deps = ["IfElse"] +git-tree-sha1 = "e7bc80dc93f50857a5d1e3c8121495852f407e6a" +uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" +version = "0.4.0" + +[[StaticArrays]] +deps = ["LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "3c76dde64d03699e074ac02eb2e8ba8254d428da" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.2.13" + +[[StatisticalTraits]] +deps = ["ScientificTypesBase"] +git-tree-sha1 = "730732cae4d3135e2f2182bd47f8d8b795ea4439" +uuid = "64bff920-2084-43da-a3e6-9bb72801c0c9" +version = "2.1.0" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[StatsAPI]] +git-tree-sha1 = "1958272568dc176a1d881acb797beb909c785510" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.0.0" + +[[StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "eb35dcc66558b2dda84079b9a1be17557d32091a" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.33.12" + +[[StatsFuns]] +deps = ["ChainRulesCore", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "95072ef1a22b057b1e80f73c2a89ad238ae4cfff" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "0.9.12" + +[[StructArrays]] +deps = ["Adapt", "DataAPI", "StaticArrays", "Tables"] +git-tree-sha1 = "2ce41e0d042c60ecd131e9fb7154a3bfadbf50d3" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.3" + +[[SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[Syslogs]] +deps = ["Printf", "Sockets"] +git-tree-sha1 = "46badfcc7c6e74535cc7d833a91f4ac4f805f86d" +uuid = "cea106d9-e007-5e6c-ad93-58fe2094e9c4" +version = "0.3.0" + +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + +[[TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"] +git-tree-sha1 = "fed34d0e71b91734bf0a7e10eb1bb05296ddbcd0" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.6.0" + +[[Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" + +[[Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[TimeZones]] +deps = ["Dates", "Downloads", "InlineStrings", "LazyArtifacts", "Mocking", "Pkg", "Printf", "RecipesBase", "Serialization", "Unicode"] +git-tree-sha1 = "8de32288505b7db196f36d27d7236464ef50dba1" +uuid = "f269a46b-ccf7-5d73-abea-4c690281aa53" +version = "1.6.2" + +[[TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "7cb456f358e8f9d102a8b25e8dfedf58fa5689bc" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.13" + +[[TranscodingStreams]] +deps = ["Random", "Test"] +git-tree-sha1 = "216b95ea110b5972db65aa90f88d8d89dcb8851c" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.9.6" + +[[URIs]] +git-tree-sha1 = "97bbe755a53fe859669cd907f2d96aee8d2c1355" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.3.0" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[UnPack]] +git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b" +uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" +version = "1.0.2" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[UnicodeFun]] +deps = ["REPL"] +git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf" +uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1" +version = "0.4.1" + +[[Wayland_jll]] +deps = ["Artifacts", "Expat_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "3e61f0b86f90dacb0bc0e73a0c5a83f6a8636e23" +uuid = "a2964d1f-97da-50d4-b82a-358c7fce9d89" +version = "1.19.0+0" + +[[Wayland_protocols_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll"] +git-tree-sha1 = "2839f1c1296940218e35df0bbb220f2a79686670" +uuid = "2381bf8a-dfd0-557d-9999-79630e7b1b91" +version = "1.18.0+4" + +[[XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "1acf5bdf07aa0907e0a37d3718bb88d4b687b74a" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.9.12+0" + +[[XSLT_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"] +git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a" +uuid = "aed1982a-8fda-507f-9586-7b0439959a61" +version = "1.1.34+0" + +[[Xorg_libX11_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] +git-tree-sha1 = "5be649d550f3f4b95308bf0183b82e2582876527" +uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" +version = "1.6.9+4" + +[[Xorg_libXau_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4e490d5c960c314f33885790ed410ff3a94ce67e" +uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" +version = "1.0.9+4" + +[[Xorg_libXcursor_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXfixes_jll", "Xorg_libXrender_jll"] +git-tree-sha1 = "12e0eb3bc634fa2080c1c37fccf56f7c22989afd" +uuid = "935fb764-8cf2-53bf-bb30-45bb1f8bf724" +version = "1.2.0+4" + +[[Xorg_libXdmcp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4fe47bd2247248125c428978740e18a681372dd4" +uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" +version = "1.1.3+4" + +[[Xorg_libXext_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3" +uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" +version = "1.3.4+4" + +[[Xorg_libXfixes_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "0e0dc7431e7a0587559f9294aeec269471c991a4" +uuid = "d091e8ba-531a-589c-9de9-94069b037ed8" +version = "5.0.3+4" + +[[Xorg_libXi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXfixes_jll"] +git-tree-sha1 = "89b52bc2160aadc84d707093930ef0bffa641246" +uuid = "a51aa0fd-4e3c-5386-b890-e753decda492" +version = "1.7.10+4" + +[[Xorg_libXinerama_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll"] +git-tree-sha1 = "26be8b1c342929259317d8b9f7b53bf2bb73b123" +uuid = "d1454406-59df-5ea1-beac-c340f2130bc3" +version = "1.1.4+4" + +[[Xorg_libXrandr_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll"] +git-tree-sha1 = "34cea83cb726fb58f325887bf0612c6b3fb17631" +uuid = "ec84b674-ba8e-5d96-8ba1-2a689ba10484" +version = "1.5.2+4" + +[[Xorg_libXrender_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96" +uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" +version = "0.9.10+4" + +[[Xorg_libpthread_stubs_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6783737e45d3c59a4a4c4091f5f88cdcf0908cbb" +uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" +version = "0.1.0+3" + +[[Xorg_libxcb_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] +git-tree-sha1 = "daf17f441228e7a3833846cd048892861cff16d6" +uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" +version = "1.13.0+3" + +[[Xorg_libxkbfile_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "926af861744212db0eb001d9e40b5d16292080b2" +uuid = "cc61e674-0454-545c-8b26-ed2c68acab7a" +version = "1.1.0+4" + +[[Xorg_xcb_util_image_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] +git-tree-sha1 = "0fab0a40349ba1cba2c1da699243396ff8e94b97" +uuid = "12413925-8142-5f55-bb0e-6d7ca50bb09b" +version = "0.4.0+1" + +[[Xorg_xcb_util_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll"] +git-tree-sha1 = "e7fd7b2881fa2eaa72717420894d3938177862d1" +uuid = "2def613f-5ad1-5310-b15b-b15d46f528f5" +version = "0.4.0+1" + +[[Xorg_xcb_util_keysyms_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] +git-tree-sha1 = "d1151e2c45a544f32441a567d1690e701ec89b00" +uuid = "975044d2-76e6-5fbe-bf08-97ce7c6574c7" +version = "0.4.0+1" + +[[Xorg_xcb_util_renderutil_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] +git-tree-sha1 = "dfd7a8f38d4613b6a575253b3174dd991ca6183e" +uuid = "0d47668e-0667-5a69-a72c-f761630bfb7e" +version = "0.3.9+1" + +[[Xorg_xcb_util_wm_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xcb_util_jll"] +git-tree-sha1 = "e78d10aab01a4a154142c5006ed44fd9e8e31b67" +uuid = "c22f9ab0-d5fe-5066-847c-f4bb1cd4e361" +version = "0.4.1+1" + +[[Xorg_xkbcomp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxkbfile_jll"] +git-tree-sha1 = "4bcbf660f6c2e714f87e960a171b119d06ee163b" +uuid = "35661453-b289-5fab-8a00-3d9160c6a3a4" +version = "1.4.2+4" + +[[Xorg_xkeyboard_config_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_xkbcomp_jll"] +git-tree-sha1 = "5c8424f8a67c3f2209646d4425f3d415fee5931d" +uuid = "33bec58e-1273-512f-9401-5d533626f822" +version = "2.27.0+4" + +[[Xorg_xtrans_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845" +uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" +version = "1.4.0+3" + +[[ZipFile]] +deps = ["Libdl", "Printf", "Zlib_jll"] +git-tree-sha1 = "3593e69e469d2111389a9bd06bac1f3d730ac6de" +uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" +version = "0.9.4" + +[[Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" + +[[Zstd_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "cc4bf3fdde8b7e3e9fa0351bdeedba1cf3b7f6e6" +uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" +version = "1.5.0+0" + +[[Zygote]] +deps = ["AbstractFFTs", "ChainRules", "ChainRulesCore", "DiffRules", "Distributed", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] +git-tree-sha1 = "0fc9959bcabc4668c403810b4e851f6b8962eac9" +uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" +version = "0.6.29" + +[[ZygoteRules]] +deps = ["MacroTools"] +git-tree-sha1 = "8c1a8e4dfacb1fd631745552c8db35d0deb09ea0" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.2" + +[[libass_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47" +uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" +version = "0.15.1+0" + +[[libfdk_aac_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55" +uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" +version = "2.0.2+0" + +[[libpng_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "94d180a6d2b5e55e447e2d27a29ed04fe79eb30c" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.38+0" + +[[libvorbis_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] +git-tree-sha1 = "c45f4e40e7aafe9d086379e5578947ec8b95a8fb" +uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" +version = "1.3.7+0" + +[[nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" + +[[p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" + +[[x264_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2" +uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" +version = "2021.5.5+0" + +[[x265_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9" +uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" +version = "3.5.0+0" + +[[xkbcommon_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Wayland_jll", "Wayland_protocols_jll", "Xorg_libxcb_jll", "Xorg_xkeyboard_config_jll"] +git-tree-sha1 = "ece2350174195bb31de1a63bea3a41ae1aa593b6" +uuid = "d8fb68d0-12a3-5cfd-a85a-d49703b185fd" +version = "0.9.1+5" diff --git a/examples/boston/env/Project.toml b/examples/boston/env/Project.toml new file mode 100644 index 00000000..9447d967 --- /dev/null +++ b/examples/boston/env/Project.toml @@ -0,0 +1,10 @@ +[deps] +EvoTrees = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5" +Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7" +MLJFlux = "094fc8d1-fd35-5302-93ea-dabda2abf845" +MLJLinearModels = "6ee0df7b-362f-4a72-a706-9e79364fb692" +MLJTuning = "03970b2e-30c4-11ea-3135-d1576263f10f" +Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" +Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" diff --git a/examples/boston/env/generate.jl b/examples/boston/env/generate.jl new file mode 100644 index 00000000..40880dba --- /dev/null +++ b/examples/boston/env/generate.jl @@ -0,0 +1,5 @@ +# Execute this julia file to generate the notebooks from ../notebook.jl + +env = @__DIR__ +joinpath(env, "..", "..", "generate.jl") |> include +generate(env, execute=false, pluto=false) diff --git a/examples/boston/notebook.jl b/examples/boston/notebook.jl new file mode 100644 index 00000000..c1691b2e --- /dev/null +++ b/examples/boston/notebook.jl @@ -0,0 +1,256 @@ +# # Building an MLJFlux regression model for the Boston house +# # price dataset + +using Pkg +Pkg.activate("env") +Pkg.instantiate() + +# **Julia version** is assumed to be 1.6.* + +using MLJ +using MLJFlux +using Flux + +MLJ.color_off() + +using Plots + +# This tutorial uses MLJ's `IteratedModel` wrapper to transform the +# MLJFlux `NeuralNetworkRegressor` into a model that **automatically +# selects the number of epochs** required to optimize an out-of-sample +# loss. + +# We also show how to include the model in a **pipeline** to carry out +# standardization of the features and target. + + +# ## Loading data + +data = OpenML.load(531); # Loads from https://www.openml.org/d/531 + +# The target `y` is `:MEDV` and everything else except `:CHAS` goes +# into the features `X`: + +y, X = unpack(data, ==(:MEDV), !=(:CHAS); rng=123); + +# We specified the seed `rng` to shuffle the observations. The Charles +# River dummy variable `:CHAS` is dropped, as not deemed to be +# relevant. + +# Inspecting the scientific types: + +scitype(y) + +#- + +schema(X) + +# We'll regard `:RAD` (index of accessibility to radial highways) as +# `Continuous` as MLJFlux models don't handle ordered factors: + +X = coerce(X, :RAD => Continuous); + +# Let's split off a test set for final testing: + +X, Xtest = partition(X, 0.7); +y, ytest = partition(y, 0.7); + +# ## Defining a builder + +# In the macro call below, `n_in` is expected to represent the number +# of inputs features and `rng` a RNG (builders are generic, ie can be +# applied to data with any number of input features): + +builder = MLJFlux.@builder begin + init=Flux.glorot_uniform(rng) + Chain(Dense(n_in, 64, relu, init=init), + Dense(64, 32, relu, init=init), + Dense(32, 1, init=init)) +end + + +# ## Defining a MLJFlux model: + +NeuralNetworkRegressor = @load NeuralNetworkRegressor + model = NeuralNetworkRegressor(builder=builder, + rng=123, + epochs=20) + + +# ## Standardization + +# The following wraps our regressor in feature and target standardizations: + +pipe = @pipeline Standardizer model target=Standardizer + +# Notice that our original neural network model is now a +# hyper-parameter of the composite `pipe`, with the automatically +# generated name, `:neural_network_regressor`. + + +# ## Choosing a learning rate + +# Let's see how the training losses look for the default optimiser. For +# MLJFlux models, `fit!` will print these losses if we bump the +# verbosity level (default is always 1): + +mach = machine(pipe, X, y) +fit!(mach, verbosity=2) + +# They are also extractable from the training report (which includes +# the pre-train loss): + +report(mach).neural_network_regressor.training_losses + +# Next, let's visually compare a few learning rates: + +plt = plot() +rates = [5e-5, 1e-4, 0.005, 0.001, 0.05] + +# By default, changing only the optimiser will not trigger a +# cold-restart when we `fit!` (to allow for adaptive learning rate +# control). So we call `fit!` with the `force=true` +# option. (Alternatively, one can change the hyper-parameter +# `pipe.neural_network_regressor.optimiser_changes_trigger_retraining` +# to `true`.) + +# We'll skip the first few losses to get a better vertical scale in +# our plot. + +foreach(rates) do η + pipe.neural_network_regressor.optimiser.eta = η + fit!(mach, force=true) + losses = report(mach).neural_network_regressor.training_losses[3:end] + plot!(1:length(losses), losses, label=η) +end +plt #!md + +#- + +savefig(joinpath("assets", "learning_rate.png")) + +# ![](assets/learing_rate.png) #md + +# We'll go with the second most conservative rate: + +pipe.neural_network_regressor.optimiser.eta = 0.0001 + + +# ## Wrapping in iteration control + +# We want a model that trains until an out-of-sample loss satisfies +# the `NumberSinceBest(6)` stopping criterion. We'll add some fallback +# stopping criterion `InvalidValue` and `TimeLimit(1/60)`, and +# controls to print traces of the losses. + +# For initializing or clearing the traces: + +clear() = begin + global losses = [] + global training_losses = [] + global epochs = [] + return nothing +end + +# And to update the traces: + +update_loss(loss) = push!(losses, loss) +update_training_loss(report) = + push!(training_losses, report.neural_network_regressor.training_losses[end]) +update_epochs(epoch) = push!(epochs, epoch) + +# The controls to apply (see +# [here](https://alan-turing-institute.github.io/MLJ.jl/dev/controlling_iterative_models/#Controls-provided) +# for the complete list): + +controls=[Step(1), + NumberSinceBest(6), + InvalidValue(), + TimeLimit(1/60), + WithLossDo(update_loss), + WithReportDo(update_training_loss), + WithIterationsDo(update_epochs)] + +# Next we create a "self-iterating" version of the pipeline. Note +# that the iteration parameter is a nested hyperparameter: + +iterated_pipe = + IteratedModel(model=pipe, + controls=controls, + resampling=Holdout(fraction_train=0.8), + iteration_parameter=:(neural_network_regressor.epochs), + measure = l2) + +# Training the wrapped model on all the train/validation data: + +clear() +mach = machine(iterated_pipe, X, y) +fit!(mach) + +# And plotting the traces: + +plot(epochs, losses, + xlab = "epoch", + ylab = "mean sum of squares error", + label="out-of-sample", + legend = :topleft); +scatter!(twinx(), epochs, training_losses, label="training", color=:red) #!md + +#- + +savefig(joinpath("assets", "loss.png")) + +# ![](assets/losses.png) #md + +# **How `IteratedModel` works.** Training an `IteratedModel` means +# holding out some data (80% in this case) so an out-of-sample loss +# can be tracked and used in the specified stopping criterion, +# `NumberSinceBest(4)`. However, once the stop is triggered, the model +# wrapped by `IteratedModel` (our pipeline model) is retrained on all +# data for the same number of iterations. Calling `predict(mach, +# Xnew)` on new data uses the updated learned parameters. + +# In other words, `iterated_model` is a "self-iterating" version of +# the original model, where `epochs` has been transformed from +# hyper-parameter to *learned* parameter. + + +# ## An evaluation the self-iterating model + +# Here's an estimate of performance of our "self-iterating" +# model: + +e = evaluate!(mach, + resampling=CV(nfolds=8), + measures=[l2, l1]) + +using Measurements +err = e.measurement[1] ± std(e.per_fold[1])/sqrt(7) +@show err + +# which we can see has substantial uncertainty (not that CV estimates +# of the uncertainty are trustworthy; see e.g., [this +# survey](https://direct.mit.edu/neco/article-abstract/10/7/1895/6224/Approximate-Statistical-Tests-for-Comparing)) + + +# ## Comparison with other models on the test set + +# Although we cannot assign them statistical significance, here are +# comparisons, on the untouched test set, of the eror of our +# self-iterating neural network regressor with a couple of other +# models trained on the same data (using default hyperparameters): + +function performance(model) + mach = machine(model, X, y) |> fit! + yhat = predict(mach, Xtest) + l2(yhat, ytest) |> mean +end +performance(iterated_pipe) + +three_models = [(@load EvoTreeRegressor)(), # tree boosting model + (@load LinearRegressor pkg=MLJLinearModels)(), + iterated_pipe] + +errs = performance.(three_models) + +(models=typeof.(three_models), mean_square_errors=errs) |> pretty diff --git a/examples/boston/notebook.unexecuted.ipynb b/examples/boston/notebook.unexecuted.ipynb new file mode 100644 index 00000000..b0dfb65b --- /dev/null +++ b/examples/boston/notebook.unexecuted.ipynb @@ -0,0 +1,1500 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Building an MLJFlux regression model for the Boston house\n", + "# price dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[32m\u001b[1m Activating\u001b[22m\u001b[39m environment at `~/GoogleDrive/Julia/MLJ/MLJFlux/examples/boston/env/Project.toml`\n" + ] + } + ], + "source": [ + "using Pkg\n", + "Pkg.activate(\"env\")\n", + "Pkg.instantiate()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Julia version** is assumed to be 1.6.*" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "using MLJ\n", + "using MLJFlux\n", + "using Flux\n", + "\n", + "MLJ.color_off()\n", + "\n", + "using Plots" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This tutorial uses MLJ's `IteratedModel` wrapper to transform the\n", + "MLJFlux `NeuralNetworkRegressor` into a model that **automatically\n", + "selects the number of epochs** required to optimize an out-of-sample\n", + "loss." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also show how to include the model in a **pipeline** to carry out\n", + "standardization of the features and target." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Loading data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "data = OpenML.load(531); # Loads from https://www.openml.org/d/531" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The target `y` is `:MEDV` and everything else except `:CHAS` goes\n", + "into the features `X`:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "y, X = unpack(data, ==(:MEDV), !=(:CHAS); rng=123);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We specified the seed `rng` to shuffle the observations. The Charles\n", + "River dummy variable `:CHAS` is dropped, as not deemed to be\n", + "relevant." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Inspecting the scientific types:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "AbstractVector{Continuous} (alias for AbstractArray{Continuous, 1})" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "scitype(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "┌─────────┬──────────────────────────────────┬───────────────┐\n", + "│\u001b[22m _.names \u001b[0m│\u001b[22m _.types \u001b[0m│\u001b[22m _.scitypes \u001b[0m│\n", + "├─────────┼──────────────────────────────────┼───────────────┤\n", + "│ CRIM │ Float64 │ Continuous │\n", + "│ ZN │ Float64 │ Continuous │\n", + "│ INDUS │ Float64 │ Continuous │\n", + "│ NOX │ Float64 │ Continuous │\n", + "│ RM │ Float64 │ Continuous │\n", + "│ AGE │ Float64 │ Continuous │\n", + "│ DIS │ Float64 │ Continuous │\n", + "│ RAD │ CategoricalValue{String, UInt32} │ Multiclass{9} │\n", + "│ TAX │ Float64 │ Continuous │\n", + "│ PTRATIO │ Float64 │ Continuous │\n", + "│ B │ Float64 │ Continuous │\n", + "│ LSTAT │ Float64 │ Continuous │\n", + "└─────────┴──────────────────────────────────┴───────────────┘\n", + "_.nrows = 506\n" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "schema(X)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll regard `:RAD` (index of accessibility to radial highways) as\n", + "`Continuous` as MLJFlux models don't handle ordered factors:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "X = coerce(X, :RAD => Continuous);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's split off a test set for final testing:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "X, Xtest = partition(X, 0.7);\n", + "y, ytest = partition(y, 0.7);" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Defining a builder" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the macro call below, `n_in` is expected to represent the number\n", + "of inputs features and `rng` a RNG (builders are generic, ie can be\n", + "applied to data with any number of input features):" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "GenericBuilder{#5#6}\n" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "builder = MLJFlux.@builder begin\n", + " init=Flux.glorot_uniform(rng)\n", + " Chain(Dense(n_in, 64, relu, init=init),\n", + " Dense(64, 32, relu, init=init),\n", + " Dense(32, 1, init=init))\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Defining a MLJFlux model:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: For silent loading, specify `verbosity=0`. \n", + "└ @ Main /Users/anthony/.julia/packages/MLJModels/4sRmw/src/loading.jl:168\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "import MLJFlux ✔\n" + ] + }, + { + "data": { + "text/plain": [ + "NeuralNetworkRegressor(\n", + " builder = GenericBuilder(\n", + " apply = var\"#5#6\"()),\n", + " optimiser = ADAM(0.001, (0.9, 0.999), IdDict{Any, Any}()),\n", + " loss = Flux.Losses.mse,\n", + " epochs = 20,\n", + " batch_size = 1,\n", + " lambda = 0.0,\n", + " alpha = 0.0,\n", + " rng = 123,\n", + " optimiser_changes_trigger_retraining = false,\n", + " acceleration = CPU1{Nothing}(nothing))" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "NeuralNetworkRegressor = @load NeuralNetworkRegressor\n", + " model = NeuralNetworkRegressor(builder=builder,\n", + " rng=123,\n", + " epochs=20)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Standardization" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following wraps our regressor in feature and target standardizations:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Treating pipeline as a `Deterministic` predictor.\n", + "│ To override, specify `prediction_type=...` (options: :deterministic, :probabilistic, :interval). \n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/composition/models/pipelines.jl:372\n" + ] + }, + { + "data": { + "text/plain": [ + "Pipeline282(\n", + " standardizer = Standardizer(\n", + " features = Symbol[],\n", + " ignore = false,\n", + " ordered_factor = false,\n", + " count = false),\n", + " neural_network_regressor = NeuralNetworkRegressor(\n", + " builder = GenericBuilder{#5#6},\n", + " optimiser = ADAM(0.001, (0.9, 0.999), IdDict{Any, Any}()),\n", + " loss = Flux.Losses.mse,\n", + " epochs = 20,\n", + " batch_size = 1,\n", + " lambda = 0.0,\n", + " alpha = 0.0,\n", + " rng = 123,\n", + " optimiser_changes_trigger_retraining = false,\n", + " acceleration = CPU1{Nothing}(nothing)),\n", + " target = Standardizer(\n", + " features = Symbol[],\n", + " ignore = false,\n", + " ordered_factor = false,\n", + " count = false))" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pipe = @pipeline Standardizer model target=Standardizer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Notice that our original neural network model is now a\n", + "hyper-parameter of the composite `pipe`, with the automatically\n", + "generated name, `:neural_network_regressor`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Choosing a learning rate" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's see how the training losses look for the default optimiser. For\n", + "MLJFlux models, `fit!` will print these losses if we bump the\n", + "verbosity level (default is always 1):" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Training Machine{Pipeline282,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Features standarized: \n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:825\n", + "┌ Info: :CRIM mu=3.689791468926554 sigma=8.486186608186063\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :ZN mu=10.392655367231638 sigma=22.078531026900794\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :INDUS mu=11.477372881355933 sigma=6.83192917494762\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :NOX mu=0.5619451977401131 sigma=0.11706875291419269\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :RM mu=6.256881355932204 sigma=0.655703581671515\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :AGE mu=69.98474576271187 sigma=27.70464405221309\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :DIS mu=3.704779096045197 sigma=2.018034269570148\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :RAD mu=5.72316384180791 sigma=2.4534591735797426\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :TAX mu=415.97457627118644 sigma=172.0508915025351\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :PTRATIO mu=18.54971751412429 sigma=2.176727910395866\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :B mu=354.88437853107337 sigma=91.70747435121616\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: :LSTAT mu=12.997796610169493 sigma=7.067143799083826\n", + "└ @ MLJModels /Users/anthony/.julia/packages/MLJModels/4sRmw/src/builtins/Transformers.jl:836\n", + "┌ Info: Training Machine{NeuralNetworkRegressor{GenericBuilder{#5#6},…},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Loss is 0.3687\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.2216\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.1775\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.1519\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.1322\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.117\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.1064\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.09521\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.08949\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.08277\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.07997\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.07727\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.0719\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.07051\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.06697\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.06423\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.06024\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.05974\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.05666\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n", + "┌ Info: Loss is 0.05577\n", + "└ @ MLJFlux /Users/anthony/.julia/packages/MLJFlux/5ENQA/src/core.jl:128\n" + ] + }, + { + "data": { + "text/plain": [ + "Machine{Pipeline282,…} trained 1 time; caches data\n", + " args: \n", + " 1:\tSource @301 ⏎ `Table{AbstractVector{Continuous}}`\n", + " 2:\tSource @846 ⏎ `AbstractVector{Continuous}`\n" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mach = machine(pipe, X, y)\n", + "fit!(mach, verbosity=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "They are also extractable from the training report (which includes\n", + "the pre-train loss):" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "21-element Vector{Float64}:\n", + " 1.2838145640873329\n", + " 0.3686760957021546\n", + " 0.2215979652158559\n", + " 0.17754900774597876\n", + " 0.15194120480021098\n", + " 0.13221864530853797\n", + " 0.11697435053172738\n", + " 0.10636408931390769\n", + " 0.095208490765527\n", + " 0.08948774179997945\n", + " 0.08276797005304384\n", + " 0.07997044943380421\n", + " 0.07726719934689869\n", + " 0.07190359246973332\n", + " 0.07050712214306228\n", + " 0.0669674161856798\n", + " 0.06422548257753624\n", + " 0.06024360408282619\n", + " 0.05974192388977129\n", + " 0.05666044439895874\n", + " 0.055767894928594526" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "report(mach).neural_network_regressor.training_losses" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, let's visually compare a few learning rates:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "5-element Vector{Float64}:\n", + " 5.0e-5\n", + " 0.0001\n", + " 0.005\n", + " 0.001\n", + " 0.05" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "plt = plot()\n", + "rates = [5e-5, 1e-4, 0.005, 0.001, 0.05]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default, changing only the optimiser will not trigger a\n", + "cold-restart when we `fit!` (to allow for adaptive learning rate\n", + "control). So we call `fit!` with the `force=true`\n", + "option. (Alternatively, one can change the hyper-parameter\n", + "`pipe.neural_network_regressor.optimiser_changes_trigger_retraining`\n", + "to `true`.)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll skip the first few losses to get a better vertical scale in\n", + "our plot." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Training Machine{Pipeline282,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{NeuralNetworkRegressor{GenericBuilder{#5#6},…},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "\u001b[33mOptimising neural net:100%[=========================] Time: 0:00:02\u001b[39m\n", + "┌ Info: Training Machine{Pipeline282,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{NeuralNetworkRegressor{GenericBuilder{#5#6},…},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "\u001b[33mOptimising neural net:100%[=========================] Time: 0:00:01\u001b[39m\n", + "┌ Info: Training Machine{Pipeline282,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{NeuralNetworkRegressor{GenericBuilder{#5#6},…},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "\u001b[33mOptimising neural net:100%[=========================] Time: 0:00:01\u001b[39m\n", + "┌ Info: Training Machine{Pipeline282,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{NeuralNetworkRegressor{GenericBuilder{#5#6},…},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "\u001b[33mOptimising neural net:100%[=========================] Time: 0:00:01\u001b[39m\n", + "┌ Info: Training Machine{Pipeline282,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{Standardizer,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{NeuralNetworkRegressor{GenericBuilder{#5#6},…},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "\u001b[33mOptimising neural net:100%[=========================] Time: 0:00:01\u001b[39m\n" + ] + }, + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "foreach(rates) do η\n", + " pipe.neural_network_regressor.optimiser.eta = η\n", + " fit!(mach, force=true)\n", + " losses = report(mach).neural_network_regressor.training_losses[3:end]\n", + " plot!(1:length(losses), losses, label=η)\n", + "end\n", + "plt" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "savefig(joinpath(\"assets\", \"learning_rate.png\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll go with the second most conservative rate:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0001" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pipe.neural_network_regressor.optimiser.eta = 0.0001" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Wrapping in iteration control" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We want a model that trains until an out-of-sample loss satisfies\n", + "the `NumberSinceBest(6)` stopping criterion. We'll add some fallback\n", + "stopping criterion `InvalidValue` and `TimeLimit(1/60)`, and\n", + "controls to print traces of the losses." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For initializing or clearing the traces:" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "clear (generic function with 1 method)" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clear() = begin\n", + " global losses = []\n", + " global training_losses = []\n", + " global epochs = []\n", + " return nothing\n", + "end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And to update the traces:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "update_epochs (generic function with 1 method)" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "update_loss(loss) = push!(losses, loss)\n", + "update_training_loss(report) =\n", + " push!(training_losses, report.neural_network_regressor.training_losses[end])\n", + "update_epochs(epoch) = push!(epochs, epoch)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The controls to apply (see\n", + "[here](https://alan-turing-institute.github.io/MLJ.jl/dev/controlling_iterative_models/#Controls-provided)\n", + "for the complete list):" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "7-element Vector{Any}:\n", + " Step(1)\n", + " NumberSinceBest(6)\n", + " InvalidValue()\n", + " TimeLimit(Dates.Millisecond(60000))\n", + " WithLossDo{typeof(update_loss)}(update_loss, false, nothing)\n", + " WithReportDo{typeof(update_training_loss)}(update_training_loss, false, nothing)\n", + " WithIterationsDo{typeof(update_epochs)}(update_epochs, false, nothing)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "controls=[Step(1),\n", + " NumberSinceBest(6),\n", + " InvalidValue(),\n", + " TimeLimit(1/60),\n", + " WithLossDo(update_loss),\n", + " WithReportDo(update_training_loss),\n", + " WithIterationsDo(update_epochs)]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we create a \"self-iterating\" version of the pipeline. Note\n", + "that the iteration parameter is a nested hyperparameter:" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DeterministicIteratedModel(\n", + " model = Pipeline282(\n", + " standardizer = Standardizer,\n", + " neural_network_regressor = NeuralNetworkRegressor{GenericBuilder{#5#6},…},\n", + " target = Standardizer),\n", + " controls = Any[Step(1), NumberSinceBest(6), InvalidValue(), TimeLimit(Dates.Millisecond(60000)), WithLossDo{typeof(update_loss)}(update_loss, false, nothing), WithReportDo{typeof(update_training_loss)}(update_training_loss, false, nothing), WithIterationsDo{typeof(update_epochs)}(update_epochs, false, nothing)],\n", + " resampling = Holdout(\n", + " fraction_train = 0.8,\n", + " shuffle = false,\n", + " rng = Random._GLOBAL_RNG()),\n", + " measure = LPLoss(p = 2),\n", + " weights = nothing,\n", + " class_weights = nothing,\n", + " operation = MLJModelInterface.predict,\n", + " retrain = false,\n", + " check_measure = true,\n", + " iteration_parameter = :(neural_network_regressor.epochs),\n", + " cache = true)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "iterated_pipe =\n", + " IteratedModel(model=pipe,\n", + " controls=controls,\n", + " resampling=Holdout(fraction_train=0.8),\n", + " iteration_parameter=:(neural_network_regressor.epochs),\n", + " measure = l2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Training the wrapped model on all the train/validation data:" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Training Machine{DeterministicIteratedModel{Pipeline282},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: final loss: 18.029846309038444\n", + "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/B3jW2/src/train.jl:29\n", + "┌ Info: Stop triggered by NumberSinceBest(6) stopping criterion. \n", + "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/B3jW2/src/stopping_controls.jl:77\n", + "┌ Info: Total of 13 iterations. \n", + "└ @ MLJIteration /Users/anthony/.julia/packages/MLJIteration/8GWjJ/src/core.jl:35\n" + ] + }, + { + "data": { + "text/plain": [ + "Machine{DeterministicIteratedModel{Pipeline282},…} trained 1 time; does not cache data\n", + " args: \n", + " 1:\tSource @311 ⏎ `Table{AbstractVector{Continuous}}`\n", + " 2:\tSource @222 ⏎ `AbstractVector{Continuous}`\n" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "clear()\n", + "mach = machine(iterated_pipe, X, y)\n", + "fit!(mach)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And plotting the traces:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + " \n", + " \n", + " \n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "plot(epochs, losses,\n", + " xlab = \"epoch\",\n", + " ylab = \"mean sum of squares error\",\n", + " label=\"out-of-sample\",\n", + " legend = :topleft);\n", + "scatter!(twinx(), epochs, training_losses, label=\"training\", color=:red)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "savefig(joinpath(\"assets\", \"loss.png\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**How `IteratedModel` works.** Training an `IteratedModel` means\n", + "holding out some data (80% in this case) so an out-of-sample loss\n", + "can be tracked and used in the specified stopping criterion,\n", + "`NumberSinceBest(4)`. However, once the stop is triggered, the model\n", + "wrapped by `IteratedModel` (our pipeline model) is retrained on all\n", + "data for the same number of iterations. Calling `predict(mach,\n", + "Xnew)` on new data uses the updated learned parameters." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In other words, `iterated_model` is a \"self-iterating\" version of\n", + "the original model, where `epochs` has been transformed from\n", + "hyper-parameter to *learned* parameter." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## An evaluation the self-iterating model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's an estimate of performance of our \"self-iterating\"\n", + "model:" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[33mEvaluating over 8 folds: 100%[=========================] Time: 0:00:09\u001b[39m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "err = 17.8 ± 2.7\n" + ] + }, + { + "data": { + "text/latex": [ + "$17.8 \\pm 2.7$" + ], + "text/plain": [ + "17.8 ± 2.7" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "e = evaluate!(mach,\n", + " resampling=CV(nfolds=8),\n", + " measures=[l2, l1])\n", + "\n", + "using Measurements\n", + "err = e.measurement[1] ± std(e.per_fold[1])/sqrt(7)\n", + "@show err" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which we can see has substantial uncertainty (not that CV estimates\n", + "of the uncertainty are trustworthy; see e.g., [this\n", + "survey](https://direct.mit.edu/neco/article-abstract/10/7/1895/6224/Approximate-Statistical-Tests-for-Comparing))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Comparison with other models on the test set" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Although we cannot assign them statistical significance, here are\n", + "comparisons, on the untouched test set, of the eror of our\n", + "self-iterating neural network regressor with a couple of other\n", + "models trained on the same data (using default hyperparameters):" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Training Machine{DeterministicIteratedModel{Pipeline282},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: final loss: 18.029846309038444\n", + "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/B3jW2/src/train.jl:29\n", + "┌ Info: Stop triggered by NumberSinceBest(6) stopping criterion. \n", + "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/B3jW2/src/stopping_controls.jl:77\n", + "┌ Info: Total of 13 iterations. \n", + "└ @ MLJIteration /Users/anthony/.julia/packages/MLJIteration/8GWjJ/src/core.jl:35\n", + "┌ Info: For silent loading, specify `verbosity=0`. \n", + "└ @ Main /Users/anthony/.julia/packages/MLJModels/4sRmw/src/loading.jl:168\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "import EvoTrees ✔\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: For silent loading, specify `verbosity=0`. \n", + "└ @ Main /Users/anthony/.julia/packages/MLJModels/4sRmw/src/loading.jl:168\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "import MLJLinearModels ✔\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "┌ Info: Training Machine{EvoTreeRegressor{Float64,…},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{LinearRegressor,…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: Training Machine{DeterministicIteratedModel{Pipeline282},…}.\n", + "└ @ MLJBase /Users/anthony/.julia/packages/MLJBase/u6vLz/src/machines.jl:403\n", + "┌ Info: final loss: 18.029846309038444\n", + "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/B3jW2/src/train.jl:29\n", + "┌ Info: Stop triggered by NumberSinceBest(6) stopping criterion. \n", + "└ @ IterationControl /Users/anthony/.julia/packages/IterationControl/B3jW2/src/stopping_controls.jl:77\n", + "┌ Info: Total of 13 iterations. \n", + "└ @ MLJIteration /Users/anthony/.julia/packages/MLJIteration/8GWjJ/src/core.jl:35\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "┌──────────────────────────────────────────┬────────────────────┐\n", + "│\u001b[1m models \u001b[0m│\u001b[1m mean_square_errors \u001b[0m│\n", + "│\u001b[90m DataType \u001b[0m│\u001b[90m Float64 \u001b[0m│\n", + "│\u001b[90m Unknown \u001b[0m│\u001b[90m Continuous \u001b[0m│\n", + "├──────────────────────────────────────────┼────────────────────┤\n", + "│ EvoTreeRegressor{Float64, Linear, Int64} │ 23.939 │\n", + "│ LinearRegressor │ 25.3434 │\n", + "│ DeterministicIteratedModel{Pipeline282} │ 14.7957 │\n", + "└──────────────────────────────────────────┴────────────────────┘\n" + ] + } + ], + "source": [ + "function performance(model)\n", + " mach = machine(model, X, y) |> fit!\n", + " yhat = predict(mach, Xtest)\n", + " l2(yhat, ytest) |> mean\n", + "end\n", + "performance(iterated_pipe)\n", + "\n", + "three_models = [(@load EvoTreeRegressor)(), # tree boosting model\n", + " (@load LinearRegressor pkg=MLJLinearModels)(),\n", + " iterated_pipe]\n", + "\n", + "errs = performance.(three_models)\n", + "\n", + "(models=typeof.(three_models), mean_square_errors=errs) |> pretty" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "*This notebook was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).*" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Julia 1.6.3", + "language": "julia", + "name": "julia-1.6" + }, + "language_info": { + "file_extension": ".jl", + "mimetype": "application/julia", + "name": "julia", + "version": "1.6.3" + } + }, + "nbformat": 4, + "nbformat_minor": 3 +} diff --git a/examples/generate.jl b/examples/generate.jl new file mode 100644 index 00000000..362774fb --- /dev/null +++ b/examples/generate.jl @@ -0,0 +1,40 @@ +# helper +function runcommand(cmd) + @info cmd + run(cmd) +end + +# `env` is the environment directory +function generate(env; execute=true, pluto=true) + quote + using Pkg + Pkg.activate(temp=true) + Pkg.add(name="Literate", rev="fe/pluto") + using Literate + + const ENVDIR = $env + const OUTDIR = joinpath(ENVDIR, "..") + const INFILE = joinpath(OUTDIR, "notebook.jl") + + # generate pluto notebook: + if $pluto + Literate.notebook(INFILE, ENVDIR, flavor=Literate.PlutoFlavor()) + runcommand(`mv $ENVDIR/notebook.jl $OUTDIR/notebook.pluto.jl`) + end + + Literate.notebook(INFILE, OUTDIR, execute=false) + runcommand( + `mv $OUTDIR/notebook.ipynb $OUTDIR/notebook.unexecuted.ipynb`) + $execute && Literate.notebook(INFILE, OUTDIR, execute=true) + + end |> eval +end + +# Pkg.add("Pluto") +# using Pluto +# Pluto.run(notebook=joinpath(OUTDIR, "notebook.pluto.jl")) + +# Pkg.add("IJulia") +# Pkg.instantiate() +# using IJulia +# IJulia.notebook(dir=OUTDIR) diff --git a/src/MLJFlux.jl b/src/MLJFlux.jl index 5a909ba4..150e031b 100644 --- a/src/MLJFlux.jl +++ b/src/MLJFlux.jl @@ -14,14 +14,14 @@ using ColorTypes using ComputationalResources using Random -include("penalized_losses.jl") +include("penalizers.jl") include("core.jl") include("builders.jl") include("types.jl") include("regressor.jl") include("classifier.jl") include("image.jl") -include("common.jl") +include("mlj_model_interface.jl") ### Package specific model traits: MLJModelInterface.metadata_pkg.((NeuralNetworkRegressor, diff --git a/src/builders.jl b/src/builders.jl index bdb63138..2c417c20 100644 --- a/src/builders.jl +++ b/src/builders.jl @@ -9,7 +9,7 @@ # input/output dimensions/shape. # Below n or (n1, n2) etc refers to network inputs, while m or (m1, -# m2) etc refers to outputs. +# m2) etc refers to outputs. abstract type Builder <: MLJModelInterface.MLJType end @@ -83,7 +83,7 @@ MLP(; hidden=(100,), σ=Flux.relu) = MLP(hidden, σ) function MLJFlux.build(mlp::MLP, rng, n_in, n_out) init=Flux.glorot_uniform(rng) - hidden = [Flux.Dense(n_in, mlp.hidden[1], mlp.σ, init=init)] + hidden = Any[Flux.Dense(n_in, mlp.hidden[1], mlp.σ, init=init)] for i ∈ 2:length(mlp.hidden) push!(hidden, Flux.Dense(mlp.hidden[i-1], mlp.hidden[i], diff --git a/src/core.jl b/src/core.jl index f4960d3e..de2a982d 100644 --- a/src/core.jl +++ b/src/core.jl @@ -37,22 +37,28 @@ end (::Mover{<:CUDALibs})(data) = Flux.gpu(data) """ - train!(loss_func, parameters, optimiser, X, y) + train!(loss, penalty, chain, optimiser, X, y) A private method. -Update the parameters of a Flux chain with parameters `parameters`, -given a Flux-style "loss function" `loss(x, y)`. Here `X` and `y` are -vectors of batches of the training data, as detailed in the -[`MLJFlux.fit!`](@ref) document string. +Update the parameters of a Flux `chain`, where: + +- `(yhat, y) -> loss(yhat, y)` is the loss function + +- `params -> penalty(params)` is a regularization penalty function + +- `X` and `y` are vectors of batches of the training data, as detailed + in the [`MLJFlux.fit!`](@ref) document string. """ -function train!(loss_func, parameters, optimiser, X, y) +function train!(loss, penalty, chain, optimiser, X, y) n_batches = length(y) training_loss = zero(Float32) for i in 1:n_batches + parameters = Flux.params(chain) gs = Flux.gradient(parameters) do - batch_loss = loss_func(X[i], y[i]) + yhat = chain(X[i]) + batch_loss = loss(yhat, y[i]) + penalty(parameters) training_loss += batch_loss return batch_loss end @@ -63,15 +69,15 @@ end """ - fit!(penalized_loss, chain, optimiser, epochs, verbosity, X, y) + fit!(loss, penalty, chain, optimiser, epochs, verbosity, X, y) A private method. -Optimize a Flux model `chain`, where `penalized_loss(xb, yb)` is the -penalized loss associated with a batch of training input features `xb` -and target observations `yb` (and generally depends on `chain`). +Optimize a Flux model `chain`, where `(yhat, y) -> loss(yhat, y)` is +the loss, and `parameters -> penalty(parameters)` is the +regularization penalty function. -Here `chain` is a `Flux.Chain` object, or other "Flux model" such that +Here `chain` is a `Flux.Chain` object, or other Flux model such that `Flux.params(chain)` returns the parameters to be optimized. `X`, the vector of input batches and `y` the vector of target @@ -89,11 +95,6 @@ batches. Specifically, it is expected that: - The vectors `X` and `y` have the same length, coinciding with the total number of training batches. -The [`MLJFlux.PenalizedLoss`](@ref) constructor is available for -defining an appropriate `penalized_loss` from an MLJFlux model and -chain (the model specifies the unpenalized Flux loss, such as `mse`, -and the regularization parameters). - Both the `chain` and the data `(X, y)` must both live on a CPU or both live on a GPU. This `fit!` method takes no responsibility for data movement. @@ -105,7 +106,7 @@ of `chain` and `history` is a vector of penalized losses - one initial loss, and one loss per epoch. """ -function fit!(penalized_loss, chain, optimiser, epochs, verbosity, X, y) +function fit!(loss, penalty, chain, optimiser, epochs, verbosity, X, y) # intitialize and start progress meter: meter = Progress(epochs+1, dt=0, desc="Optimising neural net:", @@ -115,15 +116,13 @@ function fit!(penalized_loss, chain, optimiser, epochs, verbosity, X, y) # initiate history: n_batches = length(y) - training_loss = mean(penalized_loss(X[i], y[i]) for i in 1:n_batches) - history = [training_loss,] + parameters = Flux.params(chain) + losses = (loss(chain(X[i]), y[i]) + + penalty(parameters) for i in 1:n_batches) + history = [mean(losses),] for i in 1:epochs - current_loss = train!(penalized_loss, - Flux.params(chain), - optimiser, - X, - y) + current_loss = train!(loss, penalty, chain, optimiser, X, y) verbosity < 2 || @info "Loss is $(round(current_loss; sigdigits=4))" verbosity != 1 || next!(meter) diff --git a/src/common.jl b/src/mlj_model_interface.jl similarity index 95% rename from src/common.jl rename to src/mlj_model_interface.jl index a585ae9e..bfac2987 100644 --- a/src/common.jl +++ b/src/mlj_model_interface.jl @@ -52,18 +52,18 @@ function MLJModelInterface.fit(model::MLJFluxModel, rng = true_rng(model) shape = MLJFlux.shape(model, X, y) chain = build(model, rng, shape) |> move - penalized_loss = PenalizedLoss(model, chain) - + penalty = Penalty(model) data = move.(collate(model, X, y)) optimiser = deepcopy(model.optimiser) - chain, history = fit!(penalized_loss, + chain, history = fit!(model.loss, + penalty, chain, optimiser, model.epochs, verbosity, - data[1], + data[1], data[2]) # `optimiser` is now mutated @@ -109,7 +109,7 @@ function MLJModelInterface.update(model::MLJFluxModel, epochs = model.epochs end - penalized_loss = PenalizedLoss(model, chain) + penalty = Penalty(model) # we only get to keep the optimiser "state" carried over from # previous training if we're doing a warm restart and the user has not @@ -120,7 +120,8 @@ function MLJModelInterface.update(model::MLJFluxModel, optimiser = deepcopy(model.optimiser) end - chain, history = fit!(penalized_loss, + chain, history = fit!(model.loss, + penalty, chain, optimiser, epochs, diff --git a/src/penalized_losses.jl b/src/penalizers.jl similarity index 61% rename from src/penalized_losses.jl rename to src/penalizers.jl index 477bc913..ccea2d2d 100644 --- a/src/penalized_losses.jl +++ b/src/penalizers.jl @@ -33,31 +33,30 @@ function (p::Penalizer)(A) return λ*(α*L1 + (1 - α)*L2) end + + """ - PenalizedLoss(model, chain) + Penalty(model) -Returns a callable object `p`, for returning the penalized loss on -some batch of data `(x, y)`. Specifically, `p(x, y)` returns +Returns a callable object `p`, for returning the regularization +penalty `p(w)` associated with some collection of parameters `w`. For +example, `w = params(chain)` where `chain` is some Flux +model. Here `model` is an MLJFlux model ("model" in the MLJ +sense, not the Flux sense). Specifically, `p(w)` returns - loss(chain(x), y) + sum(Penalizer(λ, α).(params(chain))) + sum(Penalizer(λ, α).w) -where `loss = model.loss`, `α = model.alpha`, `λ = model.lambda`. +where `α = model.alpha`, `λ = model.lambda`. See also [`Penalizer`](@ref) """ -struct PenalizedLoss{P} - loss +struct Penalty{P} penalizer::P - chain - params - function PenalizedLoss(model, chain) - loss = model.loss + function Penalty(model) penalizer = Penalizer(model.lambda, model.alpha) - params = Flux.params(chain) - return new{typeof(penalizer)}(loss, penalizer, chain, params) + return new{typeof(penalizer)}(penalizer) end end -(p::PenalizedLoss{Penalizer{Nothing}})(x, y) = p.loss(p.chain(x), y) -(p::PenalizedLoss)(x, y) = p.loss(p.chain(x), y) + - sum(p.penalizer(θ) for θ in p.params) +(p::Penalty{Penalizer{Nothing}})(w) = zero(Float32) +(p::Penalty)(w) = sum(p.penalizer(wt) for wt in w) diff --git a/src/types.jl b/src/types.jl index 362262ff..bf5674af 100644 --- a/src/types.jl +++ b/src/types.jl @@ -9,22 +9,22 @@ const doc_regressor(model_name) = """ Instantiate an MLJFlux model. Available hyperparameters: -- `builder`: Default = `MLJFlux.Linear(σ=Flux.relu)` (regressors) or +- `builder`: Default = `MLJFlux.Linear(σ=Flux.relu)` (regressors) or `MLJFlux.Short(n_hidden=0, dropout=0.5, σ=Flux.σ)` (classifiers) - `optimiser`: The optimiser to use for training. Default = `Flux.ADAM()` -- `loss`: The loss function used for training. Default = `Flux.mse` +- `loss`: The loss function used for training. Default = `Flux.mse` (regressors) and `Flux.crossentropy` (classifiers) -- `n_epochs`: Number of epochs to train for. Default = `10` +- `epochs`: Number of epochs to train for. Default = `10` - `batch_size`: The batch_size for the data. Default = 1 - `lambda`: The regularization strength. Default = 0. Range = [0, ∞) -- `alpha`: The L2/L1 mix of regularization. Default = 0. Range = [0, 1] +- `alpha`: The L2/L1 mix of regularization. Default = 0. Range = [0, 1] - `rng`: The random number generator (RNG) passed to builders, for weight intitialization, for example. Can be any `AbstractRNG` or diff --git a/test/builders.jl b/test/builders.jl index 6ea28d26..030cbfa0 100644 --- a/test/builders.jl +++ b/test/builders.jl @@ -76,3 +76,11 @@ end ps4 = Flux.params(chain4) @test size.(ps4) == [(3, 3, 2, 16), (16,), (3, 144), (3,)] end + +@testset_accelerated "MLP" accel begin + builder = MLJFlux.MLP(hidden = (2, 2)) + rng = StableRNGs.StableRNG(123) + nn = MLJFlux.build(builder, rng, 3, 3) + @test length(nn.layers) == 3 + @test size(nn.layers[1].weight) == (2, 3) +end diff --git a/test/classifier.jl b/test/classifier.jl index 1ce7b879..135c3020 100644 --- a/test/classifier.jl +++ b/test/classifier.jl @@ -47,11 +47,13 @@ losses = [] # check flux model is an improvement on predicting constant # distribution: + stable_rng = StableRNGs.StableRNG(123) model = MLJFlux.NeuralNetworkClassifier(epochs=50, builder=builder, optimiser=optimiser, acceleration=accel, - batch_size=10) + batch_size=10, + rng=stable_rng) @time mach = fit!(machine(model, X, y), rows=train, verbosity=0) first_last_training_loss = MLJBase.report(mach)[1][[1, end]] push!(losses, first_last_training_loss[2]) diff --git a/test/core.jl b/test/core.jl index cf5b2ab6..75e03636 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1,4 +1,5 @@ Random.seed!(123) +stable_rng = StableRNGs.StableRNG(123) rowvec(y) = y rowvec(y::Vector) = reshape(y, 1, length(y)) @@ -6,7 +7,7 @@ rowvec(y::Vector) = reshape(y, 1, length(y)) @test MLJFlux.MLJModelInterface.istransparent(Flux.ADAM(0.1)) @testset "nrows" begin - Xmatrix = rand(10, 3) + Xmatrix = rand(stable_rng, 10, 3) X = MLJBase.table(Xmatrix) @test MLJFlux.nrows(X) == 10 @test MLJFlux.nrows(Tables.columntable(X)) == 10 @@ -14,11 +15,11 @@ end @testset "collate" begin # NeuralNetworRegressor: - Xmatrix = broadcast(x->round(x, sigdigits=2), rand(10, 3)) + Xmatrix = broadcast(x->round(x, sigdigits=2), rand(stable_rng, 10, 3)) # convert to a column table: X = MLJBase.table(Xmatrix) - y = rand(10) + y = rand(stable_rng, 10) model = MLJFlux.NeuralNetworkRegressor() model.batch_size= 3 @test MLJFlux.collate(model, X, y) == @@ -37,7 +38,7 @@ end reshape([1; 0], (2,1))])) # MultitargetNeuralNetworRegressor: - ymatrix = rand(10, 2) + ymatrix = rand(stable_rng, 10, 2) y = MLJBase.table(ymatrix) # a rowaccess table model = MLJFlux.NeuralNetworkRegressor() model.batch_size= 3 @@ -53,7 +54,7 @@ end ymatrix'[:,7:9], ymatrix'[:,10:10]])) # ImageClassifier - Xmatrix = coerce(rand(6, 6, 1, 10), GrayImage) + Xmatrix = coerce(rand(stable_rng, 6, 6, 1, 10), GrayImage) y = categorical(['a', 'b', 'a', 'a', 'b', 'a', 'a', 'a', 'b', 'a']) model = MLJFlux.ImageClassifier(batch_size=2) @@ -68,7 +69,7 @@ end end -Xmatrix = rand(100, 5) +Xmatrix = rand(stable_rng, 100, 5) X = MLJBase.table(Xmatrix) y = Xmatrix[:, 1] + Xmatrix[:, 2] + Xmatrix[:, 3] + Xmatrix[:, 4] + Xmatrix[:, 5] @@ -95,7 +96,7 @@ model = MLJFlux.NeuralNetworkRegressor() # any model will do here chain_no_drop = deepcopy(chain_yes_drop) chain_no_drop.layers[2].p = 1.0 -test_input = rand(Float32, 5, 1) +test_input = rand(stable_rng, Float32, 5, 1) # check both chains have same behaviour before training: @test chain_yes_drop(test_input) == chain_no_drop(test_input) @@ -107,8 +108,9 @@ epochs = 10 move = MLJFlux.Mover(accel) Random.seed!(123) - penalized_loss = MLJFlux.PenalizedLoss(model, chain_yes_drop) - _chain_yes_drop, history = MLJFlux.fit!(penalized_loss, + penalty = MLJFlux.Penalty(model) + _chain_yes_drop, history = MLJFlux.fit!(model.loss, + penalty, chain_yes_drop, Flux.Optimise.ADAM(0.001), epochs, @@ -118,14 +120,15 @@ epochs = 10 println() Random.seed!(123) - penalized_loss = MLJFlux.PenalizedLoss(model, chain_no_drop) - _chain_no_drop, history = MLJFlux.fit!(penalized_loss, - chain_no_drop, - Flux.Optimise.ADAM(0.001), - epochs, - 0, - data[1], - data[2]) + penalty = MLJFlux.Penalty(model) + _chain_no_drop, history = MLJFlux.fit!(model.loss, + penalty, + chain_no_drop, + Flux.Optimise.ADAM(0.001), + epochs, + 0, + data[1], + data[2]) # check chains have different behaviour after training: @test !(_chain_yes_drop(test_input) ≈ diff --git a/test/image.jl b/test/image.jl index f3a1ceb9..1866b1ed 100644 --- a/test/image.jl +++ b/test/image.jl @@ -1,6 +1,7 @@ ## BASIC IMAGE TESTS GREY Random.seed!(123) +stable_rng = StableRNGs.StableRNG(123) mutable struct MyNeuralNetwork <: MLJFlux.Builder kernel1 @@ -14,26 +15,28 @@ function MLJFlux.build(model::MyNeuralNetwork, rng, ip, op, n_channels) Flux.Conv(model.kernel2, 2=>1, init=init), x->reshape(x, :, size(x)[end]), Flux.Dense(16, op, init=init)) -end +end builder = MyNeuralNetwork((2,2), (2,2)) # collection of gray images as a 4D array in WHCN format: -raw_images = rand(Float32, 6, 6, 1, 50); +raw_images = rand(stable_rng, Float32, 6, 6, 1, 50); # as a vector of Matrix{<:AbstractRGB} images = coerce(raw_images, GrayImage); @test scitype(images) == AbstractVector{GrayImage{6,6}} -labels = categorical(rand(1:5, 50)); +labels = categorical(rand(stable_rng, 1:5, 50)); losses = [] @testset_accelerated "ImageClassifier basic tests" accel begin Random.seed!(123) + stable_rng = StableRNGs.StableRNG(123) model = MLJFlux.ImageClassifier(builder=builder, epochs=10, - acceleration=accel) + acceleration=accel, + rng=stable_rng) fitresult, cache, _report = MLJBase.fit(model, 0, images, labels) @@ -45,8 +48,12 @@ losses = [] pred = MLJBase.predict(model, fitresult, images[1:6]) # try with batch_size > 1: - model = MLJFlux.ImageClassifier(builder=builder, epochs=10, batch_size=2, - acceleration=accel) + model = MLJFlux.ImageClassifier(builder=builder, + epochs=10, + batch_size=2, + acceleration=accel, + rng=stable_rng) + model.optimiser.eta = 0.005 @time fitresult, cache, _report = MLJBase.fit(model, 0, images, labels); first_last_training_loss = _report[1][[1, end]] push!(losses, first_last_training_loss[2]) @@ -61,9 +68,10 @@ losses = [] end -# check different resources (CPU1, CUDALibs, etc)) give about the same loss: +# check different resources (CPU1, CUDALibs) give about the same loss: reference = losses[1] -@test all(x->abs(x - reference)/reference < 1e-5, losses[2:end]) +@info "Losses for each computational resource: $losses" +@test all(x->abs(x - reference)/reference < 5e-4, losses[2:end]) ## MNIST IMAGES TEST @@ -100,10 +108,12 @@ losses = [] @testset_accelerated "Image MNIST" accel begin Random.seed!(123) + stable_rng = StableRNGs.StableRNG(123) model = MLJFlux.ImageClassifier(builder=MyConvBuilder(), acceleration=accel, - batch_size=50) + batch_size=50, + rng=stable_rng) @time fitresult, cache, _report = MLJBase.fit(model, 0, images[1:500], labels[1:500]); @@ -119,6 +129,7 @@ end # check different resources (CPU1, CUDALibs, etc)) give about the same loss: reference = losses[1] +@info "Losses for each computational resource: $losses" @test all(x->abs(x - reference)/reference < 0.05, losses[2:end]) @@ -127,7 +138,7 @@ reference = losses[1] builder = MyNeuralNetwork((2,2), (2,2)) # collection of color images as a 4D array in WHCN format: -raw_images = rand(Float32, 6, 6, 3, 50); +raw_images = rand(stable_rng, Float32, 6, 6, 3, 50); images = coerce(raw_images, ColorImage); @test scitype(images) == AbstractVector{ColorImage{6,6}} @@ -138,10 +149,12 @@ losses = [] @testset_accelerated "ColorImages" accel begin Random.seed!(123) + stable_rng = StableRNGs.StableRNG(123) model = MLJFlux.ImageClassifier(builder=builder, epochs=10, - acceleration=accel) + acceleration=accel, + rng=stable_rng) # tests update logic, etc (see test_utililites.jl): @test basictest(MLJFlux.ImageClassifier, images, labels, @@ -157,7 +170,8 @@ losses = [] model = MLJFlux.ImageClassifier(builder=builder, epochs=10, batch_size=2, - acceleration=accel) + acceleration=accel, + rng=stable_rng) fitresult, cache, _report = MLJBase.fit(model, 0, images, labels); @test optimisertest(MLJFlux.ImageClassifier, images, labels, @@ -167,6 +181,7 @@ end # check different resources (CPU1, CUDALibs, etc)) give about the same loss: reference = losses[1] +@info "Losses for each computational resource: $losses" @test all(x->abs(x - reference)/reference < 1e-5, losses[2:end]) true diff --git a/test/common.jl b/test/mlj_model_interface.jl similarity index 100% rename from test/common.jl rename to test/mlj_model_interface.jl diff --git a/test/penalized_losses.jl b/test/penalizers.jl similarity index 68% rename from test/penalized_losses.jl rename to test/penalizers.jl index bbc67720..68ee6b99 100644 --- a/test/penalized_losses.jl +++ b/test/penalizers.jl @@ -1,6 +1,8 @@ using Statistics +import MLJFlux +import Flux -@testset "penalties" begin +@testset "Penalizer" begin A = [-1 2; -3 4] lambda = 1 @@ -21,19 +23,13 @@ using Statistics @test penalty(A) == 0 end -@testset "penalized_losses" begin - # construct a penalized loss function: +@testset "Penalty" begin model = MLJFlux.NeuralNetworkRegressor(lambda=1, alpha=1, loss=Flux.mae) chain = Flux.Dense(3, 1, identity) - p = MLJFlux.PenalizedLoss(model, chain) - - # construct a batch: - b = 5 - x = rand(Float32, 3, b) - y = rand(Float32, 1, b) + w = Flux.params(chain) + p = MLJFlux.Penalty(model) # compare loss by hand and with penalized loss function: penalty = (sum(abs.(chain.weight)) + abs(chain.bias[1])) - yhat = chain(x) - @test p(x, y) ≈ Flux.mae(yhat, y) + penalty + @test p(w) ≈ penalty end diff --git a/test/regressor.jl b/test/regressor.jl index a3b2e088..0b6c7c7f 100644 --- a/test/regressor.jl +++ b/test/regressor.jl @@ -4,7 +4,7 @@ N = 200 X = MLJBase.table(randn(Float32, N, 5)); # TODO: replace Short2 -> Short when -# https://github.com/FluxML/Flux.jl/issues/1372 is resolved: +# https://github.com/FluxML/Flux.jl/pull/1618 is resolved: builder = Short2(σ=identity) optimiser = Flux.Optimise.ADAM() @@ -26,9 +26,11 @@ train, test = MLJBase.partition(1:N, 0.7) 0.7, accel) - # test a bit better than constant predictor + # test model is a bit better than constant predictor: + stable_rng = StableRNGs.StableRNG(123) model = MLJFlux.NeuralNetworkRegressor(builder=builder, - acceleration=accel) + acceleration=accel, + rng=stable_rng) @time fitresult, _, rpt = fit(model, 0, MLJBase.selectrows(X, train), y[train]) first_last_training_loss = rpt[1][[1, end]] @@ -70,7 +72,7 @@ losses = [] 1.0, accel) - # test a bit better than constant predictor + # test model is a bit better than constant predictor model = MLJFlux.MultitargetNeuralNetworkRegressor(acceleration=accel, builder=builder) @time fitresult, _, rpt = @@ -79,8 +81,8 @@ losses = [] push!(losses, first_last_training_loss[2]) # @show first_last_training_loss yhat = predict(model, fitresult, selectrows(X, test)) - truth = ymatrix[test] - goal = 0.9*model.loss(truth .- mean(truth), 0) + truth = ymatrix[test,:] + goal = 0.8*model.loss(truth .- mean(truth), 0) @test model.loss(Tables.matrix(yhat), truth) < goal optimisertest(MLJFlux.MultitargetNeuralNetworkRegressor, diff --git a/test/runtests.jl b/test/runtests.jl index d72b13e8..ab44a92f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -25,6 +25,7 @@ MLJFlux.gpu_isdead() && push!(EXCLUDED_RESOURCE_TYPES, CUDALibs) "Excluded tests marked as \"broken\"." # alternative version of Short builder with no dropout; see +# https://github.com/FluxML/Flux.jl/issues/1372 and # https://github.com/FluxML/Flux.jl/issues/1372 mutable struct Short2 <: MLJFlux.Builder n_hidden::Int # if zero use geometric mean of input/output @@ -44,8 +45,8 @@ seed!(123) include("test_utils.jl") -@testset "penalized_losses" begin - include("penalized_losses.jl") +@testset "penalizers" begin + include("penalizers.jl") end @testset "core" begin @@ -56,8 +57,8 @@ end include("builders.jl") end -@testset "common" begin - include("common.jl") +@testset "mlj_model_interface" begin + include("mlj_model_interface.jl") end @testset "regressor" begin diff --git a/test/test_utils.jl b/test/test_utils.jl index 942643a9..7a2728b4 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -57,9 +57,12 @@ function basictest(ModelType, X, y, builder, optimiser, threshold, accel) eval(quote + stable_rng = StableRNGs.StableRNG(123) + model = $ModelType_ex(builder=$builder, optimiser=$optimiser, - acceleration=$accel_ex) + acceleration=$accel_ex, + rng=stable_rng) fitresult, cache, _report = MLJBase.fit(model, 0, $X, $y); @@ -89,7 +92,8 @@ function basictest(ModelType, X, y, builder, optimiser, threshold, accel) model = $ModelType_ex(builder=$builder, optimiser=$optimiser, epochs=2, - acceleration=$accel_ex) + acceleration=$accel_ex, + rng=stable_rng) fitresult, cache, _report = MLJBase.fit(model, 0, $X, $y); @@ -127,6 +131,7 @@ function optimisertest(ModelType, X, y, builder, optimiser, accel) optimiser = deepcopy(optimiser) eval(quote + model = $ModelType_ex(builder=$builder, optimiser=$optimiser, acceleration=$accel_ex,