diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..700707ced --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2102b4f40..33c468554 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -35,11 +35,11 @@ jobs: - '1' - '1.6' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.version }} - - uses: actions/cache@v1 + - uses: actions/cache@v3 env: cache-name: cache-artifacts with: diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml index eca6c13ba..bf517da6f 100644 --- a/.github/workflows/Documentation.yml +++ b/.github/workflows/Documentation.yml @@ -11,7 +11,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: julia-actions/setup-julia@latest with: version: '1' diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml index f0b8bb072..4eb03b44f 100644 --- a/.github/workflows/Downstream.yml +++ b/.github/workflows/Downstream.yml @@ -22,14 +22,14 @@ jobs: - {user: SciML, repo: ModelingToolkit.jl, group: All} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.julia-version }} arch: x64 - uses: julia-actions/julia-buildpkg@latest - name: Clone Downstream - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: ${{ matrix.package.user }}/${{ matrix.package.repo }} path: downstream @@ -51,6 +51,6 @@ jobs: exit(0) # Exit immediately, as a success end - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v3 with: file: lcov.info diff --git a/.github/workflows/FormatCheck.yml b/.github/workflows/FormatCheck.yml index 2a3517a0f..f80d0b18b 100644 --- a/.github/workflows/FormatCheck.yml +++ b/.github/workflows/FormatCheck.yml @@ -21,7 +21,7 @@ jobs: with: version: ${{ matrix.julia-version }} - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 - name: Install JuliaFormatter and format # This will use the latest version by default but you can set the version like so: # diff --git a/Project.toml b/Project.toml index c5a5487eb..0d2ddfcf0 100644 --- a/Project.toml +++ b/Project.toml @@ -1,9 +1,9 @@ name = "Optimization" uuid = "7f7a1694-90dd-40f0-9382-eb1efda571ba" -version = "3.11.2" +version = "3.12.1" [deps] -ArrayInterfaceCore = "30b0a656-2188-435a-8636-2ec0e6a096e2" +ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" ConsoleProgressMonitor = "88cd18e8-d9cc-4ea6-8889-5259c0d15c8b" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" @@ -18,8 +18,7 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" [compat] -ArrayInterface = "6" -ArrayInterfaceCore = "0.1.1" +ArrayInterface = "6, 7" ConsoleProgressMonitor = "0.1" DocStringExtensions = "0.8, 0.9" LoggingExtras = "0.4, 0.5, 1" @@ -29,6 +28,3 @@ Requires = "1.0" SciMLBase = "1.79.0" TerminalLoggers = "0.1" julia = "1.6" - -[extras] -ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" diff --git a/README.md b/README.md index 7612cc59e..06b4c1882 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor%27s%20Guide-blueviolet)](https://github.com/SciML/ColPrac) [![SciML Code Style](https://img.shields.io/static/v1?label=code%20style&message=SciML&color=9558b2&labelColor=389826)](https://github.com/SciML/SciMLStyle) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7738525.svg)](https://doi.org/10.5281/zenodo.7738525) + Optimization.jl is a package with a scope that is beyond your normal global optimization package. Optimization.jl seeks to bring together all of the optimization packages it can find, local and global, into one unified Julia interface. This means, you diff --git a/docs/pages.jl b/docs/pages.jl index 3a8a194c6..1d7d2971c 100644 --- a/docs/pages.jl +++ b/docs/pages.jl @@ -1,11 +1,13 @@ pages = ["index.md", + "getting_started.md", "Tutorials" => [ - "tutorials/intro.md", - "tutorials/rosenbrock.md", "tutorials/minibatch.md", "tutorials/symbolic.md", "tutorials/constraints.md", ], + "Examples" => [ + "examples/rosenbrock.md", + ], "Basics" => [ "API/optimization_problem.md", "API/optimization_function.md", diff --git a/docs/src/tutorials/rosenbrock.md b/docs/src/examples/rosenbrock.md similarity index 98% rename from docs/src/tutorials/rosenbrock.md rename to docs/src/examples/rosenbrock.md index 5b7f149da..240124b57 100644 --- a/docs/src/tutorials/rosenbrock.md +++ b/docs/src/examples/rosenbrock.md @@ -1,6 +1,6 @@ # Solving the Rosenbrock Problem in >10 Ways -This tutorial is a demonstration of many different solvers to demonstrate the +This example is a demonstration of many different solvers to demonstrate the flexibility of Optimization.jl. This is a gauntlet of many solvers to get a feel for common workflows of the package and give copy-pastable starting points. diff --git a/docs/src/tutorials/intro.md b/docs/src/getting_started.md similarity index 99% rename from docs/src/tutorials/intro.md rename to docs/src/getting_started.md index 59354a996..b6fef6a44 100644 --- a/docs/src/tutorials/intro.md +++ b/docs/src/getting_started.md @@ -1,4 +1,4 @@ -# Basic usage +# Getting Started with Optimization in Julia In this tutorial, we introduce the basics of Optimization.jl by showing how to easily mix local optimizers from Optim.jl and global optimizers diff --git a/docs/src/optimization_packages/mathoptinterface.md b/docs/src/optimization_packages/mathoptinterface.md index bc7ac89ec..14ff18f41 100644 --- a/docs/src/optimization_packages/mathoptinterface.md +++ b/docs/src/optimization_packages/mathoptinterface.md @@ -82,11 +82,15 @@ sol = solve(prob, opt) The following shows how to use integer linear programming within `Optimization`. We will solve the classical Knapsack Problem using `Juniper.jl`. - [`Juniper.Optimizer`](https://github.com/lanl-ansi/Juniper.jl) + - Juniper requires a nonlinear optimizer to be set via the `nl_solver` option, which must be a MathOptInterface-based optimizer. See the [Juniper documentation](https://github.com/lanl-ansi/Juniper.jl) for more detail. - - Allows only for binary decisions + - The integer domain is infered based on the bounds of the variable: + + + Setting the lower bound to zero and the upper bound to one corresponds to `MOI.ZeroOne()` or a binary decision variable + + Providing other or no bounds corresponds to `MOI.Integer()` ```@example MOI v = [1.0, 2.0, 4.0, 3.0] diff --git a/lib/OptimizationMOI/Project.toml b/lib/OptimizationMOI/Project.toml index 5f73db761..891713082 100644 --- a/lib/OptimizationMOI/Project.toml +++ b/lib/OptimizationMOI/Project.toml @@ -1,7 +1,7 @@ name = "OptimizationMOI" uuid = "fd9f6733-72f4-499f-8506-86b2bdd0dea1" authors = ["Vaibhav Dixit and contributors"] -version = "0.1.7" +version = "0.1.8" [deps] MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" diff --git a/lib/OptimizationMOI/test/runtests.jl b/lib/OptimizationMOI/test/runtests.jl index 0da368727..cd3fffdea 100644 --- a/lib/OptimizationMOI/test/runtests.jl +++ b/lib/OptimizationMOI/test/runtests.jl @@ -158,25 +158,43 @@ end sol = solve(prob, AmplNLWriter.Optimizer(Ipopt_jll.amplexe)) end -@testset "MINLP" begin - v = [1.0, 2.0, 4.0, 3.0] - w = [5.0, 4.0, 3.0, 2.0] - W = 4.0 - u0 = [0.0, 0.0, 0.0, 1.0] - - optfun = OptimizationFunction((u, p) -> -v'u, cons = (res, u, p) -> res .= w'u, - Optimization.AutoForwardDiff()) - - optprob = OptimizationProblem(optfun, u0; lb = zero.(u0), ub = one.(u0), - int = ones(Bool, length(u0)), - lcons = [-Inf;], ucons = [W;]) - +@testset "Integer Support" begin nl_solver = OptimizationMOI.MOI.OptimizerWithAttributes(Ipopt.Optimizer, "print_level" => 0) minlp_solver = OptimizationMOI.MOI.OptimizerWithAttributes(Juniper.Optimizer, "nl_solver" => nl_solver) - res = solve(optprob, minlp_solver) - @test res.u == [0.0, 0.0, 1.0, 0.0] - @test res.objective == -4.0 + @testset "Binary Domain" begin + v = [1.0, 2.0, 4.0, 3.0] + w = [5.0, 4.0, 3.0, 2.0] + W = 4.0 + u0 = [0.0, 0.0, 0.0, 1.0] + + optfun = OptimizationFunction((u, p) -> -v'u, cons = (res, u, p) -> res .= w'u, + Optimization.AutoForwardDiff()) + + optprob = OptimizationProblem(optfun, u0; lb = zero.(u0), ub = one.(u0), + int = ones(Bool, length(u0)), + lcons = [-Inf;], ucons = [W;]) + + res = solve(optprob, minlp_solver) + @test res.u == [0.0, 0.0, 1.0, 0.0] + @test res.objective == -4.0 + end + + @testset "Integer Domain" begin + x = [1.0, 2.0, 4.0, 3.0] + y = [5.0, 10.0, 20.0, 15.0] + u0 = [1.0] + + optfun = OptimizationFunction((u, p) -> sum(abs2, x * u[1] .- y), + Optimization.AutoForwardDiff()) + + optprob = OptimizationProblem(optfun, u0; lb = one.(u0), ub = 6.0 .* u0, + int = ones(Bool, length(u0))) + + res = solve(optprob, minlp_solver) + @test res.u ≈ [5.0] + @test res.objective <= 5eps() + end end diff --git a/lib/OptimizationNLopt/src/OptimizationNLopt.jl b/lib/OptimizationNLopt/src/OptimizationNLopt.jl index f10781d07..73f80349b 100644 --- a/lib/OptimizationNLopt/src/OptimizationNLopt.jl +++ b/lib/OptimizationNLopt/src/OptimizationNLopt.jl @@ -77,6 +77,31 @@ function __map_optimizer_args!(prob::OptimizationProblem, opt::NLopt.Opt; return nothing end +function __nlopt_status_to_ReturnCode(status::Symbol) + if status in Symbol.([ + NLopt.SUCCESS, + NLopt.STOPVAL_REACHED, + NLopt.FTOL_REACHED, + NLopt.XTOL_REACHED, + ]) + return ReturnCode.Success + elseif status == Symbol(NLopt.MAXEVAL_REACHED) + return ReturnCode.MaxIters + elseif status == Symbol(NLopt.MAXTIME_REACHED) + return ReturnCode.MaxTime + elseif status in Symbol.([ + NLopt.OUT_OF_MEMORY, + NLopt.INVALID_ARGS, + NLopt.FAILURE, + NLopt.ROUNDOFF_LIMITED, + NLopt.FORCED_STOP, + ]) + return ReturnCode.Failure + else + return ReturnCode.Default + end +end + function SciMLBase.__solve(prob::OptimizationProblem, opt::Union{NLopt.Algorithm, NLopt.Opt}; maxiters::Union{Number, Nothing} = nothing, @@ -134,8 +159,9 @@ function SciMLBase.__solve(prob::OptimizationProblem, (minf, minx, ret) = NLopt.optimize(opt_setup, prob.u0) t1 = time() + retcode = __nlopt_status_to_ReturnCode(ret) SciMLBase.build_solution(SciMLBase.DefaultOptimizationCache(prob.f, prob.p), opt, minx, - minf; original = opt_setup, retcode = ret, + minf; original = opt_setup, retcode = retcode, solve_time = t1 - t0) end diff --git a/lib/OptimizationOptimisers/Project.toml b/lib/OptimizationOptimisers/Project.toml index c1194fe69..54cfde53d 100644 --- a/lib/OptimizationOptimisers/Project.toml +++ b/lib/OptimizationOptimisers/Project.toml @@ -1,7 +1,7 @@ name = "OptimizationOptimisers" uuid = "42dfb2eb-d2b4-4451-abcd-913932933ac1" authors = ["Vaibhav Dixit and contributors"] -version = "0.1.1" +version = "0.1.2" [deps] Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" diff --git a/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl b/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl index 70b82cd1a..6c6f22906 100644 --- a/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl +++ b/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl @@ -4,12 +4,7 @@ using Reexport, Printf, ProgressLogging @reexport using Optimisers, Optimization using Optimization.SciMLBase -const OptimisersOptimizers = Union{Descent, Adam, Momentum, Nesterov, RMSProp, - AdaGrad, AdaMax, AdaDelta, AMSGrad, NAdam, RAdam, OAdam, - AdaBelief, - WeightDecay, ClipGrad, ClipNorm, OptimiserChain} - -function SciMLBase.__solve(prob::OptimizationProblem, opt::OptimisersOptimizers, +function SciMLBase.__solve(prob::OptimizationProblem, opt::AbstractRule, data = Optimization.DEFAULT_DATA; maxiters::Number = 0, callback = (args...) -> (false), progress = false, save_best = true, kwargs...) diff --git a/lib/OptimizationPolyalgorithms/Project.toml b/lib/OptimizationPolyalgorithms/Project.toml index 36e0ea237..498edd576 100644 --- a/lib/OptimizationPolyalgorithms/Project.toml +++ b/lib/OptimizationPolyalgorithms/Project.toml @@ -1,7 +1,7 @@ name = "OptimizationPolyalgorithms" uuid = "500b13db-7e66-49ce-bda4-eed966be6282" authors = ["Vaibhav Dixit and contributors"] -version = "0.1.2" +version = "0.2.0" [deps] Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" diff --git a/src/Optimization.jl b/src/Optimization.jl index 1145d081b..13c910437 100644 --- a/src/Optimization.jl +++ b/src/Optimization.jl @@ -8,7 +8,7 @@ using Reexport @reexport using SciMLBase using Requires using Logging, ProgressLogging, ConsoleProgressMonitor, TerminalLoggers, LoggingExtras -using ArrayInterfaceCore, Base.Iterators, SparseArrays +using ArrayInterface, Base.Iterators, SparseArrays using Pkg import SciMLBase: OptimizationProblem, OptimizationFunction, AbstractADType, ObjSense, diff --git a/src/function/finitediff.jl b/src/function/finitediff.jl index c6bb929d8..a7b97a280 100644 --- a/src/function/finitediff.jl +++ b/src/function/finitediff.jl @@ -77,7 +77,7 @@ function instantiate_function(f, x, adtype::AutoFiniteDiff, p, if f.hv === nothing hv = function (H, θ, v, args...) - res = ArrayInterfaceCore.zeromatrix(θ) + res = ArrayInterface.zeromatrix(θ) hess(res, θ, args...) H .= res * v end diff --git a/src/function/forwarddiff.jl b/src/function/forwarddiff.jl index 275b832e7..fb36743ea 100644 --- a/src/function/forwarddiff.jl +++ b/src/function/forwarddiff.jl @@ -64,7 +64,7 @@ function instantiate_function(f::OptimizationFunction{true}, x, if f.hv === nothing hv = function (H, θ, v, args...) - res = ArrayInterfaceCore.zeromatrix(θ) + res = ArrayInterface.zeromatrix(θ) hess(res, θ, args...) H .= res * v end diff --git a/src/function/function.jl b/src/function/function.jl index 90c8d5a72..bd413992b 100644 --- a/src/function/function.jl +++ b/src/function/function.jl @@ -43,11 +43,10 @@ function that is not defined, an error is thrown. For more information on the use of automatic differentiation, see the documentation of the `AbstractADType` types. """ -function instantiate_function(f, x, ::AbstractADType, - p, num_cons = 0) - grad = f.grad === nothing ? nothing : (G, x) -> f.grad(G, x, p) - hess = f.hess === nothing ? nothing : (H, x) -> f.hess(H, x, p) - hv = f.hv === nothing ? nothing : (H, x, v) -> f.hv(H, x, v, p) +function instantiate_function(f, x, ::AbstractADType, p, num_cons = 0) + grad = f.grad === nothing ? nothing : (G, x, args...) -> f.grad(G, x, p, args...) + hess = f.hess === nothing ? nothing : (H, x, args...) -> f.hess(H, x, p, args...) + hv = f.hv === nothing ? nothing : (H, x, v, args...) -> f.hv(H, x, v, p, args...) cons = f.cons === nothing ? nothing : (res, x) -> f.cons(res, x, p) cons_j = f.cons_j === nothing ? nothing : (res, x) -> f.cons_j(res, x, p) cons_h = f.cons_h === nothing ? nothing : (res, x) -> f.cons_h(res, x, p) @@ -73,9 +72,9 @@ end function instantiate_function(f, cache::ReInitCache, ::AbstractADType, num_cons = 0) - grad = f.grad === nothing ? nothing : (G, x) -> f.grad(G, x, cache.p) - hess = f.hess === nothing ? nothing : (H, x) -> f.hess(H, x, cache.p) - hv = f.hv === nothing ? nothing : (H, x, v) -> f.hv(H, x, v, cache.p) + grad = f.grad === nothing ? nothing : (G, x, args...) -> f.grad(G, x, cache.p, args...) + hess = f.hess === nothing ? nothing : (H, x, args...) -> f.hess(H, x, cache.p, args...) + hv = f.hv === nothing ? nothing : (H, x, v, args...) -> f.hv(H, x, v, cache.p, args...) cons = f.cons === nothing ? nothing : (res, x) -> f.cons(res, x, cache.p) cons_j = f.cons_j === nothing ? nothing : (res, x) -> f.cons_j(res, x, cache.p) cons_h = f.cons_h === nothing ? nothing : (res, x) -> f.cons_h(res, x, cache.p) diff --git a/src/function/mtk.jl b/src/function/mtk.jl index 2b0fad7fe..83bdaf3e6 100644 --- a/src/function/mtk.jl +++ b/src/function/mtk.jl @@ -62,7 +62,7 @@ function instantiate_function(f, x, adtype::AutoModelingToolkit, p, hv = function (H, θ, v, args...) res = adtype.obj_sparse ? (eltype(θ)).(f.hess_prototype) : - ArrayInterfaceCore.zeromatrix(θ) + ArrayInterface.zeromatrix(θ) hess(res, θ, args...) H .= res * v end diff --git a/test/Project.toml b/test/Project.toml index 0c560340c..e9183b56a 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -12,6 +12,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" +SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" @@ -28,6 +29,7 @@ Optim = ">= 1.4.1" OrdinaryDiffEq = ">= 5" ReverseDiff = ">= 1.9.0" SafeTestsets = ">= 0.0.1" +SciMLSensitivity = ">= 7.0.0" Tracker = ">= 0.2" Zygote = ">= 0.5" julia = "1.5" diff --git a/test/minibatch.jl b/test/minibatch.jl index 22cdda8d2..07090857b 100644 --- a/test/minibatch.jl +++ b/test/minibatch.jl @@ -1,4 +1,5 @@ -using DiffEqFlux, Optimization, OrdinaryDiffEq, OptimizationOptimisers +using DiffEqFlux, Optimization, OrdinaryDiffEq, OptimizationOptimisers, ModelingToolkit, + SciMLSensitivity function newtons_cooling(du, u, p, t) temp = u[1] @@ -61,3 +62,40 @@ using IterTools: ncycle res1 = Optimization.solve(optprob, Optimisers.ADAM(0.05), ncycle(train_loader, numEpochs), callback = callback, maxiters = numEpochs) @test 10res1.objective < l1 + +optfun = OptimizationFunction((θ, p, batch, time_batch) -> loss_adjoint(θ, batch, + time_batch), + Optimization.AutoForwardDiff()) +optprob = OptimizationProblem(optfun, pp) +using IterTools: ncycle +res1 = Optimization.solve(optprob, Optimisers.ADAM(0.05), ncycle(train_loader, numEpochs), + callback = callback, maxiters = numEpochs) +@test 10res1.objective < l1 + +optfun = OptimizationFunction((θ, p, batch, time_batch) -> loss_adjoint(θ, batch, + time_batch), + Optimization.AutoModelingToolkit()) +optprob = OptimizationProblem(optfun, pp) +using IterTools: ncycle +@test_broken res1 = Optimization.solve(optprob, Optimisers.ADAM(0.05), + ncycle(train_loader, numEpochs), + callback = callback, maxiters = numEpochs) +# @test 10res1.objective < l1 + +function loss_grad(res, fullp, _, batch, time_batch) + pred = solve(prob, Tsit5(), p = fullp, saveat = time_batch) + res .= Array(adjoint_sensitivities(pred, Tsit5(); t = time_batch, p = fullp, + dgdu_discrete = (out, u, p, t, i) -> (out .= -2 * + (batch[i] .- + u[1])), + sensealg = InterpolatingAdjoint())[2]') +end + +optfun = OptimizationFunction((θ, p, batch, time_batch) -> loss_adjoint(θ, batch, + time_batch), + grad = loss_grad) +optprob = OptimizationProblem(optfun, pp) +using IterTools: ncycle +res1 = Optimization.solve(optprob, Optimisers.ADAM(0.05), ncycle(train_loader, numEpochs), + callback = callback, maxiters = numEpochs) +@test 10res1.objective < l1