From 738eedf1d85feb00498713a23936c7d8f53d76ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Besan=C3=A7on?= Date: Wed, 20 Sep 2023 13:42:34 +0200 Subject: [PATCH 1/6] added open loop parameterization --- src/linesearch.jl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/linesearch.jl b/src/linesearch.jl index 9edc47353..70c4788ea 100644 --- a/src/linesearch.jl +++ b/src/linesearch.jl @@ -24,14 +24,22 @@ function perform_line_search end build_linesearch_workspace(::LineSearchMethod, x, gradient) = nothing """ -Computes step size: `2/(2 + t)` at iteration `t`. +Computes step size: `l/(l + t)` at iteration `t`, given `l > 0`. + +See: +> Acceleration of Frank-Wolfe Algorithms with Open-Loop Step-Sizes, Wirth, Kerdreux, Pokutta, 2023. + """ -struct Agnostic{T<:Real} <: LineSearchMethod end +struct Agnostic{T<:Real} <: LineSearchMethod + l::Int +end -Agnostic() = Agnostic{Float64}() +Agnostic() = Agnostic{Float64}(2) + +Agnostic{T}() where {T} = Agnostic{T}(2) perform_line_search( - ::Agnostic{<:Rational}, + ls::Agnostic{<:Rational}, t, f, g!, @@ -41,9 +49,10 @@ perform_line_search( gamma_max, workspace, memory_mode::MemoryEmphasis, -) = 2 // (t + 2) +) = ls.l // (t + ls.l) + perform_line_search( - ::Agnostic{T}, + ls::Agnostic{T}, t, f, g!, @@ -53,7 +62,7 @@ perform_line_search( gamma_max, workspace, memory_mode::MemoryEmphasis, -) where {T} = T(2 / (t + 2)) +) where {T} = T(ls.l / (t + ls.l)) Base.print(io::IO, ::Agnostic) = print(io, "Agnostic") From 975f5abe189905e9d6be7990a5e16a8da0361a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Besan=C3=A7on?= Date: Fri, 29 Sep 2023 12:33:01 +0200 Subject: [PATCH 2/6] added test --- src/linesearch.jl | 4 +- test/trajectory_tests/open_loop_parametric.jl | 91 +++++++++++++++++++ 2 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 test/trajectory_tests/open_loop_parametric.jl diff --git a/src/linesearch.jl b/src/linesearch.jl index 70c4788ea..cc52f175e 100644 --- a/src/linesearch.jl +++ b/src/linesearch.jl @@ -26,15 +26,15 @@ build_linesearch_workspace(::LineSearchMethod, x, gradient) = nothing """ Computes step size: `l/(l + t)` at iteration `t`, given `l > 0`. -See: +Using `l ≥ 4` is advised only for strongly convex sets, see: > Acceleration of Frank-Wolfe Algorithms with Open-Loop Step-Sizes, Wirth, Kerdreux, Pokutta, 2023. - """ struct Agnostic{T<:Real} <: LineSearchMethod l::Int end Agnostic() = Agnostic{Float64}(2) +Agnostic(l::Int) = Agnostic{Float64}(l) Agnostic{T}() where {T} = Agnostic{T}(2) diff --git a/test/trajectory_tests/open_loop_parametric.jl b/test/trajectory_tests/open_loop_parametric.jl new file mode 100644 index 000000000..83c209e21 --- /dev/null +++ b/test/trajectory_tests/open_loop_parametric.jl @@ -0,0 +1,91 @@ +using FrankWolfe + +using Test +using LinearAlgebra + +@testset "Open-loop FW on polytope" begin + n = Int(1e2) + k = Int(1e4) + + xp = ones(n) + f(x) = norm(x - xp)^2 + function grad!(storage, x) + @. storage = 2 * (x - xp) + end + + lmo = FrankWolfe.KSparseLMO(40, 1.0) + + x0 = FrankWolfe.compute_extreme_point(lmo, zeros(n)) + + res_2 = FrankWolfe.frank_wolfe( + f, + grad!, + lmo, + copy(x0), + max_iteration=k, + line_search=FrankWolfe.Agnostic(2), + print_iter=k / 10, + epsilon=1e-5, + verbose=true, + trajectory=true, + ) + + res_10 = FrankWolfe.frank_wolfe( + f, + grad!, + lmo, + copy(x0), + max_iteration=k, + line_search=FrankWolfe.Agnostic(10), + print_iter=k / 10, + epsilon=1e-5, + verbose=true, + trajectory=true, + ) + + @test res_2[4] ≤ 0.004799839951985518 + @test res_10[4] ≤ 0.02399919272834694 + + # strongly convex set + xp2 = 10 * ones(n) + diag_term = 100 * rand(n) + covariance_matrix = LinearAlgebra.Diagonal(diag_term) + lmo2 = FrankWolfe.EllipsoidLMO(covariance_matrix) + + f2(x) = norm(x - xp2)^2 + function grad2!(storage, x) + @. storage = 2 * (x - xp2) + end + + x0 = FrankWolfe.compute_extreme_point(lmo2, randn(n)) + + res_2 = FrankWolfe.frank_wolfe( + f2, + grad2!, + lmo2, + copy(x0), + max_iteration=k, + line_search=FrankWolfe.Agnostic(2), + print_iter=k / 10, + epsilon=1e-5, + verbose=true, + trajectory=true, + ) + + res_10 = FrankWolfe.frank_wolfe( + f2, + grad2!, + lmo2, + copy(x0), + max_iteration=k, + line_search=FrankWolfe.Agnostic(10), + print_iter=k / 10, + epsilon=1e-5, + verbose=true, + trajectory=true, + ) + + @test length(res_10[end]) <= 8 + @test length(res_2[end]) <= 71 + +end From b56b8489f013cb344e99b214dd475fdabc6a5a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Besan=C3=A7on?= Date: Fri, 29 Sep 2023 13:02:21 +0200 Subject: [PATCH 3/6] replace diagonal with sparse for 1.6 compat --- test/trajectory_tests/open_loop_parametric.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/trajectory_tests/open_loop_parametric.jl b/test/trajectory_tests/open_loop_parametric.jl index 83c209e21..d6c454771 100644 --- a/test/trajectory_tests/open_loop_parametric.jl +++ b/test/trajectory_tests/open_loop_parametric.jl @@ -49,7 +49,7 @@ using LinearAlgebra # strongly convex set xp2 = 10 * ones(n) diag_term = 100 * rand(n) - covariance_matrix = LinearAlgebra.Diagonal(diag_term) + covariance_matrix = spzeros(n,n) + LinearAlgebra.Diagonal(diag_term) lmo2 = FrankWolfe.EllipsoidLMO(covariance_matrix) f2(x) = norm(x - xp2)^2 From c0b23e3df7314f7740aff3c415279d076c528aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Besan=C3=A7on?= Date: Fri, 29 Sep 2023 14:21:32 +0200 Subject: [PATCH 4/6] import --- test/trajectory_tests/open_loop_parametric.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/trajectory_tests/open_loop_parametric.jl b/test/trajectory_tests/open_loop_parametric.jl index d6c454771..a8c535967 100644 --- a/test/trajectory_tests/open_loop_parametric.jl +++ b/test/trajectory_tests/open_loop_parametric.jl @@ -2,6 +2,7 @@ using FrankWolfe using Test using LinearAlgebra +using SparseArrays @testset "Open-loop FW on polytope" begin n = Int(1e2) From eae75bbcb13a1cf2696cb8a7d9edf52a45fda04f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Besan=C3=A7on?= Date: Fri, 29 Sep 2023 14:48:08 +0200 Subject: [PATCH 5/6] dense matrix --- test/trajectory_tests/open_loop_parametric.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/trajectory_tests/open_loop_parametric.jl b/test/trajectory_tests/open_loop_parametric.jl index a8c535967..558ca051b 100644 --- a/test/trajectory_tests/open_loop_parametric.jl +++ b/test/trajectory_tests/open_loop_parametric.jl @@ -2,7 +2,6 @@ using FrankWolfe using Test using LinearAlgebra -using SparseArrays @testset "Open-loop FW on polytope" begin n = Int(1e2) @@ -50,7 +49,7 @@ using SparseArrays # strongly convex set xp2 = 10 * ones(n) diag_term = 100 * rand(n) - covariance_matrix = spzeros(n,n) + LinearAlgebra.Diagonal(diag_term) + covariance_matrix = zeros(n,n) + LinearAlgebra.Diagonal(diag_term) lmo2 = FrankWolfe.EllipsoidLMO(covariance_matrix) f2(x) = norm(x - xp2)^2 From 50e5ecedab96c30b86a45a642d3f582f1ff14f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathieu=20Besan=C3=A7on?= Date: Fri, 29 Sep 2023 15:22:26 +0200 Subject: [PATCH 6/6] relax lenght --- test/trajectory_tests/open_loop_parametric.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/trajectory_tests/open_loop_parametric.jl b/test/trajectory_tests/open_loop_parametric.jl index 558ca051b..474d98ecb 100644 --- a/test/trajectory_tests/open_loop_parametric.jl +++ b/test/trajectory_tests/open_loop_parametric.jl @@ -86,6 +86,6 @@ using LinearAlgebra ) @test length(res_10[end]) <= 8 - @test length(res_2[end]) <= 71 + @test length(res_2[end]) <= 73 end