diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e0a9ec268..555fe0854 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -66,9 +66,54 @@ jobs: - uses: julia-actions/julia-buildpkg@v1 - if: ${{ matrix.group == 'OptimizationQuadDIRECT' }} run: julia --project -e 'using Pkg; Pkg.Registry.add(RegistrySpec(url = "https://github.com/HolyLab/HolyLabRegistry.git")); Pkg.add("QuadDIRECT")' - - uses: julia-actions/julia-runtest@v1 + - name: ${{ matrix.group }} env: - GROUP: ${{ matrix.group }} + GROUP: ${{ matrix.group }} + shell: julia --color=yes --check-bounds=yes --depwarn=yes {0} + run: | + using Pkg + const GROUP = get(ENV, "GROUP", "Core") + + function dev_subpkg(subpkg) + subpkg_path = "lib/$subpkg" + Pkg.develop(PackageSpec(path = subpkg_path)) + end + + if GROUP == "Core" + Pkg.activate(".") + else + subpkg_path = "lib/${{ matrix.group }}" + Pkg.activate(subpkg_path) + end + + if VERSION < v"1.11" + @info "Preparing env" + if GROUP == "Core" + @info "Testing Core" + dev_subpkg("OptimizationBase") + dev_subpkg("OptimizationLBFGSB") + dev_subpkg("OptimizationMOI") + dev_subpkg("OptimizationOptimJL") + dev_subpkg("OptimizationOptimisers") + elseif GROUP == "OptimizationBase" + dev_subpkg("OptimizationLBFGSB") + dev_subpkg("OptimizationManopt") + elseif GROUP == "OptimizationAuglag" + dev_subpkg("OptimizationOptimisers") + elseif GROUP == "OptimizationMultistartOptimization" + dev_subpkg("OptimizationNLopt") + elseif GROUP == "GPU" || GROUP == "OptimizationPolyalgorithms" + # special case + Pkg.develop([PackageSpec(path = "lib/OptimizationOptimJL"), PackageSpec(path = "lib/OptimizationOptimisers")]) + elseif GROUP == "OptimizationNLPModels" + dev_subpkg("OptimizationMOI") + dev_subpkg("OptimizationOptimJL") + dev_subpkg("OptimizationLBFGSB") + end + end + + @info "Starting tests" + Pkg.test() - uses: julia-actions/julia-processcoverage@v1 with: directories: src,lib/OptimizationBase/src,lib/OptimizationBBO/src,lib/OptimizationCMAEvolutionStrategy/src,lib/OptimizationEvolutionary/src,lib/OptimizationGCMAES/src,lib/OptimizationIpopt/src,lib/OptimizationMadNLP/src,lib/OptimizationManopt/src,lib/OptimizationMOI/src,lib/OptimizationMetaheuristics/src,lib/OptimizationMultistartOptimization/src,lib/OptimizationNLopt/src,lib/OptimizationNOMAD/src,lib/OptimizationOptimJL/src,lib/OptimizationOptimisers/src,lib/OptimizationPolyalgorithms/src,lib/OptimizationQuadDIRECT/src,lib/OptimizationSpeedMapping/src diff --git a/docs/Project.toml b/docs/Project.toml index 21d238294..69af16b62 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -19,8 +19,8 @@ NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6" NLPModelsTest = "7998695d-6960-4d3a-85c4-e1bceb8cd856" NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" -OptimizationBBO = "3e6eede4-6085-4f62-9a71-46d9bc1eb92b" OptimizationBase = "bca83a33-5cc9-4baa-983d-23429ab6bcbb" +OptimizationBBO = "3e6eede4-6085-4f62-9a71-46d9bc1eb92b" OptimizationCMAEvolutionStrategy = "bd407f91-200f-4536-9381-e4ba712f53f8" OptimizationEvolutionary = "cb963754-43f6-435e-8d4b-99009ff27753" OptimizationGCMAES = "6f0a0517-dbc2-4a7a-8a20-99ae7f27e911" @@ -47,6 +47,26 @@ Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" +[sources] +Optimization = {path = ".."} +OptimizationBase = {path = "../lib/OptimizationBase"} +OptimizationBBO = {path = "../lib/OptimizationBBO"} +OptimizationCMAEvolutionStrategy = {path = "../lib/OptimizationCMAEvolutionStrategy"} +OptimizationEvolutionary = {path = "../lib/OptimizationEvolutionary"} +OptimizationGCMAES = {path = "../lib/OptimizationGCMAES"} +OptimizationIpopt = {path = "../lib/OptimizationIpopt"} +OptimizationMOI = {path = "../lib/OptimizationMOI"} +OptimizationManopt = {path = "../lib/OptimizationManopt"} +OptimizationMetaheuristics = {path = "../lib/OptimizationMetaheuristics"} +OptimizationNLPModels = {path = "../lib/OptimizationNLPModels"} +OptimizationNLopt = {path = "../lib/OptimizationNLopt"} +OptimizationNOMAD = {path = "../lib/OptimizationNOMAD"} +OptimizationOptimJL = {path = "../lib/OptimizationOptimJL"} +OptimizationOptimisers = {path = "../lib/OptimizationOptimisers"} +OptimizationPRIMA = {path = "../lib/OptimizationPRIMA"} +OptimizationPolyalgorithms = {path = "../lib/OptimizationPolyalgorithms"} +OptimizationSpeedMapping = {path = "../lib/OptimizationSpeedMapping"} + [compat] ADTypes = "1" AmplNLWriter = "1" @@ -66,9 +86,9 @@ ModelingToolkit = "10.23" NLPModels = "0.21" NLPModelsTest = "0.10" NLopt = "0.6, 1" -Optimization = "4, 5" +Optimization = "5" +OptimizationBase = "4" OptimizationBBO = "0.4" -OptimizationBase = "2, 3" OptimizationCMAEvolutionStrategy = "0.3" OptimizationEvolutionary = "0.4" OptimizationGCMAES = "0.3" @@ -88,7 +108,7 @@ OrdinaryDiffEq = "6" Plots = "1" Random = "1" ReverseDiff = ">= 1.9.0" -SciMLBase = "2.30.0" +SciMLBase = "2.122.1" SciMLSensitivity = "7" SymbolicAnalysis = "0.3" Symbolics = "6" diff --git a/lib/OptimizationAuglag/Project.toml b/lib/OptimizationAuglag/Project.toml index 61769a374..41f8578bb 100644 --- a/lib/OptimizationAuglag/Project.toml +++ b/lib/OptimizationAuglag/Project.toml @@ -14,14 +14,17 @@ MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" OptimizationOptimisers = "42dfb2eb-d2b4-4451-abcd-913932933ac1" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +[sources] +OptimizationOptimisers = {path = "../OptimizationOptimisers"} + [compat] ForwardDiff = "1.0.1" -OptimizationBase = "3" +OptimizationBase = "4" MLUtils = "0.4.8" LinearAlgebra = "1.10" OptimizationOptimisers = "0.3.8" Test = "1.10.0" -SciMLBase = "2.58" +SciMLBase = "2.122.1" julia = "1.10" [targets] diff --git a/lib/OptimizationAuglag/src/OptimizationAuglag.jl b/lib/OptimizationAuglag/src/OptimizationAuglag.jl index 801552ca4..3ce0df17d 100644 --- a/lib/OptimizationAuglag/src/OptimizationAuglag.jl +++ b/lib/OptimizationAuglag/src/OptimizationAuglag.jl @@ -16,12 +16,7 @@ using LinearAlgebra: norm ϵ = 1e-8 end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::AugLag) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::AugLag) = true -end +SciMLBase.has_init(::AugLag) = true SciMLBase.allowsbounds(::AugLag) = true SciMLBase.requiresgradient(::AugLag) = true SciMLBase.allowsconstraints(::AugLag) = true @@ -59,32 +54,7 @@ function __map_optimizer_args(cache::OptimizationBase.OptimizationCache, opt::Au return mapped_args end -function SciMLBase.__solve(cache::OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - AugLag, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: AugLag} maxiters = OptimizationBase._check_and_convert_maxiters(cache.solver_args.maxiters) local x @@ -121,7 +91,8 @@ function SciMLBase.__solve(cache::OptimizationCache{ if cache.callback(opt_state, x...) error("Optimization halted by callback.") end - return x[1] + sum(@. λ * cons_tmp[eq_inds] + ρ / 2 * (cons_tmp[eq_inds] .^ 2)) + 1 / (2 * ρ) * sum((max.(Ref(0.0), μ .+ (ρ .* cons_tmp[ineq_inds]))) .^ 2) + return x[1] + sum(@. λ * cons_tmp[eq_inds] + ρ / 2 * (cons_tmp[eq_inds] .^ 2)) + + 1 / (2 * ρ) * sum((max.(Ref(0.0), μ .+ (ρ .* cons_tmp[ineq_inds]))) .^ 2) end prev_eqcons = zero(λ) diff --git a/lib/OptimizationBBO/Project.toml b/lib/OptimizationBBO/Project.toml index 987bdb4e3..6068f90a9 100644 --- a/lib/OptimizationBBO/Project.toml +++ b/lib/OptimizationBBO/Project.toml @@ -14,8 +14,8 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] julia = "1.10" BlackBoxOptim = "0.6" -OptimizationBase = "3" -SciMLBase = "2.58" +OptimizationBase = "4" +SciMLBase = "2.122.1" Reexport = "1.2" [targets] diff --git a/lib/OptimizationBBO/src/OptimizationBBO.jl b/lib/OptimizationBBO/src/OptimizationBBO.jl index cb62b3324..59e507f26 100644 --- a/lib/OptimizationBBO/src/OptimizationBBO.jl +++ b/lib/OptimizationBBO/src/OptimizationBBO.jl @@ -1,21 +1,16 @@ module OptimizationBBO using Reexport -import OptimizationBase -import OptimizationBase: SciMLBase -import BlackBoxOptim -import SciMLBase: MultiObjectiveOptimizationFunction +using OptimizationBase +using SciMLBase +using BlackBoxOptim: BlackBoxOptim abstract type BBO end SciMLBase.requiresbounds(::BBO) = true SciMLBase.allowsbounds(::BBO) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::BBO) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::BBO) = true -end + +SciMLBase.has_init(opt::BBO) = true for j in string.(BlackBoxOptim.SingleObjectiveMethodNames) eval(Meta.parse("Base.@kwdef struct BBO_" * j * " <: BBO method=:" * j * " end")) @@ -36,19 +31,19 @@ function decompose_trace(opt::BlackBoxOptim.OptRunController, progress) if iszero(max_time) # we stop at either convergence or max_steps n_steps = BlackBoxOptim.num_steps(opt) - Base.@logmsg(Base.LogLevel(-1), msg, progress=n_steps/maxiters, + Base.@logmsg(Base.LogLevel(-1), msg, progress=n_steps / maxiters, _id=:OptimizationBBO) else # we stop at either convergence or max_time elapsed = BlackBoxOptim.elapsed_time(opt) - Base.@logmsg(Base.LogLevel(-1), msg, progress=elapsed/max_time, + Base.@logmsg(Base.LogLevel(-1), msg, progress=elapsed / max_time, _id=:OptimizationBBO) end end return BlackBoxOptim.best_candidate(opt) end -function __map_optimizer_args(prob::OptimizationBase.OptimizationCache, opt::BBO; +function __map_optimizer_args(prob::OptimizationCache, opt::BBO; callback = nothing, maxiters::Union{Number, Nothing} = nothing, maxtime::Union{Number, Nothing} = nothing, @@ -96,32 +91,7 @@ function map_objective(obj::BlackBoxOptim.IndexedTupleFitness) obj.orig end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - BBO, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: BBO} function _cb(trace) if cache.callback === OptimizationBase.DEFAULT_CALLBACK cb_call = false diff --git a/lib/OptimizationBase/Project.toml b/lib/OptimizationBase/Project.toml index 4576abbf6..bbc5ef39b 100644 --- a/lib/OptimizationBase/Project.toml +++ b/lib/OptimizationBase/Project.toml @@ -55,7 +55,7 @@ ModelingToolkit = "10.23" PDMats = "0.11" Reexport = "1.2" ReverseDiff = "1.14" -SciMLBase = "2.122" +SciMLBase = "2.122.1" SparseConnectivityTracer = "0.6, 1" SparseMatrixColorings = "0.4" SymbolicAnalysis = "0.3" diff --git a/lib/OptimizationBase/ext/OptimizationEnzymeExt.jl b/lib/OptimizationBase/ext/OptimizationEnzymeExt.jl index 5774c360b..7f2307a32 100644 --- a/lib/OptimizationBase/ext/OptimizationEnzymeExt.jl +++ b/lib/OptimizationBase/ext/OptimizationEnzymeExt.jl @@ -199,12 +199,17 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x, if hv == true && f.hv === nothing function hv!(H, θ, v, p = p) - x = Duplicated(θ, v) - dx = Enzyme.make_zero(x) - H .= Enzyme.autodiff( - fmode, hv_f2_alloc, Const(rmode), Duplicated(x,dx), - Const(f.f), Const(p) - )[1].dval + dθ = zero(θ) + Enzyme.make_zero!(H) + Enzyme.autodiff( + fmode, + inner_grad, + Const(rmode), + Duplicated(θ, v), + Duplicated(dθ, H), + Const(f.f), + Const(p) + ) end elseif hv == true hv! = (H, θ, v, p = p) -> f.hv(H, θ, v, p) @@ -553,13 +558,20 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x end if hv == true && f.hv === nothing + H = zero(x) function hv!(θ, v, p = p) - x = Duplicated(θ, v) - dx = Enzyme.make_zero(x) - return Enzyme.autodiff( - fmode, hv_f2_alloc, DuplicatedNoNeed, Const(rmode), Duplicated(x, dx), - Const(_f), Const(f.f), Const(p) - )[1].dval + dθ = zero(θ) + Enzyme.make_zero!(H) + Enzyme.autodiff( + fmode, + inner_grad, + Const(rmode), + Duplicated(θ, v), + Duplicated(dθ, H), + Const(f.f), + Const(p) + ) + return H end elseif hv == true hv! = (θ, v, p = p) -> f.hv(θ, v, p) diff --git a/lib/OptimizationBase/ext/OptimizationZygoteExt.jl b/lib/OptimizationBase/ext/OptimizationZygoteExt.jl index 1e26909af..2e2de093a 100644 --- a/lib/OptimizationBase/ext/OptimizationZygoteExt.jl +++ b/lib/OptimizationBase/ext/OptimizationZygoteExt.jl @@ -186,20 +186,54 @@ function OptimizationBase.instantiate_function( conshess_sparsity = f.cons_hess_prototype conshess_colors = f.cons_hess_colorvec - if cons !== nothing && cons_h == true && f.cons_h === nothing + + # Prepare constraint Hessian preparations if needed by lag_h or cons_h + if cons !== nothing && f.cons_h === nothing && (cons_h == true || lag_h == true) prep_cons_hess = [prepare_hessian( cons_oop, soadtype, x, Constant(i), strict = Val(false)) for i in 1:num_cons] + else + prep_cons_hess = nothing + end + + # Generate cons_h! functions + if cons !== nothing && f.cons_h === nothing && prep_cons_hess !== nothing + # Standard cons_h! that returns array of matrices + if cons_h == true + cons_h! = function (H, θ) + for i in 1:num_cons + hessian!(cons_oop, H[i], prep_cons_hess[i], soadtype, θ, Constant(i)) + end + end + else + cons_h! = nothing + end + + # Weighted sum dispatch for cons_h! (always created if prep_cons_hess exists) + # This is used by lag_h! when σ=0 + cons_h_weighted! = function (H::AbstractMatrix, θ, λ) + # Compute weighted sum: H = Σᵢ λᵢ∇²cᵢ + H .= zero(eltype(H)) + + # Create a single temporary matrix to reuse for all constraints + Hi = similar(H) - function cons_h!(H, θ) for i in 1:num_cons - hessian!(cons_oop, H[i], prep_cons_hess[i], soadtype, θ, Constant(i)) + if λ[i] != zero(eltype(λ)) + # Compute constraint's Hessian into temporary matrix + hessian!(cons_oop, Hi, prep_cons_hess[i], soadtype, θ, Constant(i)) + # Add weighted Hessian to result using in-place operation + # H += λ[i] * Hi + @. H += λ[i] * Hi + end end end elseif cons !== nothing && cons_h == true cons_h! = (res, θ) -> f.cons_h(res, θ, p) + cons_h_weighted! = nothing else cons_h! = nothing + cons_h_weighted! = nothing end lag_hess_prototype = f.lag_hess_prototype @@ -212,8 +246,8 @@ function OptimizationBase.instantiate_function( function lag_h!(H::AbstractMatrix, θ, σ, λ) if σ == zero(eltype(θ)) - cons_h!(H, θ) - H *= λ + # When σ=0, use the weighted sum function + cons_h_weighted!(H, θ, λ) else hessian!(lagrangian, H, lag_extras, soadtype, θ, Constant(σ), Constant(λ), Constant(p)) @@ -512,8 +546,8 @@ function OptimizationBase.instantiate_function( function lag_h!(H::AbstractMatrix, θ, σ, λ) if σ == zero(eltype(θ)) - cons_h!(H, θ) - H *= λ + # When σ=0, use the weighted sum function + cons_h_weighted!(H, θ, λ) else hessian!(lagrangian, H, lag_extras, soadtype, θ, Constant(σ), Constant(λ), Constant(p)) diff --git a/lib/OptimizationBase/src/OptimizationDIExt.jl b/lib/OptimizationBase/src/OptimizationDIExt.jl index 017506a03..9d3ec006a 100644 --- a/lib/OptimizationBase/src/OptimizationDIExt.jl +++ b/lib/OptimizationBase/src/OptimizationDIExt.jl @@ -177,19 +177,53 @@ function instantiate_function( conshess_sparsity = f.cons_hess_prototype conshess_colors = f.cons_hess_colorvec - if f.cons !== nothing && f.cons_h === nothing && cons_h == true + + # Prepare constraint Hessian preparations if needed by lag_h or cons_h + if f.cons !== nothing && f.cons_h === nothing && (cons_h == true || lag_h == true) prep_cons_hess = [prepare_hessian(cons_oop, soadtype, x, Constant(i)) for i in 1:num_cons] + else + prep_cons_hess = nothing + end + + # Generate cons_h! functions + if f.cons !== nothing && f.cons_h === nothing && prep_cons_hess !== nothing + # Standard cons_h! that returns array of matrices + if cons_h == true + cons_h! = function (H, θ) + for i in 1:num_cons + hessian!(cons_oop, H[i], prep_cons_hess[i], soadtype, θ, Constant(i)) + end + end + else + cons_h! = nothing + end + + # Weighted sum dispatch for cons_h! (always created if prep_cons_hess exists) + # This is used by lag_h! when σ=0 + cons_h_weighted! = function (H::AbstractMatrix, θ, λ) + # Compute weighted sum: H = Σᵢ λᵢ∇²cᵢ + H .= zero(eltype(H)) + + # Create a single temporary matrix to reuse for all constraints + Hi = similar(H) - function cons_h!(H, θ) for i in 1:num_cons - hessian!(cons_oop, H[i], prep_cons_hess[i], soadtype, θ, Constant(i)) + if λ[i] != zero(eltype(λ)) + # Compute constraint's Hessian into temporary matrix + hessian!(cons_oop, Hi, prep_cons_hess[i], soadtype, θ, Constant(i)) + # Add weighted Hessian to result using in-place operation + # H += λ[i] * Hi + @. H += λ[i] * Hi + end end end elseif cons_h == true && f.cons !== nothing cons_h! = (res, θ) -> f.cons_h(res, θ, p) + cons_h_weighted! = nothing else cons_h! = nothing + cons_h_weighted! = nothing end lag_hess_prototype = f.lag_hess_prototype @@ -202,8 +236,8 @@ function instantiate_function( function lag_h!(H::AbstractMatrix, θ, σ, λ) if σ == zero(eltype(θ)) - cons_h!(H, θ) - H *= λ + # When σ=0, use the weighted sum function + cons_h_weighted!(H, θ, λ) else hessian!(lagrangian, H, lag_prep, soadtype, θ, Constant(σ), Constant(λ), Constant(p)) diff --git a/lib/OptimizationBase/test/Project.toml b/lib/OptimizationBase/test/Project.toml index 25549349b..54f26bee6 100644 --- a/lib/OptimizationBase/test/Project.toml +++ b/lib/OptimizationBase/test/Project.toml @@ -1,4 +1,5 @@ [deps] +ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" @@ -15,12 +16,13 @@ Manifolds = "1cead3c2-87b3-11e9-0ccd-23c62b72b94e" ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" Optim = "429524aa-4258-5aef-a3af-852621145aeb" Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" -Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" +OptimizationLBFGSB = "22f7324a-a79d-40f2-bebe-3af60c77bd15" OptimizationManopt = "e57b7fff-7ee7-4550-b4f0-90e9476e9fb6" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" +SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5" SymbolicAnalysis = "4297ee4d-0239-47d8-ba5d-195ecdf594fe" @@ -29,17 +31,20 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" +[sources] +OptimizationLBFGSB = {path = "../../OptimizationLBFGSB"} +OptimizationManopt = {path = "../../OptimizationManopt"} + [compat] Aqua = "0.8" ComponentArrays = ">= 0.13.9" Enzyme = "0.13" IterTools = ">= 1.3.0" Lux = "1.12" -Manifolds = "0.9" +Manifolds = "0.10" Optim = ">= 1.4.1" Optimisers = ">= 0.2.5" -Optimization = "4" -OptimizationManopt = "0.0.4" -SparseConnectivityTracer = "0.6" +OptimizationLBFGSB = "1.1" +OptimizationManopt = "1.1" SymbolicAnalysis = "0.3.0" SafeTestsets = ">= 0.0.1" diff --git a/lib/OptimizationBase/test/adtests.jl b/lib/OptimizationBase/test/adtests.jl index 1a9483deb..791f6057c 100644 --- a/lib/OptimizationBase/test/adtests.jl +++ b/lib/OptimizationBase/test/adtests.jl @@ -1,5 +1,5 @@ using OptimizationBase, Test, DifferentiationInterface, SparseArrays, Symbolics -using ForwardDiff, Zygote, ReverseDiff, FiniteDiff, Tracker +using ADTypes, ForwardDiff, Zygote, ReverseDiff, FiniteDiff, Tracker using ModelingToolkit, Enzyme, Random x0 = zeros(2) @@ -27,9 +27,9 @@ g!(G1, x0) h!(H1, x0) cons = (res, x, p) -> (res[1] = x[1]^2 + x[2]^2; return nothing) -optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoModelingToolkit(), cons = cons) +optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoSymbolics(), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoModelingToolkit(), + OptimizationBase.AutoSymbolics(), nothing, 1, g = true, h = true, cons_j = true, cons_h = true) optprob.grad(G2, x0) @test G1 == G2 @@ -51,10 +51,10 @@ function con2_c(res, x, p) return nothing end optf = OptimizationFunction(rosenbrock, - OptimizationBase.AutoModelingToolkit(), + OptimizationBase.AutoSymbolics(), cons = con2_c) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoModelingToolkit(), + OptimizationBase.AutoSymbolics(), nothing, 2, g = true, h = true, cons_j = true, cons_h = true) optprob.grad(G2, x0) @test G1 == G2 @@ -196,9 +196,9 @@ optprob.cons_h(H3, x0) H2 = Array{Float64}(undef, 2, 2) optf = OptimizationFunction( - rosenbrock, OptimizationBase.AutoReverseDiff(true), cons = cons) + rosenbrock, OptimizationBase.AutoReverseDiff(; compile = true), cons = cons) optprob = OptimizationBase.instantiate_function( - optf, x0, OptimizationBase.AutoReverseDiff(true), + optf, x0, OptimizationBase.AutoReverseDiff(; compile = true), nothing, 1, g = true, h = true, hv = true, cons_j = true, cons_h = true, cons_vjp = true, cons_jvp = true, lag_h = true) @@ -402,9 +402,9 @@ end H2 = Array{Float64}(undef, 2, 2) optf = OptimizationFunction( - rosenbrock, OptimizationBase.AutoReverseDiff(true), cons = con2_c) + rosenbrock, OptimizationBase.AutoReverseDiff(; compile = true), cons = con2_c) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoReverseDiff(true), + OptimizationBase.AutoReverseDiff(; compile = true), nothing, 2, g = true, h = true, hv = true, cons_j = true, cons_h = true, cons_vjp = true, cons_jvp = true, lag_h = true) @@ -442,7 +442,7 @@ end optf = OptimizationFunction( rosenbrock, OptimizationBase.AutoForwardDiff(), cons = con2_c) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoReverseDiff(compile = true), + OptimizationBase.AutoReverseDiff(; compile = true), nothing, 2, g = true, h = true, hv = true, cons_j = true, cons_h = true, cons_vjp = true, cons_jvp = true, lag_h = true) @@ -582,12 +582,12 @@ end x0 = [0.5, 0.5, 0.5] # Create OptimizationFunction - optf = OptimizationFunction(sparse_objective, OptimizationBase.AutoSparseForwardDiff(), + optf = OptimizationFunction(sparse_objective, AutoSparse(OptimizationBase.AutoForwardDiff()), cons = sparse_constraints) # Instantiate the optimization problem optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), nothing, 2, g = true, h = true, cons_j = true, cons_h = true, lag_h = true) # Test gradient G = zeros(3) @@ -631,12 +631,12 @@ end @test lag_H ≈ lag_H_expected @test nnz(lag_H) == 5 - optf = OptimizationFunction(sparse_objective, OptimizationBase.AutoSparseReverseDiff(), + optf = OptimizationFunction(sparse_objective, AutoSparse(OptimizationBase.AutoReverseDiff()), cons = sparse_constraints) # Instantiate the optimization problem optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), nothing, 2, g = true, h = true, cons_j = true, cons_h = true, lag_h = true) # Test gradient G = zeros(3) @@ -681,12 +681,12 @@ end @test nnz(lag_H) == 5 optf = OptimizationFunction( - sparse_objective, OptimizationBase.AutoSparseReverseDiff(true), + sparse_objective, AutoSparse(OptimizationBase.AutoReverseDiff(; compile = true)), cons = sparse_constraints) # Instantiate the optimization problem optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), nothing, 2, g = true, h = true, cons_j = true, cons_h = true, lag_h = true) # Test gradient G = zeros(3) @@ -730,12 +730,12 @@ end @test lag_H ≈ lag_H_expected @test nnz(lag_H) == 5 - optf = OptimizationFunction(sparse_objective, OptimizationBase.AutoSparseFiniteDiff(), + optf = OptimizationFunction(sparse_objective, AutoSparse(OptimizationBase.AutoFiniteDiff()), cons = sparse_constraints) # Instantiate the optimization problem optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), nothing, 2, g = true, h = true, cons_j = true, cons_h = true, lag_h = true) # Test gradient G = zeros(3) @@ -959,10 +959,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoReverseDiff(true), + OptimizationBase.AutoReverseDiff(; compile = true), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoReverseDiff(true), + OptimizationBase.AutoReverseDiff(; compile = true), nothing, 1, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 @@ -976,10 +976,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2, x[2] * sin(x[1]) - x[1]] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoReverseDiff(true), + OptimizationBase.AutoReverseDiff(; compile = true), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoReverseDiff(true), + OptimizationBase.AutoReverseDiff(; compile = true), nothing, 2, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 @@ -990,10 +990,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), nothing, 1, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 @@ -1007,10 +1007,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2, x[2] * sin(x[1]) - x[1]] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), nothing, 2, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 @@ -1021,10 +1021,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoSparseFiniteDiff(), + AutoSparse(OptimizationBase.AutoFiniteDiff()), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseFiniteDiff(), + AutoSparse(OptimizationBase.AutoFiniteDiff()), nothing, 1, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0)≈G1 rtol=1e-4 @@ -1038,10 +1038,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2, x[2] * sin(x[1]) - x[1]] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoSparseFiniteDiff(), + AutoSparse(OptimizationBase.AutoFiniteDiff()), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseForwardDiff(), + AutoSparse(OptimizationBase.AutoForwardDiff()), nothing, 2, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 @@ -1052,10 +1052,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoSparseReverseDiff(), + AutoSparse(OptimizationBase.AutoReverseDiff()), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseReverseDiff(), + AutoSparse(OptimizationBase.AutoReverseDiff()), nothing, 1, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 @@ -1069,10 +1069,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2, x[2] * sin(x[1]) - x[1]] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoSparseReverseDiff(), + AutoSparse(OptimizationBase.AutoReverseDiff()), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseReverseDiff(), + AutoSparse(OptimizationBase.AutoReverseDiff()), nothing, 2, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 @@ -1083,10 +1083,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoSparseReverseDiff(true), + AutoSparse(OptimizationBase.AutoReverseDiff(; compile = true)), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseReverseDiff(true), + AutoSparse(OptimizationBase.AutoReverseDiff(; compile = true)), nothing, 1, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 @@ -1099,10 +1099,10 @@ end cons = (x, p) -> [x[1]^2 + x[2]^2, x[2] * sin(x[1]) - x[1]] optf = OptimizationFunction{false}(rosenbrock, - OptimizationBase.AutoSparseReverseDiff(true), + AutoSparse(OptimizationBase.AutoReverseDiff(; compile = true)), cons = cons) optprob = OptimizationBase.instantiate_function(optf, x0, - OptimizationBase.AutoSparseReverseDiff(true), + AutoSparse(OptimizationBase.AutoReverseDiff(; compile = true)), nothing, 2, g = true, h = true, cons_j = true, cons_h = true) @test optprob.grad(x0) == G1 diff --git a/lib/OptimizationBase/test/cvxtest.jl b/lib/OptimizationBase/test/cvxtest.jl index 701f26e2d..5ad17ee52 100644 --- a/lib/OptimizationBase/test/cvxtest.jl +++ b/lib/OptimizationBase/test/cvxtest.jl @@ -1,4 +1,4 @@ -using OptimizationBase, OptimizationBase, ForwardDiff, SymbolicAnalysis, LinearAlgebra, +using OptimizationBase, ForwardDiff, SymbolicAnalysis, LinearAlgebra, Manifolds, OptimizationManopt, OptimizationLBFGSB function f(x, p = nothing) @@ -47,6 +47,6 @@ prob = OptimizationProblem(optf, data2[1]; manifold = M, structural_analysis = t opt = OptimizationManopt.GradientDescentOptimizer() @time sol = solve(prob, opt, maxiters = 100) -@test sol.minimum < 1e-1 +@test sol.objective < 1e-1 @test sol.cache.analysis_results.objective.curvature == SymbolicAnalysis.UnknownCurvature @test sol.cache.analysis_results.objective.gcurvature == SymbolicAnalysis.GConvex diff --git a/lib/OptimizationBase/test/lag_h_sigma_zero_test.jl b/lib/OptimizationBase/test/lag_h_sigma_zero_test.jl new file mode 100644 index 000000000..61e957587 --- /dev/null +++ b/lib/OptimizationBase/test/lag_h_sigma_zero_test.jl @@ -0,0 +1,187 @@ +using OptimizationBase, Test, DifferentiationInterface +using ADTypes, ForwardDiff, ReverseDiff, Zygote + +@testset "Lagrangian Hessian with σ = 0" begin + # Test that lag_h works correctly when σ = 0 + # This is a regression test for the bug where lag_h! would fail when + # cons_h was not generated but lag_h needed to compute constraint Hessians + + x0 = [0.5, 0.5] + rosenbrock(x, p = nothing) = (1 - x[1])^2 + 100 * (x[2] - x[1]^2)^2 + + # Single constraint + cons1 = (res, x, p) -> (res[1] = x[1]^2 + x[2]^2; return nothing) + + # Two constraints + cons2 = (res, x, p) -> begin + res[1] = x[1]^2 + x[2]^2 + res[2] = x[2] * sin(x[1]) - x[1] + return nothing + end + + @testset "Single constraint with σ = 0" begin + # Create optimization function WITHOUT cons_h but WITH lag_h + optf = OptimizationFunction( + rosenbrock, + SecondOrder(AutoForwardDiff(), AutoForwardDiff()), + cons = cons1 + ) + + optprob = OptimizationBase.instantiate_function( + optf, x0, + SecondOrder(AutoForwardDiff(), AutoForwardDiff()), + nothing, 1, + g = true, h = true, + cons_j = true, + cons_h = false, # Don't generate cons_h! + lag_h = true # Only generate lag_h! + ) + + # Test with σ = 0 - this should compute only constraint Hessians + H_lag = Array{Float64}(undef, 2, 2) + λ = [2.0] # arbitrary multiplier + σ = 0.0 + + # This should work and compute H = λ[1] * ∇²c₁ + optprob.lag_h(H_lag, x0, σ, λ) + + # Expected: constraint Hessian is [2 0; 0 2] for c(x) = x₁² + x₂² + # Scaled by λ[1] = 2.0 gives [4 0; 0 4] + @test H_lag ≈ [4.0 0.0; 0.0 4.0] + + # Test with σ ≠ 0 for comparison + σ = 1.0 + optprob.lag_h(H_lag, x0, σ, λ) + + # Expected objective Hessian at x0 = [0.5, 0.5] + H_obj = zeros(2, 2) + H_obj[1, 1] = 2.0 - 400.0 * x0[2] + 1200.0 * x0[1]^2 + H_obj[1, 2] = -400.0 * x0[1] + H_obj[2, 1] = -400.0 * x0[1] + H_obj[2, 2] = 200.0 + + # Should be σ * H_obj + λ[1] * H_cons + @test H_lag ≈ H_obj + [4.0 0.0; 0.0 4.0] + end + + @testset "Two constraints with σ = 0" begin + optf = OptimizationFunction( + rosenbrock, + SecondOrder(AutoForwardDiff(), AutoForwardDiff()), + cons = cons2 + ) + + optprob = OptimizationBase.instantiate_function( + optf, x0, + SecondOrder(AutoForwardDiff(), AutoForwardDiff()), + nothing, 2, + g = true, h = true, + cons_j = true, + cons_h = false, # Don't generate cons_h! + lag_h = true # Only generate lag_h! + ) + + # Test with σ = 0 + H_lag = Array{Float64}(undef, 2, 2) + λ = [1.5, -0.5] + σ = 0.0 + + # This should compute H = λ[1] * ∇²c₁ + λ[2] * ∇²c₂ + optprob.lag_h(H_lag, x0, σ, λ) + + # Expected constraint Hessians: + # ∇²c₁ = [2 0; 0 2] for c₁(x) = x₁² + x₂² + # ∇²c₂ = [-sin(x₁)*x₂ cos(x₁); cos(x₁) 0] for c₂(x) = x₂*sin(x₁) - x₁ + # At x0 = [0.5, 0.5]: + H_c2 = zeros(2, 2) + H_c2[1, 1] = -sin(x0[1]) * x0[2] + H_c2[1, 2] = cos(x0[1]) + H_c2[2, 1] = cos(x0[1]) + H_c2[2, 2] = 0.0 + + expected = λ[1] * [2.0 0.0; 0.0 2.0] + λ[2] * H_c2 + @test H_lag ≈ expected rtol=1e-6 + end + + @testset "Different AD backends with σ = 0" begin + # Test with AutoReverseDiff + optf = OptimizationFunction( + rosenbrock, + SecondOrder(AutoForwardDiff(), AutoReverseDiff()), + cons = cons1 + ) + + optprob = OptimizationBase.instantiate_function( + optf, x0, + SecondOrder(AutoForwardDiff(), AutoReverseDiff()), + nothing, 1, + g = true, h = true, + cons_j = true, + cons_h = false, + lag_h = true + ) + + H_lag = Array{Float64}(undef, 2, 2) + λ = [3.0] + σ = 0.0 + + optprob.lag_h(H_lag, x0, σ, λ) + @test H_lag ≈ [6.0 0.0; 0.0 6.0] # 3.0 * [2 0; 0 2] + + # Test with AutoZygote + optf = OptimizationFunction( + rosenbrock, + SecondOrder(AutoForwardDiff(), AutoZygote()), + cons = cons1 + ) + + optprob = OptimizationBase.instantiate_function( + optf, x0, + SecondOrder(AutoForwardDiff(), AutoZygote()), + nothing, 1, + g = true, h = true, + cons_j = true, + cons_h = false, + lag_h = true + ) + + H_lag = Array{Float64}(undef, 2, 2) + λ = [0.5] + σ = 0.0 + + optprob.lag_h(H_lag, x0, σ, λ) + @test H_lag ≈ [1.0 0.0; 0.0 1.0] # 0.5 * [2 0; 0 2] + end + + @testset "Edge cases" begin + optf = OptimizationFunction( + rosenbrock, + SecondOrder(AutoForwardDiff(), AutoForwardDiff()), + cons = cons2 + ) + + optprob = OptimizationBase.instantiate_function( + optf, x0, + SecondOrder(AutoForwardDiff(), AutoForwardDiff()), + nothing, 2, + g = true, h = true, + cons_j = true, + cons_h = false, + lag_h = true + ) + + H_lag = Array{Float64}(undef, 2, 2) + + # Test with all λ = 0 and σ = 0 (should give zero matrix) + λ = [0.0, 0.0] + σ = 0.0 + optprob.lag_h(H_lag, x0, σ, λ) + @test all(H_lag .≈ 0.0) + + # Test with some λ = 0 (should skip those constraints) + λ = [2.0, 0.0] + σ = 0.0 + optprob.lag_h(H_lag, x0, σ, λ) + @test H_lag ≈ [4.0 0.0; 0.0 4.0] # Only first constraint contributes + end +end \ No newline at end of file diff --git a/lib/OptimizationBase/test/runtests.jl b/lib/OptimizationBase/test/runtests.jl index 70e378b3c..2d49b5985 100644 --- a/lib/OptimizationBase/test/runtests.jl +++ b/lib/OptimizationBase/test/runtests.jl @@ -6,4 +6,5 @@ using Test include("cvxtest.jl") include("matrixvalued.jl") include("solver_missing_error_messages.jl") + include("lag_h_sigma_zero_test.jl") end diff --git a/lib/OptimizationBase/test/solver_missing_error_messages.jl b/lib/OptimizationBase/test/solver_missing_error_messages.jl index 5f603cd83..ca9e7a192 100644 --- a/lib/OptimizationBase/test/solver_missing_error_messages.jl +++ b/lib/OptimizationBase/test/solver_missing_error_messages.jl @@ -1,4 +1,5 @@ using OptimizationBase, Test +using SciMLBase: NoAD import OptimizationBase: allowscallback, requiresbounds, requiresconstraints diff --git a/lib/OptimizationCMAEvolutionStrategy/Project.toml b/lib/OptimizationCMAEvolutionStrategy/Project.toml index 00a55ddef..51b8aec26 100644 --- a/lib/OptimizationCMAEvolutionStrategy/Project.toml +++ b/lib/OptimizationCMAEvolutionStrategy/Project.toml @@ -14,8 +14,8 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] CMAEvolutionStrategy = "0.2" julia = "1.10" -OptimizationBase = "3" -SciMLBase = "2.58" +OptimizationBase = "4" +SciMLBase = "2.122.1" Reexport = "1.2" [targets] diff --git a/lib/OptimizationCMAEvolutionStrategy/src/OptimizationCMAEvolutionStrategy.jl b/lib/OptimizationCMAEvolutionStrategy/src/OptimizationCMAEvolutionStrategy.jl index 90b65b975..9d372569b 100644 --- a/lib/OptimizationCMAEvolutionStrategy/src/OptimizationCMAEvolutionStrategy.jl +++ b/lib/OptimizationCMAEvolutionStrategy/src/OptimizationCMAEvolutionStrategy.jl @@ -10,18 +10,14 @@ export CMAEvolutionStrategyOpt struct CMAEvolutionStrategyOpt end SciMLBase.allowsbounds(::CMAEvolutionStrategyOpt) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::CMAEvolutionStrategyOpt) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::CMAEvolutionStrategyOpt) = true -end +SciMLBase.has_init(opt::CMAEvolutionStrategyOpt) = true SciMLBase.requiresgradient(::CMAEvolutionStrategyOpt) = false SciMLBase.requireshessian(::CMAEvolutionStrategyOpt) = false SciMLBase.requiresconsjac(::CMAEvolutionStrategyOpt) = false SciMLBase.requiresconshess(::CMAEvolutionStrategyOpt) = false -function __map_optimizer_args(prob::OptimizationBase.OptimizationCache, opt::CMAEvolutionStrategyOpt; +function __map_optimizer_args( + prob::OptimizationBase.OptimizationCache, opt::CMAEvolutionStrategyOpt; callback = nothing, maxiters::Union{Number, Nothing} = nothing, maxtime::Union{Number, Nothing} = nothing, @@ -53,32 +49,7 @@ function __map_optimizer_args(prob::OptimizationBase.OptimizationCache, opt::CMA return mapped_args end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - CMAEvolutionStrategyOpt, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: CMAEvolutionStrategyOpt} local x, cur, state function _cb(opt, y, fvals, perm) diff --git a/lib/OptimizationEvolutionary/Project.toml b/lib/OptimizationEvolutionary/Project.toml index fbb65c25c..8fc09985d 100644 --- a/lib/OptimizationEvolutionary/Project.toml +++ b/lib/OptimizationEvolutionary/Project.toml @@ -14,9 +14,9 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "4" Evolutionary = "0.11" -SciMLBase = "2.58" +SciMLBase = "2.122.1" Reexport = "1.2" [targets] diff --git a/lib/OptimizationEvolutionary/src/OptimizationEvolutionary.jl b/lib/OptimizationEvolutionary/src/OptimizationEvolutionary.jl index 2d9c4c305..6e4395e91 100644 --- a/lib/OptimizationEvolutionary/src/OptimizationEvolutionary.jl +++ b/lib/OptimizationEvolutionary/src/OptimizationEvolutionary.jl @@ -6,12 +6,7 @@ using SciMLBase SciMLBase.allowsbounds(opt::Evolutionary.AbstractOptimizer) = true SciMLBase.allowsconstraints(opt::Evolutionary.AbstractOptimizer) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Evolutionary.AbstractOptimizer) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Evolutionary.AbstractOptimizer) = true -end +SciMLBase.has_init(opt::Evolutionary.AbstractOptimizer) = true SciMLBase.requiresgradient(opt::Evolutionary.AbstractOptimizer) = false SciMLBase.requiresgradient(opt::Evolutionary.NSGA2) = false SciMLBase.requireshessian(opt::Evolutionary.AbstractOptimizer) = false @@ -76,32 +71,8 @@ function __map_optimizer_args(cache::OptimizationBase.OptimizationCache, return Evolutionary.Options(; mapped_args...) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - Evolutionary.AbstractOptimizer, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: + Evolutionary.AbstractOptimizer} local x, cur, state function _cb(trace) diff --git a/lib/OptimizationGCMAES/Project.toml b/lib/OptimizationGCMAES/Project.toml index a48a714f6..70a7fda38 100644 --- a/lib/OptimizationGCMAES/Project.toml +++ b/lib/OptimizationGCMAES/Project.toml @@ -14,8 +14,8 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] julia = "1.10" -OptimizationBase = "3" -SciMLBase = "2.58" +OptimizationBase = "4" +SciMLBase = "2.122.1" Reexport = "1.2" GCMAES = "0.1" diff --git a/lib/OptimizationGCMAES/src/OptimizationGCMAES.jl b/lib/OptimizationGCMAES/src/OptimizationGCMAES.jl index 471198447..84b0602be 100644 --- a/lib/OptimizationGCMAES/src/OptimizationGCMAES.jl +++ b/lib/OptimizationGCMAES/src/OptimizationGCMAES.jl @@ -11,12 +11,7 @@ struct GCMAESOpt end SciMLBase.requiresbounds(::GCMAESOpt) = true SciMLBase.allowsbounds(::GCMAESOpt) = true SciMLBase.allowscallback(::GCMAESOpt) = false -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::GCMAESOpt) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::GCMAESOpt) = true -end +SciMLBase.has_init(opt::GCMAESOpt) = true SciMLBase.requiresgradient(::GCMAESOpt) = true SciMLBase.requireshessian(::GCMAESOpt) = false SciMLBase.requiresconsjac(::GCMAESOpt) = false @@ -61,32 +56,7 @@ function SciMLBase.__init(prob::SciMLBase.OptimizationProblem, kwargs...) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - GCMAESOpt, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: GCMAESOpt} local x local G = similar(cache.u0) diff --git a/lib/OptimizationIpopt/Project.toml b/lib/OptimizationIpopt/Project.toml index ddf1cc31f..64c193efd 100644 --- a/lib/OptimizationIpopt/Project.toml +++ b/lib/OptimizationIpopt/Project.toml @@ -14,8 +14,8 @@ SymbolicIndexingInterface = "2efcf032-c050-4f8e-a9bb-153293bab1f5" Ipopt = "1.10.3" LinearAlgebra = "1.10.0" ModelingToolkit = "10.23" -OptimizationBase = "3" -SciMLBase = "2.90.0" +OptimizationBase = "3, 4" +SciMLBase = "2.122.1" SparseArrays = "1.10.0" SymbolicIndexingInterface = "0.3.40" Zygote = "0.7" diff --git a/lib/OptimizationIpopt/src/OptimizationIpopt.jl b/lib/OptimizationIpopt/src/OptimizationIpopt.jl index 269a127ff..7c551405e 100644 --- a/lib/OptimizationIpopt/src/OptimizationIpopt.jl +++ b/lib/OptimizationIpopt/src/OptimizationIpopt.jl @@ -166,15 +166,8 @@ https://coin-or.github.io/Ipopt/OPTIONS.html additional_options::Dict{String, Any} = Dict{String, Any}() end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - function SciMLBase.supports_opt_cache_interface(alg::IpoptOptimizer) - true - end -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - function OptimizationBase.supports_opt_cache_interface(alg::IpoptOptimizer) - true - end +function SciMLBase.has_init(alg::IpoptOptimizer) + true end function SciMLBase.requiresgradient(opt::IpoptOptimizer) diff --git a/lib/OptimizationLBFGSB/Project.toml b/lib/OptimizationLBFGSB/Project.toml index c74000abc..971ad9166 100644 --- a/lib/OptimizationLBFGSB/Project.toml +++ b/lib/OptimizationLBFGSB/Project.toml @@ -19,8 +19,8 @@ DocStringExtensions = "0.9.5" ForwardDiff = "1.0.1" LBFGSB = "0.4.1" MLUtils = "0.4.8" -OptimizationBase = "3" -SciMLBase = "2.58" +OptimizationBase = "4" +SciMLBase = "2.122.1" Zygote = "0.7.10" julia = "1.10" diff --git a/lib/OptimizationLBFGSB/src/OptimizationLBFGSB.jl b/lib/OptimizationLBFGSB/src/OptimizationLBFGSB.jl index c39ac837b..4d8a26771 100644 --- a/lib/OptimizationLBFGSB/src/OptimizationLBFGSB.jl +++ b/lib/OptimizationLBFGSB/src/OptimizationLBFGSB.jl @@ -31,12 +31,7 @@ References ϵ = 1e-8 end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::LBFGSB) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::LBFGSB) = true -end +SciMLBase.has_init(::LBFGSB) = true SciMLBase.allowsbounds(::LBFGSB) = true SciMLBase.requiresgradient(::LBFGSB) = true SciMLBase.allowsconstraints(::LBFGSB) = true @@ -78,32 +73,7 @@ function __map_optimizer_args(cache::OptimizationBase.OptimizationCache, opt::LB return mapped_args end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - LBFGSB, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: LBFGSB} maxiters = OptimizationBase._check_and_convert_maxiters(cache.solver_args.maxiters) local x @@ -142,7 +112,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ if cache.callback(opt_state, x...) error("Optimization halted by callback.") end - return x[1] + sum(@. λ * cons_tmp[eq_inds] + ρ / 2 * (cons_tmp[eq_inds] .^ 2)) + 1 / (2 * ρ) * sum((max.(Ref(0.0), μ .+ (ρ .* cons_tmp[ineq_inds]))) .^ 2) + return x[1] + sum(@. λ * cons_tmp[eq_inds] + ρ / 2 * (cons_tmp[eq_inds] .^ 2)) + + 1 / (2 * ρ) * sum((max.(Ref(0.0), μ .+ (ρ .* cons_tmp[ineq_inds]))) .^ 2) end prev_eqcons = zero(λ) diff --git a/lib/OptimizationMOI/Project.toml b/lib/OptimizationMOI/Project.toml index 730ee43bd..604ad0380 100644 --- a/lib/OptimizationMOI/Project.toml +++ b/lib/OptimizationMOI/Project.toml @@ -27,7 +27,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [compat] HiGHS = "1" -OptimizationBase = "3" +OptimizationBase = "3, 4" Test = "1.6" Symbolics = "6" AmplNLWriter = "1" @@ -36,7 +36,7 @@ Ipopt_jll = "300.1400" Juniper = "0.9" Ipopt = "1" NLopt = "1" -SciMLBase = "2.58" +SciMLBase = "2.122.1" SparseArrays = "1.6" ModelingToolkit = "10.23" SymbolicIndexingInterface = "0.3" diff --git a/lib/OptimizationMOI/src/OptimizationMOI.jl b/lib/OptimizationMOI/src/OptimizationMOI.jl index ee3c2c6af..0266d6368 100644 --- a/lib/OptimizationMOI/src/OptimizationMOI.jl +++ b/lib/OptimizationMOI/src/OptimizationMOI.jl @@ -283,17 +283,9 @@ end include("nlp.jl") include("moi.jl") -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - function SciMLBase.supports_opt_cache_interface(alg::Union{MOI.AbstractOptimizer, - MOI.OptimizerWithAttributes}) - true - end -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - function OptimizationBase.supports_opt_cache_interface(alg::Union{MOI.AbstractOptimizer, - MOI.OptimizerWithAttributes}) - true - end +function SciMLBase.has_init(alg::Union{MOI.AbstractOptimizer, + MOI.OptimizerWithAttributes}) + true end function SciMLBase.__init(prob::OptimizationProblem, diff --git a/lib/OptimizationMadNLP/Project.toml b/lib/OptimizationMadNLP/Project.toml index f5edc9109..ffd604c0b 100644 --- a/lib/OptimizationMadNLP/Project.toml +++ b/lib/OptimizationMadNLP/Project.toml @@ -1,6 +1,6 @@ name = "OptimizationMadNLP" uuid = "5d9c809f-c847-4062-9fba-1793bbfef577" -version = "1.0.0" +version = "1.1.0" authors = ["Sebastian Micluța-Câmpeanu and contributors"] [deps] @@ -19,8 +19,8 @@ LinearAlgebra = "1.10.0" MadNLP = "0.8.12" ModelingToolkit = "10.23" NLPModels = "0.21.5" -OptimizationBase = "3.3.1" -SciMLBase = "2.90.0" +OptimizationBase = "4" +SciMLBase = "2.122.1" SparseArrays = "1.10.0" SymbolicIndexingInterface = "0.3.40" Zygote = "0.7" diff --git a/lib/OptimizationMadNLP/src/OptimizationMadNLP.jl b/lib/OptimizationMadNLP/src/OptimizationMadNLP.jl index b538b11f1..b71c1178b 100644 --- a/lib/OptimizationMadNLP/src/OptimizationMadNLP.jl +++ b/lib/OptimizationMadNLP/src/OptimizationMadNLP.jl @@ -169,7 +169,8 @@ function NLPModels.hess_coord!( if !isnothing(nlp.cache.f.cons_h) && !isempty(y) # Add weighted constraint Hessians - cons_hessians = [similar(nlp.hess_buffer, eltype(nlp.hess_buffer)) for _ in 1:length(y)] + cons_hessians = [similar(nlp.hess_buffer, eltype(nlp.hess_buffer)) + for _ in 1:length(y)] nlp.cache.f.cons_h(cons_hessians, x) for (λ, H_cons) in zip(y, cons_hessians) nlp.hess_buffer .+= λ .* H_cons @@ -193,7 +194,8 @@ function NLPModels.hess_coord!( return H end -function NLPModels.jtprod!(nlp::NLPModelsAdaptor, x::AbstractVector, v::AbstractVector, Jtv::AbstractVector) +function NLPModels.jtprod!( + nlp::NLPModelsAdaptor, x::AbstractVector, v::AbstractVector, Jtv::AbstractVector) # Compute J^T * v using the AD-provided VJP (Vector-Jacobian Product) if !isnothing(nlp.cache.f.cons_vjp) && !isempty(Jtv) nlp.cache.f.cons_vjp(Jtv, x, v) @@ -201,7 +203,8 @@ function NLPModels.jtprod!(nlp::NLPModelsAdaptor, x::AbstractVector, v::Abstract return Jtv end -function NLPModels.jprod!(nlp::NLPModelsAdaptor, x::AbstractVector, v::AbstractVector, Jv::AbstractVector) +function NLPModels.jprod!( + nlp::NLPModelsAdaptor, x::AbstractVector, v::AbstractVector, Jv::AbstractVector) # Compute J * v using the AD-provided JVP (Jacobian-Vector Product) if !isnothing(nlp.cache.f.cons_jvp) && !isempty(Jv) nlp.cache.f.cons_jvp(Jv, x, v) @@ -228,21 +231,21 @@ end hessian_constant::Bool = false hessian_approximation::Type = MadNLP.ExactHessian - # Barrier - mu_init::T = 1e-1 + # Linear solver configuration + linear_solver::Union{Nothing, Type} = nothing # e.g., MumpsSolver, LapackCPUSolver, UmfpackSolver + + kkt_system::Union{Nothing, Type} = nothing # e.g. DenseKKTSystem + + mu_init::T = 0.1 # Quasi-Newton options (used when hessian_approximation is CompactLBFGS, BFGS, or DampedBFGS) - max_history::Int = 6 # Number of past gradients to store for L-BFGS - init_strategy::MadNLP.BFGSInitStrategy = MadNLP.SCALAR1 # How to initialize Hessian - init_value::T = 1.0 # Initial scaling value - sigma_min::T = 1e-8 # Minimum allowed σ (safeguard) - sigma_max::T = 1e+8 # Maximum allowed σ (safeguard) + quasi_newton_options::Union{Nothing, MadNLP.QuasiNewtonOptions} = nothing # Additional MadNLP options additional_options::Dict{Symbol, Any} = Dict{Symbol, Any}() end -SciMLBase.supports_opt_cache_interface(opt::MadNLPOptimizer) = true +SciMLBase.has_init(opt::MadNLPOptimizer) = true function SciMLBase.requiresgradient(opt::MadNLPOptimizer) true @@ -262,6 +265,9 @@ end function SciMLBase.requireslagh(opt::MadNLPOptimizer) opt.hessian_approximation === MadNLP.ExactHessian end +function SciMLBase.requiresconshess(opt::MadNLPOptimizer) + opt.hessian_approximation === MadNLP.ExactHessian +end function SciMLBase.allowsconsvjp(opt::MadNLPOptimizer) true end @@ -269,11 +275,6 @@ function SciMLBase.allowsconsjvp(opt::MadNLPOptimizer) true end -function SciMLBase.__init(prob::SciMLBase.OptimizationProblem, opt::MadNLPOptimizer; - kwargs...) - return OptimizationCache(prob, opt; kwargs...) -end - function map_madnlp_status(status::MadNLP.Status) if status in [ MadNLP.SOLVE_SUCCEEDED, @@ -367,8 +368,6 @@ function __map_optimizer_args(cache, minimize = cache.sense !== MaxSense # Default to minimization when sense is nothing or MinSense ) - nlp = NLPModelsAdaptor(cache, meta, NLPModels.Counters()) - if verbose isa Bool print_level = verbose ? MadNLP.INFO : MadNLP.WARN else @@ -380,62 +379,51 @@ function __map_optimizer_args(cache, max_iter = isnothing(maxiters) ? 3000 : maxiters max_wall_time = isnothing(maxtime) ? 1e6 : maxtime - # Create QuasiNewtonOptions if using quasi-Newton methods - quasi_newton_options = MadNLP.QuasiNewtonOptions{T}(; - opt.init_strategy, - opt.max_history, - opt.init_value, - opt.sigma_min, - opt.sigma_max - ) + # Build final options dictionary + options = Dict{Symbol, Any}(opt.additional_options) - MadNLP.MadNLPSolver(nlp; - opt.additional_options..., - print_level, tol, max_iter, max_wall_time, - opt.rethrow_error, - opt.disable_garbage_collector, - opt.blas_num_threads, - opt.output_file, - opt.file_print_level, - opt.acceptable_tol, - opt.acceptable_iter, - opt.jacobian_constant, - opt.hessian_constant, - opt.hessian_approximation, - opt.mu_init, - quasi_newton_options = quasi_newton_options, - ) + options[:mu_init] = opt.mu_init + + # Add quasi_newton_options if provided, otherwise create default + if !isnothing(opt.quasi_newton_options) + options[:quasi_newton_options] = opt.quasi_newton_options + else + # Create default quasi-Newton options + options[:quasi_newton_options] = MadNLP.QuasiNewtonOptions{T}() + end + + # Add linear_solver if provided + if !isnothing(opt.linear_solver) + options[:linear_solver] = opt.linear_solver + end + + if !isnothing(opt.kkt_system) + options[:kkt_system] = opt.kkt_system + end + + options[:rethrow_error] = opt.rethrow_error + options[:disable_garbage_collector] = opt.disable_garbage_collector + options[:blas_num_threads] = opt.blas_num_threads + options[:output_file] = opt.output_file + options[:file_print_level] = opt.file_print_level + options[:acceptable_tol] = opt.acceptable_tol + options[:acceptable_iter] = opt.acceptable_iter + options[:jacobian_constant] = opt.jacobian_constant + options[:hessian_constant] = opt.hessian_constant + options[:hessian_approximation] = opt.hessian_approximation + options[:print_level] = print_level + options[:tol] = tol + options[:max_iter] = max_iter + options[:max_wall_time] = max_wall_time + + meta, options end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: MadNLPOptimizer, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: MadNLPOptimizer} maxiters = OptimizationBase._check_and_convert_maxiters(cache.solver_args.maxiters) maxtime = OptimizationBase._check_and_convert_maxtime(cache.solver_args.maxtime) - solver = __map_optimizer_args(cache, + meta, options = __map_optimizer_args(cache, cache.opt; abstol = cache.solver_args.abstol, reltol = cache.solver_args.reltol, @@ -446,6 +434,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ callback = cache.callback ) + nlp = NLPModelsAdaptor(cache, meta, NLPModels.Counters()) + solver = MadNLP.MadNLPSolver(nlp; options...) results = MadNLP.solve!(solver) stats = OptimizationBase.OptimizationStats(; time = results.counters.total_time, diff --git a/lib/OptimizationMadNLP/test/runtests.jl b/lib/OptimizationMadNLP/test/runtests.jl index 801336773..e4af90f1e 100644 --- a/lib/OptimizationMadNLP/test/runtests.jl +++ b/lib/OptimizationMadNLP/test/runtests.jl @@ -4,7 +4,7 @@ using MadNLP using Test import Zygote, ForwardDiff, ReverseDiff using SparseArrays -using DifferentiationInterface +using DifferentiationInterface: SecondOrder using Random @testset "rosenbrock" begin @@ -50,8 +50,7 @@ end opts = [ MadNLPOptimizer(), - MadNLPOptimizer(additional_options = Dict{Symbol, Any}( - :linear_solver => LapackCPUSolver,)) + MadNLPOptimizer(linear_solver = LapackCPUSolver) ] for opt in opts @@ -59,7 +58,7 @@ end @test SciMLBase.successful_retcode(sol) # compare against Ipopt results - @test sol ≈ [0.7071678163428006, 0.7070457460302945] rtol=1e-4 + @test sol≈[0.7071678163428006, 0.7070457460302945] rtol=1e-4 end end @@ -72,7 +71,7 @@ end @testset "$ad" for ad in [ SecondOrder(AutoZygote(), AutoZygote()), SecondOrder(AutoForwardDiff(), AutoZygote()), - SecondOrder(AutoForwardDiff(), AutoReverseDiff()), + SecondOrder(AutoForwardDiff(), AutoReverseDiff()) ] optf = OptimizationFunction(objective, ad) prob = OptimizationProblem(optf, x0, p) @@ -104,7 +103,7 @@ end @testset "$ad" for ad in [ AutoSparse(SecondOrder(AutoForwardDiff(), AutoZygote())), AutoSparse(SecondOrder(AutoForwardDiff(), AutoForwardDiff())), - AutoSparse(SecondOrder(AutoForwardDiff(), AutoReverseDiff())), + AutoSparse(SecondOrder(AutoForwardDiff(), AutoReverseDiff())) ] optfunc = OptimizationFunction(objective, ad, cons = constraints) prob = OptimizationProblem(optfunc, x0; sense = OptimizationBase.MinSense, @@ -130,7 +129,7 @@ end @testset "$ad" for ad in [ SecondOrder(AutoForwardDiff(), AutoZygote()), SecondOrder(AutoForwardDiff(), AutoForwardDiff()), - SecondOrder(AutoForwardDiff(), AutoReverseDiff()), + SecondOrder(AutoForwardDiff(), AutoReverseDiff()) ] optfunc = OptimizationFunction(objective, ad, cons = constraints) prob = OptimizationProblem(optfunc, x0; sense = OptimizationBase.MinSense, @@ -139,9 +138,9 @@ end lcons = [25.0, 40.0], ucons = [Inf, 40.0]) - cache = init(prob, MadNLPOptimizer(additional_options = Dict{Symbol, Any}( - :kkt_system => MadNLP.DenseKKTSystem, - :linear_solver => LapackCPUSolver,))) + cache = init(prob, + MadNLPOptimizer(kkt_system = MadNLP.DenseKKTSystem, + linear_solver = LapackCPUSolver)) sol = OptimizationBase.solve!(cache) @@ -162,7 +161,7 @@ end # x1*x3 >= 1 function objective_sparse(x, p) - return x[1]^2 + 2*x[2]^2 + x[3]^2 + x[1]*x[3] + x[2]*x[4] + return x[1]^2 + 2 * x[2]^2 + x[3]^2 + x[1] * x[3] + x[2] * x[4] end function cons_sparse(res, x, p) @@ -220,7 +219,7 @@ end # Check constraints cons_vals = zeros(2) cons_sparse(cons_vals, sol.u, p) - @test isapprox(cons_vals[1], 4.0, atol=1e-6) # Sum constraint + @test isapprox(cons_vals[1], 4.0, atol = 1e-6) # Sum constraint @test cons_vals[2] >= 1.0 - 1e-6 # Product constraint end @@ -260,8 +259,8 @@ end # Test passing MadNLP options via additional_options opt = MadNLPOptimizer( + linear_solver = MadNLP.UmfpackSolver, additional_options = Dict{Symbol, Any}( - :linear_solver => MadNLP.UmfpackSolver, :max_iter => 200, :tol => 1e-7 ) @@ -272,9 +271,9 @@ end # Test with different options opt2 = MadNLPOptimizer( additional_options = Dict{Symbol, Any}( - :inertia_correction_method => MadNLP.InertiaFree, - :fixed_variable_treatment => MadNLP.RelaxBound - ) + :inertia_correction_method => MadNLP.InertiaFree, + :fixed_variable_treatment => MadNLP.RelaxBound + ) ) sol2 = solve(prob, opt2) @test SciMLBase.successful_retcode(sol2) @@ -287,7 +286,7 @@ end # Test that abstol overrides default tolerance sol1 = solve(prob, MadNLPOptimizer(); abstol = 1e-12) @test SciMLBase.successful_retcode(sol1) - @test sol1.u ≈ [1.0, 1.0] atol=1e-10 + @test sol1.u≈[1.0, 1.0] atol=1e-10 # Test that maxiters limits iterations sol2 = solve(prob, MadNLPOptimizer(); maxiters = 5) @@ -301,7 +300,7 @@ end end end - @testset "Priority: struct < additional_options < solve args" begin + @testset "Priority: struct < additional_options < common solve args" begin optfunc = OptimizationFunction(rosenbrock, ad) prob = OptimizationProblem(optfunc, x0, p) @@ -315,8 +314,8 @@ end ) sol = solve(prob, opt; - maxiters = 5, # Should override additional_options[:max_iter] - abstol = 1e-10) # Should override additional_options[:tol] + maxiters = 5, # Should override additional_options[:max_iter] + abstol = 1e-10) # Should override additional_options[:tol] @test sol.stats.iterations <= 5 @test sol.retcode == SciMLBase.ReturnCode.MaxIters @@ -330,7 +329,7 @@ end # Extended Rosenbrock function (n-dimensional) function extended_rosenbrock(x, p) n = length(x) - sum(100 * (x[2i] - x[2i-1]^2)^2 + (1 - x[2i-1])^2 for i in 1:div(n, 2)) + sum(100 * (x[2i] - x[2i - 1]^2)^2 + (1 - x[2i - 1])^2 for i in 1:div(n, 2)) end n = 10 # Problem dimension @@ -359,7 +358,7 @@ end hessian_approximation = variant ) - sol = solve(prob, opt; maxiters=100, verbose = false) + sol = solve(prob, opt; maxiters = 100, verbose = false) @test SciMLBase.successful_retcode(sol) @test all(isapprox.(sol.u, 1.0, atol = 1e-6)) # Solution should be all ones @@ -374,10 +373,10 @@ end opt = MadNLPOptimizer( hessian_approximation = MadNLP.CompactLBFGS, - max_history = memory_size + quasi_newton_options = MadNLP.QuasiNewtonOptions(max_history = memory_size) ) - sol = solve(prob, opt; maxiters=100, verbose = false) + sol = solve(prob, opt; maxiters = 100, verbose = false) @test SciMLBase.successful_retcode(sol) @test all(isapprox.(sol.u, 1.0, atol = 1e-6)) @@ -394,13 +393,13 @@ end # vars = [x1...xn, y1...yn, z1...zn] np = div(length(vars), 3) x = @view vars[1:np] - y = @view vars[np+1:2*np] - z = @view vars[2*np+1:3*np] + y = @view vars[(np + 1):(2 * np)] + z = @view vars[(2 * np + 1):(3 * np)] # Sum of 1/r_ij for all electron pairs energy = 0.0 - for i in 1:np-1 - for j in i+1:np + for i in 1:(np - 1) + for j in (i + 1):np dist_sq = (x[i] - x[j])^2 + (y[i] - y[j])^2 + (z[i] - z[j])^2 energy += 1.0 / sqrt(dist_sq) end @@ -412,8 +411,8 @@ end # Each electron must lie on the unit sphere np = div(length(vars), 3) x = @view vars[1:np] - y = @view vars[np+1:2*np] - z = @view vars[2*np+1:3*np] + y = @view vars[(np + 1):(2 * np)] + z = @view vars[(2 * np + 1):(3 * np)] for i in 1:np res[i] = x[i]^2 + y[i]^2 + z[i]^2 - 1.0 @@ -426,19 +425,18 @@ end theta = 2π .* rand(np) phi = π .* rand(np) - x0 = zeros(3*np) + x0 = zeros(3 * np) # x coordinates x0[1:np] = cos.(theta) .* sin.(phi) # y coordinates - x0[np+1:2*np] = sin.(theta) .* sin.(phi) + x0[(np + 1):(2 * np)] = sin.(theta) .* sin.(phi) # z coordinates - x0[2*np+1:3*np] = cos.(phi) + x0[(2 * np + 1):(3 * np)] = cos.(phi) return x0 end - @testset "N=$np electrons with $approx" for - np in [6, 8, 10], + @testset "N=$np electrons with $approx" for np in [6, 8, 10], approx in [MadNLP.CompactLBFGS, MadNLP.ExactHessian] x0 = init_electrons_on_sphere(np) @@ -466,11 +464,11 @@ end ) opt = MadNLPOptimizer( - additional_options = Dict{Symbol, Any}(:linear_solver=>LapackCPUSolver), + linear_solver = LapackCPUSolver, hessian_approximation = approx ) - sol = solve(prob, opt; abstol=1e-7, maxiters=200, verbose = false) + sol = solve(prob, opt; abstol = 1e-7, maxiters = 200, verbose = false) @test SciMLBase.successful_retcode(sol) @@ -494,30 +492,32 @@ end # Verify minimum distance between electrons x = sol.u[1:np] - y = sol.u[np+1:2*np] - z = sol.u[2*np+1:3*np] + y = sol.u[(np + 1):(2 * np)] + z = sol.u[(2 * np + 1):(3 * np)] min_dist = Inf - for i in 1:np-1 - for j in i+1:np - dist = sqrt((x[i]-x[j])^2 + (y[i]-y[j])^2 + (z[i]-z[j])^2) + for i in 1:(np - 1) + for j in (i + 1):np + dist = sqrt((x[i] - x[j])^2 + (y[i] - y[j])^2 + (z[i] - z[j])^2) min_dist = min(min_dist, dist) end end @test min_dist > 0.5 # Electrons should be well-separated end - @testset "Performance comparison: LBFGS vs Exact Hessian" begin + @testset "LBFGS vs Exact Hessian" begin # Test with moderate size to show LBFGS efficiency np = 12 # Icosahedron configuration x0 = init_electrons_on_sphere(np) - results = Dict() + results = [] - for (name, approx, ad) in [ - ("CompactLBFGS", MadNLP.CompactLBFGS, AutoForwardDiff()) - ("ExactHessian", MadNLP.ExactHessian, SecondOrder(AutoForwardDiff(), AutoForwardDiff())) - ] + for (name, approx, ad) in [("CompactLBFGS", MadNLP.CompactLBFGS, + AutoForwardDiff()) + ("ExactHessian", + MadNLP.ExactHessian, + SecondOrder( + AutoForwardDiff(), AutoForwardDiff()))] optfunc = OptimizationFunction( coulomb_potential, ad, cons = unit_sphere_constraints @@ -533,31 +533,61 @@ end ) sol = solve(prob, opt; abstol = 1e-6, maxiters = 300, verbose = false) - results[name] = ( + push!(results, name => ( objective = sol.objective, iterations = sol.stats.iterations, success = SciMLBase.successful_retcode(sol) - ) + )) end # All methods should converge - @test all(r.success for r in values(results)) + @test all(r[2].success for r in values(results)) # All should find similar objective values (icosahedron energy) # Reference: https://en.wikipedia.org/wiki/Thomson_problem - objectives = [r.objective for r in values(results)] - @test all(abs.(objectives .- 49.165253058) .< 0.1) + objectives = [r[2].objective for r in values(results)] + @testset "$(results[i][1])" for (i, o) in enumerate(objectives) + @test o ≈ 49.165253058 rtol=1e-2 + end # LBFGS methods typically need more iterations but less cost per iteration - @test results["CompactLBFGS"].iterations > 0 - @test results["ExactHessian"].iterations > 0 + @test results[1][2].iterations > results[1][2].iterations broken=true + end + + @testset "Exact Hessian and sparse KKT that hits σ == 0 in lag_h" begin + np = 12 + # x0 = init_electrons_on_sphere(np) + x0 = [-0.10518691576929745, 0.051771801773795686, -0.9003045175547166, 0.23213937667116594, -0.02874270928423086, -0.652270178114126, -0.5918025628300999, 0.2511988210810674, -0.016535391659614228, 0.5949770074227214, -0.4492781383448046, -0.29581324890382626, -0.8989309486672202, 0.10678505987872657, -0.4351575519144031, -0.9589360279618278, 0.02680807390998832, 0.40670966862867725, 0.08594698464206306, -0.9646178134393677, -0.004187961953999249, -0.09107912492873807, -0.6973104772728601, 0.40182616259664583, 0.4252750430946946, -0.9929333469713824, 0.009469988512801456, 0.1629509253594941, -0.9992272933803594, -0.6396333795127627, -0.8014878928958706, 0.08007263129768477, -0.9998545103150432, 0.7985655600140281, -0.5584865734204564, -0.8666200187082093] + + approx = MadNLP.ExactHessian + ad = SecondOrder(AutoForwardDiff(), AutoForwardDiff()) + + optfunc = OptimizationFunction( + coulomb_potential, ad, + cons = unit_sphere_constraints + ) + + prob = OptimizationProblem(optfunc, x0; + lcons = zeros(np), + ucons = zeros(np) + ) + + opt = MadNLPOptimizer( + hessian_approximation = approx, + kkt_system = MadNLP.SparseKKTSystem + ) + + sol = solve(prob, opt; abstol = 1e-6, maxiters = 300, verbose = false) + + @test SciMLBase.successful_retcode(sol) + @test sol.objective ≈ 49.165253058 rtol=1e-2 end end @testset "LBFGS with damped update" begin # Test the damped BFGS update option function simple_quadratic(x, p) - return sum(x.^2) + return sum(x .^ 2) end x0 = randn(5) @@ -568,12 +598,11 @@ end opt = MadNLPOptimizer( hessian_approximation = MadNLP.DampedBFGS, # Use damped BFGS variant - additional_options = Dict{Symbol,Any}( - :linear_solver => MadNLP.LapackCPUSolver, - :kkt_system=>MadNLP.DenseKKTSystem) + linear_solver = MadNLP.LapackCPUSolver, + kkt_system = MadNLP.DenseKKTSystem ) - sol = solve(prob, opt; maxiters=50, verbose = false) + sol = solve(prob, opt; maxiters = 50, verbose = false) @test SciMLBase.successful_retcode(sol) @test all(abs.(sol.u) .< 1e-6) # Solution should be at origin diff --git a/lib/OptimizationManopt/Project.toml b/lib/OptimizationManopt/Project.toml index 97375c1f9..f75b5edbd 100644 --- a/lib/OptimizationManopt/Project.toml +++ b/lib/OptimizationManopt/Project.toml @@ -13,6 +13,7 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" [extras] +DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" RipQP = "1e40b3f8-35eb-4cd8-8edd-3e515bb9de08" QuadraticModels = "f468eda6-eac5-11e8-05a5-ff9e497bcd19" @@ -25,14 +26,15 @@ Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" [compat] julia = "1.10" +DifferentiationInterface = "0.7" Manopt = "0.5" -OptimizationBase = "3" +OptimizationBase = "4" LinearAlgebra = "1.10" ManifoldsBase = "1" ManifoldDiff = "0.4" Manifolds = "0.10" Reexport = "1.2" -SciMLBase = "2.58" +SciMLBase = "2.122.1" [targets] -test = ["Enzyme", "ForwardDiff", "FiniteDiff", "QuadraticModels", "Random", "ReverseDiff", "RipQP", "Test", "Zygote"] +test = ["DifferentiationInterface", "Enzyme", "ForwardDiff", "FiniteDiff", "QuadraticModels", "Random", "ReverseDiff", "RipQP", "Test", "Zygote"] diff --git a/lib/OptimizationManopt/src/OptimizationManopt.jl b/lib/OptimizationManopt/src/OptimizationManopt.jl index ecae49c4e..8bee9abac 100644 --- a/lib/OptimizationManopt/src/OptimizationManopt.jl +++ b/lib/OptimizationManopt/src/OptimizationManopt.jl @@ -12,12 +12,7 @@ internal state. """ abstract type AbstractManoptOptimizer end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::AbstractManoptOptimizer) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::AbstractManoptOptimizer) = true -end +SciMLBase.has_init(opt::AbstractManoptOptimizer) = true function __map_optimizer_args!(cache::OptimizationBase.OptimizationCache, opt::AbstractManoptOptimizer; @@ -70,7 +65,7 @@ function call_manopt_optimizer( loss, gradF, x0; - hessF=nothing, # ignore that keyword for this solver + hessF = nothing, # ignore that keyword for this solver kwargs...) opts = Manopt.gradient_descent(M, loss, @@ -90,7 +85,7 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, opt::NelderMea loss, gradF, x0; - hessF=nothing, # ignore that keyword for this solver + hessF = nothing, # ignore that keyword for this solver kwargs...) opts = NelderMead(M, loss; return_state = true, kwargs...) minimizer = Manopt.get_solver_result(opts) @@ -105,7 +100,7 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, loss, gradF, x0; - hessF=nothing, # ignore that keyword for this solver + hessF = nothing, # ignore that keyword for this solver kwargs...) opts = Manopt.conjugate_gradient_descent(M, loss, @@ -126,7 +121,7 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, loss, gradF, x0; - hessF=nothing, # ignore that keyword for this solver + hessF = nothing, # ignore that keyword for this solver population_size::Int = 100, kwargs...) swarm = [x0, [rand(M) for _ in 1:(population_size - 1)]...] @@ -144,7 +139,7 @@ function call_manopt_optimizer(M::Manopt.AbstractManifold, loss, gradF, x0; - hessF=nothing, # ignore that keyword for this solver + hessF = nothing, # ignore that keyword for this solver kwargs... ) opts = quasi_Newton(M, loss, gradF, x0; return_state = true, kwargs...) @@ -159,7 +154,7 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, loss, gradF, x0; - hessF=nothing, # ignore that keyword for this solver + hessF = nothing, # ignore that keyword for this solver kwargs...) opt = cma_es(M, loss, x0; return_state = true, kwargs...) minimizer = Manopt.get_solver_result(opt) @@ -173,7 +168,7 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, loss, gradF, x0; - hessF=nothing, # ignore that keyword for this solver + hessF = nothing, # ignore that keyword for this solver kwargs...) opt = convex_bundle_method(M, loss, gradF, x0; return_state = true, kwargs...) minimizer = Manopt.get_solver_result(opt) @@ -189,11 +184,12 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, x0; hessF = nothing, kwargs...) - opt = if isnothing(hessF) - adaptive_regularization_with_cubics(M, loss, gradF, x0; return_state = true, kwargs...) + adaptive_regularization_with_cubics( + M, loss, gradF, x0; return_state = true, kwargs...) else - adaptive_regularization_with_cubics(M, loss, gradF, hessF, x0; return_state = true, kwargs...) + adaptive_regularization_with_cubics( + M, loss, gradF, hessF, x0; return_state = true, kwargs...) end minimizer = Manopt.get_solver_result(opt) return (; minimizer = minimizer, minimum = loss(M, minimizer), options = opt) @@ -224,7 +220,7 @@ function call_manopt_optimizer(M::ManifoldsBase.AbstractManifold, loss, gradF, x0; - hessF=nothing, # ignore that keyword for this solver + hessF = nothing, # ignore that keyword for this solver kwargs...) opt = Frank_Wolfe_method(M, loss, gradF, x0; return_state = true, kwargs...) minimizer = Manopt.get_solver_result(opt) @@ -240,7 +236,7 @@ function SciMLBase.requiresgradient(opt::Union{ end function SciMLBase.requireshessian(opt::Union{ AdaptiveRegularizationCubicOptimizer, TrustRegionsOptimizer}) - false + true end function build_loss(f::OptimizationFunction, prob, cb) @@ -286,31 +282,7 @@ function build_hessF(f::OptimizationFunction{true}) return h end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: AbstractManoptOptimizer, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: AbstractManoptOptimizer} local x, cur, state manifold = cache.manifold diff --git a/lib/OptimizationManopt/test/runtests.jl b/lib/OptimizationManopt/test/runtests.jl index 9e22be677..a979ef5e0 100644 --- a/lib/OptimizationManopt/test/runtests.jl +++ b/lib/OptimizationManopt/test/runtests.jl @@ -2,6 +2,7 @@ using OptimizationManopt using OptimizationBase using Manifolds using ForwardDiff, Zygote, Enzyme, FiniteDiff, ReverseDiff +using DifferentiationInterface: SecondOrder using Manopt, RipQP, QuadraticModels using Test using SciMLBase @@ -40,12 +41,12 @@ R2 = Euclidean(2) prob_forwarddiff = OptimizationProblem( optprob_forwarddiff, x0, p; manifold = R2, stepsize = stepsize) sol = OptimizationBase.solve(prob_forwarddiff, opt) - @test sol.minimum < 0.2 + @test sol.objective < 0.2 optprob_grad = OptimizationFunction(rosenbrock; grad = rosenbrock_grad!) prob_grad = OptimizationProblem(optprob_grad, x0, p; manifold = R2, stepsize = stepsize) sol = OptimizationBase.solve(prob_grad, opt) - @test sol.minimum < 0.2 + @test sol.objective < 0.2 end @testset "Nelder-Mead" begin @@ -58,7 +59,7 @@ R2 = Euclidean(2) prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = OptimizationBase.solve(prob, opt) - @test sol.minimum < 0.7 + @test sol.objective < 0.7 end @testset "Conjugate gradient descent" begin @@ -72,7 +73,7 @@ R2 = Euclidean(2) prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = OptimizationBase.solve(prob, opt, stepsize = stepsize) - @test sol.minimum < 0.5 + @test sol.objective < 0.5 end @testset "Quasi Newton" begin @@ -89,7 +90,7 @@ R2 = Euclidean(2) prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = OptimizationBase.solve(prob, opt, callback = callback, maxiters = 30) - @test sol.minimum < 1e-14 + @test sol.objective < 1e-14 end @testset "Particle swarm" begin @@ -102,7 +103,7 @@ R2 = Euclidean(2) prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = OptimizationBase.solve(prob, opt) - @test sol.minimum < 0.1 + @test sol.objective < 0.1 end @testset "CMA-ES" begin @@ -115,7 +116,7 @@ R2 = Euclidean(2) prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = OptimizationBase.solve(prob, opt) - @test sol.minimum < 0.1 + @test sol.objective < 0.1 end @testset "ConvexBundle" begin @@ -129,7 +130,7 @@ R2 = Euclidean(2) sol = OptimizationBase.solve( prob, opt, sub_problem = Manopt.convex_bundle_method_subsolver) - @test sol.minimum < 0.1 + @test sol.objective < 0.1 end # @testset "TruncatedConjugateGradientDescent" begin @@ -142,7 +143,7 @@ R2 = Euclidean(2) # prob = OptimizationProblem(optprob, x0, p; manifold = R2) # sol = OptimizationBase.solve(prob, opt) - # @test_broken sol.minimum < 0.1 + # @test_broken sol.objective < 0.1 # end @testset "AdaptiveRegularizationCubic" begin @@ -153,11 +154,12 @@ R2 = Euclidean(2) #TODO: This autodiff currently provides a Hessian that seem to not provide a Hessian # ARC Fails but also AD before that warns. So it passes _some_ hessian but a wrong one, even in format - optprob = OptimizationFunction(rosenbrock, AutoForwardDiff()) + optprob = OptimizationFunction(rosenbrock, SecondOrder(AutoForwardDiff(), AutoForwardDiff())) prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = OptimizationBase.solve(prob, opt) - @test sol.minimum < 0.1 + @test sol.objective < 0.1 + @test SciMLBase.successful_retcode(sol) broken=true end @testset "TrustRegions" begin @@ -168,11 +170,12 @@ R2 = Euclidean(2) #TODO: This autodiff currently provides a Hessian that seem to not provide a Hessian # TR Fails but also AD before that warns. So it passes _some_ hessian but a wrong one, even in format - optprob = OptimizationFunction(rosenbrock, AutoForwardDiff()) + optprob = OptimizationFunction(rosenbrock, SecondOrder(AutoForwardDiff(), AutoForwardDiff())) prob = OptimizationProblem(optprob, x0, p; manifold = R2) sol = OptimizationBase.solve(prob, opt) - @test sol.minimum < 0.1 + @test sol.objective < 0.1 + @test SciMLBase.successful_retcode(sol) broken=true end @testset "Custom constraints" begin @@ -185,6 +188,6 @@ R2 = Euclidean(2) optprob_cons = OptimizationFunction(rosenbrock; grad = rosenbrock_grad!, cons = cons) prob_cons = OptimizationProblem(optprob_cons, x0, p) #TODO: What is this? - @test_throws SciMLBase.IncompatibleOptimizerError OptimizationBase.solve(prob_cons, opt) + @test_throws OptimizationBase.IncompatibleOptimizerError OptimizationBase.solve(prob_cons, opt) end end diff --git a/lib/OptimizationMetaheuristics/Project.toml b/lib/OptimizationMetaheuristics/Project.toml index 90141e02b..16ae2e38b 100644 --- a/lib/OptimizationMetaheuristics/Project.toml +++ b/lib/OptimizationMetaheuristics/Project.toml @@ -14,9 +14,9 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "4" Metaheuristics = "3" -SciMLBase = "2.58" +SciMLBase = "2.122.1" Reexport = "1.2" [targets] diff --git a/lib/OptimizationMetaheuristics/src/OptimizationMetaheuristics.jl b/lib/OptimizationMetaheuristics/src/OptimizationMetaheuristics.jl index 65ae4c935..f672278f5 100644 --- a/lib/OptimizationMetaheuristics/src/OptimizationMetaheuristics.jl +++ b/lib/OptimizationMetaheuristics/src/OptimizationMetaheuristics.jl @@ -7,12 +7,7 @@ using SciMLBase SciMLBase.requiresbounds(opt::Metaheuristics.AbstractAlgorithm) = true SciMLBase.allowsbounds(opt::Metaheuristics.AbstractAlgorithm) = true SciMLBase.allowscallback(opt::Metaheuristics.AbstractAlgorithm) = false -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Metaheuristics.AbstractAlgorithm) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Metaheuristics.AbstractAlgorithm) = true -end +SciMLBase.has_init(opt::Metaheuristics.AbstractAlgorithm) = true function initial_population!(opt, cache, bounds, f) opt_init = deepcopy(opt) @@ -20,7 +15,8 @@ function initial_population!(opt, cache, bounds, f) Metaheuristics.optimize(f, bounds, opt_init) pop_size = opt_init.parameters.N - population_rand = [bounds[1, :] + rand(length(cache.u0)) .* (bounds[2, :] - bounds[1, :]) + population_rand = [bounds[1, :] + + rand(length(cache.u0)) .* (bounds[2, :] - bounds[1, :]) for i in 1:(pop_size - 1)] push!(population_rand, cache.u0) population_init = [Metaheuristics.create_child(x, f(x)) for x in population_rand] @@ -79,32 +75,8 @@ function SciMLBase.__init(prob::SciMLBase.OptimizationProblem, kwargs...) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - Metaheuristics.AbstractAlgorithm, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: + Metaheuristics.AbstractAlgorithm} local x maxiters = OptimizationBase._check_and_convert_maxiters(cache.solver_args.maxiters) diff --git a/lib/OptimizationMultistartOptimization/Project.toml b/lib/OptimizationMultistartOptimization/Project.toml index 195709915..adbe6b472 100644 --- a/lib/OptimizationMultistartOptimization/Project.toml +++ b/lib/OptimizationMultistartOptimization/Project.toml @@ -15,11 +15,14 @@ ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" OptimizationNLopt = "4e6fcdb7-1186-4e1f-a706-475e75c168bb" +[sources] +OptimizationNLopt = {path = "../OptimizationNLopt"} + [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "4" MultistartOptimization = "0.2, 0.3" -SciMLBase = "2.58" +SciMLBase = "2.122.1" Reexport = "1.2" [targets] diff --git a/lib/OptimizationMultistartOptimization/src/OptimizationMultistartOptimization.jl b/lib/OptimizationMultistartOptimization/src/OptimizationMultistartOptimization.jl index 13eb60b51..bab210a03 100644 --- a/lib/OptimizationMultistartOptimization/src/OptimizationMultistartOptimization.jl +++ b/lib/OptimizationMultistartOptimization/src/OptimizationMultistartOptimization.jl @@ -7,12 +7,7 @@ using SciMLBase SciMLBase.requiresbounds(opt::MultistartOptimization.TikTak) = true SciMLBase.allowsbounds(opt::MultistartOptimization.TikTak) = true SciMLBase.allowscallback(opt::MultistartOptimization.TikTak) = false -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::MultistartOptimization.TikTak) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::MultistartOptimization.TikTak) = true -end +SciMLBase.has_init(opt::MultistartOptimization.TikTak) = true function SciMLBase.__init(prob::SciMLBase.OptimizationProblem, opt::MultistartOptimization.TikTak, @@ -24,32 +19,8 @@ function SciMLBase.__init(prob::SciMLBase.OptimizationProblem, kwargs...) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - MultistartOptimization.TikTak, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: + MultistartOptimization.TikTak} local x, _loss _loss = function (θ) diff --git a/lib/OptimizationNLPModels/Project.toml b/lib/OptimizationNLPModels/Project.toml index 090b593a9..ebdfcd7a2 100644 --- a/lib/OptimizationNLPModels/Project.toml +++ b/lib/OptimizationNLPModels/Project.toml @@ -20,11 +20,16 @@ Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9" OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e" OptimizationLBFGSB = "22f7324a-a79d-40f2-bebe-3af60c77bd15" +[sources] +OptimizationMOI = {path = "../OptimizationMOI"} +OptimizationOptimJL = {path = "../OptimizationOptimJL"} +OptimizationLBFGSB = {path = "../OptimizationLBFGSB"} + [compat] julia = "1.10" NLPModels = "0.21" ADTypes = "1.7" -OptimizationBase = "3" +OptimizationBase = "3, 4" SciMLBase = "2.58" Reexport = "1.2" diff --git a/lib/OptimizationNLopt/Project.toml b/lib/OptimizationNLopt/Project.toml index aa3b509ee..4bf8737ea 100644 --- a/lib/OptimizationNLopt/Project.toml +++ b/lib/OptimizationNLopt/Project.toml @@ -17,9 +17,9 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "4" NLopt = "1.1" -SciMLBase = "2.58" +SciMLBase = "2.122.1" Reexport = "1.2" [targets] diff --git a/lib/OptimizationNLopt/src/OptimizationNLopt.jl b/lib/OptimizationNLopt/src/OptimizationNLopt.jl index aae8548e8..eea9f4791 100644 --- a/lib/OptimizationNLopt/src/OptimizationNLopt.jl +++ b/lib/OptimizationNLopt/src/OptimizationNLopt.jl @@ -8,13 +8,7 @@ using OptimizationBase: deduce_retcode (f::NLopt.Algorithm)() = f SciMLBase.allowsbounds(opt::Union{NLopt.Algorithm, NLopt.Opt}) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Union{NLopt.Algorithm, NLopt.Opt}) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Union{ - NLopt.Algorithm, NLopt.Opt}) = true -end +SciMLBase.has_init(opt::Union{NLopt.Algorithm, NLopt.Opt}) = true function SciMLBase.requiresgradient(opt::Union{NLopt.Algorithm, NLopt.Opt}) # https://github.com/JuliaOpt/NLopt.jl/blob/master/src/NLopt.jl#L18C7-L18C16 @@ -138,35 +132,8 @@ function __map_optimizer_args!(cache::OptimizationBase.OptimizationCache, opt::N return nothing end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - Union{ - NLopt.Algorithm, - NLopt.Opt - }, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: Union{ + NLopt.Algorithm, NLopt.Opt}} local x # Check if algorithm requires gradients but none are provided diff --git a/lib/OptimizationNOMAD/Project.toml b/lib/OptimizationNOMAD/Project.toml index 607622d39..50274ab08 100644 --- a/lib/OptimizationNOMAD/Project.toml +++ b/lib/OptimizationNOMAD/Project.toml @@ -13,7 +13,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "3, 4" NOMAD = "2.4.1" SciMLBase = "2.58" Reexport = "1.2" diff --git a/lib/OptimizationODE/Project.toml b/lib/OptimizationODE/Project.toml index 42a840974..615968c6d 100644 --- a/lib/OptimizationODE/Project.toml +++ b/lib/OptimizationODE/Project.toml @@ -16,11 +16,11 @@ NonlinearSolve = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" [compat] DiffEqBase = "6.190" ForwardDiff = "0.10, 1" -OptimizationBase = "3" +OptimizationBase = "4" OrdinaryDiffEq = "6.70" NonlinearSolve = "4" Reexport = "1" -SciMLBase = "2" +SciMLBase = "2.122.1" SteadyStateDiffEq = "2" julia = "1.10" diff --git a/lib/OptimizationODE/src/OptimizationODE.jl b/lib/OptimizationODE/src/OptimizationODE.jl index eb8638b9b..e511bcc88 100644 --- a/lib/OptimizationODE/src/OptimizationODE.jl +++ b/lib/OptimizationODE/src/OptimizationODE.jl @@ -26,46 +26,41 @@ end DAEMassMatrix() = DAEOptimizer(Rodas5P(autodiff = false)) - SciMLBase.requiresbounds(::ODEOptimizer) = false SciMLBase.allowsbounds(::ODEOptimizer) = false SciMLBase.allowscallback(::ODEOptimizer) = true -SciMLBase.supports_opt_cache_interface(::ODEOptimizer) = true +SciMLBase.has_init(::ODEOptimizer) = true SciMLBase.requiresgradient(::ODEOptimizer) = true SciMLBase.requireshessian(::ODEOptimizer) = false SciMLBase.requiresconsjac(::ODEOptimizer) = false SciMLBase.requiresconshess(::ODEOptimizer) = false - SciMLBase.requiresbounds(::DAEOptimizer) = false SciMLBase.allowsbounds(::DAEOptimizer) = false SciMLBase.allowsconstraints(::DAEOptimizer) = true SciMLBase.allowscallback(::DAEOptimizer) = true -SciMLBase.supports_opt_cache_interface(::DAEOptimizer) = true +SciMLBase.has_init(::DAEOptimizer) = true SciMLBase.requiresgradient(::DAEOptimizer) = true SciMLBase.requireshessian(::DAEOptimizer) = false SciMLBase.requiresconsjac(::DAEOptimizer) = true SciMLBase.requiresconshess(::DAEOptimizer) = false - function SciMLBase.__init(prob::OptimizationProblem, opt::ODEOptimizer; - callback=OptimizationBase.DEFAULT_CALLBACK, progress=false, dt=nothing, - maxiters=nothing, kwargs...) - return OptimizationCache(prob, opt; callback=callback, progress=progress, dt=dt, - maxiters=maxiters, kwargs...) + callback = OptimizationBase.DEFAULT_CALLBACK, progress = false, dt = nothing, + maxiters = nothing, kwargs...) + return OptimizationCache(prob, opt; callback = callback, progress = progress, dt = dt, + maxiters = maxiters, kwargs...) end function SciMLBase.__init(prob::OptimizationProblem, opt::DAEOptimizer; - callback=OptimizationBase.DEFAULT_CALLBACK, progress=false, dt=nothing, - maxiters=nothing, kwargs...) - return OptimizationCache(prob, opt; callback=callback, progress=progress, dt=dt, - maxiters=maxiters, kwargs...) + callback = OptimizationBase.DEFAULT_CALLBACK, progress = false, dt = nothing, + maxiters = nothing, kwargs...) + return OptimizationCache(prob, opt; callback = callback, progress = progress, dt = dt, + maxiters = maxiters, kwargs...) end -function SciMLBase.__solve( - cache::OptimizationBase.OptimizationCache{F,RC,LB,UB,LC,UC,S,O,D,P,C} - ) where {F,RC,LB,UB,LC,UC,S,O<:Union{ODEOptimizer,DAEOptimizer},D,P,C} - +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: Union{ + ODEOptimizer, DAEOptimizer}} dt = get(cache.solver_args, :dt, nothing) maxit = get(cache.solver_args, :maxiters, nothing) u0 = copy(cache.u0) @@ -109,7 +104,7 @@ function solve_ode(cache, dt, maxit, u0, p) else solve_kwargs = Dict{Symbol, Any}() end - + if !isnothing(maxit) solve_kwargs[:maxiters] = maxit end @@ -124,7 +119,8 @@ function solve_ode(cache, dt, maxit, u0, p) has_t = hasproperty(sol, :t) && !isempty(sol.t) stats = OptimizationBase.OptimizationStats( - iterations = has_destats ? get(sol.destats, :iters, 10) : (has_t ? length(sol.t) - 1 : 10), + iterations = has_destats ? get(sol.destats, :iters, 10) : + (has_t ? length(sol.t) - 1 : 10), time = has_t ? sol.t[end] : 0.0, fevals = has_destats ? get(sol.destats, :f_calls, 0) : 0, gevals = has_destats ? get(sol.destats, :iters, 0) : 0, @@ -147,11 +143,11 @@ function solve_dae_mass_matrix(cache, dt, maxit, u0, p) if m > n error("DAEOptimizer with mass matrix method requires the number of constraints to be less than or equal to the number of variables.") end - M = Diagonal([ones(n-m); zeros(m)]) + M = Diagonal([ones(n - m); zeros(m)]) function f_mass!(du, u, p_, t) cache.f.grad(du, u, p) @. du = -du - consout = @view du[(n-m)+1:end] + consout = @view du[((n - m) + 1):end] cache.f.cons(consout, u) return nothing end @@ -170,16 +166,20 @@ function solve_dae_mass_matrix(cache, dt, maxit, u0, p) else solve_kwargs = Dict{Symbol, Any}() end - + solve_kwargs[:progress] = cache.progress - if maxit !== nothing; solve_kwargs[:maxiters] = maxit; end - if dt !== nothing; solve_kwargs[:dt] = dt; end + if maxit !== nothing + solve_kwargs[:maxiters] = maxit + end + if dt !== nothing + solve_kwargs[:dt] = dt + end sol = solve(ss_prob, DynamicSS(cache.opt.solver); solve_kwargs...) # if sol.retcode ≠ ReturnCode.Success # # you may still accept Default or warn # end - u_ext = sol.u + u_ext = sol.u u_final = u_ext[1:n] return SciMLBase.build_solution(cache, cache.opt, u_final, cache.f(u_final, p); retcode = sol.retcode) @@ -199,8 +199,8 @@ function solve_dae_implicit(cache, dt, maxit, u0, p) function dae_residual!(res, du, u, p_, t) cache.f.grad(res, u, p) - @. res = du-res - consout = @view res[(n-m)+1:end] + @. res = du - res + consout = @view res[((n - m) + 1):end] cache.f.cons(consout, u) return nothing end @@ -221,11 +221,15 @@ function solve_dae_implicit(cache, dt, maxit, u0, p) else solve_kwargs = Dict{Symbol, Any}() end - + solve_kwargs[:progress] = cache.progress - if maxit !== nothing; solve_kwargs[:maxiters] = maxit; end - if dt !== nothing; solve_kwargs[:dt] = dt; end + if maxit !== nothing + solve_kwargs[:maxiters] = maxit + end + if dt !== nothing + solve_kwargs[:dt] = dt + end solve_kwargs[:initializealg] = DiffEqBase.ShampineCollocationInit() sol = solve(prob, cache.opt.solver; solve_kwargs...) @@ -236,5 +240,4 @@ function solve_dae_implicit(cache, dt, maxit, u0, p) retcode = sol.retcode) end - -end +end diff --git a/lib/OptimizationOptimJL/Project.toml b/lib/OptimizationOptimJL/Project.toml index 50968416e..9267c39c6 100644 --- a/lib/OptimizationOptimJL/Project.toml +++ b/lib/OptimizationOptimJL/Project.toml @@ -21,11 +21,11 @@ ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" [compat] julia = "1.10" PrecompileTools = "1.2" -OptimizationBase = "3" +OptimizationBase = "4" SparseArrays = "1.6" Optim = "1" Reexport = "1.2" -SciMLBase = "2.58" +SciMLBase = "2.122.1" [targets] test = ["ForwardDiff", "ModelingToolkit", "Random", "ReverseDiff", "Test", "Zygote"] diff --git a/lib/OptimizationOptimJL/src/OptimizationOptimJL.jl b/lib/OptimizationOptimJL/src/OptimizationOptimJL.jl index f969748a6..cdd25f430 100644 --- a/lib/OptimizationOptimJL/src/OptimizationOptimJL.jl +++ b/lib/OptimizationOptimJL/src/OptimizationOptimJL.jl @@ -11,17 +11,11 @@ SciMLBase.allowsbounds(opt::Optim.AbstractOptimizer) = true SciMLBase.allowsbounds(opt::Optim.SimulatedAnnealing) = false SciMLBase.requiresbounds(opt::Optim.Fminbox) = true SciMLBase.requiresbounds(opt::Optim.SAMIN) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Optim.AbstractOptimizer) = true - SciMLBase.supports_opt_cache_interface(opt::Union{Optim.Fminbox, Optim.SAMIN}) = true - SciMLBase.supports_opt_cache_interface(opt::Optim.ConstrainedOptimizer) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Optim.AbstractOptimizer) = true - OptimizationBase.supports_opt_cache_interface(opt::Union{ - Optim.Fminbox, Optim.SAMIN}) = true - OptimizationBase.supports_opt_cache_interface(opt::Optim.ConstrainedOptimizer) = true -end + +SciMLBase.has_init(opt::Optim.AbstractOptimizer) = true +SciMLBase.has_init(opt::Union{Optim.Fminbox, Optim.SAMIN}) = true +SciMLBase.has_init(opt::Optim.ConstrainedOptimizer) = true + function SciMLBase.requiresgradient(opt::Optim.AbstractOptimizer) !(opt isa Optim.ZerothOrderOptimizer) end @@ -120,28 +114,7 @@ function SciMLBase.__init(prob::OptimizationProblem, kwargs...) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P -}) where { - F, - RC, - LB, - UB, LC, UC, - S, - O <: - Optim.AbstractOptimizer, - D, - P -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: Optim.AbstractOptimizer} local x, cur, state !(cache.opt isa Optim.ZerothOrderOptimizer) && cache.f.grad === nothing && error("Use OptimizationFunction to pass the derivatives or automatically generate them with one of the autodiff backends") @@ -241,31 +214,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P -}) where { - F, - RC, - LB, - UB, LC, UC, - S, - O <: - Union{ - Optim.Fminbox, - Optim.SAMIN - }, - D, - P -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: Union{ + Optim.Fminbox, Optim.SAMIN}} local x, cur, state function _cb(trace) @@ -338,28 +288,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ original = opt_res, retcode = opt_ret, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P -}) where { - F, - RC, - LB, - UB, LC, UC, - S, - O <: - Optim.ConstrainedOptimizer, - D, - P -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: + Optim.ConstrainedOptimizer} local x, cur, state function _cb(trace) @@ -495,7 +425,8 @@ PrecompileTools.@compile_workload begin end function solve_nonnegative_least_squares(A, b, solver) - optf = OptimizationBase.OptimizationFunction(obj_f, OptimizationBase.AutoForwardDiff()) + optf = OptimizationBase.OptimizationFunction( + obj_f, OptimizationBase.AutoForwardDiff()) prob = OptimizationBase.OptimizationProblem(optf, ones(size(A, 2)), (A, b), lb = zeros(size(A, 2)), ub = Inf * ones(size(A, 2))) x = OptimizationOptimJL.solve(prob, solver, maxiters = 5000, maxtime = 100) diff --git a/lib/OptimizationOptimisers/Project.toml b/lib/OptimizationOptimisers/Project.toml index 28989ef78..43018cf9b 100644 --- a/lib/OptimizationOptimisers/Project.toml +++ b/lib/OptimizationOptimisers/Project.toml @@ -23,8 +23,8 @@ Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" [compat] julia = "1.10" -OptimizationBase = "3" -SciMLBase = "2.58" +OptimizationBase = "4" +SciMLBase = "2.122.1" Optimisers = "0.2, 0.3, 0.4" Reexport = "1.2" Logging = "1.10" diff --git a/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl b/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl index de36f25a8..b1713244d 100644 --- a/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl +++ b/lib/OptimizationOptimisers/src/OptimizationOptimisers.jl @@ -4,12 +4,7 @@ using Reexport, Logging @reexport using Optimisers, OptimizationBase using SciMLBase -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::AbstractRule) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::AbstractRule) = true -end +SciMLBase.has_init(opt::AbstractRule) = true SciMLBase.requiresgradient(opt::AbstractRule) = true SciMLBase.allowsfg(opt::AbstractRule) = true @@ -23,32 +18,7 @@ function SciMLBase.__init( save_best, progress, kwargs...) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - AbstractRule, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: AbstractRule} if OptimizationBase.isa_dataiterator(cache.p) data = cache.p dataiterate = true diff --git a/lib/OptimizationPRIMA/Project.toml b/lib/OptimizationPRIMA/Project.toml index e662802d3..4fa032440 100644 --- a/lib/OptimizationPRIMA/Project.toml +++ b/lib/OptimizationPRIMA/Project.toml @@ -16,9 +16,9 @@ ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "4" PRIMA = "0.2.0" -SciMLBase = "2.58" +SciMLBase = "2.122.1" Reexport = "1" [targets] diff --git a/lib/OptimizationPRIMA/src/OptimizationPRIMA.jl b/lib/OptimizationPRIMA/src/OptimizationPRIMA.jl index 2ba274ea2..7dba5752b 100644 --- a/lib/OptimizationPRIMA/src/OptimizationPRIMA.jl +++ b/lib/OptimizationPRIMA/src/OptimizationPRIMA.jl @@ -11,12 +11,9 @@ struct BOBYQA <: PRIMASolvers end struct LINCOA <: PRIMASolvers end struct COBYLA <: PRIMASolvers end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::PRIMASolvers) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::PRIMASolvers) = true -end +export UOBYQA, NEWUOA, BOBYQA, LINCOA, COBYLA + +SciMLBase.has_init(::PRIMASolvers) = true SciMLBase.allowsconstraints(::Union{LINCOA, COBYLA}) = true SciMLBase.allowsbounds(opt::Union{BOBYQA, LINCOA, COBYLA}) = true SciMLBase.requiresconstraints(opt::COBYLA) = true @@ -35,7 +32,7 @@ function OptimizationBase.OptimizationCache(prob::SciMLBase.OptimizationProblem, reinit_cache = OptimizationBase.ReInitCache(prob.u0, prob.p) num_cons = prob.ucons === nothing ? 0 : length(prob.ucons) if prob.f.adtype isa SciMLBase.NoAD && opt isa COBYLA - throw("We evaluate the jacobian and hessian of the constraints once to automatically detect + throw("We evaluate the jacobian and hessian of the constraints once to automatically detect linear and nonlinear constraints, please provide a valid AD backend for using COBYLA.") else if opt isa COBYLA @@ -48,12 +45,12 @@ function OptimizationBase.OptimizationCache(prob::SciMLBase.OptimizationProblem, end end - return OptimizationBase.OptimizationCache(f, reinit_cache, prob.lb, prob.ub, prob.lcons, + return OptimizationBase.OptimizationCache( + opt, f, reinit_cache, prob.lb, prob.ub, prob.lcons, prob.ucons, prob.sense, - opt, progress, callback, nothing, + progress, callback, nothing, OptimizationBase.OptimizationBase.AnalysisResults(nothing, nothing), - merge((; maxiters, maxtime, abstol, reltol), - NamedTuple(kwargs))) + merge((; maxiters, maxtime, abstol, reltol), NamedTuple(kwargs))) end function get_solve_func(opt::PRIMASolvers) @@ -70,7 +67,8 @@ function get_solve_func(opt::PRIMASolvers) end end -function __map_optimizer_args!(cache::OptimizationBase.OptimizationCache, opt::PRIMASolvers; +function __map_optimizer_args!( + cache::OptimizationBase.OptimizationCache, opt::PRIMASolvers; callback = nothing, maxiters::Union{Number, Nothing} = nothing, maxtime::Union{Number, Nothing} = nothing, @@ -109,36 +107,13 @@ function sciml_prima_retcode(rc::AbstractString) end end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: PRIMASolvers, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: PRIMASolvers} iter = 0 _loss = function (θ) x = cache.f(θ, cache.p) iter += 1 - opt_state = OptimizationBase.OptimizationState(u = θ, p = cache.p, objective = x[1], iter = iter) + opt_state = OptimizationBase.OptimizationState( + u = θ, p = cache.p, objective = x[1], iter = iter) if cache.callback(opt_state, x...) error("Optimization halted by callback.") end @@ -182,14 +157,14 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationBase.Optimization b₂ = cache.ucons[linineqsinds] (minx, - inf) = optfunc(_loss, + inf) = optfunc(_loss, cache.u0; linear_eq = (A₁, b₁), linear_ineq = (A₂, b₂), nonlinear_ineq = x -> (res = zeros(eltype(x), length(nonlininds)); - nonlincons( - res, x); - res), + nonlincons( + res, x); + res), kws...) else (minx, inf) = optfunc(_loss, cache.u0; kws...) @@ -203,5 +178,4 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationBase.Optimization stats = stats, original = inf) end -export UOBYQA, NEWUOA, BOBYQA, LINCOA, COBYLA end diff --git a/lib/OptimizationPRIMA/test/runtests.jl b/lib/OptimizationPRIMA/test/runtests.jl index 09f69f25c..0aeb3f870 100644 --- a/lib/OptimizationPRIMA/test/runtests.jl +++ b/lib/OptimizationPRIMA/test/runtests.jl @@ -16,9 +16,8 @@ using Test @test 10 * sol.objective < l1 sol = OptimizationBase.solve(prob, LINCOA(), maxiters = 1000) @test 10 * sol.objective < l1 - @test_throws SciMLBase.IncompatibleOptimizerError OptimizationBase.solve(prob, - COBYLA(), - maxiters = 1000) + @test_throws OptimizationBase.IncompatibleOptimizerError OptimizationBase.solve( + prob, COBYLA(), maxiters = 1000) function con2_c(res, x, p) res .= [x[1] + x[2], x[2] * sin(x[1]) - x[1]] diff --git a/lib/OptimizationPolyalgorithms/Project.toml b/lib/OptimizationPolyalgorithms/Project.toml index 137db158d..875166b02 100644 --- a/lib/OptimizationPolyalgorithms/Project.toml +++ b/lib/OptimizationPolyalgorithms/Project.toml @@ -13,9 +13,13 @@ OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +[sources] +OptimizationOptimisers = {path = "../OptimizationOptimisers"} +OptimizationOptimJL = {path = "../OptimizationOptimJL"} + [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "3, 4" OptimizationOptimisers = "0.3" SciMLBase = "2.58" Reexport = "1.2" diff --git a/lib/OptimizationPyCMA/Project.toml b/lib/OptimizationPyCMA/Project.toml index 2c37516bd..4eb9ec900 100644 --- a/lib/OptimizationPyCMA/Project.toml +++ b/lib/OptimizationPyCMA/Project.toml @@ -12,9 +12,9 @@ PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "4" CondaPkg = "0.2" Test = "1.10" -SciMLBase = "2.58" +SciMLBase = "2.122.1" Reexport = "1.2" PythonCall = "0.9" diff --git a/lib/OptimizationPyCMA/src/OptimizationPyCMA.jl b/lib/OptimizationPyCMA/src/OptimizationPyCMA.jl index 312732b5f..308ba23c3 100644 --- a/lib/OptimizationPyCMA/src/OptimizationPyCMA.jl +++ b/lib/OptimizationPyCMA/src/OptimizationPyCMA.jl @@ -19,12 +19,7 @@ end # Defining the SciMLBase interface for PyCMAOpt SciMLBase.allowsbounds(::PyCMAOpt) = true -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::PyCMAOpt) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::PyCMAOpt) = true -end +SciMLBase.has_init(opt::PyCMAOpt) = true SciMLBase.allowscallback(::PyCMAOpt) = true SciMLBase.requiresgradient(::PyCMAOpt) = false SciMLBase.requireshessian(::PyCMAOpt) = false @@ -43,7 +38,7 @@ function __map_optimizer_args(prob::OptimizationBase.OptimizationCache, opt::PyC end # Converting OptimizationBase.jl args to PyCMA opts - # OptimizationBase.jl kwargs will overwrite PyCMA kwargs supplied to solve() + # OptimizationBase.jl kwargs will overwrite PyCMA kwargs supplied to solve() mapped_args = Dict{String, Any}() @@ -95,32 +90,7 @@ function __map_pycma_retcode(stop_dict::Dict{String, Any}) end end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - PyCMAOpt, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: PyCMAOpt} local x # wrapping the objective function @@ -130,7 +100,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ end _cb = function (es) - opt_state = OptimizationBase.OptimizationState(; iter = pyconvert(Int, es.countiter), + opt_state = OptimizationBase.OptimizationState(; + iter = pyconvert(Int, es.countiter), u = pyconvert(Vector{Float64}, es.best.x), p = cache.p, objective = pyconvert(Float64, es.best.f), diff --git a/lib/OptimizationQuadDIRECT/Project.toml b/lib/OptimizationQuadDIRECT/Project.toml index 65b4a774b..6d0cf5582 100644 --- a/lib/OptimizationQuadDIRECT/Project.toml +++ b/lib/OptimizationQuadDIRECT/Project.toml @@ -14,7 +14,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "3, 4" SciMLBase = "2.58" Reexport = "1.2" diff --git a/lib/OptimizationSciPy/Project.toml b/lib/OptimizationSciPy/Project.toml index eb1b17cf0..4e628e430 100644 --- a/lib/OptimizationSciPy/Project.toml +++ b/lib/OptimizationSciPy/Project.toml @@ -18,8 +18,8 @@ ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" [compat] julia = "1.10" -OptimizationBase = "3" -SciMLBase = "2.58" +OptimizationBase = "4" +SciMLBase = "2.122.1" Reexport = "1.2" PythonCall = "0.9" diff --git a/lib/OptimizationSciPy/src/OptimizationSciPy.jl b/lib/OptimizationSciPy/src/OptimizationSciPy.jl index 712958b8d..0ac3f589b 100644 --- a/lib/OptimizationSciPy/src/OptimizationSciPy.jl +++ b/lib/OptimizationSciPy/src/OptimizationSciPy.jl @@ -216,32 +216,17 @@ for opt_type in [:ScipyMinimize, :ScipyDifferentialEvolution, :ScipyBasinhopping :ScipyLinprog, :ScipyMilp] @eval begin SciMLBase.allowsbounds(::$opt_type) = true - @static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::$opt_type) = true - end - @static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::$opt_type) = true - end + SciMLBase.has_init(::$opt_type) = true end end for opt_type in [:ScipyMinimizeScalar, :ScipyRootScalar, :ScipyLeastSquares] @eval begin - @static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::$opt_type) = true - end - @static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::$opt_type) = true - end + SciMLBase.has_init(::$opt_type) = true end end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(::ScipyRoot) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(::ScipyRoot) = true -end +SciMLBase.has_init(::ScipyRoot) = true function SciMLBase.requiresgradient(opt::ScipyMinimize) gradient_free = ["Nelder-Mead", "Powell", "COBYLA", "COBYQA"] @@ -298,7 +283,7 @@ function SciMLBase.__init(prob::SciMLBase.OptimizationProblem, opt::ScipyOptimiz requires_bounds = opt isa Union{ ScipyDifferentialEvolution, ScipyDirect, ScipyDualAnnealing, ScipyBrute} if requires_bounds && (isnothing(prob.lb) || isnothing(prob.ub)) - throw(SciMLBase.IncompatibleOptimizerError("$(typeof(opt)) requires bounds")) + throw(OptimizationBase.IncompatibleOptimizerError("$(typeof(opt)) requires bounds")) end if opt isa ScipyMinimizeScalar && length(prob.u0) != 1 throw(ArgumentError("ScipyMinimizeScalar requires exactly 1 variable, got $(length(prob.u0)). Use ScipyMinimize for multivariate problems.")) @@ -317,15 +302,13 @@ function SciMLBase.__init(prob::SciMLBase.OptimizationProblem, opt::ScipyOptimiz end end if !isnothing(prob.lb) && !isnothing(prob.ub) - @assert length(prob.lb) == length(prob.ub) "Bounds must have the same length" + @assert length(prob.lb)==length(prob.ub) "Bounds must have the same length" @assert all(prob.lb .<= prob.ub) "Lower bounds must be less than or equal to upper bounds" end return OptimizationCache(prob, opt; cons_tol, callback, progress, kwargs...) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyMinimize, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyMinimize} local cons_cache = nothing if !isnothing(cache.f.cons) && !isnothing(cache.lcons) cons_cache = zeros(eltype(cache.u0), length(cache.lcons)) @@ -408,8 +391,9 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, _cons_func = function (θ) θ_julia = ensure_julia_array(θ, eltype(cache.u0)) cons_cache .= zero(eltype(cons_cache)) - if hasmethod(cache.f.cons, Tuple{ - typeof(cons_cache), typeof(θ_julia), typeof(cache.p)}) + if hasmethod(cache.f.cons, + Tuple{ + typeof(cons_cache), typeof(θ_julia), typeof(cache.p)}) cache.f.cons(cons_cache, θ_julia, cache.p) else cache.f.cons(cons_cache, θ_julia) @@ -421,8 +405,9 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, cons_j_cache = zeros(eltype(cache.u0), length(lcons), length(cache.u0)) _cons_jac = function (θ) θ_julia = ensure_julia_array(θ, eltype(cache.u0)) - if hasmethod(cache.f.cons_j, Tuple{ - typeof(cons_j_cache), typeof(θ_julia), typeof(cache.p)}) + if hasmethod(cache.f.cons_j, + Tuple{ + typeof(cons_j_cache), typeof(θ_julia), typeof(cache.p)}) cache.f.cons_j(cons_j_cache, θ_julia, cache.p) else cache.f.cons_j(cons_j_cache, θ_julia) @@ -515,9 +500,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyMinimizeScalar, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyMinimizeScalar} maxtime = get(cache.solver_args, :maxtime, nothing) start_time = time() _loss = function (θ) @@ -527,7 +510,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, θ_vec = [θ] x = cache.f(θ_vec, cache.p) x = isa(x, Tuple) ? x : (x,) - opt_state = OptimizationBase.OptimizationState(u = θ_vec, p = cache.p, objective = x[1]) + opt_state = OptimizationBase.OptimizationState( + u = θ_vec, p = cache.p, objective = x[1]) if cache.callback(opt_state, x...) error("Optimization halted by callback") end @@ -584,9 +568,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyLeastSquares, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyLeastSquares} _residuals = nothing if hasfield(typeof(cache.f), :f) && (cache.f.f isa ResidualObjective) real_res = (cache.f.f)::ResidualObjective @@ -669,9 +651,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyRootScalar, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyRootScalar} x0 = cache.u0[1] maxtime = get(cache.solver_args, :maxtime, nothing) start_time = time() @@ -682,7 +662,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, θ_vec = [θ] x = cache.f(θ_vec, cache.p) x = isa(x, Tuple) ? x : (x,) - opt_state = OptimizationBase.OptimizationState(u = θ_vec, p = cache.p, objective = x[1]) + opt_state = OptimizationBase.OptimizationState( + u = θ_vec, p = cache.p, objective = x[1]) if cache.callback(opt_state, x...) error("Optimization halted by callback") end @@ -768,14 +749,12 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, try stats_dict[:iterations] = pyconvert(Int, result.iterations) catch - ; end end if pyhasattr(result, "function_calls") try stats_dict[:fevals] = pyconvert(Int, result.function_calls) catch - ; end end stats = OptimizationBase.OptimizationStats(; stats_dict...) @@ -785,9 +764,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyRoot, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyRoot} _func = _create_loss(cache, vector_output = true) kwargs = Dict{Symbol, Any}() kwargs[:method] = cache.opt.method @@ -858,9 +835,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyLinprog, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyLinprog} c = cache.f(cache.u0, cache.p) if isa(c, Tuple) c = c[1] @@ -961,9 +936,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyMilp, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyMilp} c = cache.f(cache.u0, cache.p) if isa(c, Tuple) c = c[1] @@ -995,7 +968,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, throw(ArgumentError("A, lb_con, and ub_con must all be provided for linear constraints")) end keep_feasible_flag = get(cache.solver_args, :keep_feasible, false) - constraints = scipy.optimize.LinearConstraint(A, lb_con, ub_con, keep_feasible = keep_feasible_flag) + constraints = scipy.optimize.LinearConstraint( + A, lb_con, ub_con, keep_feasible = keep_feasible_flag) end t0 = time() result = nothing @@ -1041,9 +1015,8 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyDifferentialEvolution, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: + ScipyDifferentialEvolution} _loss = _create_loss(cache) bounds = _build_bounds(cache.lb, cache.ub) maxiters = OptimizationBase._check_and_convert_maxiters(cache.solver_args.maxiters) @@ -1107,9 +1080,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyBasinhopping, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyBasinhopping} _loss = _create_loss(cache) maxiters = OptimizationBase._check_and_convert_maxiters(cache.solver_args.maxiters) bh_kwargs = Dict{Symbol, Any}() @@ -1167,9 +1138,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyDualAnnealing, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyDualAnnealing} _loss = _create_loss(cache) bounds = _build_bounds(cache.lb, cache.ub) da_kwargs = Dict{Symbol, Any}() @@ -1232,9 +1201,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyShgo, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyShgo} local cons_cache = nothing if !isnothing(cache.f.cons) && !isnothing(cache.lcons) cons_cache = zeros(eltype(cache.u0), length(cache.lcons)) @@ -1247,8 +1214,9 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, _cons_func = function (θ) θ_julia = ensure_julia_array(θ, eltype(cache.u0)) cons_cache .= zero(eltype(cons_cache)) - if hasmethod(cache.f.cons, Tuple{ - typeof(cons_cache), typeof(θ_julia), typeof(cache.p)}) + if hasmethod( + cache.f.cons, Tuple{ + typeof(cons_cache), typeof(θ_julia), typeof(cache.p)}) cache.f.cons(cons_cache, θ_julia, cache.p) else cache.f.cons(cons_cache, θ_julia) @@ -1257,7 +1225,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, end for i in 1:length(cache.lcons) if isfinite(cache.lcons[i]) - cons_func_i = let i=i, _cons_func=_cons_func + cons_func_i = let i = i, _cons_func = _cons_func θ -> _cons_func(θ)[i] - cache.lcons[i] end push!(cons_list, pydict(Dict("type" => "ineq", "fun" => cons_func_i))) @@ -1265,7 +1233,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, end for i in 1:length(cache.ucons) if isfinite(cache.ucons[i]) - cons_func_i = let i=i, _cons_func=_cons_func + cons_func_i = let i = i, _cons_func = _cons_func θ -> cache.ucons[i] - _cons_func(θ)[i] end push!(cons_list, pydict(Dict("type" => "ineq", "fun" => cons_func_i))) @@ -1328,9 +1296,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyDirect, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyDirect} _loss = _create_loss(cache) bounds = _build_bounds(cache.lb, cache.ub) maxiters = OptimizationBase._check_and_convert_maxiters(cache.solver_args.maxiters) @@ -1389,9 +1355,7 @@ function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, stats = stats) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{F, RC, LB, UB, LC, UC, S, O, D, P, - C}) where - {F, RC, LB, UB, LC, UC, S, O <: ScipyBrute, D, P, C} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: ScipyBrute} _loss = _create_loss(cache) ranges = _build_bounds(cache.lb, cache.ub) brute_kwargs = Dict{Symbol, Any}() @@ -1469,7 +1433,8 @@ function _create_loss(cache; vector_output::Bool = false) elseif isa(x, Number) x = (x,) end - opt_state = OptimizationBase.OptimizationState(u = θ_julia, p = cache.p, objective = sum(abs2, x)) + opt_state = OptimizationBase.OptimizationState( + u = θ_julia, p = cache.p, objective = sum(abs2, x)) if cache.callback(opt_state, x...) error("Optimization halted by callback") end @@ -1489,7 +1454,8 @@ function _create_loss(cache; vector_output::Bool = false) elseif isa(x, Number) x = (x,) end - opt_state = OptimizationBase.OptimizationState(u = θ_julia, p = cache.p, objective = x[1]) + opt_state = OptimizationBase.OptimizationState( + u = θ_julia, p = cache.p, objective = x[1]) if cache.callback(opt_state, x...) error("Optimization halted by callback") end diff --git a/lib/OptimizationSciPy/test/runtests.jl b/lib/OptimizationSciPy/test/runtests.jl index fcec383a7..37723c474 100644 --- a/lib/OptimizationSciPy/test/runtests.jl +++ b/lib/OptimizationSciPy/test/runtests.jl @@ -435,10 +435,10 @@ end @test sol.u ≈ [3.0] atol=1e-6 optprob = OptimizationFunction(rosenbrock, OptimizationBase.AutoZygote()) prob = OptimizationProblem(optprob, x0, _p) - @test_throws SciMLBase.IncompatibleOptimizerError solve(prob, ScipyDifferentialEvolution()) - @test_throws SciMLBase.IncompatibleOptimizerError solve(prob, ScipyDirect()) - @test_throws SciMLBase.IncompatibleOptimizerError solve(prob, ScipyDualAnnealing()) - @test_throws SciMLBase.IncompatibleOptimizerError solve(prob, ScipyBrute()) + @test_throws OptimizationBase.IncompatibleOptimizerError solve(prob, ScipyDifferentialEvolution()) + @test_throws OptimizationBase.IncompatibleOptimizerError solve(prob, ScipyDirect()) + @test_throws OptimizationBase.IncompatibleOptimizerError solve(prob, ScipyDualAnnealing()) + @test_throws OptimizationBase.IncompatibleOptimizerError solve(prob, ScipyBrute()) @test_throws ArgumentError solve(prob, ScipyBrent()) @test_throws ArgumentError solve(prob, ScipyRootScalar("brentq")) end diff --git a/lib/OptimizationSophia/Project.toml b/lib/OptimizationSophia/Project.toml index 1eb9ec882..2fa517b1d 100644 --- a/lib/OptimizationSophia/Project.toml +++ b/lib/OptimizationSophia/Project.toml @@ -20,10 +20,10 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" ComponentArrays = "0.15.29" Lux = "1.16.0" MLUtils = "0.4.8" -OptimizationBase = "3" +OptimizationBase = "4" OrdinaryDiffEqTsit5 = "1.2.0" Random = "1.10.0" -SciMLBase = "2.58" +SciMLBase = "2.122.1" SciMLSensitivity = "7.88.0" Test = "1.10.0" Zygote = "0.7.10" diff --git a/lib/OptimizationSophia/src/OptimizationSophia.jl b/lib/OptimizationSophia/src/OptimizationSophia.jl index f02dfff1e..e68702af6 100644 --- a/lib/OptimizationSophia/src/OptimizationSophia.jl +++ b/lib/OptimizationSophia/src/OptimizationSophia.jl @@ -10,15 +10,15 @@ using Random A second-order optimizer that incorporates diagonal Hessian information for faster convergence. -Based on the paper "Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training" +Based on the paper "Sophia: A Scalable Stochastic Second-order Optimizer for Language Model Pre-training" (https://arxiv.org/abs/2305.14342). Sophia uses an efficient estimate of the diagonal of the Hessian -matrix to adaptively adjust the learning rate for each parameter, achieving faster convergence than +matrix to adaptively adjust the learning rate for each parameter, achieving faster convergence than first-order methods like Adam and SGD while avoiding the computational cost of full second-order methods. ## Arguments - `η::Float64 = 1e-3`: Learning rate (step size) - - `βs::Tuple{Float64, Float64} = (0.9, 0.999)`: Exponential decay rates for the first moment (β₁) + - `βs::Tuple{Float64, Float64} = (0.9, 0.999)`: Exponential decay rates for the first moment (β₁) and diagonal Hessian (β₂) estimates - `ϵ::Float64 = 1e-8`: Small constant for numerical stability - `λ::Float64 = 1e-1`: Weight decay coefficient for L2 regularization @@ -37,18 +37,19 @@ optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoZygote()) prob = OptimizationProblem(optf, x0) # Solve with Sophia -sol = solve(prob, Sophia(η=0.01, k=5)) +sol = solve(prob, Sophia(η = 0.01, k = 5)) ``` ## Notes Sophia is particularly effective for: + - Large-scale optimization problems - Neural network training - Problems where second-order information can significantly improve convergence - + The algorithm maintains computational efficiency by only estimating the diagonal of the Hessian -matrix using a Hutchinson trace estimator with random vectors, making it more scalable than +matrix using a Hutchinson trace estimator with random vectors, making it more scalable than full second-order methods while still leveraging curvature information. """ struct Sophia @@ -60,12 +61,7 @@ struct Sophia ρ::Float64 end -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::Sophia) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::Sophia) = true -end +SciMLBase.has_init(opt::Sophia) = true SciMLBase.requiresgradient(opt::Sophia) = true SciMLBase.allowsfg(opt::Sophia) = true SciMLBase.requireshessian(opt::Sophia) = true @@ -84,32 +80,7 @@ function SciMLBase.__init(prob::OptimizationProblem, opt::Sophia; save_best, kwargs...) end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - Sophia, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: Sophia} local x, cur, state uType = eltype(cache.u0) η = uType(cache.opt.η) diff --git a/lib/OptimizationSpeedMapping/Project.toml b/lib/OptimizationSpeedMapping/Project.toml index d1f58f311..798f62c30 100644 --- a/lib/OptimizationSpeedMapping/Project.toml +++ b/lib/OptimizationSpeedMapping/Project.toml @@ -14,9 +14,9 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] julia = "1.10" -OptimizationBase = "3" +OptimizationBase = "4" SpeedMapping = "0.3" -SciMLBase = "2.58" +SciMLBase = "2.122.1" Reexport = "1.2" [targets] diff --git a/lib/OptimizationSpeedMapping/src/OptimizationSpeedMapping.jl b/lib/OptimizationSpeedMapping/src/OptimizationSpeedMapping.jl index e7fd92d10..d7646b25a 100644 --- a/lib/OptimizationSpeedMapping/src/OptimizationSpeedMapping.jl +++ b/lib/OptimizationSpeedMapping/src/OptimizationSpeedMapping.jl @@ -10,15 +10,11 @@ struct SpeedMappingOpt end SciMLBase.allowsbounds(::SpeedMappingOpt) = true SciMLBase.allowscallback(::SpeedMappingOpt) = false -@static if isdefined(SciMLBase, :supports_opt_cache_interface) - SciMLBase.supports_opt_cache_interface(opt::SpeedMappingOpt) = true -end -@static if isdefined(OptimizationBase, :supports_opt_cache_interface) - OptimizationBase.supports_opt_cache_interface(opt::SpeedMappingOpt) = true -end +SciMLBase.has_init(opt::SpeedMappingOpt) = true SciMLBase.requiresgradient(opt::SpeedMappingOpt) = true -function __map_optimizer_args(cache::OptimizationBase.OptimizationCache, opt::SpeedMappingOpt; +function __map_optimizer_args( + cache::OptimizationBase.OptimizationCache, opt::SpeedMappingOpt; callback = nothing, maxiters::Union{Number, Nothing} = nothing, maxtime::Union{Number, Nothing} = nothing, @@ -48,32 +44,7 @@ function __map_optimizer_args(cache::OptimizationBase.OptimizationCache, opt::Sp return mapped_args end -function SciMLBase.__solve(cache::OptimizationBase.OptimizationCache{ - F, - RC, - LB, - UB, - LC, - UC, - S, - O, - D, - P, - C -}) where { - F, - RC, - LB, - UB, - LC, - UC, - S, - O <: - SpeedMappingOpt, - D, - P, - C -} +function SciMLBase.__solve(cache::OptimizationCache{O}) where {O <: SpeedMappingOpt} local x _loss = function (θ) diff --git a/test/qa.jl b/test/qa.jl index 6da7ee249..9b2d16a0d 100644 --- a/test/qa.jl +++ b/test/qa.jl @@ -7,7 +7,12 @@ using Optimization, Aqua treat_as_own = [OptimizationProblem, Optimization.SciMLBase.AbstractOptimizationCache]) Aqua.test_project_extras(Optimization) - Aqua.test_stale_deps(Optimization) + if !(VERSION < v"1.11") + # in CI we need to dev packages to run the tests + # which adds stale deps + # on later versions [sources] is used instead + Aqua.test_stale_deps(Optimization) + end Aqua.test_unbound_args(Optimization) Aqua.test_undefined_exports(Optimization) end diff --git a/test/runtests.jl b/test/runtests.jl index 193d2c51b..19b7f4155 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,6 @@ -using SafeTestsets, Pkg +using SafeTestsets, Test, Pkg -const GROUP = get(ENV, "GROUP", "All") +const GROUP = get(ENV, "GROUP", "Core") function dev_subpkg(subpkg) subpkg_path = joinpath(dirname(@__DIR__), "lib", subpkg) @@ -14,41 +14,28 @@ function activate_subpkg_env(subpkg) Pkg.instantiate() end -if VERSION < v"1.11" - if GROUP == "All" || GROUP == "Core" - dev_subpkg("OptimizationBase") - dev_subpkg("OptimizationOptimJL") - dev_subpkg("OptimizationOptimisers") - dev_subpkg("OptimizationMOI") - elseif GROUP == "GPU" || GROUP == "OptimizationPolyalgorithms" - dev_subpkg("OptimizationOptimJL") - dev_subpkg("OptimizationOptimisers") - elseif GROUP == "OptimizationNLPModels" - dev_subpkg("OptimizationOptimJL") - dev_subpkg("OptimizationMOI") - end -end - @time begin - if GROUP == "All" || GROUP == "Core" - @safetestset "Quality Assurance" include("qa.jl") - @safetestset "Utils Tests" begin - include("utils.jl") - end - @safetestset "AD Tests" begin - include("ADtests.jl") - end - @safetestset "AD Performance Regression Tests" begin - include("AD_performance_regression.jl") - end - @safetestset "Optimization" begin - include("native.jl") - end - @safetestset "Mini batching" begin - include("minibatch.jl") - end - @safetestset "DiffEqFlux" begin - include("diffeqfluxtests.jl") + if GROUP == "Core" + @testset verbose=true "Optimization.jl" begin + @safetestset "Quality Assurance" include("qa.jl") + @safetestset "Utils Tests" begin + include("utils.jl") + end + @safetestset "AD Tests" begin + include("ADtests.jl") + end + @safetestset "AD Performance Regression Tests" begin + include("AD_performance_regression.jl") + end + @safetestset "Optimization" begin + include("native.jl") + end + @safetestset "Mini batching" begin + include("minibatch.jl") + end + @safetestset "DiffEqFlux" begin + include("diffeqfluxtests.jl") + end end elseif GROUP == "GPU" activate_downstream_env()