Added a new Subpackage for Augmented Lagrangian

lib/OptimizationAuglag/Project.toml

@@ -0,0 +1,18 @@
+name = "OptimizationAuglag"
+uuid = "2ea93f80-9333-43a1-a68d-1f53b957a421"
+authors = ["paramthakkar123 <[email protected]>"]
+version = "0.1.0"
+
+[deps]
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54"
+Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba"
+OptimizationOptimisers = "42dfb2eb-d2b4-4451-abcd-913932933ac1"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[compat]
+ForwardDiff = "1.0.1"
+MLUtils = "0.4.8"
+Optimization = "4.4.0"
+OptimizationOptimisers = "0.3.8"
+Test = "1.11.0"

src/auglag.jl → lib/OptimizationAuglag/src/OptimizationAuglag.jl

@@ -1,3 +1,7 @@
+module OptimizationAuglag
+
+using Optimization.SciMLBase, Optimization
+
 @kwdef struct AugLag
     inner::Any
     τ = 0.5
@@ -179,3 +183,5 @@ function SciMLBase.__solve(cache::OptimizationCache{
             stats = stats, retcode = opt_ret)
     end
 end
+
+end

lib/OptimizationAuglag/test/runtests.jl

@@ -0,0 +1,31 @@
+using MLUtils, OptimizationOptimisers
+using ForwardDiff
+using Test
+
+@testset "OptimizationAuglag.jl" begin
+    x0 = (-pi):0.001:pi
+    y0 = sin.(x0)
+    data = MLUtils.DataLoader((x0, y0), batchsize = 126)
+
+    function loss(coeffs, data)
+        ypred = [evalpoly(data[1][i], coeffs) for i in eachindex(data[1])]
+        return sum(abs2, ypred .- data[2])
+    end
+
+    function cons1(res, coeffs, p = nothing)
+        res[1] = coeffs[1] * coeffs[5] - 1
+        return nothing
+    end
+
+    optf = OptimizationFunction(loss, AutoSparseForwardDiff(), cons = cons1)
+    callback = (st, l) -> (@show l; return false)
+
+    initpars = rand(5)
+    l0 = optf(initpars, (x0, y0))
+
+    prob = OptimizationProblem(optf, initpars, data, lcons = [-Inf], ucons = [1],
+        lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0])
+    opt = solve(
+        prob, Optimization.AugLag(; inner = Adam()), maxiters = 10000, callback = callback)
+    @test opt.objective < l0
+end

src/Optimization.jl

@@ -24,7 +24,6 @@ include("utils.jl")
 include("state.jl")
 include("lbfgsb.jl")
 include("sophia.jl")
-include("auglag.jl")
 
 export solve
 

test/native.jl

@@ -51,12 +51,6 @@ prob = OptimizationProblem(optf, initpars, (x0, y0), lcons = [-Inf], ucons = [0.
 opt1 = solve(prob, Optimization.LBFGS(), maxiters = 1000, callback = callback)
 @test opt1.objective < l0
 
-prob = OptimizationProblem(optf, initpars, data, lcons = [-Inf], ucons = [1],
-    lb = [-10.0, -10.0, -10.0, -10.0, -10.0], ub = [10.0, 10.0, 10.0, 10.0, 10.0])
-opt = solve(
-    prob, Optimization.AugLag(; inner = Adam()), maxiters = 10000, callback = callback)
-@test opt.objective < l0
-
 optf1 = OptimizationFunction(loss, AutoSparseForwardDiff())
 prob1 = OptimizationProblem(optf1, rand(5), data)
 sol1 = solve(prob1, OptimizationOptimisers.Adam(), maxiters = 1000, callback = callback)