From 62037db06f959f281146f8f6d37572d3c31b2670 Mon Sep 17 00:00:00 2001 From: gaurav Date: Thu, 13 Mar 2025 16:02:04 +0530 Subject: [PATCH 01/13] add pyensmallen lbfgs --- pyproject.toml | 2 + src/optimagic/algorithms.py | 17 +++++ .../optimizers/pyensmallen_optimizers.py | 62 +++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 src/optimagic/optimizers/pyensmallen_optimizers.py diff --git a/pyproject.toml b/pyproject.toml index 40b93ff8d..ee97e9023 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ dependencies = [ "sqlalchemy>=1.3", "annotated-types", "typing-extensions", + "pyensmallen", ] dynamic = ["version"] keywords = [ @@ -334,6 +335,7 @@ ignore_errors = true [[tool.mypy.overrides]] module = [ + "pyensmallen", "pybaum", "scipy", "scipy.linalg", diff --git a/src/optimagic/algorithms.py b/src/optimagic/algorithms.py index a892f5a51..22bda379e 100644 --- a/src/optimagic/algorithms.py +++ b/src/optimagic/algorithms.py @@ -36,6 +36,7 @@ NloptVAR, ) from optimagic.optimizers.pounders import Pounders +from optimagic.optimizers.pyensmallen_optimizers import EnsmallenLBFGS from optimagic.optimizers.pygmo_optimizers import ( PygmoBeeColony, PygmoCmaes, @@ -293,6 +294,7 @@ class BoundedGradientBasedLocalScalarAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB scipy_slsqp: Type[ScipySLSQP] = ScipySLSQP scipy_truncated_newton: Type[ScipyTruncatedNewton] = ScipyTruncatedNewton @@ -847,6 +849,7 @@ class BoundedGradientBasedLocalAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB scipy_ls_dogbox: Type[ScipyLSDogbox] = ScipyLSDogbox scipy_ls_trf: Type[ScipyLSTRF] = ScipyLSTRF @@ -896,6 +899,7 @@ class GradientBasedLocalScalarAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_bfgs: Type[ScipyBFGS] = ScipyBFGS scipy_conjugate_gradient: Type[ScipyConjugateGradient] = ScipyConjugateGradient scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB @@ -963,6 +967,7 @@ class BoundedGradientBasedScalarAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_basinhopping: Type[ScipyBasinhopping] = ScipyBasinhopping scipy_dual_annealing: Type[ScipyDualAnnealing] = ScipyDualAnnealing scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB @@ -1687,6 +1692,7 @@ class BoundedLocalScalarAlgorithms(AlgoSelection): nlopt_sbplx: Type[NloptSbplx] = NloptSbplx nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB scipy_neldermead: Type[ScipyNelderMead] = ScipyNelderMead scipy_powell: Type[ScipyPowell] = ScipyPowell @@ -1950,6 +1956,7 @@ class GradientBasedLocalAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_bfgs: Type[ScipyBFGS] = ScipyBFGS scipy_conjugate_gradient: Type[ScipyConjugateGradient] = ScipyConjugateGradient scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB @@ -1992,6 +1999,7 @@ class BoundedGradientBasedAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_basinhopping: Type[ScipyBasinhopping] = ScipyBasinhopping scipy_dual_annealing: Type[ScipyDualAnnealing] = ScipyDualAnnealing scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB @@ -2061,6 +2069,7 @@ class GradientBasedScalarAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_bfgs: Type[ScipyBFGS] = ScipyBFGS scipy_basinhopping: Type[ScipyBasinhopping] = ScipyBasinhopping scipy_conjugate_gradient: Type[ScipyConjugateGradient] = ScipyConjugateGradient @@ -2592,6 +2601,7 @@ class BoundedLocalAlgorithms(AlgoSelection): nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR pounders: Type[Pounders] = Pounders + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB scipy_ls_dogbox: Type[ScipyLSDogbox] = ScipyLSDogbox scipy_ls_trf: Type[ScipyLSTRF] = ScipyLSTRF @@ -2674,6 +2684,7 @@ class LocalScalarAlgorithms(AlgoSelection): nlopt_sbplx: Type[NloptSbplx] = NloptSbplx nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_bfgs: Type[ScipyBFGS] = ScipyBFGS scipy_cobyla: Type[ScipyCOBYLA] = ScipyCOBYLA scipy_conjugate_gradient: Type[ScipyConjugateGradient] = ScipyConjugateGradient @@ -2826,6 +2837,7 @@ class BoundedScalarAlgorithms(AlgoSelection): nlopt_sbplx: Type[NloptSbplx] = NloptSbplx nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS pygmo_bee_colony: Type[PygmoBeeColony] = PygmoBeeColony pygmo_cmaes: Type[PygmoCmaes] = PygmoCmaes pygmo_compass_search: Type[PygmoCompassSearch] = PygmoCompassSearch @@ -3070,6 +3082,7 @@ class GradientBasedAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_bfgs: Type[ScipyBFGS] = ScipyBFGS scipy_basinhopping: Type[ScipyBasinhopping] = ScipyBasinhopping scipy_conjugate_gradient: Type[ScipyConjugateGradient] = ScipyConjugateGradient @@ -3263,6 +3276,7 @@ class LocalAlgorithms(AlgoSelection): nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR pounders: Type[Pounders] = Pounders + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_bfgs: Type[ScipyBFGS] = ScipyBFGS scipy_cobyla: Type[ScipyCOBYLA] = ScipyCOBYLA scipy_conjugate_gradient: Type[ScipyConjugateGradient] = ScipyConjugateGradient @@ -3335,6 +3349,7 @@ class BoundedAlgorithms(AlgoSelection): nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR pounders: Type[Pounders] = Pounders + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS pygmo_bee_colony: Type[PygmoBeeColony] = PygmoBeeColony pygmo_cmaes: Type[PygmoCmaes] = PygmoCmaes pygmo_compass_search: Type[PygmoCompassSearch] = PygmoCompassSearch @@ -3470,6 +3485,7 @@ class ScalarAlgorithms(AlgoSelection): nlopt_sbplx: Type[NloptSbplx] = NloptSbplx nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS pygmo_bee_colony: Type[PygmoBeeColony] = PygmoBeeColony pygmo_cmaes: Type[PygmoCmaes] = PygmoCmaes pygmo_compass_search: Type[PygmoCompassSearch] = PygmoCompassSearch @@ -3646,6 +3662,7 @@ class Algorithms(AlgoSelection): nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR pounders: Type[Pounders] = Pounders + ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS pygmo_bee_colony: Type[PygmoBeeColony] = PygmoBeeColony pygmo_cmaes: Type[PygmoCmaes] = PygmoCmaes pygmo_compass_search: Type[PygmoCompassSearch] = PygmoCompassSearch diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py new file mode 100644 index 000000000..4e0dfd235 --- /dev/null +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -0,0 +1,62 @@ +"""Implement ensmallen optimizers.""" + +from dataclasses import dataclass + +import numpy as np +import pyensmallen as pye +from numpy.typing import NDArray + +from optimagic import mark +from optimagic.optimization.algo_options import ( + MAX_LINE_SEARCH_STEPS, + STOPPING_MAXITER, +) +from optimagic.optimization.algorithm import Algorithm, InternalOptimizeResult +from optimagic.optimization.internal_optimization_problem import ( + InternalOptimizationProblem, +) +from optimagic.typing import ( + AggregationLevel, +) + + +@mark.minimizer( + name="ensmallen_lbfgs", + solver_type=AggregationLevel.SCALAR, + is_available=True, + is_global=False, + needs_jac=True, + needs_hess=False, + supports_parallelism=False, + supports_bounds=True, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class EnsmallenLBFGS(Algorithm): + stopping_maxiter = STOPPING_MAXITER + max_step_for_line_search = MAX_LINE_SEARCH_STEPS + # min_step_for_line_search = MIN_LINE_SEARCH_STEPS + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + optimizer = pye.L_BFGS() + + print(type(problem.fun(x0))) + + def objective_function( + x: NDArray[np.float64], grad: NDArray[np.float64] + ) -> np.float64: + grad[:] = problem.jac(x) + return np.float64(problem.fun(x)) + + raw_res = optimizer.optimize(objective_function, x0) + + res = InternalOptimizeResult( + x=raw_res, + fun=problem.fun(raw_res), + ) + + return res From de1acd0ceb068a77297b7dbee37aa3dea84dd640 Mon Sep 17 00:00:00 2001 From: gaurav Date: Thu, 13 Mar 2025 19:33:13 +0530 Subject: [PATCH 02/13] add stub --- .tools/envs/testenv-linux.yml | 1 + .tools/envs/testenv-numpy.yml | 1 + .tools/envs/testenv-others.yml | 1 + .tools/envs/testenv-pandas.yml | 1 + environment.yml | 1 + 5 files changed, 5 insertions(+) diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml index cc554a08b..993b16a5f 100644 --- a/.tools/envs/testenv-linux.yml +++ b/.tools/envs/testenv-linux.yml @@ -31,6 +31,7 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests + - pyensmallen - kaleido # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml index cea3822ac..aadd54380 100644 --- a/.tools/envs/testenv-numpy.yml +++ b/.tools/envs/testenv-numpy.yml @@ -29,6 +29,7 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests + - pyensmallen - kaleido # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml index 6b3a76db0..309208c58 100644 --- a/.tools/envs/testenv-others.yml +++ b/.tools/envs/testenv-others.yml @@ -29,6 +29,7 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests + - pyensmallen - kaleido # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index 6a5e01fda..e2457de4e 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -29,6 +29,7 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests + - pyensmallen - kaleido # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests diff --git a/environment.yml b/environment.yml index 70a63f49a..4cfa9ed66 100644 --- a/environment.yml +++ b/environment.yml @@ -40,6 +40,7 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests + - pyensmallen - kaleido # dev, tests - pre-commit>=4 # dev - -e . # dev From 622fd2c449efc86f26d4acb5d3158ef1897364cb Mon Sep 17 00:00:00 2001 From: gaurav Date: Thu, 13 Mar 2025 23:16:32 +0530 Subject: [PATCH 03/13] ensmallen lbfgs improve --- src/optimagic/algorithms.py | 8 --- .../optimizers/pyensmallen_optimizers.py | 52 ++++++++++++++----- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/optimagic/algorithms.py b/src/optimagic/algorithms.py index 22bda379e..5f25f25ca 100644 --- a/src/optimagic/algorithms.py +++ b/src/optimagic/algorithms.py @@ -294,7 +294,6 @@ class BoundedGradientBasedLocalScalarAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR - ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB scipy_slsqp: Type[ScipySLSQP] = ScipySLSQP scipy_truncated_newton: Type[ScipyTruncatedNewton] = ScipyTruncatedNewton @@ -849,7 +848,6 @@ class BoundedGradientBasedLocalAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR - ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB scipy_ls_dogbox: Type[ScipyLSDogbox] = ScipyLSDogbox scipy_ls_trf: Type[ScipyLSTRF] = ScipyLSTRF @@ -967,7 +965,6 @@ class BoundedGradientBasedScalarAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR - ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_basinhopping: Type[ScipyBasinhopping] = ScipyBasinhopping scipy_dual_annealing: Type[ScipyDualAnnealing] = ScipyDualAnnealing scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB @@ -1692,7 +1689,6 @@ class BoundedLocalScalarAlgorithms(AlgoSelection): nlopt_sbplx: Type[NloptSbplx] = NloptSbplx nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR - ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB scipy_neldermead: Type[ScipyNelderMead] = ScipyNelderMead scipy_powell: Type[ScipyPowell] = ScipyPowell @@ -1999,7 +1995,6 @@ class BoundedGradientBasedAlgorithms(AlgoSelection): nlopt_slsqp: Type[NloptSLSQP] = NloptSLSQP nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR - ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_basinhopping: Type[ScipyBasinhopping] = ScipyBasinhopping scipy_dual_annealing: Type[ScipyDualAnnealing] = ScipyDualAnnealing scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB @@ -2601,7 +2596,6 @@ class BoundedLocalAlgorithms(AlgoSelection): nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR pounders: Type[Pounders] = Pounders - ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS scipy_lbfgsb: Type[ScipyLBFGSB] = ScipyLBFGSB scipy_ls_dogbox: Type[ScipyLSDogbox] = ScipyLSDogbox scipy_ls_trf: Type[ScipyLSTRF] = ScipyLSTRF @@ -2837,7 +2831,6 @@ class BoundedScalarAlgorithms(AlgoSelection): nlopt_sbplx: Type[NloptSbplx] = NloptSbplx nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR - ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS pygmo_bee_colony: Type[PygmoBeeColony] = PygmoBeeColony pygmo_cmaes: Type[PygmoCmaes] = PygmoCmaes pygmo_compass_search: Type[PygmoCompassSearch] = PygmoCompassSearch @@ -3349,7 +3342,6 @@ class BoundedAlgorithms(AlgoSelection): nlopt_tnewton: Type[NloptTNewton] = NloptTNewton nlopt_var: Type[NloptVAR] = NloptVAR pounders: Type[Pounders] = Pounders - ensmallen_lbfgs: Type[EnsmallenLBFGS] = EnsmallenLBFGS pygmo_bee_colony: Type[PygmoBeeColony] = PygmoBeeColony pygmo_cmaes: Type[PygmoCmaes] = PygmoCmaes pygmo_compass_search: Type[PygmoCompassSearch] = PygmoCompassSearch diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py index 4e0dfd235..5a8e7b80e 100644 --- a/src/optimagic/optimizers/pyensmallen_optimizers.py +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -8,6 +8,8 @@ from optimagic import mark from optimagic.optimization.algo_options import ( + CONVERGENCE_FTOL_REL, + CONVERGENCE_GTOL_ABS, MAX_LINE_SEARCH_STEPS, STOPPING_MAXITER, ) @@ -15,9 +17,19 @@ from optimagic.optimization.internal_optimization_problem import ( InternalOptimizationProblem, ) -from optimagic.typing import ( - AggregationLevel, -) +from optimagic.typing import AggregationLevel, NonNegativeFloat, PositiveInt + +LIMITED_MEMORY_MAX_HISTORY = 10 +"""Number of memory points to be stored (default 10).""" +MIN_LINE_SEARCH_STEPS = 1e-20 +"""The minimum step of the line search.""" +MAX_LINE_SEARCH_TRIALS = 50 +"""The maximum number of trials for the line search (before giving up).""" +ARMIJO_CONSTANT = 1e-4 +"""Controls the accuracy of the line search routine for determining the Armijo +condition.""" +WOLFE_CONDITION = 0.9 +"""Parameter for detecting the Wolfe condition.""" @mark.minimizer( @@ -28,23 +40,37 @@ needs_jac=True, needs_hess=False, supports_parallelism=False, - supports_bounds=True, + supports_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, ) @dataclass(frozen=True) class EnsmallenLBFGS(Algorithm): - stopping_maxiter = STOPPING_MAXITER - max_step_for_line_search = MAX_LINE_SEARCH_STEPS - # min_step_for_line_search = MIN_LINE_SEARCH_STEPS + limited_memory_max_history: PositiveInt = LIMITED_MEMORY_MAX_HISTORY + stopping_maxiter: PositiveInt = STOPPING_MAXITER + armijo_constant: NonNegativeFloat = ARMIJO_CONSTANT # needs review + wolfe_condition: NonNegativeFloat = WOLFE_CONDITION # needs review + convergence_gtol_abs: NonNegativeFloat = CONVERGENCE_GTOL_ABS + convergence_ftol_rel: NonNegativeFloat = CONVERGENCE_FTOL_REL + max_line_search_trials: PositiveInt = MAX_LINE_SEARCH_TRIALS + min_step_for_line_search: NonNegativeFloat = MIN_LINE_SEARCH_STEPS + max_step_for_line_search: NonNegativeFloat = MAX_LINE_SEARCH_STEPS def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] ) -> InternalOptimizeResult: - optimizer = pye.L_BFGS() - - print(type(problem.fun(x0))) + optimizer = pye.L_BFGS( + numBasis=self.limited_memory_max_history, + maxIterations=self.stopping_maxiter, + armijoConstant=self.armijo_constant, + wolfe=self.wolfe_condition, + minGradientNorm=self.convergence_gtol_abs, + factr=self.convergence_ftol_rel, + maxLineSearchTrials=self.max_line_search_trials, + minStep=self.min_step_for_line_search, + maxStep=self.max_step_for_line_search, + ) def objective_function( x: NDArray[np.float64], grad: NDArray[np.float64] @@ -52,11 +78,11 @@ def objective_function( grad[:] = problem.jac(x) return np.float64(problem.fun(x)) - raw_res = optimizer.optimize(objective_function, x0) + raw = optimizer.optimize(objective_function, x0) res = InternalOptimizeResult( - x=raw_res, - fun=problem.fun(raw_res), + x=raw, # only best x is available + fun=problem.fun(raw), # best f(x) value is not available ) return res From e9515cc374a0199c7971dfe69069e54a5fe24625 Mon Sep 17 00:00:00 2001 From: gaurav Date: Tue, 18 Mar 2025 14:13:05 +0530 Subject: [PATCH 04/13] fix environment problems for windows and mac --- .github/workflows/main.yml | 8 ++++++++ .tools/envs/testenv-linux.yml | 1 - .tools/envs/testenv-numpy.yml | 1 - .tools/envs/testenv-others.yml | 1 - .tools/envs/testenv-pandas.yml | 1 - environment.yml | 1 - pyproject.toml | 1 - src/optimagic/config.py | 6 ++++++ src/optimagic/optimizers/pyensmallen_optimizers.py | 7 +++++-- 9 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d74e9eeda..cbe6cc8fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,9 +36,17 @@ jobs: python=${{ matrix.python-version }} - name: run pytest shell: bash -l {0} + if: runner.os == 'Linux' && matrix.python-version == '3.13' run: | micromamba activate optimagic pytest --cov-report=xml --cov=./ + - name: run pytest (and install pyensmallen) + shell: bash -l {0} + if: runner.os == 'Linux' && matrix.python-version < '3.13' + run: | + micromamba activate optimagic + pip install pyensmallen + pytest --cov-report=xml --cov=./ - name: Upload coverage report. if: runner.os == 'Linux' && matrix.python-version == '3.10' uses: codecov/codecov-action@v4 diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml index 993b16a5f..cc554a08b 100644 --- a/.tools/envs/testenv-linux.yml +++ b/.tools/envs/testenv-linux.yml @@ -31,7 +31,6 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - - pyensmallen - kaleido # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml index aadd54380..cea3822ac 100644 --- a/.tools/envs/testenv-numpy.yml +++ b/.tools/envs/testenv-numpy.yml @@ -29,7 +29,6 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - - pyensmallen - kaleido # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml index 309208c58..6b3a76db0 100644 --- a/.tools/envs/testenv-others.yml +++ b/.tools/envs/testenv-others.yml @@ -29,7 +29,6 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - - pyensmallen - kaleido # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index e2457de4e..6a5e01fda 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -29,7 +29,6 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - - pyensmallen - kaleido # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests diff --git a/environment.yml b/environment.yml index 4cfa9ed66..70a63f49a 100644 --- a/environment.yml +++ b/environment.yml @@ -40,7 +40,6 @@ dependencies: - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - - pyensmallen - kaleido # dev, tests - pre-commit>=4 # dev - -e . # dev diff --git a/pyproject.toml b/pyproject.toml index ee97e9023..5d370dd49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ dependencies = [ "sqlalchemy>=1.3", "annotated-types", "typing-extensions", - "pyensmallen", ] dynamic = ["version"] keywords = [ diff --git a/src/optimagic/config.py b/src/optimagic/config.py index d63ef54ac..bb1307165 100644 --- a/src/optimagic/config.py +++ b/src/optimagic/config.py @@ -91,6 +91,12 @@ else: IS_NUMBA_INSTALLED = True +try: + import pyensmallen # noqa: F401 +except ImportError: + IS_PYENSMALLEN_INSTALLED = False +else: + IS_PYENSMALLEN_INSTALLED = True # ====================================================================================== # Check if pandas version is newer or equal to version 2.1.0 diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py index 5a8e7b80e..e00fe590b 100644 --- a/src/optimagic/optimizers/pyensmallen_optimizers.py +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -3,10 +3,10 @@ from dataclasses import dataclass import numpy as np -import pyensmallen as pye from numpy.typing import NDArray from optimagic import mark +from optimagic.config import IS_PYENSMALLEN_INSTALLED from optimagic.optimization.algo_options import ( CONVERGENCE_FTOL_REL, CONVERGENCE_GTOL_ABS, @@ -19,6 +19,9 @@ ) from optimagic.typing import AggregationLevel, NonNegativeFloat, PositiveInt +if IS_PYENSMALLEN_INSTALLED: + import pyensmallen as pye + LIMITED_MEMORY_MAX_HISTORY = 10 """Number of memory points to be stored (default 10).""" MIN_LINE_SEARCH_STEPS = 1e-20 @@ -35,7 +38,7 @@ @mark.minimizer( name="ensmallen_lbfgs", solver_type=AggregationLevel.SCALAR, - is_available=True, + is_available=IS_PYENSMALLEN_INSTALLED, is_global=False, needs_jac=True, needs_hess=False, From 5b4a660d156d7107c1b8f4ffc0ce5f9ea8449366 Mon Sep 17 00:00:00 2001 From: Janos Gabler Date: Mon, 17 Mar 2025 18:01:47 +0100 Subject: [PATCH 05/13] Add how-to add an algorithm guide and improve documentation of some internal classes (#570) --- .tools/envs/testenv-linux.yml | 2 +- .tools/envs/testenv-numpy.yml | 2 +- .tools/envs/testenv-others.yml | 2 +- .tools/envs/testenv-pandas.yml | 2 +- docs/rtd_environment.yml | 3 +- .../source/explanation/internal_optimizers.md | 110 ++- .../source/how_to/how_to_add_optimizers.ipynb | 672 ++++++++++++++++++ docs/source/how_to/index.md | 1 + docs/source/reference/algo_options.md | 2 +- environment.yml | 2 +- src/optimagic/optimization/algorithm.py | 39 + .../internal_optimization_problem.py | 173 ++++- .../test_internal_optimization_problem.py | 15 + 13 files changed, 991 insertions(+), 34 deletions(-) create mode 100644 docs/source/how_to/how_to_add_optimizers.ipynb diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml index cc554a08b..67fab9017 100644 --- a/.tools/envs/testenv-linux.yml +++ b/.tools/envs/testenv-linux.yml @@ -7,7 +7,7 @@ dependencies: - petsc4py - jax - cyipopt>=1.4.0 # dev, tests - - pygmo>=2.19.0 # dev, tests + - pygmo>=2.19.0 # dev, tests, docs - nlopt # dev, tests, docs - pip # dev, tests, docs - pytest # dev, tests diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml index cea3822ac..2cd35c4e0 100644 --- a/.tools/envs/testenv-numpy.yml +++ b/.tools/envs/testenv-numpy.yml @@ -7,7 +7,7 @@ dependencies: - pandas>=2 - numpy<2 - cyipopt>=1.4.0 # dev, tests - - pygmo>=2.19.0 # dev, tests + - pygmo>=2.19.0 # dev, tests, docs - nlopt # dev, tests, docs - pip # dev, tests, docs - pytest # dev, tests diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml index 6b3a76db0..974cffec1 100644 --- a/.tools/envs/testenv-others.yml +++ b/.tools/envs/testenv-others.yml @@ -5,7 +5,7 @@ channels: - nodefaults dependencies: - cyipopt>=1.4.0 # dev, tests - - pygmo>=2.19.0 # dev, tests + - pygmo>=2.19.0 # dev, tests, docs - nlopt # dev, tests, docs - pip # dev, tests, docs - pytest # dev, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index 6a5e01fda..6d88f1016 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -7,7 +7,7 @@ dependencies: - pandas<2 - numpy<2 - cyipopt>=1.4.0 # dev, tests - - pygmo>=2.19.0 # dev, tests + - pygmo>=2.19.0 # dev, tests, docs - nlopt # dev, tests, docs - pip # dev, tests, docs - pytest # dev, tests diff --git a/docs/rtd_environment.yml b/docs/rtd_environment.yml index 1929ec914..345d083d8 100644 --- a/docs/rtd_environment.yml +++ b/docs/rtd_environment.yml @@ -4,7 +4,7 @@ channels: - conda-forge - nodefaults dependencies: - - python=3.10 + - python=3.11 - typing-extensions - pip - setuptools_scm @@ -29,6 +29,7 @@ dependencies: - plotly - nlopt - annotated-types + - pygmo>=2.19.0 - pip: - ../ - kaleido diff --git a/docs/source/explanation/internal_optimizers.md b/docs/source/explanation/internal_optimizers.md index 89871e8c4..b1ed3523c 100644 --- a/docs/source/explanation/internal_optimizers.md +++ b/docs/source/explanation/internal_optimizers.md @@ -9,48 +9,114 @@ internal optimizer interface. The advantages of using the algorithm with optimagic over using it directly are: +- You can collect the optimizer history and create criterion_plots and params_plots. +- You can use flexible formats for your start parameters (e.g. nested dicts or + namedtuples) - optimagic turns unconstrained optimizers into constrained ones. - You can use logging. - You get great error handling for exceptions in the criterion function or gradient. -- You get a parallelized and customizable numerical gradient if the user did not provide - a closed form gradient. -- You can compare your optimizer with all the other optimagic optimizers by changing - only one line of code. +- You get a parallelized and customizable numerical gradient if you don't have a closed + form gradient. +- You can compare your optimizer with all the other optimagic optimizers on our + benchmark sets. All of this functionality is achieved by transforming a more complicated user provided problem into a simpler problem and then calling "internal optimizers" to solve the transformed problem. -## The internal optimizer interface +(functions_and_classes_for_internal_optimizers)= -(to be written) +## Functions and classes for internal optimizers + +The functions and classes below are everything you need to know to add an optimizer to +optimagic. To see them in action look at +[this guide](../how_to/how_to_add_optimizers.ipynb) + +```{eval-rst} +.. currentmodule:: optimagic.mark +``` + +```{eval-rst} +.. dropdown:: mark.minimizer + + The `mark.minimizer` decorator is used to provide algorithm specific information to + optimagic. This information is used in the algorithm selection tool, for better + error handling and for processing of the user provided optimization problem. + + .. autofunction:: minimizer +``` + +```{eval-rst} +.. currentmodule:: optimagic.optimization.internal_optimization_problem +``` + +```{eval-rst} + + +.. dropdown:: InternalOptimizationProblem + + The `InternalOptimizationProblem` is optimagic's internal representation of objective + functions, derivatives, bounds, constraints, and more. This representation is already + pretty close to what most algorithms expect (e.g. parameters and bounds are flat + numpy arrays, no matter which format the user provided). + + .. autoclass:: InternalOptimizationProblem() + :members: + +``` -## Output of internal optimizers +```{eval-rst} +.. currentmodule:: optimagic.optimization.algorithm +``` + +```{eval-rst} + +.. dropdown:: InternalOptimizeResult + + This is what you need to create from the output of a wrapped algorithm. + + .. autoclass:: InternalOptimizeResult + :members: + +``` + +```{eval-rst} + +.. dropdown:: Algorithm + + .. autoclass:: Algorithm + :members: + :exclude-members: with_option_if_applicable + +``` (naming-conventions)= ## Naming conventions for algorithm specific arguments -Many optimizers have similar but slightly different names for arguments that configure -the convergence criteria, other stopping conditions, and so on. We try to harmonize -those names and their default values where possible. - -Since some optimizers support many tuning parameters we group some of them by the first -part of their name (e.g. all convergence criteria names start with `convergence`). See -{ref}`list_of_algorithms` for the signatures of the provided internal optimizers. +To make switching between different algorithm as simple as possible, we align the names +of commonly used convergence and stopping criteria. We also align the default values for +stopping and convergence criteria as much as possible. -The preferred default values can be imported from `optimagic.optimization.algo_options` -which are documented in {ref}`algo_options`. If you add a new optimizer to optimagic you -should only deviate from them if you have good reasons. +You can find the harmonized names and value [here](algo_options_docs). -Note that a complete harmonization is not possible nor desirable, because often -convergence criteria that clearly are the same are implemented slightly different for -different optimizers. However, complete transparency is possible and we try to document -the exact meaning of all options for all optimizers. +To align the names of other tuning parameters as much as possible with what is already +there, simple have a look at the optimizers we already wrapped. For example, if you are +wrapping a bfgs or lbfgs algorithm from some libray, try to look at all existing +wrappers of bfgs algorithms and use the same names for the same options. ## Algorithms that parallelize -(to be written) +Algorithms that evaluate the objective function or derivatives in parallel should only +do so via `InternalOptimizationProblem.batch_fun`, +`InternalOptimizationProblem.batch_jac` or +`InternalOptimizationProblem.batch_fun_and_jac`. + +If you parallelize in any other way, the automatic history collection will stop to work. + +In that case, call `om.mark.minimizer` with `disable_history=True`. In that case you can +either do your own history collection and add that history to `InternalOptimizeResult` +or the user has to rely on logging. ## Nonlinear constraints diff --git a/docs/source/how_to/how_to_add_optimizers.ipynb b/docs/source/how_to/how_to_add_optimizers.ipynb new file mode 100644 index 000000000..fc335ca8b --- /dev/null +++ b/docs/source/how_to/how_to_add_optimizers.ipynb @@ -0,0 +1,672 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "source": [ + "# How to add optimizers to optimagic\n", + "\n", + "This is a hands-on guide that shows you how to use custom optimizers with optimagic or\n", + "how to contribute an optimizer to the optimagic library.\n", + "\n", + "We have many [examples of optimizers](https://github.com/optimagic-dev/optimagic/tree/main/src/optimagic/optimizers) that are already part of optimagic and you can learn a lot from looking at \n", + "those. However, only looking at the final results might be a bit intimidating and does\n", + "not show the process of exploring a new optimizer library and gradually developing a \n", + "wrapper. \n", + "\n", + "This guide is there to fill the gap. It tells the story of how the `pygmo_gaco`\n", + "optimizer was added to optimagic by someone who was unfamiliar with pygmo or the \n", + "gaco algorithm. \n", + "\n", + "The steps of adding an algorithm are roughly as follows:\n", + "\n", + "1. **Understand how to use the algorithm**: Play around with the algorithm you want to \n", + "add in a notebook and solve some simple problems with it. Only move on to the next step \n", + "after you have a solid understanding. This is completely unrelated to optimagic and only\n", + "about he algorithm implementation you want to wrap. \n", + "2. **Understand how the algorithm works**: Read documentation,\n", + "research papers and other resources to find out why this algorithm was created and what \n", + "problems it is supposed to solve really well. \n", + "3. **Implement the minimal wrapper**: Learn about the `om.mark.minimizer` decorator as \n", + "well as the `om.InternalOptimizationProblem` and the `om.Algorithm` classes. Implement a \n", + "minimal version of your wrapper and test it.\n", + "4. **Complete and refactor the wrapper**: Make sure that all convergence criteria, \n", + "stopping criteria, and tuning parameters the algorithm supports can be passed to your \n", + "wrapper. Also check that the algorithm gets everything it needs to achieve maximum \n", + "performance (e.g. closed form derivatives and batch function evaluators). Now is also \n", + "the time to clean-up and refactor your code, especially if you wrap multiple optimizers \n", + "from a library.\n", + "5. **Align the wrapper with optimagic conventions**: Use harmonized names wherever \n", + "a convention exists. Think about good names everywhere else. Set stopping criteria \n", + "similar to other optimizers and try to adhere to our [design philosophy](style_guide) \n", + "when it comes to tuning parameters. \n", + "6. **Integrate your code into optimagic**: Learn how to add an optional dependency to \n", + "optimagic, where you need to put your code and how to add tests and documentation. \n", + "\n", + "\n", + "## Gen AI Policy \n", + "\n", + "It is ok to use GenAI and AI based coding assistants to speed up the process of adding \n", + "an optimizer to optimagic. They can be very useful for step 1 and 2. However, AI models \n", + "often fail completely when filling out the arguments of `om.mark.minimizer`, when you \n", + "ask them to come up with good names for tuning parameters or when you auto-generate the \n", + "documentation. \n", + "\n", + "Even for step 1 and 2 you should not use an AI Model naively, but upload a paper or \n", + "documentation page to provide context to the AI.\n", + "\n", + "Our policy is therefore:\n", + "1. Only use AI for drafts that you double-check; Never rely on AI producing correct results \n", + "2. Be transparent about your use of AI \n", + "\n", + "We will reject all Pull Requests that violate this policy. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Understand how to use the algorithm\n", + "\n", + "Understanding how to use an algorithm means that you are at least able to solve a \n", + "simple optimization problem (like a sphere function or a rosenbrock function). \n", + "\n", + "The best starting point for this are usually tutorials or example notebooks from the \n", + "documentation. An AI model can also be a good idea. \n", + "\n", + "The things you need to find out for any new algorithm are:\n", + "\n", + "1. How to code up the objective function \n", + "2. How to run an optimization at default values\n", + "3. How to pass tuning parameters \n", + "4. How to pass bounds, constraints, derivatives, batch evaluators, etc. \n", + "5. How to get results back from the optimizer\n", + "\n", + "### Objective functions in pygmo\n", + "\n", + "To add pygmo_gaco, let's start by looking at the pygmo [tutorials](https://esa.github.io/pygmo2/tutorials/tutorials.html). Objective functions are coded up via the Problem class. We skip using [pre-defined problems](https://esa.github.io/pygmo2/tutorials/using_problem.html) because they will not help us and directly go to [user defined problems](https://esa.github.io/pygmo2/tutorials/coding_udp_simple.html).\n", + "\n", + "The following is copied from the documentation:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pygmo as pg\n", + "\n", + "\n", + "class sphere_function:\n", + " def fitness(self, x):\n", + " return [sum(x * x)]\n", + "\n", + " def get_bounds(self):\n", + " return ([-1, -1], [1, 1])\n", + "\n", + "\n", + "prob = pg.problem(sphere_function())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This looks simple enough. No subclassing is required, `fitness` implements the objective\n", + "function, which returns the objective value as a list of a scalar and `get_bounds` returns \n", + "the bounds. We can immediately see how we would adjust this for any scalar objective \n", + "function. \n", + "\n", + "### How to run an optimization at default values\n", + "\n", + "After copy pasting from a few tutorials we find the following:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The initial population\n", + "pop = pg.population(prob, size=20)\n", + "# The algorithm; ker needs to be at most the population size to avoid errors\n", + "algo = pg.algorithm(pg.gaco(ker=20))\n", + "# The actual optimization process\n", + "pop = algo.evolve(pop)\n", + "# Getting the best individual in the population\n", + "best_fitness = pop.get_f()[pop.best_idx()]\n", + "print(best_fitness)\n", + "best_x = pop.get_x()[pop.best_idx()]\n", + "print(np.round(best_x, 4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It looks like the optimization worked, even though the precision is not great. The true optimal function value is 0 and the true optimal parameters are [0, 0]. But global algorithms like gaco are almost never precise, so this is good enough. \n", + "\n", + "We can also see that pygmo is really organized around concepts that are specific to genetic optimizers. Examples are `population` and `evolve`. The optimagic wrapper will hide the details (i.e. users don't have to create a population) but still allow full customization (the population size will be an algorithm specific option that can be set by the user).\n", + "\n", + "### How to pass tuning parameters\n", + "\n", + "We already saw in the previous step that tuning parameters like `ker` are passed when the \n", + "algorithm is created. \n", + "\n", + "All supported tuning parameters of gaco are listed and described \n", + "[here](https://esa.github.io/pygmo2/algorithms.html#pygmo.gaco). Unfortunately, the \n", + "description is not great so we'll have to look into the [paper](https://digital.csic.es/bitstream/10261/54957/3/Extended_ant_colony_2009.pdf) for details. \n", + "\n", + "\n", + "### How to pass bounds, constraints, derivatives, batch evaluators, etc. \n", + "\n", + "- We already saw how to pass bounds via the Problem class \n", + "- gaco does not support any other constraints, so we don't need to pass them \n", + "- gaco is derivative free, so we don't need to pass derivatives \n", + "- gaco can parallelize, so we need to find out how to pass a batch version of the \n", + "objective function\n", + "\n", + "After searching around in the pygmo documentation, we find out that our Problem needs to \n", + "be extended with a [`batch_fitness`](https://esa.github.io/pygmo2/problem.html#pygmo.problem.batch_fitness)\n", + "and our algorithm needs to know about [`pg.bfe()`](https://esa.github.io/pygmo2/bfe.html).\n", + "In our previous example it will look like this:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import pygmo as pg\n", + "\n", + "\n", + "class sphere_function:\n", + " def fitness(self, x):\n", + " return [sum(x * x)]\n", + "\n", + " def get_bounds(self):\n", + " return ([-1, -1], [1, 1])\n", + "\n", + " # dvs represents a batch of parameter vectors at which the objective function is\n", + " # evaluated. However it is stored in an unintuitive format that needs to be reshaped\n", + " # to get at the actual parameter vectors.\n", + " def batch_fitness(self, dvs):\n", + " dim = len(self.get_bounds()[0])\n", + " x_list = list(dvs.reshape(-1, dim))\n", + " # we don't actually need to parallelize to find out how batch evaluators work\n", + " # and optimagic will make it really easy to parallelize this later on.\n", + " eval_list = [self.fitness(x)[0] for x in x_list]\n", + " evals = np.array(eval_list)\n", + " return evals\n", + "\n", + "\n", + "prob = pg.problem(sphere_function())\n", + "\n", + "pop = pg.population(prob, size=20)\n", + "\n", + "# creating the algorithm now requires 3 steps\n", + "pygmo_uda = pg.gaco(ker=20)\n", + "pygmo_uda.set_bfe(pg.bfe())\n", + "algo = pg.algorithm(pygmo_uda)\n", + "\n", + "pop = algo.evolve(pop)\n", + "best_fitness = pop.get_f()[pop.best_idx()]\n", + "print(best_fitness)\n", + "best_x = pop.get_x()[pop.best_idx()]\n", + "print(np.round(best_x, 4))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For this how-to guide we leave it at this basic exploration of the pygmo library. If you actually contributed an optimizer to optimagic, you would have to explore much more and document your exploration to convince us that you understand the library you wrap in detail. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### How to get results back \n", + "\n", + "The results are stored as part of the evolved population" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Best function value: \", pop.get_f()[pop.best_idx()][0])\n", + "print(\"Best parameters: \", pop.get_x()[pop.best_idx()])\n", + "print(\"Number of function evaluations: \", pop.problem.get_fevals())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Understand how the algorithm works\n", + "\n", + "Here we want to find out as much as possible about the algorithm. Common questions \n", + "that should be answered are:\n", + "- For which kind of problems and situations was it designed?\n", + "- How does it work (intuitively)?\n", + "- Are there any papers, blogposts or other sources of information on the algorithm? \n", + "- Which tuning parameters does it have and what do they mean? \n", + "- Are there known limitations? \n", + "\n", + "### For which kind of problems and situations was it desigend \n", + "\n", + "gaco is a global optimizer that does not use derivative information. It should not be\n", + "used if you only need a local optimum or if you have derivatives. Other algorithms would \n", + "be more efficient and more precise there. \n", + "\n", + "Since gaco can evaluate the objective function in parallel it is designed for problems \n", + "with expensive objective functions. \n", + "\n", + "\n", + "### How does it work (intuitively)\n", + "\n", + "Ant colony optimization is a class of optimization algorithms modeled on the\n", + "actions of an ant colony. Artificial \"ants\" (e.g. simulation agents) locate\n", + "optimal solutions by moving through a parameter space representing all\n", + "possible solutions. Real ants lay down pheromones directing each other to\n", + "resources while exploring their environment. The simulated \"ants\" similarly\n", + "record their positions and the quality of their solutions, so that in later\n", + "simulation iterations more ants locate better solutions.\n", + "\n", + "The generalized ant colony algorithm generates future generations of ants by\n", + "using a multi-kernel gaussian distribution based on three parameters (i.e.,\n", + "pheromone values) which are computed depending on the quality of each\n", + "previous solution. The solutions are ranked through an oracle penalty\n", + "method.\n", + "\n", + "\n", + "### Are there any papers, blogposts or other sources of information on the algorithm? \n", + "\n", + "gaco was proposed in M. Schlueter, et al. (2009). Extended ant colony optimization for \n", + "non-convex mixed integer non-linear programming. Computers & Operations Research.\n", + "\n", + "See [here](https://digital.csic.es/bitstream/10261/54957/3/Extended_ant_colony_2009.pdf) for a free pdf. \n", + "\n", + "### Which tuning parameters does it have and what do they mean? \n", + "\n", + "The following is not just copied from the documentation but extended by reading the\n", + "paper. It is super important to provide as much information as possible for every \n", + "tunig parameter: \n", + "\n", + "- gen (int): number of generations.\n", + "- ker (int): number of solutions stored in the solution archive. Must be <= the population\n", + " size. \n", + "- q (float): convergence speed parameter. This parameter manages the convergence speed\n", + " towards the found minima (the smaller the faster). It must be positive and can be\n", + " larger than 1. The default is 1.0 until **threshold** is reached. Then it\n", + " is set to 0.01.\n", + "- oracle (float): oracle parameter used in the penalty method.\n", + "- acc (float): accuracy parameter for maintaining a minimum penalty\n", + " function's values distances.\n", + "- threshold (int): when the iteration counter reaches the threshold the\n", + " convergence speed is set to 0.01 automatically. To deactivate this effect\n", + " set the threshold to stopping.maxiter which is the largest allowed\n", + " value.\n", + "- n_gen_mark (int): parameter that determines the convergence speed of the standard \n", + " deviations. This must be an integer.\n", + "- impstop (int): if a positive integer is assigned here, the algorithm will count the \n", + " runs without improvements, if this number exceeds the given value, the algorithm \n", + " will be stopped.\n", + "- evalstop (int): maximum number of function evaluations.\n", + "- focus (float): this parameter makes the search for the optimum greedier\n", + " and more focused on local improvements (the higher the greedier). If the\n", + " value is very high, the search is more focused around the current best\n", + " solutions. Values larger than 1 are allowed.\n", + "- memory (bool): if True, memory is activated in the algorithm for multiple calls.\n", + "- seed (int): seed used by the internal random number generator (default is random).\n", + "\n", + "\n", + "### Are there known limitations \n", + "\n", + "No. \n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Implement the minimal wrapper\n", + "\n", + "\n", + "### Learn the relevant functions and classes\n", + "\n", + "Before you implement a minimal wrapper, you need to familiarize yourself with a few\n", + "important [classes and functions](functions_and_classes_for_internal_optimizers) \n", + "you will need. \n", + "\n", + "- The `mark.miminizer` decorator \n", + "- The `Algorithm` class \n", + "- The `InternalOptimizationProblem` class \n", + "- The `InternalOptimizeResult` class \n", + "\n", + "**Your task will be to subclass `Algorithm`. Your subclass must be decorated with\n", + "`mark.minizer` and override `Algorithm._solve_internal_problem`. `_solve_internal_problem`\n", + "takes an `InternalOptimizationProblem` and returns an `InternalOptimizeResult`**\n", + "\n", + "```{note}\n", + "Users of optimagic never create instances of `InternalOptimizationProblem` nor \n", + "do they call the `_solve_internal_problem` methods of algorithms. Instead they call \n", + "`minimize` or `maximize` which are much more convenient and flexible. \n", + "\n", + "`minimize` and `maximize` will then create an `InternalOptimizationProblem` from the \n", + "user's inputs, call the `_solve_internal_problem` method and postprocess it to create an \n", + "OptimizeResult. \n", + "\n", + "To summarize: The public `minimize` interface is optimized for user-friendliness. The \n", + "`InternalOptimizeProblem` is optimized for easy wrapping of external libraries. \n", + "```\n", + "\n", + "Below we define a heavily commented minimal version of a wrapper for pygmo's gaco \n", + "algorithm. We stay as close as possible to the pygmo examples we have worked with \n", + "before and ignore most tuning parameters for now. \n", + "\n", + "\n", + "### Write the minimal implementation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "\n", + "from numpy.typing import NDArray\n", + "\n", + "import optimagic as om\n", + "from optimagic.optimization.algorithm import Algorithm, InternalOptimizeResult\n", + "from optimagic.optimization.internal_optimization_problem import (\n", + " InternalOptimizationProblem,\n", + ")\n", + "from optimagic.typing import AggregationLevel, PositiveInt\n", + "\n", + "try:\n", + " import pygmo as pg\n", + "\n", + " IS_PYGMO_INSTALLED = True\n", + "except ImportError:\n", + " IS_PYGMO_INSTALLED = False\n", + "\n", + "\n", + "@om.mark.minimizer(\n", + " # you can pick the name; convention is lowercase with underscores\n", + " name=\"pygmo_gaco\",\n", + " # the type of problem this optimizer can solve -> scalar problems; Other optimizers\n", + " # solve likelihood or least_squares problems.\n", + " solver_type=AggregationLevel.SCALAR,\n", + " # is the optimizer available? -> only if pygmo is installed\n", + " is_available=IS_PYGMO_INSTALLED,\n", + " # is the optimizer a global optimizer? -> yes\n", + " is_global=True,\n", + " # does the optimizer need the jacobian? -> no, gaco is derivative free\n", + " needs_jac=False,\n", + " # does the optimizer need the hessian? -> no, gaco is derivative free\n", + " needs_hess=False,\n", + " # does the optimizer support parallelism? -> yes\n", + " supports_parallelism=True,\n", + " # does the optimizer support bounds? -> yes\n", + " supports_bounds=True,\n", + " # does the optimizer support linear constraints? -> no\n", + " supports_linear_constraints=False,\n", + " # does the optimizer support nonlinear constraints? -> no\n", + " supports_nonlinear_constraints=False,\n", + " # should the history be disabled? -> no\n", + " disable_history=False,\n", + ")\n", + "# All algortihms need to be frozen dataclasses.\n", + "@dataclass(frozen=True)\n", + "class PygmoGaco(Algorithm):\n", + " # for now only set one parameter to get things running. The rest will come later.\n", + " stopping_maxiter: PositiveInt = 1000\n", + " n_cores: int = 1\n", + "\n", + " def _solve_internal_problem(\n", + " self, problem: InternalOptimizationProblem, x0: NDArray[np.float64]\n", + " ) -> InternalOptimizeResult:\n", + " # create a pygmo problem from the internal optimization problem\n", + " # This is just slightly more abstract than before and actually simpler because\n", + " # we have problem.batch_fun.\n", + "\n", + " n_cores = self.n_cores\n", + "\n", + " class PygmoProblem:\n", + " def fitness(self, x):\n", + " # problem.fun is not just the `fun` that was passed to `minimize` by\n", + " # the user. It is a wrapper around fun with added error handling,\n", + " # history collection, and reparametrization to enforce constraints.\n", + " # Moreover, it always works on flat numpy arrays as parameters and\n", + " # does not have additional arguments. The magic of optimagic is to\n", + " # create this internal `fun` from the user's `fun`, so you don't have\n", + " # to deal with constraints, weird parameter formats and similar when\n", + " # implementing the wrapper.\n", + " return [problem.fun(x)]\n", + "\n", + " def get_bounds(self):\n", + " # problem.bounds is not just the `bounds` that was passed to `minimize`\n", + " # by the user, which could have been a dictionary or some other non-flat\n", + " # format. `problem.bounds` always contains flat arrays with lower and\n", + " # upper bounds because this makes it easy to write wrappers.\n", + " return (problem.bounds.lower, problem.bounds.upper)\n", + "\n", + " def batch_fitness(self, dvs):\n", + " # The processing of dvs is pygmo specific.\n", + " dim = len(self.get_bounds()[0])\n", + " x_list = list(dvs.reshape(-1, dim))\n", + " # problem.batch_fun is a parallelized version of problem.fun.\n", + " eval_list = problem.batch_fun(x_list, n_cores)\n", + " evals = np.array(eval_list)\n", + " return evals\n", + "\n", + " prob = pg.problem(PygmoProblem())\n", + " pop = pg.population(prob, size=20)\n", + " pygmo_uda = pg.gaco(ker=20)\n", + " pygmo_uda.set_bfe(pg.bfe())\n", + " algo = pg.algorithm(pygmo_uda)\n", + " pop = algo.evolve(pop)\n", + " best_fun = pop.get_f()[pop.best_idx()][0]\n", + " best_x = pop.get_x()[pop.best_idx()]\n", + " n_fun_evals = pop.problem.get_fevals()\n", + " # For now we only use a few fields of the InternalOptimizeResult.\n", + " out = InternalOptimizeResult(\n", + " x=best_x,\n", + " fun=best_fun,\n", + " n_fun_evals=n_fun_evals,\n", + " )\n", + " return out" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test the minimal wrapper directly\n", + "\n", + "So now that we have a wrapper, what do we do with it? And how can we be sure it works?\n", + "\n", + "We'll first try it out directly with the `SphereExampleInternalOptimizationProblem`. \n", + "This is only for debugging and testing purposes. A user would never create an \n", + "InternalOptimizationProblem and call an algorithm with it. It's called \"Internal\" for \n", + "a reason!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from optimagic.optimization.internal_optimization_problem import (\n", + " SphereExampleInternalOptimizationProblem,\n", + ")\n", + "\n", + "problem = SphereExampleInternalOptimizationProblem()\n", + "\n", + "gaco = PygmoGaco()\n", + "\n", + "result = gaco._solve_internal_problem(problem, x0=np.array([1.0, 1.0]))\n", + "\n", + "print(result.fun)\n", + "print(result.x)\n", + "print(result.n_fun_evals)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use the minimal wrapper in minimize\n", + "\n", + "The internal testing gives us some confidence that the wrapper works correctly and would \n", + "have been good for debugging if it didn't. But now we want to test the wrapper in the\n", + "way it would be used later: via `minimize`\n", + "\n", + "With this we also get all the benefits of optimagic, from history collection and \n", + "criterion plots to flexible parameter formats. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "res = om.minimize(\n", + " fun=lambda x: x @ x,\n", + " params=np.arange(5),\n", + " algorithm=PygmoGaco,\n", + " bounds=om.Bounds(lower=-np.ones(5), upper=np.ones(5)),\n", + ")\n", + "\n", + "om.criterion_plot(res, monotone=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4 Complete and refactor the wrapper\n", + "\n", + "To keep things simple, we left out almost all tuning parameters of the gaco algorithm \n", + "when we wrote the minimal wrapper. \n", + "\n", + "Now it's time to add them. You can add them one by one and make sure nothing breaks by \n", + "testing your wrapper after each change - both with the internal problem and via \n", + "minimize. \n", + "\n", + "Moreover, our code looks quite messy currently. Despite being a minimal wrapper, the \n", + "`_solve_internal_problem` method is quite long, unstructured and hard to read. \n", + "\n", + "The result of completing and refactoring the wrapper is too long to be repeated in the \n", + "notebook. Instead you can look at the actual [implementation in optimagic](\n", + "https://github.com/optimagic-dev/optimagic/blob/ba2678753587f91cea54de69ff76cb3dcb4257d4/src/optimagic/optimizers/pygmo_optimizers.py#L70)\n", + "\n", + "\n", + "The PygmoGaco class now contains all tuning parameters we identified in step 2 as\n", + "dataclass fields. They all have very useful type-hints that don't just show whether\n", + "a parameter is an int, str or float but also which values it can take (e.g. PositiveInt).\n", + "\n", + "`_solve_internal_problem` is now also much cleaner. It mainly maps our mor descriptive \n", + "names of tuning parameters to the old pygmo names and then calls a function called \n", + "`_minimize_pygmo` that does all the heavy lifting and can be re-used for other pygmo \n", + "optimizers. \n", + "\n", + "The arguments to `mark.minimizer` have not changed. They always need te be set correctly,\n", + "even for minimal working examples. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Align the wrapper with optimagic conventions\n", + "\n", + "To make switching between different algorithm as simple as possible, we align the names \n", + "of commonly used convergence and stopping criteria. We also align the default values for \n", + "stopping and convergence criteria as much as possible. \n", + "\n", + "You can find the harmonized names and value [here](algo_options_docs). \n", + "\n", + "To align the names of other tuning parameters as much as possible with what is already \n", + "there, simple have a look at the optimizers we already wrapped. For example, if you are \n", + "wrapping a bfgs or lbfgs algorithm from some libray, try to look at all existing wrappers \n", + "of bfgs algorithms and use the same names for the same options. \n", + "\n", + "You can see what this means for the gaco algorithm [here](\n", + "https://github.com/optimagic-dev/optimagic/blob/ba2678753587f91cea54de69ff76cb3dcb4257d4/src/optimagic/optimizers/pygmo_optimizers.py#L70)\n", + "\n", + "In the future we will provide much more extensive guidelines for harmonization. \n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Integrate your code into optimagic\n", + "\n", + "So far you could have worked in a Jupyter Notebook. Integrating your code into \n", + "optimagic only requires a few small changes:\n", + "\n", + "1. Add new dependencies to the `environment.yml` file and run \n", + "`pre-commit run --all-files`. This will trigger a script that adds the dependencies to \n", + "multiple environments we need for continuous integration. Then re-create the enviroment \n", + "to make sure that the environment is the same as we will use for continuous integration.\n", + "If your dependencies don't work on all platforms (e.g. linux only packages), skip this\n", + "entire step and reach out to a core contributor for help. \n", + "2. Save the code for your algorithm wrapper in a .py file in `optimagic.algorithms`. \n", + "Use an existing file if you wrap another algorithm from a library we already had. \n", + "Otherwise, create a new file. \n", + "3. Run `pre-commit run --all-files`. This will trigger an automatic code generation \n", + "that fully integrates your wrapper into our algorithm selection tool.\n", + "4. Run `pytest`. This will run at least a few tests for your new algorithm. Add more \n", + "tests for algorithm specific things (e.g. tests that make sure tuning parameters have \n", + "the intended effects). \n", + "5. Write documentation. The documentation should contain everything you figured out in \n", + "step 2. You can either write it into the docstring of your algorithm class (preferred, \n", + "as this is what we will do for all algorithms in the long run) or in `algorithms.md` \n", + "in the documentation. \n", + "6. Create a pull request and ask for a review. " + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "optimagic", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/source/how_to/index.md b/docs/source/how_to/index.md index fc1677cb5..2a2f362d5 100644 --- a/docs/source/how_to/index.md +++ b/docs/source/how_to/index.md @@ -24,4 +24,5 @@ how_to_logging how_to_errors_during_optimization how_to_slice_plot how_to_benchmarking +how_to_add_optimizers ``` diff --git a/docs/source/reference/algo_options.md b/docs/source/reference/algo_options.md index 367644521..2336c8003 100644 --- a/docs/source/reference/algo_options.md +++ b/docs/source/reference/algo_options.md @@ -1,4 +1,4 @@ -(algo_options)= +(algo_options_docs)= # The default algorithm options diff --git a/environment.yml b/environment.yml index 70a63f49a..83b2a0952 100644 --- a/environment.yml +++ b/environment.yml @@ -6,7 +6,7 @@ channels: dependencies: - python=3.10 # dev - cyipopt>=1.4.0 # dev, tests - - pygmo>=2.19.0 # dev, tests + - pygmo>=2.19.0 # dev, tests, docs - jupyterlab # dev, docs - nlopt # dev, tests, docs - pip # dev, tests, docs diff --git a/src/optimagic/optimization/algorithm.py b/src/optimagic/optimization/algorithm.py index 7f776cf90..125ad6799 100644 --- a/src/optimagic/optimization/algorithm.py +++ b/src/optimagic/optimization/algorithm.py @@ -67,6 +67,27 @@ def __post_init__(self) -> None: @dataclass(frozen=True) class InternalOptimizeResult: + """Internal representation of the result of an optimization problem. + + Args: + x: The optimal parameters. + fun: The function value at the optimal parameters. + success: Whether the optimization was successful. + message: A message from the optimizer. + status: The status of the optimization. + n_fun_evals: The number of function evaluations. + n_jac_evals: The number of gradient or jacobian evaluations. + n_hess_evals: The number of Hessian evaluations. + n_iterations: The number of iterations. + jac: The Jacobian of the objective function at the optimal parameters. + hess: The Hessian of the objective function at the optimal parameters. + hess_inv: The inverse of the Hessian of the objective function at the optimal + parameters. + max_constraint_violation: The maximum constraint violation. + info: Additional information from the optimizer. + + """ + x: NDArray[np.float64] fun: float | NDArray[np.float64] success: bool | None = None @@ -175,6 +196,13 @@ def algo_info(self) -> AlgoInfo: @dataclass(frozen=True) class Algorithm(ABC, metaclass=AlgorithmMeta): + """Base class for all optimization algorithms in optimagic. + + To add an optimizer to optimagic you need to subclass Algorithm and overide the + ``_solve_internal_problem`` method. + + """ + @abstractmethod def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -200,6 +228,7 @@ def __post_init__(self) -> None: object.__setattr__(self, field, value) def with_option(self, **kwargs: Any) -> Self: + """Create a modified copy with the given options.""" valid_keys = set(self.__dataclass_fields__) - {"__algo_info__"} invalid = set(kwargs) - valid_keys if invalid: @@ -210,6 +239,7 @@ def with_option(self, **kwargs: Any) -> Self: return replace(self, **kwargs) def with_stopping(self, **kwargs: Any) -> Self: + """Create a modified copy with the given stopping options.""" options = {} for k, v in kwargs.items(): if k.startswith("stopping_"): @@ -220,6 +250,7 @@ def with_stopping(self, **kwargs: Any) -> Self: return self.with_option(**options) def with_convergence(self, **kwargs: Any) -> Self: + """Create a modified copy with the given convergence options.""" options = {} for k, v in kwargs.items(): if k.startswith("convergence_"): @@ -235,6 +266,12 @@ def solve_internal_problem( x0: NDArray[np.float64], step_id: int, ) -> InternalOptimizeResult: + """Solve the internal optimization problem. + + This method is called internally by `minimize` or `maximize` to solve the + internal optimization problem and process the results. + + """ problem = problem.with_new_history().with_step_id(step_id) if problem.logger: @@ -270,6 +307,7 @@ def with_option_if_applicable(self, **kwargs: Any) -> Self: @property def name(self) -> str: + """The name of the algorithm.""" # cannot call algo_info here because it would be an infinite recursion if hasattr(self, "__algo_info__") and self.__algo_info__ is not None: return self.__algo_info__.name @@ -277,6 +315,7 @@ def name(self) -> str: @property def algo_info(self) -> AlgoInfo: + """Information about the algorithm.""" if not hasattr(self, "__algo_info__") or self.__algo_info__ is None: msg = ( f"The algorithm {self.name} does not have have the __algo_info__ " diff --git a/src/optimagic/optimization/internal_optimization_problem.py b/src/optimagic/optimization/internal_optimization_problem.py index c15e32fe0..6e0e58e46 100644 --- a/src/optimagic/optimization/internal_optimization_problem.py +++ b/src/optimagic/optimization/internal_optimization_problem.py @@ -8,12 +8,18 @@ from numpy.typing import NDArray from typing_extensions import Self +from optimagic.batch_evaluators import process_batch_evaluator from optimagic.differentiation.derivatives import first_derivative from optimagic.differentiation.numdiff_options import NumdiffOptions from optimagic.exceptions import UserFunctionRuntimeError, get_traceback from optimagic.logging.logger import LogStore from optimagic.logging.types import IterationState -from optimagic.optimization.fun_value import SpecificFunctionValue +from optimagic.optimization.fun_value import ( + LeastSquaresFunctionValue, + LikelihoodFunctionValue, + ScalarFunctionValue, + SpecificFunctionValue, +) from optimagic.optimization.history import History, HistoryEntry from optimagic.parameters.bounds import Bounds from optimagic.parameters.conversion import Converter @@ -79,11 +85,32 @@ def __init__( # ================================================================================== def fun(self, x: NDArray[np.float64]) -> float | NDArray[np.float64]: + """Evaluate the objective function at x. + + Args: + x: The parameter vector at which to evaluate the objective function. + + Returns: + The function value at x. This is a scalar for scalar problems and an array + for least squares or likelihood problems. + + """ fun_value, hist_entry = self._evaluate_fun(x) self._history.add_entry(hist_entry) return fun_value def jac(self, x: NDArray[np.float64]) -> NDArray[np.float64]: + """Evaluate the first derivative at x. + + Args: + x: The parameter vector at which to evaluate the first derivative. + + Returns: + The first derivative at x. This is a 1d array for scalar problems (the + gradient) and a 2d array for least squares or likelihood problems (the + Jacobian). + + """ jac_value, hist_entry = self._evaluate_jac(x) self._history.add_entry(hist_entry) return jac_value @@ -91,6 +118,11 @@ def jac(self, x: NDArray[np.float64]) -> NDArray[np.float64]: def fun_and_jac( self, x: NDArray[np.float64] ) -> tuple[float | NDArray[np.float64], NDArray[np.float64]]: + """Simultaneously evaluate the objective function and its first derivative. + + See .fun and .jac for details. + + """ fun_and_jac_value, hist_entry = self._evaluate_fun_and_jac(x) self._history.add_entry(hist_entry) return fun_and_jac_value @@ -101,6 +133,20 @@ def batch_fun( n_cores: int, batch_size: int | None = None, ) -> list[float | NDArray[np.float64]]: + """Parallelized batch version of .fun. + + Args: + x_list: A list of parameter vectors at which to evaluate the objective + function. + n_cores: The number of cores to use for the parallel evaluation. + batch_size: Batch size that can be used by some algorithms to simulate + the behavior under parallelization on more cores than are actually + available. Only used by `criterion_plots` and benchmark plots. + + Returns: + A list of function values at the points in x_list. See .fun for details. + + """ batch_size = n_cores if batch_size is None else batch_size batch_result = self._batch_evaluator( func=self._evaluate_fun, @@ -121,6 +167,20 @@ def batch_jac( n_cores: int, batch_size: int | None = None, ) -> list[NDArray[np.float64]]: + """Parallelized batch version of .jac. + + Args: + x_list: A list of parameter vectors at which to evaluate the first + derivative. + n_cores: The number of cores to use for the parallel evaluation. + batch_size: Batch size that can be used by some algorithms to simulate + the behavior under parallelization on more cores than are actually + available. Only used by `criterion_plots` and benchmark plots. + + Returns: + A list of first derivatives at the points in x_list. See .jac for details. + + """ batch_size = n_cores if batch_size is None else batch_size batch_result = self._batch_evaluator( @@ -141,6 +201,21 @@ def batch_fun_and_jac( n_cores: int, batch_size: int | None = None, ) -> list[tuple[float | NDArray[np.float64], NDArray[np.float64]]]: + """Parallelized batch version of .fun_and_jac. + + Args: + x_list: A list of parameter vectors at which to evaluate the objective + function and its first derivative. + n_cores: The number of cores to use for the parallel evaluation. + batch_size: Batch size that can be used by some algorithms to simulate + the behavior under parallelization on more cores than are actually + available. Only used by `criterion_plots` and benchmark plots. + + Returns: + A list of tuples containing the function value and the first derivative + at the points in x_list. See .fun_and_jac for details. + + """ batch_size = n_cores if batch_size is None else batch_size batch_result = self._batch_evaluator( func=self._evaluate_fun_and_jac, @@ -194,28 +269,34 @@ def with_step_id(self, step_id: int) -> Self: # Public attributes # ================================================================================== + @property + def bounds(self) -> InternalBounds: + """Bounds of the optimization problem.""" + return self._bounds + @property def linear_constraints(self) -> list[dict[str, Any]] | None: + # TODO: write a docstring as soon as we actually use this return self._linear_constraints @property def nonlinear_constraints(self) -> list[dict[str, Any]] | None: + """Internal dictionary representation of nonlinear constraints.""" return self._nonlinear_constraints @property def direction(self) -> Direction: + """Direction of the optimization problem.""" return self._direction @property def history(self) -> History: + """History container for the optimization problem.""" return self._history - @property - def bounds(self) -> InternalBounds: - return self._bounds - @property def logger(self) -> LogStore[Any, Any] | None: + """Logger for the optimization problem.""" return self._logger # ================================================================================== @@ -675,3 +756,85 @@ def _process_jac_value( out_value = -out_value return out_value + + +class SphereExampleInternalOptimizationProblem(InternalOptimizationProblem): + """Super simple example of an internal optimization problem. + + This can be used to test algorithm wrappers or to familiarize yourself with the + internal optimization problem interface. + + Args: + + """ + + def __init__( + self, + solver_type: AggregationLevel = AggregationLevel.SCALAR, + binding_bounds: bool = False, + ) -> None: + _fun_dict = { + AggregationLevel.SCALAR: lambda x: ScalarFunctionValue(x @ x), + AggregationLevel.LIKELIHOOD: lambda x: LikelihoodFunctionValue(x**2), + AggregationLevel.LEAST_SQUARES: lambda x: LeastSquaresFunctionValue(x), + } + + _jac_dict = { + AggregationLevel.SCALAR: lambda x: 2 * x, + AggregationLevel.LIKELIHOOD: lambda x: 2 * x, + AggregationLevel.LEAST_SQUARES: lambda x: np.eye(len(x)), + } + + fun = _fun_dict[solver_type] + jac = _jac_dict[solver_type] + fun_and_jac = lambda x: (fun(x), jac(x)) + + converter = Converter( + params_to_internal=lambda x: x, + params_from_internal=lambda x: x, + derivative_to_internal=lambda x, x0: x, + has_transforming_constraints=False, + ) + + direction = Direction.MINIMIZE + + if binding_bounds: + lb = np.arange(10, dtype=np.float64) - 7.0 + ub = np.arange(10, dtype=np.float64) - 3.0 + self._x_opt = np.array([-3, -2, -1, 0, 0, 0, 0, 0, 1, 2.0]) + else: + lb = np.full(10, -10, dtype=np.float64) + ub = np.full(10, 10, dtype=np.float64) + self._x_opt = np.zeros(10) + + bounds = InternalBounds(lb, ub) + + numdiff_options = NumdiffOptions() + + error_handling = ErrorHandling.RAISE + + error_penalty_func = fun_and_jac + + batch_evaluator = process_batch_evaluator("joblib") + + linear_constraints = None + nonlinear_constraints = None + + logger = None + + super().__init__( + fun=fun, + jac=jac, + fun_and_jac=fun_and_jac, + converter=converter, + solver_type=solver_type, + direction=direction, + bounds=bounds, + numdiff_options=numdiff_options, + error_handling=error_handling, + error_penalty_func=error_penalty_func, + batch_evaluator=batch_evaluator, + linear_constraints=linear_constraints, + nonlinear_constraints=nonlinear_constraints, + logger=logger, + ) diff --git a/tests/optimagic/optimization/test_internal_optimization_problem.py b/tests/optimagic/optimization/test_internal_optimization_problem.py index a0bb24a25..3d37149b9 100644 --- a/tests/optimagic/optimization/test_internal_optimization_problem.py +++ b/tests/optimagic/optimization/test_internal_optimization_problem.py @@ -16,6 +16,7 @@ from optimagic.optimization.internal_optimization_problem import ( InternalBounds, InternalOptimizationProblem, + SphereExampleInternalOptimizationProblem, ) from optimagic.parameters.conversion import Converter from optimagic.typing import AggregationLevel, Direction, ErrorHandling, EvalTask @@ -706,3 +707,17 @@ def test_error_in_exploration_fun_maximize(error_max_problem): ) expected = [-np.inf, -np.inf] assert np.allclose(got, expected) + + +# ====================================================================================== +# test SphereExampleInternalOptimizationProblem +# ====================================================================================== + + +def test_sphere_example_internal_optimization_problem(): + problem = SphereExampleInternalOptimizationProblem() + assert problem.fun(np.array([1, 2, 3])) == 14 + aaae(problem.jac(np.array([1, 2, 3])), np.array([2, 4, 6])) + f, j = problem.fun_and_jac(np.array([1, 2, 3])) + assert f == 14 + aaae(j, np.array([2, 4, 6])) From 1215434e1ee135432bdeaa19725a857b7ccb61d8 Mon Sep 17 00:00:00 2001 From: gaurav Date: Tue, 18 Mar 2025 21:55:05 +0530 Subject: [PATCH 06/13] change test names, add docs, use fun_and_jac instead of fun and jac --- .github/workflows/main.yml | 4 +-- docs/source/algorithms.md | 28 +++++++++++++++++++ docs/source/refs.bib | 12 ++++++++ .../optimizers/pyensmallen_optimizers.py | 5 ++-- 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cbe6cc8fd..082858ac9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,13 +34,13 @@ jobs: cache-environment: true create-args: | python=${{ matrix.python-version }} - - name: run pytest + - name: run pytest (for python 3.13) shell: bash -l {0} if: runner.os == 'Linux' && matrix.python-version == '3.13' run: | micromamba activate optimagic pytest --cov-report=xml --cov=./ - - name: run pytest (and install pyensmallen) + - name: run pytest (for python < 3.13 with pip install pyensmallen) shell: bash -l {0} if: runner.os == 'Linux' && matrix.python-version < '3.13' run: | diff --git a/docs/source/algorithms.md b/docs/source/algorithms.md index 4ec46b6b5..eb812d470 100644 --- a/docs/source/algorithms.md +++ b/docs/source/algorithms.md @@ -3913,6 +3913,34 @@ addition to optimagic when using an NLOPT algorithm. To install nlopt run 10 * (number of parameters + 1). ``` +## Optimizers from the Ensmallen C++ library + +```{eval-rst} +.. dropdown:: ensmallen_lbfgs + + .. code-block:: + + "ensmallen_lbfgs" + + Minimize a scalar function using the “LBFGS” algorithm. + + L-BFGS is an optimization algorithm in the family of quasi-Newton methods that approximates the Broyden-Fletcher-Goldfarb-Shanno (BFGS) algorithm using a limited amount of computer memory. + + Detailed description of the algorithm is given in :cite:`Matthies1979`. + + - **limited_memory_max_history** (int): Number of memory points to be stored. default is 10. + - **stopping.maxiter** (int): Maximum number of iterations for the optimization (0 means no limit and may run indefinitely). + - **armijo_constant** (float): Controls the accuracy of the line search routine for determining the Armijo condition. default is 1e-4. + - **wolfe_condition** (float): Parameter for detecting the Wolfe condition. default is 0.9. + - **convergence.gtol_abs** (float): Stop when the absolute gradient norm is smaller than this. + - **convergence.ftol_rel** (float): Stop when the relative improvement between two iterations is below this. + - **max_line_search_trials** (int): The maximum number of trials for the line search (before giving up). default is 50. + - **min_step_for_line_search** (float): The minimum step of the line search. default is 1e-20. + - **max_step_for_line_search** (float): The maximum step of the line search. default is 1e20. + + +``` + ## References ```{eval-rst} diff --git a/docs/source/refs.bib b/docs/source/refs.bib index bdae3f4f2..a7880afd2 100644 --- a/docs/source/refs.bib +++ b/docs/source/refs.bib @@ -893,4 +893,16 @@ @book{Conn2009 URL = {https://epubs.siam.org/doi/abs/10.1137/1.9780898718768}, } +@article{Matthies1979, + author = {H. Matthies and G. Strang}, + title = {The Solution of Nonlinear Finite Element Equations}, + journal = {International Journal for Numerical Methods in Engineering}, + volume = {14}, + number = {11}, + pages = {1613-1626}, + year = {1979}, + doi = {10.1002/nme.1620141104} +} + + @Comment{jabref-meta: databaseType:bibtex;} diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py index e00fe590b..3a7765378 100644 --- a/src/optimagic/optimizers/pyensmallen_optimizers.py +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -78,8 +78,9 @@ def _solve_internal_problem( def objective_function( x: NDArray[np.float64], grad: NDArray[np.float64] ) -> np.float64: - grad[:] = problem.jac(x) - return np.float64(problem.fun(x)) + fun, jac = problem.fun_and_jac(x) + grad[:] = jac + return np.float64(fun) raw = optimizer.optimize(objective_function, x0) From fed0f2d32d522bda7432c5007520c45dddfaf874 Mon Sep 17 00:00:00 2001 From: gaurav Date: Tue, 18 Mar 2025 23:47:30 +0530 Subject: [PATCH 07/13] add test --- .../optimizers/test_pyensmallen_optimizers.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/optimagic/optimizers/test_pyensmallen_optimizers.py diff --git a/tests/optimagic/optimizers/test_pyensmallen_optimizers.py b/tests/optimagic/optimizers/test_pyensmallen_optimizers.py new file mode 100644 index 000000000..7a6bdfd17 --- /dev/null +++ b/tests/optimagic/optimizers/test_pyensmallen_optimizers.py @@ -0,0 +1,22 @@ +"""Tests for pyensmallen optimizers.""" + +import numpy as np +import pytest + +import optimagic as om +from optimagic.config import IS_PYENSMALLEN_INSTALLED +from optimagic.optimization.optimize import minimize + + +@pytest.mark.skipif(not IS_PYENSMALLEN_INSTALLED, reason="pyensmallen not installed.") +def test_stop_after_one_iteration(): + algo = om.algos.ensmallen_lbfgs(stopping_maxiter=1) + expected = np.array([0, 0.81742581, 1.63485163, 2.45227744, 3.26970326]) + res = minimize( + fun=lambda x: x @ x, + fun_and_jac=lambda x: (x @ x, 2 * x), + params=np.arange(5), + algorithm=algo, + ) + + assert np.allclose(res.x, expected) From 22104fadc106eb26a0211846f00b3f316026415e Mon Sep 17 00:00:00 2001 From: gaurav Date: Wed, 19 Mar 2025 14:38:06 +0530 Subject: [PATCH 08/13] rename --- docs/source/algorithms.md | 2 +- .../optimizers/pyensmallen_optimizers.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/source/algorithms.md b/docs/source/algorithms.md index eb812d470..da5f21bee 100644 --- a/docs/source/algorithms.md +++ b/docs/source/algorithms.md @@ -3928,7 +3928,7 @@ addition to optimagic when using an NLOPT algorithm. To install nlopt run Detailed description of the algorithm is given in :cite:`Matthies1979`. - - **limited_memory_max_history** (int): Number of memory points to be stored. default is 10. + - **limited_memory_storage_length** (int): Maximum number of saved gradients used to approximate the hessian matrix.. - **stopping.maxiter** (int): Maximum number of iterations for the optimization (0 means no limit and may run indefinitely). - **armijo_constant** (float): Controls the accuracy of the line search routine for determining the Armijo condition. default is 1e-4. - **wolfe_condition** (float): Parameter for detecting the Wolfe condition. default is 0.9. diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py index 3a7765378..db06f5d5a 100644 --- a/src/optimagic/optimizers/pyensmallen_optimizers.py +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -10,6 +10,7 @@ from optimagic.optimization.algo_options import ( CONVERGENCE_FTOL_REL, CONVERGENCE_GTOL_ABS, + LIMITED_MEMORY_STORAGE_LENGTH, MAX_LINE_SEARCH_STEPS, STOPPING_MAXITER, ) @@ -22,8 +23,6 @@ if IS_PYENSMALLEN_INSTALLED: import pyensmallen as pye -LIMITED_MEMORY_MAX_HISTORY = 10 -"""Number of memory points to be stored (default 10).""" MIN_LINE_SEARCH_STEPS = 1e-20 """The minimum step of the line search.""" MAX_LINE_SEARCH_TRIALS = 50 @@ -34,6 +33,15 @@ WOLFE_CONDITION = 0.9 """Parameter for detecting the Wolfe condition.""" +STEP_SIZE = 0.001 +"""Step size for each iteration.""" +BATCH_SIZE = 32 +"""Step size for each iteration.""" +EXP_DECAY_RATE_FOR_FIRST_MOMENT = 0.9 +"""Exponential decay rate for the first moment estimates.""" +EXP_DECAY_RATE_FOR_WEIGHTED_INF_NORM = 0.999 +"""Exponential decay rate for the first moment estimates.""" + @mark.minimizer( name="ensmallen_lbfgs", @@ -50,7 +58,7 @@ ) @dataclass(frozen=True) class EnsmallenLBFGS(Algorithm): - limited_memory_max_history: PositiveInt = LIMITED_MEMORY_MAX_HISTORY + limited_memory_storage_length: PositiveInt = LIMITED_MEMORY_STORAGE_LENGTH stopping_maxiter: PositiveInt = STOPPING_MAXITER armijo_constant: NonNegativeFloat = ARMIJO_CONSTANT # needs review wolfe_condition: NonNegativeFloat = WOLFE_CONDITION # needs review @@ -64,7 +72,7 @@ def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] ) -> InternalOptimizeResult: optimizer = pye.L_BFGS( - numBasis=self.limited_memory_max_history, + numBasis=self.limited_memory_storage_length, maxIterations=self.stopping_maxiter, armijoConstant=self.armijo_constant, wolfe=self.wolfe_condition, From 86f022268371b28cfd6298e7a07747dfadab5def Mon Sep 17 00:00:00 2001 From: gauravmanmode Date: Wed, 14 May 2025 16:25:24 +0530 Subject: [PATCH 09/13] Update pyensmallen_optimizers.py --- src/optimagic/optimizers/pyensmallen_optimizers.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py index db06f5d5a..b5582bf38 100644 --- a/src/optimagic/optimizers/pyensmallen_optimizers.py +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -1,6 +1,7 @@ """Implement ensmallen optimizers.""" from dataclasses import dataclass +from typing import Any import numpy as np from numpy.typing import NDArray @@ -90,11 +91,17 @@ def objective_function( grad[:] = jac return np.float64(fun) - raw = optimizer.optimize(objective_function, x0) + # Passing a Report class to the optimizer allows us to retrieve additional info + ens_res: dict[str, Any] = dict() + report = pye.Report(resultIn=ens_res, disableOutput=True) + best_x = optimizer.optimize(objective_function, x0, report) res = InternalOptimizeResult( - x=raw, # only best x is available - fun=problem.fun(raw), # best f(x) value is not available + x=best_x, + fun=ens_res["objective_value"], + n_iterations=ens_res["iterations"], + n_fun_evals=ens_res["evaluate_calls"], + n_jac_evals=ens_res["gradient_calls"], ) return res From 9069104b58f49f388ad4121f47ce1696b01e80c2 Mon Sep 17 00:00:00 2001 From: gaurav Date: Tue, 12 Aug 2025 16:49:21 +0530 Subject: [PATCH 10/13] clean commit history --- CHANGES.md | 63 + README.md | 1 - docs/rtd_environment.yml | 10 +- docs/source/_static/js/require.js | 5 + docs/source/algorithms.md | 884 ++++++++++++- docs/source/conf.py | 67 +- docs/source/development/code_of_conduct.md | 8 +- .../cluster_robust_likelihood_inference.md | 2 +- .../estimagic/tutorials/msm_overview.ipynb | 5 +- .../source/explanation/internal_optimizers.md | 4 +- .../source/how_to/how_to_add_optimizers.ipynb | 3 +- .../how_to/how_to_algorithm_selection.ipynb | 23 +- docs/source/how_to/how_to_benchmarking.ipynb | 12 +- docs/source/how_to/how_to_bounds.ipynb | 71 +- docs/source/how_to/how_to_derivatives.ipynb | 5 +- .../how_to/how_to_document_optimizers.md | 254 ++++ .../how_to_errors_during_optimization.ipynb | 5 +- docs/source/how_to/how_to_logging.ipynb | 7 +- docs/source/how_to/how_to_multistart.ipynb | 7 +- docs/source/how_to/how_to_slice_plot.ipynb | 11 +- .../how_to/how_to_visualize_histories.ipynb | 15 +- docs/source/how_to/index.md | 1 + docs/source/reference/algo_options.md | 2 +- docs/source/reference/index.md | 1 + docs/source/reference/typing.md | 10 + docs/source/refs.bib | 175 +++ .../tutorials/optimization_overview.ipynb | 9 +- environment.yml | 9 +- pyproject.toml | 7 +- src/optimagic/algorithms.py | 465 +++++++ src/optimagic/batch_evaluators.py | 15 +- src/optimagic/config.py | 98 +- src/optimagic/deprecations.py | 13 +- src/optimagic/differentiation/derivatives.py | 6 +- .../differentiation/generate_steps.py | 19 +- .../differentiation/numdiff_options.py | 22 +- src/optimagic/examples/criterion_functions.py | 15 +- src/optimagic/exceptions.py | 4 + src/optimagic/logging/logger.py | 4 +- src/optimagic/mark.py | 9 + src/optimagic/optimization/algo_options.py | 13 + src/optimagic/optimization/algorithm.py | 6 + src/optimagic/optimization/history.py | 4 +- .../internal_optimization_problem.py | 126 +- src/optimagic/optimization/multistart.py | 43 +- .../optimization/multistart_options.py | 22 +- src/optimagic/optimization/optimize.py | 35 + src/optimagic/optimization/process_results.py | 3 + src/optimagic/optimizers/_pounders/bntr.py | 6 +- .../_pounders/pounders_auxiliary.py | 10 +- .../optimizers/bayesian_optimizer.py | 353 ++++++ src/optimagic/optimizers/bhhh.py | 2 + src/optimagic/optimizers/fides.py | 12 +- src/optimagic/optimizers/iminuit_migrad.py | 146 +++ src/optimagic/optimizers/ipopt.py | 14 +- src/optimagic/optimizers/nag_optimizers.py | 14 +- src/optimagic/optimizers/neldermead.py | 2 + .../optimizers/nevergrad_optimizers.py | 1089 +++++++++++++++++ src/optimagic/optimizers/nlopt_optimizers.py | 32 + src/optimagic/optimizers/pounders.py | 8 +- .../optimizers/pyensmallen_optimizers.py | 3 +- src/optimagic/optimizers/pygmo_optimizers.py | 44 +- src/optimagic/optimizers/scipy_optimizers.py | 160 ++- src/optimagic/optimizers/tao_optimizers.py | 28 +- src/optimagic/optimizers/tranquilo.py | 25 +- src/optimagic/parameters/bounds.py | 55 +- .../parameters/consolidate_constraints.py | 16 +- src/optimagic/parameters/conversion.py | 9 +- .../parameters/nonlinear_constraints.py | 22 + .../parameters/process_constraints.py | 4 +- src/optimagic/parameters/scale_conversion.py | 23 +- src/optimagic/parameters/space_conversion.py | 15 +- src/optimagic/parameters/tree_conversion.py | 4 +- src/optimagic/typing.py | 17 +- src/optimagic/utilities.py | 13 + src/optimagic/visualization/history_plots.py | 580 +++++---- .../visualization/plotting_utilities.py | 44 +- .../differentiation/test_numdiff_options.py | 2 +- .../optimagic/optimization/test_algorithm.py | 4 + .../optimization/test_history_collection.py | 10 +- .../test_infinite_and_incomplete_bounds.py | 27 + .../test_invalid_jacobian_value.py | 114 ++ .../optimization/test_many_algorithms.py | 119 +- .../optimagic/optimization/test_multistart.py | 4 +- .../optimization/test_with_multistart.py | 13 + .../optimizers/test_bayesian_optimizer.py | 145 +++ .../optimizers/test_iminuit_migrad.py | 95 ++ tests/optimagic/optimizers/test_nevergrad.py | 135 ++ tests/optimagic/parameters/test_bounds.py | 33 +- tests/optimagic/parameters/test_conversion.py | 4 +- .../parameters/test_tree_conversion.py | 6 +- tests/optimagic/test_mark.py | 2 + .../visualization/test_history_plots.py | 109 +- .../visualization/test_plotting_utilities.py | 50 + 94 files changed, 5608 insertions(+), 612 deletions(-) create mode 100644 docs/source/_static/js/require.js create mode 100644 docs/source/how_to/how_to_document_optimizers.md create mode 100644 docs/source/reference/typing.md create mode 100644 src/optimagic/optimizers/bayesian_optimizer.py create mode 100644 src/optimagic/optimizers/iminuit_migrad.py create mode 100644 src/optimagic/optimizers/nevergrad_optimizers.py create mode 100644 tests/optimagic/optimization/test_infinite_and_incomplete_bounds.py create mode 100644 tests/optimagic/optimization/test_invalid_jacobian_value.py create mode 100644 tests/optimagic/optimizers/test_bayesian_optimizer.py create mode 100644 tests/optimagic/optimizers/test_iminuit_migrad.py create mode 100644 tests/optimagic/optimizers/test_nevergrad.py create mode 100644 tests/optimagic/visualization/test_plotting_utilities.py diff --git a/CHANGES.md b/CHANGES.md index 0aa867272..134f0d242 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,69 @@ chronological order. We follow [semantic versioning](https://semver.org/) and al releases are available on [Anaconda.org](https://anaconda.org/optimagic-dev/optimagic). +## 0.5.2 + +This minor release adds support for two additional optimizer libraries: + +- [Nevergrad](https://github.com/facebookresearch/nevergrad): A library for + gradient-free optimization developed by Facebook Research. +- [Bayesian + Optimization](https://github.com/bayesian-optimization/BayesianOptimization): A + library for constrained bayesian global optimization with Gaussian processes. + +In addition, this release includes several bug fixes and improvements to the +documentation. Many contributions in this release were made by Google Summer of Code +(GSoC) 2025 applicants, with @gauravmanmode and @spline2hg being the accepted +contributors. + +- {gh}`620` Uses interactive plotly figures in documentation ({ghuser}`timmens`). +- {gh}`618` Improves bounds processing when no bounds are specified ({ghuser}`timmens`). +- {gh}`615` Adds pre-commit hook that checks mypy version consistency ({ghuser}`timmens`). +- {gh}`613` Exposes converter functionality ({ghuser}`spline2hg`). +- {gh}`612` Fixes results processing to work with new cobyla optimizer ({ghuser}`janosg`). +- {gh}`610` Adds `needs_bounds` and `supports_infinite_bounds` fields to algorithm info ({ghuser}`gauravmanmode`). +- {gh}`608` Adds support for plotly >= 6 ({ghuser}`hmgaudecker`, {ghuser}`timmens`). +- {gh}`607` Returns `run_explorations` results in a dataclass ({ghuser}`r3kste`). +- {gh}`605` Enhances batch evaluator checking and processing, introduces the internal + `BatchEvaluatorLiteral` literal, and updates CHANGES.md ({ghuser}`janosg`, + {ghuser}`timmens`). +- {gh}`602` Adds optimizer wrapper for bayesian-optimization package ({ghuser}`spline2hg`). +- {gh}`601` Updates pre-commit hooks and fixes mypy issues ({ghuser}`janosg`). +- {gh}`598` Fixes and adds links to GitHub in the documentation ({ghuser}`hamogu`). +- {gh}`594` Refines newly added optimizer wrappers ({ghuser}`janosg`). +- {gh}`591` Adds multiple optimizers from the nevergrad package ({ghuser}`gauravmanmode`). +- {gh}`589` Rewrites the algorithm selection pre-commit hook in pure Python to address + issues with bash scripts on Windows ({ghuser}`timmens`). +- {gh}`586` and {gh}`592` Ensure the SciPy `disp` parameter is exposed for the following + SciPy algorithms: slsqp, neldermead, powell, conjugate_gradient, newton_cg, cobyla, + truncated_newton, trust_constr ({ghuser}`sefmef`, {ghuser}`TimBerti`). +- {gh}`585` Exposes all parameters of [SciPy's + BFGS](https://docs.scipy.org/doc/scipy/reference/optimize.minimize-bfgs.html) + optimizer in optimagic ({ghuser}`TimBerti`). +- {gh}`582` Adds support for handling infinite gradients during optimization + ({ghuser}`Aziz-Shameem`). +- {gh}`579` Implements a wrapper for the PSO optimizer from the + [nevergrad](https://github.com/facebookresearch/nevergrad) package ({ghuser}`r3kste`). +- {gh}`578` Integrates the `intersphinx-registry` package into the documentation for + automatic linking to up-to-date external documentation + ({ghuser}`Schefflera-Arboricola`). +- {gh}`576` Wraps oneplusone optimizer from nevergrad ({ghuser}`gauravmanmode`, {ghuser}`gulshan-123`). +- {gh}`572` and {gh}`573` Fix bugs in error handling for parameter selector processing + and constraints checking ({ghuser}`hmgaudecker`). +- {gh}`570` Adds a how-to guide for adding algorithms to optimagic and improves internal + documentation ({ghuser}`janosg`). +- {gh}`569` Implements a threading batch evaluator ({ghuser}`spline2hg`). +- {gh}`568` Introduces an initial wrapper for the migrad optimizer from the + [iminuit](https://github.com/scikit-hep/iminuit) package ({ghuser}`spline2hg`). +- {gh}`567` Makes the `fun` argument optional when `fun_and_jac` is provided + ({ghuser}`gauravmanmode`). +- {gh}`563` Fixes a bug in input harmonization for history plotting + ({ghuser}`gauravmanmode`). +- {gh}`552` Refactors and extends the `History` class, removing the internal + `HistoryArrays` class ({ghuser}`timmens`). +- {gh}`485` Adds bootstrap weights functionality ({ghuser}`alanlujan91`). + + ## 0.5.1 This is a minor release that introduces the new algorithm selection tool and several diff --git a/README.md b/README.md index 58932dec3..aa101dfeb 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,6 @@ [![image](https://codecov.io/gh/optimagic-dev/optimagic/branch/main/graph/badge.svg)](https://codecov.io/gh/optimagic-dev/optimagic) [![image](https://results.pre-commit.ci/badge/github/optimagic-dev/optimagic/main.svg)](https://results.pre-commit.ci/latest/github/optimagic-dev/optimagic/main) [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) -[![image](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) [![image](https://pepy.tech/badge/optimagic/month)](https://pepy.tech/project/optimagic) [![image](https://img.shields.io/badge/NumFOCUS-affiliated%20project-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](https://numfocus.org/sponsored-projects/affiliated-projects) [![image](https://img.shields.io/twitter/follow/aiidateam.svg?style=social&label=Follow)](https://x.com/optimagic) diff --git a/docs/rtd_environment.yml b/docs/rtd_environment.yml index feea4b89a..c7f941372 100644 --- a/docs/rtd_environment.yml +++ b/docs/rtd_environment.yml @@ -4,16 +4,15 @@ channels: - conda-forge - nodefaults dependencies: - - python=3.11 + - python=3.12 - typing-extensions - pip - setuptools_scm - toml - - sphinx + - sphinx>=8.2.3 - sphinxcontrib-bibtex - sphinx-copybutton - sphinx-design - - sphinx-panels - ipython - ipython_genutils - myst-nb @@ -26,13 +25,12 @@ dependencies: - scipy - patsy - joblib - - plotly + - plotly>=6.2 - nlopt - annotated-types - pygmo>=2.19.0 - pip: - - ../ - - kaleido + - -e ../ - Py-BOBYQA - DFO-LS - pandas-stubs # dev, tests diff --git a/docs/source/_static/js/require.js b/docs/source/_static/js/require.js new file mode 100644 index 000000000..f4828a405 --- /dev/null +++ b/docs/source/_static/js/require.js @@ -0,0 +1,5 @@ +/** vim: et:ts=4:sw=4:sts=4 + * @license RequireJS 2.3.7 Copyright jQuery Foundation and other contributors. + * Released under MIT license, https://github.com/requirejs/requirejs/blob/master/LICENSE + */ +var requirejs,require,define;!function(global,setTimeout){var req,s,head,baseElement,dataMain,src,interactiveScript,currentlyAddingScript,mainScript,subPath,version="2.3.7",commentRegExp=/\/\*[\s\S]*?\*\/|([^:"'=]|^)\/\/.*$/gm,cjsRequireRegExp=/[^.]\s*require\s*\(\s*["']([^'"\s]+)["']\s*\)/g,jsSuffixRegExp=/\.js$/,currDirRegExp=/^\.\//,op=Object.prototype,ostring=op.toString,hasOwn=op.hasOwnProperty,isBrowser=!("undefined"==typeof window||"undefined"==typeof navigator||!window.document),isWebWorker=!isBrowser&&"undefined"!=typeof importScripts,readyRegExp=isBrowser&&"PLAYSTATION 3"===navigator.platform?/^complete$/:/^(complete|loaded)$/,defContextName="_",isOpera="undefined"!=typeof opera&&"[object Opera]"===opera.toString(),contexts={},cfg={},globalDefQueue=[],useInteractive=!1,disallowedProps=["__proto__","constructor"];function commentReplace(e,t){return t||""}function isFunction(e){return"[object Function]"===ostring.call(e)}function isArray(e){return"[object Array]"===ostring.call(e)}function each(e,t){if(e)for(var i=0;i`_ is + the workhorse algorithm of the MINUIT optimization suite, which has been widely used in the + high-energy physics community since 1975. The IMINUIT package is a Python interface to the + Minuit2 C++ library developed by CERN. + + Migrad uses a quasi-Newton method, updating the Hessian matrix iteratively + to guide the optimization. The algorithm adapts dynamically to challenging landscapes + using several key techniques: + + - **Quasi-Newton updates**: The Hessian is updated iteratively rather than recalculated at + each step, improving efficiency. + - **Steepest descent fallback**: When the Hessian update fails, Migrad falls back to steepest + descent with line search. + - **Box constraints handling**: Parameters with bounds are transformed internally to ensure + they remain within allowed limits. + - **Heuristics for numerical stability**: Special cases such as flat gradients or singular + Hessians are managed using pre-defined heuristics. + - **Stopping criteria based on Estimated Distance to Minimum (EDM)**: The optimization halts + when the predicted improvement becomes sufficiently small. + + For details see :cite:`JAMES1975343`. + + **Optimizer Parameters:** + + - **stopping.maxfun** (int): Maximum number of function evaluations. If reached, the optimization stops + but this is not counted as successful convergence. Function evaluations used for numerical gradient + calculations do not count toward this limit. Default is 1,000,000. + + - **n_restarts** (int): Number of times to restart the optimizer if convergence is not reached. + + - A value of 1 (the default) indicates that the optimizer will only run once, disabling the restart feature. + - Values greater than 1 specify the maximum number of restart attempts. +``` + +## Nevergrad Optimizers + +optimagic supports following algorithms from the +[Nevergrad](https://facebookresearch.github.io/nevergrad/index.html) library. To use +these optimizers, you need to have +[the nevergrad package](https://github.com/facebookresearch/nevergrad) installed. +(`pip install nevergrad`).\ +Two algorithms from nevergrad are not available in optimagic.\ +`SPSA (Simultaneous Perturbation Stochastic Approximation)` - This is WIP in nevergrad +and hence imprecise.\ +`AXP (AX-platfofm)` - Very slow and not recommended. + +```{eval-rst} +.. dropdown:: nevergrad_pso + + .. code-block:: + + "nevergrad_pso" + + Minimize a scalar function using the Particle Swarm Optimization algorithm. + + The Particle Swarm Optimization algorithm was originally proposed by :cite:`Kennedy1995`.The + implementation in Nevergrad is based on :cite:`Zambrano2013`. + + PSO solves an optimization problem by evolving a swarm of particles (candidate solutions) across the + search space. Each particle adjusts its position based on its own experience (cognitive component) + and the experiences of its neighbors or the swarm (social component), using velocity updates. The + algorithm iteratively guides the swarm toward promising regions of the search space. + + - **transform** (str): The transform used to map from PSO optimization space to real space. Options: + - "arctan" (default) + - "identity" + - "gaussian" + - **population\_size** (int): The number of particles in the swarm. + - **n\_cores** (int): The number of CPU cores to use for parallel computation. + - **seed** (int, optional): Random seed for reproducibility. + - **stopping\_maxfun** (int, optional): Maximum number of function evaluations. + - **inertia** (float): + Inertia weight ω. Controls the influence of a particle's previous velocity. Must be less than 1 to + avoid divergence. Default is 0.7213475204444817. + - **cognitive** (float): + Cognitive coefficient :math:`\phi_p`. Controls the influence of a particle’s own best known + position. Typical values: 1.0 to 3.0. Default is 1.1931471805599454. + - **social** (float): + Social coefficient. Denoted by :math:`\phi_g`. Controls the influence of the swarm’s best known + position. Typical values: 1.0 to 3.0. Default is 1.1931471805599454. + - **quasi\_opp\_init** (bool): Whether to use quasi-opposition initialization. Default is False. + - **speed\_quasi\_opp\_init** (bool): + Whether to apply quasi-opposition initialization to speed. Default is False. + - **special\_speed\_quasi\_opp\_init** (bool): + Whether to use special quasi-opposition initialization for speed. Default is False. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_cmaes + + .. code-block:: + + "nevergrad_cmaes" + + Minimize a scalar function using the Covariance Matrix Adaptation Evolution Strategy (CMA-ES) + algorithm. + + The CMA-ES (Covariance Matrix Adaptation Evolution Strategy) is a state-of-the-art evolutionary + algorithm designed for difficult non-linear, non-convex, black-box optimization problems in + continuous domains. It is typically applied to unconstrained or bounded optimization problems with + dimensionality between 3 and 100. CMA-ES adapts a multivariate normal distribution to approximate + the shape of the objective function. It estimates a positive-definite covariance matrix, akin to the + inverse Hessian in convex-quadratic problems, but without requiring derivatives or their + approximation. Original paper can be accessed at `cma `_. This + implementation is a python wrapper over the original code `pycma `_. + + - **scale**: Scale of the search. + - **elitist**: + Whether to switch to elitist mode (also known as (μ,λ)-CMA-ES). In elitist mode, the best point in + the population is always retained. + - **population\_size**: Population size. + - **diagonal**: Use the diagonal version of CMA, which is more efficient for high-dimensional problems. + - **high\_speed**: Use a metamodel for recommendation to speed up optimization. + - **fast\_cmaes**: + Use the fast CMA-ES implementation. Cannot be used with diagonal=True. Produces equivalent results + and is preferable for high dimensions or when objective function evaluations are fast. + - **random\_init**: If True, initialize the optimizer with random parameters. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **step\_size\_adaptive**: + Whether to adapt the step size. Can be a boolean or a string specifying the adaptation strategy. + - **CSA\_dampfac**: Damping factor for step size adaptation. + - **CMA\_dampsvec\_fade**: Damping rate for step size adaptation. + - **CSA\_squared**: Whether to use squared step sizes in updates. + - **CMA\_on**: Learning rate for the covariance matrix update. + - **CMA\_rankone**: Multiplier for the rank-one update learning rate of the covariance matrix. + - **CMA\_rankmu**: Multiplier for the rank-mu update learning rate of the covariance matrix. + - **CMA\_cmean**: Learning rate for the mean update. + - **CMA\_diagonal\_decoding**: Learning rate for the diagonal update. + - **num\_parents**: Number of parents (μ) for recombination. + - **CMA\_active**: Whether to use negative updates for the covariance matrix. + - **CMA\_mirrormethod**: Strategy for mirror sampling. Possible values are: + - **0**: Unconditional mirroring + - **1**: Selective mirroring + - **2**: Selective mirroring with delay (default) + - **CMA\_const\_trace**: How to normalize the trace of the covariance matrix. Valid values are: + - False: No normalization + - True: Normalize to 1 + - "arithm": Arithmetic mean normalization + - "geom": Geometric mean normalization + - "aeig": Arithmetic mean of eigenvalues + - "geig": Geometric mean of eigenvalues + - **CMA\_diagonal**: + Number of iterations to use diagonal covariance matrix before switching to full matrix. If False, + always use full matrix. + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **stopping\_maxiter**: Maximum number of iterations before termination. + - **stopping\_timeout**: Maximum time in seconds before termination. + - **stopping\_cov\_mat\_cond**: Maximum condition number of the covariance matrix before termination. + - **convergence\_ftol\_abs**: Absolute tolerance on function value changes for convergence. + - **convergence\_ftol\_rel**: Relative tolerance on function value changes for convergence. + - **convergence\_xtol\_abs**: Absolute tolerance on parameter changes for convergence. + - **convergence\_iter\_noimprove**: Number of iterations without improvement before termination. + - **invariant\_path**: Whether evolution path (pc) should be invariant to transformations. + - **eval\_final\_mean**: Whether to evaluate the final mean solution. + - **seed**: Seed used by the internal random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_oneplusone + + .. code-block:: + + "nevergrad_oneplusone" + + Minimize a scalar function using the One Plus One Evolutionary algorithm from Nevergrad. + + THe One Plus One evolutionary algorithm iterates to find a set of parameters that minimizes the loss + function. It does this by perturbing, or mutating, the parameters from the last iteration (the + parent). If the new (child) parameters yield a better result, then the child becomes the new parent + whose parameters are perturbed, perhaps more aggressively. If the parent yields a better result, it + remains the parent and the next perturbation is less aggressive. Originally proposed by + :cite:`Rechenberg1973`. The implementation in Nevergrad is based on the one-fifth adaptation rule, + going back to :cite:`Schumer1968. + + - **noise\_handling**: Method for handling the noise, can be + - "random": A random point is reevaluated regularly using the one-fifth adaptation rule. + - "optimistic": The best optimistic point is reevaluated regularly, embracing optimism in the face of uncertainty. + - A float coefficient can be provided to tune the regularity of these reevaluations (default is 0.05). Eg: with 0.05, each evaluation has a 5% chance (i.e., 1 in 20) of being repeated (i.e., the same candidate solution is reevaluated to better estimate its performance). (Default: `None`). + - **n\_cores**: Number of cores to use. + + stopping.maxfun: Maximum number of function evaluations. + - **mutation**: Type of mutation to apply. Available options are (Default: `"gaussian"`). + - "gaussian": Standard mutation by adding a Gaussian random variable (with progressive widening) to the best pessimistic point. + - "cauchy": Same as Gaussian but using a Cauchy distribution. + - "discrete": Mutates a randomly drawn variable (mutation occurs with probability 1/d in d dimensions, hence ~1 variable per mutation). + - "discreteBSO": Follows brainstorm optimization by gradually decreasing mutation rate from 1 to 1/d. + - "fastga": Fast Genetic Algorithm mutations from the current best. + - "doublefastga": Double-FastGA mutations from the current best :cite:`doerr2017`. + - "rls": Randomized Local Search — mutates one and only one variable. + - "portfolio": Random number of mutated bits, known as uniform mixing :cite:`dang2016`. + - "lengler": Mutation rate is a function of dimension and iteration index. + - "lengler{2|3|half|fourth}": Variants of the Lengler mutation rate adaptation. + - **sparse**: Whether to apply random mutations that set variables to zero. Default is `False`. + - **smoother**: Whether to suggest smooth mutations. Default is `False`. + - **annealing**: + Annealing schedule to apply to mutation amplitude or temperature-based control. Options are: + - "none": No annealing is applied. + - "Exp0.9": Exponential decay with rate 0.9. + - "Exp0.99": Exponential decay with rate 0.99. + - "Exp0.9Auto": Exponential decay with rate 0.9, auto-scaled based on problem horizon. + - "Lin100.0": Linear decay from 1 to 0 over 100 iterations. + - "Lin1.0": Linear decay from 1 to 0 over 1 iteration. + - "LinAuto": Linearly decaying annealing automatically scaled to the problem horizon. Default is `"none"`. + - **super\_radii**: + Whether to apply extended radii beyond standard bounds for candidate generation, enabling broader + exploration. Default is `False`. + - **roulette\_size**: + Size of the roulette wheel used for selection in the evolutionary process. Affects the sampling + diversity from past candidates. (Default: `64`) + - **antismooth**: + Degree of anti-smoothing applied to prevent premature convergence in smooth landscapes. This alters + the landscape by penalizing overly smooth improvements. (Default: `4`) + - **crossover**: Whether to include a genetic crossover step every other iteration. Default is `False`. + - **crossover\_type**: + Method used for genetic crossover between individuals in the population. Available options (Default: `"none"`): + - "none": No crossover is applied. + - "rand": Randomized selection of crossover point. + - "max": Crossover at the point with maximum fitness gain. + - "min": Crossover at the point with minimum fitness gain. + - "onepoint": One-point crossover, splitting the genome at a single random point. + - "twopoint": Two-point crossover, splitting the genome at two points and exchanging the middle section. + - **tabu\_length**: + Length of the tabu list used to prevent revisiting recently evaluated candidates in local search + strategies. Helps in escaping local minima. (Default: `1000`) + - **rotation**: + Whether to apply rotational transformations to the search space, promoting invariance to axis- + aligned structures and enhancing search performance in rotated coordinate systems. (Default: + `False`) + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` ```{eval-rst} +.. dropdown:: nevergrad_de + + .. code-block:: + + "nevergrad_de" + + Minimize a scalar function using the Differential Evolution optimizer from Nevergrad. + + Differential Evolution is typically used for continuous optimization. It uses differences between + points in the population for performing mutations in fruitful directions; it is therefore a kind of + covariance adaptation without any explicit covariance, making it very fast in high dimensions. + + - **initialization**: + Algorithm/distribution used for initialization. Can be one of: "parametrization" (uses + parametrization's sample method), "LHS" (Latin Hypercube Sampling), "QR" (Quasi-Random), "QO" + (Quasi-Orthogonal), or "SO" (Sobol sequence). + - **scale**: Scale of random component of updates. Can be a float or a string. + - **recommendation**: Criterion for selecting the best point to recommend. + - **Options**: "pessimistic", "optimistic", "mean", or "noisy". + - **crossover**: Crossover rate or strategy. Can be: + - float: Fixed crossover rate + - "dimension": 1/dimension + - "random": Random uniform rate per iteration + - "onepoint": One-point crossover + - "twopoints": Two-points crossover + - "rotated_twopoints": Rotated two-points crossover + - "parametrization": Use parametrization's recombine method + - **F1**: Differential weight #1 (scaling factor). + - **F2**: Differential weight #2 (scaling factor). + - **popsize**: Population size. Can be an integer or one of: + - "standard": max(num_workers, 30) + - "dimension": max(num_workers, 30, dimension + 1) + - "large": max(num_workers, 30, 7 * dimension) + - **high\_speed**: If True, uses a metamodel for recommendations to speed up optimization. + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_bo + + .. code-block:: + + "nevergrad_bo" + + Minimize a scalar function using the Bayes Optim algorithm. BO and PCA-BO algorithms from the + `bayes_optim `_ package PCA-BO (Principal + Component Analysis for Bayesian Optimization) is a dimensionality reduction technique for black-box + optimization. It applies PCA to the input space before performing Bayesian optimization, improving + efficiency in high dimensions by focusing on directions of greatest variance. This helps concentrate + search in informative subspaces and reduce sample complexity. :cite:`bayesoptimimpl`. + + - **init\_budget**: Number of initialization algorithm steps. + - **pca**: Whether to use the PCA transformation, defining PCA-BO rather than standard BO. + - **n\_components**: + Number of principal axes in feature space representing directions of maximum variance in the data. + Represents the percentage of explained variance (e.g., 0.95 means 95% variance retained). + - **prop\_doe\_factor**: + Percentage of the initial budget used for DoE, potentially overriding `init_budget`. For + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_emna + + .. code-block:: + + "nevergrad_emna" + + Minimize a scalar function using the Estimation of Multivariate Normal Algorithm. + + Estimation of Multivariate Normal Algorithm (EMNA), a distribution-based evolutionary algorithm that + models the search space using a multivariate Gaussian. EMNA learns the full covariance matrix of the + Gaussian sampling distribution, resulting in a cubic time complexity w.r.t. each sampling. It is + highly recommended to first attempt other more advanced optimization methods for LBO. See + :cite:`emnaimpl`. This algorithm is quite efficient in a parallel setting, i.e. when the population + size is large. + + - **isotropic**: + If True, uses an isotropic (identity covariance) Gaussian. If False, uses a separable (diagonal + covariance) Gaussian for greater flexibility in anisotropic landscapes. + - **noise\_handling**: + If True, returns the best individual found. If False (recommended for noisy problems), returns the + average of the final population to reduce noise. + - **population\_size\_adaptation**: + If True, the population size is adjusted automatically based on the optimization landscape and noise + level. + - **initial\_popsize**: Initial population size. Default: 4 x dimension.. + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_cga + + .. code-block:: + + "nevergrad_cga" + + Minimize a scalar function using the Compact Genetic Algorithm. + + The Compact Genetic Algorithm (cGA) is a memory-efficient genetic algorithm that represents the + population as a probability vector over gene values. It simulates the order-one behavior of a simple + GA with uniform crossover, updating probabilities instead of maintaining an explicit population. cGA + processes each gene independently and is well-suited for large or constrained environments. For + details see :cite:`cgaimpl`. + + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_eda + + .. code-block:: + + "nevergrad_eda" + + Minimize a scalar function using the Estimation of distribution algorithm. + + Estimation of Distribution Algorithms (EDAs) optimize by building and sampling a probabilistic model + of promising solutions. Instead of using traditional variation operators like crossover or mutation, + EDAs update a distribution based on selected individuals and sample new candidates from it. This + allows efficient exploration of complex or noisy search spaces. In short, EDAs typically do not + directly evolve populations of search points but build probabilistic models of promising solutions + by repeatedly sampling and selecting points from the underlying search space. Refer :cite:`edaimpl`. + + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_tbpsa + + .. code-block:: + + "nevergrad_tbpsa" + + Minimize a scalar function using the Test-based population size adaptation algorithm. + + TBPSA adapts population size based on fitness trend detection using linear regression. If no + significant improvement is found (via hypothesis testing), the population size is increased to + improve robustness in noisy settings. This method performs the best in many noisy optimization + problems, even in large dimensions. For more details, refer :cite:`tbpsaimpl` + + - **noise\_handling**: + If True, returns the best individual seen so far. If False (recommended for noisy problems), returns + the average of the final population to reduce the effect of noise. + - **initial\_popsize**: Initial population size. If not specified, defaults to 4 x dimension. + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_randomsearch + + .. code-block:: + + "nevergrad_randomsearch" + + Minimize a scalar function using the Random Search algorithm. + + This is a one-shot optimization method, provides random suggestions. + + - **middle\_point**: + Enforces that the first suggested point (ask) is the zero vector. i.e we add (0,0,...,0) as a first + point. + - **opposition\_mode**: Symmetrizes exploration with respect to the center. + - "opposite": enables full symmetry by always evaluating mirrored points. + - "quasi": applies randomized symmetry (less strict, more exploratory). + - None: disables any symmetric mirroring in the sampling process. + - **sampler**: + - "parametrization": uses the default sample() method of the parametrization, which samples uniformly within bounds or from a Gaussian. + - "gaussian": samples from a standard Gaussian distribution. + - "cauchy": uses a Cauchy distribution instead of Gaussian. + - **scale**: Scalar used to multiply suggested point values, or a string mode: + - "random": uses a randomized pattern for the scale. + - "auto": sigma = (1 + log(budget)) / (4 * log(dimension)); adjusts scale based on problem size. + - "autotune": sigma = sqrt(log(budget) / dimension); alternative auto-scaling based on budget and dimensionality. + - **recommendation\_rule**: Specifies how the final recommendation is chosen. + - "average_of_best": returns the average of top-performing candidates. + - "pessimistic": selects the pessimistic best (default); + - "average_of_exp_best": uses an exponential moving average of the best points. + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_samplingsearch + + .. code-block:: + + "nevergrad_samplingsearch" + + Minimize a scalar function using SamplingSearch. + + This is a one-shot optimization method, but better than random search by ensuring more uniformity. + + - **sampler**: Choice of the low-discrepancy sampler used for initial points. + - "Halton": deterministic, well-spaced sequences + - "Hammersley": similar to Halton but more uniform in low dimension + - "LHS": Latin Hypercube Sampling; ensures coverage along each axis + - **scrambled**: + If True, Adds scrambling to the search; much better in high dimension and rarely worse than the + original search. + - **middle\_point**: + If True, the first suggested point is the zero vector. Useful for initializing at the center of the + search space. + - **cauchy**: + If True, uses the inverse Cauchy distribution instead of Gaussian when projecting samples to real- + valued space (especially when no box bounds exist). + - **scale**: A float multiplier or "random". + - float: directly scales all generated points + - "random": uses a randomized scaling pattern for increased diversity + - **rescaled**: If True or a specific mode, rescales the sampling pattern. + - Ensures coverage of boundaries and may apply adaptive scaling + - Useful when original scale is too narrow or biased + - **recommendation\_rule**: How the final recommendation is chosen. + - "average_of_best": mean of the best-performing points + - "pessimistic": selects the point with best worst-case value (default) + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. Notes + ----- + - Halton is a low quality sampling method when the dimension is high; it is usually better to use Halton with scrambling. + - When the budget is known in advance, it is also better to replace Halton by Hammersley. +``` + +```{eval-rst} +.. dropdown:: nevergrad_NGOpt + + .. code-block:: + + "nevergrad_NGOpt" + + Minimize a scalar function using a Meta Optimizer from Nevergrad. Each meta optimizer combines + multiples optimizers to solve a problem. + + - **optimizer**: One of + - NGOpt + - NGOpt4 + - NGOpt8 + - NGOpt10 + - NGOpt12 + - NGOpt13 + - NGOpt14 + - NGOpt15 + - NGOpt16 + - NGOpt21 + - NGOpt36 + - NGOpt38 + - NGOpt39 + - NGOptRW + - NGOptF + - NGOptF2 + - NGOptF3 + - NGOptF5 + - NgIoh2 + - NgIoh3 + - NgIoh4 + - NgIoh5 + - NgIoh6 + - NgIoh7 + - NgIoh8 + - NgIoh9 + - NgIoh10 + - NgIoh11 + - NgIoh12 + - NgIoh13 + - NgIoh14 + - NgIoh15 + - NgIoh16 + - NgIoh17 + - NgIoh18 + - NgIoh19 + - NgIoh20 + - NgIoh21 + - NgIoh12b + - NgIoh13b + - NgIoh14b + - NgIoh15b + - NgIohRW2 + - NgIohTuned + - NgDS + - NgDS2 + - NGDSRW + - NGO + - CSEC + - CSEC10 + - CSEC11 + - Wiz + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +```{eval-rst} +.. dropdown:: nevergrad_meta + + .. code-block:: + + "nevergrad_meta" + + Minimize a scalar function using a Meta Optimizer from Nevergrad. Utilizes a combination of local + and global optimizers to find the best solution. Local optimizers like BFGS are wrappers over scipy + implementations. Each meta optimizer combines multiples optimizers to solve a problem. + + - **optimizer**: One of + - MultiBFGSPlus + - LogMultiBFGSPlus + - SqrtMultiBFGSPlus + - MultiCobylaPlus + - MultiSQPPlus + - BFGSCMAPlus + - LogBFGSCMAPlus + - SqrtBFGSCMAPlus + - SQPCMAPlus + - LogSQPCMAPlus + - SqrtSQPCMAPlus + - MultiBFGS + - LogMultiBFGS + - SqrtMultiBFGS + - MultiCobyla + - ForceMultiCobyla + - MultiSQP + - BFGSCMA + - LogBFGSCMA + - SqrtBFGSCMA + - SQPCMA + - LogSQPCMA + - SqrtSQPCMA + - FSQPCMA + - F2SQPCMA + - F3SQPCMA + - MultiDiscrete + - CMandAS2 + - CMandAS3 + - MetaCMA + - CMA + - PCEDA + - MPCEDA + - MEDA + - NoisyBandit + - Shiwa + - Carola3 + - **stopping\_maxfun**: Maximum number of function evaluations before termination. + - **n\_cores**: Number of cores to use for parallel function evaluation. + - **seed**: Seed for the random number generator for reproducibility. + - **sigma**: + Standard deviation for sampling initial population from N(0, σ²) in case bounds are not provided. +``` + +## Bayesian Optimization + +We wrap the +[BayesianOptimization](https://github.com/bayesian-optimization/BayesianOptimization) +package. To use it, you need to have +[bayesian-optimization](https://pypi.org/project/bayesian-optimization/) installed. + +```{eval-rst} +.. dropdown:: bayes_opt + + .. code-block:: + + "bayes_opt" + + Minimize a scalar function using Bayesian Optimization with Gaussian Processes. + + This optimizer wraps the BayesianOptimization package (:cite:`Nogueira2014`), which + implements Bayesian optimization using Gaussian processes to build probabilistic + models of the objective function. Bayesian optimization is particularly effective + for expensive black-box functions where gradient information is not available. + + The algorithm requires finite bounds for all parameters. + + The bayes_opt wrapper preserves the default parameter values from the underlying + BayesianOptimization package where appropriate. + + bayes_opt supports the following options: + + - **init_points** (PositiveInt): Number of random exploration points to evaluate before + starting optimization. Default is 5. + + - **n_iter** (PositiveInt): Number of Bayesian optimization iterations to perform after + the initial random exploration. Default is 25. + + - **verbose** (Literal[0, 1, 2]): Verbosity level from 0 (silent) to 2 (most verbose). Default is 0. + + - **kappa** (NonNegativeFloat): Parameter to balance exploration versus exploitation trade-off + for the Upper Confidence Bound acquisition function. Higher values mean more exploration. + This parameter is only used if the acquisition function is set to "ucb" or "upper_confidence_bound" + and when a configured instance of an AcquisitionFunction object is not passed. Default is 2.576. + + - **xi** (PositiveFloat): Parameter to balance exploration versus exploitation trade-off + for the Expected Improvement or Probability of Improvement acquisition functions. + Higher values mean more exploration. This parameter is only used if the acquisition function + is set to "ei", "expected_improvement", "poi", or "probability_of_improvement" + and when a configured instance of an AcquisitionFunction object is not passed. Default is 0.01. + + - **exploration_decay** (float | None): Rate at which exploration decays over time. + Default is None (no decay). + + - **exploration_decay_delay** (NonNegativeInt | None): Delay for decay. If None, + decay is applied from the start. Default is None. + + - **random_state** (int | None): Random seed for reproducible results. Default is None. + + - **acquisition_function** (str | AcquisitionFunction | Type[AcquisitionFunction] | None): Strategy for selecting + the next evaluation point. Options include: + - "ucb" or "upper_confidence_bound": Upper Confidence Bound + - "ei" or "expected_improvement": Expected Improvement + - "poi" or "probability_of_improvement": Probability of Improvement + Default is None (uses package default). + + - **allow_duplicate_points** (bool): Whether to allow re-evaluation of the same point. + Default is False. + + - **enable_sdr** (bool): Enable Sequential Domain Reduction, which progressively + narrows the search space around promising regions. Default is False. + + - **sdr_gamma_osc** (float): Oscillation parameter for SDR. Default is 0.7. + + - **sdr_gamma_pan** (float): Panning parameter for SDR. Default is 1.0. + + - **sdr_eta** (float): Zooming parameter for SDR. Default is 0.9. + + - **sdr_minimum_window** (NonNegativeFloat): Minimum window size for SDR. Default is 0.0. + + - **alpha** (float): Noise parameter for the Gaussian Process. Default is 1e-6. + + - **n_restarts** (int): Number of times to restart the optimizer. Default is 1. +``` + +```{eval-rst} +.. dropdown:: nevergrad_oneplusone + + .. code-block:: + + "nevergrad_oneplusone" + + Minimize a scalar function using the One Plus One Evolutionary algorithm from Nevergrad. + + THe One Plus One evolutionary algorithm iterates to find a set of parameters that minimizes the loss + function. It does this by perturbing, or mutating, the parameters from the last iteration (the + parent). If the new (child) parameters yield a better result, then the child becomes the new parent + whose parameters are perturbed, perhaps more aggressively. If the parent yields a better result, it + remains the parent and the next perturbation is less aggressive. Originally proposed by + :cite:`Rechenberg1973`. The implementation in Nevergrad is based on the one-fifth adaptation rule, + going back to :cite:`Schumer1968. + + - **noise\_handling**: Method for handling the noise, can be + - "random": A random point is reevaluated regularly using the one-fifth adaptation rule. + - "optimistic": The best optimistic point is reevaluated regularly, embracing optimism in the face of uncertainty. + - A float coefficient can be provided to tune the regularity of these reevaluations (default is 0.05). Eg: with 0.05, each evaluation has a 5% chance (i.e., 1 in 20) of being repeated (i.e., the same candidate solution is reevaluated to better estimate its performance). (Default: `None`). + - **n\_cores**: Number of cores to use. + + - **stopping.maxfun**: Maximum number of function evaluations. + - **mutation**: Type of mutation to apply. Available options are (Default: `"gaussian"`). + - "gaussian": Standard mutation by adding a Gaussian random variable (with progressive widening) to the best pessimistic point. + - "cauchy": Same as Gaussian but using a Cauchy distribution. + - "discrete": Mutates a randomly drawn variable (mutation occurs with probability 1/d in d dimensions, hence ~1 variable per mutation). + - "discreteBSO": Follows brainstorm optimization by gradually decreasing mutation rate from 1 to 1/d. + - "fastga": Fast Genetic Algorithm mutations from the current best. + - "doublefastga": Double-FastGA mutations from the current best :cite:`doerr2017`. + - "rls": Randomized Local Search — mutates one and only one variable. + - "portfolio": Random number of mutated bits, known as uniform mixing :cite:`dang2016`. + - "lengler": Mutation rate is a function of dimension and iteration index. + - "lengler{2|3|half|fourth}": Variants of the Lengler mutation rate adaptation. + - **sparse**: Whether to apply random mutations that set variables to zero. Default is `False`. + - **smoother**: Whether to suggest smooth mutations. Default is `False`. + - **annealing**: + Annealing schedule to apply to mutation amplitude or temperature-based control. Options are: + - "none": No annealing is applied. + - "Exp0.9": Exponential decay with rate 0.9. + - "Exp0.99": Exponential decay with rate 0.99. + - "Exp0.9Auto": Exponential decay with rate 0.9, auto-scaled based on problem horizon. + - "Lin100.0": Linear decay from 1 to 0 over 100 iterations. + - "Lin1.0": Linear decay from 1 to 0 over 1 iteration. + - "LinAuto": Linearly decaying annealing automatically scaled to the problem horizon. Default is `"none"`. + - **super\_radii**: + Whether to apply extended radii beyond standard bounds for candidate generation, enabling broader + exploration. Default is `False`. + - **roulette\_size**: + Size of the roulette wheel used for selection in the evolutionary process. Affects the sampling + diversity from past candidates. (Default: `64`) + - **antismooth**: + Degree of anti-smoothing applied to prevent premature convergence in smooth landscapes. This alters + the landscape by penalizing overly smooth improvements. (Default: `4`) + - **crossover**: Whether to include a genetic crossover step every other iteration. Default is `False`. + - **crossover\_type**: + Method used for genetic crossover between individuals in the population. Available options (Default: `"none"`): + - "none": No crossover is applied. + - "rand": Randomized selection of crossover point. + - "max": Crossover at the point with maximum fitness gain. + - "min": Crossover at the point with minimum fitness gain. + - "onepoint": One-point crossover, splitting the genome at a single random point. + - "twopoint": Two-point crossover, splitting the genome at two points and exchanging the middle section. + - **tabu\_length**: + Length of the tabu list used to prevent revisiting recently evaluated candidates in local search + strategies. Helps in escaping local minima. (Default: `1000`) + - **rotation**: + Whether to apply rotational transformations to the search space, promoting invariance to axis- + aligned structures and enhancing search performance in rotated coordinate systems. (Default: + `False`) + - **seed**: Seed for the random number generator for reproducibility. +``` + +## Optimizers from the Ensmallen C++ library + +optimagic supports some optimizers from the Ensmallen C++ library. Optimizers from this +library are made available in Python through the pyensmallen python wrapper. To use +optimizers from Ensmallen, you need to have +[pyensmallen](https://pypi.org/project/pyensmallen-experimental/) installed (pip install +pyensmallen_experimental). + +````{eval-rst} .. dropdown:: ensmallen_lbfgs .. code-block:: @@ -3939,7 +4733,6 @@ addition to optimagic when using an NLOPT algorithm. To install nlopt run - **max_step_for_line_search** (float): The maximum step of the line search. default is 1e20. -``` ## References @@ -3949,3 +4742,4 @@ addition to optimagic when using an NLOPT algorithm. To install nlopt run :filter: docname in docnames :style: unsrt ``` +```` diff --git a/docs/source/conf.py b/docs/source/conf.py index 150519a78..f10cbde15 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,7 +48,6 @@ "sphinx_copybutton", "myst_nb", "sphinxcontrib.bibtex", - "sphinx_panels", "sphinx_design", "sphinxcontrib.mermaid", ] @@ -67,6 +66,28 @@ bibtex_bibfiles = ["refs.bib"] autodoc_member_order = "bysource" +autodoc_class_signature = "separated" +autodoc_default_options = { + "exclude-members": "__init__", + "members": True, + "undoc-members": True, + "member-order": "bysource", + "class-doc-from": "class", +} +autodoc_preserve_defaults = True +autodoc_type_aliases = { + "PositiveInt": "optimagic.typing.PositiveInt", + "NonNegativeInt": "optimagic.typing.NonNegativeInt", + "PositiveFloat": "optimagic.typing.PositiveFloat", + "NonNegativeFloat": "optimagic.typing.NonNegativeFloat", + "NegativeFloat": "optimagic.typing.NegativeFloat", + "GtOneFloat": "optimagic.typing.GtOneFloat", + "UnitIntervalFloat": "optimagic.typing.UnitIntervalFloat", + "YesNoBool": "optimagic.typing.YesNoBool", + "DirectionLiteral": "optimagic.typing.DirectionLiteral", + "BatchEvaluatorLiteral": "optimagic.typing.BatchEvaluatorLiteral", + "ErrorHandlingLiteral": "optimagic.typing.ErrorHandlingLiteral", +} autodoc_mock_imports = [ "bokeh", @@ -86,8 +107,8 @@ ] extlinks = { - "ghuser": ("https://github.com/%s", "@"), - "gh": ("https://github.com/optimagic-dev/optimagic/pulls/%s", "#"), + "ghuser": ("https://github.com/%s", "%s"), + "gh": ("https://github.com/optimagic-dev/optimagic/pull/%s", "%s"), } intersphinx_mapping = get_intersphinx_mapping( @@ -126,7 +147,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -145,7 +166,7 @@ todo_emit_warnings = True # -- Options for myst-nb ---------------------------------------- -nb_execution_mode = "force" +nb_execution_mode = "force" # "off", "force", "cache", "auto" nb_execution_allow_errors = False nb_merge_streams = True @@ -171,7 +192,11 @@ # "default.css" will overwrite the built-in "default.css". html_css_files = ["css/termynal.css", "css/termynal_custom.css", "css/custom.css"] -html_js_files = ["js/termynal.js", "js/custom.js"] +html_js_files = [ + "js/termynal.js", + "js/custom.js", + "js/require.js", +] # Add any paths that contain custom static files (such as style sheets) here, relative @@ -188,8 +213,11 @@ # If true, the index is split into individual pages for each letter. html_split_index = False +# If true, links to the source (either copied by sphinx on on github) +html_copy_source = True + # If true, links to the reST sources are added to the pages. -html_show_sourcelink = False +html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. html_show_sphinx = True @@ -212,4 +240,29 @@ "color-brand-primary": "#f04f43", "color-brand-content": "#f04f43", }, + "source_repository": "https://github.com/optimagic-dev/optimagic", + "source_branch": "main", + "source_directory": "docs/source/", + "footer_icons": [ + { + "name": "GitHub", + "url": "https://github.com/optimagic-dev/optimagic", + "html": """ + + + + """, + "class": "", + }, + { + "name": "Zulip", + "url": "https://ose.zulipchat.com/#narrow/channel/221432-optimagic", + "html": """ + + + """, + "class": "", + }, + ], } diff --git a/docs/source/development/code_of_conduct.md b/docs/source/development/code_of_conduct.md index 7e716cd10..893392e82 100644 --- a/docs/source/development/code_of_conduct.md +++ b/docs/source/development/code_of_conduct.md @@ -1,4 +1,8 @@ (coc)= -```{include} ../../../CODE_OF_CONDUCT.md -``` +## Code of Conduct + +The optimagic project has a [Code of Conduct][conduct] to which all contributors must +adhere. See details in the [written policy statement][conduct]. + +[conduct]: https://github.com/optimagic-dev/optimagic/blob/main/.github/CODE_OF_CONDUCT.md diff --git a/docs/source/estimagic/explanation/cluster_robust_likelihood_inference.md b/docs/source/estimagic/explanation/cluster_robust_likelihood_inference.md index b3bb14beb..6fe14146f 100644 --- a/docs/source/estimagic/explanation/cluster_robust_likelihood_inference.md +++ b/docs/source/estimagic/explanation/cluster_robust_likelihood_inference.md @@ -5,4 +5,4 @@ (to be written.) In case of an urgent request for this guide, feel free to open an issue -\[here\](). +\[here\](). diff --git a/docs/source/estimagic/tutorials/msm_overview.ipynb b/docs/source/estimagic/tutorials/msm_overview.ipynb index e2c0fc104..9735ea619 100644 --- a/docs/source/estimagic/tutorials/msm_overview.ipynb +++ b/docs/source/estimagic/tutorials/msm_overview.ipynb @@ -44,6 +44,9 @@ "source": [ "import numpy as np\n", "import pandas as pd\n", + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"\n", "\n", "import estimagic as em\n", "\n", @@ -307,7 +310,7 @@ "fig = lollipop_plot(sensitivity_data)\n", "\n", "fig = fig.update_layout(height=500, width=900)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] } ], diff --git a/docs/source/explanation/internal_optimizers.md b/docs/source/explanation/internal_optimizers.md index b1ed3523c..6ff5d17a1 100644 --- a/docs/source/explanation/internal_optimizers.md +++ b/docs/source/explanation/internal_optimizers.md @@ -98,7 +98,9 @@ To make switching between different algorithm as simple as possible, we align th of commonly used convergence and stopping criteria. We also align the default values for stopping and convergence criteria as much as possible. -You can find the harmonized names and value [here](algo_options_docs). +```{eval-rst} +You can find the harmonized names and value here: :ref:`algo_options`. +``` To align the names of other tuning parameters as much as possible with what is already there, simple have a look at the optimizers we already wrapped. For example, if you are diff --git a/docs/source/how_to/how_to_add_optimizers.ipynb b/docs/source/how_to/how_to_add_optimizers.ipynb index fc335ca8b..fd90a72e5 100644 --- a/docs/source/how_to/how_to_add_optimizers.ipynb +++ b/docs/source/how_to/how_to_add_optimizers.ipynb @@ -644,7 +644,8 @@ "step 2. You can either write it into the docstring of your algorithm class (preferred, \n", "as this is what we will do for all algorithms in the long run) or in `algorithms.md` \n", "in the documentation. \n", - "6. Create a pull request and ask for a review. " + "6. Create a pull request [in the optimagic repository](https://github.com/optimagic-dev/optimagic)\n", + "and ask for a review. " ] } ], diff --git a/docs/source/how_to/how_to_algorithm_selection.ipynb b/docs/source/how_to/how_to_algorithm_selection.ipynb index 0dfc58307..1490f3134 100644 --- a/docs/source/how_to/how_to_algorithm_selection.ipynb +++ b/docs/source/how_to/how_to_algorithm_selection.ipynb @@ -114,6 +114,13 @@ "`.Available` instead of `.All`." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An even more fine-grained way of filtering is described in [Filtering Algorithms Using Bounds](filtering_algorithms_using_bounds)." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -137,7 +144,11 @@ "source": [ "import warnings\n", "\n", - "warnings.filterwarnings(\"ignore\")" + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"" ] }, { @@ -214,7 +225,7 @@ " )\n", "\n", "fig = om.criterion_plot(results, max_evaluations=8)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -263,7 +274,7 @@ "outputs": [], "source": [ "fig = om.criterion_plot(results)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -313,7 +324,7 @@ " )\n", "\n", "fig = om.criterion_plot(results)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -345,7 +356,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "optimagic-docs", "language": "python", "name": "python3" }, @@ -359,7 +370,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.15" + "version": "3.10.16" } }, "nbformat": 4, diff --git a/docs/source/how_to/how_to_benchmarking.ipynb b/docs/source/how_to/how_to_benchmarking.ipynb index f0b3ddb1c..b6b19bf5c 100644 --- a/docs/source/how_to/how_to_benchmarking.ipynb +++ b/docs/source/how_to/how_to_benchmarking.ipynb @@ -40,6 +40,10 @@ "metadata": {}, "outputs": [], "source": [ + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"\n", + "\n", "import optimagic as om" ] }, @@ -124,7 +128,7 @@ " results=results,\n", ")\n", "\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -158,7 +162,7 @@ " x_precision=0.001,\n", ")\n", "\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -185,7 +189,7 @@ " problem_subset=[\"rosenbrock_good_start\", \"box_3d\"],\n", ")\n", "\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -214,7 +218,7 @@ " stopping_criterion=\"x\",\n", ")\n", "\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { diff --git a/docs/source/how_to/how_to_bounds.ipynb b/docs/source/how_to/how_to_bounds.ipynb index b87a15be2..9e587c06b 100644 --- a/docs/source/how_to/how_to_bounds.ipynb +++ b/docs/source/how_to/how_to_bounds.ipynb @@ -223,13 +223,78 @@ "cell_type": "markdown", "id": "17", "metadata": {}, + "source": [ + "(filtering_algorithms_using_bounds)=\n", + "\n", + "## Filtering algorithms\n", + "\n", + "It is further possible to filter algorithms based on whether they support bounds, if bounds are required to run, and if infinite bounds are supported. The AlgoInfo class provides all information about the chosen algorithm, which can be accessed with algo.algo_info... . Suppose we are looking for a optimizer that supports bounds and strictly require them for the algorithm to run properly.\n", + "\n", + "To find all algorithms that support bounds and cannot run without bounds, we can simply do:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18", + "metadata": {}, + "outputs": [], + "source": [ + "from optimagic.algorithms import AVAILABLE_ALGORITHMS\n", + "\n", + "algos_with_bounds_support = [\n", + " algo\n", + " for name, algo in AVAILABLE_ALGORITHMS.items()\n", + " if algo.algo_info.supports_bounds\n", + "]\n", + "my_selection = [\n", + " algo for algo in algos_with_bounds_support if algo.algo_info.needs_bounds\n", + "]\n", + "my_selection[0:3]" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "Similarly, to find all algorithms that support infinite values in bounds , we can do:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "my_selection2 = [\n", + " algo\n", + " for algo in algos_with_bounds_support\n", + " if algo.algo_info.supports_infinite_bounds\n", + "]\n", + "my_selection2[0:3]" + ] + }, + { + "cell_type": "markdown", + "id": "21", + "metadata": {}, + "source": [ + "In case you you forget to specify bounds for a optimizer that strictly requires them or pass infinite values in bounds to a optimizer which does not support them, optimagic will raise an `IncompleteBoundsError`. " + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, "source": [ "## Coming from scipy" ] }, { "cell_type": "markdown", - "id": "18", + "id": "23", "metadata": {}, "source": [ "If `params` is a flat numpy array, you can also provide bounds in any format that \n", @@ -240,7 +305,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "optimagic-docs", "language": "python", "name": "python3" }, @@ -254,7 +319,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.10.16" } }, "nbformat": 4, diff --git a/docs/source/how_to/how_to_derivatives.ipynb b/docs/source/how_to/how_to_derivatives.ipynb index f18558482..17cf4fae1 100644 --- a/docs/source/how_to/how_to_derivatives.ipynb +++ b/docs/source/how_to/how_to_derivatives.ipynb @@ -38,6 +38,9 @@ "outputs": [], "source": [ "import numpy as np\n", + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"\n", "\n", "import optimagic as om\n", "\n", @@ -245,7 +248,7 @@ " )\n", "\n", "fig = om.criterion_plot(results)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { diff --git a/docs/source/how_to/how_to_document_optimizers.md b/docs/source/how_to/how_to_document_optimizers.md new file mode 100644 index 000000000..a7481f0e5 --- /dev/null +++ b/docs/source/how_to/how_to_document_optimizers.md @@ -0,0 +1,254 @@ +# How to document optimizers + +This guide shows you how to document algorithms in optimagic using our new documentation +system. We'll walk through the process step-by-step using the `ScipyLBFGSB` optimizer as +a complete example. + +## When to Use This Guide + +Use this guide when you need to: + +- Document a new algorithm you've added to optimagic +- Migrate existing algorithm documentation from the old split system (docstrings + + `algorithms.md`) to the new system +- Update or improve existing algorithm documentation + +If you're adding a completely new optimizer to optimagic, start with the "How to Add +Optimizers guide" first, then use this guide to document your algorithm properly. + +## Why the New Documentation System? + +Previously, algorithm documentation was scattered across multiple places: + +- Basic descriptions in the algorithm class docstrings +- Detailed parameter descriptions in `algorithms.md` +- Usage examples separate from the algorithm definitions + +This made it hard to maintain consistency and keep documentation up-to-date. The new +system centralizes nearly all documentation in the algorithm code itself, making it: + +- Easier to maintain (documentation lives next to code) +- More consistent (unified format across all algorithms) +- Auto-generated (parameter lists appear automatically in docs) +- Type-safe (documentation matches actual parameter types) + +## The Documentation System Components + +Our documentation system has three main parts: + +1. **Algorithm Class Documentation**: A comprehensive docstring in the algorithm + dataclass that explains what the algorithm does, how it works, and when to use it +1. **Parameter Documentation**: Detailed docstrings for each parameter with mathematical + formulations when needed +1. **Usage Integration**: A section in `algorithms.md` that show how to use the + algorithm + +Let's walk through documenting an algorithm from start to finish. + +## Example: Documenting ScipyLBFGSB + +We'll use the `ScipyLBFGSB` optimizer to show you exactly how to document an algorithm. +This is a real example from the optimagic codebase, so you can follow along and see the +results. + +### Step 1: Understand Your Algorithm + +Before writing documentation, make sure you understand: + +- What the algorithm does mathematically +- What problems it's designed to solve +- How its parameters affect behavior +- Any performance characteristics or limitations + +For L-BFGS-B, this means understanding it's a quasi-Newton method for bound-constrained +optimization that approximates the Hessian using gradient history. + +```{eval-rst} + +.. note:: + If you are simply migrating an existing algorithm, you can mostly rely on the + existing documentation in the algorithm class docstring and `algorithms.md`. + +``` + +### Step 2: Write the Algorithm Class Documentation + +The algorithm class docstring is the most important part. It should give users +everything they need to decide whether to use this algorithm. + +Here's how we document `ScipyLBFGSB`: + +```python +# src/optimagic/optimizers/scipy_optimizers.py +class ScipyLBFGSB(Algorithm): + """Minimize a scalar differentiable function using the L-BFGS-B algorithm. + + The optimizer is taken from scipy, which calls the Fortran code written by the + original authors of the algorithm. The Fortran code includes the corrections + and improvements that were introduced in a follow up paper. + + lbfgsb is a limited memory version of the original bfgs algorithm, that deals with + lower and upper bounds via an active set approach. + + The lbfgsb algorithm is well suited for differentiable scalar optimization problems + with up to several hundred parameters. + + It is a quasi-newton line search algorithm. At each trial point it evaluates the + criterion function and its gradient to find a search direction. It then approximates + the hessian using the stored history of gradients and uses the hessian to calculate + a candidate step size. Then it uses a gradient based line search algorithm to + determine the actual step length. Since the algorithm always evaluates the gradient + and criterion function jointly, the user should provide a ``fun_and_jac`` function + that exploits the synergies in the calculation of criterion and gradient. + + The lbfgsb algorithm is almost perfectly scale invariant. Thus, it is not necessary + to scale the parameters. + + """ +``` + +**What makes this docstring effective:** + +- **Clear first line**: States exactly what the algorithm does +- **Implementation details**: Explains it uses scipy's Fortran implementation +- **Algorithm classification**: Identifies it as a quasi-Newton method +- **Problem suitability**: Explains what problems it's good for +- **How it works**: Brief explanation of the algorithm's approach +- **Performance characteristics**: Mentions scale invariance +- **Usage advice**: Suggests using `fun_and_jac` for efficiency + +### Step 3: Document Individual Parameters + +Each parameter needs clear documentation explaining what it controls and how it affects +the algorithm's behavior. + +```python +# Basic parameter documentation +stopping_maxiter: PositiveInt = STOPPING_MAXITER +"""Maximum number of iterations.""" + +# Parameter with mathematical formulation +convergence_ftol_rel: NonNegativeFloat = CONVERGENCE_FTOL_REL +r"""Converge if the relative change in the objective function is less than this +value. More formally, this is expressed as. + +.. math:: + + \frac{f^k - f^{k+1}}{\max\{|f^k|, |f^{k+1}|, 1\}} \leq + \textsf{convergence_ftol_rel}. + +""" + +# Parameter with external library context +limited_memory_storage_length: PositiveInt = LIMITED_MEMORY_STORAGE_LENGTH +"""The maximum number of variable metric corrections used to define the limited +memory matrix. This is the 'maxcor' parameter in the SciPy documentation. + +The default value is taken from SciPy's L-BFGS-B implementation. Larger values use +more memory but may converge faster for some problems. + +""" +``` + +**Key principles for parameter documentation:** + +- **Start with a clear description** of what the parameter controls +- **Add mathematical formulations** when they clarify the exact meaning (use `r"""` for + raw strings with LaTeX) +- **Include external library context** when relevant (e.g., "Default value is taken from + SciPy") +- **Explain performance implications** when they matter +- **Use proper type annotations** that match the parameter's constraints + +### Step 4: Integrate into `algorithms.md` + +The final step is integrating your documented algorithm into the main documentation. +This creates a dropdown section that shows users how to use the algorithm. + +Add the following to `docs/source/algorithms.md` in an `eval-rst` block: + +```text +.. dropdown:: scipy_lbfgsb + + **How to use this algorithm:** + + .. code-block:: python + + import optimagic as om + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm=om.algos.scipy_lbfgsb(stopping_maxiter=1_000, ...), + ) + + or using the string interface: + + .. code-block:: python + + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm="scipy_lbfgsb", + algo_options={"stopping_maxiter": 1_000, ...}, + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.scipy_optimizers.ScipyLBFGSB +``` + +**What this section provides:** + +- **The dropdown button and title**: Makes it easy to find the algorithm +- **Concrete usage examples** showing both the object and string interfaces +- **Algorithm-specific parameter** in the usage example +- **Auto-generated documentation** via the `autoclass` directive that pulls in your + docstrings + +## Working with Existing Documentation + +If you're migrating an algorithm that already has documentation: + +### Finding Existing Content + +Look for existing documentation in: + +- **Algorithm class docstrings**: Usually basic descriptions +- **`docs/source/algorithms.md`**: Detailed parameter descriptions and examples +- **Research papers**: For mathematical formulations and background +- **External library docs**: For default values and parameter meanings + +### Migration Strategy + +1. **Start with the algorithm class**: Move the best description from `algorithms.md` to + the class docstring +1. **Update and expand**: Add missing information about performance, usage, etc. +1. **Move parameter docs**: Transfer parameter descriptions from `algorithms.md` to + individual parameter docstrings +1. **Verify accuracy**: Check that all information is current and correct +1. **Create new integration**: Replace the old `algorithms.md` section with the new + dropdown format + +## Common Pitfalls to Avoid + +- **Don't copy-paste generic descriptions**: Each algorithm needs specific, detailed + documentation +- **Don't skip mathematical formulations**: When convergence criteria or parameters have + precise mathematical definitions, include them +- **Don't ignore external library context**: Always mention where default values come + from +- **Don't use vague parameter descriptions**: "Controls the algorithm behavior" is not + helpful +- **Don't forget performance implications**: Users need to understand trade-offs between + parameters + +## Getting Help + +If you're stuck or need clarification: + +- Look at existing well-documented algorithms like `ScipyLBFGSB` +- Check the {ref}`style_guide` for coding conventions +- Ask questions in GitHub issues or discussions + +The goal is to make optimagic's algorithm documentation the best resource for +understanding and using optimization algorithms effectively. diff --git a/docs/source/how_to/how_to_errors_during_optimization.ipynb b/docs/source/how_to/how_to_errors_during_optimization.ipynb index 8a69f85c3..407bbddb9 100644 --- a/docs/source/how_to/how_to_errors_during_optimization.ipynb +++ b/docs/source/how_to/how_to_errors_during_optimization.ipynb @@ -49,8 +49,11 @@ "import warnings\n", "\n", "import numpy as np\n", + "import plotly.io as pio\n", "from scipy.optimize import minimize as scipy_minimize\n", "\n", + "pio.renderers.default = \"notebook_connected\"\n", + "\n", "import optimagic as om\n", "\n", "warnings.simplefilter(\"ignore\")" @@ -217,7 +220,7 @@ "fig = go.Figure()\n", "fig.add_trace(go.Scatter(x=grid, y=values))\n", "fig.add_trace(go.Scatter(x=grid, y=dummy_values))\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { diff --git a/docs/source/how_to/how_to_logging.ipynb b/docs/source/how_to/how_to_logging.ipynb index dea47d11f..2ec956332 100644 --- a/docs/source/how_to/how_to_logging.ipynb +++ b/docs/source/how_to/how_to_logging.ipynb @@ -30,6 +30,9 @@ "from pathlib import Path\n", "\n", "import numpy as np\n", + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"\n", "\n", "import optimagic as om" ] @@ -217,7 +220,7 @@ "outputs": [], "source": [ "fig = om.criterion_plot(\"my_log.db\")\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -227,7 +230,7 @@ "outputs": [], "source": [ "fig = om.params_plot(\"my_log.db\", selector=lambda x: x[1:3])\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] } ], diff --git a/docs/source/how_to/how_to_multistart.ipynb b/docs/source/how_to/how_to_multistart.ipynb index 73ff8f171..203ef063c 100644 --- a/docs/source/how_to/how_to_multistart.ipynb +++ b/docs/source/how_to/how_to_multistart.ipynb @@ -36,6 +36,9 @@ "outputs": [], "source": [ "import numpy as np\n", + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"\n", "\n", "import optimagic as om\n", "\n", @@ -127,7 +130,7 @@ ")\n", "\n", "fig = om.criterion_plot(res, monotone=True)\n", - "fig.show(\"png\")" + "fig.show()" ] }, { @@ -178,7 +181,7 @@ ")\n", "\n", "fig = om.criterion_plot(res)\n", - "fig.show(\"png\")" + "fig.show()" ] }, { diff --git a/docs/source/how_to/how_to_slice_plot.ipynb b/docs/source/how_to/how_to_slice_plot.ipynb index b30de384a..558f2cef2 100644 --- a/docs/source/how_to/how_to_slice_plot.ipynb +++ b/docs/source/how_to/how_to_slice_plot.ipynb @@ -31,6 +31,9 @@ "outputs": [], "source": [ "import numpy as np\n", + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"\n", "\n", "import optimagic as om" ] @@ -77,7 +80,7 @@ " params=params,\n", " bounds=bounds,\n", ")\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -122,13 +125,13 @@ " # number of gridpoints in each dimension\n", " n_gridpoints=50,\n", ")\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "optimagic", "language": "python", "name": "python3" }, @@ -142,7 +145,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.10.18" } }, "nbformat": 4, diff --git a/docs/source/how_to/how_to_visualize_histories.ipynb b/docs/source/how_to/how_to_visualize_histories.ipynb index b9da50889..144a35dbf 100644 --- a/docs/source/how_to/how_to_visualize_histories.ipynb +++ b/docs/source/how_to/how_to_visualize_histories.ipynb @@ -23,6 +23,9 @@ "outputs": [], "source": [ "import numpy as np\n", + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"\n", "\n", "import optimagic as om" ] @@ -67,7 +70,7 @@ "outputs": [], "source": [ "fig = om.criterion_plot(results[\"scipy_neldermead\"])\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -86,7 +89,7 @@ "outputs": [], "source": [ "fig = om.criterion_plot(results)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -111,7 +114,7 @@ " # show only the current best function value\n", " monotone=True,\n", ")\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -130,7 +133,7 @@ "outputs": [], "source": [ "fig = om.params_plot(results[\"scipy_neldermead\"])\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -155,7 +158,7 @@ " # select only the last three parameters\n", " selector=lambda x: x[2:],\n", ")\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -194,7 +197,7 @@ "outputs": [], "source": [ "fig = om.criterion_plot(res, max_evaluations=1000, monotone=True)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] } ], diff --git a/docs/source/how_to/index.md b/docs/source/how_to/index.md index 2a2f362d5..911762e43 100644 --- a/docs/source/how_to/index.md +++ b/docs/source/how_to/index.md @@ -25,4 +25,5 @@ how_to_errors_during_optimization how_to_slice_plot how_to_benchmarking how_to_add_optimizers +how_to_document_optimizers ``` diff --git a/docs/source/reference/algo_options.md b/docs/source/reference/algo_options.md index 2336c8003..367644521 100644 --- a/docs/source/reference/algo_options.md +++ b/docs/source/reference/algo_options.md @@ -1,4 +1,4 @@ -(algo_options_docs)= +(algo_options)= # The default algorithm options diff --git a/docs/source/reference/index.md b/docs/source/reference/index.md index 3ee07cee1..728a29d38 100644 --- a/docs/source/reference/index.md +++ b/docs/source/reference/index.md @@ -218,4 +218,5 @@ maxdepth: 1 utilities algo_options batch_evaluators +typing ``` diff --git a/docs/source/reference/typing.md b/docs/source/reference/typing.md new file mode 100644 index 000000000..1a13cdf6f --- /dev/null +++ b/docs/source/reference/typing.md @@ -0,0 +1,10 @@ +(typing)= + +# Types + +```{eval-rst} + +.. automodule:: optimagic.typing + :members: + +``` diff --git a/docs/source/refs.bib b/docs/source/refs.bib index a7880afd2..bf6dda45b 100644 --- a/docs/source/refs.bib +++ b/docs/source/refs.bib @@ -893,6 +893,7 @@ @book{Conn2009 URL = {https://epubs.siam.org/doi/abs/10.1137/1.9780898718768}, } + @article{Matthies1979, author = {H. Matthies and G. Strang}, title = {The Solution of Nonlinear Finite Element Equations}, @@ -904,5 +905,179 @@ @article{Matthies1979 doi = {10.1002/nme.1620141104} } +@article{JAMES1975343, +title = {Minuit - a system for function minimization and analysis of the parameter errors and correlations}, +journal = {Computer Physics Communications}, +volume = {10}, +number = {6}, +pages = {343-367}, +year = {1975}, +issn = {0010-4655}, +doi = {https://doi.org/10.1016/0010-4655(75)90039-9}, +url = {https://www.sciencedirect.com/science/article/pii/0010465575900399}, +author = {F. James and M. Roos} +} + + +@misc{Hansen2023, +title={The CMA Evolution Strategy: A Tutorial}, +author={Nikolaus Hansen}, +year={2023}, +eprint={1604.00772}, +archivePrefix={arXiv}, +primaryClass={cs.LG}, +url={https://arxiv.org/abs/1604.00772}, +} + +@InProceedings{Kennedy1995, + author={Kennedy, J. and Eberhart, R.}, + booktitle={Proceedings of ICNN'95 - International Conference on Neural Networks}, + title={Particle swarm optimization}, + year={1995}, + volume={4}, + pages={1942-1948 vol.4}, + keywords={Particle swarm optimization;Birds;Educational institutions;Marine animals;Testing;Humans;Genetic algorithms;Optimization methods;Artificial neural networks;Performance evaluation}, + doi={10.1109/ICNN.1995.488968}, +} + +@InProceedings{Zambrano2013, + author = {Zambrano-Bigiarini, Mauricio and Clerc, Maurice and Rojas, Rodrigo}, + booktitle = {2013 IEEE Congress on Evolutionary Computation}, + title = {Standard Particle Swarm Optimisation 2011 at CEC-2013: A baseline for future PSO improvements}, + year = {2013}, + pages = {2337-2344}, + keywords = {Optimization;Standards;Benchmark testing;Topology;Algorithm design and analysis;Convergence;Equations;particle swarm optimization;SPSO-2011;CEC-2013;random topology;rotational invariance;benchmark testing;evolutionary computation;optimization}, + doi = {10.1109/CEC.2013.6557848}, +} + +@inbook{randomsearch2010, +author = {Zabinsky, Zelda}, +year = {2010}, +month = {06}, +pages = {}, +title = {Random Search Algorithms}, +isbn = {9780470400531}, +doi = {10.1002/9780470400531.eorms0704} +} + +@INPROCEEDINGS{spsaimpl, + author={Rastogi, Pushpendre and Zhu, Jingyi and Spall, James C.}, + booktitle={2016 Annual Conference on Information Science and Systems (CISS)}, + title={Efficient implementation of enhanced adaptive simultaneous perturbation algorithms}, + year={2016}, + volume={}, + number={}, + pages={298-303}, + keywords={Estimation;Algorithm design and analysis;Adaptive Estimation;Simultaneous Perturbation Stochastic Approximation (SPSA);Woodbury Matrix Identity}, + doi={10.1109/CISS.2016.7460518}} + +@inproceedings{tbpsaimpl, +author = {Hellwig, Michael and Beyer, Hans-Georg}, +year = {2016}, +month = {09}, +pages = {}, +title = {Evolution under Strong Noise: A Self-Adaptive Evolution Strategy Can Reach the Lower Performance Bound - the pcCMSA-ES}, +volume = {9921}, +isbn = {9783319458229}, +doi = {10.1007/978-3-319-45823-6_3} +} + +@ARTICLE{cgaimpl, + author={Harik, G.R. and Lobo, F.G. and Goldberg, D.E.}, + journal={IEEE Transactions on Evolutionary Computation}, + title={The compact genetic algorithm}, + year={1999}, + volume={3}, + number={4}, + pages={287-297}, + keywords={Genetic algorithms;Algorithm design and analysis;Laboratories;Computer simulation;Genetic engineering;Probability distribution;Computational modeling;History;Convergence;Mathematical model}, + doi={10.1109/4235.797971}} + +@inproceedings{bayesoptimimpl, +author = {Raponi, Elena and Wang, Hao and Bujny, Mariusz and Boria, Simonetta and Doerr, Carola}, +title = {High Dimensional Bayesian Optimization Assisted by Principal Component Analysis}, +year = {2020}, +isbn = {978-3-030-58111-4}, +publisher = {Springer-Verlag}, +address = {Berlin, Heidelberg}, +url = {https://doi.org/10.1007/978-3-030-58112-1_12}, +doi = {10.1007/978-3-030-58112-1_12}, +abstract = {Bayesian Optimization (BO) is a surrogate-assisted global optimization technique that has been successfully applied in various fields, e.g., automated machine learning and design optimization. Built upon a so-called infill-criterion and Gaussian Process regression (GPR), the BO technique suffers from a substantial computational complexity and hampered convergence rate as the dimension of the search spaces increases. Scaling up BO for high-dimensional optimization problems remains a challenging task.In this paper, we propose to tackle the scalability of BO by hybridizing it with a Principal Component Analysis (PCA), resulting in a novel PCA-assisted BO (PCA-BO) algorithm. Specifically, the PCA procedure learns a linear transformation from all the evaluated points during the run and selects dimensions in the transformed space according to the variability of evaluated points. We then construct the GPR model, and the infill-criterion in the space spanned by the selected dimensions.We assess the performance of our PCA-BO in terms of the empirical convergence rate and CPU time on multi-modal problems from the COCO benchmark framework. The experimental results show that PCA-BO can effectively reduce the CPU time incurred on high-dimensional problems, and maintains the convergence rate on problems with an adequate global structure. PCA-BO therefore provides a satisfactory trade-off between the convergence rate and computational efficiency opening new ways to benefit from the strength of BO approaches in high dimensional numerical optimization.}, +booktitle = {Parallel Problem Solving from Nature – PPSN XVI: 16th International Conference, PPSN 2020, Leiden, The Netherlands, September 5-9, 2020, Proceedings, Part I}, +pages = {169–183}, +numpages = {15}, +keywords = {Dimensionality reduction, Principal Component Analysis, Black-box optimization, Bayesian optimization}, +location = {Leiden, The Netherlands} +} + +@book{Rechenberg1973, + author = {Rechenberg, Ingo}, + title = {Evolutionsstrategie: Optimierung technischer Systeme nach Prinzipien der biologischen Evolution}, + publisher = {Frommann-Holzboog Verlag}, + year = {1973}, + url = {https://gwern.net/doc/reinforcement-learning/exploration/1973-rechenberg.pdf}, + address = {Stuttgart}, + note = {[Evolution Strategy: Optimization of Technical Systems According to the Principles of Biological Evolution]} +} + +@article{Schumer1968, + author={Schumer, M. and Steiglitz, K.}, + journal={IEEE Transactions on Automatic Control}, + title={Adaptive step size random search}, + year={1968}, + volume={13}, + number={3}, + pages={270-276}, + keywords={Minimization methods;Gradient methods;Search methods;Adaptive control;Communication systems;Q measurement;Cost function;Newton method;Military computing}, + doi={10.1109/TAC.1968.1098903} +} + +@misc{edaimpl, + title={Theory of Estimation-of-Distribution Algorithms}, + author={Martin S. Krejca and Carsten Witt}, + year={2018}, + eprint={1806.05392}, + archivePrefix={arXiv}, + primaryClass={cs.NE}, + url={https://arxiv.org/abs/1806.05392}, +} + +@book{emnaimpl, +author = {Larranaga, Pedro and Lozano, Jose}, +year = {2002}, +month = {01}, +pages = {}, +title = {Estimation of Distribution Algorithms: A New Tool for Evolutionary Computation}, +isbn = {9781461356042}, +journal = {Genetic algorithms and evolutionary computation ; 2}, +doi = {10.1007/978-1-4615-1539-5} +} + +@Misc{Nogueira2014, + author={Fernando Nogueira}, + title={{Bayesian Optimization}: Open source constrained global optimization tool for {Python}}, + year={2014--}, + url="https://github.com/bayesian-optimization/BayesianOptimization" +} + +@article{Stander2002, + author={Stander, Nielen and Craig, Kenneth}, + year={2002}, + month={06}, + pages={}, + title={On the robustness of a simple domain reduction scheme for simulation-based optimization}, + volume={19}, + journal={International Journal for Computer-Aided Engineering and Software (Eng. Comput.)}, + doi={10.1108/02644400210430190} +} + +@inproceedings{gardner2014bayesian, + title={Bayesian optimization with inequality constraints.}, + author={Gardner, Jacob R and Kusner, Matt J and Xu, Zhixiang Eddie and Weinberger, Kilian Q and Cunningham, John P}, + booktitle={ICML}, + volume={2014}, + pages={937--945}, + year={2014} +} @Comment{jabref-meta: databaseType:bibtex;} diff --git a/docs/source/tutorials/optimization_overview.ipynb b/docs/source/tutorials/optimization_overview.ipynb index 250b9f567..05888bf16 100644 --- a/docs/source/tutorials/optimization_overview.ipynb +++ b/docs/source/tutorials/optimization_overview.ipynb @@ -17,6 +17,9 @@ "source": [ "import numpy as np\n", "import pandas as pd\n", + "import plotly.io as pio\n", + "\n", + "pio.renderers.default = \"notebook_connected\"\n", "\n", "import optimagic as om" ] @@ -106,7 +109,7 @@ "source": [ "results = {\"lbfgsb\": lbfgsb_res, \"nelder_mead\": nm_res}\n", "fig = om.criterion_plot(results, max_evaluations=300)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -128,7 +131,7 @@ " # optionally select a subset of parameters to plot\n", " selector=lambda params: params[\"c\"],\n", ")\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { @@ -398,7 +401,7 @@ "outputs": [], "source": [ "fig = om.criterion_plot(res)\n", - "fig.show(renderer=\"png\")" + "fig.show()" ] }, { diff --git a/environment.yml b/environment.yml index 34ab4604b..6bb4f01db 100644 --- a/environment.yml +++ b/environment.yml @@ -20,7 +20,7 @@ dependencies: - joblib # run, tests - numpy >= 2 # run, tests - pandas # run, tests - - plotly<6.0.0 # run, tests + - plotly>=6.2 # run, tests - pybaum>=0.1.2 # run, tests - scipy>=1.2.1 # run, tests - sqlalchemy # run, tests @@ -37,12 +37,17 @@ dependencies: - jinja2 # dev, tests - furo # dev, docs - annotated-types # dev, tests + - iminuit # dev, tests + - cma # dev, tests - pip: # dev, tests, docs + - bayesian-optimization>=2.0.4 # dev, tests + # - nevergrad # incompatible with bayesian-optimization>=2.0.4 - DFO-LS>=1.5.3 # dev, tests - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - - kaleido # dev, tests + - kaleido>=1.0 # dev, tests - pre-commit>=4 # dev + - bayes_optim # dev, tests - -e . # dev # type stubs - pandas-stubs # dev, tests diff --git a/pyproject.toml b/pyproject.toml index 5d370dd49..dcad32bea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ "joblib", "numpy", "pandas", - "plotly<6.0.0", + "plotly", "pybaum>=0.1.2", "scipy>=1.2.1", "sqlalchemy>=1.3", @@ -334,7 +334,7 @@ ignore_errors = true [[tool.mypy.overrides]] module = [ - "pyensmallen", + "pyensmallen_experimental", "pybaum", "scipy", "scipy.linalg", @@ -379,5 +379,8 @@ module = [ "optimagic._version", "annotated_types", "pdbp", + "iminuit", + "nevergrad", + "yaml", ] ignore_missing_imports = true diff --git a/src/optimagic/algorithms.py b/src/optimagic/algorithms.py index 5f25f25ca..b681f59fe 100644 --- a/src/optimagic/algorithms.py +++ b/src/optimagic/algorithms.py @@ -12,11 +12,28 @@ from typing import Type, cast from optimagic.optimization.algorithm import Algorithm +from optimagic.optimizers.bayesian_optimizer import BayesOpt from optimagic.optimizers.bhhh import BHHH from optimagic.optimizers.fides import Fides +from optimagic.optimizers.iminuit_migrad import IminuitMigrad from optimagic.optimizers.ipopt import Ipopt from optimagic.optimizers.nag_optimizers import NagDFOLS, NagPyBOBYQA from optimagic.optimizers.neldermead import NelderMeadParallel +from optimagic.optimizers.nevergrad_optimizers import ( + NevergradBayesOptim, + NevergradCGA, + NevergradCMAES, + NevergradDifferentialEvolution, + NevergradEDA, + NevergradEMNA, + NevergradMeta, + NevergradNGOpt, + NevergradOnePlusOne, + NevergradPSO, + NevergradRandomSearch, + NevergradSamplingSearch, + NevergradTBPSA, +) from optimagic.optimizers.nlopt_optimizers import ( NloptBOBYQA, NloptCCSAQ, @@ -171,6 +188,19 @@ def Scalar( @dataclass(frozen=True) class BoundedGlobalGradientFreeParallelScalarAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -287,6 +317,7 @@ def Scalar(self) -> BoundedGradientBasedLocalNonlinearConstrainedScalarAlgorithm @dataclass(frozen=True) class BoundedGradientBasedLocalScalarAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_lbfgsb: Type[NloptLBFGSB] = NloptLBFGSB @@ -363,6 +394,20 @@ def Scalar(self) -> BoundedGlobalGradientFreeNonlinearConstrainedScalarAlgorithm @dataclass(frozen=True) class BoundedGlobalGradientFreeScalarAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -402,6 +447,19 @@ def Parallel(self) -> BoundedGlobalGradientFreeParallelScalarAlgorithms: @dataclass(frozen=True) class BoundedGlobalGradientFreeParallelAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -457,6 +515,19 @@ def Scalar(self) -> GlobalGradientFreeNonlinearConstrainedParallelScalarAlgorith @dataclass(frozen=True) class GlobalGradientFreeParallelScalarAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -604,6 +675,19 @@ def Scalar(self) -> BoundedGradientFreeNonlinearConstrainedParallelScalarAlgorit @dataclass(frozen=True) class BoundedGradientFreeParallelScalarAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -698,6 +782,19 @@ def Scalar(self) -> BoundedGlobalNonlinearConstrainedParallelScalarAlgorithms: @dataclass(frozen=True) class BoundedGlobalParallelScalarAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -841,6 +938,7 @@ def NonlinearConstrained( @dataclass(frozen=True) class BoundedGradientBasedLocalAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_lbfgsb: Type[NloptLBFGSB] = NloptLBFGSB @@ -890,6 +988,7 @@ def Scalar(self) -> GradientBasedLocalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class GradientBasedLocalScalarAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_lbfgsb: Type[NloptLBFGSB] = NloptLBFGSB @@ -958,6 +1057,7 @@ def Scalar(self) -> BoundedGradientBasedNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class BoundedGradientBasedScalarAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_lbfgsb: Type[NloptLBFGSB] = NloptLBFGSB @@ -1022,6 +1122,20 @@ def Local(self) -> GradientBasedLocalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class BoundedGlobalGradientFreeAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -1085,6 +1199,20 @@ def Scalar(self) -> GlobalGradientFreeNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class GlobalGradientFreeScalarAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -1128,6 +1256,19 @@ def Parallel(self) -> GlobalGradientFreeParallelScalarAlgorithms: @dataclass(frozen=True) class GlobalGradientFreeParallelAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -1292,7 +1433,21 @@ def Scalar(self) -> BoundedGradientFreeNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class BoundedGradientFreeScalarAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM @@ -1364,6 +1519,19 @@ def Parallel(self) -> BoundedGradientFreeLeastSquaresParallelAlgorithms: @dataclass(frozen=True) class BoundedGradientFreeParallelAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pounders: Type[Pounders] = Pounders pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -1445,6 +1613,19 @@ def Scalar(self) -> GradientFreeNonlinearConstrainedParallelScalarAlgorithms: @dataclass(frozen=True) class GradientFreeParallelScalarAlgorithms(AlgoSelection): neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -1513,6 +1694,20 @@ def Scalar(self) -> BoundedGlobalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class BoundedGlobalScalarAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -1561,6 +1756,19 @@ def Parallel(self) -> BoundedGlobalParallelScalarAlgorithms: @dataclass(frozen=True) class BoundedGlobalParallelAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -1629,6 +1837,19 @@ def Scalar(self) -> GlobalNonlinearConstrainedParallelScalarAlgorithms: @dataclass(frozen=True) class GlobalParallelScalarAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -1676,6 +1897,7 @@ def Scalar(self) -> BoundedLocalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class BoundedLocalScalarAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA @@ -1862,6 +2084,19 @@ def Scalar(self) -> BoundedNonlinearConstrainedParallelScalarAlgorithms: @dataclass(frozen=True) class BoundedParallelScalarAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -1945,6 +2180,7 @@ def Scalar(self) -> GlobalGradientBasedScalarAlgorithms: class GradientBasedLocalAlgorithms(AlgoSelection): bhhh: Type[BHHH] = BHHH fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_lbfgsb: Type[NloptLBFGSB] = NloptLBFGSB @@ -1988,6 +2224,7 @@ def Scalar(self) -> GradientBasedLocalScalarAlgorithms: @dataclass(frozen=True) class BoundedGradientBasedAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_lbfgsb: Type[NloptLBFGSB] = NloptLBFGSB @@ -2057,6 +2294,7 @@ def Scalar(self) -> GradientBasedNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class GradientBasedScalarAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_lbfgsb: Type[NloptLBFGSB] = NloptLBFGSB @@ -2119,6 +2357,20 @@ def Local(self) -> GradientBasedLikelihoodLocalAlgorithms: @dataclass(frozen=True) class GlobalGradientFreeAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -2204,8 +2456,22 @@ def Scalar(self) -> GradientFreeLocalScalarAlgorithms: @dataclass(frozen=True) class BoundedGradientFreeAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM @@ -2300,8 +2566,22 @@ def Scalar(self) -> GradientFreeNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class GradientFreeScalarAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM @@ -2382,6 +2662,19 @@ def Parallel(self) -> GradientFreeLeastSquaresParallelAlgorithms: @dataclass(frozen=True) class GradientFreeParallelAlgorithms(AlgoSelection): neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pounders: Type[Pounders] = Pounders pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -2421,6 +2714,20 @@ def Scalar(self) -> GradientFreeParallelScalarAlgorithms: @dataclass(frozen=True) class BoundedGlobalAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -2502,6 +2809,20 @@ def Scalar(self) -> GlobalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class GlobalScalarAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -2554,6 +2875,19 @@ def Parallel(self) -> GlobalParallelScalarAlgorithms: @dataclass(frozen=True) class GlobalParallelAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -2581,6 +2915,7 @@ def Scalar(self) -> GlobalParallelScalarAlgorithms: @dataclass(frozen=True) class BoundedLocalAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA @@ -2663,6 +2998,7 @@ def Scalar(self) -> LocalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class LocalScalarAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel @@ -2813,9 +3149,24 @@ def Scalar(self) -> BoundedNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class BoundedScalarAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA @@ -2916,6 +3267,19 @@ def Parallel(self) -> BoundedLeastSquaresParallelAlgorithms: @dataclass(frozen=True) class BoundedParallelAlgorithms(AlgoSelection): + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pounders: Type[Pounders] = Pounders pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -3017,6 +3381,19 @@ def Scalar(self) -> NonlinearConstrainedParallelScalarAlgorithms: @dataclass(frozen=True) class ParallelScalarAlgorithms(AlgoSelection): neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen scipy_brute: Type[ScipyBrute] = ScipyBrute @@ -3068,6 +3445,7 @@ def Local(self) -> LeastSquaresLocalParallelAlgorithms: class GradientBasedAlgorithms(AlgoSelection): bhhh: Type[BHHH] = BHHH fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_lbfgsb: Type[NloptLBFGSB] = NloptLBFGSB @@ -3121,9 +3499,23 @@ def Scalar(self) -> GradientBasedScalarAlgorithms: @dataclass(frozen=True) class GradientFreeAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM @@ -3194,6 +3586,20 @@ def Scalar(self) -> GradientFreeScalarAlgorithms: @dataclass(frozen=True) class GlobalAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_crs2_lm: Type[NloptCRS2LM] = NloptCRS2LM nlopt_direct: Type[NloptDirect] = NloptDirect nlopt_esch: Type[NloptESCH] = NloptESCH @@ -3252,6 +3658,7 @@ def Scalar(self) -> GlobalScalarAlgorithms: class LocalAlgorithms(AlgoSelection): bhhh: Type[BHHH] = BHHH fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA @@ -3322,10 +3729,25 @@ def Scalar(self) -> LocalScalarAlgorithms: @dataclass(frozen=True) class BoundedAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA @@ -3457,10 +3879,25 @@ def Scalar(self) -> NonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class ScalarAlgorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA @@ -3590,6 +4027,19 @@ def Local(self) -> LikelihoodLocalAlgorithms: @dataclass(frozen=True) class ParallelAlgorithms(AlgoSelection): neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA pounders: Type[Pounders] = Pounders pygmo_gaco: Type[PygmoGaco] = PygmoGaco pygmo_pso_gen: Type[PygmoPsoGen] = PygmoPsoGen @@ -3631,12 +4081,27 @@ def Scalar(self) -> ParallelScalarAlgorithms: @dataclass(frozen=True) class Algorithms(AlgoSelection): + bayes_opt: Type[BayesOpt] = BayesOpt bhhh: Type[BHHH] = BHHH fides: Type[Fides] = Fides + iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel + nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim + nevergrad_cga: Type[NevergradCGA] = NevergradCGA + nevergrad_cmaes: Type[NevergradCMAES] = NevergradCMAES + nevergrad_de: Type[NevergradDifferentialEvolution] = NevergradDifferentialEvolution + nevergrad_eda: Type[NevergradEDA] = NevergradEDA + nevergrad_emna: Type[NevergradEMNA] = NevergradEMNA + nevergrad_meta: Type[NevergradMeta] = NevergradMeta + nevergrad_NGOpt: Type[NevergradNGOpt] = NevergradNGOpt + nevergrad_oneplusone: Type[NevergradOnePlusOne] = NevergradOnePlusOne + nevergrad_pso: Type[NevergradPSO] = NevergradPSO + nevergrad_randomsearch: Type[NevergradRandomSearch] = NevergradRandomSearch + nevergrad_samplingsearch: Type[NevergradSamplingSearch] = NevergradSamplingSearch + nevergrad_tbpsa: Type[NevergradTBPSA] = NevergradTBPSA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_ccsaq: Type[NloptCCSAQ] = NloptCCSAQ nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA diff --git a/src/optimagic/batch_evaluators.py b/src/optimagic/batch_evaluators.py index 3911a9a92..778562fe8 100644 --- a/src/optimagic/batch_evaluators.py +++ b/src/optimagic/batch_evaluators.py @@ -17,9 +17,10 @@ import threading from typing import Any, Callable, Literal, TypeVar, cast +from optimagic import deprecations from optimagic.config import DEFAULT_N_CORES as N_CORES from optimagic.decorators import catch, unpack -from optimagic.typing import BatchEvaluator, ErrorHandling +from optimagic.typing import BatchEvaluator, BatchEvaluatorLiteral, ErrorHandling T = TypeVar("T") @@ -260,10 +261,12 @@ def _check_inputs( def process_batch_evaluator( - batch_evaluator: Literal["joblib", "pathos", "threading"] - | BatchEvaluator = "joblib", + batch_evaluator: BatchEvaluatorLiteral | BatchEvaluator = "joblib", ) -> BatchEvaluator: - batch_evaluator = "joblib" if batch_evaluator is None else batch_evaluator + if batch_evaluator is None: + deprecations.throw_none_valued_batch_evaluator_warning() + batch_evaluator = "joblib" + if callable(batch_evaluator): out = batch_evaluator elif isinstance(batch_evaluator, str): @@ -275,8 +278,8 @@ def process_batch_evaluator( out = cast(BatchEvaluator, threading_batch_evaluator) else: raise ValueError( - "Invalid batch evaluator requested. Currently only 'pathos' and " - "'joblib' are supported." + "Invalid batch evaluator requested. Currently only 'pathos', 'joblib', " + "and 'threading' are supported." ) else: raise TypeError("batch_evaluator must be a callable or string.") diff --git a/src/optimagic/config.py b/src/optimagic/config.py index bb1307165..d4eb25636 100644 --- a/src/optimagic/config.py +++ b/src/optimagic/config.py @@ -1,3 +1,4 @@ +import importlib.util from pathlib import Path import pandas as pd @@ -15,88 +16,31 @@ CRITERION_PENALTY_SLOPE = 0.1 CRITERION_PENALTY_CONSTANT = 100 -# ====================================================================================== -# Check Available Packages -# ====================================================================================== - -try: - from petsc4py import PETSc # noqa: F401 -except ImportError: - IS_PETSC4PY_INSTALLED = False -else: - IS_PETSC4PY_INSTALLED = True - -try: - import nlopt # noqa: F401 -except ImportError: - IS_NLOPT_INSTALLED = False -else: - IS_NLOPT_INSTALLED = True - -try: - import pybobyqa # noqa: F401 -except ImportError: - IS_PYBOBYQA_INSTALLED = False -else: - IS_PYBOBYQA_INSTALLED = True - -try: - import dfols # noqa: F401 -except ImportError: - IS_DFOLS_INSTALLED = False -else: - IS_DFOLS_INSTALLED = True -try: - import pygmo # noqa: F401 -except ImportError: - IS_PYGMO_INSTALLED = False -else: - IS_PYGMO_INSTALLED = True +def _is_installed(module_name: str) -> bool: + """Return True if the given module is installed, otherwise False.""" + return importlib.util.find_spec(module_name) is not None -try: - import cyipopt # noqa: F401 -except ImportError: - IS_CYIPOPT_INSTALLED = False -else: - IS_CYIPOPT_INSTALLED = True - -try: - import fides # noqa: F401 -except ImportError: - IS_FIDES_INSTALLED = False -else: - IS_FIDES_INSTALLED = True - -try: - import jax # noqa: F401 -except ImportError: - IS_JAX_INSTALLED = False -else: - IS_JAX_INSTALLED = True - - -try: - import tranquilo # noqa: F401 -except ImportError: - IS_TRANQUILO_INSTALLED = False -else: - IS_TRANQUILO_INSTALLED = True +# ====================================================================================== +# Check Available Packages +# ====================================================================================== -try: - import numba # noqa: F401 -except ImportError: - IS_NUMBA_INSTALLED = False -else: - IS_NUMBA_INSTALLED = True +IS_PETSC4PY_INSTALLED = _is_installed("petsc4py") +IS_NLOPT_INSTALLED = _is_installed("nlopt") +IS_PYBOBYQA_INSTALLED = _is_installed("pybobyqa") +IS_DFOLS_INSTALLED = _is_installed("dfols") +IS_PYGMO_INSTALLED = _is_installed("pygmo") +IS_CYIPOPT_INSTALLED = _is_installed("cyipopt") +IS_FIDES_INSTALLED = _is_installed("fides") +IS_JAX_INSTALLED = _is_installed("jax") +IS_TRANQUILO_INSTALLED = _is_installed("tranquilo") +IS_NUMBA_INSTALLED = _is_installed("numba") +IS_IMINUIT_INSTALLED = _is_installed("iminuit") +IS_NEVERGRAD_INSTALLED = _is_installed("nevergrad") +IS_BAYESOPT_INSTALLED = _is_installed("bayes_opt") +IS_PYENSMALLEN_INSTALLED = _is_installed("pyensmallen") -try: - import pyensmallen # noqa: F401 -except ImportError: - IS_PYENSMALLEN_INSTALLED = False -else: - IS_PYENSMALLEN_INSTALLED = True # ====================================================================================== # Check if pandas version is newer or equal to version 2.1.0 diff --git a/src/optimagic/deprecations.py b/src/optimagic/deprecations.py index 5e8802637..344dc283f 100644 --- a/src/optimagic/deprecations.py +++ b/src/optimagic/deprecations.py @@ -183,6 +183,15 @@ def throw_dict_access_future_warning(attribute, obj_name): warnings.warn(msg, FutureWarning) +def throw_none_valued_batch_evaluator_warning(): + msg = ( + "Passing `None` as the `batch_evaluator` is deprecated and will be " + "removed in optimagic version 0.6.0. Please use the string 'joblib' instead to " + "use the joblib batch evaluator by default." + ) + warnings.warn(msg, FutureWarning) + + def replace_and_warn_about_deprecated_algo_options(algo_options): if not isinstance(algo_options, dict): return algo_options @@ -362,11 +371,11 @@ def throw_dict_constraints_future_warning_if_required( if not isinstance(constraints, list): constraints = [constraints] - types = [ + types_or_none = [ constraint.get("type", None) if isinstance(constraint, dict) else None for constraint in constraints ] - types = list(set(types) - {None}) + types = [t for t in types_or_none if t is not None] if types: msg = ( diff --git a/src/optimagic/differentiation/derivatives.py b/src/optimagic/differentiation/derivatives.py index f0f9d7653..a521703c2 100644 --- a/src/optimagic/differentiation/derivatives.py +++ b/src/optimagic/differentiation/derivatives.py @@ -23,7 +23,7 @@ from optimagic.parameters.block_trees import hessian_to_block_tree, matrix_to_block_tree from optimagic.parameters.bounds import Bounds, get_internal_bounds, pre_process_bounds from optimagic.parameters.tree_registry import get_registry -from optimagic.typing import PyTree +from optimagic.typing import BatchEvaluatorLiteral, PyTree @dataclass(frozen=True) @@ -95,7 +95,7 @@ def first_derivative( f0: PyTree | None = None, n_cores: int = DEFAULT_N_CORES, error_handling: Literal["continue", "raise", "raise_strict"] = "continue", - batch_evaluator: Literal["joblib", "pathos"] | Callable = "joblib", + batch_evaluator: BatchEvaluatorLiteral | Callable = "joblib", unpacker: Callable[[Any], PyTree] | None = None, # deprecated lower_bounds: PyTree | None = None, @@ -400,7 +400,7 @@ def second_derivative( f0: PyTree | None = None, n_cores: int = DEFAULT_N_CORES, error_handling: Literal["continue", "raise", "raise_strict"] = "continue", - batch_evaluator: Literal["joblib", "pathos"] | Callable = "joblib", + batch_evaluator: BatchEvaluatorLiteral | Callable = "joblib", unpacker: Callable[[Any], PyTree] | None = None, # deprecated lower_bounds: PyTree | None = None, diff --git a/src/optimagic/differentiation/generate_steps.py b/src/optimagic/differentiation/generate_steps.py index ca34b5769..35628d542 100644 --- a/src/optimagic/differentiation/generate_steps.py +++ b/src/optimagic/differentiation/generate_steps.py @@ -3,6 +3,8 @@ import numpy as np +from optimagic.utilities import fast_numpy_full + class Steps(NamedTuple): pos: np.ndarray @@ -90,12 +92,21 @@ def generate_steps( ) min_steps = base_steps if min_steps is None else min_steps - assert (bounds.upper - bounds.lower >= 2 * min_steps).all(), ( + lower_bounds = bounds.lower + upper_bounds = bounds.upper + # None-valued bounds are handled by instantiating them as an -inf and inf array. In + # the future, this should be handled more gracefully. + if lower_bounds is None: + lower_bounds = fast_numpy_full(len(x), fill_value=-np.inf) + if upper_bounds is None: + upper_bounds = fast_numpy_full(len(x), fill_value=np.inf) + + assert (upper_bounds - lower_bounds >= 2 * min_steps).all(), ( "min_steps is too large to fit into bounds." ) - upper_step_bounds = bounds.upper - x - lower_step_bounds = bounds.lower - x + upper_step_bounds = upper_bounds - x + lower_step_bounds = lower_bounds - x pos = step_ratio ** np.arange(n_steps) * base_steps.reshape(-1, 1) neg = -pos.copy() @@ -105,7 +116,7 @@ def generate_steps( x, pos, neg, method, lower_step_bounds, upper_step_bounds ) - if np.isfinite(bounds.lower).any() or np.isfinite(bounds.upper).any(): + if np.isfinite(lower_bounds).any() or np.isfinite(upper_bounds).any(): pos, neg = _rescale_to_accomodate_bounds( base_steps, pos, neg, lower_step_bounds, upper_step_bounds, min_steps ) diff --git a/src/optimagic/differentiation/numdiff_options.py b/src/optimagic/differentiation/numdiff_options.py index 75d7d25d7..d6c2ff3c2 100644 --- a/src/optimagic/differentiation/numdiff_options.py +++ b/src/optimagic/differentiation/numdiff_options.py @@ -4,8 +4,10 @@ from typing_extensions import NotRequired +from optimagic.batch_evaluators import process_batch_evaluator from optimagic.config import DEFAULT_N_CORES from optimagic.exceptions import InvalidNumdiffOptionsError +from optimagic.typing import BatchEvaluatorLiteral @dataclass(frozen=True) @@ -21,8 +23,8 @@ class NumdiffOptions: min_steps: The minimum step size to use for numerical differentiation. If None, the default minimum step size will be used. n_cores: The number of cores to use for numerical differentiation. - batch_evaluator: The batch evaluator to use for numerical differentiation. Can - be "joblib" or "pathos", or a custom function. + batch_evaluator: The evaluator to use for batch evaluation. Allowed are + "joblib", "pathos", and "threading", or a custom callable. Raises: InvalidNumdiffError: If the numdiff options cannot be processed, e.g. because @@ -37,7 +39,7 @@ class NumdiffOptions: scaling_factor: float = 1 min_steps: float | None = None n_cores: int = DEFAULT_N_CORES - batch_evaluator: Literal["joblib", "pathos"] | Callable = "joblib" # type: ignore + batch_evaluator: BatchEvaluatorLiteral | Callable = "joblib" # type: ignore def __post_init__(self) -> None: _validate_attribute_types_and_values(self) @@ -51,7 +53,7 @@ class NumdiffOptionsDict(TypedDict): scaling_factor: NotRequired[float] min_steps: NotRequired[float | None] n_cores: NotRequired[int] - batch_evaluator: NotRequired[Literal["joblib", "pathos"] | Callable] # type: ignore + batch_evaluator: NotRequired[BatchEvaluatorLiteral | Callable] # type: ignore def pre_process_numdiff_options( @@ -139,14 +141,12 @@ def _validate_attribute_types_and_values(options: NumdiffOptions) -> None: "must be an integer greater than 0." ) - if not callable(options.batch_evaluator) and options.batch_evaluator not in { - "joblib", - "pathos", - }: + try: + process_batch_evaluator(options.batch_evaluator) + except Exception as e: raise InvalidNumdiffOptionsError( - f"Invalid numdiff `batch_evaluator`: {options.batch_evaluator}. Batch " - "evaluator must be a callable or one of 'joblib', 'pathos'." - ) + f"Invalid batch evaluator: {options.batch_evaluator}." + ) from e class NumdiffPurpose(str, Enum): diff --git a/src/optimagic/examples/criterion_functions.py b/src/optimagic/examples/criterion_functions.py index 1ac139c6a..bb925d399 100644 --- a/src/optimagic/examples/criterion_functions.py +++ b/src/optimagic/examples/criterion_functions.py @@ -105,16 +105,11 @@ def rosenbrock_scalar(params: PyTree) -> float: def rosenbrock_gradient(params: PyTree) -> PyTree: """Calculate gradient of rosenbrock function.""" x = _get_x(params) - l1 = np.delete(x, [-1]) - l1 = np.append(l1, 0) - l2 = np.insert(x, 0, 0) - l2 = np.delete(l2, [1]) - l3 = np.insert(x, 0, 0) - l3 = np.delete(l3, [-1]) - l4 = np.delete(x, [0]) - l4 = np.append(l4, 0) - l5 = np.full((len(x) - 1), 2) - l5 = np.append(l5, 0) # type: ignore[assignment] + l1 = np.append(np.delete(x, [-1]), 0) + l2 = np.delete(np.insert(x, 0, 0), [1]) + l3 = np.delete(np.insert(x, 0, 0), [-1]) + l4 = np.append(np.delete(x, [0]), 0) + l5 = np.append(np.full((len(x) - 1), 2), 0) flat = 100 * (4 * (l1**3) + 2 * l2 - 2 * (l3**2) - 4 * (l4 * x)) + 2 * l1 - l5 return _unflatten_gradient(flat, params) diff --git a/src/optimagic/exceptions.py b/src/optimagic/exceptions.py index 69e4757a7..7a7dfb75d 100644 --- a/src/optimagic/exceptions.py +++ b/src/optimagic/exceptions.py @@ -47,6 +47,10 @@ class InvalidBoundsError(OptimagicError): """Exception for invalid user provided bounds.""" +class IncompleteBoundsError(OptimagicError): + """Exception when user provided bounds are incomplete.""" + + class InvalidScalingError(OptimagicError): """Exception for invalid user provided scaling.""" diff --git a/src/optimagic/logging/logger.py b/src/optimagic/logging/logger.py index 7165022a4..9d6238340 100644 --- a/src/optimagic/logging/logger.py +++ b/src/optimagic/logging/logger.py @@ -190,7 +190,7 @@ def _build_history_dataframe(self) -> pd.DataFrame: # For numpy arrays with ndim = 0, tolist() returns a scalar, which violates the # type hinting list[Any] from above. As history["time"] is always a list, this # case is safe to ignore. - history["time"] = times.tolist() # type: ignore[assignment] + history["time"] = times.tolist() df = pd.DataFrame(history) df = df.merge( @@ -242,7 +242,7 @@ def _extract_best_history( best_idx, level="step" ) - def _to_dict(pandas_obj: pd.DataFrame | pd.Series) -> dict[str, Any]: # type:ignore + def _to_dict(pandas_obj: pd.DataFrame | pd.Series) -> dict[str, Any]: if isinstance(pandas_obj, pd.DataFrame): result = pandas_obj.to_dict(orient="list") else: diff --git a/src/optimagic/mark.py b/src/optimagic/mark.py index 6f0f08c29..a3567fcb3 100644 --- a/src/optimagic/mark.py +++ b/src/optimagic/mark.py @@ -73,8 +73,10 @@ def minimizer( is_global: bool, needs_jac: bool, needs_hess: bool, + needs_bounds: bool, supports_parallelism: bool, supports_bounds: bool, + supports_infinite_bounds: bool, supports_linear_constraints: bool, supports_nonlinear_constraints: bool, disable_history: bool = False, @@ -95,12 +97,17 @@ def minimizer( needs_hess: Whether the algorithm needs some kind of second derivative. This is not yet implemented and will be False for all currently wrapped algorithms. + needs_bounds: Whether the algorithm needs bounds to run. This is different from + supports_bounds in that algorithms that support bounds can run without + requiring them. supports_parallelism: Whether the algorithm supports parallelism. This needs to be True if the algorithm previously took `n_cores` and/or `batch_evaluator` as arguments. supports_bounds: Whether the algorithm supports bounds. This needs to be True if the algorithm previously took `lower_bounds` and/or `upper_bounds` as arguments. + supports_infinite_bounds: Whether the algorithm supports infinite values in + bounds. supports_linear_constraints: Whether the algorithm supports linear constraints. This is not yet implemented and will be False for all currently wrapped algorithms. @@ -119,8 +126,10 @@ def decorator(cls: AlgorithmSubclass) -> AlgorithmSubclass: is_global=is_global, needs_jac=needs_jac, needs_hess=needs_hess, + needs_bounds=needs_bounds, supports_parallelism=supports_parallelism, supports_bounds=supports_bounds, + supports_infinite_bounds=supports_infinite_bounds, supports_linear_constraints=supports_linear_constraints, supports_nonlinear_constraints=supports_nonlinear_constraints, disable_history=disable_history, diff --git a/src/optimagic/optimization/algo_options.py b/src/optimagic/optimization/algo_options.py index 1846ba081..2f7c6fca5 100644 --- a/src/optimagic/optimization/algo_options.py +++ b/src/optimagic/optimization/algo_options.py @@ -122,6 +122,19 @@ """ +N_RESTARTS = 1 +"""int: Number of times to restart the optimizer if convergence is not reached. + This parameter controls how many times the optimization process is restarted + in an attempt to achieve convergence. + + - A value of 1 (the default) indicates that the optimizer will only run once, + disabling the restart feature. + - Values greater than 1 specify the maximum number of restart attempts. + + Note: This is distinct from `STOPPING_MAXITER`, which limits the number of + iterations within a single optimizer run, not the number of restarts. +""" + def get_population_size(population_size, x, lower_bound=10): """Default population size for genetic algorithms.""" diff --git a/src/optimagic/optimization/algorithm.py b/src/optimagic/optimization/algorithm.py index 125ad6799..ac83fe4dd 100644 --- a/src/optimagic/optimization/algorithm.py +++ b/src/optimagic/optimization/algorithm.py @@ -26,8 +26,10 @@ class AlgoInfo: is_global: bool needs_jac: bool needs_hess: bool + needs_bounds: bool supports_parallelism: bool supports_bounds: bool + supports_infinite_bounds: bool supports_linear_constraints: bool supports_nonlinear_constraints: bool disable_history: bool = False @@ -46,10 +48,14 @@ def __post_init__(self) -> None: report.append("needs_jac must be a bool") if not isinstance(self.needs_hess, bool): report.append("needs_hess must be a bool") + if not isinstance(self.needs_bounds, bool): + report.append("needs_bounds must be a bool") if not isinstance(self.supports_parallelism, bool): report.append("supports_parallelism must be a bool") if not isinstance(self.supports_bounds, bool): report.append("supports_bounds must be a bool") + if not isinstance(self.supports_infinite_bounds, bool): + report.append("supports_infinite_bounds must be a bool") if not isinstance(self.supports_linear_constraints, bool): report.append("supports_linear_constraints must be a bool") if not isinstance(self.supports_nonlinear_constraints, bool): diff --git a/src/optimagic/optimization/history.py b/src/optimagic/optimization/history.py index 70f93adb1..966316826 100644 --- a/src/optimagic/optimization/history.py +++ b/src/optimagic/optimization/history.py @@ -409,7 +409,7 @@ def _get_flat_param_names(param: PyTree) -> list[str]: if fast_path: # Mypy raises an error here because .tolist() returns a str for zero-dimensional # arrays, but the fast path is only taken for 1d arrays, so it can be ignored. - return np.arange(param.size).astype(str).tolist() # type: ignore[return-value] + return np.arange(param.size).astype(str).tolist() registry = get_registry(extended=True) return leaf_names(param, registry=registry) @@ -530,7 +530,7 @@ def _get_batch_starts_and_stops(batch_ids: list[int]) -> tuple[list[int], list[i """ ids_arr = np.array(batch_ids, dtype=np.int64) indices = np.where(ids_arr[:-1] != ids_arr[1:])[0] + 1 - list_indices: list[int] = indices.tolist() # type: ignore[assignment] + list_indices: list[int] = indices.tolist() starts = [0, *list_indices] stops = [*starts[1:], len(batch_ids)] return starts, stops diff --git a/src/optimagic/optimization/internal_optimization_problem.py b/src/optimagic/optimization/internal_optimization_problem.py index 6e0e58e46..3d630c7bf 100644 --- a/src/optimagic/optimization/internal_optimization_problem.py +++ b/src/optimagic/optimization/internal_optimization_problem.py @@ -2,7 +2,7 @@ import warnings from copy import copy from dataclasses import asdict, dataclass, replace -from typing import Any, Callable, cast +from typing import Any, Callable, Literal, cast import numpy as np from numpy.typing import NDArray @@ -274,6 +274,57 @@ def bounds(self) -> InternalBounds: """Bounds of the optimization problem.""" return self._bounds + @property + def converter(self) -> Converter: + """Converter between external and internal parameter representation. + + The converter transforms parameters between their user-provided + representation (the external representation) and the flat numpy array used + by the optimizer (the internal representation). + + This transformation includes: + - Flattening and unflattening of pytree structures. + - Applying parameter constraints via reparametrizations. + - Scaling and unscaling of parameter values. + + The Converter object provides the following main attributes: + + - ``params_to_internal``: Callable that converts a pytree of external + parameters to a flat numpy array of internal parameters. + - ``params_from_internal``: Callable that converts a flat numpy array of + internal parameters to a pytree of external parameters. + - ``derivative_to_internal``: Callable that converts the derivative + from the external parameter space to the internal space. + - ``has_transforming_constraints``: Boolean that is True if the conversion + involves constraints that are handled by reparametrization. + + Examples: + The converter is particularly useful for algorithms that require initial + values in the internal (flat) parameter space, while allowing the user + to specify these values in the more convenient external (pytree) format. + + Here's how an optimization algorithm might use the converter internally + to prepare parameters for the optimizer: + + >>> from optimagic.optimization.internal_optimization_problem import ( + ... SphereExampleInternalOptimizationProblem + ... ) + >>> import numpy as np + >>> + >>> # Optimization problem instance. + >>> problem = SphereExampleInternalOptimizationProblem() + >>> + >>> # User provided parameters in external format. + >>> user_params = np.array([1.0, 2.0, 3.0]) + >>> + >>> # Convert to internal format for optimization algorithms. + >>> internal_params = problem.converter.params_to_internal(user_params) + >>> internal_params + array([1., 2., 3.]) + + """ + return self._converter + @property def linear_constraints(self) -> list[dict[str, Any]] | None: # TODO: write a docstring as soon as we actually use this @@ -281,7 +332,26 @@ def linear_constraints(self) -> list[dict[str, Any]] | None: @property def nonlinear_constraints(self) -> list[dict[str, Any]] | None: - """Internal dictionary representation of nonlinear constraints.""" + """Internal representation of nonlinear constraints. + + Compared to the user provided constraints, we have done the following + transformations: + + 1. The constraint a <= g(x) <= b is transformed to h(x) >= 0, where h(x) is + - h(x) = g(x), if a == 0 and b == inf + - h(x) = g(x) - a, if a != 0 and b == inf + - h(x) = (g(x) - a, -g(x) + b) >= 0, if a != 0 and b != inf. + + 2. The equality constraint g(x) = v is transformed to h(x) >= 0, where + h(x) = (g(x) - v, -g(x) + v). + + 3. Vector constraints are transformed to a list of scalar constraints. + g(x) = (g1(x), g2(x), ...) >= 0 is transformed to (g1(x) >= 0, g2(x) >= 0, ...). + + 4. The constraint function (defined on a selection of user-facing parameters) is + transformed to be evaluated on the internal parameters. + + """ return self._nonlinear_constraints @property @@ -471,6 +541,9 @@ def _pure_evaluate_jac( out_jac = _process_jac_value( value=jac_value, direction=self._direction, converter=self._converter, x=x ) + _assert_finite_jac( + out_jac=out_jac, jac_value=jac_value, params=params, origin="jac" + ) stop_time = time.perf_counter() @@ -543,6 +616,13 @@ def func(x: NDArray[np.float64]) -> SpecificFunctionValue: warnings.warn(msg) fun_value, jac_value = self._error_penalty_func(x) + _assert_finite_jac( + out_jac=jac_value, + jac_value=jac_value, + params=self._converter.params_from_internal(x), + origin="numerical", + ) + algo_fun_value, hist_fun_value = _process_fun_value( value=fun_value, # type: ignore solver_type=self._solver_type, @@ -682,6 +762,10 @@ def _pure_evaluate_fun_and_jac( if self._direction == Direction.MAXIMIZE: out_jac = -out_jac + _assert_finite_jac( + out_jac=out_jac, jac_value=jac_value, params=params, origin="fun_and_jac" + ) + stop_time = time.perf_counter() hist_entry = HistoryEntry( @@ -705,6 +789,44 @@ def _pure_evaluate_fun_and_jac( return (algo_fun_value, out_jac), hist_entry, log_entry +def _assert_finite_jac( + out_jac: NDArray[np.float64], + jac_value: PyTree, + params: PyTree, + origin: Literal["numerical", "jac", "fun_and_jac"], +) -> None: + """Check for infinite and NaN values in the Jacobian and raise an error if found. + + Args: + out_jac: internal processed Jacobian to check for finiteness. + jac_value: original Jacobian value as returned by the user function, + params: user-facing parameter representation at evaluation point. + origin: Source of Jacobian calculation, for the error message. + + Raises: + UserFunctionRuntimeError: + If any infinite or NaN values are found in the Jacobian. + + """ + if not np.all(np.isfinite(out_jac)): + if origin == "jac" or "fun_and_jac": + msg = ( + "The optimization failed because the derivative provided via " + f"{origin} contains infinite or NaN values." + "\nPlease validate the derivative function." + ) + elif origin == "numerical": + msg = ( + "The optimization failed because the numerical derivative " + "(computed using fun) contains infinite or NaN values." + "\nPlease validate the criterion function or try a different optimizer." + ) + msg += ( + f"\nParameters at evaluation point: {params}\nJacobian values: {jac_value}" + ) + raise UserFunctionRuntimeError(msg) + + def _process_fun_value( value: SpecificFunctionValue, solver_type: AggregationLevel, diff --git a/src/optimagic/optimization/multistart.py b/src/optimagic/optimization/multistart.py index 9b7544430..e990dd79f 100644 --- a/src/optimagic/optimization/multistart.py +++ b/src/optimagic/optimization/multistart.py @@ -12,7 +12,7 @@ """ import warnings -from dataclasses import replace +from dataclasses import dataclass, replace from typing import Literal import numpy as np @@ -85,8 +85,8 @@ def run_multistart_optimization( scheduled_steps = scheduled_steps[1:] - sorted_sample = exploration_res["sorted_sample"] - sorted_values = exploration_res["sorted_values"] + sorted_sample = exploration_res.sorted_sample + sorted_values = exploration_res.sorted_values stopping_maxopt = options.stopping_maxopt if stopping_maxopt > len(sorted_sample): @@ -172,7 +172,7 @@ def single_optimization(x0, step_id): "start_parameters": state["start_history"], "local_optima": state["result_history"], "exploration_sample": sorted_sample, - "exploration_results": exploration_res["sorted_values"], + "exploration_results": sorted_values, } raw_res = state["best_res"] @@ -288,12 +288,27 @@ def _draw_exploration_sample( return sample_scaled +@dataclass(frozen=True) +class _InternalExplorationResult: + """Exploration result of the multistart optimization. + + Attributes: + sorted_values: List of sorted function values. + sorted_sample: 2d numpy array where each row is the internal parameter + vector corresponding to the sorted function values. + + """ + + sorted_values: list[float] + sorted_sample: NDArray[np.float64] + + def run_explorations( internal_problem: InternalOptimizationProblem, sample: NDArray[np.float64], n_cores: int, step_id: int, -) -> dict[str, NDArray[np.float64]]: +) -> _InternalExplorationResult: """Do the function evaluations for the exploration phase. Args: @@ -305,11 +320,11 @@ def run_explorations( step_id: The identifier of the exploration step. Returns: - dict: A dictionary with the the following entries: - "sorted_values": 1d numpy array with sorted function values. Invalid - function values are excluded. - "sorted_sample": 2d numpy array with corresponding internal parameter - vectors. + A data object containing + - sorted_values: List of sorted function values. Invalid function values are + excluded. + - sorted_sample: 2d numpy array where each row is the internal parameter + vector corresponding to the sorted function values. """ internal_problem = internal_problem.with_step_id(step_id) @@ -334,10 +349,10 @@ def run_explorations( # of the sign switch. sorting_indices = np.argsort(valid_values) - out = { - "sorted_values": valid_values[sorting_indices], - "sorted_sample": valid_sample[sorting_indices], - } + out = _InternalExplorationResult( + sorted_values=valid_values[sorting_indices].tolist(), + sorted_sample=valid_sample[sorting_indices], + ) return out diff --git a/src/optimagic/optimization/multistart_options.py b/src/optimagic/optimization/multistart_options.py index 5c5899815..450c2a951 100644 --- a/src/optimagic/optimization/multistart_options.py +++ b/src/optimagic/optimization/multistart_options.py @@ -9,7 +9,7 @@ from optimagic.batch_evaluators import process_batch_evaluator from optimagic.deprecations import replace_and_warn_about_deprecated_multistart_options from optimagic.exceptions import InvalidMultistartError -from optimagic.typing import BatchEvaluator, PyTree +from optimagic.typing import BatchEvaluator, BatchEvaluatorLiteral, PyTree # ====================================================================================== # Public Options @@ -45,8 +45,8 @@ class MultistartOptions: for convergence. Determines the maximum relative distance two parameter vecctors can have to be considered equal. Defaults to 0.01. n_cores: The number of cores to use for parallelization. Defaults to 1. - batch_evaluator: The evaluator to use for batch evaluation. Allowed are "joblib" - and "pathos", or a custom callable. + batch_evaluator: The evaluator to use for batch evaluation. Allowed are + "joblib", "pathos", and "threading", or a custom callable. batch_size: The batch size for batch evaluation. Must be larger than n_cores or None. seed: The seed for the random number generator. @@ -71,7 +71,7 @@ class MultistartOptions: convergence_xtol_rel: float | None = None convergence_max_discoveries: int = 2 n_cores: int = 1 - batch_evaluator: Literal["joblib", "pathos"] | BatchEvaluator = "joblib" + batch_evaluator: BatchEvaluatorLiteral | BatchEvaluator = "joblib" batch_size: int | None = None seed: int | np.random.Generator | None = None error_handling: Literal["raise", "continue"] | None = None @@ -100,7 +100,7 @@ class MultistartOptionsDict(TypedDict): convergence_xtol_rel: NotRequired[float | None] convergence_max_discoveries: NotRequired[int] n_cores: NotRequired[int] - batch_evaluator: NotRequired[Literal["joblib", "pathos"] | BatchEvaluator] + batch_evaluator: NotRequired[BatchEvaluatorLiteral | BatchEvaluator] batch_size: NotRequired[int | None] seed: NotRequired[int | np.random.Generator | None] error_handling: NotRequired[Literal["raise", "continue"] | None] @@ -247,14 +247,12 @@ def _validate_attribute_types_and_values(options: MultistartOptions) -> None: "must be a positive integer." ) - if not callable(options.batch_evaluator) and options.batch_evaluator not in ( - "joblib", - "pathos", - ): + try: + process_batch_evaluator(options.batch_evaluator) + except Exception as e: raise InvalidMultistartError( - f"Invalid batch evaluator: {options.batch_evaluator}. Batch evaluator " - "must be a Callable or one of 'joblib' or 'pathos'." - ) + f"Invalid batch evaluator: {options.batch_evaluator}." + ) from e if options.batch_size is not None and ( not isinstance(options.batch_size, int) or options.batch_size < options.n_cores diff --git a/src/optimagic/optimization/optimize.py b/src/optimagic/optimization/optimize.py index 7935de635..cd3b0caab 100644 --- a/src/optimagic/optimization/optimize.py +++ b/src/optimagic/optimization/optimize.py @@ -17,12 +17,14 @@ from pathlib import Path from typing import Any, Callable, Sequence, Type, cast +import numpy as np from scipy.optimize import Bounds as ScipyBounds from optimagic.batch_evaluators import process_batch_evaluator from optimagic.constraints import Constraint from optimagic.differentiation.numdiff_options import NumdiffOptions, NumdiffOptionsDict from optimagic.exceptions import ( + IncompleteBoundsError, InvalidFunctionError, ) from optimagic.logging.logger import LogReader, LogStore @@ -555,6 +557,38 @@ def _optimize(problem: OptimizationProblem) -> OptimizeResult: else: logger = None + # ================================================================================== + # Strict checking if bounds are required and infinite values in bounds + # ================================================================================== + if problem.algorithm.algo_info.supports_bounds: + bounds_missing = ( + internal_params.lower_bounds is None or internal_params.upper_bounds is None + ) + + # Check for infinite values in bounds arrays (only possible in mixed cases now) + infinite_values_in_bounds = False + if internal_params.lower_bounds is not None: + infinite_values_in_bounds |= np.isinf(internal_params.lower_bounds).any() + if internal_params.upper_bounds is not None: + infinite_values_in_bounds |= np.isinf(internal_params.upper_bounds).any() + + # Case 1: Algorithm needs bounds but none provided + if problem.algorithm.algo_info.needs_bounds and bounds_missing: + raise IncompleteBoundsError( + f"Algorithm {problem.algorithm.name} requires bounds for all " + "parameters. Please provide finite lower and upper bounds." + ) + + # Case 2: Algorithm doesn't support infinite bounds but they are present + if ( + not problem.algorithm.algo_info.supports_infinite_bounds + and infinite_values_in_bounds + ): + raise IncompleteBoundsError( + f"Algorithm {problem.algorithm.name} does not support infinite bounds. " + "Please provide finite bounds for all parameters." + ) + # ================================================================================== # Do some things that require internal parameters or bounds # ================================================================================== @@ -589,6 +623,7 @@ def _optimize(problem: OptimizationProblem) -> OptimizeResult: lower=internal_params.lower_bounds, upper=internal_params.upper_bounds, ) + # ================================================================================== # Create a batch evaluator # ================================================================================== diff --git a/src/optimagic/optimization/process_results.py b/src/optimagic/optimization/process_results.py index 64d764174..f9a2c191e 100644 --- a/src/optimagic/optimization/process_results.py +++ b/src/optimagic/optimization/process_results.py @@ -137,6 +137,9 @@ def _process_multistart_info( solver_type: AggregationLevel, extra_fields: ExtraResultFields, ) -> MultistartInfo: + # The `info` dictionary is obtained from the `multistart_info` field of the + # InternalOptimizeResult returned by `run_multistart_optimization` function. + starts = [converter.params_from_internal(x) for x in info["start_parameters"]] optima = [] diff --git a/src/optimagic/optimizers/_pounders/bntr.py b/src/optimagic/optimizers/_pounders/bntr.py index fe039e7cb..341cfcaa9 100644 --- a/src/optimagic/optimizers/_pounders/bntr.py +++ b/src/optimagic/optimizers/_pounders/bntr.py @@ -67,9 +67,9 @@ def bntr( x_candidate (np.ndarray): Initial guess for the solution of the subproblem. conjugate_gradient_method (str): Method for computing the conjugate gradient step. Available conjugate gradient methods are: - - "cg" - - "steihaug_toint" - - "trsbox" (default) + - "cg" + - "steihaug_toint" + - "trsbox" (default) maxiter (int): Maximum number of iterations. If reached, terminate. maxiter_gradient_descent (int): Maximum number of steepest descent iterations to perform when the trust-region subsolver BNTR is used. diff --git a/src/optimagic/optimizers/_pounders/pounders_auxiliary.py b/src/optimagic/optimizers/_pounders/pounders_auxiliary.py index 223c028fe..f3c73177f 100644 --- a/src/optimagic/optimizers/_pounders/pounders_auxiliary.py +++ b/src/optimagic/optimizers/_pounders/pounders_auxiliary.py @@ -36,7 +36,7 @@ def create_initial_residual_model(history, accepted_index, delta): Returns: ResidualModel: Residual model containing the initial parameters for - ``linear_terms`` and ``square_terms``. + ``linear_terms`` and ``square_terms``. """ center_info = { @@ -221,9 +221,9 @@ def solve_subproblem( - "gqtpar" (does not support bound constraints) conjugate_gradient_method (str): Method for computing the conjugate gradient step. Available conjugate gradient methods are: - - "cg" - - "steihaug_toint" - - "trsbox" (default) + - "cg" + - "steihaug_toint" + - "trsbox" (default) maxiter (int): Maximum number of iterations to perform when solving the trust-region subproblem. maxiter_gradient_descent (int): Maximum number of gradient descent iterations @@ -336,7 +336,7 @@ def find_affine_points( If *project_x_onto_null* is False, it is an array filled with zeros. project_x_onto_null (int): Indicator whether to calculate the QR decomposition of *model_improving_points* and multiply it - with vector *x_projected*. + with vector *x_projected*. delta (float): Delta, current trust-region radius. theta1 (float): Threshold for adding the current x candidate to the model. c (float): Threshold for acceptance of the norm of our current x candidate. diff --git a/src/optimagic/optimizers/bayesian_optimizer.py b/src/optimagic/optimizers/bayesian_optimizer.py new file mode 100644 index 000000000..3de716a7f --- /dev/null +++ b/src/optimagic/optimizers/bayesian_optimizer.py @@ -0,0 +1,353 @@ +"""Implement Bayesian optimization using bayes_opt.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal, Type + +import numpy as np +from numpy.typing import NDArray +from scipy.optimize import NonlinearConstraint + +from optimagic import mark +from optimagic.config import IS_BAYESOPT_INSTALLED +from optimagic.exceptions import NotInstalledError +from optimagic.optimization.algo_options import N_RESTARTS +from optimagic.optimization.algorithm import Algorithm, InternalOptimizeResult +from optimagic.optimization.internal_optimization_problem import ( + InternalBounds, + InternalOptimizationProblem, +) +from optimagic.typing import ( + AggregationLevel, + NonNegativeFloat, + NonNegativeInt, + PositiveFloat, + PositiveInt, + UnitIntervalFloat, +) + +if TYPE_CHECKING: + from bayes_opt import BayesianOptimization + from bayes_opt.acquisition import AcquisitionFunction + + +@mark.minimizer( + name="bayes_opt", + solver_type=AggregationLevel.SCALAR, + is_available=IS_BAYESOPT_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=True, + supports_parallelism=False, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, # temp + disable_history=False, +) +@dataclass(frozen=True) +class BayesOpt(Algorithm): + init_points: PositiveInt = 5 + n_iter: PositiveInt = 25 + verbose: Literal[0, 1, 2] = 0 + kappa: NonNegativeFloat = 2.576 + xi: PositiveFloat = 0.01 + exploration_decay: UnitIntervalFloat | None = None + exploration_decay_delay: NonNegativeInt | None = None + random_state: int | None = None + acquisition_function: ( + str | AcquisitionFunction | Type[AcquisitionFunction] | None + ) = None + allow_duplicate_points: bool = False + enable_sdr: bool = False + sdr_gamma_osc: float = 0.7 + sdr_gamma_pan: float = 1.0 + sdr_eta: float = 0.9 + sdr_minimum_window: NonNegativeFloat = 0.0 + alpha: float = 1e-6 + n_restarts: int = N_RESTARTS + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_BAYESOPT_INSTALLED: + raise NotInstalledError( + "To use the 'bayes_opt' optimizer you need to install bayes_opt. " + "Use 'pip install bayesian-optimization'. " + "Check the documentation for more details: " + "https://bayesian-optimization.github.io/BayesianOptimization/index.html" + ) + + from bayes_opt import BayesianOptimization + + pbounds = _process_bounds(problem.bounds) + + acq = _process_acquisition_function( + acquisition_function=self.acquisition_function, + kappa=self.kappa, + xi=self.xi, + exploration_decay=self.exploration_decay, + exploration_decay_delay=self.exploration_decay_delay, + random_state=self.random_state, + ) + + constraint = None + constraint = self._process_constraints(problem.nonlinear_constraints) + + def objective(**kwargs: dict[str, float]) -> float: + x = _extract_params_from_kwargs(kwargs) + return -float( + problem.fun(x) + ) # Negate to convert minimization to maximization + + bounds_transformer = None + if self.enable_sdr: + from bayes_opt import SequentialDomainReductionTransformer + + bounds_transformer = SequentialDomainReductionTransformer( + gamma_osc=self.sdr_gamma_osc, + gamma_pan=self.sdr_gamma_pan, + eta=self.sdr_eta, + minimum_window=self.sdr_minimum_window, + ) + + optimizer = BayesianOptimization( + f=objective, + pbounds=pbounds, + acquisition_function=acq, + constraint=constraint, + random_state=self.random_state, + verbose=self.verbose, + bounds_transformer=bounds_transformer, + allow_duplicate_points=self.allow_duplicate_points, + ) + + # Set Gaussian Process parameters + optimizer.set_gp_params(alpha=self.alpha, n_restarts_optimizer=self.n_restarts) + + # Use initial point as first probe + probe_params = {f"param{i}": float(val) for i, val in enumerate(x0)} + optimizer.probe( + params=probe_params, + lazy=True, + ) + optimizer.maximize( + init_points=self.init_points, + n_iter=self.n_iter, + ) + + res = _process_bayes_opt_result(optimizer=optimizer, x0=x0, problem=problem) + return res + + def _process_constraints( + self, constraints: list[dict[str, Any]] | None + ) -> NonlinearConstraint | None: + """Temporarily skip processing of nonlinear constraints. + + Args: + constraints: List of constraint dictionaries from the problem + + Returns: + None. Nonlinear constraint processing is deferred. + + """ + # TODO: Implement proper handling of nonlinear constraints in future. + return None + + +def _process_bounds(bounds: InternalBounds) -> dict[str, tuple[float, float]]: + """Process bounds for bayesian optimization. + + Args: + bounds: Internal bounds object. + + Returns: + Dictionary mapping parameter names to (lower, upper) bound tuples. + + Raises: + ValueError: If bounds are None or infinite. + + """ + if not ( + bounds.lower is not None + and bounds.upper is not None + and np.all(np.isfinite(bounds.lower)) + and np.all(np.isfinite(bounds.upper)) + ): + raise ValueError( + "Bayesian optimization requires finite bounds for all parameters. " + "Bounds cannot be None or infinite." + ) + + return { + f"param{i}": (lower, upper) + for i, (lower, upper) in enumerate(zip(bounds.lower, bounds.upper, strict=True)) + } + + +def _extract_params_from_kwargs(params_dict: dict[str, Any]) -> NDArray[np.float64]: + """Extract parameters from kwargs dictionary. + + Args: + params_dict: Dictionary with parameter values. + + Returns: + Array of parameter values. + + """ + return np.array(list(params_dict.values())) + + +def _process_acquisition_function( + acquisition_function: ( + str | AcquisitionFunction | Type[AcquisitionFunction] | None + ), + kappa: NonNegativeFloat, + xi: PositiveFloat, + exploration_decay: float | None, + exploration_decay_delay: NonNegativeInt | None, + random_state: int | None, +) -> AcquisitionFunction | None: + """Create and return the appropriate acquisition function. + + Args: + acquisition_function: The acquisition function specification. + Can be one of the following: + - A string: "upper_confidence_bound" (or "ucb"), "expected_improvement" + (or "ei"), "probability_of_improvement" (or "poi") + - An instance of `AcquisitionFunction` + - A class inheriting from `AcquisitionFunction` + - None (uses the default acquisition function from the bayes_opt package) + kappa: Exploration-exploitation trade-off parameter for Upper Confidence Bound + acquisition function. Higher values favor exploration over exploitation. + xi: Exploration-exploitation trade-off parameter for Expected Improvement and + Probability of Improvement acquisition functions. Higher values favor + exploration over exploitation. + exploration_decay: Rate at which exploration parameters (kappa or xi) decay + over time. None means no decay is applied. + exploration_decay_delay: Number of iterations before starting the decay. + None means decay is applied from the start. + random_state: Random seed for reproducibility. + + Returns: + The configured acquisition function instance or None for default. + + Raises: + ValueError: If acquisition_function is an invalid string. + TypeError: If acquisition_function is not a string, an AcquisitionFunction + instance, a class inheriting from AcquisitionFunction, or None. + + """ + + from bayes_opt import acquisition + + acquisition_function_aliases = { + "ucb": "ucb", + "upper_confidence_bound": "ucb", + "ei": "ei", + "expected_improvement": "ei", + "poi": "poi", + "probability_of_improvement": "poi", + } + + if acquisition_function is None: + return None + + elif isinstance(acquisition_function, str): + acq_name = acquisition_function.lower() + + if acq_name not in acquisition_function_aliases: + raise ValueError( + f"Invalid acquisition_function string: '{acquisition_function}'. " + f"Must be one of: {', '.join(acquisition_function_aliases.keys())}" + ) + + canonical_name = acquisition_function_aliases[acq_name] + + if canonical_name == "ucb": + return acquisition.UpperConfidenceBound( + kappa=kappa, + exploration_decay=exploration_decay, + exploration_decay_delay=exploration_decay_delay, + random_state=random_state, + ) + elif canonical_name == "ei": + return acquisition.ExpectedImprovement( + xi=xi, + exploration_decay=exploration_decay, + exploration_decay_delay=exploration_decay_delay, + random_state=random_state, + ) + elif canonical_name == "poi": + return acquisition.ProbabilityOfImprovement( + xi=xi, + exploration_decay=exploration_decay, + exploration_decay_delay=exploration_decay_delay, + random_state=random_state, + ) + else: + raise ValueError(f"Unhandled canonical name: {canonical_name}") + + # If acquisition_function is an instance of AcquisitionFunction class + elif isinstance(acquisition_function, acquisition.AcquisitionFunction): + return acquisition_function + + # If acquisition_function is a class inheriting from AcquisitionFunction + elif isinstance(acquisition_function, type) and issubclass( + acquisition_function, acquisition.AcquisitionFunction + ): + return acquisition_function() + + else: + raise TypeError( + "acquisition_function must be None, a string, " + "an AcquisitionFunction instance, or a class inheriting from " + "AcquisitionFunction. " + f"Got type: {type(acquisition_function).__name__}" + ) + + +def _process_bayes_opt_result( + optimizer: BayesianOptimization, + x0: NDArray[np.float64], + problem: InternalOptimizationProblem, +) -> InternalOptimizeResult: + """Convert BayesianOptimization result to InternalOptimizeResult format. + + Args: + optimizer: The BayesianOptimization instance after optimization + x0: Initial parameter values + problem: The internal optimization problem + + Returns: + InternalOptimizeResult with processed results + + """ + n_evals = len(optimizer.space) + + if optimizer.max is not None: + best_params = optimizer.max["params"] + best_x = _extract_params_from_kwargs(best_params) + best_y = -optimizer.max["target"] # Un-negate the result + success = True + message = "Optimization succeeded" + else: + best_x = x0 + best_y = float(problem.fun(x0)) + success = False + message = ( + "Optimization did not succeed " + "returning the initial point as the best available result." + ) + + return InternalOptimizeResult( + x=best_x, + fun=best_y, + success=success, + message=message, + n_iterations=n_evals, + n_fun_evals=n_evals, + n_jac_evals=0, + ) diff --git a/src/optimagic/optimizers/bhhh.py b/src/optimagic/optimizers/bhhh.py index f0528c461..c7e3a1539 100644 --- a/src/optimagic/optimizers/bhhh.py +++ b/src/optimagic/optimizers/bhhh.py @@ -21,8 +21,10 @@ is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, diff --git a/src/optimagic/optimizers/fides.py b/src/optimagic/optimizers/fides.py index a5af078d0..4d90814c0 100644 --- a/src/optimagic/optimizers/fides.py +++ b/src/optimagic/optimizers/fides.py @@ -29,9 +29,6 @@ PositiveInt, ) -if IS_FIDES_INSTALLED: - from fides import Optimizer, hessian_approximation - @mark.minimizer( name="fides", @@ -40,8 +37,10 @@ is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -161,6 +160,8 @@ def fides_internal( "You can install it with `pip install fides>=0.7.4`." ) + from fides import Optimizer + fides_options = { "delta_init": trustregion_initial_radius, "eta": trustregion_increase_threshold, @@ -181,6 +182,9 @@ def fides_internal( hessian_instance = _create_hessian_updater_from_user_input(hessian_update_strategy) + lower_bounds = np.full(len(x), -np.inf) if lower_bounds is None else lower_bounds + upper_bounds = np.full(len(x), np.inf) if upper_bounds is None else upper_bounds + opt = Optimizer( fun=fun_and_jac, lb=lower_bounds, @@ -246,6 +250,8 @@ def _process_exitflag(exitflag): def _create_hessian_updater_from_user_input(hessian_update_strategy): + from fides import hessian_approximation + hessians_needing_residuals = ( hessian_approximation.FX, hessian_approximation.SSM, diff --git a/src/optimagic/optimizers/iminuit_migrad.py b/src/optimagic/optimizers/iminuit_migrad.py new file mode 100644 index 000000000..4c6490e74 --- /dev/null +++ b/src/optimagic/optimizers/iminuit_migrad.py @@ -0,0 +1,146 @@ +"""Implement the MIGRAD algorithm from iminuit.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Optional + +import numpy as np +from numpy.typing import NDArray + +from optimagic import mark +from optimagic.config import IS_IMINUIT_INSTALLED +from optimagic.exceptions import NotInstalledError +from optimagic.optimization.algo_options import ( + N_RESTARTS, + STOPPING_MAXFUN, +) +from optimagic.optimization.algorithm import Algorithm, InternalOptimizeResult +from optimagic.optimization.internal_optimization_problem import ( + InternalOptimizationProblem, +) +from optimagic.typing import AggregationLevel + +if TYPE_CHECKING: + from iminuit import Minuit + + +@mark.minimizer( + name="iminuit_migrad", + solver_type=AggregationLevel.SCALAR, + is_available=IS_IMINUIT_INSTALLED, + is_global=False, + needs_jac=True, + needs_hess=False, + needs_bounds=False, + supports_parallelism=False, + supports_bounds=True, + supports_infinite_bounds=True, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class IminuitMigrad(Algorithm): + stopping_maxfun: int = STOPPING_MAXFUN + n_restarts: int = N_RESTARTS + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, params: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_IMINUIT_INSTALLED: + raise NotInstalledError( # pragma: no cover + "To use the 'iminuit_migrad` optimizer you need to install iminuit. " + "Use 'pip install iminuit' or 'conda install -c conda-forge iminuit'. " + "Check the iminuit documentation for more details: " + "https://scikit-hep.org/iminuit/install.html" + ) + from iminuit import Minuit + + def wrapped_objective(x: NDArray[np.float64]) -> float: + return float(problem.fun(x)) + + m = Minuit(wrapped_objective, params, grad=problem.jac) + + bounds = _convert_bounds_to_minuit_limits( + problem.bounds.lower, problem.bounds.upper + ) + + for i, (lower, upper) in enumerate(bounds): + if lower is not None or upper is not None: + m.limits[i] = (lower, upper) + + m.migrad( + ncall=self.stopping_maxfun, + iterate=self.n_restarts, + ) + + res = _process_minuit_result(m) + return res + + +def _process_minuit_result(minuit_result: Minuit) -> InternalOptimizeResult: + """Convert iminuit result to optimagic's internal result format.""" + + res = InternalOptimizeResult( + x=np.array(minuit_result.values), + fun=minuit_result.fval, + success=minuit_result.valid, + message=repr(minuit_result.fmin), + n_fun_evals=minuit_result.nfcn, + n_jac_evals=minuit_result.ngrad, + n_hess_evals=None, + n_iterations=minuit_result.nfcn, + status=None, + jac=None, + hess=None, + hess_inv=np.array(minuit_result.covariance), + max_constraint_violation=None, + info=None, + history=None, + ) + return res + + +def _convert_bounds_to_minuit_limits( + lower_bounds: Optional[NDArray[np.float64]], + upper_bounds: Optional[NDArray[np.float64]], +) -> list[tuple[Optional[float], Optional[float]]]: + """Convert optimization bounds to Minuit-compatible limit format. + + Transforms numpy arrays of bounds into List of tuples as expected by iminuit. + Handles special values like np.inf, -np.inf, and np.nan by converting + them to None where appropriate, as required by Minuit's limits API. + + Parameters + ---------- + lower_bounds : Optional[NDArray[np.float64]] + Array of lower bounds for parameters. + upper_bounds : Optional[NDArray[np.float64]] + Array of upper bounds for parameters. + + Returns + ------- + list[tuple[Optional[float], Optional[float]]] + List of (lower, upper) limit tuples in Minuit format, where: + - None indicates unbounded (equivalent to infinity) + - Float values represent actual bounds + + Notes + ----- + Minuit expects bounds as tuples of (lower, upper) where: + - `None` indicates no bound (equivalent to -inf or +inf) + - A finite float value indicates a specific bound + - Bounds can be asymmetric (e.g., one side bounded, one side not) + + """ + if lower_bounds is None or upper_bounds is None: + return [] + + return [ + ( + None if np.isneginf(lower) or np.isnan(lower) else float(lower), + None if np.isposinf(upper) or np.isnan(upper) else float(upper), + ) + for lower, upper in zip(lower_bounds, upper_bounds, strict=True) + ] diff --git a/src/optimagic/optimizers/ipopt.py b/src/optimagic/optimizers/ipopt.py index 2442ebe78..ba23074c0 100644 --- a/src/optimagic/optimizers/ipopt.py +++ b/src/optimagic/optimizers/ipopt.py @@ -30,9 +30,6 @@ YesNoBool, ) -if IS_CYIPOPT_INSTALLED: - import cyipopt - @mark.minimizer( name="ipopt", @@ -41,8 +38,10 @@ is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -347,11 +346,14 @@ class Ipopt(Algorithm): def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] ) -> InternalOptimizeResult: - if not self.algo_info.is_available: + if not IS_CYIPOPT_INSTALLED: raise NotInstalledError( - "The 'ipopt' algorithm requires the cyipopt package to be installed. " - "You can it with: `conda install -c conda-forge cyipopt`." + "The 'ipopt' algorithm requires the cyipopt package to be installed.\n" + "You can install it with: `conda install -c conda-forge cyipopt`." ) + + import cyipopt + if self.acceptable_tol <= self.convergence_ftol_rel: raise ValueError( "The acceptable tolerance must be larger than the desired tolerance." diff --git a/src/optimagic/optimizers/nag_optimizers.py b/src/optimagic/optimizers/nag_optimizers.py index 8c8d6875a..5546e847b 100644 --- a/src/optimagic/optimizers/nag_optimizers.py +++ b/src/optimagic/optimizers/nag_optimizers.py @@ -31,13 +31,6 @@ ) from optimagic.utilities import calculate_trustregion_initial_radius -if IS_PYBOBYQA_INSTALLED: - import pybobyqa - -if IS_DFOLS_INSTALLED: - import dfols - - CONVERGENCE_MINIMAL_TRUSTREGION_RADIUS_TOLERANCE = 1e-8 """float: Stop when the lower trust region radius falls below this value.""" @@ -342,8 +335,10 @@ is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -484,6 +479,8 @@ def nag_dfols_internal( "For additional installation instructions visit: ", r"https://numericalalgorithmsgroup.github.io/dfols/build/html/install.html", ) + import dfols + if trustregion_method_to_replace_extra_points == "momentum": trustregion_use_momentum = True elif trustregion_method_to_replace_extra_points in ["geometry_improving", None]: @@ -643,8 +640,10 @@ def nag_dfols_internal( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -784,6 +783,7 @@ def nag_pybobyqa_internal( r"https://numericalalgorithmsgroup.github.io/pybobyqa/build/html/" "install.html", ) + import pybobyqa if convergence_criterion_value is None: convergence_criterion_value = -np.inf diff --git a/src/optimagic/optimizers/neldermead.py b/src/optimagic/optimizers/neldermead.py index 233f61370..4e45da13a 100644 --- a/src/optimagic/optimizers/neldermead.py +++ b/src/optimagic/optimizers/neldermead.py @@ -31,8 +31,10 @@ is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=True, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=True, diff --git a/src/optimagic/optimizers/nevergrad_optimizers.py b/src/optimagic/optimizers/nevergrad_optimizers.py new file mode 100644 index 000000000..16166b0a9 --- /dev/null +++ b/src/optimagic/optimizers/nevergrad_optimizers.py @@ -0,0 +1,1089 @@ +"""Implement optimizers from the nevergrad package.""" + +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np +from numpy.typing import NDArray + +from optimagic import mark +from optimagic.config import IS_NEVERGRAD_INSTALLED +from optimagic.exceptions import NotInstalledError +from optimagic.optimization.algo_options import ( + CONVERGENCE_FTOL_ABS, + CONVERGENCE_FTOL_REL, + CONVERGENCE_XTOL_ABS, + STOPPING_MAXFUN_GLOBAL, + STOPPING_MAXITER, +) +from optimagic.optimization.algorithm import Algorithm, InternalOptimizeResult +from optimagic.optimization.internal_optimization_problem import ( + InternalOptimizationProblem, +) +from optimagic.typing import ( + AggregationLevel, + NonNegativeFloat, + NonNegativeInt, + PositiveFloat, + PositiveInt, +) + +if TYPE_CHECKING: + import nevergrad as ng + + +NEVERGRAD_NOT_INSTALLED_ERROR = ( + "This optimizer requires the 'nevergrad' package to be installed. " + "You can install it with `pip install nevergrad`. " + "Visit https://facebookresearch.github.io/nevergrad/getting_started.html " + "for more detailed installation instructions." +) + + +@mark.minimizer( + name="nevergrad_pso", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradPSO(Algorithm): + transform: Literal["arctan", "gaussian", "identity"] = "arctan" + population_size: int | None = None + n_cores: int = 1 + seed: int | None = None + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + inertia: float = 0.5 / math.log(2.0) + cognitive: float = 0.5 + math.log(2.0) + social: float = 0.5 + math.log(2.0) + quasi_opp_init: bool = False + speed_quasi_opp_init: bool = False + special_speed_quasi_opp_init: bool = False + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.ConfPSO( + transform=self.transform, + popsize=self.population_size, + omega=self.inertia, + phip=self.cognitive, + phig=self.social, + qo=self.quasi_opp_init, + sqo=self.speed_quasi_opp_init, + so=self.special_speed_quasi_opp_init, + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + + return res + + +@mark.minimizer( + name="nevergrad_cmaes", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradCMAES(Algorithm): + scale: NonNegativeFloat = 1.0 + elitist: bool = False + population_size: int | None = None + diagonal: bool = False + high_speed: bool = False + fast_cmaes: bool = False + random_init: bool = False + n_cores: PositiveInt = 1 + step_size_adaptive: bool | str = True + CSA_dampfac: PositiveFloat = 1.0 + CMA_dampsvec_fade: PositiveFloat = 0.1 + CSA_squared: bool = False + CMA_on: float = 1.0 + CMA_rankone: float = 1.0 + CMA_rankmu: float = 1.0 + CMA_cmean: float = 1.0 + CMA_diagonal_decoding: float = 0.0 + num_parents: int | None = None + CMA_active: bool = True + CMA_mirrormethod: Literal[0, 1, 2] = 2 + CMA_const_trace: bool | Literal["arithm", "geom", "aeig", "geig"] = False + CMA_diagonal: int | bool = False + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + stopping_maxiter: PositiveInt = STOPPING_MAXITER + stopping_maxtime: PositiveFloat = float("inf") + stopping_cov_mat_cond: NonNegativeFloat = 1e14 + convergence_ftol_abs: NonNegativeFloat = CONVERGENCE_FTOL_ABS + convergence_ftol_rel: NonNegativeFloat = CONVERGENCE_FTOL_REL + convergence_xtol_abs: NonNegativeFloat = CONVERGENCE_XTOL_ABS + convergence_iter_noimprove: PositiveInt | None = None + invariant_path: bool = False + eval_final_mean: bool = True + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + cma_options = { + "AdaptSigma": self.step_size_adaptive, + "CSA_dampfac": self.CSA_dampfac, + "CMA_dampsvec_fade": self.CMA_dampsvec_fade, + "CSA_squared": self.CSA_squared, + "CSA_invariant_path": self.invariant_path, + "CMA_on": self.CMA_on, + "CMA_rankone": self.CMA_rankone, + "CMA_rankmu": self.CMA_rankmu, + "CMA_cmean": self.CMA_cmean, + "CMA_diagonal_decoding": self.CMA_diagonal_decoding, + "CMA_mu": self.num_parents, + "CMA_active": self.CMA_active, + "CMA_mirrormethod": self.CMA_mirrormethod, + "CMA_const_trace": self.CMA_const_trace, + "CMA_diagonal": self.CMA_diagonal, + "maxfevals": self.stopping_maxfun, + "maxiter": self.stopping_maxiter, + "timeout": self.stopping_maxtime, + "tolconditioncov": self.stopping_cov_mat_cond, + "tolfun": self.convergence_ftol_abs, + "tolfunrel": self.convergence_ftol_rel, + "tolx": self.convergence_xtol_abs, + "tolstagnation": self.convergence_iter_noimprove, + "eval_final_mean": self.eval_final_mean, + } + + configured_optimizer = ng.optimizers.ParametrizedCMA( + scale=self.scale, + popsize=self.population_size, + elitist=self.elitist, + diagonal=self.diagonal, + high_speed=self.high_speed, + fcmaes=self.fast_cmaes, + inopts=cma_options, + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_oneplusone", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradOnePlusOne(Algorithm): + noise_handling: ( + Literal["random", "optimistic"] + | tuple[Literal["random", "optimistic"], float] + | None + ) = None + mutation: Literal[ + "gaussian", + "cauchy", + "discrete", + "fastga", + "rls", + "doublefastga", + "adaptive", + "coordinatewise_adaptive", + "portfolio", + "discreteBSO", + "lengler", + "lengler2", + "lengler3", + "lenglerhalf", + "lenglerfourth", + "doerr", + "lognormal", + "xlognormal", + "xsmalllognormal", + "tinylognormal", + "smalllognormal", + "biglognormal", + "hugelognormal", + ] = "gaussian" + annealing: ( + Literal[ + "none", "Exp0.9", "Exp0.99", "Exp0.9Auto", "Lin100.0", "Lin1.0", "LinAuto" + ] + | None + ) = None + sparse: bool = False + super_radii: bool = False + smoother: bool = False + roulette_size: PositiveInt = 64 + antismooth: NonNegativeInt = 4 + crossover: bool = False + crossover_type: ( + Literal["none", "rand", "max", "min", "onepoint", "twopoint"] | None + ) = None + tabu_length: NonNegativeInt = 1000 + rotation: bool = False + seed: int | None = None + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.ParametrizedOnePlusOne( + noise_handling=self.noise_handling, + mutation=self.mutation, + crossover=self.crossover, + rotation=self.rotation, + annealing=self.annealing or "none", + sparse=self.sparse, + smoother=self.smoother, + super_radii=self.super_radii, + roulette_size=self.roulette_size, + antismooth=self.antismooth, + crossover_type=self.crossover_type or "none", + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + + return res + + +@mark.minimizer( + name="nevergrad_de", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradDifferentialEvolution(Algorithm): + initialization: Literal["parametrization", "LHS", "QR", "QO", "SO"] = ( + "parametrization" + ) + scale: float | str = 1.0 + recommendation: Literal["pessimistic", "optimistic", "mean", "noisy"] = ( + "pessimistic" + ) + crossover: ( + float + | Literal[ + "dimension", + "random", + "onepoint", + "twopoints", + "rotated_twopoints", + "parametrization", + ] + ) = 0.5 + F1: PositiveFloat = 0.8 + F2: PositiveFloat = 0.8 + population_size: int | Literal["standard", "dimension", "large"] = "standard" + high_speed: bool = False + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.DifferentialEvolution( + scale=self.scale, + recommendation=self.recommendation, + crossover=self.crossover, + F1=self.F1, + F2=self.F2, + popsize=self.population_size, + high_speed=self.high_speed, + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_bo", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradBayesOptim(Algorithm): + init_budget: int | None = None + pca: bool = False + n_components: NonNegativeFloat = 0.95 + prop_doe_factor: NonNegativeFloat | None = 1 + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: int | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.BayesOptim( + init_budget=self.init_budget, + pca=self.pca, + n_components=self.n_components, + prop_doe_factor=self.prop_doe_factor, + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_emna", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradEMNA(Algorithm): + isotropic: bool = True + noise_handling: bool = True + population_size_adaptation: bool = False + initial_popsize: int | None = None + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.EMNA( + isotropic=self.isotropic, + naive=self.noise_handling, + population_size_adaptation=self.population_size_adaptation, + initial_popsize=self.initial_popsize, + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_cga", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradCGA(Algorithm): + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.cGA + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_eda", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradEDA(Algorithm): + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.EDA + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_tbpsa", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradTBPSA(Algorithm): + noise_handling: bool = True + initial_popsize: int | None = None + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.ParametrizedTBPSA( + naive=self.noise_handling, + initial_popsize=self.initial_popsize, + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_randomsearch", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradRandomSearch(Algorithm): + middle_point: bool = False + opposition_mode: Literal["opposite", "quasi"] | None = None + sampler: Literal["parametrization", "gaussian", "cauchy"] = "parametrization" + scale: PositiveFloat | Literal["random", "auto", "autotune"] = "auto" + recommendation_rule: Literal[ + "average_of_best", "pessimistic", "average_of_exp_best" + ] = "pessimistic" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.RandomSearchMaker( + stupid=False, + middle_point=self.middle_point, + opposition_mode=self.opposition_mode, + sampler=self.sampler, + scale=self.scale, + recommendation_rule=self.recommendation_rule, + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=None, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_samplingsearch", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradSamplingSearch(Algorithm): + sampler: Literal["Halton", "LHS", "Hammersley"] = "Halton" + scrambled: bool = False + middle_point: bool = False + cauchy: bool = False + scale: bool | NonNegativeFloat = 1.0 + rescaled: bool = False + recommendation_rule: Literal["average_of_best", "pessimistic"] = "pessimistic" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = ng.optimizers.SamplingSearch( + sampler=self.sampler, + scrambled=self.scrambled, + middle_point=self.middle_point, + cauchy=self.cauchy, + scale=self.scale, + rescaled=self.rescaled, + recommendation_rule=self.recommendation_rule, + ) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=None, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + return res + + +@mark.minimizer( + name="nevergrad_NGOpt", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradNGOpt(Algorithm): + optimizer: Literal[ + "NGOpt", + "NGOpt4", + "NGOpt8", + "NGOpt10", + "NGOpt12", + "NGOpt13", + "NGOpt14", + "NGOpt15", + "NGOpt16", + "NGOpt21", + "NGOpt36", + "NGOpt38", + "NGOpt39", + "NGOptRW", + "NGOptF", + "NGOptF2", + "NGOptF3", + "NGOptF5", + "NgIoh2", + "NgIoh3", + "NgIoh4", + "NgIoh5", + "NgIoh6", + "NgIoh7", + "NgIoh11", + "NgIoh14", + "NgIoh13", + "NgIoh15", + "NgIoh12", + "NgIoh16", + "NgIoh17", + "NgIoh21", + "NgIoh20", + "NgIoh19", + "NgIoh18", + "NgIoh10", + "NgIoh9", + "NgIoh8", + "NgIoh12b", + "NgIoh13b", + "NgIoh14b", + "NgIoh15b", + "NgDS", + "NgDS2", + "NGDSRW", + "NGO", + "NgIohRW2", + "NgIohTuned", + "CSEC", + "CSEC10", + "CSEC11", + "Wiz", + ] = "NGOpt" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = getattr(ng.optimizers, self.optimizer) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + + return res + + +@mark.minimizer( + name="nevergrad_meta", + solver_type=AggregationLevel.SCALAR, + is_available=IS_NEVERGRAD_INSTALLED, + is_global=True, + needs_jac=False, + needs_hess=False, + needs_bounds=False, + supports_parallelism=True, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class NevergradMeta(Algorithm): + optimizer: Literal[ + "MultiBFGSPlus", + "LogMultiBFGSPlus", + "SqrtMultiBFGSPlus", + "MultiCobylaPlus", + "MultiSQPPlus", + "BFGSCMAPlus", + "LogBFGSCMAPlus", + "SqrtBFGSCMAPlus", + "SQPCMAPlus", + "LogSQPCMAPlus", + "SqrtSQPCMAPlus", + "MultiBFGS", + "LogMultiBFGS", + "SqrtMultiBFGS", + "MultiCobyla", + "ForceMultiCobyla", + "MultiSQP", + "BFGSCMA", + "LogBFGSCMA", + "SqrtBFGSCMA", + "SQPCMA", + "LogSQPCMA", + "SqrtSQPCMA", + "FSQPCMA", + "F2SQPCMA", + "F3SQPCMA", + "MultiDiscrete", + "CMandAS2", + "CMandAS3", + "MetaCMA", + "CMA", + "PCEDA", + "MPCEDA", + "MEDA", + "NoisyBandit", + "Shiwa", + "Carola3", + ] = "Shiwa" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN_GLOBAL + n_cores: PositiveInt = 1 + seed: int | None = None + sigma: float | None = None + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + if not IS_NEVERGRAD_INSTALLED: + raise NotInstalledError(NEVERGRAD_NOT_INSTALLED_ERROR) + + import nevergrad as ng + + configured_optimizer = getattr(ng.optimizers, self.optimizer) + + res = _nevergrad_internal( + problem=problem, + x0=x0, + configured_optimizer=configured_optimizer, + stopping_maxfun=self.stopping_maxfun, + n_cores=self.n_cores, + seed=self.seed, + sigma=self.sigma, + nonlinear_constraints=problem.nonlinear_constraints, + ) + + return res + + +def _nevergrad_internal( + problem: InternalOptimizationProblem, + x0: NDArray[np.float64], + n_cores: int, + configured_optimizer: "ng.optimization.base.ConfiguredOptimizer", + stopping_maxfun: int, + seed: int | None, + sigma: float | None, + nonlinear_constraints: list[dict[str, Any]] | None, +) -> InternalOptimizeResult: + """Internal helper function for nevergrad. + + Handle the optimization loop. + + Args: + problem (InternalOptimizationProblem): Internal optimization problem to solve. + x0 (np.ndarray): Initial parameter vector of shape (n_params,). + n_cores (int): Number of processes used to parallelize the function + evaluations. + configured_optimizer (ConfiguredOptimizer): Nevergrad optimizer instance + configured with options. + stopping_maxfun (int): Maximum number of function evaluations. + seed (int): Random seed for reproducibility. Defaults to None. + + Returns: + InternalOptimizeResult: Internal optimization result + + """ + + import nevergrad as ng + + param = ng.p.Array( + init=x0, + ) + + param.set_bounds( + lower=problem.bounds.lower, + upper=problem.bounds.upper, + ) + + # In case bounds are not provided, the initial population is sampled + # from a gaussian with mean = 0 and sigma = 1, + # which can be set through this method. + param.set_mutation(sigma=sigma) + + instrum = ng.p.Instrumentation(param) + + if seed is not None: + instrum.random_state.seed(seed) + + optimizer = configured_optimizer( + parametrization=instrum, budget=stopping_maxfun, num_workers=n_cores + ) + + ### Skip handling of non_linear constraints until improve constraint handling. + # if nonlinear_constraints: + # constraints = _process_nonlinear_constraints(nonlinear_constraints) + ### + + # optimization loop using the ask-and-tell interface + while optimizer.num_ask < stopping_maxfun: + x_list = [ + optimizer.ask() + for _ in range(min(n_cores, stopping_maxfun - optimizer.num_ask)) + ] + + losses = problem.batch_fun([x.value[0][0] for x in x_list], n_cores=n_cores) + + if not nonlinear_constraints: + for x, loss in zip(x_list, losses, strict=True): + optimizer.tell(x, loss) + + ### Skip handling of non_linear constraints until improve constraint handling. + # else: + # constraint_violations = _batch_constraint_evaluations( + # constraints, [x.value[0][0] for x in x_list], n_cores + # ) + # for x, loss, cv in zip(x_list, losses, constraint_violations, strict=True): + # optimizer.tell(x, loss, cv) + ### + + recommendation = optimizer.provide_recommendation() + best_x = recommendation.value[0][0] + loss = recommendation.loss + + # In case of CMA, loss is not provided by the optimizer, in that case, + # evaluate it manually using problem.fun + if loss is None: + loss = problem.fun(best_x) + + result = InternalOptimizeResult( + x=best_x, + fun=loss, + success=True, + n_fun_evals=optimizer.num_ask, + n_jac_evals=0, + n_hess_evals=0, + ) + + return result + + +### Skip handling of non_linear constraints until improve constraint handling. + +# def _process_nonlinear_constraints( +# constraints: list[dict[str, Any]], +# ) -> list[dict[str, Any]]: +# """Process stacked inequality constraints as single constraints. + +# Returns a list of single constraints. + +# """ +# processed_constraints = [] +# for c in constraints: +# new = _vector_to_list_of_scalar(c) +# processed_constraints.extend(new) +# return processed_constraints + + +# def _get_constraint_evaluations( +# constraints: list[dict[str, Any]], x: NDArray[np.float64] +# ) -> list[NDArray[np.float64]]: +# """In optimagic, inequality constraints are internally defined as g(x) >= 0. +# Nevergrad uses h(x) <= 0 hence a sign flip is required. Passed equality +# constraints are treated as inequality constraints with lower bound equal to +# value. Return a list of constraint evaluations at x. + +# """ +# results = [-c["fun"](x) for c in constraints] +# results = [np.atleast_1d(i) for i in results] +# return results + + +# def _batch_constraint_evaluations( +# constraints: list[dict[str, Any]], x_list: list[Any], n_cores: int +# ) -> list[list[NDArray[np.float64]]]: +# """Batch version of _get_constraint_evaluations.""" +# batch = process_batch_evaluator("joblib") +# func = partial(_get_constraint_evaluations, constraints) +# results = batch(func=func, arguments=[x for x in x_list], n_cores=n_cores) +# return results +### diff --git a/src/optimagic/optimizers/nlopt_optimizers.py b/src/optimagic/optimizers/nlopt_optimizers.py index 0889f6493..ce716fb0f 100644 --- a/src/optimagic/optimizers/nlopt_optimizers.py +++ b/src/optimagic/optimizers/nlopt_optimizers.py @@ -43,8 +43,10 @@ is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -82,8 +84,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -119,8 +123,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -158,8 +164,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -197,8 +205,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -236,8 +246,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -283,8 +295,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -322,8 +336,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -361,8 +377,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -400,8 +418,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -444,8 +464,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -488,8 +510,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -527,8 +551,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -588,8 +614,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -627,8 +655,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -666,8 +696,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, diff --git a/src/optimagic/optimizers/pounders.py b/src/optimagic/optimizers/pounders.py index 87b652225..2fa2650b2 100644 --- a/src/optimagic/optimizers/pounders.py +++ b/src/optimagic/optimizers/pounders.py @@ -45,8 +45,10 @@ is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=True, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -244,9 +246,9 @@ def internal_solve_pounders( conjugate_gradient_method_sub (str): Method for computing the conjugate gradient step ("bntr"). Available conjugate gradient methods are: - - "cg" - - "steihaug_toint" - - "trsbox" (default) + - "cg" + - "steihaug_toint" + - "trsbox" (default) maxiter_sub (int): Maximum number of iterations in the trust-region subproblem. maxiter_gradient_descent_sub (int): Maximum number of gradient descent iterations to perform ("bntr"). diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py index b5582bf38..89e66a1a1 100644 --- a/src/optimagic/optimizers/pyensmallen_optimizers.py +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -21,8 +21,9 @@ ) from optimagic.typing import AggregationLevel, NonNegativeFloat, PositiveInt +# use pyensmallen_experimental for testing purpose if IS_PYENSMALLEN_INSTALLED: - import pyensmallen as pye + import pyensmallen_experimental as pye MIN_LINE_SEARCH_STEPS = 1e-20 """The minimum step of the line search.""" diff --git a/src/optimagic/optimizers/pygmo_optimizers.py b/src/optimagic/optimizers/pygmo_optimizers.py index 44e6c7339..b8e9b84ed 100644 --- a/src/optimagic/optimizers/pygmo_optimizers.py +++ b/src/optimagic/optimizers/pygmo_optimizers.py @@ -21,7 +21,7 @@ import warnings from dataclasses import dataclass -from typing import Any, List, Literal +from typing import TYPE_CHECKING, Any, List, Literal import numpy as np from numpy.typing import NDArray @@ -47,10 +47,8 @@ STOPPING_MAX_ITERATIONS_GENETIC = 250 -try: +if TYPE_CHECKING: import pygmo as pg -except ImportError: - pass @mark.minimizer( @@ -60,8 +58,10 @@ is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=True, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -126,8 +126,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -172,8 +174,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -255,8 +259,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -301,8 +307,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -445,8 +453,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -541,8 +551,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -613,8 +625,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -672,8 +686,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -777,8 +793,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=True, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -882,8 +900,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -933,8 +953,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -1010,8 +1032,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -1052,8 +1076,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -1112,8 +1138,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -1171,8 +1199,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -1290,6 +1320,8 @@ def _minimize_pygmo( def _create_pygmo_problem( problem: InternalOptimizationProblem, dim: int, n_cores: int ) -> pg.problem: + import pygmo as pg + class Problem: def fitness(self, x): return [problem.fun(x)] @@ -1314,6 +1346,8 @@ def _create_algorithm( method: str, algo_options: dict[str, Any], n_cores: int ) -> pg.algorithm: """Create a pygmo algorithm.""" + import pygmo as pg + pygmo_uda = getattr(pg, method) algo = pygmo_uda(**algo_options) try: @@ -1335,6 +1369,8 @@ def _create_population( seed: int | None, discard_start_params: bool, ) -> pg.population: + import pygmo as pg + if not discard_start_params: population_size = population_size - 1 diff --git a/src/optimagic/optimizers/scipy_optimizers.py b/src/optimagic/optimizers/scipy_optimizers.py index 1aea5d32c..b35aef29d 100644 --- a/src/optimagic/optimizers/scipy_optimizers.py +++ b/src/optimagic/optimizers/scipy_optimizers.py @@ -33,9 +33,11 @@ """ +from __future__ import annotations + import functools from dataclasses import dataclass -from typing import Any, Callable, List, Literal, Tuple +from typing import Any, Callable, List, Literal, SupportsInt, Tuple import numpy as np import scipy @@ -74,6 +76,7 @@ from optimagic.typing import ( AggregationLevel, BatchEvaluator, + BatchEvaluatorLiteral, NegativeFloat, NonNegativeFloat, NonNegativeInt, @@ -90,20 +93,78 @@ is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, ) @dataclass(frozen=True) class ScipyLBFGSB(Algorithm): + """Minimize a scalar differentiable function using the L-BFGS-B algorithm. + + The optimizer is taken from scipy, which calls the Fortran code written by the + original authors of the algorithm. The Fortran code includes the corrections + and improvements that were introduced in a follow up paper. + + lbfgsb is a limited memory version of the original bfgs algorithm, that deals with + lower and upper bounds via an active set approach. + + The lbfgsb algorithm is well suited for differentiable scalar optimization problems + with up to several hundred parameters. + + It is a quasi-newton line search algorithm. At each trial point it evaluates the + criterion function and its gradient to find a search direction. It then approximates + the hessian using the stored history of gradients and uses the hessian to calculate + a candidate step size. Then it uses a gradient based line search algorithm to + determine the actual step length. Since the algorithm always evaluates the gradient + and criterion function jointly, the user should provide a ``fun_and_jac`` function + that exploits the synergies in the calculation of criterion and gradient. + + The lbfgsb algorithm is almost perfectly scale invariant. Thus, it is not necessary + to scale the parameters. + + """ + convergence_ftol_rel: NonNegativeFloat = CONVERGENCE_FTOL_REL + r"""Converge if the relative change in the objective function is less than this + value. More formally, this is expressed as. + + .. math:: + + \frac{f^k - f^{k+1}}{\max\{{|f^k|, |f^{k+1}|, 1}\}} \leq + \textsf{convergence_ftol_rel}. + + """ + convergence_gtol_abs: NonNegativeFloat = CONVERGENCE_GTOL_ABS + """Converge if the absolute values in the gradient of the objective function are + less than this value.""" + stopping_maxfun: PositiveInt = STOPPING_MAXFUN + """Maximum number of function evaluations.""" + stopping_maxiter: PositiveInt = STOPPING_MAXITER + """Maximum number of iterations.""" + limited_memory_storage_length: PositiveInt = LIMITED_MEMORY_STORAGE_LENGTH + """The maximum number of variable metric corrections used to define the limited + memory matrix. This is the 'maxcor' parameter in the SciPy documentation. + + The default value is taken from SciPy's L-BFGS-B implementation. Larger values use + more memory but may converge faster for some problems. + + """ + max_line_search_steps: PositiveInt = MAX_LINE_SEARCH_STEPS + """The maximum number of line search steps. This is the 'maxls' parameter in the + SciPy documentation. + + The default value is taken from SciPy's L-BFGS-B implementation. + + """ def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -135,8 +196,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -145,6 +208,7 @@ def _solve_internal_problem( class ScipySLSQP(Algorithm): convergence_ftol_abs: NonNegativeFloat = CONVERGENCE_SECOND_BEST_FTOL_ABS stopping_maxiter: PositiveInt = STOPPING_MAXITER + display: bool = False def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -152,6 +216,7 @@ def _solve_internal_problem( options = { "maxiter": self.stopping_maxiter, "ftol": self.convergence_ftol_abs, + "disp": self.display, } raw_res = scipy.optimize.minimize( fun=problem.fun_and_jac, @@ -173,8 +238,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -186,6 +253,7 @@ class ScipyNelderMead(Algorithm): convergence_ftol_abs: NonNegativeFloat = CONVERGENCE_SECOND_BEST_FTOL_ABS convergence_xtol_abs: NonNegativeFloat = CONVERGENCE_SECOND_BEST_XTOL_ABS adaptive: bool = False + display: bool = False def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -197,6 +265,7 @@ def _solve_internal_problem( "fatol": self.convergence_ftol_abs, # TODO: Benchmark if adaptive = True works better "adaptive": self.adaptive, + "disp": self.display, } raw_res = scipy.optimize.minimize( fun=problem.fun, @@ -216,8 +285,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -228,6 +299,7 @@ class ScipyPowell(Algorithm): convergence_ftol_rel: NonNegativeFloat = CONVERGENCE_FTOL_REL stopping_maxfun: PositiveInt = STOPPING_MAXFUN stopping_maxiter: PositiveInt = STOPPING_MAXITER + display: bool = False def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -237,6 +309,7 @@ def _solve_internal_problem( "ftol": self.convergence_ftol_rel, "maxfev": self.stopping_maxfun, "maxiter": self.stopping_maxiter, + "disp": self.display, } raw_res = scipy.optimize.minimize( fun=problem.fun, @@ -256,8 +329,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -267,6 +342,10 @@ class ScipyBFGS(Algorithm): convergence_gtol_abs: NonNegativeFloat = CONVERGENCE_GTOL_ABS stopping_maxiter: PositiveInt = STOPPING_MAXITER norm: NonNegativeFloat = np.inf + convergence_xtol_rel: NonNegativeFloat = CONVERGENCE_XTOL_REL + display: bool = False + armijo_condition: NonNegativeFloat = 1e-4 + curvature_condition: NonNegativeFloat = 0.9 def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -275,6 +354,10 @@ def _solve_internal_problem( "gtol": self.convergence_gtol_abs, "maxiter": self.stopping_maxiter, "norm": self.norm, + "xrtol": self.convergence_xtol_rel, + "disp": self.display, + "c1": self.armijo_condition, + "c2": self.curvature_condition, } raw_res = scipy.optimize.minimize( fun=problem.fun_and_jac, x0=x0, method="BFGS", jac=True, options=options @@ -290,8 +373,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -301,6 +386,7 @@ class ScipyConjugateGradient(Algorithm): convergence_gtol_abs: NonNegativeFloat = CONVERGENCE_GTOL_ABS stopping_maxiter: PositiveInt = STOPPING_MAXITER norm: NonNegativeFloat = np.inf + display: bool = False def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -309,6 +395,7 @@ def _solve_internal_problem( "gtol": self.convergence_gtol_abs, "maxiter": self.stopping_maxiter, "norm": self.norm, + "disp": self.display, } raw_res = scipy.optimize.minimize( fun=problem.fun_and_jac, x0=x0, method="CG", jac=True, options=options @@ -324,8 +411,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -334,6 +423,7 @@ def _solve_internal_problem( class ScipyNewtonCG(Algorithm): convergence_xtol_rel: NonNegativeFloat = CONVERGENCE_XTOL_REL stopping_maxiter: PositiveInt = STOPPING_MAXITER + display: bool = False def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -341,6 +431,7 @@ def _solve_internal_problem( options = { "xtol": self.convergence_xtol_rel, "maxiter": self.stopping_maxiter, + "disp": self.display, } raw_res = scipy.optimize.minimize( fun=problem.fun_and_jac, @@ -360,8 +451,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -371,6 +464,7 @@ class ScipyCOBYLA(Algorithm): convergence_xtol_rel: NonNegativeFloat = CONVERGENCE_XTOL_REL stopping_maxiter: PositiveInt = STOPPING_MAXITER trustregion_initial_radius: PositiveFloat | None = None + display: bool = False def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -384,6 +478,7 @@ def _solve_internal_problem( options = { "maxiter": self.stopping_maxiter, "rhobeg": radius, + "disp": self.display, } # cannot handle equality constraints @@ -410,8 +505,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -433,12 +530,15 @@ def _solve_internal_problem( else: tr_solver_options = self.tr_solver_options + lower_bounds = -np.inf if problem.bounds.lower is None else problem.bounds.lower + upper_bounds = np.inf if problem.bounds.upper is None else problem.bounds.upper + raw_res = scipy.optimize.least_squares( fun=problem.fun, x0=x0, # This optimizer does not work with fun_and_jac jac=problem.jac, - bounds=(problem.bounds.lower, problem.bounds.upper), + bounds=(lower_bounds, upper_bounds), method="trf", max_nfev=self.stopping_maxfun, ftol=self.convergence_ftol_rel, @@ -458,8 +558,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -481,12 +583,15 @@ def _solve_internal_problem( else: tr_solver_options = self.tr_solver_options + lower_bounds = -np.inf if problem.bounds.lower is None else problem.bounds.lower + upper_bounds = np.inf if problem.bounds.upper is None else problem.bounds.upper + raw_res = scipy.optimize.least_squares( fun=problem.fun, x0=x0, # This optimizer does not work with fun_and_jac jac=problem.jac, - bounds=(problem.bounds.lower, problem.bounds.upper), + bounds=(lower_bounds, upper_bounds), method="dogbox", max_nfev=self.stopping_maxfun, ftol=self.convergence_ftol_rel, @@ -506,8 +611,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -544,8 +651,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -563,6 +672,7 @@ class ScipyTruncatedNewton(Algorithm): criterion_rescale_factor: float = -1 # TODO: Check type hint for `func_min_estimate` func_min_estimate: float = 0 + display: bool = False def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -578,6 +688,7 @@ def _solve_internal_problem( "eta": self.line_search_severity, "accuracy": self.finite_difference_precision, "rescale": self.criterion_rescale_factor, + "disp": self.display, } raw_res = scipy.optimize.minimize( @@ -599,8 +710,10 @@ def _solve_internal_problem( is_global=False, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -612,6 +725,7 @@ class ScipyTrustConstr(Algorithm): convergence_xtol_rel: NonNegativeFloat = CONVERGENCE_XTOL_REL stopping_maxiter: PositiveInt = STOPPING_MAXITER trustregion_initial_radius: PositiveFloat | None = None + display: bool = False def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -626,6 +740,7 @@ def _solve_internal_problem( "maxiter": self.stopping_maxiter, "xtol": self.convergence_xtol_rel, "initial_tr_radius": trustregion_initial_radius, + "disp": self.display, } # cannot handle equality constraints @@ -652,10 +767,10 @@ def process_scipy_result(scipy_res: ScipyOptimizeResult) -> InternalOptimizeResu fun=scipy_res.fun, success=bool(scipy_res.success), message=str(scipy_res.message), - n_fun_evals=scipy_res.get("nfev"), - n_jac_evals=scipy_res.get("njev"), - n_hess_evals=scipy_res.get("nhev"), - n_iterations=scipy_res.get("nit"), + n_fun_evals=_int_if_not_none(scipy_res.get("nfev")), + n_jac_evals=_int_if_not_none(scipy_res.get("njev")), + n_hess_evals=_int_if_not_none(scipy_res.get("nhev")), + n_iterations=_int_if_not_none(scipy_res.get("nit")), # TODO: Pass on more things once we can convert them to external status=None, jac=None, @@ -668,6 +783,12 @@ def process_scipy_result(scipy_res: ScipyOptimizeResult) -> InternalOptimizeResu return res +def _int_if_not_none(value: SupportsInt | None) -> int | None: + if value is None: + return None + return int(value) + + def _get_scipy_constraints(constraints): """Transform internal nonlinear constraints to scipy readable format. @@ -695,8 +816,10 @@ def _internal_to_scipy_constraint(c): is_global=True, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -775,8 +898,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=True, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=True, @@ -786,7 +911,7 @@ class ScipyBrute(Algorithm): n_grid_points: PositiveInt = 20 polishing_function: Callable | None = None n_cores: PositiveInt = 1 - batch_evaluator: Literal["joblib", "pathos"] | BatchEvaluator = "joblib" + batch_evaluator: BatchEvaluatorLiteral | BatchEvaluator = "joblib" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -824,8 +949,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=True, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=True, @@ -866,7 +993,7 @@ class ScipyDifferentialEvolution(Algorithm): ) = "latinhypercube" convergence_ftol_abs: NonNegativeFloat = CONVERGENCE_SECOND_BEST_FTOL_ABS n_cores: PositiveInt = 1 - batch_evaluator: Literal["joblib", "pathos"] | BatchEvaluator = "joblib" + batch_evaluator: BatchEvaluatorLiteral | BatchEvaluator = "joblib" def _solve_internal_problem( self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] @@ -900,8 +1027,10 @@ def _solve_internal_problem( is_global=True, needs_jac=True, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=True, disable_history=False, @@ -1006,8 +1135,10 @@ def _solve_internal_problem( is_global=True, needs_jac=True, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -1086,8 +1217,10 @@ def _solve_internal_problem( is_global=True, needs_jac=False, needs_hess=False, + needs_bounds=True, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -1135,8 +1268,13 @@ def _get_workers(n_cores, batch_evaluator): return out -def _get_scipy_bounds(bounds: InternalBounds) -> ScipyBounds: - return ScipyBounds(lb=bounds.lower, ub=bounds.upper) +def _get_scipy_bounds(bounds: InternalBounds) -> ScipyBounds | None: + if bounds.lower is None and bounds.upper is None: + return None + + lower = bounds.lower if bounds.lower is not None else -np.inf + upper = bounds.upper if bounds.upper is not None else np.inf + return ScipyBounds(lb=lower, ub=upper) def process_scipy_result_old(scipy_results_obj): diff --git a/src/optimagic/optimizers/tao_optimizers.py b/src/optimagic/optimizers/tao_optimizers.py index 8ea2401b5..b36e71778 100644 --- a/src/optimagic/optimizers/tao_optimizers.py +++ b/src/optimagic/optimizers/tao_optimizers.py @@ -22,11 +22,6 @@ from optimagic.typing import AggregationLevel, NonNegativeFloat, PositiveInt from optimagic.utilities import calculate_trustregion_initial_radius -try: - from petsc4py import PETSc -except ImportError: - pass - @mark.minimizer( name="tao_pounders", @@ -35,8 +30,10 @@ is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=False, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, @@ -108,9 +105,10 @@ def tao_pounders( raise NotInstalledError( "The 'tao_pounders' algorithm requires the petsc4py package to be " "installed. If you are using Linux or MacOS, install the package with " - "'conda install -c conda-forge petsc4py. The package is not available on " + "'conda install -c conda-forge petsc4py'. The package is not available on " "Windows. Windows users can use optimagics 'pounders' algorithm instead." ) + from petsc4py import PETSc first_eval = criterion(x) n_errors = len(first_eval) @@ -150,10 +148,15 @@ def func_tao(tao, x, resid_out): # noqa: ARG001 raise ValueError("The initial trust region radius must be > 0.") tao.setInitialTrustRegionRadius(trustregion_initial_radius) - # Add bounds. - lower_bounds = _initialise_petsc_array(lower_bounds) - upper_bounds = _initialise_petsc_array(upper_bounds) - tao.setVariableBounds(lower_bounds, upper_bounds) + # Add bounds if provided. + if lower_bounds is not None or upper_bounds is not None: + if lower_bounds is None: + lower_bounds = np.full(len(x), -np.inf) + if upper_bounds is None: + upper_bounds = np.full(len(x), np.inf) + lower_bounds = _initialise_petsc_array(lower_bounds) + upper_bounds = _initialise_petsc_array(upper_bounds) + tao.setVariableBounds(lower_bounds, upper_bounds) # Put the starting values into the container and pass them to the optimizer. tao.setInitial(_x) @@ -195,7 +198,8 @@ def func_tao(tao, x, resid_out): # noqa: ARG001 results = _process_pounders_results(residuals_out, tao) # Destroy petsc objects for memory reasons. - for obj in [tao, _x, residuals_out, lower_bounds, upper_bounds]: + petsc_bounds = [b for b in (lower_bounds, upper_bounds) if b is not None] + for obj in [tao, _x, residuals_out, *petsc_bounds]: obj.destroy() return results @@ -210,6 +214,8 @@ def _initialise_petsc_array(len_or_array): array of equal length and fill in the values. """ + from petsc4py import PETSc + length = len_or_array if isinstance(len_or_array, int) else len(len_or_array) array = PETSc.Vec().create(PETSc.COMM_WORLD) diff --git a/src/optimagic/optimizers/tranquilo.py b/src/optimagic/optimizers/tranquilo.py index 0e269bc3b..becffe5f4 100644 --- a/src/optimagic/optimizers/tranquilo.py +++ b/src/optimagic/optimizers/tranquilo.py @@ -1,5 +1,7 @@ +from __future__ import annotations + from dataclasses import dataclass -from typing import Any, Callable, Literal +from typing import TYPE_CHECKING, Callable, Literal import numpy as np from numpy.typing import NDArray @@ -19,7 +21,7 @@ PositiveInt, ) -if IS_TRANQUILO_INSTALLED: +if TYPE_CHECKING: from tranquilo.options import ( AcceptanceOptions, FilterOptions, @@ -31,17 +33,6 @@ SubsolverOptions, VarianceEstimatorOptions, ) - from tranquilo.tranquilo import _tranquilo -else: - AcceptanceOptions = Any - FilterOptions = Any - FitterOptions = Any - NoiseAdaptationOptions = Any - RadiusOptions = Any - SamplerOptions = Any - StagnationOptions = Any - SubsolverOptions = Any - VarianceEstimatorOptions = Any @mark.minimizer( @@ -51,8 +42,10 @@ is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=True, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=True, @@ -177,6 +170,8 @@ def _solve_internal_problem( "to be installed. You can install it with " "'conda install -c conda-forge tranquilo'." ) + from tranquilo.tranquilo import _tranquilo + raw_res = _tranquilo( functype="scalar", criterion=problem.fun, @@ -241,8 +236,10 @@ def _solve_internal_problem( is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=True, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=True, @@ -365,6 +362,8 @@ def _solve_internal_problem( "to be installed. You can install it with " "'conda install -c conda-forge tranquilo'." ) + from tranquilo.tranquilo import _tranquilo + raw_res = _tranquilo( functype="least_squares", criterion=problem.fun, diff --git a/src/optimagic/parameters/bounds.py b/src/optimagic/parameters/bounds.py index b8b3f48c0..344dca4f4 100644 --- a/src/optimagic/parameters/bounds.py +++ b/src/optimagic/parameters/bounds.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Sequence +from typing import Any, Literal, Sequence import numpy as np from numpy.typing import NDArray @@ -12,6 +12,7 @@ from optimagic.exceptions import InvalidBoundsError from optimagic.parameters.tree_registry import get_registry from optimagic.typing import PyTree, PyTreeRegistry +from optimagic.utilities import fast_numpy_full @dataclass(frozen=True) @@ -60,8 +61,8 @@ def pre_process_bounds( def _process_bounds_sequence(bounds: Sequence[tuple[float, float]]) -> Bounds: - lower = np.full(len(bounds), -np.inf) - upper = np.full(len(bounds), np.inf) + lower = fast_numpy_full(len(bounds), fill_value=-np.inf) + upper = fast_numpy_full(len(bounds), fill_value=np.inf) for i, (lb, ub) in enumerate(bounds): if lb is not None: @@ -76,14 +77,14 @@ def get_internal_bounds( bounds: Bounds | None = None, registry: PyTreeRegistry | None = None, add_soft_bounds: bool = False, -) -> tuple[NDArray[np.float64], NDArray[np.float64]]: +) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None]: """Create consolidated and flattened bounds for params. If params is a DataFrame with value column, the user provided bounds are extended with bounds from the params DataFrame. - If no bounds are available the entry is set to minus np.inf for the lower bound and - np.inf for the upper bound. + If no bounds are provided, we return None. If some bounds are available the missing + entries are set to -np.inf for the lower bound and np.inf for the upper bound. The bounds provided in `bounds` override bounds provided in params if both are specified (in the case where params is a DataFrame with bounds as a column). @@ -109,10 +110,11 @@ def get_internal_bounds( add_soft_bounds=add_soft_bounds, ) if fast_path: - return _get_fast_path_bounds( - params=params, - bounds=bounds, - ) + return _get_fast_path_bounds(bounds) + + # Handling of None-valued bounds in the slow path needs to be improved. Currently, + # None-valued bounds are replaced with arrays of np.inf and -np.inf, and then + # translated back to None if all entries are non-finite. registry = get_registry(extended=True) if registry is None else registry n_params = len(tree_leaves(params, registry=registry)) @@ -149,11 +151,18 @@ def get_internal_bounds( msg = "Invalid bounds. Some lower bounds are larger than upper bounds." raise InvalidBoundsError(msg) + if np.isinf(lower_flat).all(): + lower_flat = None # type: ignore[assignment] + if np.isinf(upper_flat).all(): + upper_flat = None # type: ignore[assignment] + return lower_flat, upper_flat def _update_bounds_and_flatten( - nan_tree: PyTree, bounds: PyTree, kind: str + nan_tree: PyTree, + bounds: PyTree, + kind: Literal["lower_bound", "upper_bound", "soft_lower_bound", "soft_upper_bound"], ) -> NDArray[np.float64]: """Flatten bounds array and update it with bounds from params. @@ -213,7 +222,7 @@ def _is_fast_path(params: PyTree, bounds: Bounds, add_soft_bounds: bool) -> bool if not _is_1d_array(params): out = False - for bound in bounds.lower, bounds.upper: + for bound in (bounds.lower, bounds.upper): if not (_is_1d_array(bound) or bound is None): out = False return out @@ -224,21 +233,27 @@ def _is_1d_array(candidate: Any) -> bool: def _get_fast_path_bounds( - params: PyTree, bounds: Bounds -) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + bounds: Bounds, +) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None]: if bounds.lower is None: - # faster than np.full - lower_bounds = np.array([-np.inf] * len(params)) + lower_bounds = None else: lower_bounds = bounds.lower.astype(float) + if np.isinf(lower_bounds).all(): + lower_bounds = None if bounds.upper is None: - # faster than np.full - upper_bounds = np.array([np.inf] * len(params)) + upper_bounds = None else: upper_bounds = bounds.upper.astype(float) - - if (lower_bounds > upper_bounds).any(): + if np.isinf(upper_bounds).all(): + upper_bounds = None + + if ( + lower_bounds is not None + and upper_bounds is not None + and (lower_bounds > upper_bounds).any() + ): msg = "Invalid bounds. Some lower bounds are larger than upper bounds." raise InvalidBoundsError(msg) diff --git a/src/optimagic/parameters/consolidate_constraints.py b/src/optimagic/parameters/consolidate_constraints.py index 21e52a90f..f942c8b56 100644 --- a/src/optimagic/parameters/consolidate_constraints.py +++ b/src/optimagic/parameters/consolidate_constraints.py @@ -11,7 +11,10 @@ import pandas as pd from optimagic.exceptions import InvalidConstraintError -from optimagic.utilities import number_of_triangular_elements_to_dimension +from optimagic.utilities import ( + fast_numpy_full, + number_of_triangular_elements_to_dimension, +) def consolidate_constraints( @@ -25,8 +28,8 @@ def consolidate_constraints( constraints have been rewritten as linear constraints and pairwise_equality constraints have been rewritten as equality constraints. parvec (np.ndarray): 1d numpy array with parameters. - lower_bounds (np.ndarray): 1d numpy array with lower_bounds - upper_bounds (np.ndarray): 1d numpy array wtih upper_bounds + lower_bounds (np.ndarray | None): 1d numpy array with lower_bounds + upper_bounds (np.ndarray | None): 1d numpy array with upper_bounds param_names (list): Names of parameters. Used for error messages. Returns: @@ -37,6 +40,13 @@ def consolidate_constraints( constraints. """ + # None-valued bounds are handled by instantiating them as an -inf and inf array. In + # the future, this should be handled more gracefully. + if lower_bounds is None: + lower_bounds = fast_numpy_full(len(parvec), fill_value=-np.inf) + if upper_bounds is None: + upper_bounds = fast_numpy_full(len(parvec), fill_value=np.inf) + raw_eq, other_constraints = _split_constraints(constraints, "equality") equality_constraints = _consolidate_equality_constraints(raw_eq) diff --git a/src/optimagic/parameters/conversion.py b/src/optimagic/parameters/conversion.py index 2c8061228..c86498627 100644 --- a/src/optimagic/parameters/conversion.py +++ b/src/optimagic/parameters/conversion.py @@ -36,8 +36,7 @@ def get_converter( Args: params (pytree): The user provided parameters. constraints (list): The user provided constraints. - lower_bounds (pytree): The user provided lower_bounds - upper_bounds (pytree): The user provided upper bounds + bounds (Bounds): The user provided bounds. func_eval (float or pytree): An evaluation of ``func`` at ``params``. Used to flatten the derivative output. solver_type: Used to determine how the derivative output has to be @@ -46,8 +45,6 @@ def get_converter( performed. derivative_eval (dict, pytree or None): Evaluation of the derivative of func at params. Used for consistency checks. - soft_lower_bounds (pytree): As lower_bounds - soft_upper_bounds (pytree): As upper_bounds add_soft_bounds (bool): Whether soft bounds should be added to the internal_params @@ -182,12 +179,12 @@ def _fast_derivative_to_internal( ) if bounds is None or bounds.lower is None: - lower_bounds = np.full(len(params), -np.inf) + lower_bounds = None else: lower_bounds = bounds.lower.astype(float) if bounds is None or bounds.upper is None: - upper_bounds = np.full(len(params), np.inf) + upper_bounds = None else: upper_bounds = bounds.upper.astype(float) diff --git a/src/optimagic/parameters/nonlinear_constraints.py b/src/optimagic/parameters/nonlinear_constraints.py index 7ea5b08e0..3479246ac 100644 --- a/src/optimagic/parameters/nonlinear_constraints.py +++ b/src/optimagic/parameters/nonlinear_constraints.py @@ -23,6 +23,28 @@ def process_nonlinear_constraints( ): """Process and prepare nonlinear constraints for internal use. + A user-provided nonlinear constraint consists of a function that is evaluated on a + selection of parameters returning a scalar or vector that must either be equal to + a fixed value (equality constraint) or smaller and larger than or equal to a lower + and upper bound (inequality constraint). + + This function processes the nonlinear constraints in the following way: + + 1. The constraint a <= g(x) <= b is transformed to h(x) >= 0, where h(x) is + - h(x) = g(x), if a == 0 and b == inf + - h(x) = g(x) - a, if a != 0 and b == inf + - h(x) = (g(x) - a, -g(x) + b) >= 0, if a != 0 and b != inf. + + 2. The equality constraint g(x) = v is transformed to h(x) >= 0, where + h(x) = (g(x) - v, -g(x) + v). + + 3. Vector constraints are transformed to a list of scalar constraints. + g(x) = (g1(x), g2(x), ...) >= 0 is transformed to (g1(x) >= 0, g2(x) >= 0, ...). + + 4. The constraint function (defined on a selection of user-facing parameters) is + transformed to be evaluated on the internal parameters. + + Args: nonlinear_constraints (list[dict]): List of dictionaries, each representing a nonlinear constraint. diff --git a/src/optimagic/parameters/process_constraints.py b/src/optimagic/parameters/process_constraints.py index f4ef85582..df53e7176 100644 --- a/src/optimagic/parameters/process_constraints.py +++ b/src/optimagic/parameters/process_constraints.py @@ -42,8 +42,8 @@ def process_constraints( have already been consolidated into an ``"index"`` field that selects the same parameters from the flattened_parameter vector. params_vec (np.ndarray): Flattened version of params. - lower_bounds (np.ndarray): Lower bounds for params_vec. - upper_bounds (np.ndarray): Upper bounds for params_vec. + lower_bounds (np.ndarray | None): Lower bounds for params_vec. + upper_bounds (np.ndarray | None): Upper bounds for params_vec. param_names (list): Names of the flattened parameters. Only used to produce good error messages. diff --git a/src/optimagic/parameters/scale_conversion.py b/src/optimagic/parameters/scale_conversion.py index 1f88c294c..f05e80ec2 100644 --- a/src/optimagic/parameters/scale_conversion.py +++ b/src/optimagic/parameters/scale_conversion.py @@ -83,10 +83,20 @@ def get_scale_converter( else: _soft_upper = None + if internal_params.lower_bounds is not None: + _lower_bounds = converter.params_to_internal(internal_params.lower_bounds) + else: + _lower_bounds = None + + if internal_params.upper_bounds is not None: + _upper_bounds = converter.params_to_internal(internal_params.upper_bounds) + else: + _upper_bounds = None + params = InternalParams( values=converter.params_to_internal(internal_params.values), - lower_bounds=converter.params_to_internal(internal_params.lower_bounds), - upper_bounds=converter.params_to_internal(internal_params.upper_bounds), + lower_bounds=_lower_bounds, + upper_bounds=_upper_bounds, names=internal_params.names, soft_lower_bounds=_soft_lower, soft_upper_bounds=_soft_upper, @@ -107,6 +117,15 @@ def calculate_scaling_factor_and_offset( raw_factor = np.clip(np.abs(x), options.clipping_value, np.inf) scaling_offset = None elif options.method == "bounds": + if ( + lower_bounds is None + or np.isinf(lower_bounds).any() + or upper_bounds is None + or np.isinf(upper_bounds).any() + ): + raise ValueError( + "To use the 'bounds' scaling method, all bounds must be finite." + ) raw_factor = upper_bounds - lower_bounds scaling_offset = lower_bounds else: diff --git a/src/optimagic/parameters/space_conversion.py b/src/optimagic/parameters/space_conversion.py index 73ed5edd3..6bc86ec04 100644 --- a/src/optimagic/parameters/space_conversion.py +++ b/src/optimagic/parameters/space_conversion.py @@ -61,10 +61,12 @@ def get_space_converter( SpaceConverter: The space converter. InternalParams: Dataclass with entries: - value (np.ndarray): Internal parameter values. - - lower_bounds (np.ndarray): Lower bounds on the internal params. - - upper_bounds (np.ndarray): Upper bounds on the internal params. - - soft_lower_bounds (np.ndarray): Soft lower bounds on the internal params. - - soft_upper_bounds (np.ndarray): Soft upper bounds on the internal params. + - lower_bounds (np.ndarray | None): Lower bounds on the internal params. + - upper_bounds (np.ndarray | None): Upper bounds on the internal params. + - soft_lower_bounds (np.ndarray | None): Soft lower bounds on the internal + params. + - soft_upper_bounds (np.ndarray | None): Soft upper bounds on the internal + params. - name (list): List of names of the external parameters. - free_mask (np.ndarray): Boolean mask representing which external parameter is free. @@ -145,7 +147,6 @@ def get_space_converter( soft_lower_bounds=_soft_lower, soft_upper_bounds=_soft_upper, ) - return converter, params @@ -514,8 +515,8 @@ def post_replace_jacobian(post_replacements): @dataclass(frozen=True) class InternalParams: values: np.ndarray - lower_bounds: np.ndarray - upper_bounds: np.ndarray + lower_bounds: np.ndarray | None + upper_bounds: np.ndarray | None soft_lower_bounds: np.ndarray | None = None soft_upper_bounds: np.ndarray | None = None names: list | None = None diff --git a/src/optimagic/parameters/tree_conversion.py b/src/optimagic/parameters/tree_conversion.py index 0c990dd31..2e29fd87e 100644 --- a/src/optimagic/parameters/tree_conversion.py +++ b/src/optimagic/parameters/tree_conversion.py @@ -179,8 +179,8 @@ class TreeConverter(NamedTuple): class FlatParams(NamedTuple): values: np.ndarray - lower_bounds: np.ndarray - upper_bounds: np.ndarray + lower_bounds: np.ndarray | None + upper_bounds: np.ndarray | None soft_lower_bounds: np.ndarray | None = None soft_upper_bounds: np.ndarray | None = None names: list | None = None diff --git a/src/optimagic/typing.py b/src/optimagic/typing.py index 889152f79..443bad959 100644 --- a/src/optimagic/typing.py +++ b/src/optimagic/typing.py @@ -14,7 +14,7 @@ ) import numpy as np -from annotated_types import Ge, Gt, Lt +from annotated_types import Ge, Gt, Le, Lt from numpy._typing import NDArray PyTree = Any @@ -115,15 +115,28 @@ def __call__( PositiveInt = Annotated[int, Gt(0)] +"""Type alias for positive integers (greater than 0).""" NonNegativeInt = Annotated[int, Ge(0)] +"""Type alias for non-negative integers (greater than or equal to 0).""" PositiveFloat = Annotated[float, Gt(0)] +"""Type alias for positive floats (greater than 0).""" NonNegativeFloat = Annotated[float, Ge(0)] +"""Type alias for non-negative floats (greater than or equal to 0).""" NegativeFloat = Annotated[float, Lt(0)] +"""Type alias for negative floats (less than 0).""" GtOneFloat = Annotated[float, Gt(1)] +"""Type alias for floats greater than 1.""" +UnitIntervalFloat = Annotated[float, Gt(0), Le(1)] +"""Type alias for floats in (0, 1].""" YesNoBool = Literal["yes", "no"] | bool +"""Type alias for boolean values represented as 'yes' or 'no' strings or as boolean +values.""" DirectionLiteral = Literal["minimize", "maximize"] -BatchEvaluatorLiteral = Literal["joblib", "pathos"] +"""Type alias for optimization direction, either 'minimize' or 'maximize'.""" +BatchEvaluatorLiteral = Literal["joblib", "pathos", "threading"] +"""Type alias for batch evaluator types, can be 'joblib', 'pathos', or 'threading'.""" ErrorHandlingLiteral = Literal["raise", "continue"] +"""Type alias for error handling strategies, can be 'raise' or 'continue'.""" @dataclass(frozen=True) diff --git a/src/optimagic/utilities.py b/src/optimagic/utilities.py index 4b9e1a748..447a09d70 100644 --- a/src/optimagic/utilities.py +++ b/src/optimagic/utilities.py @@ -5,12 +5,25 @@ import cloudpickle import numpy as np import pandas as pd +from numpy.typing import NDArray from scipy.linalg import ldl, qr with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) +def fast_numpy_full(length: int, fill_value: float) -> NDArray[np.float64]: + """Return a new array of given length, filled with fill_value. + + Empirically, this is faster than using np.full for small arrays. + + """ + if length < 18: + return np.array([fill_value] * length, dtype=np.float64) + else: + return np.full(length, fill_value=fill_value, dtype=np.float64) + + def chol_params_to_lower_triangular_matrix(params): dim = number_of_triangular_elements_to_dimension(len(params)) mat = np.zeros((dim, dim)) diff --git a/src/optimagic/visualization/history_plots.py b/src/optimagic/visualization/history_plots.py index cb64a4e94..696ac0c1d 100644 --- a/src/optimagic/visualization/history_plots.py +++ b/src/optimagic/visualization/history_plots.py @@ -1,5 +1,6 @@ import inspect import itertools +from dataclasses import dataclass from pathlib import Path from typing import Any @@ -13,160 +14,87 @@ from optimagic.optimization.history import History from optimagic.optimization.optimize_result import OptimizeResult from optimagic.parameters.tree_registry import get_registry -from optimagic.typing import Direction +from optimagic.typing import IterationHistory, PyTree + +OptimizeResultOrPath = OptimizeResult | str | Path def criterion_plot( - results, - names=None, - max_evaluations=None, - template=PLOTLY_TEMPLATE, - palette=PLOTLY_PALETTE, - stack_multistart=False, - monotone=False, - show_exploration=False, -): + results: OptimizeResultOrPath + | list[OptimizeResultOrPath] + | dict[str, OptimizeResultOrPath], + names: list[str] | str | None = None, + max_evaluations: int | None = None, + template: str = PLOTLY_TEMPLATE, + palette: list[str] | str = PLOTLY_PALETTE, + stack_multistart: bool = False, + monotone: bool = False, + show_exploration: bool = False, +) -> go.Figure: """Plot the criterion history of an optimization. Args: - results (Union[List, Dict][Union[OptimizeResult, pathlib.Path, str]): A (list or - dict of) optimization results with collected history. If dict, then the - key is used as the name in a legend. - names (Union[List[str], str]): Names corresponding to res or entries in res. - max_evaluations (int): Clip the criterion history after that many entries. - template (str): The template for the figure. Default is "plotly_white". - palette (Union[List[str], str]): The coloring palette for traces. Default is - "qualitative.Plotly". - stack_multistart (bool): Whether to combine multistart histories into a single - history. Default is False. - monotone (bool): If True, the criterion plot becomes monotone in the sense - that only that at each iteration the current best criterion value is - displayed. Default is False. - show_exploration (bool): If True, exploration samples of a multistart - optimization are visualized. Default is False. + results: A (list or dict of) optimization results with collected history. + If dict, then the key is used as the name in a legend. + names: Names corresponding to res or entries in res. + max_evaluations: Clip the criterion history after that many entries. + template: The template for the figure. Default is "plotly_white". + palette: The coloring palette for traces. Default is "qualitative.Set2". + stack_multistart: Whether to combine multistart histories into a single history. + Default is False. + monotone: If True, the criterion plot becomes monotone in the sense that at each + iteration the current best criterion value is displayed. Default is False. + show_exploration: If True, exploration samples of a multistart optimization are + visualized. Default is False. Returns: - plotly.graph_objs._figure.Figure: The figure. + The figure object containing the criterion plot. """ # ================================================================================== # Process inputs - # ================================================================================== - - results = _harmonize_inputs_to_dict(results, names) if not isinstance(palette, list): palette = [palette] - palette = itertools.cycle(palette) - - fun_or_monotone_fun = "monotone_fun" if monotone else "fun" - - # ================================================================================== - # Extract plotting data from results objects / data base - # ================================================================================== - - data = [] - for name, res in results.items(): - if isinstance(res, OptimizeResult): - _data = _extract_plotting_data_from_results_object( - res, stack_multistart, show_exploration, plot_name="criterion_plot" - ) - elif isinstance(res, (str, Path)): - _data = _extract_plotting_data_from_database( - res, stack_multistart, show_exploration - ) - else: - msg = "results must be (or contain) an OptimizeResult or a path to a log" - f"file, but is type {type(res)}." - raise TypeError(msg) + palette_cycle = itertools.cycle(palette) - _data["name"] = name - data.append(_data) + dict_of_optimize_results_or_paths = _harmonize_inputs_to_dict(results, names) # ================================================================================== - # Create figure - # ================================================================================== + # Extract backend-agnostic plotting data from results - fig = go.Figure() - - plot_multistart = ( - len(data) == 1 and data[0]["is_multistart"] and not stack_multistart + list_of_optimize_data = _retrieve_optimization_data( + results=dict_of_optimize_results_or_paths, + stack_multistart=stack_multistart, + show_exploration=show_exploration, ) - # ================================================================================== - # Plot multistart paths - - if plot_multistart: - scatter_kws = { - "connectgaps": True, - "showlegend": False, - } - - for i, local_history in enumerate(data[0]["local_histories"]): - history = getattr(local_history, fun_or_monotone_fun) - - if max_evaluations is not None and len(history) > max_evaluations: - history = history[:max_evaluations] - - trace = go.Scatter( - x=np.arange(len(history)), - y=history, - mode="lines", - name=str(i), - line_color="#bab0ac", - **scatter_kws, - ) - fig.add_trace(trace) + lines, multistart_lines = _extract_criterion_plot_lines( + data=list_of_optimize_data, + max_evaluations=max_evaluations, + palette_cycle=palette_cycle, + stack_multistart=stack_multistart, + monotone=monotone, + ) # ================================================================================== - # Plot main optimization objects - - for _data in data: - if stack_multistart and _data["stacked_local_histories"] is not None: - _history = _data["stacked_local_histories"] - else: - _history = _data["history"] + # Generate the plotly figure - history = getattr(_history, fun_or_monotone_fun) - - if max_evaluations is not None and len(history) > max_evaluations: - history = history[:max_evaluations] - - scatter_kws = { - "connectgaps": True, - "showlegend": not plot_multistart, - } - - _color = next(palette) - if not isinstance(_color, str): - msg = "highlight_palette needs to be a string or list of strings, but its " - f"entry is of type {type(_color)}." - raise TypeError(msg) - - line_kws = { - "color": _color, - } - - trace = go.Scatter( - x=np.arange(len(history)), - y=history, - mode="lines", - name="best result" if plot_multistart else _data["name"], - line=line_kws, - **scatter_kws, - ) - fig.add_trace(trace) - - fig.update_layout( + plot_config = PlotConfig( template=template, - xaxis_title_text="No. of criterion evaluations", - yaxis_title_text="Criterion value", legend={"yanchor": "top", "xanchor": "right", "y": 0.95, "x": 0.95}, ) + + fig = _plotly_line_plot(lines + multistart_lines, plot_config) return fig -def _harmonize_inputs_to_dict(results, names): +def _harmonize_inputs_to_dict( + results: OptimizeResultOrPath + | list[OptimizeResultOrPath] + | dict[str, OptimizeResultOrPath], + names: list[str] | str | None, +) -> dict[str, OptimizeResult | str | Path]: """Convert all valid inputs for results and names to dict[str, OptimizeResult].""" # convert scalar case to list case if not isinstance(names, list) and names is not None: @@ -187,7 +115,8 @@ def _harmonize_inputs_to_dict(results, names): # unlabeled iterable of results else: - names = range(len(results)) if names is None else names + if names is None: + names = [str(i) for i in range(len(results))] results_dict = dict(zip(names, results, strict=False)) # convert keys to strings @@ -234,7 +163,7 @@ def params_plot( # ================================================================================== if isinstance(result, OptimizeResult): - data = _extract_plotting_data_from_results_object( + data = _retrieve_optimization_data_from_results_object( result, stack_multistart=True, show_exploration=show_exploration, @@ -242,19 +171,19 @@ def params_plot( ) start_params = result.start_params elif isinstance(result, (str, Path)): - data = _extract_plotting_data_from_database( + data = _retrieve_optimization_data_from_database( result, stack_multistart=True, show_exploration=show_exploration, ) - start_params = data["start_params"] + start_params = data.start_params else: raise TypeError("result must be an OptimizeResult or a path to a log file.") - if data["stacked_local_histories"] is not None: - history = data["stacked_local_histories"].params + if data.stacked_local_histories is not None: + history = data.stacked_local_histories.params else: - history = data["history"].params + history = data.history.params # ================================================================================== # Create figure @@ -298,30 +227,94 @@ def params_plot( return fig -def _extract_plotting_data_from_results_object( - res, stack_multistart, show_exploration, plot_name -): - """Extract data for plotting from results object. +@dataclass(frozen=True) +class _PlottingMultistartHistory: + """Data container for an optimization history and metadata. Contains local histories + in case of multistart optimization. + + This dataclass is only used internally. + + """ + + history: History + name: str | None + start_params: PyTree + is_multistart: bool + local_histories: list[History] | list[IterationHistory] | None + stacked_local_histories: History | None + + +def _retrieve_optimization_data( + results: dict[str, OptimizeResult | str | Path], + stack_multistart: bool, + show_exploration: bool, +) -> list[_PlottingMultistartHistory]: + """Retrieve data for criterion plot from results (OptimizeResult or database). Args: - res (OptmizeResult): An optimization results object. - stack_multistart (bool): Whether to combine multistart histories into a single - history. Default is False. - show_exploration (bool): If True, exploration samples of a multistart - optimization are visualized. Default is False. - plot_name (str): Name of the plotting function that calls this function. Used - for rasing errors. + results: A dict of optimization results with collected history. + The key is used as the name in a legend. + stack_multistart: Whether to combine multistart histories into a single history. + Default is False. + show_exploration: If True, exploration samples of a multistart optimization are + visualized. Default is False. Returns: - dict: - - "history": The results history - - "direction": maximize or minimize - - "is_multistart": Whether the optimization used multistart - - "local_histories": All other multistart histories except for 'history'. If not - available is None. If show_exploration is True, the exploration phase is - added as the first entry. - - "stacked_local_histories": If stack_multistart is True the local histories - are stacked into a single one. + A list of objects containing the history, metadata, and local histories of each + optimization result. + + """ + data = [] + for name, res in results.items(): + if isinstance(res, OptimizeResult): + _data = _retrieve_optimization_data_from_results_object( + res=res, + stack_multistart=stack_multistart, + show_exploration=show_exploration, + plot_name="criterion_plot", + res_name=name, + ) + elif isinstance(res, (str, Path)): + _data = _retrieve_optimization_data_from_database( + res=res, + stack_multistart=stack_multistart, + show_exploration=show_exploration, + res_name=name, + ) + else: + msg = ( + "results must be (or contain) an OptimizeResult or a path to a log " + f"file, but is type {type(res)}." + ) + raise TypeError(msg) + + data.append(_data) + + return data + + +def _retrieve_optimization_data_from_results_object( + res: OptimizeResult, + stack_multistart: bool, + show_exploration: bool, + plot_name: str, + res_name: str | None = None, +) -> _PlottingMultistartHistory: + """Retrieve optimization data from results object. + + Args: + res: An optimization results object. + stack_multistart: Whether to combine multistart histories into a single history. + Default is False. + show_exploration: If True, exploration samples of a multistart optimization are + visualized. Default is False. + plot_name: Name of the plotting function that calls this function. Used for + raising errors. + res_name: Name of the result. + + Returns: + A data object containing the history, metadata, and local histories of the + optimization result. """ if res.history is None: @@ -329,76 +322,82 @@ def _extract_plotting_data_from_results_object( "collection by setting collect_history=True when calling maximize or minimize." raise ValueError(msg) - is_multistart = res.multistart_info is not None - - if is_multistart: - local_histories = [opt.history for opt in res.multistart_info.local_optima] + if res.multistart_info: + local_histories = [ + opt.history + for opt in res.multistart_info.local_optima + if opt.history is not None + ] + + if stack_multistart: + stacked = _get_stacked_local_histories(local_histories, res.direction) + if show_exploration: + fun = res.multistart_info.exploration_results[::-1] + stacked.fun + params = res.multistart_info.exploration_sample[::-1] + stacked.params + + stacked = History( + direction=stacked.direction, + fun=fun, + params=params, + # TODO: This needs to be fixed + start_time=len(fun) * [None], # type: ignore + stop_time=len(fun) * [None], # type: ignore + batches=len(fun) * [None], # type: ignore + task=len(fun) * [None], # type: ignore + ) + else: + stacked = None else: local_histories = None - - if stack_multistart and local_histories is not None: - stacked = _get_stacked_local_histories(local_histories, res.direction) - if show_exploration: - fun = res.multistart_info.exploration_results.tolist()[::-1] + stacked.fun - params = res.multistart_info.exploration_sample[::-1] + stacked.params - - stacked = History( - direction=stacked.direction, - fun=fun, - params=params, - # TODO: This needs to be fixed - start_time=len(fun) * [None], - stop_time=len(fun) * [None], - batches=len(fun) * [None], - task=len(fun) * [None], - ) - else: stacked = None - data = { - "history": res.history, - "direction": Direction(res.direction), - "is_multistart": is_multistart, - "local_histories": local_histories, - "stacked_local_histories": stacked, - } + data = _PlottingMultistartHistory( + history=res.history, + name=res_name, + start_params=res.start_params, + is_multistart=res.multistart_info is not None, + local_histories=local_histories, + stacked_local_histories=stacked, + ) return data -def _extract_plotting_data_from_database(res, stack_multistart, show_exploration): - """Extract data for plotting from database. +def _retrieve_optimization_data_from_database( + res: str | Path, + stack_multistart: bool, + show_exploration: bool, + res_name: str | None = None, +) -> _PlottingMultistartHistory: + """Retrieve optimization data from a database. Args: - res (str or pathlib.Path): A path to an optimization database. - stack_multistart (bool): Whether to combine multistart histories into a single - history. Default is False. - show_exploration (bool): If True, exploration samples of a multistart - optimization are visualized. Default is False. + res: A path to an optimization database. + stack_multistart: Whether to combine multistart histories into a single history. + Default is False. + show_exploration: If True, exploration samples of a multistart optimization are + visualized. Default is False. + res_name: Name of the result. Returns: - dict: - - "history": The results history - - "direction": maximize or minimize - - "is_multistart": Whether the optimization used multistart - - "local_histories": All other multistart histories except for 'history'. If not - available is None. If show_exploration is True, the exploration phase is - added as the first entry. - - "stacked_local_histories": If stack_multistart is True the local histories - are stacked into a single one. + A data object containing the history, metadata, and local histories of the + optimization result. """ - reader = LogReader.from_options(SQLiteLogOptions(res)) + reader: LogReader = LogReader.from_options(SQLiteLogOptions(res)) _problem_table = reader.problem_df direction = _problem_table["direction"].tolist()[-1] - _history, local_histories, exploration = reader.read_multistart_history(direction) + multistart_history = reader.read_multistart_history(direction) + _history = multistart_history.history + local_histories = multistart_history.local_histories + exploration = multistart_history.exploration if stack_multistart and local_histories is not None: stacked = _get_stacked_local_histories(local_histories, direction, _history) if show_exploration: - stacked["params"] = exploration["params"][::-1] + stacked["params"] - stacked["criterion"] = exploration["criterion"][::-1] + stacked["criterion"] + stacked["params"] = exploration["params"][::-1] + stacked["params"] # type: ignore + stacked["criterion"] = exploration["criterion"][::-1] + stacked["criterion"] # type: ignore else: stacked = None @@ -409,23 +408,27 @@ def _extract_plotting_data_from_database(res, stack_multistart, show_exploration start_time=_history["time"], # TODO (@janosg): Retrieve `stop_time` from `hist` once it is available. # https://github.com/optimagic-dev/optimagic/pull/553 - stop_time=len(_history["fun"]) * [None], - task=len(_history["fun"]) * [None], + stop_time=len(_history["fun"]) * [None], # type: ignore + task=len(_history["fun"]) * [None], # type: ignore batches=list(range(len(_history["fun"]))), ) - data = { - "history": history, - "direction": direction, - "is_multistart": local_histories is not None, - "local_histories": local_histories, - "stacked_local_histories": stacked, - "start_params": reader.read_start_params(), - } + data = _PlottingMultistartHistory( + history=history, + name=res_name, + start_params=reader.read_start_params(), + is_multistart=local_histories is not None, + local_histories=local_histories, + stacked_local_histories=stacked, + ) return data -def _get_stacked_local_histories(local_histories, direction, history=None): +def _get_stacked_local_histories( + local_histories: list[History] | list[IterationHistory], + direction: Any, + history: History | IterationHistory | None = None, +) -> History: """Stack local histories. Local histories is a list of dictionaries, each of the same structure. We transform @@ -433,7 +436,7 @@ def _get_stacked_local_histories(local_histories, direction, history=None): append the best history at the end. """ - stacked = {"criterion": [], "params": [], "runtime": []} + stacked: dict[str, list[Any]] = {"criterion": [], "params": [], "runtime": []} for hist in local_histories: stacked["criterion"].extend(hist.fun) stacked["params"].extend(hist.params) @@ -453,7 +456,156 @@ def _get_stacked_local_histories(local_histories, direction, history=None): # TODO (@janosg): Retrieve `stop_time` from `hist` once it is available for the # IterationHistory. # https://github.com/optimagic-dev/optimagic/pull/553 - stop_time=len(stacked["criterion"]) * [None], - task=len(stacked["criterion"]) * [None], + stop_time=len(stacked["criterion"]) * [None], # type: ignore + task=len(stacked["criterion"]) * [None], # type: ignore batches=list(range(len(stacked["criterion"]))), ) + + +@dataclass(frozen=True) +class LineData: + """Data of a single line. + + Attributes: + x: The x-coordinates of the points. + y: The y-coordinates of the points. + color: The color of the line. Default is None. + name: The name of the line. Default is None. + show_in_legend: Whether to show the line in the legend. Default is True. + + """ + + x: np.ndarray + y: np.ndarray + color: str | None = None + name: str | None = None + show_in_legend: bool = True + + +def _extract_criterion_plot_lines( + data: list[_PlottingMultistartHistory], + max_evaluations: int | None, + palette_cycle: "itertools.cycle[str]", + stack_multistart: bool, + monotone: bool, +) -> tuple[list[LineData], list[LineData]]: + """Extract lines for criterion plot from data. + + Args: + data: Data retrieved from results or database. + max_evaluations: Clip the criterion history after that many entries. + palette_cycle: Cycle of colors for plotting. + stack_multistart: Whether to combine multistart histories into a single + history. Default is False. + monotone: If True, the criterion plot becomes monotone in the sense that at each + iteration the current best criterion value is displayed. + + Returns: + Tuple containing + - lines: Main optimization paths. + - multistart_lines: Multistart optimization paths. + + """ + fun_or_monotone_fun = "monotone_fun" if monotone else "fun" + + # Collect multistart optimization paths + multistart_lines: list[LineData] = [] + + plot_multistart = len(data) == 1 and data[0].is_multistart and not stack_multistart + + if plot_multistart and data[0].local_histories: + for i, local_history in enumerate(data[0].local_histories): + history = getattr(local_history, fun_or_monotone_fun) + + if max_evaluations is not None and len(history) > max_evaluations: + history = history[:max_evaluations] + + line_data = LineData( + x=np.arange(len(history)), + y=history, + color="#bab0ac", + name=str(i), + show_in_legend=False, + ) + multistart_lines.append(line_data) + + # Collect main optimization paths + lines: list[LineData] = [] + + for _data in data: + if stack_multistart and _data.stacked_local_histories is not None: + _history = _data.stacked_local_histories + else: + _history = _data.history + + history = getattr(_history, fun_or_monotone_fun) + + if max_evaluations is not None and len(history) > max_evaluations: + history = history[:max_evaluations] + + _color = next(palette_cycle) + if not isinstance(_color, str): + msg = "highlight_palette needs to be a string or list of strings, but its " + f"entry is of type {type(_color)}." + raise TypeError(msg) + + line_data = LineData( + x=np.arange(len(history)), + y=history, + color=_color, + name="best result" if plot_multistart else _data.name, + show_in_legend=not plot_multistart, + ) + lines.append(line_data) + + return lines, multistart_lines + + +@dataclass(frozen=True) +class PlotConfig: + """Configuration settings for figure. + + Attributes: + template: The template for the figure. + legend: Configuration for the legend. + + """ + + template: str + legend: dict[str, Any] + + +def _plotly_line_plot(lines: list[LineData], plot_config: PlotConfig) -> go.Figure: + """Create a plotly line plot from the given lines and plot configuration. + + Args: + lines: Data for lines to be plotted. + plot_config: Configuration for the plot. + + Returns: + The figure object containing the lines. + + """ + + fig = go.Figure() + + for line in lines: + trace = go.Scatter( + x=line.x, + y=line.y, + name=line.name, + mode="lines", + line_color=line.color, + showlegend=line.show_in_legend, + connectgaps=True, + ) + fig.add_trace(trace) + + fig.update_layout( + template=plot_config.template, + xaxis_title_text="No. of criterion evaluations", + yaxis_title_text="Criterion value", + legend=plot_config.legend, + ) + + return fig diff --git a/src/optimagic/visualization/plotting_utilities.py b/src/optimagic/visualization/plotting_utilities.py index eea622d9c..3d37c3b97 100644 --- a/src/optimagic/visualization/plotting_utilities.py +++ b/src/optimagic/visualization/plotting_utilities.py @@ -1,5 +1,8 @@ +import base64 +import collections.abc import itertools from copy import deepcopy +from typing import Any import numpy as np import plotly.graph_objects as go @@ -102,8 +105,9 @@ def combine_plots( ub = [] for f in plots: for d in f.data: - lb.append(np.min(d["y"])) - ub.append(np.max(d["y"])) + y = _ensure_array_from_plotly_data(d["y"]) + lb.append(np.min(y)) + ub.append(np.max(y)) ub = np.max(ub) lb = np.min(lb) y_range = ub - lb @@ -115,8 +119,9 @@ def combine_plots( ub = [] for f in plots: for d in f.data: - lb.append(np.min(d["x"])) - ub.append(np.max(d["x"])) + x = _ensure_array_from_plotly_data(d["x"]) + lb.append(np.min(x)) + ub.append(np.max(x)) x_upper = np.max(ub) x_lower = np.min(lb) fig.update_xaxes(range=[x_lower, x_upper]) @@ -328,3 +333,34 @@ def get_layout_kwargs(layout_kwargs, legend_kwargs, title_kwargs, template, show if layout_kwargs: default_kwargs.update(layout_kwargs) return default_kwargs + + +def _ensure_array_from_plotly_data(data: Any) -> np.ndarray: + """Ensure that data is a numpy array, including decoding Plotly v6+ base64 format. + + Args: + data: Can be a numpy array, (nested) sequence (e.g., list of lists), or a + dict with 'bdata' and 'dtype' keys (Plotly v6+ format). + + Returns: + Data as a numpy array. + + Raises: + ValueError: If input cannot be interpreted as an array. + + """ + if isinstance(data, np.ndarray): + return data + elif isinstance(data, dict) and "bdata" in data and "dtype" in data: + return _decode_base64_data(data["bdata"], dtype=data["dtype"]) + elif isinstance(data, collections.abc.Sequence): + try: + return np.array(data, dtype=np.float64) + except Exception: + pass + raise ValueError("Failed to convert input to numpy array.") + + +def _decode_base64_data(b64data: str, dtype: str) -> np.ndarray: + decoded = base64.b64decode(b64data) + return np.frombuffer(decoded, dtype=np.dtype(dtype)) diff --git a/tests/optimagic/differentiation/test_numdiff_options.py b/tests/optimagic/differentiation/test_numdiff_options.py index 9ffc37616..796e27c61 100644 --- a/tests/optimagic/differentiation/test_numdiff_options.py +++ b/tests/optimagic/differentiation/test_numdiff_options.py @@ -80,6 +80,6 @@ def test_numdiff_options_invalid_n_cores(): def test_numdiff_options_invalid_batch_evaluator(): with pytest.raises( - InvalidNumdiffOptionsError, match="Invalid numdiff `batch_evaluator`:" + InvalidNumdiffOptionsError, match="Invalid batch evaluator: invalid" ): NumdiffOptions(batch_evaluator="invalid") diff --git a/tests/optimagic/optimization/test_algorithm.py b/tests/optimagic/optimization/test_algorithm.py index 96a531ea9..71bae8a79 100644 --- a/tests/optimagic/optimization/test_algorithm.py +++ b/tests/optimagic/optimization/test_algorithm.py @@ -25,8 +25,10 @@ {"is_global": "no"}, {"needs_jac": "yes"}, {"needs_hess": "no"}, + {"needs_bounds": "no"}, {"supports_parallelism": "yes"}, {"supports_bounds": "no"}, + {"supports_infinite_bounds": "no"}, {"supports_linear_constraints": "yes"}, {"supports_nonlinear_constraints": "no"}, {"disable_history": "no"}, @@ -42,8 +44,10 @@ def test_algo_info_validation(kwargs): "is_global": True, "needs_jac": True, "needs_hess": True, + "needs_bounds": True, "supports_parallelism": True, "supports_bounds": True, + "supports_infinite_bounds": True, "supports_linear_constraints": True, "supports_nonlinear_constraints": True, "disable_history": True, diff --git a/tests/optimagic/optimization/test_history_collection.py b/tests/optimagic/optimization/test_history_collection.py index 743b8cf43..0adb6a521 100644 --- a/tests/optimagic/optimization/test_history_collection.py +++ b/tests/optimagic/optimization/test_history_collection.py @@ -33,12 +33,18 @@ def test_history_collection_with_parallelization(algorithm, tmp_path): path = tmp_path / "log.db" + algo_options = {"n_cores": 2} + if algorithm == "nevergrad_pso": + algo_options["stopping_maxfun"] = 15 + else: + algo_options["stopping_maxiter"] = 3 + collected_hist = minimize( fun=mark.least_squares(lambda x: x), params=np.arange(5), algorithm=algorithm, bounds=Bounds(lower=lb, upper=ub), - algo_options={"n_cores": 2, "stopping_maxiter": 3}, + algo_options=algo_options, logging=SQLiteLogOptions(path=path, if_database_exists="replace"), ).history @@ -57,8 +63,10 @@ def test_history_collection_with_parallelization(algorithm, tmp_path): is_global=False, needs_jac=False, needs_hess=False, + needs_bounds=False, supports_parallelism=True, supports_bounds=False, + supports_infinite_bounds=False, supports_linear_constraints=False, supports_nonlinear_constraints=False, disable_history=False, diff --git a/tests/optimagic/optimization/test_infinite_and_incomplete_bounds.py b/tests/optimagic/optimization/test_infinite_and_incomplete_bounds.py new file mode 100644 index 000000000..b83cb51ae --- /dev/null +++ b/tests/optimagic/optimization/test_infinite_and_incomplete_bounds.py @@ -0,0 +1,27 @@ +import numpy as np +import pytest + +from optimagic import mark +from optimagic.config import IS_NEVERGRAD_INSTALLED +from optimagic.optimization.optimize import minimize + + +@mark.least_squares +def sos(x): + return x + + +@pytest.mark.skipif( + not IS_NEVERGRAD_INSTALLED, + reason="nevergrad not installed", +) +def test_no_bounds_with_nevergrad(): + res = minimize( + fun=sos, + params=np.arange(3), + algorithm="nevergrad_cmaes", + collect_history=True, + skip_checks=True, + algo_options={"seed": 12345}, + ) + print(res) diff --git a/tests/optimagic/optimization/test_invalid_jacobian_value.py b/tests/optimagic/optimization/test_invalid_jacobian_value.py new file mode 100644 index 000000000..19e186591 --- /dev/null +++ b/tests/optimagic/optimization/test_invalid_jacobian_value.py @@ -0,0 +1,114 @@ +import numpy as np +import pytest + +from optimagic.exceptions import UserFunctionRuntimeError +from optimagic.optimization.optimize import minimize + +# ====================================================================================== +# Test setup: +# -------------------------------------------------------------------------------------- +# We test that minimize raises an error if the user function returns a jacobian +# containing invalid values (np.inf, np.nan). To test that this works not only at +# the start parameters, we create jac functions that return invalid values if the +# parameter norm becomes smaller than one. +# ====================================================================================== + + +@pytest.fixture +def params(): + return {"a": 1, "b": np.array([3, 4])} + + +def sphere(params): + return params["a"] ** 2 + (params["b"] ** 2).sum() + + +def sphere_gradient(params): + return { + "a": 2 * params["a"], + "b": 2 * params["b"], + } + + +def sphere_and_gradient(params): + return sphere(params), sphere_gradient(params) + + +def params_norm(params): + squared_norm = params["a"] ** 2 + np.linalg.norm(params["b"]) ** 2 + return np.sqrt(squared_norm) + + +def get_invalid_jac(invalid_jac_value): + """Get function that returns invalid jac if the parameter norm < 1.""" + + def jac(params): + if params_norm(params) < 1: + return invalid_jac_value + else: + return sphere_gradient(params) + + return jac + + +def get_invalid_fun_and_jac(invalid_jac_value): + """Get function that returns invalid fun and jac if the parameter norm < 1.""" + + def fun_and_jac(params): + if params_norm(params) < 1: + return sphere(params), invalid_jac_value + else: + return sphere_and_gradient(params) + + return fun_and_jac + + +INVALID_JACOBIAN_VALUES = [ + {"a": np.inf, "b": 2 * np.array([1, 2])}, + {"a": 1, "b": 2 * np.array([np.inf, 2])}, + {"a": np.nan, "b": 2 * np.array([1, 2])}, + {"a": 1, "b": 2 * np.array([np.nan, 2])}, +] + + +# ====================================================================================== +# Test Invalid Jacobian raises proper error with jac argument +# ====================================================================================== + + +@pytest.mark.parametrize("invalid_jac_value", INVALID_JACOBIAN_VALUES) +def test_minimize_with_invalid_jac(invalid_jac_value, params): + with pytest.raises( + UserFunctionRuntimeError, + match=( + "The optimization failed because the derivative provided via jac " + "contains infinite or NaN values." + ), + ): + minimize( + fun=sphere, + params=params, + algorithm="scipy_lbfgsb", + jac=get_invalid_jac(invalid_jac_value), + ) + + +# ====================================================================================== +# Test Invalid Jacobian raises proper error with fun_and_jac argument +# ====================================================================================== + + +@pytest.mark.parametrize("invalid_jac_value", INVALID_JACOBIAN_VALUES) +def test_minimize_with_invalid_fun_and_jac(invalid_jac_value, params): + with pytest.raises( + UserFunctionRuntimeError, + match=( + "The optimization failed because the derivative provided via fun_and_jac " + "contains infinite or NaN values." + ), + ): + minimize( + params=params, + algorithm="scipy_lbfgsb", + fun_and_jac=get_invalid_fun_and_jac(invalid_jac_value), + ) diff --git a/tests/optimagic/optimization/test_many_algorithms.py b/tests/optimagic/optimization/test_many_algorithms.py index 47fbe7553..d082783af 100644 --- a/tests/optimagic/optimization/test_many_algorithms.py +++ b/tests/optimagic/optimization/test_many_algorithms.py @@ -13,23 +13,62 @@ from optimagic import mark from optimagic.algorithms import AVAILABLE_ALGORITHMS, GLOBAL_ALGORITHMS +from optimagic.optimization.algorithm import Algorithm from optimagic.optimization.optimize import minimize from optimagic.parameters.bounds import Bounds -LOCAL_ALGORITHMS = { - key: value - for key, value in AVAILABLE_ALGORITHMS.items() - if key not in GLOBAL_ALGORITHMS and key != "bhhh" +AVAILABLE_LOCAL_ALGORITHMS = { + name: algo + for name, algo in AVAILABLE_ALGORITHMS.items() + if name not in GLOBAL_ALGORITHMS and name != "bhhh" } -GLOBAL_ALGORITHMS_AVAILABLE = [ - name for name in AVAILABLE_ALGORITHMS if name in GLOBAL_ALGORITHMS +AVAILABLE_GLOBAL_ALGORITHMS = { + name: algo + for name, algo in AVAILABLE_ALGORITHMS.items() + if name in GLOBAL_ALGORITHMS +} + +AVAILABLE_BOUNDED_ALGORITHMS = { + name: algo + for name, algo in AVAILABLE_LOCAL_ALGORITHMS.items() + if algo.algo_info.supports_bounds +} + + +def _is_stochastic(algo: Algorithm) -> bool: + return hasattr(algo, "seed") + + +LOCAL_STOCHASTIC_ALGORITHMS = [ + name for name, algo in AVAILABLE_LOCAL_ALGORITHMS.items() if _is_stochastic(algo) +] + +LOCAL_DETERMINISTIC_ALGORITHMS = [ + name + for name, algo in AVAILABLE_LOCAL_ALGORITHMS.items() + if not _is_stochastic(algo) +] + +GLOBAL_STOCHASTIC_ALGORITHMS = [ + name for name, algo in AVAILABLE_GLOBAL_ALGORITHMS.items() if _is_stochastic(algo) +] + +GLOBAL_DETERMINISTIC_ALGORITHMS = [ + name + for name, algo in AVAILABLE_GLOBAL_ALGORITHMS.items() + if not _is_stochastic(algo) +] + +BOUNDED_STOCHASTIC_ALGORITHMS = [ + name for name, algo in AVAILABLE_BOUNDED_ALGORITHMS.items() if _is_stochastic(algo) ] -BOUNDED_ALGORITHMS = [] -for name, algo in LOCAL_ALGORITHMS.items(): - if algo.algo_info.supports_bounds: - BOUNDED_ALGORITHMS.append(name) +BOUNDED_DETERMINISTIC_ALGORITHMS = [ + name + for name, algo in AVAILABLE_BOUNDED_ALGORITHMS.items() + if not _is_stochastic(algo) +] @mark.least_squares @@ -37,21 +76,35 @@ def sos(x): return x -@pytest.mark.parametrize("algorithm", LOCAL_ALGORITHMS) -def test_algorithm_on_sum_of_squares(algorithm): +@pytest.mark.parametrize("algorithm", LOCAL_DETERMINISTIC_ALGORITHMS) +def test_deterministic_algorithm_on_sum_of_squares(algorithm): + res = minimize( + fun=sos, + params=np.arange(3), + algorithm=algorithm, + collect_history=True, + skip_checks=True, + ) + assert res.success in [True, None] + aaae(res.params, np.zeros(3), decimal=4) + + +@pytest.mark.parametrize("algorithm", LOCAL_STOCHASTIC_ALGORITHMS) +def test_stochastic_algorithm_on_sum_of_squares(algorithm): res = minimize( fun=sos, params=np.arange(3), algorithm=algorithm, collect_history=True, skip_checks=True, + algo_options={"seed": 12345}, ) assert res.success in [True, None] aaae(res.params, np.zeros(3), decimal=4) -@pytest.mark.parametrize("algorithm", BOUNDED_ALGORITHMS) -def test_algorithm_on_sum_of_squares_with_binding_bounds(algorithm): +@pytest.mark.parametrize("algorithm", BOUNDED_DETERMINISTIC_ALGORITHMS) +def test_deterministic_algorithm_on_sum_of_squares_with_binding_bounds(algorithm): res = minimize( fun=sos, params=np.array([3, 2, -3]), @@ -67,6 +120,24 @@ def test_algorithm_on_sum_of_squares_with_binding_bounds(algorithm): aaae(res.params, np.array([1, 0, -1]), decimal=decimal) +@pytest.mark.parametrize("algorithm", BOUNDED_STOCHASTIC_ALGORITHMS) +def test_stochastic_algorithm_on_sum_of_squares_with_binding_bounds(algorithm): + res = minimize( + fun=sos, + params=np.array([3, 2, -3]), + bounds=Bounds( + lower=np.array([1, -np.inf, -np.inf]), upper=np.array([np.inf, np.inf, -1]) + ), + algorithm=algorithm, + collect_history=True, + skip_checks=True, + algo_options={"seed": 12345}, + ) + assert res.success in [True, None] + decimal = 3 + aaae(res.params, np.array([1, 0, -1]), decimal=decimal) + + skip_msg = ( "The very slow tests of global algorithms are only run on linux which always " "runs much faster in continuous integration." @@ -74,8 +145,23 @@ def test_algorithm_on_sum_of_squares_with_binding_bounds(algorithm): @pytest.mark.skipif(sys.platform == "win32", reason=skip_msg) -@pytest.mark.parametrize("algorithm", GLOBAL_ALGORITHMS_AVAILABLE) -def test_global_algorithms_on_sum_of_squares(algorithm): +@pytest.mark.parametrize("algorithm", GLOBAL_DETERMINISTIC_ALGORITHMS) +def test_deterministic_global_algorithm_on_sum_of_squares(algorithm): + res = minimize( + fun=sos, + params=np.array([0.35, 0.35]), + bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), + algorithm=algorithm, + collect_history=False, + skip_checks=True, + ) + assert res.success in [True, None] + aaae(res.params, np.array([0.2, 0]), decimal=1) + + +@pytest.mark.skipif(sys.platform == "win32", reason=skip_msg) +@pytest.mark.parametrize("algorithm", GLOBAL_STOCHASTIC_ALGORITHMS) +def test_stochastic_global_algorithm_on_sum_of_squares(algorithm): res = minimize( fun=sos, params=np.array([0.35, 0.35]), @@ -83,6 +169,7 @@ def test_global_algorithms_on_sum_of_squares(algorithm): algorithm=algorithm, collect_history=False, skip_checks=True, + algo_options={"seed": 12345}, ) assert res.success in [True, None] aaae(res.params, np.array([0.2, 0]), decimal=1) diff --git a/tests/optimagic/optimization/test_multistart.py b/tests/optimagic/optimization/test_multistart.py index 06ec00236..a6a2f90e2 100644 --- a/tests/optimagic/optimization/test_multistart.py +++ b/tests/optimagic/optimization/test_multistart.py @@ -84,8 +84,8 @@ def with_step_id(self, step_id): exp_values = np.array([-9, -1]) exp_sample = np.array([[4, 5], [0, 1]]) - aaae(calculated["sorted_sample"], exp_sample) - aaae(calculated["sorted_values"], exp_values) + aaae(calculated.sorted_sample, exp_sample) + aaae(calculated.sorted_values, exp_values) def test_get_batched_optimization_sample(): diff --git a/tests/optimagic/optimization/test_with_multistart.py b/tests/optimagic/optimization/test_with_multistart.py index d9f8325d7..bc4d083b1 100644 --- a/tests/optimagic/optimization/test_with_multistart.py +++ b/tests/optimagic/optimization/test_with_multistart.py @@ -80,6 +80,7 @@ def test_multistart_optimization_with_sum_of_squares_at_defaults( assert hasattr(res, "multistart_info") ms_info = res.multistart_info assert len(ms_info.exploration_sample) == 400 + assert isinstance(ms_info.exploration_results, list) assert len(ms_info.exploration_results) == 400 assert all(isinstance(entry, float) for entry in ms_info.exploration_results) assert all(isinstance(entry, OptimizeResult) for entry in ms_info.local_optima) @@ -310,3 +311,15 @@ def ackley(x): "convergence_max_discoveries": 10, }, ) + + +@pytest.mark.slow +def test_with_batch_evaluator(params): + options = om.MultistartOptions(batch_evaluator="threading") + + minimize( + fun=sos_scalar, + params=params, + algorithm="scipy_lbfgsb", + multistart=options, + ) diff --git a/tests/optimagic/optimizers/test_bayesian_optimizer.py b/tests/optimagic/optimizers/test_bayesian_optimizer.py new file mode 100644 index 000000000..918bac092 --- /dev/null +++ b/tests/optimagic/optimizers/test_bayesian_optimizer.py @@ -0,0 +1,145 @@ +"""Unit tests for Bayesian optimizer helper functions.""" + +import numpy as np +import pytest + +from optimagic.config import IS_BAYESOPT_INSTALLED +from optimagic.optimization.internal_optimization_problem import InternalBounds + +if IS_BAYESOPT_INSTALLED: + from bayes_opt import acquisition + + from optimagic.optimizers.bayesian_optimizer import ( + _extract_params_from_kwargs, + _process_acquisition_function, + _process_bounds, + ) + + +def test_extract_params_from_kwargs(): + """Test basic parameter extraction from kwargs dictionary.""" + params_dict = {"param0": 1.0, "param1": 2.0, "param2": 3.0} + result = _extract_params_from_kwargs(params_dict) + np.testing.assert_array_equal(result, np.array([1.0, 2.0, 3.0])) + + +def test_process_bounds_valid(): + """Test processing valid bounds for Bayesian optimization.""" + bounds = InternalBounds(lower=np.array([-1.0, 0.0]), upper=np.array([1.0, 2.0])) + result = _process_bounds(bounds) + expected = {"param0": (-1.0, 1.0), "param1": (0.0, 2.0)} + assert result == expected + + +def test_process_bounds_none(): + """Test processing bounds with None values.""" + bounds = InternalBounds(lower=None, upper=np.array([1.0, 2.0])) + with pytest.raises( + ValueError, match="Bayesian optimization requires finite bounds" + ): + _process_bounds(bounds) + + +def test_process_bounds_infinite(): + """Test processing bounds with infinite values.""" + bounds = InternalBounds(lower=np.array([-1.0, 0.0]), upper=np.array([1.0, np.inf])) + with pytest.raises( + ValueError, match="Bayesian optimization requires finite bounds" + ): + _process_bounds(bounds) + + +@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +def test_process_acquisition_function_none(): + """Test processing None acquisition function.""" + result = _process_acquisition_function( + acquisition_function=None, + kappa=2.576, + xi=0.01, + exploration_decay=None, + exploration_decay_delay=None, + random_state=None, + ) + assert result is None + + +@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +@pytest.mark.parametrize( + "acq_name, expected_class", + [ + ("ucb", acquisition.UpperConfidenceBound), + ("upper_confidence_bound", acquisition.UpperConfidenceBound), + ("ei", acquisition.ExpectedImprovement), + ("expected_improvement", acquisition.ExpectedImprovement), + ("poi", acquisition.ProbabilityOfImprovement), + ("probability_of_improvement", acquisition.ProbabilityOfImprovement), + ], +) +def test_process_acquisition_function_string(acq_name, expected_class): + """Test processing string acquisition function.""" + result = _process_acquisition_function( + acquisition_function=acq_name, + kappa=2.576, + xi=0.01, + exploration_decay=None, + exploration_decay_delay=None, + random_state=None, + ) + assert isinstance(result, expected_class) + + +@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +def test_process_acquisition_function_invalid_string(): + """Test processing invalid string acquisition function.""" + with pytest.raises(ValueError, match="Invalid acquisition_function string"): + _process_acquisition_function( + acquisition_function="acq", + kappa=2.576, + xi=0.01, + exploration_decay=None, + exploration_decay_delay=None, + random_state=None, + ) + + +@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +def test_process_acquisition_function_instance(): + """Test processing acquisition function instance.""" + acq_instance = acquisition.UpperConfidenceBound() + result = _process_acquisition_function( + acquisition_function=acq_instance, + kappa=2.576, + xi=0.01, + exploration_decay=None, + exploration_decay_delay=None, + random_state=None, + ) + assert result is acq_instance + + +@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +def test_process_acquisition_function_class(): + """Test processing acquisition function class.""" + result = _process_acquisition_function( + acquisition_function=acquisition.UpperConfidenceBound, + kappa=2.576, + xi=0.01, + exploration_decay=None, + exploration_decay_delay=None, + random_state=None, + ) + assert isinstance(result, acquisition.UpperConfidenceBound) + + +@pytest.mark.skipif(not IS_BAYESOPT_INSTALLED, reason="bayes_opt not installed") +def test_process_acquisition_function_invalid_type(): + """Test processing invalid acquisition function type.""" + with pytest.raises(TypeError, match="acquisition_function must be None, a string"): + _process_acquisition_function( + acquisition_function=123, + kappa=2.576, + xi=0.01, + exploration_decay=None, + exploration_decay_delay=None, + random_state=None, + ) diff --git a/tests/optimagic/optimizers/test_iminuit_migrad.py b/tests/optimagic/optimizers/test_iminuit_migrad.py new file mode 100644 index 000000000..48e435ef4 --- /dev/null +++ b/tests/optimagic/optimizers/test_iminuit_migrad.py @@ -0,0 +1,95 @@ +"""Test suite for the iminuit migrad optimizer.""" + +import numpy as np +import pytest +from numpy.testing import assert_array_almost_equal as aaae + +from optimagic.config import IS_IMINUIT_INSTALLED +from optimagic.optimization.optimize import minimize +from optimagic.optimizers.iminuit_migrad import ( + IminuitMigrad, + _convert_bounds_to_minuit_limits, +) + + +def sphere(x): + return (x**2).sum() + + +def sphere_grad(x): + return 2 * x + + +def test_convert_bounds_unbounded(): + """Test converting unbounded bounds.""" + lower = np.array([-np.inf, -np.inf]) + upper = np.array([np.inf, np.inf]) + limits = _convert_bounds_to_minuit_limits(lower, upper) + + assert len(limits) == 2 + assert limits[0] == (None, None) + assert limits[1] == (None, None) + + +def test_convert_bounds_lower_only(): + """Test converting lower bounds only.""" + lower = np.array([1.0, 2.0]) + upper = np.array([np.inf, np.inf]) + limits = _convert_bounds_to_minuit_limits(lower, upper) + + assert len(limits) == 2 + assert limits[0] == (1.0, None) + assert limits[1] == (2.0, None) + + +def test_convert_bounds_upper_only(): + """Test converting upper bounds only.""" + lower = np.array([-np.inf, -np.inf]) + upper = np.array([1.0, 2.0]) + limits = _convert_bounds_to_minuit_limits(lower, upper) + + assert len(limits) == 2 + assert limits[0] == (None, 1.0) + assert limits[1] == (None, 2.0) + + +def test_convert_bounds_two_sided(): + """Test converting two-sided bounds.""" + lower = np.array([1.0, -2.0]) + upper = np.array([2.0, -1.0]) + limits = _convert_bounds_to_minuit_limits(lower, upper) + + assert len(limits) == 2 + assert limits[0] == (1.0, 2.0) + assert limits[1] == (-2.0, -1.0) + + +def test_convert_bounds_mixed(): + """Test converting mixed bounds (some infinite, some finite).""" + lower = np.array([-np.inf, 0.0, 1.0]) + upper = np.array([1.0, np.inf, 2.0]) + limits = _convert_bounds_to_minuit_limits(lower, upper) + + assert len(limits) == 3 + assert limits[0] == (None, 1.0) + assert limits[1] == (0.0, None) + assert limits[2] == (1.0, 2.0) + + +@pytest.mark.skipif(not IS_IMINUIT_INSTALLED, reason="iminuit not installed.") +def test_iminuit_migrad(): + """Test basic optimization with sphere function.""" + x0 = np.array([1.0, 2.0, 3.0]) + algorithm = IminuitMigrad() + + res = minimize( + fun=sphere, + jac=sphere_grad, + algorithm=algorithm, + x0=x0, + ) + + assert res.success + aaae(res.x, np.zeros(3), decimal=6) + assert res.n_fun_evals > 0 + assert res.n_jac_evals > 0 diff --git a/tests/optimagic/optimizers/test_nevergrad.py b/tests/optimagic/optimizers/test_nevergrad.py new file mode 100644 index 000000000..af351c005 --- /dev/null +++ b/tests/optimagic/optimizers/test_nevergrad.py @@ -0,0 +1,135 @@ +"""Test helper functions for nevergrad optimizers.""" + +from typing import get_args + +import numpy as np +import pytest +from numpy.testing import assert_array_almost_equal as aaae + +from optimagic import algorithms, mark +from optimagic.config import IS_NEVERGRAD_INSTALLED +from optimagic.optimization.optimize import minimize +from optimagic.parameters.bounds import Bounds + +if IS_NEVERGRAD_INSTALLED: + import nevergrad as ng + + +@mark.least_squares +def sos(x): + return x + + +### Nonlinear constraints on hold until improved handling. +# def dummy_func(): +# return lambda x: x + + +# vec_constr = [ +# { +# "type": "ineq", +# "fun": lambda x: [np.prod(x) + 1.0, 2.0 - np.prod(x)], +# "jac": dummy_func, +# "n_constr": 2, +# } +# ] + +# constrs = [ +# { +# "type": "ineq", +# "fun": lambda x: np.prod(x) + 1.0, +# "jac": dummy_func, +# "n_constr": 1, +# }, +# { +# "type": "ineq", +# "fun": lambda x: 2.0 - np.prod(x), +# "jac": dummy_func, +# "n_constr": 1, +# }, +# ] + + +# def test_process_nonlinear_constraints(): +# got = _process_nonlinear_constraints(vec_constr) +# assert len(got) == 2 + + +# def test_get_constraint_evaluations(): +# x = np.array([1, 1]) +# got = _get_constraint_evaluations(constrs, x) +# expected = [np.array([-2.0]), np.array([-1.0])] +# assert got == expected + + +# def test_batch_constraint_evaluations(): +# x = np.array([1, 1]) +# x_list = [x] * 2 +# got = _batch_constraint_evaluations(constrs, x_list, 2) +# expected = [[np.array([-2.0]), np.array([-1.0])]] * 2 +# assert got == expected +### + + +# test if all optimizers listed in Literal type hint are valid attributes +@pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") +def test_meta_optimizers_are_valid(): + opt = algorithms.NevergradMeta + optimizers = get_args(opt.__annotations__["optimizer"]) + for optimizer in optimizers: + try: + getattr(ng.optimizers, optimizer) + except AttributeError: + pytest.fail(f"Optimizer '{optimizer}' not found in Nevergrad") + + +@pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") +def test_ngopt_optimizers_are_valid(): + opt = algorithms.NevergradNGOpt + optimizers = get_args(opt.__annotations__["optimizer"]) + for optimizer in optimizers: + try: + getattr(ng.optimizers, optimizer) + except AttributeError: + pytest.fail(f"Optimizer '{optimizer}' not found in Nevergrad") + + +# list of available optimizers in nevergrad_meta +NEVERGRAD_META = get_args(algorithms.NevergradMeta.__annotations__["optimizer"]) +# list of available optimizers in nevergrad_ngopt +NEVERGRAD_NGOPT = get_args(algorithms.NevergradNGOpt.__annotations__["optimizer"]) + + +# test stochastic_global_algorithm_on_sum_of_squares +@pytest.mark.slow +@pytest.mark.parametrize("algorithm", NEVERGRAD_META) +@pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") +def test_meta_optimizers_with_stochastic_global_algorithm_on_sum_of_squares(algorithm): + res = minimize( + fun=sos, + params=np.array([0.35, 0.35]), + bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), + algorithm=algorithms.NevergradMeta(algorithm), + collect_history=False, + skip_checks=True, + algo_options={"seed": 12345}, + ) + assert res.success in [True, None] + aaae(res.params, np.array([0.2, 0]), decimal=1) + + +@pytest.mark.slow +@pytest.mark.parametrize("algorithm", NEVERGRAD_NGOPT) +@pytest.mark.skipif(not IS_NEVERGRAD_INSTALLED, reason="nevergrad not installed") +def test_ngopt_optimizers_with_stochastic_global_algorithm_on_sum_of_squares(algorithm): + res = minimize( + fun=sos, + params=np.array([0.35, 0.35]), + bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), + algorithm=algorithms.NevergradNGOpt(algorithm), + collect_history=False, + skip_checks=True, + algo_options={"seed": 12345}, + ) + assert res.success in [True, None] + aaae(res.params, np.array([0.2, 0]), decimal=1) diff --git a/tests/optimagic/parameters/test_bounds.py b/tests/optimagic/parameters/test_bounds.py index 791706d56..3c2ae9a62 100644 --- a/tests/optimagic/parameters/test_bounds.py +++ b/tests/optimagic/parameters/test_bounds.py @@ -4,7 +4,12 @@ from numpy.testing import assert_array_equal from optimagic.exceptions import InvalidBoundsError -from optimagic.parameters.bounds import Bounds, get_internal_bounds, pre_process_bounds +from optimagic.parameters.bounds import ( + Bounds, + _get_fast_path_bounds, + get_internal_bounds, + pre_process_bounds, +) @pytest.fixture() @@ -81,10 +86,9 @@ def test_get_bounds_no_arguments(pytree_params): got_lower, got_upper = get_internal_bounds(pytree_params) expected_lower = np.array([-np.inf] + 3 * [0] + 4 * [-np.inf]) - expected_upper = np.full(8, np.inf) assert_array_equal(got_lower, expected_lower) - assert_array_equal(got_upper, expected_upper) + assert got_upper is None def test_get_bounds_with_lower_bounds(pytree_params): @@ -95,10 +99,9 @@ def test_get_bounds_with_lower_bounds(pytree_params): got_lower, got_upper = get_internal_bounds(pytree_params, bounds=bounds) expected_lower = np.array([0.1] + 3 * [0] + 4 * [-np.inf]) - expected_upper = np.full(8, np.inf) assert_array_equal(got_lower, expected_lower) - assert_array_equal(got_upper, expected_upper) + assert got_upper is None def test_get_bounds_with_upper_bounds(pytree_params): @@ -118,10 +121,8 @@ def test_get_bounds_with_upper_bounds(pytree_params): def test_get_bounds_numpy(array_params): got_lower, got_upper = get_internal_bounds(array_params) - expected = np.array([np.inf, np.inf]) - - assert_array_equal(got_lower, -expected) - assert_array_equal(got_upper, expected) + assert got_lower is None + assert got_upper is None def test_get_bounds_numpy_error(array_params): @@ -132,3 +133,17 @@ def test_get_bounds_numpy_error(array_params): array_params, bounds=bounds, ) + + +def test_get_fast_path_bounds_both_none(): + got_lower, got_upper = _get_fast_path_bounds(Bounds(lower=None, upper=None)) + assert got_lower is None + assert got_upper is None + + +def test_get_fast_path_bounds_lower_none(): + got_lower, got_upper = _get_fast_path_bounds( + bounds=Bounds(lower=None, upper=np.array([1, 2, 3])), + ) + assert_array_equal(got_lower, None) + assert_array_equal(got_upper, np.array([1, 2, 3])) diff --git a/tests/optimagic/parameters/test_conversion.py b/tests/optimagic/parameters/test_conversion.py index da4942443..b1cc3d1d2 100644 --- a/tests/optimagic/parameters/test_conversion.py +++ b/tests/optimagic/parameters/test_conversion.py @@ -23,8 +23,8 @@ def test_get_converter_fast_case(): ) aaae(internal.values, np.arange(3)) - aaae(internal.lower_bounds, np.full(3, -np.inf)) - aaae(internal.upper_bounds, np.full(3, np.inf)) + assert internal.lower_bounds is None + assert internal.upper_bounds is None aaae(converter.params_to_internal(np.arange(3)), np.arange(3)) aaae(converter.params_from_internal(np.arange(3)), np.arange(3)) diff --git a/tests/optimagic/parameters/test_tree_conversion.py b/tests/optimagic/parameters/test_tree_conversion.py index 7f22c918e..705ad0835 100644 --- a/tests/optimagic/parameters/test_tree_conversion.py +++ b/tests/optimagic/parameters/test_tree_conversion.py @@ -79,15 +79,15 @@ def test_tree_conversion_fast_path(solver_type): converter, flat_params = get_tree_converter( params=np.arange(3), - bounds=None, + bounds=Bounds(lower=None, upper=np.arange(3) + 1), func_eval=func_eval, derivative_eval=derivative_eval, solver_type=solver_type, ) aae(flat_params.values, np.arange(3)) - aae(flat_params.lower_bounds, np.full(3, -np.inf)) - aae(flat_params.upper_bounds, np.full(3, np.inf)) + assert flat_params.lower_bounds is None + aae(flat_params.upper_bounds, np.arange(3) + 1) assert flat_params.names == list(map(str, range(3))) aae(converter.params_flatten(np.arange(3)), np.arange(3)) diff --git a/tests/optimagic/test_mark.py b/tests/optimagic/test_mark.py index f13b2c1ff..4c183d429 100644 --- a/tests/optimagic/test_mark.py +++ b/tests/optimagic/test_mark.py @@ -57,8 +57,10 @@ def test_mark_minimizer(): is_global=True, needs_jac=True, needs_hess=True, + needs_bounds=True, supports_parallelism=True, supports_bounds=True, + supports_infinite_bounds=True, supports_linear_constraints=True, supports_nonlinear_constraints=True, disable_history=False, diff --git a/tests/optimagic/visualization/test_history_plots.py b/tests/optimagic/visualization/test_history_plots.py index 85d6b18e0..28cdceb6e 100644 --- a/tests/optimagic/visualization/test_history_plots.py +++ b/tests/optimagic/visualization/test_history_plots.py @@ -3,13 +3,18 @@ import numpy as np import pytest +from numpy.testing import assert_array_equal import optimagic as om from optimagic.logging import SQLiteLogOptions from optimagic.optimization.optimize import minimize from optimagic.parameters.bounds import Bounds from optimagic.visualization.history_plots import ( + LineData, + _extract_criterion_plot_lines, _harmonize_inputs_to_dict, + _PlottingMultistartHistory, + _retrieve_optimization_data, criterion_plot, params_plot, ) @@ -27,8 +32,10 @@ def minimize_result(): params=np.arange(5), algorithm=algorithm, bounds=bounds, - multistart=om.MultistartOptions( - n_samples=1000, convergence_max_discoveries=5 + multistart=( + om.MultistartOptions(n_samples=1000, convergence_max_discoveries=5) + if multistart + else None ), ) res.append(_res) @@ -187,3 +194,101 @@ def test_harmonize_inputs_to_dict_str_input(): def test_harmonize_inputs_to_dict_path_input(): path = Path("test.db") assert _harmonize_inputs_to_dict(results=path, names=None) == {"0": path} + + +def _compare_plotting_multistart_history_with_result( + data: _PlottingMultistartHistory, res: om.OptimizeResult, res_name: str +): + assert_array_equal(data.history.fun, res.history.fun) + assert data.name == res_name + assert_array_equal(data.start_params, res.start_params) + assert data.is_multistart == (res.multistart_info is not None) + + +def test_retrieve_data_from_result(minimize_result): + res = minimize_result[False][0] + results = {"bla": res} + + data = _retrieve_optimization_data( + results=results, stack_multistart=False, show_exploration=False + ) + + assert isinstance(data, list) and len(data) == 1 + assert isinstance(data[0], _PlottingMultistartHistory) + _compare_plotting_multistart_history_with_result( + data=data[0], res=res, res_name="bla" + ) + + +def test_retrieve_data_from_logged_result(tmp_path): + res = minimize( + fun=lambda x: x @ x, + params=np.arange(2), + algorithm="scipy_lbfgsb", + logging=SQLiteLogOptions(tmp_path / "test.db", fast_logging=True), + ) + results = {"logged": tmp_path / "test.db"} + + data = _retrieve_optimization_data( + results=results, stack_multistart=False, show_exploration=False + ) + + assert isinstance(data, list) and len(data) == 1 + assert isinstance(data[0], _PlottingMultistartHistory) + _compare_plotting_multistart_history_with_result( + data=data[0], res=res, res_name="logged" + ) + + +@pytest.mark.parametrize("stack_multistart", [True, False]) +def test_retrieve_data_from_multistart_result(minimize_result, stack_multistart): + res = minimize_result[True][0] + results = {"multistart": res} + + data = _retrieve_optimization_data( + results=results, stack_multistart=stack_multistart, show_exploration=False + ) + + assert isinstance(data, list) and len(data) == 1 + + assert data[0].is_multistart + assert len(data[0].local_histories) == 5 + + if stack_multistart: + assert_array_equal( + data[0].stacked_local_histories.fun, + np.concatenate([hist.fun for hist in data[0].local_histories]), + ) + else: + assert data[0].stacked_local_histories is None + + +def test_extract_criterion_plot_lines(minimize_result): + res = minimize_result[True][0] + results = {"multistart": res} + data = _retrieve_optimization_data( + results=results, stack_multistart=False, show_exploration=False + ) + + palette_cycle = itertools.cycle(["red", "green", "blue"]) + + lines, multistart_lines = _extract_criterion_plot_lines( + data=data, + max_evaluations=None, + palette_cycle=palette_cycle, + stack_multistart=False, + monotone=False, + ) + + history = res.history.fun + + assert isinstance(lines, list) and len(lines) == 1 + assert isinstance(lines[0], LineData) + + assert_array_equal(lines[0].x, np.arange(len(history))) + assert_array_equal(lines[0].y, history) + + assert isinstance(multistart_lines, list) and all( + isinstance(line, LineData) for line in multistart_lines + ) + assert len(multistart_lines) == 5 diff --git a/tests/optimagic/visualization/test_plotting_utilities.py b/tests/optimagic/visualization/test_plotting_utilities.py new file mode 100644 index 000000000..2c6e4a95b --- /dev/null +++ b/tests/optimagic/visualization/test_plotting_utilities.py @@ -0,0 +1,50 @@ +import base64 + +import numpy as np +import pytest +from numpy.testing import assert_array_equal + +from optimagic.visualization.plotting_utilities import ( + _decode_base64_data, + _ensure_array_from_plotly_data, +) + + +def test_decode_base64_data(): + expected = np.arange(10, dtype=float) + encoded = base64.b64encode(expected.tobytes()).decode("ascii") + got = _decode_base64_data(encoded, dtype="float") + assert_array_equal(expected, got) + + +def test_ensure_array_from_plotly_data_case_array(): + expected = np.arange(10, dtype=float) + got = _ensure_array_from_plotly_data(expected) + assert_array_equal(expected, got) + + +def test_ensure_array_from_plotly_data_case_list(): + expected = np.arange(10, dtype=float) + got = _ensure_array_from_plotly_data(expected.tolist()) + assert_array_equal(expected, got) + + +def test_ensure_array_from_plotly_data_case_base64(): + expected = np.arange(10, dtype=float) + encoded = base64.b64encode(expected.tobytes()).decode("ascii") + got = _ensure_array_from_plotly_data({"bdata": encoded, "dtype": "float"}) + assert_array_equal(expected, got) + + +@pytest.mark.parametrize( + "invalid_input", + [ + None, + "not a valid input", + 1234, + [{"a": 1}, {"b": 2}], + ], +) +def test_ensure_array_from_plotly_data_case_invalid(invalid_input): + with pytest.raises(ValueError, match="Failed to convert input to numpy array."): + _ensure_array_from_plotly_data(invalid_input) From 0b30f38d2dc3060b939a46283d31bed4571c7028 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:24:33 +0000 Subject: [PATCH 11/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/algorithms.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/algorithms.md b/docs/source/algorithms.md index 85960ad44..01bbc91cb 100644 --- a/docs/source/algorithms.md +++ b/docs/source/algorithms.md @@ -4742,4 +4742,4 @@ pyensmallen_experimental). :filter: docname in docnames :style: unsrt ``` -```` \ No newline at end of file +```` From 80a2b9584c77e7242998920804ff3427cef68b56 Mon Sep 17 00:00:00 2001 From: gauravmanmode Date: Tue, 12 Aug 2025 16:57:41 +0530 Subject: [PATCH 12/13] use pyensmallen --- src/optimagic/optimizers/pyensmallen_optimizers.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py index 89e66a1a1..ddf0d8e92 100644 --- a/src/optimagic/optimizers/pyensmallen_optimizers.py +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -21,9 +21,8 @@ ) from optimagic.typing import AggregationLevel, NonNegativeFloat, PositiveInt -# use pyensmallen_experimental for testing purpose if IS_PYENSMALLEN_INSTALLED: - import pyensmallen_experimental as pye + import pyensmallen as pye MIN_LINE_SEARCH_STEPS = 1e-20 """The minimum step of the line search.""" @@ -92,17 +91,11 @@ def objective_function( grad[:] = jac return np.float64(fun) - # Passing a Report class to the optimizer allows us to retrieve additional info - ens_res: dict[str, Any] = dict() - report = pye.Report(resultIn=ens_res, disableOutput=True) best_x = optimizer.optimize(objective_function, x0, report) res = InternalOptimizeResult( x=best_x, - fun=ens_res["objective_value"], - n_iterations=ens_res["iterations"], - n_fun_evals=ens_res["evaluate_calls"], - n_jac_evals=ens_res["gradient_calls"], + fun=problem.fun(best_x), ) return res From d0c86f1c99eea54905785b759556e797ec14055a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:28:13 +0000 Subject: [PATCH 13/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/optimagic/optimizers/pyensmallen_optimizers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/optimagic/optimizers/pyensmallen_optimizers.py b/src/optimagic/optimizers/pyensmallen_optimizers.py index ddf0d8e92..571d4dfc4 100644 --- a/src/optimagic/optimizers/pyensmallen_optimizers.py +++ b/src/optimagic/optimizers/pyensmallen_optimizers.py @@ -1,7 +1,6 @@ """Implement ensmallen optimizers.""" from dataclasses import dataclass -from typing import Any import numpy as np from numpy.typing import NDArray