diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml index 398c56cce..717e20209 100644 --- a/.tools/envs/testenv-linux.yml +++ b/.tools/envs/testenv-linux.yml @@ -36,6 +36,7 @@ dependencies: - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - bayes_optim # dev, tests + - gradient_free_optimizers # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests diff --git a/.tools/envs/testenv-nevergrad.yml b/.tools/envs/testenv-nevergrad.yml index 874b9fa5e..d6cbf2d12 100644 --- a/.tools/envs/testenv-nevergrad.yml +++ b/.tools/envs/testenv-nevergrad.yml @@ -33,12 +33,13 @@ dependencies: - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - bayes_optim # dev, tests + - gradient_free_optimizers # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests - types-jinja2 # dev, tests - sqlalchemy-stubs # dev, tests - sphinxcontrib-mermaid # dev, tests, docs - - -e ../../ - bayesian_optimization==1.4.0 - nevergrad + - -e ../../ diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml index c54dc010f..6d6cba6a5 100644 --- a/.tools/envs/testenv-numpy.yml +++ b/.tools/envs/testenv-numpy.yml @@ -34,6 +34,7 @@ dependencies: - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - bayes_optim # dev, tests + - gradient_free_optimizers # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests - types-jinja2 # dev, tests diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml index 308d142aa..76e10889f 100644 --- a/.tools/envs/testenv-others.yml +++ b/.tools/envs/testenv-others.yml @@ -34,6 +34,7 @@ dependencies: - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - bayes_optim # dev, tests + - gradient_free_optimizers # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml index bccee25c6..832c09c34 100644 --- a/.tools/envs/testenv-pandas.yml +++ b/.tools/envs/testenv-pandas.yml @@ -34,6 +34,7 @@ dependencies: - fides==0.7.4 # dev, tests - kaleido>=1.0 # dev, tests - bayes_optim # dev, tests + - gradient_free_optimizers # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests - types-jinja2 # dev, tests diff --git a/.tools/envs/testenv-plotly.yml b/.tools/envs/testenv-plotly.yml index eccdf512d..0de5c1de5 100644 --- a/.tools/envs/testenv-plotly.yml +++ b/.tools/envs/testenv-plotly.yml @@ -33,11 +33,12 @@ dependencies: - Py-BOBYQA # dev, tests - fides==0.7.4 # dev, tests - bayes_optim # dev, tests + - gradient_free_optimizers # dev, tests - pandas-stubs # dev, tests - types-cffi # dev, tests - types-openpyxl # dev, tests - types-jinja2 # dev, tests - sqlalchemy-stubs # dev, tests - sphinxcontrib-mermaid # dev, tests, docs - - -e ../../ - kaleido<0.3 + - -e ../../ diff --git a/docs/source/algorithms.md b/docs/source/algorithms.md index bd8837b9a..8a162a2a9 100644 --- a/docs/source/algorithms.md +++ b/docs/source/algorithms.md @@ -4699,6 +4699,252 @@ package. To use it, you need to have aligned structures and enhancing search performance in rotated coordinate systems. (Default: `False`) - **seed**: Seed for the random number generator for reproducibility. + +``` + +## Gradient Free Optimizers + +Optimizers from the +[gradient_free_optimizers](https://github.com/SimonBlanke/Gradient-Free-Optimizers?tab=readme-ov-file) +package are available in optimagic. To use it, you need to have +[gradient_free_optimizers](https://pypi.org/project/gradient_free_optimizers) installed. + +```{eval-rst} +.. dropdown:: Common options across all optimizers + + .. autoclass:: optimagic.optimizers.gradient_free_optimizers.GFOCommonOptions + +``` + +```{eval-rst} +.. dropdown:: gfo_hillclimbing + + **How to use this algorithm.** + + .. code-block:: python + + import optimagic as om + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm=om.algos.gfo_hillclimbing(stopping_maxiter=1_000, ...), + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + or using the string interface: + + .. code-block:: python + + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm="gfo_hillclimbing", + algo_options={"stopping_maxiter": 1_000, ...}, + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.gradient_free_optimizers.GFOHillClimbing + +``` + +```{eval-rst} +.. dropdown:: gfo_stochastichillclimbing + + **How to use this algorithm.** + + .. code-block:: python + + import optimagic as om + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm=om.algos.gfo_stochastichillclimbing(stopping_maxiter=1_000, ...), + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + or using the string interface: + + .. code-block:: python + + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm="gfo_stochastichillclimbing", + algo_options={"stopping_maxiter": 1_000, ...}, + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.gradient_free_optimizers.GFOStochasticHillClimbing + +``` + +```{eval-rst} +.. dropdown:: gfo_repulsinghillclimbing + + **How to use this algorithm.** + + .. code-block:: python + + import optimagic as om + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm=om.algos.gfo_repulsinghillclimbing(stopping_maxiter=1_000, ...), + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + or using the string interface: + + .. code-block:: python + + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm="gfo_repulsinghillclimbing", + algo_options={"stopping_maxiter": 1_000, ...}, + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.gradient_free_optimizers.GFORepulsingHillClimbing + +``` + +```{eval-rst} +.. dropdown:: gfo_randomrestarthillclimbing + + **How to use this algorithm.** + + .. code-block:: python + + import optimagic as om + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm=om.algos.gfo_randomrestarthillclimbing(stopping_maxiter=1_000, ...), + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + or using the string interface: + + .. code-block:: python + + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm="gfo_randomrestarthillclimbing", + algo_options={"stopping_maxiter": 1_000, ...}, + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.gradient_free_optimizers.GFORandomRestartHillClimbing + +``` + +```{eval-rst} +.. dropdown:: gfo_simulatedannealing + + **How to use this algorithm.** + + .. code-block:: python + + import optimagic as om + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm=om.algos.gfo_simulatedannealing(stopping_maxiter=1_000, ...), + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + or using the string interface: + + .. code-block:: python + + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm="gfo_simulatedannealing", + algo_options={"stopping_maxiter": 1_000, ...}, + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.gradient_free_optimizers.GFOSimulatedAnnealing + +``` + +```{eval-rst} +.. dropdown:: gfo_downhillsimplex + + **How to use this algorithm.** + + .. code-block:: python + + import optimagic as om + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm=om.algos.gfo_downhillsimplex(stopping_maxiter=1_000, ...), + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + or using the string interface: + + .. code-block:: python + + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm="gfo_downhillsimplex", + algo_options={"stopping_maxiter": 1_000, ...}, + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.gradient_free_optimizers.GFODownhillSimplex + +``` + +```{eval-rst} +.. dropdown:: gfo_pso + + **How to use this algorithm.** + + .. code-block:: python + + import optimagic as om + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm=om.algos.gfo_pso(stopping_maxiter=1_000, ...), + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + or using the string interface: + + .. code-block:: python + + om.minimize( + fun=lambda x: x @ x, + params=[1.0, 2.0, 3.0], + algorithm="gfo_pso", + algo_options={"stopping_maxiter": 1_000, ...}, + bounds = om.Bounds(lower = np.array([1,1,1]), upper=np.array([5,5,5])) + ) + + **Description and available options:** + + .. autoclass:: optimagic.optimizers.gradient_free_optimizers.GFOParticleSwarmOptimization + ``` ## References diff --git a/environment.yml b/environment.yml index 6bb4f01db..8dda31e0d 100644 --- a/environment.yml +++ b/environment.yml @@ -48,6 +48,7 @@ dependencies: - kaleido>=1.0 # dev, tests - pre-commit>=4 # dev - bayes_optim # dev, tests + - gradient_free_optimizers # dev, tests - -e . # dev # type stubs - pandas-stubs # dev, tests diff --git a/pyproject.toml b/pyproject.toml index c74752252..777d7b2f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -381,5 +381,7 @@ module = [ "iminuit", "nevergrad", "yaml", + "gradient_free_optimizers", + "gradient_free_optimizers.optimizers.base_optimizer", ] ignore_missing_imports = true diff --git a/src/optimagic/algorithms.py b/src/optimagic/algorithms.py index f86792478..4fd4f65a3 100644 --- a/src/optimagic/algorithms.py +++ b/src/optimagic/algorithms.py @@ -15,6 +15,14 @@ from optimagic.optimizers.bayesian_optimizer import BayesOpt from optimagic.optimizers.bhhh import BHHH from optimagic.optimizers.fides import Fides +from optimagic.optimizers.gfo_optimizers import ( + GFODownhillSimplex, + GFOHillClimbing, + GFORandomRestartHillClimbing, + GFORepulsingHillClimbing, + GFOSimulatedAnnealing, + GFOStochasticHillClimbing, +) from optimagic.optimizers.iminuit_migrad import IminuitMigrad from optimagic.optimizers.ipopt import Ipopt from optimagic.optimizers.nag_optimizers import NagDFOLS, NagPyBOBYQA @@ -556,6 +564,16 @@ def Scalar(self) -> BoundedGradientFreeLocalNonlinearConstrainedScalarAlgorithms @dataclass(frozen=True) class BoundedGradientFreeLocalScalarAlgorithms(AlgoSelection): + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA nlopt_cobyla: Type[NloptCOBYLA] = NloptCOBYLA @@ -1291,6 +1309,16 @@ def Scalar(self) -> GlobalGradientFreeParallelScalarAlgorithms: @dataclass(frozen=True) class BoundedGradientFreeLocalAlgorithms(AlgoSelection): + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA @@ -1340,6 +1368,16 @@ def Scalar(self) -> GradientFreeLocalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class GradientFreeLocalScalarAlgorithms(AlgoSelection): + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel nlopt_bobyqa: Type[NloptBOBYQA] = NloptBOBYQA @@ -1432,6 +1470,16 @@ def Scalar(self) -> BoundedGradientFreeNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class BoundedGradientFreeScalarAlgorithms(AlgoSelection): bayes_opt: Type[BayesOpt] = BayesOpt + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim nevergrad_cga: Type[NevergradCGA] = NevergradCGA @@ -1895,6 +1943,16 @@ def Scalar(self) -> BoundedLocalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class BoundedLocalScalarAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA @@ -2412,6 +2470,16 @@ def Scalar(self) -> GlobalGradientFreeScalarAlgorithms: @dataclass(frozen=True) class GradientFreeLocalAlgorithms(AlgoSelection): + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel @@ -2453,6 +2521,16 @@ def Scalar(self) -> GradientFreeLocalScalarAlgorithms: @dataclass(frozen=True) class BoundedGradientFreeAlgorithms(AlgoSelection): bayes_opt: Type[BayesOpt] = BayesOpt + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim @@ -2563,6 +2641,16 @@ def Scalar(self) -> GradientFreeNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class GradientFreeScalarAlgorithms(AlgoSelection): bayes_opt: Type[BayesOpt] = BayesOpt + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel nevergrad_bo: Type[NevergradBayesOptim] = NevergradBayesOptim @@ -2911,6 +2999,16 @@ def Scalar(self) -> GlobalParallelScalarAlgorithms: @dataclass(frozen=True) class BoundedLocalAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_dfols: Type[NagDFOLS] = NagDFOLS @@ -2994,6 +3092,16 @@ def Scalar(self) -> LocalNonlinearConstrainedScalarAlgorithms: @dataclass(frozen=True) class LocalScalarAlgorithms(AlgoSelection): fides: Type[Fides] = Fides + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA @@ -3146,6 +3254,16 @@ def Scalar(self) -> BoundedNonlinearConstrainedScalarAlgorithms: class BoundedScalarAlgorithms(AlgoSelection): bayes_opt: Type[BayesOpt] = BayesOpt fides: Type[Fides] = Fides + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA @@ -3494,6 +3612,16 @@ def Scalar(self) -> GradientBasedScalarAlgorithms: @dataclass(frozen=True) class GradientFreeAlgorithms(AlgoSelection): bayes_opt: Type[BayesOpt] = BayesOpt + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) nag_dfols: Type[NagDFOLS] = NagDFOLS nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA neldermead_parallel: Type[NelderMeadParallel] = NelderMeadParallel @@ -3652,6 +3780,16 @@ def Scalar(self) -> GlobalScalarAlgorithms: class LocalAlgorithms(AlgoSelection): bhhh: Type[BHHH] = BHHH fides: Type[Fides] = Fides + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_dfols: Type[NagDFOLS] = NagDFOLS @@ -3724,6 +3862,16 @@ def Scalar(self) -> LocalScalarAlgorithms: class BoundedAlgorithms(AlgoSelection): bayes_opt: Type[BayesOpt] = BayesOpt fides: Type[Fides] = Fides + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_dfols: Type[NagDFOLS] = NagDFOLS @@ -3874,6 +4022,16 @@ def Scalar(self) -> NonlinearConstrainedScalarAlgorithms: class ScalarAlgorithms(AlgoSelection): bayes_opt: Type[BayesOpt] = BayesOpt fides: Type[Fides] = Fides + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_pybobyqa: Type[NagPyBOBYQA] = NagPyBOBYQA @@ -4076,6 +4234,16 @@ class Algorithms(AlgoSelection): bayes_opt: Type[BayesOpt] = BayesOpt bhhh: Type[BHHH] = BHHH fides: Type[Fides] = Fides + gfo_downhillsimplex: Type[GFODownhillSimplex] = GFODownhillSimplex + gfo_hillclimbing: Type[GFOHillClimbing] = GFOHillClimbing + gfo_randomrestarthillclimbing: Type[GFORandomRestartHillClimbing] = ( + GFORandomRestartHillClimbing + ) + gfo_repulsinghillclimbing: Type[GFORepulsingHillClimbing] = GFORepulsingHillClimbing + gfo_simulatedannealing: Type[GFOSimulatedAnnealing] = GFOSimulatedAnnealing + gfo_stochastichillclimbing: Type[GFOStochasticHillClimbing] = ( + GFOStochasticHillClimbing + ) iminuit_migrad: Type[IminuitMigrad] = IminuitMigrad ipopt: Type[Ipopt] = Ipopt nag_dfols: Type[NagDFOLS] = NagDFOLS diff --git a/src/optimagic/config.py b/src/optimagic/config.py index ce6cd4d60..d6a288ad0 100644 --- a/src/optimagic/config.py +++ b/src/optimagic/config.py @@ -39,6 +39,7 @@ def _is_installed(module_name: str) -> bool: IS_IMINUIT_INSTALLED = _is_installed("iminuit") IS_NEVERGRAD_INSTALLED = _is_installed("nevergrad") IS_BAYESOPT_INSTALLED = _is_installed("bayes_opt") +IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED = _is_installed("gradient_free_optimizers") # ====================================================================================== diff --git a/src/optimagic/optimization/internal_optimization_problem.py b/src/optimagic/optimization/internal_optimization_problem.py index 3d630c7bf..43325d143 100644 --- a/src/optimagic/optimization/internal_optimization_problem.py +++ b/src/optimagic/optimization/internal_optimization_problem.py @@ -960,3 +960,137 @@ def __init__( nonlinear_constraints=nonlinear_constraints, logger=logger, ) + + +class SphereExampleInternalOptimizationProblemWithConverter( + InternalOptimizationProblem +): + """Super simple example of an internal optimization problem with PyTree Converter. + Note: params should be a dict with key-value pairs `"x{i}" : val . + eg. `{'x0': 1, 'x1': 2, ...}`. + + The converter.params_to_internal method converts tree like + `{'x0': 1, 'x1': 2, 'x2': 3 ...}` to flat array `[1,2,3 ...]` . + + The converter.params_from_internal method converts flat array `[1,2,3 ...]` + to tree like `{'x0': 1, 'x1': 2, 'x2': 3 ...}`. + + The converter.derivative_to_internal converts derivative trees + {'x0': 2,'x1': 4, } to flat arrays [2,4] and jacobian tree + `{ "x0": {"x0": 1, "x1": 0, }, + "x1": {"x0": 0, "x1": 1, }` + to NDArray [[1, 0,], [0, 1, ],]. }. + This can be used to test algorithm wrappers or to familiarize yourself + with the internal optimization problem interface. + + Args: + + """ + + def __init__( + self, + solver_type: AggregationLevel = AggregationLevel.SCALAR, + binding_bounds: bool = False, + ) -> None: + def sphere(params: PyTree) -> SpecificFunctionValue: + out = sum([params[f"x{i}"] ** 2 for i in range(len(params))]) + return ScalarFunctionValue(out) + + def ls_sphere(params: PyTree) -> SpecificFunctionValue: + out = [params[f"x{i}"] for i in range(len(params))] + return LeastSquaresFunctionValue(out) + + def likelihood_sphere(params: PyTree) -> SpecificFunctionValue: + out = [params[f"x{i}"] ** 2 for i in range(len(params))] + return LikelihoodFunctionValue(out) + + _fun_dict = { + AggregationLevel.SCALAR: sphere, + AggregationLevel.LIKELIHOOD: likelihood_sphere, + AggregationLevel.LEAST_SQUARES: ls_sphere, + } + + def sphere_gradient(params: PyTree) -> PyTree: + return {params[f"x{i}"]: 2 * v for i, v in enumerate(params.values())} + + def likelihood_sphere_gradient(params: PyTree) -> PyTree: + return {params[f"x{i}"]: 2 * v for i, v in enumerate(params.values())} + + def ls_sphere_jac(params: PyTree) -> PyTree: + return { + f"x{i}": {f"x{j}": 1 if i == j else 0 for j in range(len(params))} + for i in range(len(params)) + } + + _jac_dict = { + AggregationLevel.SCALAR: sphere_gradient, + AggregationLevel.LIKELIHOOD: likelihood_sphere_gradient, + AggregationLevel.LEAST_SQUARES: ls_sphere_jac, + } + + fun = _fun_dict[solver_type] + jac = _jac_dict[solver_type] + fun_and_jac = lambda x: (fun(x), jac(x)) + + def params_flatten(params: PyTree) -> NDArray[np.float64]: + return np.array([v for v in params.values()]).astype(float) + + def params_unflatten(x: NDArray[np.float64]) -> PyTree: + return {f"x{i}": v for i, v in enumerate(x)} + + def derivative_flatten(tree: PyTree, x: NDArray[np.float64]) -> Any: + if solver_type == AggregationLevel.LEAST_SQUARES: + out = [list(row.values()) for row in tree.values()] + return np.array(out) + else: + return params_flatten(tree) + + converter = Converter( + params_to_internal=params_flatten, + params_from_internal=params_unflatten, + derivative_to_internal=derivative_flatten, + has_transforming_constraints=False, + ) + + direction = Direction.MINIMIZE + + if binding_bounds: + lb = np.arange(10, dtype=np.float64) - 7.0 + ub = np.arange(10, dtype=np.float64) - 3.0 + self._x_opt = np.array([-3, -2, -1, 0, 0, 0, 0, 0, 1, 2.0]) + else: + lb = np.full(10, -10, dtype=np.float64) + ub = np.full(10, 10, dtype=np.float64) + self._x_opt = np.zeros(10) + + bounds = InternalBounds(lb, ub) + + numdiff_options = NumdiffOptions() + + error_handling = ErrorHandling.RAISE + + error_penalty_func = fun_and_jac + + batch_evaluator = process_batch_evaluator("joblib") + + linear_constraints = None + nonlinear_constraints = None + + logger = None + + super().__init__( + fun=fun, + jac=jac, + fun_and_jac=fun_and_jac, + converter=converter, + solver_type=solver_type, + direction=direction, + bounds=bounds, + numdiff_options=numdiff_options, + error_handling=error_handling, + error_penalty_func=error_penalty_func, + batch_evaluator=batch_evaluator, + linear_constraints=linear_constraints, + nonlinear_constraints=nonlinear_constraints, + logger=logger, + ) diff --git a/src/optimagic/optimizers/gfo_optimizers.py b/src/optimagic/optimizers/gfo_optimizers.py new file mode 100644 index 000000000..f166fa83e --- /dev/null +++ b/src/optimagic/optimizers/gfo_optimizers.py @@ -0,0 +1,738 @@ +from __future__ import annotations + +from dataclasses import dataclass, fields +from functools import partial +from typing import TYPE_CHECKING, Any, Literal + +import numpy as np +from numpy.typing import NDArray + +from optimagic import mark +from optimagic.config import IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED +from optimagic.optimization.algo_options import ( + STOPPING_MAXFUN_GLOBAL, +) +from optimagic.optimization.algorithm import Algorithm, InternalOptimizeResult +from optimagic.optimization.internal_optimization_problem import ( + InternalBounds, + InternalOptimizationProblem, +) +from optimagic.parameters.conversion import Converter +from optimagic.typing import ( + AggregationLevel, + NonNegativeFloat, + PositiveFloat, + PositiveInt, + PyTree, +) + +if TYPE_CHECKING: + from gradient_free_optimizers.optimizers.base_optimizer import BaseOptimizer + + +@dataclass(frozen=True) +class GFOCommonOptions: + """Common options for all optimizers from GFO.""" + + n_grid_points: PositiveInt | PyTree = 200 + """Number of grid points per dimension. + + If an integer is provided, it will be used for all dimensions. + + """ + + n_init: PositiveInt = 10 + """Number of initialization steps to run. + + Accordingly, N positions will be initialized at the vertices and remaining + initialized randmoly in the search space. + + """ + + stopping_maxiter: PositiveInt = STOPPING_MAXFUN_GLOBAL + """Maximum number of iterations.""" + + stopping_maxtime: NonNegativeFloat | None = None + """Maximum time in seconds before termination.""" + + stopping_funval: float | None = None + """"Stop the optimization if the objective function is less than this value.""" + + convergence_iter_noimprove: PositiveInt = 1000 # need to set high + """Number of iterations without improvement before termination.""" + + convergence_ftol_abs: NonNegativeFloat | None = None + """Converge if the absolute change in the objective function is less than this + value.""" + + convergence_ftol_rel: NonNegativeFloat | None = None + """Converge if the relative change in the objective function is less than this + value.""" + + caching: bool = True + """Whether to cache evaluated param and function values in a dictionary for + lookup.""" + + warm_start: list[PyTree] | None = None + """List of additional start points for the optimization run.""" + + verbosity: Literal["progress_bar", "print_results", "print_times"] | bool = False + """Determines what part of the optimization information will be printed.""" + + seed: int | None = None + """Random seed for reproducibility.""" + + def common_options(self) -> GFOCommonOptions: + """Return a GFOCommonOptions instance with only the common options.""" + return GFOCommonOptions( + **{ + field.name: getattr(self, field.name) + for field in fields(GFOCommonOptions) + } + ) + + +@mark.minimizer( + name="gfo_hillclimbing", + solver_type=AggregationLevel.SCALAR, + is_available=IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED, + is_global=False, + needs_jac=False, + needs_hess=False, + needs_bounds=True, + supports_parallelism=False, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class GFOHillClimbing(Algorithm, GFOCommonOptions): + """Minimize a scalar function using the HillClimbing algorithm. + + This algorithm is a Python implementation of the HillClimbing algorithm through the + gradient_free_optimizers package. + + Hill climbing is a local search algorithm suited for exploring combinatorial search + spaces. + + It starts at an initial point, which is often chosen randomly and continues to move + to positions within its neighbourhood with a better solution. It has no method + against getting stuck in local optima. + + """ + + epsilon: PositiveFloat = 0.03 + """The step-size of the hill climbing algorithm. If step_size is too large the newly + selected positions will be at the edge of the search space. + + If its value is very low it might not find new positions. + + """ + + distribution: Literal["normal", "laplace", "logistic", "gumbel"] = "normal" + """The mathematical distribution the algorithm draws samples from. + + All available distributions are taken from the numpy-package. + + """ + + n_neighbours: PositiveInt = 3 + """The number of positions the algorithm explores from its current postion before + setting its current position to the best of those neighbour positions. + + If the value of n_neighbours is large the hill-climbing-based algorithm will take a + lot of time to choose the next position to move to, but the choice will probably be + a good one. It might be a prudent approach to increase n_neighbours of the search- + space has a lot of dimensions, because there are more possible directions to move + to. + + """ + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + import gradient_free_optimizers as gfo + + opt = gfo.HillClimbingOptimizer + optimizer = partial( + opt, + epsilon=self.epsilon, + distribution=self.distribution, + n_neighbours=self.n_neighbours, + ) + res = _gfo_internal( + common_options=self.common_options(), + problem=problem, + x0=x0, + optimizer=optimizer, + ) + + return res + + +@mark.minimizer( + name="gfo_stochastichillclimbing", + solver_type=AggregationLevel.SCALAR, + is_available=IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED, + is_global=False, + needs_jac=False, + needs_hess=False, + needs_bounds=True, + supports_parallelism=False, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class GFOStochasticHillClimbing(Algorithm, GFOCommonOptions): + """Minimize a scalar function using the Stochastic Hill Climbing algorithm. + + This algorithm is a Python implementation of the StochasticHillClimbing algorithm + through the gradient_free_optimizers package. + + Stochastic hill climbing extends the normal hill climbing by a simple method against + getting stuck in local optima. + + """ + + epsilon: PositiveFloat = 0.027 + """The step-size of the hill climbing algorithm.If step_size is too large the newly + selected positions will be at the edge of the search space. + + If its value is very low it might not find new positions. + + """ + + distribution: Literal["normal", "laplace", "logistic", "gumbel"] = "normal" + """The mathematical distribution the algorithm draws samples from. + + All available distributions are taken from the numpy-package. + + """ + + n_neighbours: PositiveInt = 3 + """The number of positions the algorithm explores from its current postion before + setting its current position to the best of those neighbour positions. + + If the value of n_neighbours is large the hill-climbing-based algorithm will take a + lot of time to choose the next position to move to, but the choice will probably be + a good one. It might be a prudent approach to increase n_neighbours of the search- + space has a lot of dimensions, because there are more possible directions to move + to. + + """ + + p_accept: NonNegativeFloat = 0.5 + """The probability factor used in the equation to calculate if a worse position is + accepted as the new position. + + If the new score is not better than the previous one the algorithm accepts worse + positions with probability p_accept. + + .. math:: + score_{normalized} = norm * \\frac{score_{current} - score_{new}} + {score_{current} + score_{new}} + .. math:: + p = \\exp^{-score_{normalized}} + + If p is less than p_accept the new position gets accepted anyways. + + """ + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + import gradient_free_optimizers as gfo + + opt = gfo.StochasticHillClimbingOptimizer + optimizer = partial( + opt, + epsilon=self.epsilon, + distribution=self.distribution, + n_neighbours=self.n_neighbours, + p_accept=self.p_accept, + ) + res = _gfo_internal( + common_options=self.common_options(), + problem=problem, + x0=x0, + optimizer=optimizer, + ) + + return res + + +@mark.minimizer( + name="gfo_repulsinghillclimbing", + solver_type=AggregationLevel.SCALAR, + is_available=IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED, + is_global=False, + needs_jac=False, + needs_hess=False, + needs_bounds=True, + supports_parallelism=False, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class GFORepulsingHillClimbing(Algorithm, GFOCommonOptions): + """Minimize a scalar function using the Repulsing Hill Climbing algorithm. + + This algorithm is a Python implementation of the Repulsing Hill Climbing algorithm + through the gradient_free_optimizers package. + + The algorithm inherits from the Hill climbing which is a local search algorithm but + always activates its methods to espace local optima. + + """ + + epsilon: PositiveFloat = 0.003 + """The step-size of the hill climbing algorithm. If step_size is too large the newly + selected positions will be at the edge of the search space. + + If its value is very low it might not find new positions. + + """ + + distribution: Literal["normal", "laplace", "logistic", "gumbel"] = "normal" + """The mathematical distribution the algorithm draws samples from. + + All available distributions are taken from the numpy-package. + + """ + + n_neighbours: PositiveInt = 3 + """The number of positions the algorithm explores from its current position before + setting its current position to the best of those neighbour positions.""" + + repulsion_factor: PositiveFloat = 2 + """The algorithm increases the step size by multiplying it with the repulsion_factor + for the next iteration. This way the algorithm escapes the region that does not + offer better positions. + + .. math:: + \\epsilon = \\epsilon * {repulsion factor} + + """ + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + import gradient_free_optimizers as gfo + + opt = gfo.RepulsingHillClimbingOptimizer + optimizer = partial( + opt, + epsilon=self.epsilon, + distribution=self.distribution, + n_neighbours=self.n_neighbours, + repulsion_factor=self.repulsion_factor, + ) + res = _gfo_internal( + common_options=self.common_options(), + problem=problem, + x0=x0, + optimizer=optimizer, + ) + + return res + + +@mark.minimizer( + name="gfo_randomrestarthillclimbing", + solver_type=AggregationLevel.SCALAR, + is_available=IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED, + is_global=False, + needs_jac=False, + needs_hess=False, + needs_bounds=True, + supports_parallelism=False, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class GFORandomRestartHillClimbing(Algorithm, GFOCommonOptions): + """Minimize a scalar function using the Random Restart Hill Climbing algorithm. + + This algorithm is a Python implementation of the Random Restart Hill Climbing + algorithm through the gradient_free_optimizers package. + + The random restart hill climbing works by starting a hill climbing search and + jumping to a random new position after n_iter_restart iterations. Those restarts + should prevent the algorithm getting stuck in local optima. + + """ + + epsilon: PositiveFloat = 0.022 + """The step-size of the hill climbing algorithm.If step_size is too large the newly + selected positions will be at the edge of the search space. + + If its value is very low it might not find new positions. + + """ + + distribution: Literal["normal", "laplace", "logistic", "gumbel"] = "normal" + """The mathematical distribution the algorithm draws samples from. + + All available distributions are taken from the numpy-package. + + """ + + n_neighbours: PositiveInt = 3 + """The number of positions the algorithm explores from its current postion before + setting its current position to the best of those neighbour positions. + + If the value of n_neighbours is large the hill-climbing-based algorithm will take a + lot of time to choose the next position to move to, but the choice will probably be + a good one. It might be a prudent approach to increase n_neighbours of the search- + space has a lot of dimensions, because there are more possible directions to move + to. + + """ + + n_iter_restart: PositiveInt = 10 + """The number of iterations the algorithm performs before jumping to a random + position.""" + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + import gradient_free_optimizers as gfo + + opt = gfo.RandomRestartHillClimbingOptimizer + optimizer = partial( + opt, + epsilon=self.epsilon, + distribution=self.distribution, + n_neighbours=self.n_neighbours, + n_iter_restart=self.n_iter_restart, + ) + res = _gfo_internal( + common_options=self.common_options(), + problem=problem, + x0=x0, + optimizer=optimizer, + ) + + return res + + +@mark.minimizer( + name="gfo_simulatedannealing", + solver_type=AggregationLevel.SCALAR, + is_available=IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED, + is_global=False, + needs_jac=False, + needs_hess=False, + needs_bounds=True, + supports_parallelism=False, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class GFOSimulatedAnnealing(Algorithm, GFOCommonOptions): + """Minimize a scalar function using the Simulated Annealing algorithm. + + This algorithm is a Python implementation of Simulated Annealing through the + gradient_free_optimizers package. + + Simulated annealing chooses its next possible position similar to hill climbing, but + it accepts worse results with a probability that decreases with time. It simulates a + temperature that decreases with each iteration, similar to a material cooling down. + + """ + + epsilon: PositiveFloat = 0.03 + """The step-size of the algorithm. + + If step_size is too large the newly selected positions will be at the edge of the + search space. If its value is very low it might not find new positions. + + """ + + distribution: Literal["normal", "laplace", "logistic", "gumbel"] = "normal" + """The mathematical distribution the algorithm draws samples from. + + All available distributions are taken from the numpy-package. + + """ + + n_neighbours: PositiveInt = 3 + """The number of positions the algorithm explores from its current position before + setting its current position to the best of those neighbour positions.""" + + start_temp: PositiveFloat = 1 + """The start_temp is a factor for the probability p of accepting a worse position. + + .. math:: + p = \\exp^{-\\frac{score_{normalized}}{temp}} + + """ + + annealing_rate: PositiveFloat = 0.215 + """Rate at which the temperatur-value of the algorithm decreases. An annealing rate + above 1 increases the temperature over time. + + .. math:: + start\\_temp \\leftarrow start\\_temp * annealing\\_rate + + """ + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + import gradient_free_optimizers as gfo + + opt = gfo.SimulatedAnnealingOptimizer + optimizer = partial( + opt, + epsilon=self.epsilon, + distribution=self.distribution, + n_neighbours=self.n_neighbours, + start_temp=self.start_temp, + annealing_rate=self.annealing_rate, + ) + res = _gfo_internal( + common_options=self.common_options(), + problem=problem, + x0=x0, + optimizer=optimizer, + ) + return res + + +@mark.minimizer( + name="gfo_downhillsimplex", # nelder_mead + solver_type=AggregationLevel.SCALAR, + is_available=IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED, + is_global=False, + needs_jac=False, + needs_hess=False, + needs_bounds=True, + supports_parallelism=False, + supports_bounds=True, + supports_infinite_bounds=False, + supports_linear_constraints=False, + supports_nonlinear_constraints=False, + disable_history=False, +) +@dataclass(frozen=True) +class GFODownhillSimplex(Algorithm, GFOCommonOptions): + """Minimize a scalar function using the Downhill Simplex algorithm. + + This algorithm is a Python implementation of the Downhill Simplex algorithm through + the gradient_free_optimizers package. + + The Downhill simplex or Nelder mead algorithm works by grouping `number of + dimensions + 1` positions into a simplex, which can explore the search-space by + changing shape. The simplex changes shape by reflecting, expanding, contracting or + shrinking via the alpha, gamma, beta or sigma parameters. It needs at least `number + of dimensions + 1` initial positions to form a simplex in the search-space and the + movement of the positions in the simplex are affected by each other. + + """ + + simplex_reflection: PositiveFloat = 1 + """The reflection parameter of the simplex algorithm.""" + + simplex_expansion: PositiveFloat = 2 + """The expansion parameter of the simplex algorithm.""" + + simplex_contraction: PositiveFloat = 0.5 + """The contraction parameter of the simplex algorithm.""" + + simplex_shrinking: PositiveFloat = 0.5 + """The shrinking parameter of the simplex algorithm.""" + + def _solve_internal_problem( + self, problem: InternalOptimizationProblem, x0: NDArray[np.float64] + ) -> InternalOptimizeResult: + import gradient_free_optimizers as gfo + + opt = gfo.DownhillSimplexOptimizer + optimizer = partial( + opt, + alpha=self.simplex_reflection, + gamma=self.simplex_expansion, + beta=self.simplex_contraction, + sigma=self.simplex_shrinking, + ) + res = _gfo_internal( + common_options=self.common_options(), + problem=problem, + x0=x0, + optimizer=optimizer, + ) + return res + + +def _gfo_internal( + common_options: GFOCommonOptions, + problem: InternalOptimizationProblem, + x0: NDArray[np.float64], + optimizer: BaseOptimizer, +) -> InternalOptimizeResult: + """Internal helper function. + + Define the search space and inital params, define the objective function and run + optimization. + + """ + # Use common options from GFOCommonOptions + common = common_options + + # set early stopping criterion + early_stopping = { + "n_iter_no_change": common.convergence_iter_noimprove, + "tol_abs": common.convergence_ftol_abs, + "tol_rel": common.convergence_ftol_rel, + } + + # define search space, initial params, population, constraints + opt = optimizer( + search_space=_get_search_space_gfo( + problem.bounds, common.n_grid_points, problem.converter + ), + initialize=_get_initialize_gfo( + x0, common.n_init, common.warm_start, problem.converter + ), + constraints=_get_gfo_constraints(), + random_state=common.seed, + ) + + # define objective function, negate to perform minimize + def objective_function(para: dict[str, float]) -> float | NDArray[np.float64]: + x = np.array(opt.conv.para2value(para)) + return -problem.fun(x) + + # negate in case of minimize + stopping_funval = ( + -1 * common.stopping_funval if common.stopping_funval is not None else None + ) + + # run optimization + opt.search( + objective_function=objective_function, + n_iter=common.stopping_maxiter, + max_time=common.stopping_maxtime, + max_score=stopping_funval, + early_stopping=early_stopping, + memory=common.caching, + memory_warm_start=None, + verbosity=common.verbosity, + ) + + return _process_result_gfo(opt) + + +def _get_search_space_gfo( + bounds: InternalBounds, n_grid_points: PositiveInt | PyTree, converter: Converter +) -> dict[str, NDArray[np.float64]]: + """Create search space. + + Args: + bounds: Internal Bounds + n_grid_points: number of grid points in each dimension + Returns: + dict: search_space dictionary + + """ + search_space = {} + if bounds.lower is not None and bounds.upper is not None: + dim = len(bounds.lower) + upper = bounds.upper + lower = bounds.lower + + if isinstance(n_grid_points, int): + n_grid_points = [n_grid_points] * dim + else: + n_grid_points = converter.params_to_internal(n_grid_points) + + for i in range(dim): + step = (upper[i] - lower[i]) / n_grid_points[i] + search_space[f"x{i}"] = np.arange(lower[i], upper[i], step) + + return search_space + + +def _process_result_gfo(opt: "BaseOptimizer") -> InternalOptimizeResult: + """Process result. + + Args: + opt: Optimizer instance after optimization run is complete + + Returns: + InternalOptimizeResult: Internal optimization result. + + """ + res = InternalOptimizeResult( + x=np.array(opt.best_value), + fun=-opt.best_score, # negate once again + success=True, + n_fun_evals=len(opt.eval_times), + n_jac_evals=0, + n_hess_evals=0, + n_iterations=opt.n_iter_search, + ) + + return res + + +def _get_gfo_constraints() -> list[Any]: + """Process constraints.""" + return [] + + +def _get_initialize_gfo( + x0: NDArray[np.float64], + n_init: PositiveInt, + warm_start: list[PyTree] | None, + converter: Converter, +) -> dict[str, Any]: + """Set initial params x0, additional start points for the optimization run or the + initial_population. + + Args: + x0: initial param + + Returns: + dict: initialize dictionary with initial parameters set + + """ + init = _value2para(x0) + x_list = [init] + if warm_start is not None: + internal_values = [converter.params_to_internal(x) for x in warm_start] + warm_start = [_value2para(x) for x in internal_values] + x_list += warm_start + initialize = {"warm_start": x_list, "vertices": n_init} + return initialize + + +def _value2para(x: NDArray[np.float64]) -> dict[str, float]: + """Convert values to dict. + + Args: + x: Array of parameter values + + Returns: + dict: Dictionary of parameter values with key-value pair as { x{i} : x[i]} + + """ + para = {} + for i in range(len(x)): + para[f"x{i}"] = x[i] + return para diff --git a/tests/optimagic/optimization/test_internal_optimization_problem.py b/tests/optimagic/optimization/test_internal_optimization_problem.py index 3d37149b9..0a8f7bc72 100644 --- a/tests/optimagic/optimization/test_internal_optimization_problem.py +++ b/tests/optimagic/optimization/test_internal_optimization_problem.py @@ -17,6 +17,7 @@ InternalBounds, InternalOptimizationProblem, SphereExampleInternalOptimizationProblem, + SphereExampleInternalOptimizationProblemWithConverter, ) from optimagic.parameters.conversion import Converter from optimagic.typing import AggregationLevel, Direction, ErrorHandling, EvalTask @@ -721,3 +722,12 @@ def test_sphere_example_internal_optimization_problem(): f, j = problem.fun_and_jac(np.array([1, 2, 3])) assert f == 14 aaae(j, np.array([2, 4, 6])) + + +def test_sphere_example_internal_optimization_problem_with_converter(): + problem = SphereExampleInternalOptimizationProblemWithConverter() + assert problem.fun(np.array([1, 2, 3])) == 14 + aaae(problem.jac(np.array([1, 2, 3])), np.array([2, 4, 6])) + f, j = problem.fun_and_jac(np.array([1, 2, 3])) + assert f == 14 + aaae(j, np.array([2, 4, 6])) diff --git a/tests/optimagic/optimization/test_many_algorithms.py b/tests/optimagic/optimization/test_many_algorithms.py index d082783af..43709f383 100644 --- a/tests/optimagic/optimization/test_many_algorithms.py +++ b/tests/optimagic/optimization/test_many_algorithms.py @@ -13,62 +13,39 @@ from optimagic import mark from optimagic.algorithms import AVAILABLE_ALGORITHMS, GLOBAL_ALGORITHMS -from optimagic.optimization.algorithm import Algorithm from optimagic.optimization.optimize import minimize from optimagic.parameters.bounds import Bounds -AVAILABLE_LOCAL_ALGORITHMS = { - name: algo +AVAILABLE_LOCAL_ALGORITHMS = [ + name for name, algo in AVAILABLE_ALGORITHMS.items() if name not in GLOBAL_ALGORITHMS and name != "bhhh" -} +] -AVAILABLE_GLOBAL_ALGORITHMS = { - name: algo +AVAILABLE_BOUNDED_ALGORITHMS = [ + name for name, algo in AVAILABLE_ALGORITHMS.items() - if name in GLOBAL_ALGORITHMS -} - -AVAILABLE_BOUNDED_ALGORITHMS = { - name: algo - for name, algo in AVAILABLE_LOCAL_ALGORITHMS.items() if algo.algo_info.supports_bounds -} - - -def _is_stochastic(algo: Algorithm) -> bool: - return hasattr(algo, "seed") +] +PRECISION_LOOKUP = {"scipy_trust_constr": 3} -LOCAL_STOCHASTIC_ALGORITHMS = [ - name for name, algo in AVAILABLE_LOCAL_ALGORITHMS.items() if _is_stochastic(algo) -] -LOCAL_DETERMINISTIC_ALGORITHMS = [ - name - for name, algo in AVAILABLE_LOCAL_ALGORITHMS.items() - if not _is_stochastic(algo) -] +@pytest.fixture +def algo(algorithm): + return AVAILABLE_ALGORITHMS[algorithm] -GLOBAL_STOCHASTIC_ALGORITHMS = [ - name for name, algo in AVAILABLE_GLOBAL_ALGORITHMS.items() if _is_stochastic(algo) -] -GLOBAL_DETERMINISTIC_ALGORITHMS = [ - name - for name, algo in AVAILABLE_GLOBAL_ALGORITHMS.items() - if not _is_stochastic(algo) -] +def _get_seed(algo): + "Fix seed if algorithm is stochastic" + return {"seed": 12345} if hasattr(algo, "seed") else {} -BOUNDED_STOCHASTIC_ALGORITHMS = [ - name for name, algo in AVAILABLE_BOUNDED_ALGORITHMS.items() if _is_stochastic(algo) -] -BOUNDED_DETERMINISTIC_ALGORITHMS = [ - name - for name, algo in AVAILABLE_BOUNDED_ALGORITHMS.items() - if not _is_stochastic(algo) -] +def _get_required_decimals(algorithm, algo): + if algorithm in PRECISION_LOOKUP: + return PRECISION_LOOKUP[algorithm] + else: + return 1 if algo.algo_info.is_global else 4 @mark.least_squares @@ -76,66 +53,77 @@ def sos(x): return x -@pytest.mark.parametrize("algorithm", LOCAL_DETERMINISTIC_ALGORITHMS) -def test_deterministic_algorithm_on_sum_of_squares(algorithm): - res = minimize( - fun=sos, - params=np.arange(3), - algorithm=algorithm, - collect_history=True, - skip_checks=True, - ) - assert res.success in [True, None] - aaae(res.params, np.zeros(3), decimal=4) +def _get_params_and_binding_bounds(algo): + params = np.array([3, 2, -3]) + if algo.algo_info.supports_infinite_bounds: + bounds = Bounds( + lower=np.array([1, -np.inf, -np.inf]), upper=np.array([np.inf, np.inf, -1]) + ) + else: + bounds = Bounds(lower=np.array([1, -10, -10]), upper=np.array([10, 10, -1])) + expected = np.array([1, 0, -1]) + return params, bounds, expected + +# Tests all bounded algorithms with binding bounds +@pytest.mark.parametrize("algorithm", AVAILABLE_BOUNDED_ALGORITHMS) +def test_sum_of_squares_with_binding_bounds(algorithm, algo): + params, bounds, expected = _get_params_and_binding_bounds(algo) + algo_options = _get_seed(algo) + decimal = _get_required_decimals(algorithm, algo) -@pytest.mark.parametrize("algorithm", LOCAL_STOCHASTIC_ALGORITHMS) -def test_stochastic_algorithm_on_sum_of_squares(algorithm): res = minimize( fun=sos, - params=np.arange(3), + params=params, + bounds=bounds, algorithm=algorithm, collect_history=True, + algo_options=algo_options, skip_checks=True, - algo_options={"seed": 12345}, ) assert res.success in [True, None] - aaae(res.params, np.zeros(3), decimal=4) + aaae(res.params, expected, decimal) -@pytest.mark.parametrize("algorithm", BOUNDED_DETERMINISTIC_ALGORITHMS) -def test_deterministic_algorithm_on_sum_of_squares_with_binding_bounds(algorithm): - res = minimize( - fun=sos, - params=np.array([3, 2, -3]), - bounds=Bounds( - lower=np.array([1, -np.inf, -np.inf]), upper=np.array([np.inf, np.inf, -1]) - ), - algorithm=algorithm, - collect_history=True, - skip_checks=True, - ) - assert res.success in [True, None] - decimal = 3 - aaae(res.params, np.array([1, 0, -1]), decimal=decimal) +def _get_params_and_bounds_on_local(algo): + params = np.arange(3) + bounds = None + expected = np.zeros(3) + if algo.algo_info.needs_bounds: + bounds = Bounds(lower=np.full(3, -10), upper=np.full(3, 10)) + return params, bounds, expected + +# Test all local algorithms without bounds unless needed +@pytest.mark.parametrize("algorithm", AVAILABLE_LOCAL_ALGORITHMS) +def test_sum_of_squares_on_local_algorithms(algorithm, algo): + params, bounds, expected = _get_params_and_bounds_on_local(algo) + algo_options = _get_seed(algo) + decimal = _get_required_decimals(algorithm, algo) -@pytest.mark.parametrize("algorithm", BOUNDED_STOCHASTIC_ALGORITHMS) -def test_stochastic_algorithm_on_sum_of_squares_with_binding_bounds(algorithm): res = minimize( fun=sos, - params=np.array([3, 2, -3]), - bounds=Bounds( - lower=np.array([1, -np.inf, -np.inf]), upper=np.array([np.inf, np.inf, -1]) - ), + params=params, + bounds=bounds, algorithm=algorithm, collect_history=True, + algo_options=algo_options, skip_checks=True, - algo_options={"seed": 12345}, ) assert res.success in [True, None] - decimal = 3 - aaae(res.params, np.array([1, 0, -1]), decimal=decimal) + aaae(res.params, expected, decimal) + + +def _get_params_and_bounds_on_global_and_bounded(algo): + if algo.algo_info.is_global: + params = np.array([0.35, 0.35]) + bounds = Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])) + expected = np.array([0.2, 0]) + else: + params = np.arange(3) + bounds = Bounds(lower=np.full(3, -10), upper=np.full(3, 10)) + expected = np.zeros(3) + return params, bounds, expected skip_msg = ( @@ -144,44 +132,22 @@ def test_stochastic_algorithm_on_sum_of_squares_with_binding_bounds(algorithm): ) +# Test all global algorithms and local algorithms with bounds @pytest.mark.skipif(sys.platform == "win32", reason=skip_msg) -@pytest.mark.parametrize("algorithm", GLOBAL_DETERMINISTIC_ALGORITHMS) -def test_deterministic_global_algorithm_on_sum_of_squares(algorithm): - res = minimize( - fun=sos, - params=np.array([0.35, 0.35]), - bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), - algorithm=algorithm, - collect_history=False, - skip_checks=True, - ) - assert res.success in [True, None] - aaae(res.params, np.array([0.2, 0]), decimal=1) - +@pytest.mark.parametrize("algorithm", AVAILABLE_BOUNDED_ALGORITHMS) +def test_sum_of_squares_on_global_and_bounded_algorithms(algorithm, algo): + params, bounds, expected = _get_params_and_bounds_on_global_and_bounded(algo) + algo_options = _get_seed(algo) + decimal = _get_required_decimals(algorithm, algo) -@pytest.mark.skipif(sys.platform == "win32", reason=skip_msg) -@pytest.mark.parametrize("algorithm", GLOBAL_STOCHASTIC_ALGORITHMS) -def test_stochastic_global_algorithm_on_sum_of_squares(algorithm): res = minimize( fun=sos, - params=np.array([0.35, 0.35]), - bounds=Bounds(lower=np.array([0.2, -0.5]), upper=np.array([1, 0.5])), + params=params, + bounds=bounds, algorithm=algorithm, - collect_history=False, + collect_history=True, + algo_options=algo_options, skip_checks=True, - algo_options={"seed": 12345}, ) assert res.success in [True, None] - aaae(res.params, np.array([0.2, 0]), decimal=1) - - -def test_nag_dfols_starting_at_optimum(): - # From issue: https://github.com/optimagic-dev/optimagic/issues/538 - params = np.zeros(2, dtype=float) - res = minimize( - fun=sos, - params=params, - algorithm="nag_dfols", - bounds=Bounds(-1 * np.ones_like(params), np.ones_like(params)), - ) - aaae(res.params, params) + aaae(res.params, expected, decimal) diff --git a/tests/optimagic/optimizers/test_gfo_optimizers.py b/tests/optimagic/optimizers/test_gfo_optimizers.py new file mode 100644 index 000000000..b05ceae17 --- /dev/null +++ b/tests/optimagic/optimizers/test_gfo_optimizers.py @@ -0,0 +1,77 @@ +import numpy as np +import pytest + +from optimagic.config import IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED +from optimagic.optimization.internal_optimization_problem import ( + SphereExampleInternalOptimizationProblemWithConverter, +) +from optimagic.optimizers.gfo_optimizers import ( + GFOCommonOptions, + _get_gfo_constraints, + _get_initialize_gfo, + _get_search_space_gfo, + _gfo_internal, + _value2para, +) +from optimagic.parameters.bounds import Bounds + +problem = SphereExampleInternalOptimizationProblemWithConverter() + + +def test_get_gfo_constraints(): + got = _get_gfo_constraints() + expected = [] + assert got == expected + + +def test_get_initialize_gfo(): + x0 = np.array([1, 0, 1]) + x1 = [ + {"x0": 1, "x1": 2, "x2": 3}, + ] + n_init = 20 + got = _get_initialize_gfo(x0, n_init, x1, problem.converter) + expected = { + "warm_start": [ + {"x0": 1, "x1": 0, "x2": 1}, # x0 + {"x0": 1, "x1": 2, "x2": 3}, + ], # x1 + "vertices": n_init, + } + assert got == expected + + +def test_get_search_space_gfo(): + bounds = Bounds(lower=np.array([-10, -10]), upper=np.array([10, 10])) + n_grid_points = { + "x0": 4, + "x1": 4, + } + got = _get_search_space_gfo(bounds, n_grid_points, problem.converter) + expected = { + "x0": np.array([-10.0, -5.0, 0.0, 5.0]), + "x1": np.array([-10.0, -5.0, 0.0, 5.0]), + } + assert len(got.keys()) == 2 + assert np.all(got["x0"] == expected["x0"]) + assert np.all(got["x1"] == expected["x1"]) + + +def test_value2para(): + assert _value2para(np.array([0, 1, 2])) == {"x0": 0, "x1": 1, "x2": 2} + + +@pytest.mark.skipif( + not IS_GRADIENT_FREE_OPTIMIZERS_INSTALLED, reason="gfo not installed" +) +def test_gfo_internal(): + from gradient_free_optimizers import DownhillSimplexOptimizer + + res = _gfo_internal( + common_options=GFOCommonOptions(), + problem=problem, + x0=np.full(10, 2), + optimizer=DownhillSimplexOptimizer, + ) + + assert np.all(res.x == np.full(10, 0)) diff --git a/tests/optimagic/optimizers/test_nag_optimizers.py b/tests/optimagic/optimizers/test_nag_optimizers.py index b01f06123..f12850f2b 100644 --- a/tests/optimagic/optimizers/test_nag_optimizers.py +++ b/tests/optimagic/optimizers/test_nag_optimizers.py @@ -1,10 +1,15 @@ +import numpy as np import pytest +from optimagic import mark +from optimagic.optimization.optimize import minimize from optimagic.optimizers.nag_optimizers import ( _build_options_dict, _change_evals_per_point_interface, _get_fast_start_method, ) +from optimagic.parameters.bounds import Bounds +from tests.estimagic.test_bootstrap import aaae def test_change_evals_per_point_interface_none(): @@ -67,3 +72,20 @@ def test_build_options_dict_invalid_key(): user_input = {"other_key": 0} with pytest.raises(ValueError): _build_options_dict(user_input, default) + + +@mark.least_squares +def sos(x): + return x + + +def test_nag_dfols_starting_at_optimum(): + # From issue: https://github.com/optimagic-dev/optimagic/issues/538 + params = np.zeros(2, dtype=float) + res = minimize( + fun=sos, + params=params, + algorithm="nag_dfols", + bounds=Bounds(-1 * np.ones_like(params), np.ones_like(params)), + ) + aaae(res.params, params) diff --git a/tests/optimagic/optimization/test_pygmo_optimizers.py b/tests/optimagic/optimizers/test_pygmo_optimizers.py similarity index 100% rename from tests/optimagic/optimization/test_pygmo_optimizers.py rename to tests/optimagic/optimizers/test_pygmo_optimizers.py