ENH remove safe_import_context (#46)

Author: tomMoral

benchmark_utils/__init__.py

@@ -1,13 +1,8 @@
 # `benchmark_utils` is a module in which you can define code to reuse in
 # the benchmark objective, datasets, and solvers. The folder should have the
 # name `benchmark_utils`, and code defined inside will be importable using
-# the usual import syntax. To import external packages in this file, use a
-# `safe_import_context` named "import_ctx", as follows:
-
-from benchopt.utils import safe_import_context
-
-with safe_import_context() as import_ctx:
-    import numpy as np
+# the usual import syntax.
+import numpy as np
 
 
 def gradient_ols(X, y, beta):

datasets/simulated.py

@@ -1,11 +1,6 @@
-from benchopt import BaseDataset, safe_import_context
+from benchopt import BaseDataset
 
-
-# Protect the import with `safe_import_context()`. This allows:
-# - skipping import to speed up autocompletion in CLI.
-# - getting requirements info when all dependencies are not installed.
-with safe_import_context() as import_ctx:
-    import numpy as np
+import numpy as np
 
 
 # All datasets must be named `Dataset` and inherit from `BaseDataset`
@@ -26,7 +21,7 @@ class Dataset(BaseDataset):
     }
 
     # List of packages needed to run the dataset. See the corresponding
-    # section in objective.py
+    # section in objective.py. This is an optional attribute.
     requirements = []
 
     def get_data(self):

objective.py

@@ -1,10 +1,7 @@
-from benchopt import BaseObjective, safe_import_context
+from benchopt import BaseObjective
 
-# Protect the import with `safe_import_context()`. This allows:
-# - skipping import to speed up autocompletion in CLI.
-# - getting requirements info when all dependencies are not installed.
-with safe_import_context() as import_ctx:
-    import numpy as np
+import numpy as np
+from benchmark_utils import value_ols, gradient_ols
 
 
 # The benchmark objective must be named `Objective` and
@@ -17,49 +14,42 @@ class Objective(BaseObjective):
     # URL of the main repo for this benchmark.
     url = "https://github.com/#ORG/#BENCHMARK_NAME"
 
-    # List of parameters for the objective. The benchmark will consider
-    # the cross product for each key in the dictionary.
-    # All parameters 'p' defined here are available as 'self.p'.
-    # This means the OLS objective will have a parameter `self.whiten_y`.
-    parameters = {
-        'whiten_y': [False, True],
-    }
-
     # List of packages needed to run the benchmark.
     # They are installed with conda; to use pip, use 'pip:packagename'. To
     # install from a specific conda channel, use 'channelname:packagename'.
     # Packages that are not necessary to the whole benchmark but only to some
     # solvers or datasets should be declared in Dataset or Solver (see
     # simulated.py and python-gd.py).
-    # Example syntax: requirements = ['numpy', 'pip:jax', 'pytorch:pytorch']
+    # Example syntax: requirements = ['numpy', 'pip::jax', 'pytorch::pytorch']
     requirements = ["numpy"]
 
     # Minimal version of benchopt required to run this benchmark.
     # Bump it up if the benchmark depends on a new feature of benchopt.
-    min_benchopt_version = "1.5"
+    min_benchopt_version = "1.7"
 
     def set_data(self, X, y):
         # The keyword arguments of this function are the keys of the dictionary
         # returned by `Dataset.get_data`. This defines the benchmark's
         # API to pass data. This is customizable for each benchmark.
         self.X, self.y = X, y
 
-        # `set_data` can be used to preprocess the data. For instance,
-        # if `whiten_y` is True, remove the mean of `y`.
-        if self.whiten_y:
-            y -= y.mean(axis=0)
-
     def evaluate_result(self, beta):
         # The keyword arguments of this function are the keys of the
         # dictionary returned by `Solver.get_result`. This defines the
         # benchmark's API to pass solvers' result. This is customizable for
         # each benchmark.
-        diff = self.y - self.X @ beta
+
+        # Here we can compute any metric to evaluate the quality of the
+        # solution. We compute the value of the objective function and the
+        # norm of the gradient.
+        grad = gradient_ols(self.X, self.y, beta)
+        value = value_ols(self.X, self.y, beta)
 
         # This method can return many metrics in a dictionary. One of these
         # metrics needs to be `value` for convergence detection purposes.
         return dict(
-            value=.5 * diff @ diff,
+            value=value,
+            grad_norm=np.linalg.vector_norm(grad),
         )
 
     def get_one_result(self):

solvers/python-gd.py

@@ -1,13 +1,10 @@
-from benchopt import BaseSolver, safe_import_context
+from benchopt import BaseSolver
 
-# Protect the import with `safe_import_context()`. This allows:
-# - skipping import to speed up autocompletion in CLI.
-# - getting requirements info when all dependencies are not installed.
-with safe_import_context() as import_ctx:
-    import numpy as np
+import numpy as np
 
-    # import your reusable functions here
-    from benchmark_utils import gradient_ols
+# Reusable function can be imported from the benchmark_utils module, which is
+# dynamically installed when running the benchmark.
+from benchmark_utils import gradient_ols
 
 
 # The benchmark solvers must be named `Solver` and
@@ -17,16 +14,27 @@ class Solver(BaseSolver):
     # Name to select the solver in the CLI and to display the results.
     name = 'GD'
 
+    # List of packages needed to run the solver. See the corresponding
+    # section in objective.py. This is an optional attribute.
+    requirements = []
+
     # List of parameters for the solver. The benchmark will consider
     # the cross product for each key in the dictionary.
     # All parameters 'p' defined here are available as 'self.p'.
     parameters = {
-        'scale_step': [1, 1.99],
+        'learning_rate': [0.1, 0.5],
     }
 
-    # List of packages needed to run the solver. See the corresponding
-    # section in objective.py
-    requirements = []
+    # Evaluation strategy for the performance curve.
+    # It describe when how and when the solver will be evaluated.
+    # You can also use `iteration`, `tolerance` or `run_once`, as described in
+    # https://benchopt.github.io/performance_curves.html
+    # For optimization solvers, we recommend to use 'callback' which can be
+    # used regularly to log the progress of the solver and implement a stopping
+    # criterion.
+    # For machine learning methods, `run_once` is usually more adapted, to
+    # evaluate method only at the end of the training phase.
+    sampling_strategy = 'callback'
 
     def set_objective(self, X, y):
         # Define the information received by each solver from the objective.
@@ -36,22 +44,23 @@ def set_objective(self, X, y):
         # It is customizable for each benchmark.
         self.X, self.y = X, y
 
-    def run(self, n_iter):
-        # This is the function that is called to evaluate the solver.
-        # It runs the algorithm for a given a number of iterations `n_iter`.
-        # You can also use a `tolerance` or a `callback`, as described in
-        # https://benchopt.github.io/performance_curves.html
-
-        L = np.linalg.norm(self.X, ord=2) ** 2
-        step_size = self.scale_step / L
-        beta = np.zeros(self.X.shape[1])
-        for _ in range(n_iter):
-            beta -= step_size * gradient_ols(self.X, self.y, beta)
+    def run(self, callback):
+        # This is the function that is called to run the method.
+        # When using ``sampling_strategy='callback'``, the function is provided
+        # with a ``callback`` function that must be called regularly to
+        # log the progress of the solver. The callback function returns
+        # ``True`` until the solver should stop.
+        # See https://benchopt.github.io/guide/auto_stop.html for more details.
 
-        self.beta = beta
+        self.beta = np.zeros(self.X.shape[1])
+        while callback():
+            self.beta -= self.learning_rate * gradient_ols(
+                self.X, self.y, self.beta
+            )
 
     def get_result(self):
-        # Return the result from one optimization run.
+        # Return the result of the method.
+        #
         # The outputs of this function is a dictionary which defines the
         # keyword arguments for `Objective.evaluate_result`
         # This defines the benchmark's API for solvers' results.