Fix v1.3.2 bug keyboard interrupt bug fix

.circleci/config.yml

@@ -273,7 +273,7 @@ jobs:
   # Misc
   coverage:
     docker:
-      - image: circleci/python:3.6
+      - image: circleci/python:3.7
     steps:
       - checkout
       - run: sudo pip install nox --progress-bar off
@@ -288,19 +288,19 @@ workflows:
       - test_macos:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+              py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
       - test_linux:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+              py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
       - test_win:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+              py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
       - test_linux_omc_dev:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+              py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
 
   plugin_tests:
@@ -309,17 +309,17 @@ workflows:
       - test_plugin_linux:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+              py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
               test_plugin: [<< pipeline.parameters.test_plugins >>]
       - test_plugin_macos:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+              py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
               test_plugin: [<< pipeline.parameters.test_plugins >>]
       - test_plugin_win:
           matrix:
             parameters:
-              py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+              py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
               test_plugin: [<< pipeline.parameters.test_plugins >>]
 
 

NEWS.md

@@ -1,3 +1,24 @@
+1.3.2 (2023-02-22)
+==================
+
+### Features
+
+- Add a `hydra.utils.get_object` function that gives users access to Hydra's dotpath-lookup machinery. ([#2139](https://github.com/facebookresearch/hydra/issues/2139))
+- Allow config_path to specify a non-relative module path, by starting with `pkg://` ([#2564](https://github.com/facebookresearch/hydra/issues/2564))
+
+### Maintenance Changes
+
+- Drop support for python3.6 ([#2304](https://github.com/facebookresearch/hydra/issues/2304))
+
+
+1.3.1 (2022-12-20)
+==================
+
+### Bug Fixes
+
+- Relax OmegaConf pin allowing OmegaConf 2.3 to be installed ([#2510](https://github.com/facebookresearch/hydra/issues/2510))
+
+
 1.3.0 (2022-12-08)
 ==================
 

README.md

@@ -46,10 +46,11 @@
 
 #### Stable
 
-**Hydra 1.2** is the stable version of Hydra.
-- [Documentation](https://hydra.cc/docs/intro)
+**Hydra 1.3** is the stable version of Hydra.
+- [Documentation](https://hydra.cc/docs/1.3/intro/)
 - Installation : `pip install hydra-core --upgrade`
 
+See the [NEWS.md](NEWS.md) file for a summary of recent changes to Hydra.
 
 ### License
 Hydra is licensed under [MIT License](LICENSE).
@@ -60,12 +61,15 @@ Hydra is licensed under [MIT License](LICENSE).
 * [hydra-zen](https://github.com/mit-ll-responsible-ai/hydra-zen): Pythonic utilities for working with Hydra. Dynamic config generation capabilities, enhanced config store features, a Python API for launching Hydra jobs, and more.
 * [lightning-hydra-template](https://github.com/ashleve/lightning-hydra-template): user-friendly template combining Hydra with [Pytorch-Lightning](https://github.com/Lightning-AI/lightning) for ML experimentation.
 * [hydra-torch](https://github.com/pytorch/hydra-torch): [configen](https://github.com/facebookresearch/hydra/tree/main/tools/configen)-generated configuration classes enabling type-safe PyTorch configuration for Hydra apps.
+* NVIDIA's DeepLearningExamples repository contains a Hydra Launcher plugin, the [distributed_launcher](https://github.com/NVIDIA/DeepLearningExamples/tree/9c34e35c218514b8607d7cf381d8a982a01175e9/Tools/PyTorch/TimeSeriesPredictionPlatform/distributed_launcher), which makes use of the pytorch [distributed.launch](https://pytorch.org/docs/stable/distributed.html#launch-utility) API.
 
 #### Ask questions in Github Discussions or StackOverflow (Use the tag #fb-hydra or #omegaconf):
 * [Github Discussions](https://github.com/facebookresearch/hydra/discussions)
 * [StackOverflow](https://stackexchange.com/filters/391828/hydra-questions)
 * [Twitter](https://twitter.com/Hydra_Framework)
 
+Check out the Meta AI [blog post](https://ai.facebook.com/blog/reengineering-facebook-ais-deep-learning-platforms-for-interoperability/) to learn about how Hydra fits into Meta's efforts to reengineer deep learning platforms for interoperability.
+
 ### Citing Hydra
 If you use Hydra in your research please use the following BibTeX entry:
 ```BibTeX

examples/advanced/config_search_path/conf/config.yaml

@@ -4,7 +4,7 @@ defaults:
 
 hydra:
   searchpath:
-    - pkg://additonal_conf
+    - pkg://additional_conf
     # You can also use file based schema, for example:
     # - file:///etc/my_app
     # - file://${oc.env:HOME}/.my_app

examples/advanced/hydra_app_example/tests/test_installed_app.py

@@ -8,7 +8,6 @@
 # when installed and executed from the command line.
 # You don't need to repeat those tests in your own app.
 class TestAppOutput:
-
     # Testing the Hydra app as an executable script
     # This only works if we are at the root of the hydra-app-example
     def test_python_run(self, tmpdir: str) -> None:

examples/plugins/example_configsource_plugin/hydra_plugins/example_configsource_plugin/example_configsource_plugin.py

@@ -105,7 +105,6 @@ def is_config(self, config_path: str) -> bool:
         return config_path in configs
 
     def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[str]:
-
         groups: Dict[str, List[str]] = {
             "": ["dataset", "level1", "optimizer"],
             "dataset": [],

hydra/__init__.py

@@ -1,7 +1,7 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
 
 # Source of truth for Hydra's version
-__version__ = "1.3.0"
+__version__ = "1.3.2"
 from hydra import utils
 from hydra.errors import MissingConfigException
 from hydra.main import main

hydra/_internal/callbacks.py

@@ -63,3 +63,14 @@ def on_job_end(
             reverse=True,
             **kwargs,
         )
+    ## added helper for keyboard interrupt
+    def safe_invoke_on_interrupt(callbacks: "Callbacks", cfg: DictConfig, job_return: Optional["JobReturn"] = None) -> None:
+        """
+        Safely invoke on_job_end and on_run_end callbacks when KeyboardInterrupt occurs.
+        Does not raise further exceptions.
+        """
+        try:
+            callbacks.on_job_end(config=cfg, job_return=job_return)
+            callbacks.on_run_end(config=cfg, job_return=job_return)
+        except Exception as e:
+            warnings.warn(f"Interrupt callbacks failed: {e}")

hydra/_internal/config_repository.py

@@ -58,7 +58,6 @@ def initialize_sources(self, config_search_path: ConfigSearchPath) -> None:
 
 
 class ConfigRepository(IConfigRepository):
-
     config_search_path: ConfigSearchPath
     sources: List[ConfigSource]
 

hydra/_internal/core_plugins/basic_sweeper.py

@@ -107,7 +107,6 @@ def split_overrides_to_chunks(
     def split_arguments(
         overrides: List[Override], max_batch_size: Optional[int]
     ) -> List[List[List[str]]]:
-
         lists = []
         final_overrides = OrderedDict()
         for override in overrides:

hydra/_internal/defaults_list.py

@@ -710,7 +710,6 @@ def _create_defaults_list(
     prepend_hydra: bool,
     skip_missing: bool,
 ) -> Tuple[List[ResultDefault], DefaultsTreeNode]:
-
     root = _create_root(config_name=config_name, with_hydra=prepend_hydra)
 
     defaults_tree = _create_defaults_tree(

hydra/_internal/hydra.py

@@ -620,7 +620,6 @@ def _print_all_info(
         overrides: List[str],
         run_mode: RunMode = RunMode.RUN,
     ) -> None:
-
         from .. import __version__
 
         self._log_header(f"Hydra {__version__}", filler="=")

hydra/_internal/utils.py

@@ -37,7 +37,6 @@ def _get_module_name_override() -> Optional[str]:
 def detect_calling_file_or_module_from_task_function(
     task_function: Any,
 ) -> Tuple[Optional[str], Optional[str]]:
-
     # if function is decorated, unwrap it
     while hasattr(task_function, "__wrapped__"):
         task_function = task_function.__wrapped__
@@ -65,7 +64,6 @@ def detect_calling_file_or_module_from_task_function(
 def detect_calling_file_or_module_from_stack_frame(
     stack_depth: int,
 ) -> Tuple[Optional[str], Optional[str]]:
-
     stack = inspect.stack()
     frame = stack[stack_depth]
     if is_notebook() and "_dh" in frame[0].f_globals:
@@ -102,7 +100,6 @@ def is_notebook() -> bool:
 
 
 def detect_task_name(calling_file: Optional[str], calling_module: Optional[str]) -> str:
-
     if calling_file is not None:
         target_file = os.path.basename(calling_file)
         task_name = get_valid_filename(os.path.splitext(target_file)[0])
@@ -123,9 +120,13 @@ def compute_search_path_dir(
     calling_module: Optional[str],
     config_path: Optional[str],
 ) -> Optional[str]:
-    if config_path is not None and os.path.isabs(config_path):
-        search_path_dir = config_path
-    elif calling_file is not None:
+    if config_path is not None:
+        if os.path.isabs(config_path):
+            return config_path
+        if config_path.startswith("pkg://"):
+            return config_path
+
+    if calling_file is not None:
         abs_base_dir = realpath(dirname(calling_file))
 
         if config_path is not None:
@@ -310,7 +311,6 @@ def _run_hydra(
     config_name: Optional[str],
     caller_stack_depth: int = 2,
 ) -> None:
-
     from hydra.core.global_hydra import GlobalHydra
 
     from .hydra import Hydra

hydra/compose.py

@@ -13,7 +13,7 @@
 
 def compose(
     config_name: Optional[str] = None,
-    overrides: List[str] = [],
+    overrides: Optional[List[str]] = None,
     return_hydra_config: bool = False,
     strict: Optional[bool] = None,
 ) -> DictConfig:
@@ -25,6 +25,10 @@ def compose(
     :param strict: DEPRECATED. If false, returned config has struct mode disabled.
     :return: the composed config
     """
+
+    if overrides is None:
+        overrides = []
+
     assert (
         GlobalHydra().is_initialized()
     ), "GlobalHydra is not initialized, use @hydra.main() or call one of the hydra initialization methods first"

hydra/core/override_parser/types.py

@@ -291,7 +291,6 @@ def _convert_value(value: ParsedElementType) -> Optional[ElementType]:
         if isinstance(value, list):
             return [Override._convert_value(x) for x in value]
         elif isinstance(value, dict):
-
             return {
                 # We ignore potential type mismatch here so as to let OmegaConf
                 # raise an explicit error in case of invalid type.

hydra/core/plugins.py

@@ -121,7 +121,6 @@ def _instantiate(self, config: DictConfig) -> Plugin:
 
     @staticmethod
     def is_in_toplevel_plugins_module(clazz: str) -> bool:
-
         return clazz.startswith("hydra_plugins.") or clazz.startswith(
             "hydra._internal.core_plugins."
         )
@@ -164,7 +163,6 @@ def instantiate_launcher(
     def _scan_all_plugins(
         modules: List[Any],
     ) -> Tuple[List[Type[Plugin]], ScanStats]:
-
         stats = ScanStats()
         stats.total_time = timer()
 

hydra/core/utils.py

@@ -124,7 +124,7 @@ def run_job(
     with read_write(config.hydra.runtime):
         with open_dict(config.hydra.runtime):
             config.hydra.runtime.output_dir = os.path.abspath(output_dir)
-
+    
     # update Hydra config
     HydraConfig.instance().set_config(config)
     _chdir = None
@@ -143,10 +143,10 @@ def run_job(
         assert isinstance(overrides, list)
         ret.overrides = overrides
         # handle output directories here
+
         Path(str(output_dir)).mkdir(parents=True, exist_ok=True)
 
         _chdir = hydra_cfg.hydra.job.chdir
-
         if _chdir is None:
             if version.base_at_least("1.2"):
                 _chdir = False
@@ -185,16 +185,23 @@ def run_job(
             try:
                 ret.return_value = task_function(task_cfg)
                 ret.status = JobStatus.COMPLETED
+           ## fix for keyboard interrupt
+            except KeyboardInterrupt:
+                import warnings
+                from hydra._internal.callbacks import safe_invoke_on_interrupt
+
+                ret.return_value = KeyboardInterrupt("Job interrupted by user (Ctrl+C)")
+                ret.status = JobStatus.FAILED
+                safe_invoke_on_interrupt(callbacks, config, ret)
+                raise
+
             except Exception as e:
                 ret.return_value = e
                 ret.status = JobStatus.FAILED
 
         ret.task_name = JobRuntime.instance().get("name")
-
         _flush_loggers()
-
         callbacks.on_job_end(config=config, job_return=ret)
-
         return ret
     finally:
         HydraConfig.instance().cfg = orig_hydra_cfg
@@ -318,4 +325,4 @@ def _flush_loggers() -> None:
             h_weak_ref().flush()
         except Exception:
             # ignore exceptions thrown during flushing
-            pass
+            pass

hydra/errors.py

@@ -33,7 +33,7 @@ class MissingConfigException(IOError, ConfigCompositionException):
     def __init__(
         self,
         message: str,
-        missing_cfg_file: Optional[str],
+        missing_cfg_file: Optional[str] = None,
         options: Optional[Sequence[str]] = None,
     ) -> None:
         super(MissingConfigException, self).__init__(message)

hydra/experimental/callbacks.py

@@ -12,6 +12,8 @@
 
 
 class LogJobReturnCallback(Callback):
+    """Log the job's return value or error upon job end"""
+
     def __init__(self) -> None:
         self.log = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
 
@@ -27,12 +29,15 @@ def on_job_end(
 
 
 class PickleJobInfoCallback(Callback):
+    """Pickle the job config/return-value in ${output_dir}/{config,job_return}.pickle"""
+
     output_dir: Path
 
     def __init__(self) -> None:
         self.log = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
 
     def on_job_start(self, config: DictConfig, **kwargs: Any) -> None:
+        """Pickle the job's config in ${output_dir}/config.pickle."""
         self.output_dir = Path(config.hydra.runtime.output_dir) / Path(
             config.hydra.output_subdir
         )
@@ -43,6 +48,7 @@ def on_job_start(self, config: DictConfig, **kwargs: Any) -> None:
     def on_job_end(
         self, config: DictConfig, job_return: JobReturn, **kwargs: Any
     ) -> None:
+        """Pickle the job's return value in ${output_dir}/job_return.pickle."""
         filename = "job_return.pickle"
         self._save_pickle(obj=job_return, filename=filename, output_dir=self.output_dir)
         self.log.info(f"Saving job_return in {self.output_dir / filename}")