diff --git a/.circleci/config.yml b/.circleci/config.yml index 80007e80ae5..635aa80ffc0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -273,7 +273,7 @@ jobs: # Misc coverage: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 steps: - checkout - run: sudo pip install nox --progress-bar off @@ -288,19 +288,19 @@ workflows: - test_macos: matrix: parameters: - py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - test_linux: matrix: parameters: - py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - test_win: matrix: parameters: - py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - test_linux_omc_dev: matrix: parameters: - py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] plugin_tests: @@ -309,17 +309,17 @@ workflows: - test_plugin_linux: matrix: parameters: - py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] test_plugin: [<< pipeline.parameters.test_plugins >>] - test_plugin_macos: matrix: parameters: - py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] test_plugin: [<< pipeline.parameters.test_plugins >>] - test_plugin_win: matrix: parameters: - py_version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + py_version: ["3.7", "3.8", "3.9", "3.10", "3.11"] test_plugin: [<< pipeline.parameters.test_plugins >>] diff --git a/NEWS.md b/NEWS.md index 844ac4c4d33..3a8bdda2628 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,24 @@ +1.3.2 (2023-02-22) +================== + +### Features + +- Add a `hydra.utils.get_object` function that gives users access to Hydra's dotpath-lookup machinery. ([#2139](https://github.com/facebookresearch/hydra/issues/2139)) +- Allow config_path to specify a non-relative module path, by starting with `pkg://` ([#2564](https://github.com/facebookresearch/hydra/issues/2564)) + +### Maintenance Changes + +- Drop support for python3.6 ([#2304](https://github.com/facebookresearch/hydra/issues/2304)) + + +1.3.1 (2022-12-20) +================== + +### Bug Fixes + +- Relax OmegaConf pin allowing OmegaConf 2.3 to be installed ([#2510](https://github.com/facebookresearch/hydra/issues/2510)) + + 1.3.0 (2022-12-08) ================== diff --git a/README.md b/README.md index 49a357da34d..7b41040881b 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,11 @@ #### Stable -**Hydra 1.2** is the stable version of Hydra. -- [Documentation](https://hydra.cc/docs/intro) +**Hydra 1.3** is the stable version of Hydra. +- [Documentation](https://hydra.cc/docs/1.3/intro/) - Installation : `pip install hydra-core --upgrade` +See the [NEWS.md](NEWS.md) file for a summary of recent changes to Hydra. ### License Hydra is licensed under [MIT License](LICENSE). @@ -60,12 +61,15 @@ Hydra is licensed under [MIT License](LICENSE). * [hydra-zen](https://github.com/mit-ll-responsible-ai/hydra-zen): Pythonic utilities for working with Hydra. Dynamic config generation capabilities, enhanced config store features, a Python API for launching Hydra jobs, and more. * [lightning-hydra-template](https://github.com/ashleve/lightning-hydra-template): user-friendly template combining Hydra with [Pytorch-Lightning](https://github.com/Lightning-AI/lightning) for ML experimentation. * [hydra-torch](https://github.com/pytorch/hydra-torch): [configen](https://github.com/facebookresearch/hydra/tree/main/tools/configen)-generated configuration classes enabling type-safe PyTorch configuration for Hydra apps. +* NVIDIA's DeepLearningExamples repository contains a Hydra Launcher plugin, the [distributed_launcher](https://github.com/NVIDIA/DeepLearningExamples/tree/9c34e35c218514b8607d7cf381d8a982a01175e9/Tools/PyTorch/TimeSeriesPredictionPlatform/distributed_launcher), which makes use of the pytorch [distributed.launch](https://pytorch.org/docs/stable/distributed.html#launch-utility) API. #### Ask questions in Github Discussions or StackOverflow (Use the tag #fb-hydra or #omegaconf): * [Github Discussions](https://github.com/facebookresearch/hydra/discussions) * [StackOverflow](https://stackexchange.com/filters/391828/hydra-questions) * [Twitter](https://twitter.com/Hydra_Framework) +Check out the Meta AI [blog post](https://ai.facebook.com/blog/reengineering-facebook-ais-deep-learning-platforms-for-interoperability/) to learn about how Hydra fits into Meta's efforts to reengineer deep learning platforms for interoperability. + ### Citing Hydra If you use Hydra in your research please use the following BibTeX entry: ```BibTeX diff --git a/examples/advanced/config_search_path/additonal_conf/__init__.py b/examples/advanced/config_search_path/additional_conf/__init__.py similarity index 100% rename from examples/advanced/config_search_path/additonal_conf/__init__.py rename to examples/advanced/config_search_path/additional_conf/__init__.py diff --git a/examples/advanced/config_search_path/additonal_conf/dataset/imagenet.yaml b/examples/advanced/config_search_path/additional_conf/dataset/imagenet.yaml similarity index 100% rename from examples/advanced/config_search_path/additonal_conf/dataset/imagenet.yaml rename to examples/advanced/config_search_path/additional_conf/dataset/imagenet.yaml diff --git a/examples/advanced/config_search_path/conf/config.yaml b/examples/advanced/config_search_path/conf/config.yaml index 37c1bb1bc78..7aaa75417cb 100644 --- a/examples/advanced/config_search_path/conf/config.yaml +++ b/examples/advanced/config_search_path/conf/config.yaml @@ -4,7 +4,7 @@ defaults: hydra: searchpath: - - pkg://additonal_conf + - pkg://additional_conf # You can also use file based schema, for example: # - file:///etc/my_app # - file://${oc.env:HOME}/.my_app diff --git a/examples/advanced/hydra_app_example/tests/test_installed_app.py b/examples/advanced/hydra_app_example/tests/test_installed_app.py index cd3bd6ab6fe..633410099af 100644 --- a/examples/advanced/hydra_app_example/tests/test_installed_app.py +++ b/examples/advanced/hydra_app_example/tests/test_installed_app.py @@ -8,7 +8,6 @@ # when installed and executed from the command line. # You don't need to repeat those tests in your own app. class TestAppOutput: - # Testing the Hydra app as an executable script # This only works if we are at the root of the hydra-app-example def test_python_run(self, tmpdir: str) -> None: diff --git a/examples/plugins/example_configsource_plugin/hydra_plugins/example_configsource_plugin/example_configsource_plugin.py b/examples/plugins/example_configsource_plugin/hydra_plugins/example_configsource_plugin/example_configsource_plugin.py index 347676ab061..69e904cc8d7 100644 --- a/examples/plugins/example_configsource_plugin/hydra_plugins/example_configsource_plugin/example_configsource_plugin.py +++ b/examples/plugins/example_configsource_plugin/hydra_plugins/example_configsource_plugin/example_configsource_plugin.py @@ -105,7 +105,6 @@ def is_config(self, config_path: str) -> bool: return config_path in configs def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[str]: - groups: Dict[str, List[str]] = { "": ["dataset", "level1", "optimizer"], "dataset": [], diff --git a/hydra/__init__.py b/hydra/__init__.py index 07297ca70ef..cb91a833c59 100644 --- a/hydra/__init__.py +++ b/hydra/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved # Source of truth for Hydra's version -__version__ = "1.3.0" +__version__ = "1.3.2" from hydra import utils from hydra.errors import MissingConfigException from hydra.main import main diff --git a/hydra/_internal/callbacks.py b/hydra/_internal/callbacks.py index 36226bc90d4..a6d69d7fbea 100644 --- a/hydra/_internal/callbacks.py +++ b/hydra/_internal/callbacks.py @@ -63,3 +63,14 @@ def on_job_end( reverse=True, **kwargs, ) + ## added helper for keyboard interrupt + def safe_invoke_on_interrupt(callbacks: "Callbacks", cfg: DictConfig, job_return: Optional["JobReturn"] = None) -> None: + """ + Safely invoke on_job_end and on_run_end callbacks when KeyboardInterrupt occurs. + Does not raise further exceptions. + """ + try: + callbacks.on_job_end(config=cfg, job_return=job_return) + callbacks.on_run_end(config=cfg, job_return=job_return) + except Exception as e: + warnings.warn(f"Interrupt callbacks failed: {e}") \ No newline at end of file diff --git a/hydra/_internal/config_repository.py b/hydra/_internal/config_repository.py index 7b3c3138d83..3beb5e43238 100644 --- a/hydra/_internal/config_repository.py +++ b/hydra/_internal/config_repository.py @@ -58,7 +58,6 @@ def initialize_sources(self, config_search_path: ConfigSearchPath) -> None: class ConfigRepository(IConfigRepository): - config_search_path: ConfigSearchPath sources: List[ConfigSource] diff --git a/hydra/_internal/core_plugins/basic_sweeper.py b/hydra/_internal/core_plugins/basic_sweeper.py index d92256f47c8..afa2b506045 100644 --- a/hydra/_internal/core_plugins/basic_sweeper.py +++ b/hydra/_internal/core_plugins/basic_sweeper.py @@ -107,7 +107,6 @@ def split_overrides_to_chunks( def split_arguments( overrides: List[Override], max_batch_size: Optional[int] ) -> List[List[List[str]]]: - lists = [] final_overrides = OrderedDict() for override in overrides: diff --git a/hydra/_internal/defaults_list.py b/hydra/_internal/defaults_list.py index 8ca6e86d54f..b4b0d92c950 100644 --- a/hydra/_internal/defaults_list.py +++ b/hydra/_internal/defaults_list.py @@ -710,7 +710,6 @@ def _create_defaults_list( prepend_hydra: bool, skip_missing: bool, ) -> Tuple[List[ResultDefault], DefaultsTreeNode]: - root = _create_root(config_name=config_name, with_hydra=prepend_hydra) defaults_tree = _create_defaults_tree( diff --git a/hydra/_internal/hydra.py b/hydra/_internal/hydra.py index b9716d54787..eb2d1c7db77 100644 --- a/hydra/_internal/hydra.py +++ b/hydra/_internal/hydra.py @@ -620,7 +620,6 @@ def _print_all_info( overrides: List[str], run_mode: RunMode = RunMode.RUN, ) -> None: - from .. import __version__ self._log_header(f"Hydra {__version__}", filler="=") diff --git a/hydra/_internal/utils.py b/hydra/_internal/utils.py index 3b331f74c56..739033faf4e 100644 --- a/hydra/_internal/utils.py +++ b/hydra/_internal/utils.py @@ -37,7 +37,6 @@ def _get_module_name_override() -> Optional[str]: def detect_calling_file_or_module_from_task_function( task_function: Any, ) -> Tuple[Optional[str], Optional[str]]: - # if function is decorated, unwrap it while hasattr(task_function, "__wrapped__"): task_function = task_function.__wrapped__ @@ -65,7 +64,6 @@ def detect_calling_file_or_module_from_task_function( def detect_calling_file_or_module_from_stack_frame( stack_depth: int, ) -> Tuple[Optional[str], Optional[str]]: - stack = inspect.stack() frame = stack[stack_depth] if is_notebook() and "_dh" in frame[0].f_globals: @@ -102,7 +100,6 @@ def is_notebook() -> bool: def detect_task_name(calling_file: Optional[str], calling_module: Optional[str]) -> str: - if calling_file is not None: target_file = os.path.basename(calling_file) task_name = get_valid_filename(os.path.splitext(target_file)[0]) @@ -123,9 +120,13 @@ def compute_search_path_dir( calling_module: Optional[str], config_path: Optional[str], ) -> Optional[str]: - if config_path is not None and os.path.isabs(config_path): - search_path_dir = config_path - elif calling_file is not None: + if config_path is not None: + if os.path.isabs(config_path): + return config_path + if config_path.startswith("pkg://"): + return config_path + + if calling_file is not None: abs_base_dir = realpath(dirname(calling_file)) if config_path is not None: @@ -310,7 +311,6 @@ def _run_hydra( config_name: Optional[str], caller_stack_depth: int = 2, ) -> None: - from hydra.core.global_hydra import GlobalHydra from .hydra import Hydra diff --git a/hydra/compose.py b/hydra/compose.py index 0ced80fd059..09b150d6e2a 100644 --- a/hydra/compose.py +++ b/hydra/compose.py @@ -13,7 +13,7 @@ def compose( config_name: Optional[str] = None, - overrides: List[str] = [], + overrides: Optional[List[str]] = None, return_hydra_config: bool = False, strict: Optional[bool] = None, ) -> DictConfig: @@ -25,6 +25,10 @@ def compose( :param strict: DEPRECATED. If false, returned config has struct mode disabled. :return: the composed config """ + + if overrides is None: + overrides = [] + assert ( GlobalHydra().is_initialized() ), "GlobalHydra is not initialized, use @hydra.main() or call one of the hydra initialization methods first" diff --git a/hydra/core/override_parser/types.py b/hydra/core/override_parser/types.py index af8071b4cdf..3f41bf516cb 100644 --- a/hydra/core/override_parser/types.py +++ b/hydra/core/override_parser/types.py @@ -291,7 +291,6 @@ def _convert_value(value: ParsedElementType) -> Optional[ElementType]: if isinstance(value, list): return [Override._convert_value(x) for x in value] elif isinstance(value, dict): - return { # We ignore potential type mismatch here so as to let OmegaConf # raise an explicit error in case of invalid type. diff --git a/hydra/core/plugins.py b/hydra/core/plugins.py index be1cdd5b799..e63a35eaebd 100644 --- a/hydra/core/plugins.py +++ b/hydra/core/plugins.py @@ -121,7 +121,6 @@ def _instantiate(self, config: DictConfig) -> Plugin: @staticmethod def is_in_toplevel_plugins_module(clazz: str) -> bool: - return clazz.startswith("hydra_plugins.") or clazz.startswith( "hydra._internal.core_plugins." ) @@ -164,7 +163,6 @@ def instantiate_launcher( def _scan_all_plugins( modules: List[Any], ) -> Tuple[List[Type[Plugin]], ScanStats]: - stats = ScanStats() stats.total_time = timer() diff --git a/hydra/core/utils.py b/hydra/core/utils.py index 43ff3649a76..6cb5a0bcb32 100644 --- a/hydra/core/utils.py +++ b/hydra/core/utils.py @@ -124,7 +124,7 @@ def run_job( with read_write(config.hydra.runtime): with open_dict(config.hydra.runtime): config.hydra.runtime.output_dir = os.path.abspath(output_dir) - + # update Hydra config HydraConfig.instance().set_config(config) _chdir = None @@ -143,10 +143,10 @@ def run_job( assert isinstance(overrides, list) ret.overrides = overrides # handle output directories here + Path(str(output_dir)).mkdir(parents=True, exist_ok=True) _chdir = hydra_cfg.hydra.job.chdir - if _chdir is None: if version.base_at_least("1.2"): _chdir = False @@ -185,16 +185,23 @@ def run_job( try: ret.return_value = task_function(task_cfg) ret.status = JobStatus.COMPLETED + ## fix for keyboard interrupt + except KeyboardInterrupt: + import warnings + from hydra._internal.callbacks import safe_invoke_on_interrupt + + ret.return_value = KeyboardInterrupt("Job interrupted by user (Ctrl+C)") + ret.status = JobStatus.FAILED + safe_invoke_on_interrupt(callbacks, config, ret) + raise + except Exception as e: ret.return_value = e ret.status = JobStatus.FAILED ret.task_name = JobRuntime.instance().get("name") - _flush_loggers() - callbacks.on_job_end(config=config, job_return=ret) - return ret finally: HydraConfig.instance().cfg = orig_hydra_cfg @@ -318,4 +325,4 @@ def _flush_loggers() -> None: h_weak_ref().flush() except Exception: # ignore exceptions thrown during flushing - pass + pass \ No newline at end of file diff --git a/hydra/errors.py b/hydra/errors.py index 16849a2f550..427e8375506 100644 --- a/hydra/errors.py +++ b/hydra/errors.py @@ -33,7 +33,7 @@ class MissingConfigException(IOError, ConfigCompositionException): def __init__( self, message: str, - missing_cfg_file: Optional[str], + missing_cfg_file: Optional[str] = None, options: Optional[Sequence[str]] = None, ) -> None: super(MissingConfigException, self).__init__(message) diff --git a/hydra/experimental/callbacks.py b/hydra/experimental/callbacks.py index 42f60c9c8d6..8001ff551d6 100644 --- a/hydra/experimental/callbacks.py +++ b/hydra/experimental/callbacks.py @@ -12,6 +12,8 @@ class LogJobReturnCallback(Callback): + """Log the job's return value or error upon job end""" + def __init__(self) -> None: self.log = logging.getLogger(f"{__name__}.{self.__class__.__name__}") @@ -27,12 +29,15 @@ def on_job_end( class PickleJobInfoCallback(Callback): + """Pickle the job config/return-value in ${output_dir}/{config,job_return}.pickle""" + output_dir: Path def __init__(self) -> None: self.log = logging.getLogger(f"{__name__}.{self.__class__.__name__}") def on_job_start(self, config: DictConfig, **kwargs: Any) -> None: + """Pickle the job's config in ${output_dir}/config.pickle.""" self.output_dir = Path(config.hydra.runtime.output_dir) / Path( config.hydra.output_subdir ) @@ -43,6 +48,7 @@ def on_job_start(self, config: DictConfig, **kwargs: Any) -> None: def on_job_end( self, config: DictConfig, job_return: JobReturn, **kwargs: Any ) -> None: + """Pickle the job's return value in ${output_dir}/job_return.pickle.""" filename = "job_return.pickle" self._save_pickle(obj=job_return, filename=filename, output_dir=self.output_dir) self.log.info(f"Saving job_return in {self.output_dir / filename}") diff --git a/hydra/extra/pytest_plugin.py b/hydra/extra/pytest_plugin.py index f01d5dbca93..6366d2f8a1f 100644 --- a/hydra/extra/pytest_plugin.py +++ b/hydra/extra/pytest_plugin.py @@ -21,19 +21,21 @@ def hydra_restore_singletons() -> Generator[None, None, None]: @fixture(scope="function") -def hydra_sweep_runner() -> Callable[ - [ - Optional[str], - Optional[str], - Optional[TaskFunction], - Optional[str], - Optional[str], - Optional[List[str]], - Optional[Path], - bool, - ], - SweepTaskFunction, -]: +def hydra_sweep_runner() -> ( + Callable[ + [ + Optional[str], + Optional[str], + Optional[TaskFunction], + Optional[str], + Optional[str], + Optional[List[str]], + Optional[Path], + bool, + ], + SweepTaskFunction, + ] +): def _( calling_file: Optional[str], calling_module: Optional[str], @@ -59,17 +61,19 @@ def _( @fixture(scope="function") -def hydra_task_runner() -> Callable[ - [ - Optional[str], - Optional[str], - Optional[str], - Optional[str], - Optional[List[str]], - bool, - ], - TaskTestFunction, -]: +def hydra_task_runner() -> ( + Callable[ + [ + Optional[str], + Optional[str], + Optional[str], + Optional[str], + Optional[List[str]], + bool, + ], + TaskTestFunction, + ] +): def _( calling_file: Optional[str], calling_module: Optional[str], diff --git a/hydra/main.py b/hydra/main.py index 9b19301d61a..134bec0b570 100644 --- a/hydra/main.py +++ b/hydra/main.py @@ -48,7 +48,11 @@ def main( version_base: Optional[str] = _UNSPECIFIED_, ) -> Callable[[TaskFunction], Any]: """ - :param config_path: The config path, a directory relative to the declaring python file. + :param config_path: The config path, a directory where Hydra will search for + config files. This path is added to Hydra's searchpath. + Relative paths are interpreted relative to the declaring python + file. Alternatively, you can use the prefix `pkg://` to specify + a python package to add to the searchpath. If config_path is None no directory is added to the Config search path. :param config_name: The name of the config (usually the file name without the .yaml extension) """ diff --git a/hydra/plugins/completion_plugin.py b/hydra/plugins/completion_plugin.py index 0f3ab8837f5..328f03623ba 100644 --- a/hydra/plugins/completion_plugin.py +++ b/hydra/plugins/completion_plugin.py @@ -240,7 +240,6 @@ def _query(self, config_name: Optional[str], line: str) -> List[str]: ) config_matches: List[str] = [] if not exact_match: - run_mode = RunMode.MULTIRUN if parsed_args.multirun else RunMode.RUN config_matches = [] try: diff --git a/hydra/test_utils/launcher_common_tests.py b/hydra/test_utils/launcher_common_tests.py index cb13cd988bc..af2346a489d 100644 --- a/hydra/test_utils/launcher_common_tests.py +++ b/hydra/test_utils/launcher_common_tests.py @@ -215,7 +215,6 @@ def test_sweep_with_custom_resolver( overrides: List[str], tmpdir: Path, ) -> None: - overrides1 = ["hydra/launcher=" + launcher_name] + overrides def task_func(c: DictConfig) -> Any: diff --git a/hydra/utils.py b/hydra/utils.py index 33c1ac1dc8d..93d3f5f22f1 100644 --- a/hydra/utils.py +++ b/hydra/utils.py @@ -19,6 +19,14 @@ def get_class(path: str) -> type: + """ + Look up a class based on a dotpath. + Fails if the path does not point to a class. + + >>> import my_module + >>> from hydra.utils import get_class + >>> assert get_class("my_module.MyClass") is my_module.MyClass + """ try: cls = _locate(path) if not isinstance(cls, type): @@ -28,11 +36,19 @@ def get_class(path: str) -> type: ) return cls except Exception as e: - log.error(f"Error initializing class at {path}: {e}") + log.error(f"Error getting class at {path}: {e}") raise e def get_method(path: str) -> Callable[..., Any]: + """ + Look up a callable based on a dotpath. + Fails if the path does not point to a callable object. + + >>> import my_module + >>> from hydra.utils import get_method + >>> assert get_method("my_module.my_function") is my_module.my_function + """ try: obj = _locate(path) if not callable(obj): @@ -51,6 +67,23 @@ def get_method(path: str) -> Callable[..., Any]: get_static_method = get_method +def get_object(path: str) -> Any: + """ + Look up an entity based on the dotpath. + Does not perform any type checks on the entity. + + >>> import my_module + >>> from hydra.utils import get_object + >>> assert get_object("my_module.my_object") is my_module.my_object + """ + try: + obj = _locate(path) + return obj + except Exception as e: + log.error(f"Error getting object at {path} : {e}") + raise e + + def get_original_cwd() -> str: """ :return: the original working directory the Hydra application was launched from diff --git a/noxfile.py b/noxfile.py index dd2f9e30d63..2a19efc60b0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -83,11 +83,11 @@ def find_dirs(path: str) -> Iterator[str]: yield fullname -def _print_installed_omegaconf_version(session: Session) -> None: +def print_installed_package_version(session: Session, package_name: str) -> None: pip_list: str = session.run("pip", "list", silent=True) for line in pip_list.split("\n"): - if "omegaconf" in line: - print(f"Installed omegaconf version: {line}") + if package_name in line: + print(f"Installed {package_name} version: {line}") def install_hydra(session: Session, cmd: List[str]) -> None: @@ -98,7 +98,7 @@ def install_hydra(session: Session, cmd: List[str]) -> None: if USE_OMEGACONF_DEV_VERSION: session.install("--pre", "omegaconf", silent=SILENT) session.run(*cmd, ".", silent=SILENT) - _print_installed_omegaconf_version(session) + print_installed_package_version(session, "omegaconf") if not SILENT: session.install("pipdeptree", silent=SILENT) session.run("pipdeptree", "-p", "hydra-core") @@ -116,6 +116,7 @@ def install_selected_plugins( def install_plugin(session: Session, install_cmd: List[str], plugin: Plugin) -> None: + maybe_install_torch(session, plugin) cmd = install_cmd + [plugin.abspath] session.run(*cmd, silent=SILENT) if not SILENT: @@ -124,6 +125,28 @@ def install_plugin(session: Session, install_cmd: List[str], plugin: Plugin) -> session.run("python", "-c", f"import {plugin.module}") +def maybe_install_torch(session: Session, plugin: Plugin) -> None: + if plugin_requires_torch(plugin): + install_cpu_torch(session) + print_installed_package_version(session, "torch") + + +def plugin_requires_torch(plugin: Plugin) -> bool: + """Determine whether the given plugin depends on pytorch as a requirement""" + return '"torch"' in Path(plugin.setup_py).read_text() + + +def install_cpu_torch(session: Session) -> None: + """ + Install the CPU version of pytorch. + This is a much smaller download size than the normal version `torch` package hosted on pypi. + The smaller download prevents our CI jobs from timing out. + """ + session.install( + "torch", "--extra-index-url", "https://download.pytorch.org/whl/cpu" + ) + + def pytest_args(*args: str) -> List[str]: ret = ["pytest"] ret.extend(args) @@ -163,45 +186,48 @@ def get_plugin_os_names(classifiers: List[str]) -> List[str]: @functools.lru_cache() def list_plugins(directory: str) -> List[Plugin]: blacklist = [".isort.cfg", "examples"] - _plugins = [ - {"dir_name": x, "path": x} + _plugin_directories = [ + x for x in sorted(os.listdir(os.path.join(BASE, directory))) if x not in blacklist ] - # Install read-version in base python environment, needed to run setup.py + # Install bootstrap deps in base python environment subprocess.check_output( - [sys.executable, "-m", "pip", "install", "read-version"], + [ + sys.executable, + "-m", + "pip", + "install", + "read-version", # needed to read data from setup.py + "toml", # so read-version can read pyproject.toml + ], ) plugins: List[Plugin] = [] - for plugin in _plugins: - - abspath = os.path.join(BASE, directory, plugin["path"]) + for dir_name in _plugin_directories: + abspath = os.path.join(BASE, directory, dir_name) setup_py = os.path.join(abspath, "setup.py") - plugin_name = subprocess.check_output( - [sys.executable, setup_py, "--name"], - universal_newlines=True, - ).strip() - classifiers = subprocess.check_output( - [sys.executable, setup_py, "--classifiers"], + name_and_classifiers: List[str] = subprocess.check_output( + [sys.executable, setup_py, "--name", "--classifiers"], universal_newlines=True, ).splitlines() + name, classifiers = name_and_classifiers[0], name_and_classifiers[1:] if "hydra_plugins" in os.listdir(abspath): - module = "hydra_plugins." + plugin["dir_name"] + module = "hydra_plugins." + dir_name source_dir = "hydra_plugins" else: - module = plugin["dir_name"] - source_dir = plugin["dir_name"] + module = dir_name + source_dir = dir_name plugins.append( Plugin( - name=plugin_name, + name=name, abspath=abspath, source_dir=source_dir, module=module, - dir_name=plugin["dir_name"], + dir_name=dir_name, setup_py=setup_py, classifiers=classifiers, ) diff --git a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py index 01c90e91a05..0aeff862f93 100644 --- a/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py +++ b/plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/config.py @@ -34,7 +34,6 @@ class ExperimentConfig: @dataclass class ClientConfig: - verbose_logging: bool = False # set random seed here to make Ax results reproducible @@ -43,7 +42,6 @@ class ClientConfig: @dataclass class AxConfig: - # max_trials is application-specific. Tune it for your use case max_trials: int = 10 early_stop: EarlyStopConfig = field(default_factory=EarlyStopConfig) diff --git a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py index bb34581c383..70f9ba38a85 100644 --- a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py +++ b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/_impl.py @@ -117,7 +117,6 @@ def setup( ) def sweep(self, arguments: List[str]) -> None: - assert self.config is not None assert self.launcher is not None assert self.job_idx is not None diff --git a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py index 5709136d1bb..de3f1f08e86 100644 --- a/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py +++ b/plugins/hydra_nevergrad_sweeper/hydra_plugins/hydra_nevergrad_sweeper/config.py @@ -35,7 +35,6 @@ class ScalarConfigSpec: @dataclass class OptimConf: - # name of the Nevergrad optimizer to use. Here is a sample: # - "OnePlusOne" extremely simple and robust, especially at low budget, but # tends to converge early. diff --git a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/config.py b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/config.py index 6769d4dec0e..03fde5a12d0 100644 --- a/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/config.py +++ b/plugins/hydra_optuna_sweeper/hydra_plugins/hydra_optuna_sweeper/config.py @@ -118,7 +118,6 @@ class MOTPESamplerConfig(SamplerConfig): @dataclass class DistributionConfig: - # Type of distribution. "int", "float" or "categorical" type: DistributionType diff --git a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core_aws.py b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core_aws.py index 5ceec12090c..bcc0da62c8d 100644 --- a/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core_aws.py +++ b/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/_core_aws.py @@ -125,7 +125,6 @@ def launch_jobs( **launcher.create_update_cluster, ) with tempfile.TemporaryDirectory() as local_tmp_download_dir: - with ray_tmp_dir(config, launcher.ray_cfg.run_env.name) as remote_tmp_dir: sdk.rsync( config, diff --git a/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py b/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py index ca9dce70535..f114e67b424 100644 --- a/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py +++ b/plugins/hydra_ray_launcher/integration_test_tools/setup_integration_test_ami.py @@ -22,7 +22,6 @@ def _run_command(command: str) -> str: def run(py_version): - _run_command("rm /home/ubuntu/ray_bootstrap_config.yaml") _run_command(f"conda create -n hydra_{py_version} python={py_version} -y") diff --git a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py index 8aa8f6f60f3..1efc8e4ce86 100644 --- a/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py +++ b/plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py @@ -16,7 +16,6 @@ class BaseSubmititLauncher(Launcher): - _EXECUTOR = "abstract" def __init__(self, **params: Any) -> None: diff --git a/requirements/requirements.txt b/requirements/requirements.txt index f3ad733e9e1..20f72cd4de6 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,4 @@ -omegaconf~=2.2 +omegaconf>=2.2,<2.4 antlr4-python3-runtime==4.9.* importlib-resources;python_version<'3.9' packaging diff --git a/setup.py b/setup.py index 347c402d63f..0fe3f7222b2 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,6 @@ classifiers=[ "License :: OSI Approved :: MIT License", "Development Status :: 4 - Beta", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", diff --git a/tests/instantiate/__init__.py b/tests/instantiate/__init__.py index ba305f8e9ae..257d117d711 100644 --- a/tests/instantiate/__init__.py +++ b/tests/instantiate/__init__.py @@ -436,3 +436,6 @@ def recisinstance(got: Any, expected: Any) -> bool: for key in expected._fields ) return True + + +an_object = object() diff --git a/tests/instantiate/test_helpers.py b/tests/instantiate/test_helpers.py index 7e4de9ef111..2c48b1cf23a 100644 --- a/tests/instantiate/test_helpers.py +++ b/tests/instantiate/test_helpers.py @@ -8,7 +8,7 @@ from pytest import mark, param from hydra._internal.utils import _locate -from hydra.utils import get_class, get_method +from hydra.utils import get_class, get_method, get_object from tests.instantiate import ( AClass, Adam, @@ -16,6 +16,7 @@ ASubclass, NestingClass, Parameters, + an_object, ) from .module_shadowed_by_function import a_function @@ -131,6 +132,7 @@ ), id="import_assertion_error", ), + param("tests.instantiate.an_object", an_object, id="object"), ], ) def test_locate(name: str, expected: Any) -> None: @@ -174,6 +176,14 @@ def test_locate_relative_import_fails(name: str) -> None: ), id="module-error", ), + param( + "tests.instantiate.an_object", + raises( + ValueError, + match="Located non-callable of type 'object' while loading 'tests.instantiate.an_object'", + ), + id="object-error", + ), ], ) def test_get_method(path: str, expected: Any) -> None: @@ -205,6 +215,14 @@ def test_get_method(path: str, expected: Any) -> None: ), id="module-error", ), + param( + "tests.instantiate.an_object", + raises( + ValueError, + match="Located non-class of type 'object' while loading 'tests.instantiate.an_object'", + ), + id="object-error", + ), ], ) def test_get_class(path: str, expected: Any) -> None: @@ -213,3 +231,38 @@ def test_get_class(path: str, expected: Any) -> None: get_class(path) else: assert get_class(path) == expected + + +@mark.parametrize( + "path,expected", + [ + param("tests.instantiate.AClass", AClass, id="class"), + param("builtins.print", print, id="callable"), + param( + "datetime", + datetime, + id="module-error", + ), + param("tests.instantiate.an_object", an_object, id="object"), + param( + "builtins.int.not_found", + raises( + ImportError, + match=dedent( + r""" + Error loading 'builtins\.int\.not_found': + AttributeError\("type object 'int' has no attribute 'not_found'",?\) + Are you sure that 'not_found' is an attribute of 'builtins\.int'\? + """ + ).strip(), + ), + id="builtin_attribute_error", + ), + ], +) +def test_get_object(path: str, expected: Any) -> None: + if isinstance(expected, RaisesContext): + with expected: + get_object(path) + else: + assert get_object(path) == expected diff --git a/tests/instantiate/test_instantiate.py b/tests/instantiate/test_instantiate.py index cde77d8af78..5e0654cecaf 100644 --- a/tests/instantiate/test_instantiate.py +++ b/tests/instantiate/test_instantiate.py @@ -691,7 +691,6 @@ def test_instantiate_bad_adam_conf(instantiate_func: Any, recwarn: Any) -> None: def test_instantiate_with_missing_module(instantiate_func: Any) -> None: - _target_ = "tests.instantiate.ClassWithMissingModule" with raises( InstantiationException, diff --git a/tests/test_config_search_path.py b/tests/test_config_search_path.py index d34d1ede6ab..d64d4d11851 100644 --- a/tests/test_config_search_path.py +++ b/tests/test_config_search_path.py @@ -172,6 +172,7 @@ def test_prepend( "../conf", realpath(os.path.join(os.getcwd(), "../conf")), ), + (None, "package.module", "pkg://some/conf", "pkg://some/conf"), ], ) def test_compute_search_path_dir( diff --git a/tests/test_env_defaults.py b/tests/test_env_defaults.py index c3b78680550..796bffc0b70 100644 --- a/tests/test_env_defaults.py +++ b/tests/test_env_defaults.py @@ -7,7 +7,6 @@ def test_env_defaults(tmpdir: Path) -> None: - cmd = [ "tests/test_apps/custom_env_defaults/my_app.py", "hydra.run.dir=" + str(tmpdir), diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 00000000000..c4a0b64b650 --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,14 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import pickle + +from hydra.errors import MissingConfigException + + +def test_pickle_missing_config_exception() -> None: + exception = MissingConfigException("msg", "filename", ["option1", "option2"]) + x = pickle.dumps(exception) + loaded = pickle.loads(x) # nosec + assert isinstance(loaded, MissingConfigException) + assert loaded.args == ("msg",) + assert loaded.missing_cfg_file == "filename" + assert loaded.options == ["option1", "option2"] diff --git a/tests/test_examples/test_tutorials_basic.py b/tests/test_examples/test_tutorials_basic.py index 33723ffb66d..941e3f9fd1d 100644 --- a/tests/test_examples/test_tutorials_basic.py +++ b/tests/test_examples/test_tutorials_basic.py @@ -312,7 +312,6 @@ def test_sweeping_example( def test_advanced_ad_hoc_composition( monkeypatch: Any, tmpdir: Path, args: List[str], expected: Any ) -> None: - monkeypatch.setenv("USER", "test_user") cmd = [ "examples/advanced/ad_hoc_composition/hydra_compose_example.py", diff --git a/tests/test_hydra.py b/tests/test_hydra.py index 4618af26fed..c75e3d39ba3 100644 --- a/tests/test_hydra.py +++ b/tests/test_hydra.py @@ -845,13 +845,13 @@ def test_help( "overrides,expected", [ param( - ["--info", "searchpath", "hydra.searchpath=['pkg://additonal_conf']"], - r".*hydra.searchpath in command-line\s+|\s+pkg://additonal_conf.*", + ["--info", "searchpath", "hydra.searchpath=['pkg://additional_conf']"], + r".*hydra.searchpath in command-line\s+|\s+pkg://additional_conf.*", id="searchpath config from command-line", ), param( ["--info", "searchpath"], - r".*hydra.searchpath in main\s+|\s+pkg://additonal_conf.*", + r".*hydra.searchpath in main\s+|\s+pkg://additional_conf.*", id="print info with searchpath config", ), param( @@ -1824,7 +1824,6 @@ def test_hydra_mode( ] ) if error: - expected = normalize_newlines(expected_output) ret = run_with_error(cmd) assert_regex_match( @@ -1834,7 +1833,6 @@ def test_hydra_mode( to_name="Actual output", ) elif warning: - out, err = run_python_script(cmd, allow_warnings=True) assert_regex_match( from_line=expected_output, diff --git a/tools/configen/configen/config.py b/tools/configen/configen/config.py index 3ff6b00cfa0..39bd4691f76 100644 --- a/tools/configen/configen/config.py +++ b/tools/configen/configen/config.py @@ -22,7 +22,6 @@ class ModuleConf: @dataclass class ConfigenConf: - # output dir output_dir: str = MISSING diff --git a/tools/configen/configen/configen.py b/tools/configen/configen/configen.py index 438ea73635b..c553dad357b 100644 --- a/tools/configen/configen/configen.py +++ b/tools/configen/configen/configen.py @@ -89,7 +89,6 @@ class ClassInfo: def is_incompatible(type_: Type[Any]) -> bool: - opt = _resolve_optional(type_) # Unions are not supported (Except Optional) if not opt[0] and is_union_annotation(type_): @@ -131,7 +130,6 @@ def is_incompatible(type_: Type[Any]) -> bool: def get_default_flags(module: ModuleConf) -> List[Parameter]: - def_flags: List[Parameter] = [] if module.default_flags._convert_ is not None: diff --git a/website/docs/advanced/compose_api.md b/website/docs/advanced/compose_api.md index 140e50823da..e18721d4876 100644 --- a/website/docs/advanced/compose_api.md +++ b/website/docs/advanced/compose_api.md @@ -28,9 +28,9 @@ Doing so forfeits many of the benefits of Hydra ### Initialization methods There are 3 initialization methods: -- `initialize()`: Initialize with a config path relative to the caller -- `initialize_config_module()` : Initialize with config_module (absolute) -- `initialize_config_dir()` : Initialize with a config_dir on the file system (absolute) +- initialize(): Initialize with a config path relative to the caller +- initialize_config_module(): Initialize with config_module (absolute) +- initialize_config_dir(): Initialize with a config_dir on the file system (absolute) All 3 can be used as methods or contexts. When used as methods, they are initializing Hydra globally and should only be called once. diff --git a/website/docs/advanced/instantiate_objects/overview.md b/website/docs/advanced/instantiate_objects/overview.md index 314b568a80b..01f88be9d97 100644 --- a/website/docs/advanced/instantiate_objects/overview.md +++ b/website/docs/advanced/instantiate_objects/overview.md @@ -392,3 +392,51 @@ from hydra.utils import instantiate # instantiate({"_target_": "len"}, [1,2,3]) # this gives an InstantiationException instantiate({"_target_": "builtins.len"}, [1,2,3]) # this works, returns the number 3 ``` + +### Dotpath lookup machinery + +Hydra looks up a given `_target_` by attempting to find a module that +corresponds to a prefix of the given dotpath and then looking for an object in +that module corresponding to the dotpath's tail. For example, to look up a `_target_` +given by the dotpath `"my_module.my_nested_module.my_object"`, hydra first locates +the module `my_module.my_nested_module`, then find `my_object` inside that nested module. + +Hydra exposes an API allowing direct use of this dotpath lookup machinery. +The following three functions, which can be imported from the hydra.utils module, +accept a string-typed dotpath as an argument and return the located class/callable/object: +```python +def get_class(path: str) -> type: + """ + Look up a class based on a dotpath. + Fails if the path does not point to a class. + + >>> import my_module + >>> from hydra.utils import get_class + >>> assert get_class("my_module.MyClass") is my_module.MyClass + """ + ... + +def get_method(path: str) -> Callable[..., Any]: + """ + Look up a callable based on a dotpath. + Fails if the path does not point to a callable object. + + >>> import my_module + >>> from hydra.utils import get_method + >>> assert get_method("my_module.my_function") is my_module.my_function + """ + ... + +# Alias for get_method +get_static_method = get_method + +def get_object(path: str) -> Any: + """ + Look up a callable based on a dotpath. + + >>> import my_module + >>> from hydra.utils import get_object + >>> assert get_object("my_module.my_object") is my_module.my_object + """ + ... +``` diff --git a/website/docs/advanced/search_path.md b/website/docs/advanced/search_path.md index 829d0ced9f2..4ca261d3dce 100644 --- a/website/docs/advanced/search_path.md +++ b/website/docs/advanced/search_path.md @@ -94,7 +94,7 @@ defaults: hydra: searchpath: - - pkg://additonal_conf + - pkg://additional_conf # You can also use file based schema: # - file:///etc/my_app # - file://${oc.env:HOME}/.my_app @@ -148,7 +148,7 @@ dataset: `hydra.searchpath` can be defined or overridden via the command line as well: ```bash title="command line override" -python my_app.py 'hydra.searchpath=[pkg://additonal_conf]' +python my_app.py 'hydra.searchpath=[pkg://additional_conf]' ``` #### Overriding `--config-dir` from the command line diff --git a/website/docs/advanced/terminology.md b/website/docs/advanced/terminology.md index 905231bbec7..2ec6d954fad 100644 --- a/website/docs/advanced/terminology.md +++ b/website/docs/advanced/terminology.md @@ -12,7 +12,7 @@ They can be grouped by placing them in [Config Groups](#config-group). ### Config files Config files are form of input configs in [YAML](https://yaml.org/) format. They can exist in the file system or in a Python module. -```yaml title="Example config file +```yaml title="Example config file" user: name: James Bond age: 7 diff --git a/website/docs/configure_hydra/Intro.md b/website/docs/configure_hydra/Intro.md index 8792894053d..2c46b3b206c 100644 --- a/website/docs/configure_hydra/Intro.md +++ b/website/docs/configure_hydra/Intro.md @@ -30,7 +30,7 @@ You can view the Hydra config structure You can view the Hydra config using `--cfg hydra`:
- $ python my_app.p --cfg hydra (Click to expand) + $ python my_app.py --cfg hydra (Click to expand) ```yaml hydra: @@ -73,7 +73,22 @@ def my_app(cfg: DictConfig) -> None: print(HydraConfig.get().job.name) ``` -The following variables are populated at runtime. +### Top-level Hydra settings +The following fields are present at the top level of the Hydra Config. +- mode: Optional, one of `RUN` or `MULTIRUN`. See [multirun](/tutorials/basic/running_your_app/2_multirun.md) for more info. +- **searchpath**: A list of paths that Hydra searches in order to find configs. + See [overriding `hydra.searchpath`](advanced/search_path.md#overriding-hydrasearchpath-config) +- **job_logging** and **hydra_logging**: Configure logging settings. + See [logging](/tutorials/basic/running_your_app/4_logging.md) and [customizing logging](logging.md). +- **sweeper**: [Sweeper](/tutorials/basic/running_your_app/2_multirun.md#sweeper) plugin settings. Defaults to basic sweeper. +- **launcher**: [Launcher](/tutorials/basic/running_your_app/2_multirun.md#launcher) plugin settings. Defaults to basic launcher. +- **callbacks**: [Experimental callback support](/experimental/callbacks.md). +- **help**: Configures your app's `--help` CLI flag. See [customizing application's help](app_help.md). +- **hydra_help**: Configures the `--hydra-help` CLI flag. +- **output_subdir**: Configures the `.hydra` subdirectory name. + See [changing or disabling the output subdir](/tutorials/basic/running_your_app/3_working_directory.md#changing-or-disabling-hydras-output-subdir). +- **verbose**: Configures per-file DEBUG-level logging. + See [logging](/tutorials/basic/running_your_app/4_logging.md). ### hydra.job: The **hydra.job** node is used for configuring some aspects of your job. @@ -118,25 +133,6 @@ Fields under **hydra.overrides** are populated automatically and should not be o See [Output/Working directory](/tutorials/basic/running_your_app/3_working_directory.md). - **hydra**: Contains a list of the command-line `hydra` config overrides used. -### hydra.mode -See [multirun](/tutorials/basic/running_your_app/2_multirun.md) for more info. - -### Other Hydra settings -The following fields are present at the top level of the Hydra Config. -- **searchpath**: A list of paths that Hydra searches in order to find configs. - See [overriding `hydra.searchpath`](advanced/search_path.md#overriding-hydrasearchpath-config) -- **job_logging** and **hydra_logging**: Configure logging settings. - See [logging](/tutorials/basic/running_your_app/4_logging.md) and [customizing logging](logging.md). -- **sweeper**: [Sweeper](/tutorials/basic/running_your_app/2_multirun.md#sweeper) plugin settings. Defaults to basic sweeper. -- **launcher**: [Launcher](/tutorials/basic/running_your_app/2_multirun.md#launcher) plugin settings. Defaults to basic launcher. -- **callbacks**: [Experimental callback support](/experimental/callbacks.md). -- **help**: Configures your app's `--help` CLI flag. See [customizing application's help](app_help.md). -- **hydra_help**: Configures the `--hydra-help` CLI flag. -- **output_subdir**: Configures the `.hydra` subdirectory name. - See [changing or disabling the output subdir](/tutorials/basic/running_your_app/3_working_directory.md#changing-or-disabling-hydras-output-subdir). -- **verbose**: Configures per-file DEBUG-level logging. - See [logging](/tutorials/basic/running_your_app/4_logging.md). - ### Resolvers provided by Hydra Hydra provides the following [OmegaConf resolvers](https://omegaconf.readthedocs.io/en/latest/usage.html#resolvers) by default. diff --git a/website/docs/experimental/rerun.md b/website/docs/experimental/rerun.md index f521148cbb0..353330fe9bc 100644 --- a/website/docs/experimental/rerun.md +++ b/website/docs/experimental/rerun.md @@ -20,7 +20,7 @@ Hydra Callback for saving the job info: hydra: callbacks: save_job_info: - _target_: hydra.experimental.pickle_job_info_callback.PickleJobInfoCallback + _target_: hydra.experimental.callbacks.PickleJobInfoCallback ``` diff --git a/website/docs/intro.md b/website/docs/intro.md index 93cd566ccaa..d176bc5f82c 100644 --- a/website/docs/intro.md +++ b/website/docs/intro.md @@ -27,7 +27,8 @@ Use the version switcher in the top bar to switch between documentation versions | | Version | Release notes | Python Versions | | -------|---------------------------|-------------------------------------------------------------------------------------| -------------------| -| ►| 1.2 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.2.0) | **3.6 - 3.10** | +| ►| 1.3 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.3.0) | **3.6 - 3.11** | +| | 1.2 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.2.0) | **3.6 - 3.10** | | | 1.1 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.1) | **3.6 - 3.9** | | | 1.0 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.0.7) | **3.6 - 3.8** | | | 0.11 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v0.11.3) | **2.7, 3.5 - 3.8** | diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 15759c49192..6c7d0936f09 100755 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -18,8 +18,9 @@ module.exports = { projectName: 'hydra', // Usually your repo name. customFields: { githubLinkVersionToBaseUrl: { - // TODO: Update once a branch is cut for 1.3 - "1.3": "https://github.com/facebookresearch/hydra/blob/main/", + // TODO: Update once a branch is cut for 1.4 + "1.4": "https://github.com/facebookresearch/hydra/blob/main/", + "1.3": "https://github.com/facebookresearch/hydra/blob/1.3_branch/", "1.2": "https://github.com/facebookresearch/hydra/blob/1.2_branch/", "1.1": "https://github.com/facebookresearch/hydra/blob/1.1_branch/", "1.0": "https://github.com/facebookresearch/hydra/blob/1.0_branch/", diff --git a/website/versioned_docs/version-0.11/intro.md b/website/versioned_docs/version-0.11/intro.md index 47acfb9fb6c..5f4085a8f5b 100644 --- a/website/versioned_docs/version-0.11/intro.md +++ b/website/versioned_docs/version-0.11/intro.md @@ -26,9 +26,11 @@ Use the version switcher in the top bar to switch between documentation versions | | Version | Release notes | Python Versions | | -------|---------------------------|-------------------------------------------------------------------------------------| -------------------| -| | 1.1 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.0) | **3.6 - 3.9** | -| | 1.0 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.0.0rc1) | **3.6 - 3.8** | -| ►| 0.11 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/0.11.0) | **2.7, 3.5 - 3.8** | +| | 1.3 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.3.0) | **3.6 - 3.11** | +| | 1.2 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.2.0) | **3.6 - 3.10** | +| | 1.1 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.1) | **3.6 - 3.9** | +| | 1.0 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.0.7) | **3.6 - 3.8** | +| ►| 0.11 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v0.11.3) | **2.7, 3.5 - 3.8** | ## Quick start guide This guide will show you some of the most important features of Hydra. diff --git a/website/versioned_docs/version-1.0/intro.md b/website/versioned_docs/version-1.0/intro.md index aab05d81509..aed906e7c9a 100644 --- a/website/versioned_docs/version-1.0/intro.md +++ b/website/versioned_docs/version-1.0/intro.md @@ -26,9 +26,11 @@ Use the version switcher in the top bar to switch between documentation versions | | Version | Release notes | Python Versions | | -------|---------------------------|-------------------------------------------------------------------------------------| -------------------| -| | 1.1 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.0) | **3.6 - 3.9** | -| ►| 1.0 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.0.0rc1) | **3.6 - 3.8** | -| | 0.11 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/0.11.0) | **2.7, 3.5 - 3.8** | +| | 1.3 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.3.0) | **3.6 - 3.11** | +| | 1.2 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.2.0) | **3.6 - 3.10** | +| | 1.1 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.1) | **3.6 - 3.9** | +| ►| 1.0 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.0.7) | **3.6 - 3.8** | +| | 0.11 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v0.11.3) | **2.7, 3.5 - 3.8** | ## Quick start guide This guide will show you some of the most important features of Hydra. diff --git a/website/versioned_docs/version-1.1/advanced/search_path.md b/website/versioned_docs/version-1.1/advanced/search_path.md index cfe3b4185bc..48325d54e6b 100644 --- a/website/versioned_docs/version-1.1/advanced/search_path.md +++ b/website/versioned_docs/version-1.1/advanced/search_path.md @@ -59,7 +59,7 @@ In this example, we add a second config directory - `additional_conf`, next to t │   ├── config.yaml │   └── dataset │   └── cifar10.yaml -├── additonal_conf +├── additional_conf │   ├── __init__.py │   └── dataset │   └── imagenet.yaml @@ -83,7 +83,7 @@ if __name__ == "__main__": `conf/config.yaml` is the primary config for `my_app.py`, config groups `cifar10` and `imagenet` are under different folders. -We can add `additonal_conf` to `hydra.searchpath` for Hydra to discover `dataset/imagenet`. +We can add `additional_conf` to `hydra.searchpath` for Hydra to discover `dataset/imagenet`.
@@ -94,7 +94,7 @@ defaults: hydra: searchpath: - - pkg://additonal_conf + - pkg://additional_conf # You can also use file based schema: # - file:///etc/my_app # - file://${oc.env:HOME}/.my_app @@ -148,7 +148,7 @@ dataset: `hydra.searchpath` can be defined or overridden via the command line as well: ```bash title="command line override" -python my_app.py 'hydra.searchpath=[pkg://additonal_conf]' +python my_app.py 'hydra.searchpath=[pkg://additional_conf]' ``` #### Overriding `--config-dir` from the command line diff --git a/website/versioned_docs/version-1.1/intro.md b/website/versioned_docs/version-1.1/intro.md index 63954c1a395..016677c176e 100644 --- a/website/versioned_docs/version-1.1/intro.md +++ b/website/versioned_docs/version-1.1/intro.md @@ -27,7 +27,9 @@ Use the version switcher in the top bar to switch between documentation versions | | Version | Release notes | Python Versions | | -------|---------------------------|-------------------------------------------------------------------------------------| -------------------| -| ►| 1.1 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.1) | **3.6 - 3.9** | +| | 1.3 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.3.0) | **3.6 - 3.11** | +| | 1.2 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.2.0) | **3.6 - 3.10** | +| ►| 1.1 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.1) | **3.6 - 3.9** | | | 1.0 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.0.7) | **3.6 - 3.8** | | | 0.11 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v0.11.3) | **2.7, 3.5 - 3.8** | diff --git a/website/versioned_docs/version-1.2/advanced/search_path.md b/website/versioned_docs/version-1.2/advanced/search_path.md index 829d0ced9f2..4ca261d3dce 100644 --- a/website/versioned_docs/version-1.2/advanced/search_path.md +++ b/website/versioned_docs/version-1.2/advanced/search_path.md @@ -94,7 +94,7 @@ defaults: hydra: searchpath: - - pkg://additonal_conf + - pkg://additional_conf # You can also use file based schema: # - file:///etc/my_app # - file://${oc.env:HOME}/.my_app @@ -148,7 +148,7 @@ dataset: `hydra.searchpath` can be defined or overridden via the command line as well: ```bash title="command line override" -python my_app.py 'hydra.searchpath=[pkg://additonal_conf]' +python my_app.py 'hydra.searchpath=[pkg://additional_conf]' ``` #### Overriding `--config-dir` from the command line diff --git a/website/versioned_docs/version-1.2/experimental/rerun.md b/website/versioned_docs/version-1.2/experimental/rerun.md index f521148cbb0..353330fe9bc 100644 --- a/website/versioned_docs/version-1.2/experimental/rerun.md +++ b/website/versioned_docs/version-1.2/experimental/rerun.md @@ -20,7 +20,7 @@ Hydra Callback for saving the job info: hydra: callbacks: save_job_info: - _target_: hydra.experimental.pickle_job_info_callback.PickleJobInfoCallback + _target_: hydra.experimental.callbacks.PickleJobInfoCallback ``` diff --git a/website/versioned_docs/version-1.2/intro.md b/website/versioned_docs/version-1.2/intro.md index 93cd566ccaa..ceb8282c976 100644 --- a/website/versioned_docs/version-1.2/intro.md +++ b/website/versioned_docs/version-1.2/intro.md @@ -27,7 +27,8 @@ Use the version switcher in the top bar to switch between documentation versions | | Version | Release notes | Python Versions | | -------|---------------------------|-------------------------------------------------------------------------------------| -------------------| -| ►| 1.2 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.2.0) | **3.6 - 3.10** | +| | 1.3 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.3.0) | **3.6 - 3.11** | +| ►| 1.2 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.2.0) | **3.6 - 3.10** | | | 1.1 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.1) | **3.6 - 3.9** | | | 1.0 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.0.7) | **3.6 - 3.8** | | | 0.11 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v0.11.3) | **2.7, 3.5 - 3.8** | diff --git a/website/versioned_docs/version-1.3/advanced/compose_api.md b/website/versioned_docs/version-1.3/advanced/compose_api.md new file mode 100644 index 00000000000..e18721d4876 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/compose_api.md @@ -0,0 +1,127 @@ +--- +id: compose_api +title: Compose API +sidebar_label: Compose API +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + +The compose API can compose a config similarly to `@hydra.main()` anywhere in the code. +Prior to calling compose(), you have to initialize Hydra: This can be done by using the standard `@hydra.main()` +or by calling one of the initialization methods listed below. + +### When to use the Compose API + +The Compose API is useful when `@hydra.main()` is not applicable. +For example: + +- Inside a Jupyter notebook ([Example](jupyter_notebooks.md)) +- Inside a unit test ([Example](unit_testing.md)) +- In parts of your application that does not have access to the command line (Example). +- To compose multiple configuration objects (Example with Ray). + + + +### Initialization methods +There are 3 initialization methods: +- initialize(): Initialize with a config path relative to the caller +- initialize_config_module(): Initialize with config_module (absolute) +- initialize_config_dir(): Initialize with a config_dir on the file system (absolute) + +All 3 can be used as methods or contexts. +When used as methods, they are initializing Hydra globally and should only be called once. +When used as contexts, they are initializing Hydra within the context can be used multiple times. +Like @hydra.main() all three support the [version_base](../upgrades/version_base.md) parameter +to define the compatability level to use. + +### Code example +```python +from hydra import compose, initialize +from omegaconf import OmegaConf + +if __name__ == "__main__": + # context initialization + with initialize(version_base=None, config_path="conf", job_name="test_app"): + cfg = compose(config_name="config", overrides=["db=mysql", "db.user=me"]) + print(OmegaConf.to_yaml(cfg)) + + # global initialization + initialize(version_base=None, config_path="conf", job_name="test_app") + cfg = compose(config_name="config", overrides=["db=mysql", "db.user=me"]) + print(OmegaConf.to_yaml(cfg)) +``` +### API Documentation + +```python title="Compose API" +def compose( + config_name: Optional[str] = None, + overrides: List[str] = [], + return_hydra_config: bool = False, +) -> DictConfig: + """ + :param config_name: the name of the config + (usually the file name without the .yaml extension) + :param overrides: list of overrides for config file + :param return_hydra_config: True to return the hydra config node in the result + :return: the composed config + """ +``` + +```python title="Relative initialization" +def initialize( + version_base: Optional[str], + config_path: Optional[str] = None, + job_name: Optional[str] = "app", + caller_stack_depth: int = 1, +) -> None: + """ + Initializes Hydra and add the config_path to the config search path. + config_path is relative to the parent of the caller. + Hydra detects the caller type automatically at runtime. + + Supported callers: + - Python scripts + - Python modules + - Unit tests + - Jupyter notebooks. + :param version_base: compatability level to use. + :param config_path: path relative to the parent of the caller + :param job_name: the value for hydra.job.name (By default it is automatically detected based on the caller) + :param caller_stack_depth: stack depth of the caller, defaults to 1 (direct caller). + """ +``` + +```python title="Initialzing with config module" +def initialize_config_module( + config_module: str, + version_base: Optional[str], + job_name: str = "app" +) -> None: + """ + Initializes Hydra and add the config_module to the config search path. + The config module must be importable (an __init__.py must exist at its top level) + :param config_module: absolute module name, for example "foo.bar.conf". + :param version_base: compatability level to use. + :param job_name: the value for hydra.job.name (default is 'app') + """ +``` +```python title="Initialzing with config directory" +def initialize_config_dir( + config_dir: str, + version_base: Optional[str], + job_name: str = "app" +) -> None: + """ + Initializes Hydra and add an absolute config dir to the to the config search path. + The config_dir is always a path on the file system and is must be an absolute path. + Relative paths will result in an error. + :param config_dir: absolute file system path + :param version_base: compatability level to use. + :param job_name: the value for hydra.job.name (default is 'app') + """ +``` + diff --git a/website/versioned_docs/version-1.3/advanced/decorating_main.md b/website/versioned_docs/version-1.3/advanced/decorating_main.md new file mode 100644 index 00000000000..c36c81f2674 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/decorating_main.md @@ -0,0 +1,61 @@ +--- +id: decorating_main +title: Decorating the main function +--- + +`@hydra.main` can be used over decorated functions: + +
+
+ +```python title="app.py" +from omegaconf import DictConfig, OmegaConf +import hydra + +from mypackage import mydecorator + +@hydra.main(version_base=None) +@mydecorator +def my_app(cfg: DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) +``` +
+ +
+ +```python title="mypackage.py" +from functools import wraps +from typing import Callable + +from omegaconf import DictConfig + +def mydecorator(func: Callable) -> Callable: + @wraps(func) + def inner_decorator(cfg: DictConfig): + print(OmegaConf.to_yaml(cfg)) # do some stuff + return func(cfg) # pass cfg to decorated function + + return inner_decorator +``` + +
+ +Decorator should be compatible with rules below: + +1. It should be created using `@functools.wraps` decorator +2. It should accept an argument of type `omegaconf.DictConfig` and pass it to the underlying function + +:::info +`@functools.wraps` passes name and docstring of underlying function to the decorated one. + +But more importantly, it saves underlying function into `__wrapped__` attribute of decorated function, +which is used by `hydra.main` to calculate the absolute path of config file. + +Without `@wraps` hydra will fail while searching for the config path: + +```log +hydra.errors.MissingConfigException: Primary config directory not found. +Check that the config directory '/usr/local/lib/python3.7/conf' exists and readable +``` + +::: diff --git a/website/versioned_docs/version-1.3/advanced/defaults_list.md b/website/versioned_docs/version-1.3/advanced/defaults_list.md new file mode 100644 index 00000000000..fc3b7ff1e91 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/defaults_list.md @@ -0,0 +1,296 @@ +--- +id: defaults_list +title: The Defaults List +--- + +## Introduction + +:::important +Many of the features described in this page are new. Please report any issues. +::: + +The Defaults List is a list in an input config that instructs Hydra how to build the output config. +Each input config can have a Defaults List as a top level element. The Defaults List itself +is not a part of output config. + +```text title="Defaults List YAML syntax" +defaults: + (- CONFIG|GROUP_DEFAULT)* + +CONFIG : (CONFIG_GROUP/)?CONFIG_NAME(@PACKAGE)? +GROUP_DEFAULT : [optional|override]? CONFIG_GROUP(@PACKAGE)?: OPTION +OPTION : CONFIG_NAME|CONFIG_NAMES|null +``` + +*CONFIG* : A config to use when creating the output config. e.g. `db/mysql`, `db/mysql@backup`. + +*GROUP_DEFAULT* : An *overridable* config. e.g. `db: mysql`, `db@backup: mysql`. +- ***override*** : Overrides the option of a previously defined GROUP_DEFAULT. +- ***optional*** : By default, an OPTION that do not exist causes an error; optional suppresses the error. +- ***null*** : A place-holder for a future override. If it is not overridden the entry is ignored. + +*CONFIG_NAME*: The name of a config, without the file system extension. e.g. `mysql` and not `mysql.yaml`. + +*CONFIG_NAMES* : A list of config names. e.g. `[mysql, sqlite]` + +*CONFIG_GROUP* : A path to a set of configs. +The path is relative to the containing config. +It can be made absolute by prefixing it with a `/`. +The path separator is `/` regardless of the operating system. + +*OPTION*: The currently selected *CONFIG_NAME* or *CONFIG_NAMES* from a *CONFIG_GROUP*. + +*PACKAGE* : Where to place the content of the config within the output config. +It is relative to the Package of the containing config by default. See [Packages](overriding_packages.md). + +## An example + +```text title="Config directory structure" +├── server +│ ├── db +│ │ ├── mysql.yaml +│ │ └── sqlite.yaml +│ └── apache.yaml +└── config.yaml +``` +Input configs: +
+
+ +```yaml title="config.yaml" +defaults: + - server/apache + +debug: false + + + +``` +
+ +
+ +```yaml title="server/apache.yaml" +defaults: + - db: mysql + +name: apache + + + +``` +
+ +
+ +```yaml title="server/db/mysql.yaml" +name: mysql +``` + +```yaml title="server/db/sqlite.yaml" +name: sqlite +``` +
+ +Output config: +```yaml title="$ python my_app.py" +server: + db: + name: mysql + name: apache +debug: false +``` + +## Overriding Config Group options +A Config Group's option can be overridden using a new *GROUP_DEFAULT* with the ***override*** keyword. +If a Group Default is overridden more than once, the last one, in depth first order, wins. + +Extending the previous example: + +
+
+ +```yaml title="config.yaml" {3} +defaults: + - server/apache + - override server/db: sqlite + +debug: false +``` +
+
+ +```yaml title="$ python my_app.py" {2,3} +server: + db: + name: sqlite + name: apache +debug: false +``` +
+
+ +A Config Group's option can also be overridden via the command line. e.g: +``` +$ python my_app.py server/db=sqlite +``` + +## Composition order +The Defaults List is ordered: +- If multiple configs define the same value, the last one wins. +- If multiple configs contribute to the same dictionary, the result is the combined dictionary. + +By default, the content of a config is overriding the content of configs in the defaults list. + +
+
+ +```yaml title="config.yaml" {5} +defaults: + - db: mysql + +db: + host: backup +``` + +
+ +
+ +```yaml title="Result: db.host from config" {3} +db: + driver: mysql # db/mysql.yaml + host: backup # config.yaml + port: 3306 # db/mysql.yaml + +``` + +
+
+ +The `_self_` entry determines the relative position of **this** config in the Defaults List. +If it is not specified, it is added automatically as the last item. + +
+
+ +```yaml title="config.yaml" {2,6} +defaults: + - _self_ + - db: mysql # Overrides this config + +db: + host: backup +``` +
+
+ +```yaml title="Result: All values from db/mysql" {3} +db: + driver: mysql # db/mysql.yaml + host: localhost # db/mysql.yaml + port: 3306 # db/mysql.yaml + + +``` +
+
+ +With `_self_` at the top of the Defaults List, the host field defined in *config.yaml* now precedes the host field defined +in *db/mysql.yaml*, and as a result is overridden. + +## Interpolation in the Defaults List + +Config Group Options can be selected using interpolation. +```yaml +defaults: + - server: apache + - db: mysql + - combination_specific_config: ${server}_${db} # apache_mysql +``` +Interpolation keys can be config groups with any @package overrides. +For example: `${db/engine}`, `${db@backup}` + +The selected option for *combination_specific_config* depends on the final selected options for *db* and *server*. +e.g., If *db* is overridden to *sqlite*, *combination_specific_config* will become *apache_sqlite*. + +#### Restrictions: + + - Interpolation keys in the Defaults List cannot reference values in the Final Config Object (it does not yet exist). + - Defaults List interpolation keys are absolute (even in nested configs). + - The subtree expanded by an Interpolated Config may not contain Default List overrides. + +See [Patterns/Specializing Configs](/patterns/specializing_config.md) for more information. + +## Debugging the Defaults List +Hydra's config composition process is as follows: + + - The Defaults Tree is created. + - The Final Defaults List is created via a DFS walk of the Defaults Tree. + - The Output Config is composed from the entries in the Final Defaults List. + +You can inspect these artifacts via command line flags: + +- `--info defaults-tree` shows the Defaults Tree. +- `--info defaults` Shows the Final Defaults List. +- `--cfg job|hydra|all` Shows the Output Config. + +Example outputs: +
python my_app.py --info defaults-tree + +```yaml title="" +: + hydra/config: + hydra/hydra_logging: default + hydra/job_logging: default + hydra/launcher: basic + hydra/sweeper: basic + hydra/output: default + hydra/help: default + hydra/hydra_help: default + _self_ + config: + server/apache: + server/db: mysql + _self_ + _self_ +``` +
+
python my_app.py --info defaults + +```text +Defaults List +************* +| Config path | Package | _self_ | Parent | +------------------------------------------------------------------------------- +| hydra/hydra_logging/default | hydra.hydra_logging | False | hydra/config | +| hydra/job_logging/default | hydra.job_logging | False | hydra/config | +| hydra/launcher/basic | hydra.launcher | False | hydra/config | +| hydra/sweeper/basic | hydra.sweeper | False | hydra/config | +| hydra/output/default | hydra | False | hydra/config | +| hydra/help/default | hydra.help | False | hydra/config | +| hydra/hydra_help/default | hydra.hydra_help | False | hydra/config | +| hydra/config | hydra | True | | +| server/db/mysql | server.db | False | server/apache | +| server/apache | server | True | config | +| config | | True | | +------------------------------------------------------------------------------- +``` +
+
python my_app.py --cfg job + +```yaml +server: + db: + name: mysql + name: apache +debug: false +``` +
+ +## Related topics +- [Packages](overriding_packages.md) +- [Common Patterns/Extending Configs](patterns/extending_configs.md) +- [Common Patterns/Configuring Experiments](patterns/configuring_experiments.md) +- [Selecting multiple configs from a Config Group](patterns/select_multiple_configs_from_config_group.md) + diff --git a/website/versioned_docs/version-1.3/advanced/hydra-command-line-flags.md b/website/versioned_docs/version-1.3/advanced/hydra-command-line-flags.md new file mode 100644 index 00000000000..1452b6cb808 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/hydra-command-line-flags.md @@ -0,0 +1,37 @@ +--- +id: hydra-command-line-flags +title: Hydra's command line flags +--- + +Hydra is using the command line for two things: +- Controlling Hydra +- Configuring your application (See [Override Grammar](override_grammar/basic.md)) + +Arguments prefixed by - or -- control Hydra; the rest are used to configure the application. + +Information about Hydra: +- **--hydra-help**: Shows Hydra specific flags +- **--version**: Show Hydra's version and exit + +Information provided by the Hydra app: +- **--help,-h**: Shows the application's help. This can be [customized](configure_hydra/app_help.md). + +Debugging assistance: +- **--cfg,-c**: Show config instead of running. Takes as parameter one of `job`, `hydra` or `all`. +- **--resolve**: Used in conjunction with the `--cfg` flag; resolve interpolations in the config before printing it. +- **--package,-p**: Used in conjunction with --cfg to select a specific config package to show. +- **--info,-i**: Print Hydra information. This includes installed plugins, Config Search Path, Defaults List, generated config and more. + + +Running Hydra applications: +- **--run,-r**: Run is the default mode and is not normally needed. +- **--multirun,-m**: Run multiple jobs with the configured launcher and sweeper. See [Multi-run](/tutorials/basic/running_your_app/2_multirun.md). +

+- **--config-path,-cp**: Overrides the `config_path` specified in `hydra.main()`. The `config_path` is absolute or relative to the Python file declaring `@hydra.main()`. +- **--config-name,-cn**: Overrides the `config_name` specified in `hydra.main()`. +- **--config-dir,-cd**: Adds an additional config directory to the [config search path](search_path.md). +This is useful for installed apps that want to allow their users to provide additional configs. + +Misc: +- **--shell-completion,-sc**: Install or Uninstall [shell tab completion](/tutorials/basic/running_your_app/6_tab_completion.md). + diff --git a/website/versioned_docs/version-1.3/advanced/instantiate_objects/config_files.md b/website/versioned_docs/version-1.3/advanced/instantiate_objects/config_files.md new file mode 100644 index 00000000000..bcc654f87f2 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/instantiate_objects/config_files.md @@ -0,0 +1,90 @@ +--- +id: config_files +title: Config files example +sidebar_label: Config files example +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +This example demonstrates the use of config files to instantiated objects. + +```python +class DBConnection: + def connect(self): + ... + +class MySQLConnection(DBConnection): + def __init__(self, host: str, user: str, password: str) -> None: + self.host = host + self.user = user + self.password = password + + def connect(self) -> None: + print(f"MySQL connecting to {self.host}") + + +class PostgreSQLConnection(DBConnection): + def __init__(self, host: str, user: str, password: str, database: str) -> None: + self.host = host + self.user = user + self.password = password + self.database = database + + def connect(self) -> None: + print(f"PostgreSQL connecting to {self.host}") +``` + +To support this, we can have a parallel config structure: +```text +conf/ +├── config.yaml +└── db + ├── mysql.yaml + └── postgresql.yaml +``` + +Config files: +
+ +
+ +```yaml title="db/mysql.yaml" +_target_: my_app.MySQLConnection +host: localhost +user: root +password: 1234 + +``` + +
+ +
+ +```yaml title="db/postgresql.yaml" +_target_: my_app.PostgreSQLConnection +host: localhost +user: root +password: 1234 +database: tutorial +``` + +
+
+ + +```yaml title="config.yaml" +defaults: + - db: mysql +``` + + + +With this, you can instantiate the object from the configuration with a single line of code: +```python +@hydra.main(version_base=None, config_path="conf", config_name="config") +def my_app(cfg): + connection = hydra.utils.instantiate(cfg.db) + connection.connect() +``` \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/advanced/instantiate_objects/overview.md b/website/versioned_docs/version-1.3/advanced/instantiate_objects/overview.md new file mode 100644 index 00000000000..bb9ab08b218 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/instantiate_objects/overview.md @@ -0,0 +1,432 @@ +--- +id: overview +title: Instantiating objects with Hydra +sidebar_label: Overview +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +One of the best ways to drive different behavior in an application is to instantiate different implementations of an interface. +The code using the instantiated object only knows the interface which remains constant, but the behavior +is determined by the actual object instance. + +Hydra provides `hydra.utils.instantiate()` (and its alias `hydra.utils.call()`) for instantiating objects and calling functions. Prefer `instantiate` for creating objects and `call` for invoking functions. + +Call/instantiate supports: +- Constructing an object by calling the `__init__` method +- Calling functions, static functions, class methods and other callable global objects + +
Instantiate API (Expand for details) + +```python +def instantiate(config: Any, *args: Any, **kwargs: Any) -> Any: + """ + :param config: An config object describing what to call and what params to use. + In addition to the parameters, the config must contain: + _target_ : target class or callable name (str) + And may contain: + _args_: List-like of positional arguments to pass to the target + _recursive_: Construct nested objects as well (bool). + True by default. + may be overridden via a _recursive_ key in + the kwargs + _convert_: Conversion strategy + none : Passed objects are DictConfig and ListConfig, default + partial : Passed objects are converted to dict and list, with + the exception of Structured Configs (and their fields). + object : Passed objects are converted to dict and list. + Structured Configs are converted to instances of the + backing dataclass / attr class. + all : Passed objects are dicts, lists and primitives without + a trace of OmegaConf containers. Structured configs + are converted to dicts / lists too. + _partial_: If True, return functools.partial wrapped method or object + False by default. Configure per target. + :param args: Optional positional parameters pass-through + :param kwargs: Optional named parameters to override + parameters in the config object. Parameters not present + in the config objects are being passed as is to the target. + IMPORTANT: dataclasses instances in kwargs are interpreted as config + and cannot be used as passthrough + :return: if _target_ is a class name: the instantiated object + if _target_ is a callable: the return value of the call + """ + +# Alias for instantiate +call = instantiate +``` + +

+ +The config passed to these functions must have a key called `_target_`, with the value of a fully qualified class name, class method, static method or callable. +For convenience, `None` config results in a `None` object. + +**Named arguments** : Config fields (except reserved fields like `_target_`) are passed as named arguments to the target. +Named arguments in the config can be overridden by passing named argument with the same name in the `instantiate()` call-site. + +**Positional arguments** : The config may contain a `_args_` field representing positional arguments to pass to the target. +The positional arguments can be overridden together by passing positional arguments in the `instantiate()` call-site. + + + +### Simple usage +Your application might have an Optimizer class: +```python title="Example class" +class Optimizer: + algo: str + lr: float + + def __init__(self, algo: str, lr: float) -> None: + self.algo = algo + self.lr = lr +``` + +
+ +
+ +```yaml title="Config" +optimizer: + _target_: my_app.Optimizer + algo: SGD + lr: 0.01 + + + + +``` + + +
+ +
+ +```python title="Instantiation" +opt = instantiate(cfg.optimizer) +print(opt) +# Optimizer(algo=SGD,lr=0.01) + +# override parameters on the call-site +opt = instantiate(cfg.optimizer, lr=0.2) +print(opt) +# Optimizer(algo=SGD,lr=0.2) +``` + +
+
+ + +### Recursive instantiation +Let's add a Dataset and a Trainer class. The trainer holds a Dataset and an Optimizer instances. +```python title="Additional classes" +class Dataset: + name: str + path: str + + def __init__(self, name: str, path: str) -> None: + self.name = name + self.path = path + + +class Trainer: + def __init__(self, optimizer: Optimizer, dataset: Dataset) -> None: + self.optimizer = optimizer + self.dataset = dataset +``` + +With the following config, you can instantiate the whole thing with a single call: +```yaml title="Example config" +trainer: + _target_: my_app.Trainer + optimizer: + _target_: my_app.Optimizer + algo: SGD + lr: 0.01 + dataset: + _target_: my_app.Dataset + name: Imagenet + path: /datasets/imagenet +``` + +Hydra will instantiate nested objects recursively by default. +```python +trainer = instantiate(cfg.trainer) +print(trainer) +# Trainer( +# optimizer=Optimizer(algo=SGD,lr=0.01), +# dataset=Dataset(name=Imagenet, path=/datasets/imagenet) +# ) +``` +You can override parameters for nested objects: +```python +trainer = instantiate( + cfg.trainer, + optimizer={"lr": 0.3}, + dataset={"name": "cifar10", "path": "/datasets/cifar10"}, +) +print(trainer) +# Trainer( +# optimizer=Optimizer(algo=SGD,lr=0.3), +# dataset=Dataset(name=cifar10, path=/datasets/cifar10) +# ) +``` + +Similarly, positional arguments of nested objects can be overridden: +```python +obj = instantiate( + cfg.object, + # pass 1 and 2 as positional arguments to the target object + 1, 2, + # pass 3 and 4 as positional arguments to a nested child object + child={"_args_": [3, 4]}, +) +``` + +### Disable recursive instantiation +You can disable recursive instantiation by setting `_recursive_` to `False` in the config node or in the call-site +In that case the Trainer object will receive an OmegaConf DictConfig for nested dataset and optimizer instead of the instantiated objects. +```python +optimizer = instantiate(cfg.trainer, _recursive_=False) +print(optimizer) +``` + +Output: +```python +Trainer( + optimizer={ + '_target_': 'my_app.Optimizer', 'algo': 'SGD', 'lr': 0.01 + }, + dataset={ + '_target_': 'my_app.Dataset', 'name': 'Imagenet', 'path': '/datasets/imagenet' + } +) +``` + +### Parameter conversion strategies +By default, the parameters passed to the target are either primitives (int, +float, bool etc) or OmegaConf containers (`DictConfig`, `ListConfig`). +OmegaConf containers have many advantages over primitive dicts and lists, +including convenient attribute access for keys, +[duck-typing as instances of dataclasses or attrs classes](https://omegaconf.readthedocs.io/en/latest/structured_config.html), and +support for [variable interpolation](https://omegaconf.readthedocs.io/en/latest/usage.html#variable-interpolation) +and [custom resolvers](https://omegaconf.readthedocs.io/en/latest/custom_resolvers.html). +If the callable targeted by `instantiate` leverages OmegaConf's features, it +will make sense to pass `DictConfig` and `ListConfig` instances directly to +that callable. + +That being said, in many cases it's desired to pass normal Python dicts and +lists, rather than `DictConfig` or `ListConfig` instances, as arguments to your +callable. You can change instantiate's argument conversion strategy using the +`_convert_` parameter. Supported values are: + +- `"none"` : Default behavior, Use OmegaConf containers +- `"partial"` : Convert OmegaConf containers to dict and list, except + Structured Configs, which remain as DictConfig instances. +- `"object"` : Convert OmegaConf containers to dict and list, except Structured + Configs, which are converted to instances of the backing dataclass / attr + class using `OmegaConf.to_object`. +- `"all"` : Convert everything to primitive containers + +The conversion strategy applies recursively to all subconfigs of the instantiation target. +Here is an example demonstrating the various conversion strategies: + +```python +from dataclasses import dataclass +from omegaconf import DictConfig, OmegaConf +from hydra.utils import instantiate + +@dataclass +class Foo: + a: int = 123 + +class MyTarget: + def __init__(self, foo, bar): + self.foo = foo + self.bar = bar + +cfg = OmegaConf.create( + { + "_target_": "__main__.MyTarget", + "foo": Foo(), + "bar": {"b": 456}, + } +) + +obj_none = instantiate(cfg, _convert_="none") +assert isinstance(obj_none, MyTarget) +assert isinstance(obj_none.foo, DictConfig) +assert isinstance(obj_none.bar, DictConfig) + +obj_partial = instantiate(cfg, _convert_="partial") +assert isinstance(obj_partial, MyTarget) +assert isinstance(obj_partial.foo, DictConfig) +assert isinstance(obj_partial.bar, dict) + +obj_object = instantiate(cfg, _convert_="object") +assert isinstance(obj_object, MyTarget) +assert isinstance(obj_object.foo, Foo) +assert isinstance(obj_object.bar, dict) + +obj_all = instantiate(cfg, _convert_="all") +assert isinstance(obj_none, MyTarget) +assert isinstance(obj_all.foo, dict) +assert isinstance(obj_all.bar, dict) +``` + +Passing the `_convert_` keyword argument to `instantiate` has the same effect as defining +a `_convert_` attribute on your config object. Here is an example creating +instances of `MyTarget` that are equivalent to the above: + +```python +cfg_none = OmegaConf.create({..., "_convert_": "none"}) +obj_none = instantiate(cfg_none) + +cfg_partial = OmegaConf.create({..., "_convert_": "partial"}) +obj_partial = instantiate(cfg_partial) + +cfg_object = OmegaConf.create({..., "_convert_": "object"}) +obj_object = instantiate(cfg_object) + +cfg_all = OmegaConf.create({..., "_convert_": "all"}) +obj_all = instantiate(cfg_all) +``` + +### Partial Instantiation +Sometimes you may not set all parameters needed to instantiate an object from the configuration, in this case you can set +`_partial_` to be `True` to get a `functools.partial` wrapped object or method, then complete initializing the object in +the application code. Here is an example: + +```python title="Example classes" +class Optimizer: + algo: str + lr: float + + def __init__(self, algo: str, lr: float) -> None: + self.algo = algo + self.lr = lr + + def __repr__(self) -> str: + return f"Optimizer(algo={self.algo},lr={self.lr})" + + +class Model: + def __init__(self, optim_partial: Any, lr: float): + super().__init__() + self.optim = optim_partial(lr=lr) + self.lr = lr + + def __repr__(self) -> str: + return f"Model(Optimizer={self.optim},lr={self.lr})" +``` + +
+ +
+ +```yaml title="Config" +model: + _target_: my_app.Model + optim_partial: + _partial_: true + _target_: my_app.Optimizer + algo: SGD + lr: 0.01 +``` + + +
+ +
+ +```python title="Instantiation" +model = instantiate(cfg.model) +print(model) +# "Model(Optimizer=Optimizer(algo=SGD,lr=0.01),lr=0.01) +``` + +
+
+ +If you are repeatedly instantiating the same config, +using `_partial_=True` may provide a significant speedup as compared with regular (non-partial) instantiation. +```python +factory = instantiate(config, _partial_=True) +obj = factory() +``` +In the above example, repeatedly calling `factory` would be faster than repeatedly calling `instantiate(config)`. +A caveat of this approach is that the same keyword arguments would be re-used in each call to `factory`. +```python +class Foo: + ... + +class Bar: + def __init__(self, foo): + self.foo = foo + +bar_conf = { + "_target_": "__main__.Bar", + "foo": {"_target_": "__main__.Foo"}, +} + +bar_factory = instantiate(bar_conf, _partial_=True) +bar1 = bar_factory() +bar2 = bar_factory() + +assert bar1 is not bar2 +assert bar1.foo is bar2.foo # the `Foo` instance is re-used here +``` +This does not apply if `_partial_=False`, +in which case a new `Foo` instance would be created with each call to `instantiate`. + + +### Instantiation of builtins + +The value of `_target_` passed to `instantiate` should be a "dotpath" pointing +to some callable that can be looked up via a combination of `import` and `getattr`. +If you want to target one of Python's [built-in functions](https://docs.python.org/3/library/functions.html) (such as `len` or `print` or `divmod`), +you will need to provide a dotpath looking up that function in Python's [`builtins`](https://docs.python.org/3/library/builtins.html) module. +```python +from hydra.utils import instantiate +# instantiate({"_target_": "len"}, [1,2,3]) # this gives an InstantiationException +instantiate({"_target_": "builtins.len"}, [1,2,3]) # this works, returns the number 3 +``` + +### Dotpath lookup machinery + +Hydra looks up a given `_target_` by attempting to find a module that +corresponds to a prefix of the given dotpath and then looking for an object in +that module corresponding to the dotpath's tail. For example, to look up a `_target_` +given by the dotpath `"my_module.my_nested_module.my_object"`, hydra first locates +the module `my_module.my_nested_module`, then find `my_object` inside that nested module. + +Hydra exposes an API allowing direct use of this dotpath lookup machinery. +The following two functions, which can be imported from the hydra.utils module, +accept a string-typed dotpath as an argument and return the located class/callable/object: +```python +def get_class(path: str) -> type: + """ + Look up a class based on a dotpath. + Fails if the path does not point to a class. + + >>> import my_module + >>> from hydra.utils import get_class + >>> assert get_class("my_module.MyClass") is my_module.MyClass + """ + ... + +def get_method(path: str) -> Callable[..., Any]: + """ + Look up a callable based on a dotpath. + Fails if the path does not point to a callable object. + + >>> import my_module + >>> from hydra.utils import get_method + >>> assert get_method("my_module.my_function") is my_module.my_function + """ + ... + +# Alias for get_method +get_static_method = get_method +``` diff --git a/website/versioned_docs/version-1.3/advanced/instantiate_objects/structured_config.md b/website/versioned_docs/version-1.3/advanced/instantiate_objects/structured_config.md new file mode 100644 index 00000000000..892cd8f25c0 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/instantiate_objects/structured_config.md @@ -0,0 +1,95 @@ +--- +id: structured_config +title: Structured Configs example +sidebar_label: Structured Configs example +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +This example demonstrates the use of Structured Configs to instantiated objects. + +#### Example usage + +```python title="my_app.py" +class DBConnection: + def __init__(self, driver: str, host: str, port: int) -> None: + self.driver = driver + self.host = host + self.port = port + + def connect(self) -> None: + print(f"{self.driver} connecting to {self.host}") + +class MySQLConnection(DBConnection): + def __init__(self, driver: str, host: str, port: int) -> None: + super().__init__(driver=driver, host=host, port=port) + +class PostgreSQLConnection(DBConnection): + def __init__(self, driver: str, host: str, port: int, timeout: int) -> None: + super().__init__(driver=driver, host=host, port=port) + self.timeout = timeout + +@dataclass +class DBConfig: + driver: str = MISSING + host: str = "localhost" + port: int = 80 + +@dataclass +class MySQLConfig(DBConfig): + _target_: str = "my_app.MySQLConnection" + driver: str = "MySQL" + port: int = 1234 + +@dataclass +class PostGreSQLConfig(DBConfig): + _target_: str = "my_app.PostgreSQLConnection" + driver: str = "PostgreSQL" + port: int = 5678 + timeout: int = 10 + +@dataclass +class Config: + defaults: List[Any] = field(default_factory=lambda: [{"db": "mysql"}]) + db: DBConfig = MISSING + + +cs = ConfigStore.instance() +cs.store(name="config", node=Config) +cs.store(group="db", name="mysql", node=MySQLConfig) +cs.store(group="db", name="postgresql", node=PostGreSQLConfig) + +@hydra.main(version_base=None, config_name="config") +def my_app(cfg: Config) -> None: + connection = instantiate(cfg.db) + connection.connect() + +if __name__ == "__main__": + my_app() +``` + + +#### Sample Output + +
+ +
+ +```bash +$ python my_app.py +MySQL connecting to localhost:1234 +``` + +
+ +
+ +```bash +$ python my_app.py db=postgresql +PostgreSQL connecting to localhost:5678 +``` + +
+
diff --git a/website/versioned_docs/version-1.3/advanced/jupyter_notebooks.md b/website/versioned_docs/version-1.3/advanced/jupyter_notebooks.md new file mode 100644 index 00000000000..f64b36d4b0f --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/jupyter_notebooks.md @@ -0,0 +1,11 @@ +--- +id: jupyter_notebooks +title: Hydra in Jupyter Notebooks +--- + +Hydra supports config composition inside Jupyter notebooks via the [Compose API](compose_api.md). + +Run the Notebook in a the Binder to see a live demo, or open the Notebook source on GitHub. + +[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/facebookresearch/hydra/main?filepath=examples%2jupyter_notebooks) +[![Notebook source](https://img.shields.io/badge/-Notebooks%20source-informational)](https://github.com/facebookresearch/hydra//tree/main/examples/jupyter_notebooks/) diff --git a/website/versioned_docs/version-1.3/advanced/override_grammar/basic.md b/website/versioned_docs/version-1.3/advanced/override_grammar/basic.md new file mode 100644 index 00000000000..c2295b4c852 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/override_grammar/basic.md @@ -0,0 +1,330 @@ +--- +id: basic +hide_title: true +sidebar_label: Basic Override syntax +--- + +import GithubLink from "@site/src/components/GithubLink" + +## Basic Override syntax +You can manipulate your configuration with overrides (via the command line or the Compose API). This includes: +- Modifying the `Defaults List` +- Modifying the config object + +Overrides matching a config group are modifying the `Defaults List`; +The rest are manipulating the config object. + +## Basic examples +### Modifying the Config Object +- Overriding a config value : `foo.bar=value` +- Appending a config value : `+foo.bar=value` +- Appending or overriding a config value : `++foo.bar=value` +- Removing a config value : `~foo.bar`, `~foo.bar=value` + +### Modifying the Defaults List +- Overriding selected Option: `db=mysql`, `server/db=mysql` +- Appending to Defaults List: `+db=mysql`, `+server/db=mysql` +- Deleting from Defaults List: `~db`, `~db=mysql`, `~server/db`, `~server/db=mysql` + +## Grammar +Hydra supports a rich [DSL](https://en.wikipedia.org/wiki/Domain-specific_language) in the command line. +Below are the parser rules from grammar. You can see the full Lexer and Parser definitions on GitHub. + +```antlr4 title="OverrideParser.g4" +// High-level command-line override. + +override: ( + key EQUAL value? // key=value, key= (for empty value) + | TILDE key (EQUAL value?)? // ~key | ~key=value + | PLUS PLUS? key EQUAL value? // +key= | +key=value | ++key=value +) EOF; + +// Key: +key : packageOrGroup (AT package)?; // key | group@pkg + +packageOrGroup: package | ID (SLASH ID)+; // db, hydra/launcher +package: ( | ID | KEY_SPECIAL | DOT_PATH); // db, $db, hydra.launcher, or the empty (for _global_ package) + +// Elements (that may be swept over). + +value: element | simpleChoiceSweep; + +element: + primitive + | listContainer + | dictContainer + | function +; + +simpleChoiceSweep: + element (COMMA element)+ // value1,value2,value3 +; + +// Functions. + +argName: ID EQUAL; +function: ID POPEN (argName? element (COMMA argName? element )* )? PCLOSE; + +// Data structures. + +listContainer: BRACKET_OPEN // [], [1,2,3], [a,b,[1,2]] + (element(COMMA element)*)? +BRACKET_CLOSE; + +dictContainer: BRACE_OPEN (dictKeyValuePair (COMMA dictKeyValuePair)*)? BRACE_CLOSE; // {}, {a:10,b:20} +dictKeyValuePair: dictKey COLON element; + +// Primitive types. + +primitive: + QUOTED_VALUE // 'hello world', "hello world" + | ( ID // foo-bar_10 + | NULL // null, NULL + | INT // 0, 10, -20, 1_000_000 + | FLOAT // 3.14, -20.0, 1e-1, -10e3 + | BOOL // true, TrUe, false, False + | INTERPOLATION // ${foo.bar}, ${oc.env:USER,me} + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ?, | + | COLON // : + | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, + | WS // whitespaces + )+; + +// Same as `primitive` except that `COLON` and `INTERPOLATION` are not allowed. +dictKey: + ( ID // foo-bar_10 + | NULL // null, NULL + | INT // 0, 10, -20, 1_000_000 + | FLOAT // 3.14, -20.0, 1e-1, -10e3 + | BOOL // true, TrUe, false, False + | UNQUOTED_CHAR // /, -, \, +, ., $, %, *, @, ?, | + | ESC // \\, \(, \), \[, \], \{, \}, \:, \=, \ , \\t, \, + | WS // whitespaces + )+; +``` + +## Elements +### Key +Key is the component before the =. A few examples: +```shell script +foo.bar # A config key +hydra/launcher # A config group +group@pkg # A config group assigned to the package pkg +group@pkg1:pkg2 # A config group changing the package from pkg1 to pkg2 +``` + +### Quoted values +Hydra supports both double quotes and single quoted values. +Quoted strings can accept any value between the quotes, but some characters need escaping: +* to include a single quote in a single quoted string, use `\'` (for double quotes in a double quoted string, use `\"`) +* any sequence of `\` characters preceding a quote (either an escaped quote as described in the previous point, or the closing quote) + must be escaped by doubling the number of `\` + +
+
+ +```python title="Double quotes" +"hello there" +"escaped \"double quote\"" +"the path is C:\\\"some folder\"\\" +"1,2,3" +"{a:10} ${xyz}" +"'single quoted string'" +``` + +
+ +
+ +```python title="Single quotes" +'hello there' +'escaped \'single quote\'' +'the path is C:\\\'some folder\'\\' +'1,2,3' +'{a:10} ${xyz}' +'"double quoted string"' +``` +
+
+ +It may be necessary to use multiple pairs of quotes to prevent your +shell from consuming quotation marks before they are passed to hydra. + +```shell +$ python my_app.py '+foo="{a: 10}"' +foo: '{a: 10}' + +$ python my_app.py '+foo={a: 10}' +foo: + a: 10 + +``` + +Here are some best practices around quoting in CLI overrides: +- Quote the whole key=value pair with single quotes, as in the first two + examples above. These quotes are for the benefit of the shell. +- Do not quote keys. +- Only quote values if they contain a space. It will work if you always quote + values, but it will turn numbers/dicts/lists into strings (as in the first + example above). +- When you are quoting values, use double quotes to avoid collision with the + outer single quoted consumed by the shell. + +### Whitespaces in unquoted values +Unquoted Override values can contain non leading or trailing whitespaces. +For example, `msg=hello world` is a legal override (key is `msg` and value is the string `hello world`). +Normally, your shell will interpret values with whitespaces as being multiple parameters (`key=a b` would be interpreted as `key=a` and `b`). +To prevent this you can quote them with a single quote. For example: + +```shell +$ python my_app.py 'msg=hello world' +``` + +Note that trailing and leading whitespace are ignored, the above is equivalent to: + +```shell +$ python my_app.py 'msg= hello world ' +``` + +### Escaped characters in unquoted values +Hydra's parser considers some characters to be illegal in unquoted strings. +These otherwise special characters may be included in unquoted values by escaping them with a `\`. +These characters are: `\()[]{}:=, \t` (the last two ones being the whitespace and tab characters). + +As an example, in the following `dir` is set to the string `job{a=1,b=2,c=3}`: + +```shell +$ python my_app.py 'dir=job\{a\=1\,b\=2\,c\=3\}' +``` + +As an alternative to escaping special characters with a backslash, the value containing the special character may be quoted: + +```shell +$ python my_app.py 'dir=A[B' # parser error +$ python my_app.py 'dir="A[B"' # ok +$ python my_app.py 'dir=A\[B' # ok +``` + +### Primitives +- `id` : oompa10, loompa_12 +- `null`: null +- `int`: 10, -20, 0, 1_000_000. +- `float`: 3.14, -10e6, inf, -inf, nan. +- `bool`: true, false +- `dot_path`: foo.bar +- `interpolation`: ${foo.bar}, ${oc.env:USER,me} + +Constants (null, true, false, inf, nan) are case-insensitive. + +:::important +Always single-quote interpolations in the shell, to prevent replacement with shell variables: +```shell +$ python my_app.py 'dir=/root/${name}' +``` +In addition, more complex interpolations containing special characters may require being passed within a quoted value +(note the extra double quotes surrounding the interpolation): +```shell +$ python my_app.py 'dir="${get_dir: {root: /root, name: ${name}}}"' +``` +::: + +## Dictionaries and Lists + +### Lists +```python +foo=[1,2,3] +nested=[a,[b,[c]]] +``` + +### Dictionaries +```python +foo={a:10,b:20} +nested={a:10,b:{c:30,d:40}} +``` + +Dictionaries are merged, not assigned. The following example illustrates the point: +
+
+ +```yaml title="Input config" +db: + driver: mysql + user: ??? + pass: ??? +``` + +
+ +
+ + +```yaml title="db={user:root,pass:1234}" +db: + driver: mysql + user: root + pass: 1234 +``` + +
+
+ + +:::important +Always single-quote overrides that contains dicts and lists in the shell. +::: + +### Sweeper syntax +A choice sweep is comma separated list with two or more elements: +```shell script +key=a,b # Simple sweep: ChoiceSweep(a, b) +key="a,b","c,d" # Elements can be quoted strings, ChoiceSweep("a,b", "c,d") +key=[a,b],[c,d] # Elements can be real lists, ChoiceSweep([a,b], [c,d]) +key={a:10, b:20},{c:30,d:40} # And dictionaries: ChoiceSweep({a:10, b:20}, {c:30,d:40}) +``` +More sweeping options are described in the [Extended Grammar page](extended.md). + +:::important +You may need to quote your choice sweep in the shell. +::: + + +### Functions +Hydra supports several functions in the command line. +See the [Extended Grammar page](extended.md) for more information. + +## Working with your shell +All shells interprets command line inputs and may change what is passed to the process. +A good way to determine what the shell is doing to your command is to `echo` it. +```shell script +# bash output +$ echo foo_{a:10,b:20} ${HOME} [b,c]* +foo_a:10 foo_b:20 /home/omry build_helpers +$ echo 'foo_{a:10,b:20}' '${HOME}' '[b,c]*' +foo_{a:10,b:20} ${HOME} [b,c]* +``` + +If in doubt, quote a command line element with a **single quote** (`'`). + +If you want to pass quotes to Hydra in a shell quoted string, it's best to pass double quotes. +```shell script +$ echo '"hello world"' +"hello world" +``` + +You can use some shell specific commands to change their behavior, but the cost will be that their behavior will change. +### Bash +You can disable braces expansion, filename generation (globing) and hist expansion. Please note that this will change +your shell behavior for the current session. +```shell script +$ set +o braceexpand -o noglob +o histexpand +$ echo key1={a:10,b:20} key2=${HOME} key=[b]* +key1={a:10,b:20} key2=/home/omry key=[b]* +# does not help with () though: +$ echo key=choice(a,b,c) +bash: syntax error near unexpected token '(' +$ echo 'key=choice(a,b,c)' +key=choice(a,b,c) +``` + +### Other shells +Send a PR to add information about your favorite shell here. diff --git a/website/versioned_docs/version-1.3/advanced/override_grammar/extended.md b/website/versioned_docs/version-1.3/advanced/override_grammar/extended.md new file mode 100644 index 00000000000..d0d266c97da --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/override_grammar/extended.md @@ -0,0 +1,383 @@ +--- +id: extended +sidebar_label: Extended Override syntax +hide_title: true +--- + +## Extended Override syntax +Hydra Overrides supports functions. +When calling a function, one can optionally name parameters. This is following the Python +convention of naming parameters. + +
+
+ +```python title="Example function" +def func(a:int, b:str) -> bool: + ... + + +``` +
+ +
+ +```python title="Calling function" +func(10,foo) # Positional only +func(a=10,b=foo) # Named only +func(10,b=foo) # Mixed +func(a=10,foo) # Error +``` + +
+
+ + +Note the lack of quotes in the examples above. Despite some similarities, this is **not Python**. + +:::important +Hydra supports very specific functions. If you would like to have +another function added, please file an issue and explain the use case. +::: + +## Sweeps +Sweep overrides are used by Sweepers to determine what to do. For example, +one can instruct the Basic Sweeper to sweep over all combinations of the +ranges `num1=range(0,3)` and `num2=range(0,3)` - resulting in `9` jobs, each getting a +different pair of numbers from `0`, `1` and `2`. + +### Choice sweep +```python title="Signature" +def choice( + *args: Union[str, int, float, bool, Dict[Any, Any], List[Any], ChoiceSweep] +) -> ChoiceSweep: + """ + A choice sweep over the specified values + """ +``` +Choice sweeps are the most common sweeps. +A choice sweep is described in one of two equivalent forms. +```python title="Examples" +db=mysql,postgresql # a comma separated list of two or more elements. +db=choice(mysql,postgresql) # choice +``` +### Glob choice sweep +```python title="Signature" +def glob( + include: Union[List[str], str], exclude: Optional[Union[List[str], str]] = None +) -> Glob: + """ + A glob selects from all options in the config group. + inputs are in glob format. e.g: *, foo*, *foo. + :param include: a string or a list of strings to use as include globs + :param exclude: a string or a list of strings to use as exclude globs + :return: A Glob object + """ +``` +Assuming the config group **schema** with the options **school**, **support** and **warehouse**: +```python title="Examples" +schema=glob(*) # school,support,warehouse +schema=glob(*,exclude=support) # school,warehouse +schema=glob([s*,w*],exclude=school) # support,warehouse +``` + +### Range sweep +Unlike Python, Hydra's range can be used with both integer and floating-point numbers. +In both cases, the range represents a discrete list of values. +```python title="Signature" +def range( + start: Union[int, float], stop: Optional[Union[int, float]] = None, step: Union[int, float] = 1 +) -> RangeSweep: + """ + Range is defines a sweeep over a range of integer or floating-point values. + For a positive step, the contents of a range r are determined by the formula + r[i] = start + step*i where i >= 0 and r[i] < stop. + For a negative step, the contents of the range are still determined by the formula + r[i] = start + step*i, but the constraints are i >= 0 and r[i] > stop. + """ +``` + +```python title="Examples" +num=range(5) # 0,1,2,3,4 +num=range(0,5) # 0,1,2,3,4 +num=range(0,5,2) # 0,2,4 +num=range(0,10,3.3) # 0.0,3.3,6.6,9.9 +num=range(-5,step=-1) # 0,-1,-2,-3,-4 +``` + +### Interval sweep +An interval sweep represents all the floating point value between two values. +This is used by optimizing sweepers like Ax and Nevergrad. The basic sweeper does not support interval. + +```python title="Signature" +def interval(start: Union[int, float], end: Union[int, float]) -> IntervalSweep: + """ + A continuous interval between two floating point values. + value=interval(x,y) is interpreted as x <= value < y + """ +``` +```python title="Examples" +interval(1.0,5.0) # 1.0 <= x < 5.0 +interval(1,5) # 1.0 <= x < 5.0, auto-cast to floats +``` + +### Tag +With tags you can add arbitrary metadata to a sweep. The metadata can be used by advanced sweepers. +```python title="Signature" +def tag(*args: Union[str, Union[Sweep]], sweep: Optional[Sweep] = None) -> Sweep: + """ + Tags the sweep with a list of string tags. + """ +``` +```python title="Examples" +tag(log,interval(0,1)) # 1.0 <= x < 1.0, tags=[log] +tag(foo,bar,interval(0,1)) # 1.0 <= x < 1.0, tags=[foo,bar] +``` + +## Reordering lists and sweeps + +### sort +```python title="Signature" +def sort( + *args: Union[ElementType, ChoiceSweep, RangeSweep], + sweep: Optional[Union[ChoiceSweep, RangeSweep]] = None, + list: Optional[List[Any]] = None, + reverse: bool = False, +) -> Any: + """ + Sort an input list or sweep. + reverse=True reverses the order + """ +``` +```python title="Examples" +# sweep +sort(1,3,2) # ChoiceSweep(1,2,3) +sort(1,3,2,reverse=true) # ChoiceSweep(3,2,1) +sort(choice(1,2,3)) # ChoiceSweep(1,2,3) +sort(sweep=choice(1,2,3)) # ChoiceSweep(1,2,3) +sort(choice(1,2,3),reverse=true) # ChoiceSweep(3,2,1) +sort(range(10,1)) # range in ascending order +sort(range(1,10),reverse=true) # range in descending order + +# lists +sort([1,3,2]) # [1,2,3] +sort(list=[1,3,2]) # [1,2,3] +sort(list=[1,3,2], reverse=true) # [3,2,1] + +# single value returned as is +sort(1) # 1 +``` + +### shuffle +```python title="Signature" +def shuffle( + *args: Union[ElementType, ChoiceSweep, RangeSweep], + sweep: Optional[Union[ChoiceSweep, RangeSweep]] = None, + list: Optional[List[Any]] = None, +) -> Union[List[Any], ChoiceSweep, RangeSweep]: + """ + Shuffle input list or sweep (does not support interval) + """ +``` +```python title="Examples" +shuffle(a,b,c) # shuffled a,b,c +shuffle(choice(a,b,c)), shuffle(sweep=choice(a,b,c)) # shuffled choice(a,b,c) +shuffle(range(1,10)) # shuffled range(1,10) +shuffle([a,b,c]), shuffle(list=[a,b,c]) # shuffled list [a,b,c] +``` + +## Type casting +You can cast values and sweeps to `int`, `float`, `bool` or `str`. +```python title="Example" +int(3.14) # 3 (int) +int(value=3.14) # 3 (int) +float(10) # 10.0 (float) +str(10) # "10" (str) +bool(1) # true (bool) +float(range(1,10)) # range(1.0,10.0) +str([1,2,3]) # ['1','2','3'] +str({a:10}) # {a:'10'} +``` + +Below are pseudo code snippets that illustrates the differences between Python's casting and Hydra's casting. + +#### Casting string to bool + +
+
+ +```python title="Python" +def bool(value: Any) -> bool: + if isinstance(value, str): + return len(value) > 0 + else: + return bool(value) + + + + +``` +
+ +
+ +```python title="Hydra" +def bool(s: str) -> bool: + if isinstance(value, str): + if value.lower() == "false": + return False + elif value.lower() == "true": + return True + else: + raise ValueError() + return bool(value) +``` + +
+
+ +#### Casting lists +Casting lists results in a list where each element is recursively cast. +Failure to cast an element in the list fails the cast of the list. + +
+
+ +```python title="Python" +def cast_int(value: Any): + if isinstance(value, list): + raise TypeError() + else: + return int(v) + + +``` +
+ +
+ +```python title="Hydra" +def cast_int(value: Any): + if isinstance(v, list): + return list(map(cast_int, v)) + else: + return int(v) + + +``` + +
+
+ + +#### Casting dicts +Casting dicts results in a dict where values are recursively cast, but keys are unchanged. +Failure to cast a value in the dict fails the cast of the entire dict. + +
+
+ +```python title="Python" +def cast_int(value: Any): + if isinstance(value, dict): + raise TypeError() + else: + return int(v) + + +``` +
+ +
+ +```python title="Hydra" +def cast_int(value: Any): + if isinstance(value, dict): + return apply_to_values( + value, cast_int + ) + else: + return int(v) +``` + +
+
+ +#### Casting ranges +Ranges can be cast to float or int, resulting in start, stop and step being cast and thus the range elements being cast. + +
+
+ +```python title="Python" +def cast_int(value: Any): + if isinstance(value, RangeSweep): + raise TypeError() + else: + return int(v) + + + + +``` +
+ +
+ +```python title="Hydra" +def cast_int(value: Any): + if isinstance(value, RangeSweep): + return RangeSweep( + start=cast_int(value.start), + stop=cast_int(value.stop), + step=cast_int(value.step), + ) + else: + return int(v) +``` + +
+
+ +### Conversion matrix +Below is the conversion matrix from various inputs to all supported types. +Input are grouped by type. + + +[//]: # (Conversion matrix source: https://docs.google.com/document/d/1JDZGHKk4PrZHqsTTS6ao-DQOu2eVD4ULR6uAxVUR-WI/edit#) + +| | int() | float() | str() | bool() | +|-------------------- |------------- |------------------- |------------------- |----------------------- | +| 10 | 10 | 10.0 | “10” | true | +| 0 | 0 | 0.0 | “0” | false | +| 10.0 | 10 | 10.0 | “10.0” | true | +| 0.0 | 0 | 0.0 | “0.0” | false | +| inf | error | inf | ‘inf’ | true | +| nan | error | nan | ‘nan’ | true | +| 1e6 | 1,000,000 | 1e6 | ‘1000000.0’ | true | +| foo | error | error | foo | error | +| “” (empty string) | error | error | “” | error | +| “10” | 10 | 10.0 | “10” | error | +| “10.0” | error | 10.0 | “10.0” | error | +| “true” | error | error | “true” | true | +| “false” | error | error | “false” | false | +| “[1,2,3]” | error | error | “[1,2,3]” | error | +| “{a:10}” | error | error | “{a:10}” | error | +| true | 1 | 1.0 | “true” | true | +| false | 0 | 0.0 | “false” | false | +| [] | [] | [] | [] | [] | +| [0,1,2] | [0,1,2] | [0.0,1.0,2.0] | [“0”,”1”,”2”] | [false,true,true] | +| [1,[2]] | [1,[2]] | [1.0,[2.0]] | [“1”,[“2”]] | [true,[true]] | +| [a,1] | error | error | [“a”,”1”] | error | +| {} | {} | {} | {} | {} | +| {a:10} | {a:10} | {a:10.0} | {a:”10”} | {a: true} | +| {a:[0,1,2]} | {a:[0,1,2]} | {a:[0.0,1.0,2.-]} | {a:[“0”,”1”,”2”]} | {a:[false,true,true]} | +| {a:10,b:xyz} | error | error | {a:”10”,b:”xyz”} | error | +| choice(0,1) | choice(0,1) | choice(0.0,1.0) | choice(“0”,“1”) | choice(false,true) | +| choice(a,b) | error | error | choice(“a”,”b”) | error | +| choice(1,a) | error | error | choice(“1”,”a”) | error | +| interval(1.0, 2.0) | interval(1, 2)| interval(1.0, 2.0) | error | error | +| interval(1, 2) | interval(1, 2)| interval(1.0, 2.0) | error | error | +| range(1,10) | range(1,10) | range(1.0,10.0) | error | error | +| range(1.0, 10.0) | range(1,10) | range(1.0,10.0) | error | error | + diff --git a/website/versioned_docs/version-1.3/advanced/overriding_packages.md b/website/versioned_docs/version-1.3/advanced/overriding_packages.md new file mode 100644 index 00000000000..fc278c42048 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/overriding_packages.md @@ -0,0 +1,175 @@ +--- +id: overriding_packages +title: Packages +--- + +The package determines where the content of each input config is placed in the output config. +The default package of an input config is derived from its Config Group. e.g. The default package of `server/db/mysql.yaml` is `server.db`. + +The default package can be overridden [in the Defaults List](#overriding-packages-using-the-defaults-list) +or via a [Package Directive](#overriding-the-package-via-the-package-directive) at the top of the config file. +Changing the package of a config can be useful when using a config from another library, or when using the same +config group twice in the same app. + +The priority for determining the final package for a config is as follows: +1. The package specified in the Defaults List (relative to the package of the including config) +2. The package specified in the Package Directive (absolute) +3. The default package + +We will use the following configs in the examples below: +
+
+ +```yaml title="config.yaml" +defaults: + - server/apache + +debug: false + + + +``` +
+ +
+ +```yaml title="server/apache.yaml" +defaults: + - db: mysql + +name: apache + + + +``` +
+ +
+ +```yaml title="server/db/mysql.yaml" +name: mysql +``` + +```yaml title="server/db/sqlite.yaml" +name: sqlite +``` +
+ +```text title="Config directory structure" +├── server +│ ├── db +│ │ ├── mysql.yaml +│ │ └── sqlite.yaml +│ └── apache.yaml +└── config.yaml +``` + + +### An example using only default packages +The default package of *config.yaml* is the global package, of *server/apache.yaml* is *server* and of *server/db/mysql.yaml* is *server.db*. +
+
+ +```yaml title="$ python my_app.py" {1-2} +server: + db: + name: mysql + name: apache +debug: false +``` +
+ +### Overriding packages using the Defaults List +By default, packages specified in the Defaults List are relative to the package of containing config. +As a consequence, overriding a package relocates the entire subtree. + +
+
+ +```yaml title="config.yaml" {2} +defaults: + - server/apache@admin + +debug: false + +``` +
+
+ +```yaml title="server/apache.yaml" {2} +defaults: + - db@backup: mysql + +name: apache + +``` +
+
+ +```yaml title="Output config" {1-4} +admin: + backup: + name: mysql + name: apache +debug: false +``` +
+ +Note that content of *server/apache.yaml* is relocated to *admin* +and the content of *server/db/mysql.yaml* to *admin.backup*. + +#### Default List package keywords +We will use this example, replacing *<@PACKAGE>* to demonstrate different cases: +```yaml title="config_group/config.yaml" +defaults: + - /server/db<@PACKAGE>: mysql +``` + +Without a package override, the resulting package is `config_group.server.db`. +With the **@\_here\_** keyword, The resulting package is the same as the containing config (`config_group`). +##### Absolute keywords: +* **@\_group\_**: \_group\_ is the absolute default package of the config (`server.db`) +* **@\_global\_**: The global package. Anything following \_global\_ is absolute. + e.g. **@\_global\_.foo** becomes `foo`. + +### Overriding the package via the package directive + +The @package directive changes the package of a config file. The package specified by a @package directive is always absolute. + +```yaml title="server/db/mysql.yaml" {1} +# @package foo.bar +name: mysql +``` + +To change the package to the global (empty) package, use the keyword `_global_`. + +### Using a config group more than once +The following example adds the `server/db/mysql` config in the packages `src` and `dst`. + +
+
+ +```yaml title="config.yaml" +defaults: + - server/db@src: mysql + - server/db@dst: mysql + +``` +
+ +```yaml title="$ python my_app.py" +src: + name: mysql +dst: + name: mysql +``` +
+ +When overriding config groups with a non-default package, the package must be used: +```yaml title="$ python my_app.py server/db@src=sqlite" +src: + name: sqlite +dst: + name: mysql +``` + diff --git a/website/versioned_docs/version-1.3/advanced/packaging.md b/website/versioned_docs/version-1.3/advanced/packaging.md new file mode 100644 index 00000000000..5f82a5dde6b --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/packaging.md @@ -0,0 +1,37 @@ +--- +id: app_packaging +title: Application packaging +sidebar_label: Application packaging +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + + + +You can package your Hydra application along with its configuration. +An example standalone application is included in the repo. + +Run it with: +```yaml +$ python examples/advanced/hydra_app_example/hydra_app/main.py +dataset: + name: imagenet + path: /datasets/imagenet +``` + +Install it with: +```text +$ pip install examples/advanced/hydra_app_example +... +Successfully installed hydra-app-0.1 +``` + +Once installed, run the installed app with: +```yaml +$ hydra_app +dataset: + name: imagenet + path: /datasets/imagenet +``` + +The installed app will use the packaged configuration files. diff --git a/website/versioned_docs/version-1.3/advanced/plugins/develop.md b/website/versioned_docs/version-1.3/advanced/plugins/develop.md new file mode 100644 index 00000000000..d4b9805d350 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/plugins/develop.md @@ -0,0 +1,66 @@ +--- +id: develop +title: Plugin development +sidebar_label: Plugin development +--- + +:::info +If you develop plugins, please join the Plugin developer announcement chat channel. +::: + + +import GithubLink from "@site/src/components/GithubLink" + +Hydra plugins must be registered before they can be used. There are two ways to register a plugin: +- via the automatic plugin discovery process, which discovers plugins located in the `hydra_plugins` namespace package +- by calling the `register` method on Hydra's `Plugins` singleton class + +## Automatic Plugin discovery process + +If you create a Plugin and want it to be discovered automatically by Hydra, keep the following things in mind: +- Hydra plugins can be either a standalone Python package, or a part of your existing Python package. + In both cases - They should be in the namespace module `hydra_plugins` (This is a top level module, Your plugin will __NOT__ be discovered if you place it in `mylib.hydra_plugins`). +- Do __NOT__ place an `__init__.py` file in `hydra_plugins` (doing so may break other installed Hydra plugins). + +The plugin discovery process runs whenever Hydra starts. During plugin discovery, Hydra scans for plugins in all the submodules of `hydra_plugins`. Hydra will import each module and look for plugins defined in that module. +Any module under `hydra_plugins` that is slow to import will slow down the startup of __ALL__ Hydra applications. +Plugins with expensive imports can exclude individual files from Hydra's plugin discovery process by prefixing them with `_` (but not `__`). +For example, the file `_my_plugin_lib.py` would not be imported and scanned, while `my_plugin_lib.py` would be. + +## Plugin registration via the `Plugins.register` method + +Plugins can be manually registered by calling the `register` method on the instance of Hydra's `Plugins` singleton class. +```python +from hydra.core.plugins import Plugins +from hydra.plugins.plugin import Plugin + +class MyPlugin(Plugin): + ... + +def register_my_plugin() -> None: + """Hydra users should call this function before invoking @hydra.main""" + Plugins.instance().register(MyPlugin) +``` + +## Getting started + +The best way to get started developing a Hydra plugin is to base your new plugin on one of the example plugins: +- Copy the subtree of the relevant example plugin into a standalone project. +- Edit `setup.py`, rename the plugin module, for example from `hydra_plugins.example_xyz_plugin` to `hydra_plugins.my_xyz_plugin`. +- Install the new plugin (Run this in the plugin directory: `pip install -e .`) +- Run the included example app and make sure that the plugin is discovered: +```shell +$ python example/my_app.py --info plugins +Installed Hydra Plugins +*********************** + ... + Launcher: + --------- + MyLauncher + ... +``` +- Run the example application to see that that your plugin is doing something. +- *[Optional]* If you want the plugin be embedded in your existing application/library, move the `hydra_plugins` directory + and make sure that it's included as a namespace module in your final Python package. See the `setup.py` + file included with the example plugin for hints (typically this involves using `find_namespace_packages(include=["hydra_plugins.*"])`). +- Hack on your plugin, Ensure that the recommended tests and any tests you want to add are passing. diff --git a/website/versioned_docs/version-1.3/advanced/plugins/intro.md b/website/versioned_docs/version-1.3/advanced/plugins/intro.md new file mode 100644 index 00000000000..6c7524cec40 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/plugins/intro.md @@ -0,0 +1,47 @@ +--- +id: overview +title: Plugins Overview +sidebar_label: Plugins Overview +--- + +import GithubLink from "@site/src/components/GithubLink" + +Hydra can be extended via plugins. +The example plugins here can help you get started with plugin development. + +## Plugin types +Hydra has several plugin types: +### Sweeper +A sweeper is responsible for converting command line arguments list into multiple jobs. +For example, the basic built-in sweeper takes arguments like: +``` +batch_size=128 optimizer=nesterov,adam learning_rate=0.01,0.1 +``` + +And creates 4 jobs with the following parameters: +``` +batch_size=128 optimizer=nesterov learning_rate=0.01 +batch_size=128 optimizer=nesterov learning_rate=0.1 +batch_size=128 optimizer=adam learning_rate=0.01 +batch_size=128 optimizer=adam learning_rate=0.1 +``` + +### Launcher +Launchers are responsible for launching a job to a specific environment. +A Launcher takes a batch of argument lists like the one above and launches a job for each one. +The job uses those arguments to compose its configuration. +The basic launcher simply launches the job locally. + +### SearchPathPlugin +A config path plugin can manipulate the search path. +This can be used to influence the default Hydra configuration to be more appropriate to a specific environment, +or just add new entries to the search path to make more configurations available to the Hydra app. + +SearchPathPlugin plugins are discovered automatically by Hydra and are being called to manipulate the search path before +the configuration is composed. + +Many other plugins also implement SearchPathPlugin to add their configuration to the config search path once they are installed. + +### ConfigSource +ConfigSource plugins can be used to allow Hydra to access configuration in non-standard locations when composing the config. +This can be used to enable access to an in-house private config store, or as a way to access configs from public sources like GitHub or S3. diff --git a/website/versioned_docs/version-1.3/advanced/search_path.md b/website/versioned_docs/version-1.3/advanced/search_path.md new file mode 100644 index 00000000000..4ca261d3dce --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/search_path.md @@ -0,0 +1,165 @@ +--- +id: search_path +title: Config Search Path +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + +The Config Search Path is a list of paths that Hydra searches in order to find **non-primary** configs. It is +similar to the Python `PYTHONPATH`. + - When a config is requested, The first matching config in the search path is used. + - Each search path element has a schema prefix such as `file://` or `pkg://` that corresponds to a `ConfigSourcePlugin`. + - `file://` points to a file system path. It can either be an absolute path or a relative path. + Relative path will be resolved to absolute based on the current working dir. Path separator is `/` on all Operating + Systems. + - `pkg://` points to an importable Python module, with `.` being the separator. `__init__.py` files are needed in + directories for Python to treat them as packages. + +You can inspect the search path and the configurations loaded by Hydra via the `--info` flag: + +```bash +$ python my_app.py --info searchpath +``` + +There are a few ways to modify the config search path, enabling Hydra to access configuration in +different locations. +Use a combination of the methods described below: + +#### Using `@hydra.main()` +Using the `config_path` parameter `@hydra.main()`. The `config_path` is relative to location of the Python script. + +#### Overriding `hydra.searchpath` config + + + +In some cases you may want to add multiple locations to the search path. +For example, an app may want to read the configs from an additional Python module or +an additional directory on the file system. Another example is in unit testing, +where the defaults list in a config loaded from the `tests/configs` folder may +make reference to another config from the `app/configs` folder. If the +`config_path` or `config_dir` argument passed to `@hydra.main` or to one of the +[initialization methods](compose_api.md#initialization-methods) points to +`tests/configs`, the configs located in `app/configs` will not be discoverable +unless Hydra's search path is modified. + +You can configure `hydra.searchpath` in your primary config or from the command line. +:::info +hydra.searchpath can **only** be configured in the primary config. Attempting to configure it in other +configs will result in an error. +::: + +In this example, we add a second config directory - `additional_conf`, next to the first config directory: + +
+
+ +```bash +├── __init__.py +├── conf +│   ├── config.yaml +│   └── dataset +│   └── cifar10.yaml +├── additional_conf +│   ├── __init__.py +│   └── dataset +│   └── imagenet.yaml +└── my_app.py +``` +
+
+ +```python title="my_app.py" + +@hydra.main(version_base=None, config_path="conf", config_name="config") +def my_app(cfg: DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) + + +if __name__ == "__main__": + my_app() +``` +
+
+ +`conf/config.yaml` is the primary config for `my_app.py`, config groups `cifar10` and `imagenet` are +under different folders. +We can add `additional_conf` to `hydra.searchpath` for Hydra to discover `dataset/imagenet`. + +
+
+ +```yaml title="config.yaml" +defaults: + - dataset: cifar10 + +hydra: + searchpath: + - pkg://additional_conf + # You can also use file based schema: + # - file:///etc/my_app + # - file://${oc.env:HOME}/.my_app +``` + +
+ +
+ +```python title="my_app.py output" +dataset: + name: cifar10 + path: /datasets/cifar10 + + + + + + +``` +
+
+ +Overriding `dataset=imagenet` from the commandline: + +
+
+ +```bash title="command line override" +python my_app.py dataset=imagenet + + +``` + +
+ +
+ +```python title="my_app.py output" +dataset: + name: imagenet + path: /datasets/imagenet +``` +
+
+ + + + + +`hydra.searchpath` can be defined or overridden via the command line as well: + +```bash title="command line override" +python my_app.py 'hydra.searchpath=[pkg://additional_conf]' +``` + +#### Overriding `--config-dir` from the command line +This is a less flexible alternative to `hydra.searchpath`. +See this [page](hydra-command-line-flags.md) for more info. + + +#### Creating a `SearchPathPlugin` + + + +Framework authors may want to add their configs to the search path automatically once their package is installed, +eliminating the need for any actions from the users. +This can be achieved using a `SearchPathPlugin`. Check the example plugin linked above for more details. diff --git a/website/versioned_docs/version-1.3/advanced/terminology.md b/website/versioned_docs/version-1.3/advanced/terminology.md new file mode 100644 index 00000000000..2ec6d954fad --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/terminology.md @@ -0,0 +1,131 @@ +--- +id: terminology +title: Terminology +--- +This page describes some common concepts in Hydra, most of which are covered in greater details throughout the documentation. +Examples of many of the following concepts are in the [Examples section](#example-of-core-concepts). + +## Input Configs +Input configs are building blocks used to construct the [Output Config](#output-config) consumed by the application. +They can be grouped by placing them in [Config Groups](#config-group). + +### Config files +Config files are form of input configs in [YAML](https://yaml.org/) format. They can exist in the file system or +in a Python module. +```yaml title="Example config file" +user: + name: James Bond + age: 7 +``` + +### Structured Config +This term has two common meanings: +1. A class decorated with [@dataclass](https://docs.python.org/3/library/dataclasses.html) or [@attr.s](https://www.attrs.org/en/stable/), or an instance of such a class which is intended to be used as config. +2. A Config object initialized from a class or object as defined in 1. Structured Configs provide additional type information that enables static and runtime type checking. + +The two primary patterns for using Structured Configs are: +- As an [Input Config](#input-configs). +- As a schema validating Config Files and command line arguments. + +```python title="Example Schema" +@dataclass +class User: + name: str + age: int +``` + +## Other configs +**Primary Config**: The input config named in [**@hydra.main()**](../tutorials/basic/your_first_app/2_config_file.md) or in the [**Compose API**](compose_api.md). +**Output Config**: A config composed from the [Input Configs](#input-configs) and [Overrides](#overrides) by **@hydra.main()**, or the Compose API. + +## Overrides +[Overrides](override_grammar/basic.md) are strings that can be used to manipulate the config composition process. +This includes updating, adding and deleting config values and [Defaults List](#defaults-list) options. + +Overrides can be used in the command line and in the [Compose API](compose_api.md). +In the examples below, `key=value` is an override: +
+
+ +```shell title="Override in the command line" +$ python my_app.py key=value + +``` + +
+
+ +```python title="Override used in the Compose API" +cfg = compose(config_name, + overrides=["key=value"]) +``` + +
+
+ +## Defaults List +A list in an [Input Config](#input-configs) that instructs Hydra how compose the config. +```yaml title="Defaults List in a YAML config" +defaults: + - db: mysql # An overridable defaults list entry + - schema/school # A non-overridable defaults list entry +``` + +## Config Group +A Config Group is a directory in the [Config Search Path](#config-search-path) that contains [Input Configs](#input-configs). +Config Groups can be nested, and in that case the path elements are separated by a forward slash ('/') regardless of the operating system. + +## Config Group Option +An Input Config in a Config Group. When used in a Defaults List, a Config Group Option can be a single Input Config, or a list of Input Configs from the same Config Group. + +## Package +A Package is the path to node in a config. By default, the Package of a Config Group Option is derived from the Config Group. +*e.g:* Configs in **mi6/agent** will have the package **mi6.agent** by default. + + +The [Package Directive](overriding_packages.md#overriding-the-package-via-the-package-directive) specifies the root [Package](#package) of a [Config File](#input-configs). It can appear at the top of YAML config file. + +## Example of Core Concepts + +
+
+ +```yaml title="config.yaml" +defaults: + - mi6/agent: james_bond + +``` + +
+ +
+ +```yaml title="mi6/agent/james_bond.yaml" {1} +# @package bond.james +codename: '007' + +``` + +
+
+ +```yaml title="Output config" {1,2} +bond: + james: + codename: '007' +``` +
+
+ +- [Input Configs](#input-configs): **config.yaml**, **mi6/agent/james_bond.yaml** +- [Config Group](#config-group): mi6/agent +- [Config Group Option](#config-group-option): james_bond +- [Packages](#package): ****, **mi6**, **mi6.agent**, **mi6.agent.codename** +- [Package directive](#package-directive): **# @package bond.james**, overriding the default Package for the containing Input Config + +## Config Search Path +The [Config Search Path](search_path.md) is a list of paths that are searched in order to find configs. It is similar to +the Python [PYTHONPATH](https://docs.python.org/3/using/cmdline.html#envvar-PYTHONPATH). + +## Plugins +[Plugins](plugins/intro.md) extend Hydra's capabilities. Hydra has several plugin types, for example Launcher and Sweeper. diff --git a/website/versioned_docs/version-1.3/advanced/unit_testing.md b/website/versioned_docs/version-1.3/advanced/unit_testing.md new file mode 100644 index 00000000000..bc0325a4c58 --- /dev/null +++ b/website/versioned_docs/version-1.3/advanced/unit_testing.md @@ -0,0 +1,32 @@ +--- +id: unit_testing +title: Hydra in Unit Tests +--- + +import GithubLink from "@site/src/components/GithubLink" + +Use `initialize()`, `initialize_config_module()` or `initialize_config_dir()` in conjunction with `compose()` +to compose configs inside your unit tests. +Be sure to read the [Compose API documentation](compose_api.md). + +The Hydra example application contains an example test. + +```python title="Testing example with initialize()" +from hydra import initialize, compose +# 1. initialize will add config_path the config search path within the context +# 2. The module with your configs should be importable. +# it needs to have a __init__.py (can be empty). +# 3. THe config path is relative to the file calling initialize (this file) +def test_with_initialize() -> None: + with initialize(version_base=None, config_path="../hydra_app/conf"): + # config is relative to a module + cfg = compose(config_name="config", overrides=["app.user=test_user"]) + assert cfg == { + "app": {"user": "test_user", "num1": 10, "num2": 20}, + "db": {"host": "localhost", "port": 3306}, + } +``` + +For an idea about how to modify Hydra's search path when using `compose` in +unit tests, see the page on +[overriding the `hydra.searchpath` config](search_path.md#overriding-hydrasearchpath-config). diff --git a/website/versioned_docs/version-1.3/configure_hydra/Intro.md b/website/versioned_docs/version-1.3/configure_hydra/Intro.md new file mode 100644 index 00000000000..2c46b3b206c --- /dev/null +++ b/website/versioned_docs/version-1.3/configure_hydra/Intro.md @@ -0,0 +1,156 @@ +--- +id: intro +title: Overview +sidebar_label: Introduction +--- + +import GithubLink from "@site/src/components/GithubLink" + +Hydra is highly configurable. Many of its aspects and subsystems can be configured, including: +* The Launcher +* The Sweeper +* Logging +* Output directory patterns +* Application help (--help and --hydra-help) + +The Hydra config can be customized using the same methods you are already familiar with from the tutorial. +You can include some Hydra config snippet in your own config to override it directly, or compose in different +configurations provided by plugins or by your own code. You can also override everything in Hydra from the command +line just like with your own configuration. + +The Hydra configuration itself is composed from multiple config files. here is a partial list: +```yaml title="hydra/config" +defaults: + - job_logging : default # Job's logging config + - launcher: basic # Launcher config + - sweeper: basic # Sweeper config + - output: default # Output directory +``` +You can view the Hydra config structure here. + +You can view the Hydra config using `--cfg hydra`: +
+ $ python my_app.py --cfg hydra (Click to expand) + +```yaml +hydra: + run: + dir: outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + launcher: + _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + hydra_logging: + version: 1 + formatters: + ... +``` +
+ + +## Accessing the Hydra config +The Hydra config is large. To reduce clutter in your own config it's being deleted from the config object +Hydra is passing to the function annotated by `@hydra.main()`. + +There are two ways to access the Hydra config: + +#### In your config, using the `hydra` resolver: +```yaml +config_name: ${hydra:job.name} +``` +Pay close attention to the syntax: The resolver name is `hydra`, and the `key` is passed after the colon. + +#### In your code, using the HydraConfig singleton. +```python +from hydra.core.hydra_config import HydraConfig + +@hydra.main() +def my_app(cfg: DictConfig) -> None: + print(HydraConfig.get().job.name) +``` + +### Top-level Hydra settings +The following fields are present at the top level of the Hydra Config. +- mode: Optional, one of `RUN` or `MULTIRUN`. See [multirun](/tutorials/basic/running_your_app/2_multirun.md) for more info. +- **searchpath**: A list of paths that Hydra searches in order to find configs. + See [overriding `hydra.searchpath`](advanced/search_path.md#overriding-hydrasearchpath-config) +- **job_logging** and **hydra_logging**: Configure logging settings. + See [logging](/tutorials/basic/running_your_app/4_logging.md) and [customizing logging](logging.md). +- **sweeper**: [Sweeper](/tutorials/basic/running_your_app/2_multirun.md#sweeper) plugin settings. Defaults to basic sweeper. +- **launcher**: [Launcher](/tutorials/basic/running_your_app/2_multirun.md#launcher) plugin settings. Defaults to basic launcher. +- **callbacks**: [Experimental callback support](/experimental/callbacks.md). +- **help**: Configures your app's `--help` CLI flag. See [customizing application's help](app_help.md). +- **hydra_help**: Configures the `--hydra-help` CLI flag. +- **output_subdir**: Configures the `.hydra` subdirectory name. + See [changing or disabling the output subdir](/tutorials/basic/running_your_app/3_working_directory.md#changing-or-disabling-hydras-output-subdir). +- **verbose**: Configures per-file DEBUG-level logging. + See [logging](/tutorials/basic/running_your_app/4_logging.md). + +### hydra.job: +The **hydra.job** node is used for configuring some aspects of your job. +Below is a short summary of the fields in **hydra.job**. +You can find more details in the [Job Configuration](job.md) page. + +Fields under **hydra.job**: +- **name** : Job name, defaults to the Python file name without the suffix. can be overridden. +- **override_dirname** : Pathname derived from the overrides for this job +- **chdir**: If `True`, Hydra calls `os.chdir(output_dir)` before calling back to the user's main function. +- **id** : Job ID in the underlying jobs system (SLURM etc) +- **num** : job serial number in sweep +- **config_name** : The name of the config used by the job (Output only) +- **env_set**: Environment variable to set for the launched job +- **env_copy**: Environment variable to copy from the launching machine +- **config**: fine-grained configuration for job + +### hydra.run: +Used in single-run mode (i.e. when the `--multirun` command-line flag is omitted). +See [configuration for run](workdir.md#configuration-for-run). +- **dir**: used to specify the output directory. + +### hydra.sweep: +Used in multi-run mode (i.e. when the `--multirun` command-line flag is given) +See [configuration for multirun](workdir.md#configuration-for-multirun). +- **dir**: used to specify the output directory common to all jobs in the multirun sweep +- **subdir**: used to specify the a pattern for creation of job-specific subdirectory + +### hydra.runtime: +Fields under **hydra.runtime** are populated automatically and should not be overridden. +- **version**: Hydra's version +- **cwd**: Original working directory the app was executed from +- **output_dir**: This is the directory created by Hydra for saving logs and + yaml config files, as configured by [customizing the working directory pattern](workdir.md). +- **choices**: A dictionary containing the final config group choices. +- **config_sources**: The final list of config sources used to compose the config. + +### hydra.overrides +Fields under **hydra.overrides** are populated automatically and should not be overridden. +- **task**: Contains a list of the command-line overrides used, except `hydra` config overrides. + Contains the same information as the `.hydra/overrides.yaml` file. + See [Output/Working directory](/tutorials/basic/running_your_app/3_working_directory.md). +- **hydra**: Contains a list of the command-line `hydra` config overrides used. + + +### Resolvers provided by Hydra +Hydra provides the following [OmegaConf resolvers](https://omegaconf.readthedocs.io/en/latest/usage.html#resolvers) by default. + +**hydra**: Interpolates into the `hydra` config node. e.g. Use `${hydra:job.name}` to get the Hydra job name. + +**now**: Creates a string representing the current time using +[strftime](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior). +e.g. for formatting the time you can use something like`${now:%H-%M-%S}`. + +**python_version**: Return a string representing the runtime python version by calling `sys.version_info`. +Takes an optional argument of a string with the values major, minor or macro. +e.g: +```yaml +default: ${python_version:} # 3.8 +major: ${python_version:major} # 3 +minor: ${python_version:minor} # 3.8 +micro: ${python_version:micro} # 3.8.2 +``` + +You can learn more about OmegaConf here. diff --git a/website/versioned_docs/version-1.3/configure_hydra/app_help.md b/website/versioned_docs/version-1.3/configure_hydra/app_help.md new file mode 100644 index 00000000000..72d903a09d8 --- /dev/null +++ b/website/versioned_docs/version-1.3/configure_hydra/app_help.md @@ -0,0 +1,85 @@ +--- +id: app_help +title: Customizing Application's help +sidebar_label: Customizing Application's help +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +Hydra provides two different help options: +* `--help` : Application specific help +* `--hydra-help` Hydra specific help. + +Example output of `--help`: +```text +$ python my_app.py --help +== AwesomeApp == + +This is AwesomeApp! +You can choose a db driver by appending +== Configuration groups == +Compose your configuration from those groups (db=mysql) + +db: mysql, postgresql + + +== Config == +This is the config generated for this run. +You can override everything, for example: +python my_app.py db.user=foo db.pass=bar +------- +db: + driver: mysql + user: omry + pass: secret + +------- + +Powered by Hydra (https://hydra.cc) +Use --hydra-help to view Hydra specific help +``` + +This output is generated from the following config group option (selected in `config.yaml` to be used by default): +```yaml title="hydra/help/my_app_help.yaml" +# App name, override to match the name your app is known by +app_name: AwesomeApp + +# Help header, customize to describe your app to your users +header: == ${hydra.help.app_name} == + +footer: |- + Powered by Hydra (https://hydra.cc) + Use --hydra-help to view Hydra specific help + +# Basic Hydra flags: +# $FLAGS_HELP +# +# Config groups, choose one of: +# $APP_CONFIG_GROUPS: All config groups that does not start with hydra/. +# $HYDRA_CONFIG_GROUPS: All the Hydra config groups (starts with hydra/) +# +# Configuration generated with overrides: +# $CONFIG : Generated config +# +template: |- + ${hydra.help.header} + + This is ${hydra.help.app_name}! + You can choose a db driver by appending + == Configuration groups == + Compose your configuration from those groups (db=mysql) + + $APP_CONFIG_GROUPS + + == Config == + This is the config generated for this run. + You can override everything, for example: + python my_app.py db.user=foo db.pass=bar + ------- + $CONFIG + ------- + + ${hydra.help.footer} +``` \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/configure_hydra/job.md b/website/versioned_docs/version-1.3/configure_hydra/job.md new file mode 100644 index 00000000000..84d2c603eec --- /dev/null +++ b/website/versioned_docs/version-1.3/configure_hydra/job.md @@ -0,0 +1,115 @@ +--- +id: job +title: Job Configuration +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + +The job configuration resides in `hydra.job`. +The Structured Config is below, the latest definition is here. + +
Expand definition + +```python +# job runtime information will be populated here +@dataclass +class JobConf: + # Job name, populated automatically unless specified by the user (in config or cli) + name: str = MISSING + + # Change current working dir to the output dir. + chdir: bool = True + + # Concatenation of job overrides that can be used as a part + # of the directory name. + # This can be configured in hydra.job.config.override_dirname + override_dirname: str = MISSING + + # Job ID in underlying scheduling system + id: str = MISSING + + # Job number if job is a part of a sweep + num: int = MISSING + + # The config name used by the job + config_name: Optional[str] = MISSING + + # Environment variables to set remotely + env_set: Dict[str, str] = field(default_factory=dict) + # Environment variables to copy from the launching machine + env_copy: List[str] = field(default_factory=list) + + # Job config + @dataclass + class JobConfig: + @dataclass + # configuration for the ${hydra.job.override_dirname} runtime variable + class OverrideDirname: + kv_sep: str = "=" + item_sep: str = "," + exclude_keys: List[str] = field(default_factory=list) + + override_dirname: OverrideDirname = field(default_factory=OverrideDirname) + + config: JobConfig = field(default_factory=JobConfig) +``` +
+ +### hydra.job.name + + +The job name is used by different things in Hydra, such as the log file name (`${hydra.job.name}.log`). +It is normally derived from the Python file name (The job name of the file `train.py` is `train`). +You can override it via the command line, or your config file. + +### hydra.job.chdir + +Decides whether Hydra changes the current working directory to the output directory for each job. +Learn more at the [Output/Working directory](/tutorials/basic/running_your_app/3_working_directory.md#disable-changing-current-working-dir-to-jobs-output-dir) page. + + +### hydra.job.override_dirname +Enables the creation of an output directory which is based on command line overrides. +Learn more at the [Customizing Working Directory](/configure_hydra/workdir.md) page. + +### hydra.job.id +The job ID is populated by the active Hydra launcher. For the basic launcher, the job ID is just a serial job number. +Other launchers will set it to an ID that makes sense like SLURM job ID. + +### hydra.job.num +Serial job number within this current sweep run. (0 to n-1). + +### hydra.job.config_name +The config name used by the job, this is populated automatically to match the config name in `@hydra.main()`. + +### hydra.job.env_set +A `Dict[str, str]` that is used to set the environment variables of the running job. +Some common use cases are to automatically set environment variables that are affecting underlying libraries. +For example, the following will disables multithreading in Intel IPP and MKL: +```yaml +hydra: + job: + env_set: + OMP_NUM_THREADS: 1 +``` + +Another example, is to use interpolation to automatically set the rank +for [Torch Distributed](https://pytorch.org/tutorials/intermediate/dist_tuto.html) run to match the job number +in the sweep. + +```yaml +hydra: + job: + env_set: + RANK: ${hydra:job.num} +``` + +### hydra.job.env_copy +In some cases you want to automatically copy local environment variables to the running job environment variables. +This is particularly useful for remote runs. +```yaml +hydra: + job: + env_copy: + - AWS_KEY +``` diff --git a/website/versioned_docs/version-1.3/configure_hydra/logging.md b/website/versioned_docs/version-1.3/configure_hydra/logging.md new file mode 100644 index 00000000000..0118989050c --- /dev/null +++ b/website/versioned_docs/version-1.3/configure_hydra/logging.md @@ -0,0 +1,52 @@ +--- +id: logging +title: Customizing logging +sidebar_label: Customizing logging +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +Hydra is configuring Python standard logging library with the dictConfig method. You can learn more about it [here](https://docs.python.org/3/howto/logging.html). +There are two logging configurations, one for Hydra itself and one for the executed jobs. + +This example demonstrates how to customize the logging behavior of your Hydra app, by making the following changes +to the default logging behavior: + + * Outputs only to stdout (no log file) + * Output a simpler log line pattern + +```yaml title="config.yaml" +defaults: + - override hydra/job_logging: custom +``` + +```yaml title="hydra/job_logging/custom.yaml" +version: 1 +formatters: + simple: + format: '[%(levelname)s] - %(message)s' +handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout +root: + handlers: [console] + +disable_existing_loggers: false +``` + +This is what the default logging looks like: +``` +$ python my_app.py hydra/job_logging=default +[2020-08-24 13:43:26,761][__main__][INFO] - Info level message +``` + +And this is what the custom logging looks like: +```text +$ python my_app.py +[INFO] - Info level message +``` + diff --git a/website/versioned_docs/version-1.3/configure_hydra/workdir.md b/website/versioned_docs/version-1.3/configure_hydra/workdir.md new file mode 100644 index 00000000000..e082b373fe5 --- /dev/null +++ b/website/versioned_docs/version-1.3/configure_hydra/workdir.md @@ -0,0 +1,171 @@ +--- +id: workdir +title: Customizing working directory pattern +sidebar_label: Customizing working directory pattern +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +Hydra automatically creates an output directory used to store log files and +save yaml configs. This directory can be configured by setting `hydra.run.dir` +(for single hydra runs) or `hydra.sweep.dir`/`hydra.sweep.subdir` (for multirun +sweeps). At runtime, the path of the output directory can be +[accessed](Intro.md#accessing-the-hydra-config) via the `hydra.runtime.output_dir` variable. +Below are a few examples of customizing output directory patterns. + +### Configuration for run + +Run output directory grouped by date: +```yaml +hydra: + run: + dir: ./outputs/${now:%Y-%m-%d}/${now:%H-%M-%S} +``` + +Run output directory grouped by job name: +```yaml +hydra: + run: + dir: outputs/${hydra.job.name}/${now:%Y-%m-%d_%H-%M-%S} +``` + +Run output directory can contain user configuration variables: +```yaml +hydra: + run: + dir: outputs/${now:%Y-%m-%d_%H-%M-%S}/opt:${optimizer.type} +``` + +### Configuration for multirun +We will run the application with same command but different configurations: + +```bash +python my_app.py --multirun a=a1,a2,a3 +``` + + +Default multirun dir configurations: +
+
+ +```yaml title="config.yaml" +hydra: + sweep: + dir: multirun/${now:%Y-%m-%d}/${now:%H-%M-%S} + subdir: ${hydra.job.num} + +``` +
+
+ +```bash title="workding dir created" +$ tree my_app -d +my_app +├── 0 +├── 1 +└── 2 +``` +
+
+ + +Similar configuration patterns in run can be applied to config multirun dir as well. + +For example, multirun output directory grouped by job name, and sub dir by job num: +
+
+ +```yaml title="config.yaml" +hydra: + sweep: + dir: ${hydra.job.name} + subdir: ${hydra.job.num} + +``` +
+
+ +```bash title="workding dir created" +$ tree my_app -d +my_app +├── 0 +├── 1 +└── 2 +``` +
+
+ + +### Using `hydra.job.override_dirname` + + + +This field is populated automatically using your command line arguments and is typically being used as a part of your +output directory pattern. It is meant to be used along with the configuration for working dir, especially +in `hydra.sweep.subdir`. + +If we run the example application with the following commandline overrides and configs: + +```bash +python my_app.py --multirun batch_size=32 learning_rate=0.1,0.01 +``` + + +
+
+ +```yaml title="config.yaml" +hydra: + sweep: + dir: multirun + subdir: ${hydra.job.override_dirname} +``` +
+
+ +```bash title="working dir created" +$ tree multirun -d +multirun +├── batch_size=32,learning_rate=0.01 +└── batch_size=32,learning_rate=0.1 +``` +
+
+ +You can further customized the output dir creation by configuring`hydra.job.override_dirname`. + +In particular, the separator char `=` and the item separator char `,` can be modified by overriding +`hydra.job.config.override_dirname.kv_sep` and `hydra.job.config.override_dirname.item_sep`. +Command line override keys can also be automatically excluded from the generated `override_dirname`. + +An example of a case where the exclude is useful is a random seed. + +```yaml +hydra: + run: + dir: output/${hydra.job.override_dirname}/seed=${seed} + job: + config: + override_dirname: + exclude_keys: + - seed +``` +With this configuration, running +```bash +$ python my_app.py --multirun batch_size=32 learning_rate=0.1,0.01 seed=1,2 +``` + +Would result in a directory structure like: +``` +$ tree multirun -d +multirun +├── batch_size=32,learning_rate=0.01 +│   ├── seed=1 +│   └── seed=2 +└── batch_size=32,learning_rate=0.1 + ├── seed=1 + └── seed=2 +``` + diff --git a/website/versioned_docs/version-1.3/development/documentation.md b/website/versioned_docs/version-1.3/development/documentation.md new file mode 100644 index 00000000000..cadcd8938bc --- /dev/null +++ b/website/versioned_docs/version-1.3/development/documentation.md @@ -0,0 +1,71 @@ +--- +id: documentation +title: Documentation +sidebar_label: Documentation +--- + +import GithubLink from "@site/src/components/GithubLink" + +## NEWS Entries +The NEWS.md file is managed using `towncrier` and all non-trivial changes +must be accompanied by a news entry. + +To add an entry to the news file, first, you need to have created an issue +describing the change you want to make. A Pull Request itself *may* function as +such, but it is preferred to have a dedicated issue (for example, in case the +PR ends up rejected due to code quality reasons). + +Once you have an issue or pull request, you take the number, and you create a +file inside the ``news/`` directory (in case the change is directly related to Hydra) +or in the ``news/`` directory of the relevant plugin. The file is named after the +issue number with one of the following extensions: +* `api_change` : API Change (Renames, deprecations, and removals) +* `feature` : Addition of a new feature +* `bugfix` : Fixing of a bug +* `docs` : Addition or updates to documentation +* `config` : Changes or addition to the configuration structure +* `maintenance` : Changes that improve the maintainability of the code + +If your issue or PR number is ``1234`` and this change is fixing a bug, you would +create a file ``news/1234.bugfix``. PRs can span multiple categories by creating +multiple files (for instance, if you added a feature and deprecated/removed the +old feature at the same time, you would create ``news/NNNN.feature`` and +``news/NNNN.api_change``). Likewise, if a PR touches multiple issues/PRs, you may +create a file for each of them with the exact same contents, and Towncrier will +deduplicate them. + + +### Contents of a NEWS entry +The contents of this file are markdown formatted text that will be used +as the content of the news file entry. You do not need to reference the issue +or PR numbers here as towncrier will automatically add a reference to all of +the affected issues when rendering the news file. + +To maintain a consistent style in the `NEWS.md` file, it is +preferred to keep the news entry to the point, in sentence case, shorter than +80 characters and in an imperative tone -- an entry should complete the sentence +"This change will ...". In rare cases, where one line is not enough, use a +summary line in an imperative tone followed by a blank line separating it +from a description of the feature/change in one or more paragraphs, each wrapped +at 80 characters. Remember that a news entry is meant for end users and should +only contain details relevant to an end user. + +## Website setup + +The website is built using [Docusaurus 2](https://v2.docusaurus.io/). +Run the following commands from the `website` directory. + +### Install + +``` +$ yarn +``` +### Local Development + +``` +$ yarn start +``` + +This command starts a local development server and opens up a browser window. Most changes are reflected live without having to restart the server. + +For more details, refer here. diff --git a/website/versioned_docs/version-1.3/development/overview.md b/website/versioned_docs/version-1.3/development/overview.md new file mode 100644 index 00000000000..704c375d58b --- /dev/null +++ b/website/versioned_docs/version-1.3/development/overview.md @@ -0,0 +1,29 @@ +--- +id: overview +title: Developer Guide Overview +--- + +This guide assumes you have checked-out the [repository](https://github.com/facebookresearch/hydra). +It is recommended that you install Hydra in a virtual environment like [conda](https://docs.conda.io/en/latest/) or [virtualenv](https://virtualenv.pypa.io/en/latest/). + +## Environment setup +Install [Miniconda](https://docs.conda.io/en/latest/miniconda.html) and create an empty Conda environment with: +``` +conda create -n hydra38 python=3.8 -qy +``` + +:::info NOTE +The core Hydra framework supports Python 3.6 or newer. You may need to create additional environments for different Python versions if CI detect issues on a supported version of Python. +::: + +Activate the environment: +``` +conda activate hydra38 +``` +From the source tree, install Hydra in development mode with the following commands: +```bash +# install development dependencies +pip install -r requirements/dev.txt +# install Hydra in development (editable) mode +pip install -e . +``` diff --git a/website/versioned_docs/version-1.3/development/release.md b/website/versioned_docs/version-1.3/development/release.md new file mode 100644 index 00000000000..57b96be5b6c --- /dev/null +++ b/website/versioned_docs/version-1.3/development/release.md @@ -0,0 +1,17 @@ +--- +id: release +title: Release process +sidebar_label: Release process +--- + +The release process may be automated in the future. + +- Checkout main +- Update the Hydra version in `hydra/__init__.py` +- Update NEWS.md with towncrier +- Create a wheel and source dist for hydra-core: `python -m build` +- Upload pip package: `python -m twine upload dist/*` +- Update the link to the latest stable release in `website/docs/intro.md` +- If you are creating a new release branch: + - [tag a new versioned copy of the docs using docusaurus](https://docusaurus.io/docs/versioning#tagging-a-new-version) + - update `website/docusaurus.config.js` with a pointer to the new release branch on github diff --git a/website/versioned_docs/version-1.3/development/style_guide.md b/website/versioned_docs/version-1.3/development/style_guide.md new file mode 100644 index 00000000000..b2652f60c8d --- /dev/null +++ b/website/versioned_docs/version-1.3/development/style_guide.md @@ -0,0 +1,28 @@ +--- +id: style_guide +title: Style Guide +sidebar_label: Style Guide +--- + +The code need to pass verification by the following tools: + - `black .` : Automatic code formatting for Python + - `flake8` : PEP8 compliance checker for Python, this includes copyright header verification + - `isort .` : Ensure imports are sorted properly + - `mypy --strict .` : Ensures code passes strict type checking + - `yamllint .` : Ensures that yaml files are syntactically correct and properly indented. + +The easiest way to run the required verifications is: + - `nox -s lint` : for the Hydra core + - `nox -s lint_plugins` : for the included plugins + +isort is a bit tricky to run for plugins. the best way to get it to sort the plugins imports is with the FIX environment +variable: +``` +$ FIX=1 nox -s lint_plugins +``` + +It is also recommended that you install pre-commit hooks (use `pre-commit install`). +pre-commit will execute some of the above tests when you commit your code locally. +You can disable it by appending `-n` to your commit command: `git commit -m wip -n` + +Pull requests that do not lint will fail the automated testing. \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/development/testing.md b/website/versioned_docs/version-1.3/development/testing.md new file mode 100644 index 00000000000..13e4dbf45f8 --- /dev/null +++ b/website/versioned_docs/version-1.3/development/testing.md @@ -0,0 +1,38 @@ +--- +id: testing +title: Testing +sidebar_label: Testing +--- + +Hydra uses [nox](https://github.com/theacodes/nox) - a build automation tool - to manage tests, linting, code coverage, etc. +The command `nox` will run all the configured sessions. List the sessions using `nox -l` and +run specific sessions with `nox -s NAME` (you may need to quote the session name in some cases) + +## Testing with pytest +Run `pytest` at the repository root to run all the Hydra core tests. +To run the tests of individual plugins, use `pytest plugins/NAME` (The plugin must be installed). + +:::info NOTE +Some plugins support fewer versions of Python than the Hydra core. +::: + +## Testing with nox +See `nox -l`. a few examples: +* `nox -s test_core` will test Hydra core on all supported Python versions +* `nox -s "test_plugins-3.8"` will test plugins on Python 3.8. +* `nox -s "test_plugins-3.8"` will test plugins on Python 3.8. + +The `noxfile.py` is checking some environment variables to decide what to run. For example, +to test a single plugin: +```shell {4} +$ PLUGINS=hydra_colorlog nox -s test_plugins-3.8 +Operating system : Linux +NOX_PYTHON_VERSIONS : ['3.6', '3.7', '3.8', '3.9'] +PLUGINS : ['hydra_colorlog'] +SKIP_CORE_TESTS : False +FIX : False +VERBOSE : 0 +INSTALL_EDITABLE_MODE : 0 +nox > Running session test_plugins-3.8 +... +``` \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/experimental/callbacks.md b/website/versioned_docs/version-1.3/experimental/callbacks.md new file mode 100644 index 00000000000..9913c140c9e --- /dev/null +++ b/website/versioned_docs/version-1.3/experimental/callbacks.md @@ -0,0 +1,206 @@ +--- +id: callbacks +title: Callbacks +sidebar_label: Callbacks +--- + +import GithubLink from "@site/src/components/GithubLink" +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +The Callback interface enables custom +code to be triggered by various Hydra events. + +To use the callback API, one should import Hydra's `Callback` class: +```python +from hydra.experimental.callback import Callback +``` +Users can then create subclasses of this `Callback` class, overriding one or more of +the methods defined by `Callback`. For the methods of a subclass to be called at the +appropriate time, the subclass must be registered with Hydra in the `hydra.callbacks` config + (see examples below). + +The full API exposed by the `hydra.experimental.callback.Callback` class is listed below: + +
Events supported (Click to expand) + +```python +from hydra.types import TaskFunction + +class Callback: + def on_run_start(self, config: DictConfig, **kwargs: Any) -> None: + """ + Called in RUN mode before job/application code starts. `config` is composed with overrides. + Some `hydra.runtime` configs are not populated yet. + See hydra.core.utils.run_job for more info. + """ + ... + + def on_run_end(self, config: DictConfig, **kwargs: Any) -> None: + """ + Called in RUN mode after job/application code returns. + """ + ... + + def on_multirun_start(self, config: DictConfig, **kwargs: Any) -> None: + """ + Called in MULTIRUN mode before any job starts. + When using a launcher, this will be executed on local machine before any Sweeper/Launcher is initialized. + """ + ... + + def on_multirun_end(self, config: DictConfig, **kwargs: Any) -> None: + """ + Called in MULTIRUN mode after all jobs returns. + When using a launcher, this will be executed on local machine. + """ + ... + + def on_job_start(self, config: DictConfig, *, task_function: TaskFunction, **kwargs: Any) -> None: + """ + Called in both RUN and MULTIRUN modes, once for each Hydra job (before running application code). + This is called from within `hydra.core.utils.run_job`. In the case of remote launching, this will be executed + on the remote server along with your application code. The `task_function` argument is the function + decorated with `@hydra.main`. + """ + ... + + def on_job_end( + self, config: DictConfig, job_return: JobReturn, **kwargs: Any + ) -> None: + """ + Called in both RUN and MULTIRUN modes, once for each Hydra job (after running + application code). + This is called from within `hydra.core.utils.run_job`. In the case of remote launching, this will be executed + on the remote server after your application code. + + `job_return` contains info that could be useful for logging or post-processing. + See hydra.core.utils.JobReturn for more. + """ + ... +``` +
+ +### Configure Callback + +Say we have `MyCallback` so after every job ends we can upload a certain file to a S3 bucket. +For simplicity we include this Callback class within the application, in real life you should have the +Callback in a separate file. +Running the application, we can see our custom method `on_job_end` was called. + +
+
+ +```python title="my_app.py" +class MyCallback(Callback): + def __init__(self, bucket: str, file_path: str) -> None: + self.bucket = bucket + self.file_path = file_path + + def on_job_end(self, config: DictConfig, **kwargs: Any) -> None: + print(f"Job ended,uploading...") + # uploading... + +@hydra.main(version_base=None, config_path="conf", config_name="config") +def my_app(cfg: DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) + + +if __name__ == "__main__": + my_app() +``` +
+
+ +```commandline title="output" + +$ python my_app.py +foo: bar + +Job ended,uploading... + + + + + + + + + + + +``` +
+
+ +Now let's take a look at the configurations. + +
+
+ +```commandline title="$ tree conf" +conf +├── config.yaml +└── hydra + └── callbacks + └── my_callback.yaml + + +``` +
+
+ +```commandline title="conf/config.yaml" +defaults: + - /hydra/callbacks: + - my_callback + +foo: bar + + +``` +
+
+ +```commandline title="conf/hydra/callbacks/my_callback.yaml" +# @package _global_ +hydra: + callbacks: + my_callback: + _target_: my_app.MyCallback + bucket: my_s3_bucket + file_path: ./test.pt +``` +
+
+ + +### Callback ordering +The `on_run_start` or `on_multirun_start` method will get called first, +followed by `on_job_start` (called once for each job). +After each job `on_job_end` is called, and finally either `on_run_end` or +`on_multirun_end` is called one time before the application exits. + +In the `hydra.callbacks` section of your config, you can use a list to register multiple callbacks. They will be called in the final composed order for `start` events and +in reversed order for `end` events. So, for example, suppose we have the following composed config: +```commandline title="python my_app.py --cfg hydra -p hydra.callbacks" +# @package hydra.callbacks +my_callback1: + _target_: my_app.MyCallback1 + param1: val1 +my_callback2: + _target_: my_app.MyCallback2 + param2: val2 +``` +Before each job starts, `MyCallback1.on_job_start` will get called first, +followed by `MyCallback2.on_job_start`. +After each job ends, `MyCallback2.on_job_end` will get called first, +followed by `MyCallback1.on_job_end`. + + +### Example callbacks + +We've included some example callbacks here: +- `LogJobReturnCallback` is especially useful for logging errors when running on a remote cluster (e.g. slurm.) +- `PickleJobInfoCallback` can be used to reproduce a Hydra job. See [here](/experimental/rerun.md) for more. \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/experimental/intro.md b/website/versioned_docs/version-1.3/experimental/intro.md new file mode 100644 index 00000000000..3c4e3931788 --- /dev/null +++ b/website/versioned_docs/version-1.3/experimental/intro.md @@ -0,0 +1,10 @@ +--- +id: intro +title: Introduction +sidebar_label: Introduction +--- +Experimental features are new features in Hydra that are considered experimental because their API may have not yet +stabilized. + +Those features should all work, but code relying on them may break in future versions as they evolve. +Experimental features are expected be promoted out of experimental once they deemed stable and complete enough. diff --git a/website/versioned_docs/version-1.3/experimental/rerun.md b/website/versioned_docs/version-1.3/experimental/rerun.md new file mode 100644 index 00000000000..353330fe9bc --- /dev/null +++ b/website/versioned_docs/version-1.3/experimental/rerun.md @@ -0,0 +1,91 @@ +--- +id: rerun +title: Re-run a job from previous config +sidebar_label: Re-run +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +:::caution +This is an experimental feature. Please read through this page to understand what is supported. +::: + +We use the example app linked above for demonstration. To save the configs for re-run, first use the experimental +Hydra Callback for saving the job info: + + +```yaml title="config.yaml" +hydra: + callbacks: + save_job_info: + _target_: hydra.experimental.callbacks.PickleJobInfoCallback +``` + + + + +```python title="Example function" +@hydra.main(version_base=None, config_path=".", config_name="config") +def my_app(cfg: DictConfig) -> None: + log.info(f"output_dir={HydraConfig.get().runtime.output_dir}") + log.info(f"cfg.foo={cfg.foo}") +``` + + +Run the example app: +```commandline +$ python my_app.py +[2022-03-16 14:51:30,905][hydra.experimental.pickle_job_info_callback][INFO] - Saving job configs in /Users/jieru/workspace/hydra/examples/experimental/outputs/2022-03-16/14-51-30/.hydra/config.pickle +[2022-03-16 14:51:30,906][__main__][INFO] - Output_dir=/Users/jieru/workspace/hydra/examples/experimental/outputs/2022-03-16/14-51-30 +[2022-03-16 14:51:30,906][__main__][INFO] - cfg.foo=bar +[2022-03-16 14:51:30,906][hydra.experimental.pickle_job_info_callback][INFO] - Saving job_return in /Users/jieru/workspace/hydra/examples/experimental/outputs/2022-03-16/14-51-30/.hydra/job_return.pickle +``` +The Callback saves `config.pickle` in `.hydra` sub dir, this is what we will use for rerun. + +Now rerun the app +```commandline +$ OUTPUT_DIR=/Users/jieru/workspace/hydra/examples/experimental/outputs/2022-03-16/14-51-30/.hydra/ +$ python my_app.py --experimental-rerun $OUTPUT_DIR/config.pickle +/Users/jieru/workspace/hydra/hydra/main.py:23: UserWarning: Experimental rerun CLI option. + warnings.warn(msg, UserWarning) +[2022-03-16 14:59:21,666][__main__][INFO] - Output_dir=/Users/jieru/workspace/hydra/examples/experimental/outputs/2022-03-16/14-51-30 +[2022-03-16 14:59:21,666][__main__][INFO] - cfg.foo=bar +``` +You will notice `my_app.log` is updated with the logging from the second run, but Callbacks are not called this time. Read on to learn more. + + +### Important Notes +This is an experimental feature. Please reach out if you have any question. +- Only single run is supported. +- `--experimental-rerun` cannot be used with other command-line options or overrides. They will simply be ignored. +- Rerun passes in a cfg_passthrough directly to your application, this means except for logging, no other `hydra.main` +functions are called (such as change working dir, or calling callbacks.) +- The configs are preserved and reconstructed to the best efforts. Meaning we can only guarantee that the `cfg` object +itself passed in by `hydra.main` stays the same across runs. However, configs are resolved lazily. Meaning we cannot +guarantee your application will behave the same if your application resolves configs during run time. In the following example, +`cfg.time_now` will resolve to different value every run. + +
+
+ +```yaml title="config.yaml" +time_now: ${now:%H-%M-%S} + + + +``` + +
+ +
+ +```python title="Example function" +@hydra.main(version_base=None, config_path=".", config_name="config") +def my_app(cfg: DictConfig) -> None: + val = cfg.time_now + # the rest of the application +``` +
+
diff --git a/website/versioned_docs/version-1.3/index.html b/website/versioned_docs/version-1.3/index.html new file mode 100644 index 00000000000..a0ed6ecad0c --- /dev/null +++ b/website/versioned_docs/version-1.3/index.html @@ -0,0 +1,14 @@ + + + + + + + Hydra - Docs + + + If you are not redirected automatically, follow this link. + + diff --git a/website/versioned_docs/version-1.3/intro.md b/website/versioned_docs/version-1.3/intro.md new file mode 100644 index 00000000000..d176bc5f82c --- /dev/null +++ b/website/versioned_docs/version-1.3/intro.md @@ -0,0 +1,166 @@ +--- +id: intro +title: Getting started +sidebar_label: Getting started +--- + +import useBaseUrl from '@docusaurus/useBaseUrl'; +import Link from '@docusaurus/Link'; + +## Introduction +Hydra is an open-source Python framework that simplifies the development of research and other complex applications. +The key feature is the ability to dynamically create a hierarchical configuration by composition and override it through config files and the command line. +The name Hydra comes from its ability to run multiple similar jobs - much like a Hydra with multiple heads. + +### Key features: + +* Hierarchical configuration composable from multiple sources +* Configuration can be specified or overridden from the command line +* Dynamic command line tab completion +* Run your application locally or launch it to run remotely +* Run multiple jobs with different arguments with a single command + +## Versions + +Hydra supports Linux, Mac and Windows. +Use the version switcher in the top bar to switch between documentation versions. + +| | Version | Release notes | Python Versions | +| -------|---------------------------|-------------------------------------------------------------------------------------| -------------------| +| ►| 1.3 (Stable) | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.3.0) | **3.6 - 3.11** | +| | 1.2 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.2.0) | **3.6 - 3.10** | +| | 1.1 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.1.1) | **3.6 - 3.9** | +| | 1.0 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v1.0.7) | **3.6 - 3.8** | +| | 0.11 | [Release notes](https://github.com/facebookresearch/hydra/releases/tag/v0.11.3) | **2.7, 3.5 - 3.8** | + + +## Quick start guide +This guide will show you some of the most important features you get by writing your application as a Hydra app. +If you only want to use Hydra for config composition, check out Hydra's [compose API](advanced/compose_api.md) for an alternative. +Please also read the full [tutorial](tutorials/basic/your_first_app/1_simple_cli.md) to gain a deeper understanding. + +### Installation +```commandline +pip install hydra-core --upgrade +``` + +### Basic example +Config: +```yaml title="conf/config.yaml" +db: + driver: mysql + user: omry + pass: secret +``` +Application: +```python {4-6} title="my_app.py" +import hydra +from omegaconf import DictConfig, OmegaConf + +@hydra.main(version_base=None, config_path="conf", config_name="config") +def my_app(cfg : DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) + +if __name__ == "__main__": + my_app() +``` +You can learn more about OmegaConf [here](https://omegaconf.readthedocs.io/en/latest/usage.html#access-and-manipulation) later. + +`config.yaml` is loaded automatically when you run your application +```yaml +$ python my_app.py +db: + driver: mysql + pass: secret + user: omry +``` + +You can override values in the loaded config from the command line: +```yaml {4-5} +$ python my_app.py db.user=root db.pass=1234 +db: + driver: mysql + user: root + pass: 1234 +``` + +### Composition example +You may want to alternate between two different databases. To support this create a `config group` named db, +and place one config file for each alternative inside: +The directory structure of our application now looks like: +```text +├── conf +│   ├── config.yaml +│   ├── db +│   │   ├── mysql.yaml +│   │   └── postgresql.yaml +│   └── __init__.py +└── my_app.py +``` + +Here is the new config: +```yaml title="conf/config.yaml" +defaults: + - db: mysql +``` + +`defaults` is a special directive telling Hydra to use db/mysql.yaml when composing the configuration object. +The resulting cfg object is a composition of configs from defaults with configs specified in your `config.yaml`. + +You can now choose which database configuration to use and override values from the command line: +```yaml +$ python my_app.py db=postgresql db.timeout=20 +db: + driver: postgresql + pass: drowssap + timeout: 20 + user: postgres_user +``` +You can have as many config groups as you need. + +### Multirun +You can run your function multiple times with different configuration easily with the `--multirun|-m` flag. + + +``` +$ python my_app.py --multirun db=mysql,postgresql +[HYDRA] Sweep output dir : multirun/2020-01-09/01-16-29 +[HYDRA] Launching 2 jobs locally +[HYDRA] #0 : db=mysql +db: + driver: mysql + pass: secret + user: omry + +[HYDRA] #1 : db=postgresql +db: + driver: postgresql + pass: drowssap + timeout: 10 + user: postgres_user +``` + +There is a whole lot more to Hydra. Read the [tutorial](tutorials/basic/your_first_app/1_simple_cli.md) to learn more. + +## Other stuff +### Community +Ask questions on github or StackOverflow (Use the tag #fb-hydra): +* [github](https://github.com/facebookresearch/hydra/discussions) +* [StackOverflow](https://stackoverflow.com/questions/tagged/fb-hydra) + +Follow Hydra on Twitter and Facebook: +* [Facebook page](https://www.facebook.com/Hydra-Framework-109364473802509/) +* [Twitter](https://twitter.com/Hydra_Framework) + + +### Citing Hydra +If you use Hydra in your research please use the following BibTeX entry: +```text +@Misc{Yadan2019Hydra, + author = {Omry Yadan}, + title = {Hydra - A framework for elegantly configuring complex applications}, + howpublished = {Github}, + year = {2019}, + url = {https://github.com/facebookresearch/hydra} +} +``` diff --git a/website/versioned_docs/version-1.3/patterns/configuring_experiments.md b/website/versioned_docs/version-1.3/patterns/configuring_experiments.md new file mode 100644 index 00000000000..b59cb96aeef --- /dev/null +++ b/website/versioned_docs/version-1.3/patterns/configuring_experiments.md @@ -0,0 +1,186 @@ +--- +id: configuring_experiments +title: Configuring Experiments +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +### Problem +A common problem is maintaining multiple configurations of an application. This can get especially +tedious when the configuration differences span multiple dimensions. +This pattern shows how to cleanly support multiple configurations, with each configuration file only specifying +the changes to the master (default) configuration. + +### Solution +Create a config file specifying the overrides to the default configuration, and then call it via the command line. +e.g. `$ python my_app.py +experiment=fast_mode`. + +To avoid clutter, we place the experiment config files in dedicated config group called *experiment*. + +### Example +In this example, we will create configurations for each of the server and database pairings that we want to benchmark. + +The default configuration is: + +
+
+ +```yaml title="config.yaml" +defaults: + - db: mysql + - server: apache + + + + + +``` +
+
+ +```yaml title="db/mysql.yaml" +name: mysql +``` + +```yaml title="server/apache.yaml" +name: apache +port: 80 +``` +
+ + +
+ +```yaml title="db/sqlite.yaml" +name: sqlite +``` + +```yaml title="server/nginx.yaml" +name: nginx +port: 80 +``` +
+
+ + + +
+
+ +```text title="Directory structure" +├── config.yaml +├── db +│ ├── mysql.yaml +│ └── sqlite.yaml +└── server + ├── apache.yaml + └── nginx.yaml +``` +
+
+ +```yaml title="$ python my_app.py" +db: + name: mysql +server: + name: apache + port: 80 + + +``` +
+
+ +The benchmark config files specify the deltas from the default configuration: + +
+
+ +```yaml title="experiment/aplite.yaml" +# @package _global_ +defaults: + - override /db: sqlite + + +server: + port: 8080 +``` +
+
+ +```yaml title="experiment/nglite.yaml" +# @package _global_ +defaults: + - override /db: sqlite + - override /server: nginx + +server: + port: 8080 +``` +
+
+ +
+
+ +```yaml title="$ python my_app.py +experiment=aplite" +db: + name: sqlite +server: + name: apache + port: 8080 +``` +
+
+ +```yaml title="$ python my_app.py +experiment=nglite" +db: + name: sqlite +server: + name: nginx + port: 8080 +``` +
+ +
+ +Key concepts: +* **\# @package \_global\_** + Changes specified in this config should be interpreted as relative to the \_global\_ package. + We could instead place *nglite.yaml* and *aplite.yaml* next to *config.yaml* and omit this line. +* **The overrides of /db and /server are absolute paths.** + This is necessary because they are outside of the experiment directory. + +Running the experiments from the command line requires prefixing the experiment choice with a `+`. +The experiment config group is an addition, not an override. + +### Sweeping over experiments + +This approach also enables sweeping over those experiments to easily compare their results: + +```text title="$ python my_app.py --multirun +experiment=aplite,nglite" +[HYDRA] Launching 2 jobs locally +[HYDRA] #0 : +experiment=aplite +db: + name: sqlite +server: + name: apache + port: 8080 + +[HYDRA] #1 : +experiment=nglite +db: + name: sqlite +server: + name: nginx + port: 8080 +``` + +To run all the experiments, use the [glob](../advanced/override_grammar/extended.md#glob-choice-sweep) syntax: +```text title="$ python my_app.py --multirun '+experiment=glob(*)'" +[HYDRA] #0 : +experiment=aplite +... +[HYDRA] #1 : +experiment=nglite +... +``` diff --git a/website/versioned_docs/version-1.3/patterns/configuring_plugins.md b/website/versioned_docs/version-1.3/patterns/configuring_plugins.md new file mode 100644 index 00000000000..e957bb4122a --- /dev/null +++ b/website/versioned_docs/version-1.3/patterns/configuring_plugins.md @@ -0,0 +1,189 @@ +--- +id: configuring_plugins +title: Configuring Plugins +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + +Hydra plugins usually come with sensible defaults which work with minimal configuration. +There are two primary ways to customize the configuration of a plugin: +- Overriding it directly in your primary config +- Extending the config and using it from your primary config. + +The first method is the simpler, but it makes it harder to switch to a different plugin configuration. +The second method is a bit more complicated, but makes it easier to switch between different plugin configurations. + + +The following methods apply to all Hydra plugins. In the following examples, we will configure an imaginary Launcher plugin +`MoonLauncher`. The Launcher has two modes: `falcon9`, which actually launches the application to the Moon and +`sim` which simulates a launch. + +The config schema for MoonLauncher looks like: + + + +
+
+ +```python +@dataclass +class Falcon9Conf: + ton_fuel: int = 10 + + + + +``` +
+
+ +```python +@dataclass +class Simulation: + ton_fuel: int = 10 + window_size: + width: 1024 + height: 768 + +``` +
+
+ + + +### Overriding in primary config +We can directly override Launcher config in primary config. + +
+
+ +```yaml title="config.yaml" +a: 1 + +hydra: + launcher: + ton_fuel: 2 + + + + + + + + + +``` +
+
+ +```commandline title="command-line override" +hydra/launcher=falcon9 + + +``` +```yaml title="resulting launcher config" {3} +hydra: + launcher: + ton_fuel: 2 + + + +``` +
+
+ +```commandline title="command-line override" +hydra/launcher=sim + + +``` +```yaml title="resulting launcher config" {3} +hydra: + launcher: + ton_fuel: 2 + window_size: + width: 1024 + height: 768 +``` +
+
+ + +This approach makes the assumption that the Launcher used has all the fields we are overriding. +If we wanted to override a field that exists in the Simulation Launcher but not in the Falcon9 Launcher, +like `window_size.width`, we would no longer be able to use the Falcon9 Launcher! The next section solves this problem. + +### Extending plugin default config + +This section assumes that you are familiar with the contents of [Common Patterns/Extending Configs](patterns/extending_configs.md). + +Extending plugin default config has several advantages: +- Separate configuration concerns, keep primary config clean. +- Easier to switch between different plugin configurations. +- Provides flexibility when a Plugin has different modes +that requires different schema. + + +Say that we want to override certain values for different Launcher mode: + +
+
+ +```yaml title="hydra/launcher/my_falcon9.yaml" {4} +defaults: + - falcon9 + +ton_fuel: 2 + + +``` +
+
+ +```yaml title="hydra/sweeper/my_sim.yaml" {5} +defaults: + - sim + +window_size: + width: 768 + +``` +
+
+ +We can easily user command-line overrides to get the configuration needed: +
+ +
+ +```commandline title="command-line override" +hydra/launcher=my_falcon9 + + +``` +```yaml title="resulting launcher config" {3} +hydra: + launcher: + ton_fuel: 2 + + + +``` +
+
+ +```commandline title="command-line override" +hydra/launcher=my_sim + + +``` +```yaml title="resulting launcher config" {5} +hydra: + launcher: + ton_fuel: 10 + window_size: + width: 768 + height: 768 +``` +
+
diff --git a/website/versioned_docs/version-1.3/patterns/extending_configs.md b/website/versioned_docs/version-1.3/patterns/extending_configs.md new file mode 100644 index 00000000000..56971db55b5 --- /dev/null +++ b/website/versioned_docs/version-1.3/patterns/extending_configs.md @@ -0,0 +1,77 @@ +--- +id: extending_configs +title: Extending Configs +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +A common pattern is to extend an existing config, overriding and/or adding new config values to it. +The extension is done by including the base configuration, and then overriding the chosen values in the current config. + +:::info +This page assumes that you are familiar with the contents of [The Defaults List](../advanced/defaults_list.md). +::: + +#### Extending a config from the same config group: + +
+
+ +```yaml title="config.yaml" +defaults: + - db: mysql + + + + + +``` +
+
+ +```yaml title="db/mysql.yaml" {2} +defaults: + - base_mysql + +user: omry +password: secret +port: 3307 +encoding: utf8 +``` +
+
+ +```yaml title="db/base_mysql.yaml" +host: localhost +port: 3306 +user: ??? +password: ??? + + + +``` +
+
+ +Output: +```yaml title="$ python my_app.py" +db: + host: localhost # from db/base_mysql + port: 3307 # overridden by db/mysql.yaml + user: omry # populated by db/mysql.yaml + password: secret # populated by db/mysql.yaml + encoding: utf8 # added by db/mysql.yaml +``` + +#### Extending a config from another config group: +To extend a config from a different config group, include it using an absolute path (/), and override +the package to *\_here\_*. (*\_here\_* is described in [Packages](../advanced/overriding_packages.md#default-list-package-keywords)) + +```yaml title="db/mysql.yaml" {2} +defaults: + - /db_schema/base_mysql@_here_ +``` + +It is otherwise identical to extending a config within the same config group. diff --git a/website/versioned_docs/version-1.3/patterns/select_multiple_configs_from_config_group.md b/website/versioned_docs/version-1.3/patterns/select_multiple_configs_from_config_group.md new file mode 100644 index 00000000000..28bc82c1b77 --- /dev/null +++ b/website/versioned_docs/version-1.3/patterns/select_multiple_configs_from_config_group.md @@ -0,0 +1,177 @@ +--- +id: select_multiple_configs_from_config_group +title: Selecting multiple configs from a Config Group +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +### Problem +In some scenarios, one may need to select multiple configs from the same Config Group. + +### Solution +Use a list of config names as the value of the config group in the Defaults List or in the command line. + +### Example + +In this example, we configure a server. The server can host multiple websites at the same time. + +
+
+ +```text title="Config directory" +├── config.yaml +└── server + ├── apache.yaml + └── site + ├── amazon.yaml + ├── fb.yaml + └── google.yaml +``` +
+
+ +```yaml title="config.yaml" +defaults: + - server/apache + + + + + +``` +
+ +
+ +```yaml title="server/apache.yaml" {3,4} +defaults: + - site: + - fb + - google + +host: localhost +port: 443 +``` +
+ +
+ +```yaml title="server/site/amazon.yaml" +amazon: + domain: amazon.com +``` +
+
+ +```yaml title="server/site/fb.yaml" +fb: + domain: facebook.com +``` +
+
+ +```yaml title="server/site/google.yaml" +google: + domain: google.com +``` +
+
+ +Output: +```yaml title="$ python my_app.py" {3,5} +server: + site: + fb: + domain: facebook.com + google: + domain: google.com + host: localhost + port: 443 +``` + +Override the selected sites from the command line by passing a list. e.g: +```yaml title="$ python my_app.py 'server/site=[google,amazon]'" {3,5} +server: + site: + google: + domain: google.com + amazon: + domain: amazon.com + host: localhost + port: 443 +``` + + +### Overriding packages +You can relocate the package of all the configs in the list. e.g: + +
+
+ +```yaml title="server/apache.yaml" {2} +defaults: + - site@https: + - fb + - google + + +``` +
+ +
+ +```yaml title="$ python my_app.py" {2} +server: + https: + fb: + domain: facebook.com + google: + domain: google.com +``` +
+
+ +When overriding the selected configs of config groups with overridden packages you need to use the package. e.g: +```yaml title="$ python my_app.py server/site@server.https=amazon" +server: + https: + amazon: + domain: amazon.com + host: localhost + port: 443 +``` + + +### Implementation considerations + +A nested list in the Defaults List is interpreted as a list of non-overridable configs: + +
+
+ +```yaml title="server/apache.yaml" {3,4} +defaults: + - site: + - fb + - google +``` +
+
+ +```yaml title="Equivalent to" {2,3} +defaults: + - site/fb + - site/google + +``` +
+
+ +All default package for all the configs in `server/site` is `server.site`. +This example uses an explicit nesting level inside each of the website configs to prevent them stepping over one another: +```yaml title="server/site/amazon.yaml" {1} +amazon: + ... +``` diff --git a/website/versioned_docs/version-1.3/patterns/specializing_config.md b/website/versioned_docs/version-1.3/patterns/specializing_config.md new file mode 100644 index 00000000000..084b7d3fc23 --- /dev/null +++ b/website/versioned_docs/version-1.3/patterns/specializing_config.md @@ -0,0 +1,84 @@ +--- +id: specializing_config +title: Specializing configuration +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +In some cases the desired configuration should depend on other configuration choices. +For example, You may want to use only 5 layers in your Alexnet model if the dataset of choice is cifar10, and the default 7 otherwise. + +We can start with a config that looks like this: +### initial config.yaml +```yaml +defaults: + - dataset: imagenet + - model: alexnet +``` + +We want to specialize the config based on the choice of the selected dataset and model: +Furthermore, we only want to do it for cifar10 and alexnet and not for 3 other combinations. + +OmegaConf supports value interpolation, we can construct a value that would - at runtime - be a function of other values. +The idea is that we can add another element to the defaults list that would load a file name that depends on those two values: +### modified config.yaml +```yaml +defaults: + - dataset: imagenet + - model: alexnet + - optional dataset_model: ${dataset}_${model} +``` + +Let's break this down: +#### dataset_model +The key `dataset_model` is an arbitrary directory, it can be anything unique that makes sense, including nested directory like `dataset/model`. + +#### ${dataset}_${model} +the value `${dataset}_${model}` is using OmegaConf's [variable interpolation](https://omegaconf.readthedocs.io/en/latest/usage.html#variable-interpolation) syntax. +At runtime, that value would resolve to *imagenet_alexnet*, or *cifar_resnet* - depending on the values of defaults.dataset and defaults.model. + +:::info +This is non-standard interpolation and there are some subtle differences and limitations. +::: + + +#### optional +By default, Hydra fails with an error if a config specified in the defaults does not exist. +In this case we only want to specialize cifar10 + alexnet, not all 4 combinations. +the keyword `optional` tells Hydra to just continue if it can't find this file. + +When specializing config, you usually want to only specify what's different, and not the whole thing. +We want the model for alexnet, when trained on cifar - to have 5 layers. + +### dataset_model/cifar10_alexnet.yaml +```yaml +# @package _global_ + +model: + num_layers: 5 +``` + +Let's check. Running with the default uses imagenet, so we don't get the specialized version of: + +```yaml +$ python example.py +dataset: + name: imagenet + path: /datasets/imagenet +model: + num_layers: 7 + type: alexnet +``` + +Running with cifar10 dataset, we do get 5 for num_layers: +```yaml +$ python example.py dataset=cifar10 +dataset: + name: cifar10 + path: /datasets/cifar10 +model: + num_layers: 5 + type: alexnet +``` diff --git a/website/versioned_docs/version-1.3/patterns/write_protect_config_node.md b/website/versioned_docs/version-1.3/patterns/write_protect_config_node.md new file mode 100644 index 00000000000..77b5947e106 --- /dev/null +++ b/website/versioned_docs/version-1.3/patterns/write_protect_config_node.md @@ -0,0 +1,56 @@ +--- +id: write_protect_config_node +title: Read-only config +--- +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +### Problem +Sometimes you want to prevent a config node from being changed accidentally. + +### Solution +Structured Configs can enable it by passing [frozen=True](https://omegaconf.readthedocs.io/en/latest/structured_config.html#frozen) in the dataclass definition. +Using Structured Configs, you can annotate a dataclass as frozen. This is recursive and applies to all child nodes. + +This will prevent modifications via code, command line overrides and config composition. + +
+
+ +```python title="frozen.py" {1} +@dataclass(frozen=True) +class SerialPort: + baud_rate: int = 19200 + data_bits: int = 8 + stop_bits: int = 1 + + +cs = ConfigStore.instance() +cs.store(name="config", node=SerialPort) + + +@hydra.main(version_base=None, config_name="config") +def my_app(cfg: SerialPort) -> None: + print(cfg) + + +if __name__ == "__main__": + my_app() +``` +
+ +```shell script title="Output" +$ python frozen.py data_bits=10 +Error merging override data_bits=10 +Cannot change read-only config container + full_key: data_bits + object_type=SerialPort +``` +
+ + + diff --git a/website/versioned_docs/version-1.3/plugins/ax_sweeper.md b/website/versioned_docs/version-1.3/plugins/ax_sweeper.md new file mode 100644 index 00000000000..0eddd3877ab --- /dev/null +++ b/website/versioned_docs/version-1.3/plugins/ax_sweeper.md @@ -0,0 +1,125 @@ +--- +id: ax_sweeper +title: Ax Sweeper plugin +sidebar_label: Ax Sweeper plugin +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + +[![PyPI](https://img.shields.io/pypi/v/hydra-ax-sweeper)](https://img.shields.io/pypi/v/hydra-ax-sweeper) +![PyPI - License](https://img.shields.io/pypi/l/hydra-ax-sweeper) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hydra-ax-sweeper) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/hydra-ax-sweeper.svg)](https://pypistats.org/packages/hydra-ax-sweeper) + + +This plugin provides a mechanism for Hydra applications to use the [Adaptive Experimentation Platform, aka Ax](https://ax.dev/). Ax can optimize any experiment - machine learning experiments, A/B tests, and simulations. + +### Installation +```commandline +pip install hydra-ax-sweeper --upgrade +``` + +### Usage +Once installed, add `hydra/sweeper=ax` to your command line. Alternatively, override `hydra/sweeper` in your config: + +```yaml +defaults: + - override hydra/sweeper: ax +``` + +We include an example of how to use this plugin. The file example/banana.py +implements the [Rosenbrock function (aka Banana function)](https://en.wikipedia.org/wiki/Rosenbrock_function). +The return value of the function should be the value that we want to optimize. + + +To compute the best parameters for the Banana function, clone the code and run the following command in the `plugins/hydra_ax_sweeper` directory: + +``` +python example/banana.py -m 'banana.x=int(interval(-5, 5))' 'banana.y=interval(-5, 10.1)' +``` + +The output of a run looks like: + +``` +[HYDRA] AxSweeper is optimizing the following parameters: +banana.x: range=[-5, 5] +banana.y: range=[-5.0, 10.1] +ax.modelbridge.dispatch_utils: Using Bayesian Optimization generation strategy: GenerationStrategy(name='Sobol+GPEI', steps=[Sobol for 5 trials, GPEI for subsequent trials]). Iterations after 5 will take longer to generate due to model-fitting. +[HYDRA] AxSweeper is launching 5 jobs +[HYDRA] Launching 5 jobs locally +[HYDRA] #0 : banana.x=2 banana.y=-0.988 +[__main__][INFO] - Banana_Function(x=2, y=-0.988)=2488.883 +[HYDRA] #1 : banana.x=-1 banana.y=7.701 +[__main__][INFO] - Banana_Function(x=-1, y=7.701)=4493.987 +[HYDRA] #2 : banana.x=-1 banana.y=-3.901 +[__main__][INFO] - Banana_Function(x=-1, y=-3.901)=2406.259 +[HYDRA] #3 : banana.x=-1 banana.y=0.209 +[__main__][INFO] - Banana_Function(x=-1, y=0.209)=66.639 +[HYDRA] #4 : banana.x=4 banana.y=-4.557 +[__main__][INFO] - Banana_Function(x=4, y=-4.557)=42270.006 +[HYDRA] New best value: 66.639, best parameters: {'banana.x': -1, 'banana.y': 0.209} +``` + +In this example, we set the range of `x` parameter as an integer in the interval `[-5, 5]` and the range of `y` parameter as a float in the interval `[-5, 10.1]`. Note that in the case of `x`, we used `int(interval(...))` and hence only integers are sampled. In the case of `y`, we used `interval(...)` which refers to a floating-point interval. Other supported formats are fixed parameters (e.g.` banana.x=5.0`), choice parameters (eg `banana.x=choice(1,2,3)`) and range (eg `banana.x=range(1, 10)`). Note that `interval`, `choice` etc. are functions provided by Hydra, and you can read more about them [here](/advanced/override_grammar/extended.md). An important thing to remember is, use [`interval`](/advanced/override_grammar/extended.md#interval-sweep) when we want Ax to sample values from an interval. [`RangeParameter`](https://ax.dev/api/ax.html#ax.RangeParameter) in Ax is equivalent to `interval` in Hydra. Remember to use `int(interval(...))` if you want to sample only integer points from the interval. [`range`](/advanced/override_grammar/extended.md#range-sweep) can be used as an alternate way of specifying choice parameters. For example `python example/banana.py -m banana.x=choice(1, 2, 3, 4)` is equivalent to `python example/banana.py -m banana.x=range(1, 5)`. + +The values of the `x` and `y` parameters can also be set using the config file `plugins/hydra_ax_sweeper/example/conf/config.yaml`. For instance, the configuration corresponding to the commandline arguments is as follows: + +``` +banana.x: + type: range + bounds: [-5, 5] + +banana.y: + type: range + bounds: [-5, 10.1] +``` + +To sample in log space, you can tag the commandline override with `log`. E.g. `python example/banana.py -m banana.x=tag(log, interval(1, 1000))`. You can set `log_scale: true` in the input config to achieve the same. +``` +banana.z: + type: range + bounds: [1, 100] + log_scale: true +``` + +In general, the plugin supports setting all the Ax supported [Parameters](https://ax.dev/api/core.html?highlight=range#module-ax.core.parameter) in the config. According to the [Ax documentation](https://ax.dev/api/service.html#ax.service.ax_client.AxClient.create_experiment), the required elements in the config are: + +* `name` - Name of the parameter. It is of type string. +* `type` - Type of the parameter. It can take the following values: `range`, `fixed`, or `choice`. +* `bounds` - Required only for the `range` parameters. It should be a list of two values, with the lower bound first. +* `values` - Required only for the `choice` parameters. It should be a list of values. +* `value` - Required only for the `fixed` parameters. It should be a single value. + +Note that if you want to sample integers in the range `-5` to `5`, you need to specify the range as `int(interval(-5, 5))` (in the command line) or `[-5, 5]` (in config). If you want to sample floats in range `-5` to `5`, you need to specify the range as `interval(-5, 5)` (in the command line) or `[-5.0, 5.0]` (in config). + +The Ax Sweeper assumes the optimized function is a noisy function with unknown measurement uncertainty. +This can be changed by overriding the `is_noisy` parameter to False, which specifies that each measurement is exact, i.e., each measurement has a measurement uncertainty of zero. + +If measurement uncertainty is known or can be estimated (e.g., via a heuristic or via the [standard error of the mean](https://en.wikipedia.org/wiki/Standard_error) of repeated measurements), the measurement function can return the tuple `(measurement_value, measurement_uncertainty)` instead of a scalar value. + +The parameters for the optimization process can also be set in the config file. Specifying the Ax config is optional. You can discover the Ax Sweeper parameters with: + +```yaml title="$ python your_app.py hydra/sweeper=ax --cfg hydra -p hydra.sweeper" +# @package hydra.sweeper +_target_: hydra_plugins.hydra_ax_sweeper.ax_sweeper.AxSweeper +max_batch_size: null +ax_config: + max_trials: 10 + early_stop: + minimize: true + max_epochs_without_improvement: 10 + epsilon: 1.0e-05 + experiment: + name: null + objective_name: objective + minimize: true + parameter_constraints: null + outcome_constraints: null + status_quo: null + client: + verbose_logging: false + random_seed: null + is_noisy: true + params: {} +``` +There are several standard approaches for configuring plugins. Check [this page](../patterns/configuring_plugins.md) for more information. \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/plugins/colorlog.md b/website/versioned_docs/version-1.3/plugins/colorlog.md new file mode 100644 index 00000000000..79ebe1eb000 --- /dev/null +++ b/website/versioned_docs/version-1.3/plugins/colorlog.md @@ -0,0 +1,35 @@ +--- +id: colorlog +title: Colorlog plugin +sidebar_label: Colorlog plugin +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + +[![PyPI](https://img.shields.io/pypi/v/hydra-colorlog)](https://pypi.org/project/hydra-colorlog/) +![PyPI - License](https://img.shields.io/pypi/l/hydra-colorlog) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hydra-colorlog) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/hydra-colorlog.svg)](https://pypistats.org/packages/hydra-colorlog) + +Adds colorlog colored logs for `hydra/job_logging` and `hydra/hydra_logging`. + + +### Installation +```commandline +pip install hydra_colorlog --upgrade +``` + +### Usage +Override `hydra/job_logging` and `hydra/hydra_logging` in your config: + +```yaml +defaults: + - override hydra/job_logging: colorlog + - override hydra/hydra_logging: colorlog +``` + +There are several standard approaches for configuring plugins. Check [this page](../patterns/configuring_plugins.md) for more information. + +See included example application. + +![Colored log output](/plugins/colorlog/colorlog.png) diff --git a/website/versioned_docs/version-1.3/plugins/joblib_launcher.md b/website/versioned_docs/version-1.3/plugins/joblib_launcher.md new file mode 100644 index 00000000000..7323f3599da --- /dev/null +++ b/website/versioned_docs/version-1.3/plugins/joblib_launcher.md @@ -0,0 +1,69 @@ +--- +id: joblib_launcher +title: Joblib Launcher plugin +sidebar_label: Joblib Launcher plugin +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + +[![PyPI](https://img.shields.io/pypi/v/hydra-joblib-launcher)](https://pypi.org/project/hydra-joblib-launcher/) +![PyPI - License](https://img.shields.io/pypi/l/hydra-joblib-launcher) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hydra-joblib-launcher) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/hydra-joblib-launcher.svg)](https://pypistats.org/packages/hydra-joblib-launcher) + +The Joblib Launcher plugin provides a launcher for parallel tasks based on [`Joblib.Parallel`](https://joblib.readthedocs.io/en/latest/parallel.html). + +### Installation +```commandline +pip install hydra-joblib-launcher --upgrade +``` + +### Usage +Once installed, add `hydra/launcher=joblib` to your command line. Alternatively, override `hydra/launcher` in your config: + +```yaml +defaults: + - override hydra/launcher: joblib +``` +By default, process-based parallelism using all available CPU cores is used. By overriding the default configuration, it is e.g. possible limit the number of parallel executions. + +The JobLibLauncherConf backing the config is defined here: + +You can discover the Joblib Launcher parameters with: +```yaml title="$ python your_app.py hydra/launcher=joblib --cfg hydra -p hydra.launcher" +# @package hydra.launcher +_target_: hydra_plugins.hydra_joblib_launcher.joblib_launcher.JoblibLauncher +n_jobs: 10 +backend: null +prefer: processes +require: null +verbose: 0 +timeout: null +pre_dispatch: 2*n_jobs +batch_size: auto +temp_folder: null +max_nbytes: null +mmap_mode: r +``` +There are several standard approaches for configuring plugins. Check [this page](../patterns/configuring_plugins.md) for more information. + +See [`Joblib.Parallel` documentation](https://joblib.readthedocs.io/en/latest/parallel.html) for full details about the parameters above. + +
+ +An example application using this launcher is provided in the plugin repository. + +Starting the app with `python my_app.py --multirun task=1,2,3,4,5` will launch five parallel executions: + +```text +$ python my_app.py --multirun task=1,2,3,4,5 +[HYDRA] Joblib.Parallel(n_jobs=-1,verbose=0,timeout=None,pre_dispatch=2*n_jobs,batch_size=auto,temp_folder=None,max_nbytes=None,mmap_mode=r,backend=loky) is launching 5 jobs +[HYDRA] Launching jobs, sweep output dir : multirun/2020-02-18/10-00-00 +[__main__][INFO] - Process ID 14336 executing task 2 ... +[__main__][INFO] - Process ID 14333 executing task 1 ... +[__main__][INFO] - Process ID 14334 executing task 3 ... +[__main__][INFO] - Process ID 14335 executing task 4 ... +[__main__][INFO] - Process ID 14337 executing task 5 ... +``` diff --git a/website/versioned_docs/version-1.3/plugins/nevergrad_sweeper.md b/website/versioned_docs/version-1.3/plugins/nevergrad_sweeper.md new file mode 100644 index 00000000000..2f672115595 --- /dev/null +++ b/website/versioned_docs/version-1.3/plugins/nevergrad_sweeper.md @@ -0,0 +1,174 @@ +--- +id: nevergrad_sweeper +title: Nevergrad Sweeper plugin +sidebar_label: Nevergrad Sweeper plugin +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + +[![PyPI](https://img.shields.io/pypi/v/hydra-nevergrad-sweeper)](https://pypi.org/project/hydra-nevergrad-sweeper/) +![PyPI - License](https://img.shields.io/pypi/l/hydra-nevergrad-sweeper) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hydra-nevergrad-sweeper) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/hydra-nevergrad-sweeper.svg)](https://pypistats.org/packages/hydra-nevergrad-sweeper) + +[Nevergrad](https://facebookresearch.github.io/nevergrad/) is a derivative-free optimization platform providing a library of state-of-the-art algorithms for hyperparameter search. This plugin provides Hydra applications a mechanism to use [Nevergrad](https://facebookresearch.github.io/nevergrad/) algorithms to optimize experiment/application parameters. + +### Installation +```commandline +pip install hydra-nevergrad-sweeper --upgrade +``` + +### Usage +Once installed, add `hydra/sweeper=nevergrad` to your command. Alternatively, override `hydra/sweeper` in your config: + +```yaml +defaults: + - override hydra/sweeper: nevergrad +``` + +The default configuration is defined and documented here. +There are several standard approaches for configuring plugins. Check [this page](../patterns/configuring_plugins.md) for more information. + +## Example of training using Nevergrad hyperparameter search + +We include an example of how to use this plugin. The file example/my_app.py implements an example of minimizing a (dummy) function using a mixture of continuous and discrete parameters. + +You can discover the Nevergrad sweeper parameters with: +```yaml title="$ python your_app hydra/sweeper=nevergrad --cfg hydra -p hydra.sweeper" +# @package hydra.sweeper +_target_: hydra_plugins.hydra_nevergrad_sweeper.core.NevergradSweeper +optim: + optimizer: NGOpt + budget: 80 + num_workers: 10 + noisy: false + maximize: false + seed: null + max_failure_rate: 0.0 +parametrization: + db: + - mnist + - cifar + lr: + init: 0.02 + step: 2.0 + log: true + dropout: + lower: 0.0 + upper: 1.0 + batch_size: + lower: 4 + upper: 16 + integer: true +``` + +The function decorated with `@hydra.main()` returns a float which we want to minimize, the minimum is 0 and reached for: +```yaml +db: mnist +lr: 0.12 +dropout: 0.33 +batch_size=4 +``` + +To run hyperparameter search and look for the best parameters for this function, clone the code and run the following command in the `plugins/hydra_nevergrad_sweeper` directory: +```bash +python example/my_app.py -m +``` + +You can also override the search space parametrization: +```bash +python example/my_app.py --multirun db=mnist,cifar batch_size=4,8,16 \ +'lr=tag(log, interval(0.001, 1))' 'dropout=interval(0,1)' +``` + +The initialization of the sweep and the first 5 evaluations (out of 100) look like this: + +```text +[2020-10-08 20:13:53,592][HYDRA] NevergradSweeper(optimizer=NGOpt, budget=100, num_workers=10) minimization +[2020-10-08 20:13:53,593][HYDRA] with parametrization Dict(batch_size=Choice(choices=Tuple(4,8,16),weights=Array{(1,3)}),db=Choice(choices=Tuple(mnist,cifar),weights=Array{(1,2)}),dropout=Scalar{Cl(0,1,b)}[sigma=Log{exp=2.0}],lr=Log{exp=3.162277660168379,Cl(0.001,1,b)}):{'db': 'mnist', 'lr': 0.03162277660168379, 'dropout': 0.5, 'batch_size': 8} +[2020-10-08 20:13:53,593][HYDRA] Sweep output dir: multirun/2020-10-08/20-13-53 +[2020-10-08 20:13:55,023][HYDRA] Launching 10 jobs locally +[2020-10-08 20:13:55,023][HYDRA] #0 : db=mnist lr=0.03162277660168379 dropout=0.5 batch_size=16 +[2020-10-08 20:13:55,217][__main__][INFO] - dummy_training(dropout=0.500, lr=0.032, db=mnist, batch_size=16) = 13.258 +[2020-10-08 20:13:55,218][HYDRA] #1 : db=cifar lr=0.018178519762066934 dropout=0.5061074452336254 batch_size=4 +[2020-10-08 20:13:55,408][__main__][INFO] - dummy_training(dropout=0.506, lr=0.018, db=cifar, batch_size=4) = 0.278 +[2020-10-08 20:13:55,409][HYDRA] #2 : db=cifar lr=0.10056825918734161 dropout=0.6399687427725211 batch_size=4 +[2020-10-08 20:13:55,595][__main__][INFO] - dummy_training(dropout=0.640, lr=0.101, db=cifar, batch_size=4) = 0.329 +[2020-10-08 20:13:55,596][HYDRA] #3 : db=mnist lr=0.06617542958182834 dropout=0.5059497416026679 batch_size=8 +[2020-10-08 20:13:55,812][__main__][INFO] - dummy_training(dropout=0.506, lr=0.066, db=mnist, batch_size=8) = 5.230 +[2020-10-08 20:13:55,813][HYDRA] #4 : db=mnist lr=0.16717013388679514 dropout=0.6519070394318255 batch_size=4 +... +[2020-10-08 20:14:27,988][HYDRA] Best parameters: db=cifar lr=0.11961221693764439 dropout=0.37285878409770895 batch_size=4 +``` + + +and the final 2 evaluations look like this: +```text +[HYDRA] #8 : db=mnist batch_size=4 lr=0.094 dropout=0.381 +[__main__][INFO] - my_app.py(dropout=0.381, lr=0.094, db=mnist, batch_size=4) = 1.077 +[HYDRA] #9 : db=mnist batch_size=4 lr=0.094 dropout=0.381 +[__main__][INFO] - my_app.py(dropout=0.381, lr=0.094, db=mnist, batch_size=4) = 1.077 +[HYDRA] Best parameters: db=mnist batch_size=4 lr=0.094 dropout=0.381 +``` + + +The run also creates an `optimization_results.yaml` file in your sweep folder with the parameters recommended by the optimizer: +```yaml +best_evaluated_result: 0.381 + +best_evaluated_params: + batch_size: 4 + db: mnist + dropout: 0.381 + lr: 0.094 + +name: nevergrad +``` + +## Defining the parameters + +The plugin supports two types of parameters: [Choices](https://facebookresearch.github.io/nevergrad/parametrization_ref.html#nevergrad.p.Choice) and [Scalars](https://facebookresearch.github.io/nevergrad/parametrization_ref.html#nevergrad.p.Scalar). They can be defined either through config file or commandline override. + +### Defining through commandline override +Hydra provides an override parser that supports rich syntax. More documentation can be found in ([OverrideGrammer/Basic](../advanced/override_grammar/basic.md)) and ([OverrideGrammer/Extended](../advanced/override_grammar/extended.md)). We recommend you go through them first before proceeding with this doc. + +#### Choices +To override a field with choices: +```commandline +'key=1,5' +'key=shuffle(range(1, 8))' +'key=range(1,5)' +``` + +You can tag an override with ```ordered``` to indicate it's a [```TransitionChoice```](https://facebookresearch.github.io/nevergrad/parametrization_ref.html#nevergrad.p.TransitionChoice) +```commandline +`key=tag(ordered, choice(1,2,3))` +``` + +#### Scalar +```commandline +`key=interval(1,12)` # Interval are float by default +`key=int(interval(1,8))` # Scalar bounds cast to an int +`key=tag(log, interval(1,12))` # call ng.p.Log if tagged with log +``` + +### Defining through config file +#### Choices +Choices are defined with a list in a config file. + +```yaml +db: + - mnist + - cifar +``` +#### Scalars +Scalars can be defined in config files, with fields: + - `init`: optional initial value + - `lower` : optional lower bound + - `upper`: optional upper bound + - `log`: set to `true` for log distributed values + - `step`: optional step size for looking for better parameters. In linear mode, this is an additive step; in logarithmic mode it is multiplicative. + - `integer`: set to `true` for integers (favor floats over integers whenever possible) + +Providing only `lower` and `upper` bound will set the initial value to the middle of the range and the step to a sixth of the range. +**Note**: unbounded scalars (scalars with no upper and/or lower bounds) can only be defined through a config file. diff --git a/website/versioned_docs/version-1.3/plugins/optuna_sweeper.md b/website/versioned_docs/version-1.3/plugins/optuna_sweeper.md new file mode 100644 index 00000000000..b8364809679 --- /dev/null +++ b/website/versioned_docs/version-1.3/plugins/optuna_sweeper.md @@ -0,0 +1,324 @@ +--- +id: optuna_sweeper +title: Optuna Sweeper plugin +sidebar_label: Optuna Sweeper plugin +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + +[![PyPI](https://img.shields.io/pypi/v/hydra-optuna-sweeper)](https://pypi.org/project/hydra-optuna-sweeper/) +![PyPI - License](https://img.shields.io/pypi/l/hydra-optuna-sweeper) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hydra-optuna-sweeper) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/hydra-optuna-sweeper.svg)](https://pypistats.org/packages/hydra-optuna-sweeper) + + +This plugin enables Hydra applications to utilize [Optuna](https://optuna.org) for the optimization of the parameters of experiments. + +## Installation + +This plugin requires `hydra-core>=1.1.0`. Please install it with the following command: + +```commandline +pip install hydra-core --upgrade +``` + +You can install the plugin via pip: + +```commandline +pip install hydra-optuna-sweeper --upgrade +``` +There are several standard approaches for configuring plugins. Check [this page](../patterns/configuring_plugins.md) for more information. + +## Usage + +Please set `hydra/sweeper` to `optuna` in your config file. + +```yaml +defaults: + - override hydra/sweeper: optuna +``` + +Alternatively, add `hydra/sweeper=optuna` option to your command line. + +The default configuration is here. + +## Example 1: Single-Objective Optimization + +We include an example in this directory. `example/sphere.py` implements a simple benchmark function to be minimized. + + + + +You can discover the Optuna sweeper parameters with: + +```yaml title="python example/sphere.py hydra/sweeper=optuna --cfg hydra -p hydra.sweeper" +# @package hydra.sweeper +sampler: + _target_: optuna.samplers.TPESampler + seed: 123 + consider_prior: true + prior_weight: 1.0 + consider_magic_clip: true + consider_endpoints: false + n_startup_trials: 10 + n_ei_candidates: 24 + multivariate: false + warn_independent_sampling: true +_target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper +direction: minimize +storage: null +study_name: sphere +n_trials: 20 +n_jobs: 1 +max_failure_rate: 0.0 +params: + x: range(-5.5,5.5,step=0.5) + y: choice(-5,0,5) +``` + +The function decorated with `@hydra.main()` returns a float which we want to minimize, the minimum is 0 and reached for: +```yaml +x: 0 +y: 0 +``` + +To run optimization, clone the code and run the following command in the `plugins/hydra_optuna_sweeper` directory: + +```commandline +python example/sphere.py --multirun +``` + +You can also override the search space parametrization: + +```commandline +python example/sphere.py --multirun 'x=interval(-5.0, 5.0)' 'y=interval(0, 10)' +``` + +You might find the `optimization_results.yaml` file (i.e. best params and best value) under `multirun` logs folder: + +```yaml +name: optuna +best_params: + x: 0.0 + 'y': 0 +best_value: 0.0 +``` + + +## Sampler configuration +This plugin supports Optuna's [samplers](https://optuna.readthedocs.io/en/stable/reference/samplers.html). +You can change the sampler by overriding `hydra/sweeper/sampler` or change sampler settings within `hydra.sweeper.sampler`. + +## Search space configuration + +This plugin supports Optuna's [distributions](https://optuna.readthedocs.io/en/stable/reference/distributions.html) to configure search spaces. They can be defined either through commandline override or config file. + +### Configuring through commandline override + +Hydra provides a override parser that support rich syntax. Please refer to [OverrideGrammer/Basic](../advanced/override_grammar/basic.md) and [OverrideGrammer/Extended](../advanced/override_grammar/extended.md) for details. + +#### Interval override + +By default, `interval` is converted to [`UniformDistribution`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.distributions.UniformDistribution.html). You can use [`IntUniformDistribution`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.distributions.IntUniformDistribution.html), [`LogUniformDistribution`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.distributions.LogUniformDistribution.html) or [`IntLogUniformDistribution`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.distributions.IntLogUniformDistribution.html) by casting the interval to `int` and tagging it with `log`. + +
Example for interval override + +```commandline +python example/sphere.py --multirun 'x=int(interval(-5.0, 5.0))' 'y=tag(log, interval(1, 10))' +``` + +The output is as follows: + +```commandline +[HYDRA] Study name: sphere +[HYDRA] Storage: None +[HYDRA] Sampler: TPESampler +[HYDRA] Directions: ['minimize'] +[HYDRA] Launching 1 jobs locally +[HYDRA] #0 : x=-3 y=1.6859762540733367 +[HYDRA] Launching 1 jobs locally +[HYDRA] #1 : x=1 y=5.237816870668193 +... +[HYDRA] Best parameters: {'x': 0, 'y': 1.0929184723430116} +[HYDRA] Best value: 1.1944707871885822 +``` + +
+ +#### Range override + +`range` is converted to [`IntUniformDistribution`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.distributions.IntUniformDistribution.html). If you apply `shuffle` to `range`, [`CategoricalDistribution`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.distributions.CategoricalDistribution.html) is used instead. +If any of `range`'s start, stop or step is of type float, it will be converted to [`DiscreteUniformDistribution`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.distributions.DiscreteUniformDistribution.html) + +
Example for range override + +```commandline +python example/sphere.py --multirun 'x=range(-5.0, 5.0)' 'y=shuffle(range(-5, 5))' +``` + +The output is as follows: + +```commandline +[HYDRA] Study name: sphere +[HYDRA] Storage: None +[HYDRA] Sampler: TPESampler +[HYDRA] Directions: ['minimize'] +[HYDRA] Launching 1 jobs locally +[HYDRA] #0 : x=-3 y=-4 +[HYDRA] Launching 1 jobs locally +[HYDRA] #1 : x=1 y=-1 +... +[HYDRA] Best parameters: {'x': 0, 'y': -1} +[HYDRA] Best value: 1.0 +``` + +
+ +#### Choice override + +`choice` is converted to [`CategoricalDistribution`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.distributions.CategoricalDistribution.html). + +
Example for choice override + +```commandline +python example/sphere.py --multirun 'x=choice(-5.0, 0.0, 5.0)' 'y=choice(0, 1, 2, 3, 4, 5)' +``` + +The output is as follows: + +```commandline +[HYDRA] Study name: sphere +[HYDRA] Storage: None +[HYDRA] Sampler: TPESampler +[HYDRA] Directions: ['minimize'] +[HYDRA] Launching 1 jobs locally +[HYDRA] #0 : x=5.0 y=5 +[HYDRA] Launching 1 jobs locally +[HYDRA] #1 : x=5.0 y=2 +... +[HYDRA] Best parameters: {'x': 0.0, 'y': 0} +[HYDRA] Best value: 0.0 +``` + +
+ +### Configuring through config file + +The syntax in config file is consistent with the above commandline override. For example, a commandline override +`x=range(1,4)` can be expressed in config file as `x: range(1,4)` under the `hydra.sweeper.params` node. + +## Example 2: Multi-Objective Optimization + +In the same directory, `example/multi-objective.py` implements a simple benchmark function, which has two objective values. We want to minimize two objectives simultaneously. + +You can discover the Optuna sweeper parameters with: + +```commandline +python example/multi-objective.py hydra/sweeper=optuna --cfg hydra -p hydra.sweeper +``` + +
Configuration of the multi-objective optimization example + +```yaml +# @package hydra.sweeper +sampler: + _target_: optuna.samplers.NSGAIISampler + seed: 123 + population_size: 50 + mutation_prob: null + crossover_prob: 0.9 + swapping_prob: 0.5 + constraints_func: null +_target_: hydra_plugins.hydra_optuna_sweeper.optuna_sweeper.OptunaSweeper +direction: +- minimize +- minimize +storage: null +study_name: multi-objective +n_trials: 20 +n_jobs: 1 +params: + x: range(0, 5, step=0.5) + y: range(0, 3, step=0.5) +``` +
+ + +To run the optimization, use the following command in the `plugins/hydra_optuna_sweeper` directory: + +```commandline +python example/multi-objective.py --multirun +``` + +For problems with trade-offs between two different objectives, there may be no single solution that simultaneously minimizes both objectives. Instead, we obtained a set of solutions, namely [Pareto optimal solutions](https://en.wikipedia.org/wiki/Pareto_efficiency), that show the best trade-offs possible between the objectives. In the following figure, the blue dots show the Pareto optimal solutions in the optimization results. + +![Pareto-optimal solutions](/plugins/optuna_sweeper/multi_objective_result.png) + +## EXPERIMENTAL: Custom-Search-Space Optimization + +Hydra's Optuna Sweeper allows users to provide a hook for custom search space configuration. +This means you can work directly with the `optuna.trial.Trial` object to suggest parameters. +To use this feature, define a python function with signature `Callable[[DictConfig, optuna.trial.Trial], None]` +and set the `hydra.sweeper.custom_search_space` key in your config to target that function. + +You can find a full example in the same directory as before, where `example/custom-search-space-objective.py` implements a benchmark function to be minimized. +The example shows the use of Optuna's [pythonic search spaces](https://optuna.readthedocs.io/en/stable/tutorial/10_key_features/002_configurations.html) in combination with Hydra. +Part of the search space configuration is defined in config files, and part of it is written in Python. + +
Example: Custom search space configuration + +```yaml +defaults: + - override hydra/sweeper: optuna + +hydra: + sweeper: + sampler: + seed: 123 + direction: minimize + study_name: custom-search-space + storage: null + n_trials: 20 + n_jobs: 1 + + params: + x: range(-5.5, 5.5, 0.5) + y: choice(-5, 0, 5) + # `custom_search_space` should be a dotpath pointing to a + # callable that provides search-space configuration logic: + custom_search_space: .custom-search-space-objective.configure + +x: 1 +y: 1 +z: 100 +max_z_difference_from_x: 0.5 +``` +```python +# example/custom-search-space-objective.py + +... + +def configure(cfg: DictConfig, trial: Trial) -> None: + x_value = trial.params["x"] + trial.suggest_float( + "z", + x_value - cfg.max_z_difference_from_x, + x_value + cfg.max_z_difference_from_x, + ) + trial.suggest_float("+w", 0.0, 1.0) # note +w here, not w as w is a new parameter + +... +``` + +
+ +The method that `custom_search_space` points to must accepts both a DictConfig with already set options and a trial object which needs further configuration. In this example we limit `z` the difference between `x` and `z` to be no more than 0.5. +Note that this `custom_search_space` API should be considered experimental and is subject to change. + +### Order of trial configuration +Configuring a trial object is done in the following sequence: + - search space parameters are set from the `hydra.sweeper.params` key in the config + - Command line overrides are set + - `custom_search_space` parameters are set + +It is not allowed to set search space parameters in the `custom_search_space` method for parameters which have a fixed value from command line overrides. [Trial.user_attrs](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial.user_attrs) can be inspected to find any of such fixed parameters. \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/plugins/ray_launcher.md b/website/versioned_docs/version-1.3/plugins/ray_launcher.md new file mode 100644 index 00000000000..d131c095a2a --- /dev/null +++ b/website/versioned_docs/version-1.3/plugins/ray_launcher.md @@ -0,0 +1,335 @@ +--- +id: ray_launcher +title: Ray Launcher plugin +sidebar_label: Ray Launcher plugin +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + +[![PyPI](https://img.shields.io/pypi/v/hydra-ray-launcher)](https://pypi.org/project/hydra-ray-launcher/) +![PyPI - License](https://img.shields.io/pypi/l/hydra-ray-launcher) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hydra-ray-launcher) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/hydra-ray-launcher.svg)](https://pypistats.org/packages/hydra-ray-launcher) + +The Ray Launcher plugin provides 2 launchers: `ray_aws` and `ray`. + `ray_aws` launches jobs remotely on AWS and is built on top of [ray autoscaler sdk](https://docs.ray.io/en/releases-1.3.0/cluster/sdk.html). `ray` launches jobs on your local machine or existing ray cluster. + +### Installation + +```commandline +$ pip install hydra-ray-launcher --upgrade +``` + +### Usage +Once installed, add `hydra/launcher=ray_aws` or `hydra/launcher=ray` to your command line. Alternatively, override `hydra/launcher` in your config: + +```yaml +defaults: + - override hydra/launcher: ray_aws +``` +There are several standard approaches for configuring plugins. Check [this page](../patterns/configuring_plugins.md) for more information. + +### `ray_aws` launcher + +:::important +`ray_aws` launcher is built on top of ray's [autoscaler sdk](https://docs.ray.io/en/releases-1.3.0/cluster/sdk.html). To get started, you need to +[config your AWS credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). +`ray autoscaler sdk` expects your AWS credentials have certain permissions for [`EC2`](https://aws.amazon.com/ec2) and [`IAM`](https://aws.amazon.com/iam). Read [this](https://github.com/ray-project/ray/issues/9327) for more information. +::: + +`ray autoscaler sdk` expects a configuration for the EC2 cluster; we've schematized the configs in here + +
Discover ray_aws launcher's config + +```commandline +$ python my_app.py hydra/launcher=ray_aws --cfg hydra -p hydra.launcher +# @package hydra.launcher +# @package hydra.launcher +_target_: hydra_plugins.hydra_ray_launcher.ray_aws_launcher.RayAWSLauncher +env_setup: + pip_packages: + omegaconf: ${ray_pkg_version:omegaconf} + hydra_core: ${ray_pkg_version:hydra} + ray: ${ray_pkg_version:ray} + cloudpickle: ${ray_pkg_version:cloudpickle} + pickle5: 0.0.11 + hydra_ray_launcher: 1.2.0.dev1 + commands: + - conda create -n hydra_${python_version:micro} python=${python_version:micro} -y + - echo 'export PATH="$HOME/anaconda3/envs/hydra_${python_version:micro}/bin:$PATH"' + >> ~/.bashrc +ray: + init: + address: null + remote: {} + cluster: + cluster_name: default + min_workers: 0 + upscaling_speed: 1.0 + max_workers: 1 + initial_workers: 0 + autoscaling_mode: default + target_utilization_fraction: 0.8 + idle_timeout_minutes: 5 + docker: + image: '' + container_name: '' + pull_before_run: true + run_options: [] + provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a,us-west-2b + cache_stopped_nodes: false + key_pair: + key_name: hydra-${oc.env:USER,user} + auth: + ssh_user: ubuntu + available_node_types: + ray.head.default: + resources: {} + node_config: + InstanceType: m5.large + ImageId: ami-0a2363a9cff180a64 + ray.worker.default: + min_workers: 0 + max_workers: 2 + resources: {} + node_config: + InstanceType: m5.large + ImageId: ami-0a2363a9cff180a64 + InstanceMarketOptions: + MarketType: spot + head_node_type: ray.head.default + file_mounts: {} + initialization_commands: [] + cluster_synced_files: [] + setup_commands: [] + head_setup_commands: [] + worker_setup_commands: [] + head_start_ray_commands: + - ray stop + - ulimit -n 65536;ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml + worker_start_ray_commands: + - ray stop + - ulimit -n 65536; ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 + run_env: auto +stop_cluster: true +sync_up: + source_dir: null + target_dir: null + include: [] + exclude: [] +sync_down: + source_dir: null + target_dir: null + include: [] + exclude: [] +logging: + log_style: auto + color_mode: auto + verbosity: 0 +create_update_cluster: + no_restart: false + restart_only: false + no_config_cache: false +teardown_cluster: + workers_only: false + keep_min_workers: false +``` +
+ + +#### Examples + +The following examples can be found here. + +
Simple app + +```commandline +$ python my_app.py --multirun task=1,2,3 +[HYDRA] Ray Launcher is launching 3 jobs, +[HYDRA] #0 : task=1 +[HYDRA] #1 : task=2 +[HYDRA] #2 : task=3 +[HYDRA] Pickle for jobs: /var/folders/n_/9qzct77j68j6n9lh0lw3vjqcn96zxl/T/tmpqqg4v4i7/job_spec.pkl +Cluster: default +... +INFO services.py:1172 -- View the Ray dashboard at http://localhost:8265 +(pid=3374) [__main__][INFO] - Executing task 1 +(pid=3374) [__main__][INFO] - Executing task 2 +(pid=3374) [__main__][INFO] - Executing task 3 +... +[HYDRA] Stopping cluster now. (stop_cluster=true) +[HYDRA] Deleted the cluster (provider.cache_stopped_nodes=false) +Destroying cluster. Confirm [y/N]: y [automatic, due to --yes] +... +No nodes remaining. + +``` +
+ + +
Upload & Download from remote cluster + +If your application is dependent on multiple modules, you can configure `hydra.launcher.sync_up` to upload dependency modules to the remote cluster. +You can also configure `hydra.launcher.sync_down` to download output from remote cluster if needed. This functionality is built on top of `rsync`, `include` and `exclude` is consistent with how it works in `rsync`. + +```commandline +$ python train.py --multirun random_seed=1,2,3 +[HYDRA] Ray Launcher is launching 3 jobs, +[HYDRA] #0 : random_seed=1 +[HYDRA] #1 : random_seed=2 +[HYDRA] #2 : random_seed=3 +[HYDRA] Pickle for jobs: /var/folders/n_/9qzct77j68j6n9lh0lw3vjqcn96zxl/T/tmptdkye9of/job_spec.pkl +Cluster: default +... +INFO services.py:1172 -- View the Ray dashboard at http://localhost:8265 +(pid=1772) [__main__][INFO] - Start training... +(pid=1772) [INFO] - Init my model +(pid=1772) [INFO] - Created dir for checkpoints. dir=checkpoint +(pid=1772) [__main__][INFO] - Start training... +(pid=1772) [INFO] - Init my model +(pid=1772) [INFO] - Created dir for checkpoints. dir=checkpoint +(pid=1772) [__main__][INFO] - Start training... +(pid=1772) [INFO] - Init my model +(pid=1772) [INFO] - Created dir for checkpoints. dir=checkpoint +Loaded cached provider configuration +... +[HYDRA] Output: receiving file list ... done +16-32-25/ +16-32-25/0/ +16-32-25/0/checkpoint/ +16-32-25/0/checkpoint/checkpoint_1.pt +16-32-25/1/ +16-32-25/1/checkpoint/ +16-32-25/1/checkpoint/checkpoint_2.pt +16-32-25/2/ +16-32-25/2/checkpoint/ +16-32-25/2/checkpoint/checkpoint_3.pt +... +[HYDRA] Stopping cluster now. (stop_cluster=true) +[HYDRA] Deleted the cluster (provider.cache_stopped_nodes=false) +Destroying cluster. Confirm [y/N]: y [automatic, due to --yes] +... +No nodes remaining. + +``` +
+ + +##### Manage Cluster LifeCycle +You can manage the Ray EC2 cluster lifecycle by configuring the flags provided by the plugin: + +- Default setting (no need to specify on commandline): delete cluster after job finishes remotely: + ```commandline + hydra.launcher.stop_cluster=true + hydra.launcher.ray.cluster.provider.cache_stopped_nodes=false + hydra.launcher.teardown_cluster.workers_only=false + hydra.launcher.teardown_cluster.keep_min_workers=false + ``` + +- Keep cluster running after jobs finishes remotely + ```commandline + hydra.launcher.stop_cluster=false + ``` + +- Power off EC2 instances and control node termination using `hydra.launcher.ray.cluster.provider.cache_stopped_nodes` +and `hydra.launcher.teardown_cluster.workers_only` + + | cache_stopped_nodes | workers_only | behavior | + |---------------------|--------------|----------| + | false | false | All nodes are terminated | + | false | true | Keeps head node running and terminates only worker node | + | true | false | Keeps both head node and worker node and stops both of them | + | true | true | Keeps both head node and worker node and stops only worker node | + +- Keep `hydra.launcher.ray.cluster.min_workers` worker nodes +and delete the rest of the worker nodes + ```commandline + hydra.launcher.teardown_cluster.keep_min_workers=true + ``` + +Additionally, you can configure how to create or update the cluster: + +- Default config: run setup commands, restart Ray and use +the config cache if available + ```commandline + hydra.launcher.create_update_cluster.no_restart=false + hydra.launcher.create_update_cluster.restart_only=false + hydra.launcher.create_update_cluster.no_config_cache=false + ``` + +- Skip restarting Ray services when updating the cluster config + ```commandline + hydra.launcher.create_update_cluster.no_restart=true + ``` + +- Skip running setup commands and only restart Ray (cannot be used with +`hydra.launcher.create_update_cluster.no_restart`) + ```commandline + hydra.launcher.create_update_cluster.restart_only=true + ``` + +- Fully resolve all environment settings from the cloud provider again + ```commandline + hydra.launcher.create_update_cluster.no_config_cache=true + ``` + + +##### Configure Ray Logging +You can manage Ray specific logging by configuring the flags provided by the plugin: + +- Default config: use minimal verbosity and automatically +detect whether to use pretty-print and color mode + ```commandline + hydra.launcher.logging.log_style="auto" + hydra.launcher.logging.color_mode="auto" + hydra.launcher.logging.verbosity=0 + ``` + +- Disable pretty-print + ```commandline + hydra.launcher.logging.log_style="record" + ``` + +- Disable color mode + ```commandline + hydra.launcher.logging.color_mode="false" + ``` + +- Increase Ray logging verbosity + ```commandline + hydra.launcher.logging.verbosity=3 + ``` + +### `ray` launcher + +`ray` launcher lets you launch application on your ray cluster or local machine. You can easily config how your jobs are executed by changing `ray` launcher's configuration here + `~/hydra/plugins/hydra_ray_launcher/hydra_plugins/hydra_ray_launcher/conf/hydra/launcher/ray.yaml` + + The example application starts a new ray cluster. +```commandline +$ python my_app.py --multirun hydra/launcher=ray +[HYDRA] Ray Launcher is launching 1 jobs, sweep output dir: multirun/2020-11-10/15-16-28 +[HYDRA] Initializing ray with config: {} +INFO services.py:1164 -- View the Ray dashboard at http://127.0.0.1:8266 +[HYDRA] #0 : +(pid=97801) [__main__][INFO] - Executing task 1 +``` + +You can run the example application on your existing ray cluster as well by overriding `hydra.launcher.ray.init.address`: +```commandline +$ python my_app.py --multirun hydra/launcher=ray hydra.launcher.ray.init.address=localhost:6379' +[HYDRA] Ray Launcher is launching 1 jobs, sweep output dir: multirun/2020-11-10/15-13-32 +[HYDRA] Initializing ray with config: {'num_cpus': None, 'num_gpus': None, 'address': 'localhost:6379'} +INFO worker.py:633 -- Connecting to existing Ray cluster at address: 10.30.99.17:6379 +[HYDRA] #0 : +(pid=93358) [__main__][INFO] - Executing task 1 +``` + +### Configure `ray.init()` and `ray.remote()` +Ray launcher is built on top of [`ray.init()`](https://docs.ray.io/en/master/package-ref.html?highlight=ray.remote#ray-init) +and [`ray.remote()`](https://docs.ray.io/en/master/package-ref.html?highlight=ray.remote#ray-remote). +You can configure `ray` by overriding `hydra.launcher.ray.init` and `hydra.launcher.ray.remote`. +Check out an example config. diff --git a/website/versioned_docs/version-1.3/plugins/rq_launcher.md b/website/versioned_docs/version-1.3/plugins/rq_launcher.md new file mode 100644 index 00000000000..bb81988fd31 --- /dev/null +++ b/website/versioned_docs/version-1.3/plugins/rq_launcher.md @@ -0,0 +1,111 @@ +--- +id: rq_launcher +title: RQ Launcher plugin +sidebar_label: RQ Launcher plugin +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + +[![PyPI](https://img.shields.io/pypi/v/hydra-rq-launcher)](https://pypi.org/project/hydra-rq-launcher/) +![PyPI - License](https://img.shields.io/pypi/l/hydra-rq-launcher) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hydra-rq-launcher) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/hydra-rq-launcher.svg)](https://pypistats.org/packages/hydra-rq-launcher) + +The RQ Launcher plugin provides a launcher for distributed execution and job queuing based on [Redis Queue (RQ)](https://python-rq.org). + +RQ launcher allows parallelizing across multiple nodes and scheduling jobs in queues. Usage of this plugin requires a [Redis server](https://redis.io/topics/quickstart). When parallelisation on a single node is intended, the Joblib launcher may be preferable, since it works without a database. + + +### Installation +```commandline +pip install hydra-rq-launcher --upgrade +``` +Usage of this plugin requires a [Redis server](https://redis.io/topics/quickstart). + +Note that RQ does [not support Windows](https://python-rq.org/docs/#limitations). + +### Usage +Once installed, add `hydra/launcher=rq` to your command line. Alternatively, override `hydra/launcher` in your config: + +```yaml +defaults: + - override hydra/launcher: rq +``` + +The default configuration is as follows: + +```yaml title="$ python your_app.py hydra/launcher=rq --cfg hydra -p hydra.launcher" +# @package hydra.launcher +_target_: hydra_plugins.hydra_rq_launcher.rq_launcher.RQLauncher +enqueue: + job_timeout: null + ttl: null + result_ttl: null + failure_ttl: null + at_front: false + job_id: null + description: null +queue: default +redis: + host: ${oc.env:REDIS_HOST,localhost} + port: ${oc.env:REDIS_PORT,6379} + db: ${oc.env:REDIS_DB,0} + password: ${oc.env:REDIS_PASSWORD,null} + ssl: ${oc.env:REDIS_SSL,False} + ssl_ca_certs: ${oc.env:REDIS_SSL_CA_CERTS,null + mock: ${oc.env:REDIS_MOCK,False} +stop_after_enqueue: false +wait_polling: 1.0 +``` + +Further descriptions on the variables can be found in the plugin config file, defined here. There are several standard approaches for configuring plugins. Check [this page](../patterns/configuring_plugins.md) for more information. + +The plugin is using environment variables to store Redis connection information. The environment variables `REDIS_HOST`, `REDIS_PORT`, `REDIS_DB`, and `REDIS_PASSWORD`, are used for the host address, port, database, and password of the server, respectively. Support for Redis SSL connections is controlled through `REDIS_SSL` and `REDIS_SSL_CA_CERTS`; see [redis-py](https://github.com/andymccurdy/redis-py#ssl-connections) documentation. + +For example, they might be set as follows when using `bash` or `zsh` as a shell: + +```commandline +export REDIS_HOST="localhost" +export REDIS_PORT="6379" +export REDIS_DB="0" +export REDIS_PASSWORD="" +``` + +Enable SSL by setting the relevant environment variables. e.g: +```commandline +export REDIS_SSL=true +export REDIS_SSL_CA_CERTS=/etc/ssl/certs/ca-certificates.crt +``` + +Assuming configured environment variables, workers connecting to the Redis server can be launched using: + +```commandline +rq worker --url redis://:$REDIS_PASSWORD@$REDIS_HOST:$REDIS_PORT/$REDIS_DB +``` + +An example application using this launcher is provided in the plugin repository. + +Starting the app with `python my_app.py --multirun task=1,2,3,4,5` will enqueue five jobs to be processed by worker instances: + +```text +$ python my_app.py --multirun task=1,2,3,4,5 + +[HYDRA] RQ Launcher is enqueuing 5 job(s) in queue : default +[HYDRA] Sweep output dir : multirun/2020-06-15/18-00-00 +[HYDRA] Enqueued 13b3da4e-03f7-4d16-9ca8-cfb3c48afeae +[HYDRA] #1 : task=1 +[HYDRA] Enqueued 00c6a32d-e5a4-432c-a0f3-b9d4ef0dd585 +[HYDRA] #2 : task=2 +[HYDRA] Enqueued 63b90f27-0711-4c95-8f63-70164fd850df +[HYDRA] #3 : task=3 +[HYDRA] Enqueued b1d49825-8b28-4516-90ca-8106477e1eb1 +[HYDRA] #4 : task=4 +[HYDRA] Enqueued ed96bdaa-087d-4c7f-9ecb-56daf948d5e2 +[HYDRA] #5 : task=5 +[HYDRA] Finished enqueuing +[HYDRA] Polling job statuses every 1.0 sec +``` + +Note that any dependencies need to be installed in the Python environment used to run the RQ worker. For serialization of jobs [`cloudpickle`](https://github.com/cloudpickle/cloudpickle) is used. + +The [RQ documentation](https://python-rq.org/) holds further information on [job monitoring](http://python-rq.org/docs/monitoring/), which can be done via console or [web interfaces](https://github.com/nvie/rq-dashboard), and provides [patterns](https://python-rq.org/patterns/) for worker and exception handling. diff --git a/website/versioned_docs/version-1.3/plugins/submitit_launcher.md b/website/versioned_docs/version-1.3/plugins/submitit_launcher.md new file mode 100644 index 00000000000..59e2d0294a2 --- /dev/null +++ b/website/versioned_docs/version-1.3/plugins/submitit_launcher.md @@ -0,0 +1,124 @@ +--- +id: submitit_launcher +title: Submitit Launcher plugin +sidebar_label: Submitit Launcher plugin +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + +[![PyPI](https://img.shields.io/pypi/v/hydra-submitit-launcher)](https://pypi.org/project/hydra-submitit-launcher/) +![PyPI - License](https://img.shields.io/pypi/l/hydra-submitit-launcher) +![PyPI - Python Version](https://img.shields.io/pypi/pyversions/hydra-submitit-launcher) +[![PyPI - Downloads](https://img.shields.io/pypi/dm/hydra-submitit-launcher.svg)](https://pypistats.org/packages/hydra-submitit-launcher) + +The Submitit Launcher plugin provides a [SLURM](https://slurm.schedmd.com/documentation.html) Launcher based on [Submitit](https://github.com/facebookincubator/submitit). + +### Installation +```commandline +pip install hydra-submitit-launcher --upgrade +``` + + +### Usage +Once installed, add `hydra/launcher=submitit_slurm` to your command line. Alternatively, override `hydra/launcher` in your config: + +```yaml +defaults: + - override hydra/launcher: submitit_slurm +``` +There are several standard approaches for configuring plugins. Check [this page](../patterns/configuring_plugins.md) for more information. +Note that this plugin expects a valid environment in the target host. Usually this means a shared file system between +the launching host and the target host. + +The Submitit Plugin implements 2 different launchers: `submitit_slurm` to run on a SLURM cluster, and `submitit_local` for basic local tests. + +
Discover the SLURM Launcher parameters (Expand) + +```yaml title="$ python your_app.py hydra/launcher=submitit_slurm --cfg hydra -p hydra.launcher" +# @package hydra.launcher +submitit_folder: ${hydra.sweep.dir}/.submitit/%j +timeout_min: 60 +cpus_per_task: null +gpus_per_node: null +tasks_per_node: 1 +mem_gb: null +nodes: 1 +name: ${hydra.job.name} +_target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.SlurmLauncher +partition: null +qos: null +comment: null +constraint: null +exclude: null +gres: null +cpus_per_gpu: null +gpus_per_task: null +mem_per_gpu: null +mem_per_cpu: null +account: null +signal_delay_s: 120 +max_num_timeout: 0 +additional_parameters: {} +array_parallelism: 256 +setup: null +srun_args: null +``` +
+
Discover the Local Launcher parameters (Expand) + +```yaml title="$ python example/my_app.py hydra/launcher=submitit_local --cfg hydra -p hydra.launcher" +# @package hydra.launcher +_target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher +submitit_folder: ${hydra.sweep.dir}/.submitit/%j +timeout_min: 60 +cpus_per_task: 1 +gpus_per_node: 0 +tasks_per_node: 1 +mem_gb: 4 +nodes: 1 +name: ${hydra.job.name} +``` +
+ +
+You can set all these parameters in your configuration file and/or override them in the command-line: + +```text +python foo.py --multirun hydra/launcher=submitit_slurm hydra.launcher.timeout_min=3 +``` +For more details, including descriptions for each parameter, check out the config file. +You can also check the [Submitit documentation](https://github.com/facebookincubator/submitit). + + +**Caution**: use of `--multirun` is required for the launcher to be picked up. + +### Example + +An example application using this launcher is provided in the plugin repository. + +Starting the app with `python my_app.py task=1,2,3 --multirun` (see [Multi-run](../tutorials/basic/running_your_app/2_multirun.md) for details) will launch 3 executions (you can override the launcher to run locally for testing by adding `hydra/launcher=submitit_local`): + +```text +$ python my_app.py task=1,2,3 --multirun +[HYDRA] Sweep output dir : multirun/2020-05-28/15-05-22 +[HYDRA] #0 : task=1 +[HYDRA] #1 : task=2 +[HYDRA] #2 : task=3 +``` +You will be able to see the output of the app in the output dir: +```commandline +$ tree +. +├── 0 +│   └── my_app.log +├── 1 +│   └── my_app.log +├── 2 +│   └── my_app.log +└── multirun.yaml + +$ cat 0/my_app.log +[2020-05-28 15:05:23,511][__main__][INFO] - Process ID 15887 executing task 1 ... +[2020-05-28 15:05:24,514][submitit][INFO] - Job completed successfully +``` + diff --git a/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/2_multirun.md b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/2_multirun.md new file mode 100644 index 00000000000..a17990d1f80 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/2_multirun.md @@ -0,0 +1,108 @@ +--- +id: multi-run +title: Multi-run +sidebar_label: Multi-run +--- + +Sometimes you want to run the same application with multiple different configurations. +E.g. running a performance test on each of the databases with each of the schemas. + +You can multirun a Hydra application via either commandline or configuration: + +### Configure `hydra.mode` (new in Hydra 1.2) +You can configure `hydra.mode` in any supported way. The legal values are `RUN` and `MULTIRUN`. +The following shows how to override from the command-line and sweep over all combinations of the dbs and schemas. +Setting `hydra.mode=MULTIRUN` in your input config would make your application multi-run by default. + +```text title="$ python my_app.py hydra.mode=MULTIRUN db=mysql,postgresql schema=warehouse,support,school" +[2021-01-20 17:25:03,317][HYDRA] Launching 6 jobs locally +[2021-01-20 17:25:03,318][HYDRA] #0 : db=mysql schema=warehouse +[2021-01-20 17:25:03,458][HYDRA] #1 : db=mysql schema=support +[2021-01-20 17:25:03,602][HYDRA] #2 : db=mysql schema=school +[2021-01-20 17:25:03,755][HYDRA] #3 : db=postgresql schema=warehouse +[2021-01-20 17:25:03,895][HYDRA] #4 : db=postgresql schema=support +[2021-01-20 17:25:04,040][HYDRA] #5 : db=postgresql schema=school +``` +The printed configurations have been omitted for brevity. + +### `--multirun (-m)` from the command-line +You can achieve the above from command-line as well: +```commandline +python my_app.py --multirun db=mysql,postgresql schema=warehouse,support,school +``` +or +```commandline +python my_app.py -m db=mysql,postgresql schema=warehouse,support,school +``` + +You can access `hydra.mode` at runtime to determine whether the application is in RUN or MULTIRUN mode. Check [here](/configure_hydra/Intro.md) +on how to access Hydra config at run time. + +If conflicts arise (e.g., `hydra.mode=RUN` and the application was run with `--multirun`), Hydra will determine the value of `hydra.mode` +at run time. The following table shows what runtime `hydra.mode` value you'd get with different input configs and commandline combinations. + +[//]: # (Conversion matrix) + +| | No multirun commandline flag | --multirun ( -m) | +|-------------------- |-------------------------------------|-------------------------------------| +|hydra.mode=RUN | RunMode.RUN | RunMode.MULTIRUN (with UserWarning) | +|hydra.mode=MULTIRUN | RunMode.MULTIRUN | RunMode.MULTIRUN | +|hydra.mode=None (default) | RunMode.RUN | RunMode.MULTIRUN | + + +:::important +Hydra composes configs lazily at job launching time. If you change code or configs after launching a job/sweep, the final +composed configs might be impacted. +::: + +### Sweeping via `hydra.sweeper.params` + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +You can also define sweeping in the input configs by overriding +`hydra.sweeper.params`. Using the above example, the same multirun could be achieved via the following config. + +```yaml +hydra: + sweeper: + params: + db: mysql,postgresql + schema: warehouse,support,school +``` + +The syntax are consistent for both input configs and commandline overrides. +If a sweep is specified in both an input config and at the command line, +then the commandline sweep will take precedence over the sweep defined +in the input config. If we run the same application with the above input config and a new commandline override: + +```text title="$ python my_app.py -m db=mysql" +[2021-01-20 17:25:03,317][HYDRA] Launching 3 jobs locally +[2021-01-20 17:25:03,318][HYDRA] #0 : db=mysql schema=warehouse +[2021-01-20 17:25:03,458][HYDRA] #1 : db=mysql schema=support +[2021-01-20 17:25:03,602][HYDRA] #2 : db=mysql schema=school +``` +:::info +The above configuration methods only apply to Hydra's default `BasicSweeper` for now. For other sweepers, please check out the +corresponding documentations. +::: + +### Additional sweep types +Hydra supports other kinds of sweeps, e.g.: +```python +x=range(1,10) # 1-9 +schema=glob(*) # warehouse,support,school +schema=glob(*,exclude=w*) # support,school +``` +See the [Extended Override syntax](/advanced/override_grammar/extended.md) for details. + +### Sweeper +The default sweeping logic is built into Hydra. Additional sweepers are available as plugins. +For example, the [Ax Sweeper](/plugins/ax_sweeper.md) can automatically find the best parameter combination! + +### Launcher +By default, Hydra runs your multi-run jobs locally and serially. +Other launchers are available as plugins for launching in parallel and on different clusters. For example, the [JobLib Launcher](/plugins/joblib_launcher.md) +can execute the different parameter combinations in parallel on your local machine using multi-processing. + diff --git a/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/3_working_directory.md b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/3_working_directory.md new file mode 100644 index 00000000000..5b4780ac024 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/3_working_directory.md @@ -0,0 +1,112 @@ +--- +id: working_directory +title: Output/Working directory +sidebar_label: Output/Working directory +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +Hydra solves the problem of your needing to specify a new output directory for each run, by +creating a directory for each run and executing your code within that working directory. + +The working directory is used to: +* Store the output for the application (For example, a database dump file) +* Store the Hydra output for the run (Configuration, Logs etc) + +Every time you run the app, a new working directory is created: + +Python file: `my_app.py` +```python +import os +import hydra +from omegaconf import DictConfig + +@hydra.main(version_base=None) +def my_app(cfg: DictConfig) -> None: + print("Working directory : {}".format(os.getcwd())) + +$ python my_app.py +Working directory : /home/omry/dev/hydra/outputs/2019-09-25/15-16-17 + +$ python my_app.py +Working directory : /home/omry/dev/hydra/outputs/2019-09-25/15-16-19 +``` + +Let's take a look at one of the working directories: +```text +$ tree outputs/2019-09-25/15-16-17 +outputs/2019-09-25/15-16-17 +├── .hydra +│ ├── config.yaml +│ ├── hydra.yaml +│ └── overrides.yaml +└── my_app.log +``` + +We have the Hydra output directory (`.hydra` by default), and the application log file. +Inside the Hydra output directory we have: +* `config.yaml`: A dump of the user specified configuration +* `hydra.yaml`: A dump of the Hydra configuration +* `overrides.yaml`: The command line overrides used + +And in the main output directory: +* `my_app.log`: A log file created for this run + +### Disable changing current working dir to job's output dir + +By default, Hydra's `@hydra.main` decorator makes a call to `os.chdir` before passing control to the user's decorated main function. +Set `hydra.job.chdir=False` to disable this behavior. +```bash +# check current working dir +$ pwd +/home/omry/dev/hydra + +# working dir remains the same +$ python my_app.py hydra.job.chdir=False +Working directory : /home/omry/dev/hydra + +# output dir and files are still created, even if `chdir` is disabled: +$ tree -a outputs/2021-10-25/09-46-26/ +outputs/2021-10-25/09-46-26/ +├── .hydra +│   ├── config.yaml +│   ├── hydra.yaml +│   └── overrides.yaml +└── my_app.log +``` + + +### Changing or disabling Hydra's output subdir +You can change the `.hydra` subdirectory name by overriding `hydra.output_subdir`. +You can disable its creation by overriding `hydra.output_subdir` to `null`. + + +### Accessing the original working directory in your application + +With `hydra.job.chdir=True`, you can still access the original working directory by importing `get_original_cwd()` and `to_absolute_path()` in `hydra.utils`: + +```python +from hydra.utils import get_original_cwd, to_absolute_path + +@hydra.main(version_base=None) +def my_app(_cfg: DictConfig) -> None: + print(f"Current working directory : {os.getcwd()}") + print(f"Orig working directory : {get_original_cwd()}") + print(f"to_absolute_path('foo') : {to_absolute_path('foo')}") + print(f"to_absolute_path('/foo') : {to_absolute_path('/foo')}") + +if __name__ == "__main__": + my_app() +``` + +```text title="$ python examples/tutorial/8_working_directory/original_cwd.py" +Current working directory : /Users/omry/dev/hydra/outputs/2019-10-23/10-53-03 +Original working directory : /Users/omry/dev/hydra +to_absolute_path('foo') : /Users/omry/dev/hydra/foo +to_absolute_path('/foo') : /foo +``` + + +The name of the generated working directories can be [customized](/configure_hydra/workdir.md). diff --git a/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/4_logging.md b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/4_logging.md new file mode 100644 index 00000000000..5ad115909fc --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/4_logging.md @@ -0,0 +1,67 @@ +--- +id: logging +title: Logging +sidebar_label: Logging +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +People often do not use Python logging due to the setup cost. +Hydra solves this by configuring the Python logging for you. + + +By default, Hydra logs at the INFO level to both the console and a log file in the automatic working directory. + +An example of logging with Hydra: + +```python +import logging +from omegaconf import DictConfig +import hydra + +# A logger for this file +log = logging.getLogger(__name__) + +@hydra.main() +def my_app(_cfg: DictConfig) -> None: + log.info("Info level message") + log.debug("Debug level message") + +if __name__ == "__main__": + my_app() + +$ python my_app.py +[2019-06-27 00:52:46,653][__main__][INFO] - Info level message + +``` +You can enable DEBUG level logging from the command line by overriding `hydra.verbose`. + +`hydra.verbose` can be a Boolean, a String or a List: + +Examples: +* `hydra.verbose=true` : Sets the log level of **all** loggers to `DEBUG` +* `hydra.verbose=NAME` : Sets the log level of the logger `NAME` to `DEBUG`. + Equivalent to `import logging; logging.getLogger(NAME).setLevel(logging.DEBUG)`. +* `hydra.verbose=[NAME1,NAME2]`: Sets the log level of the loggers `NAME1` and `NAME2` to `DEBUG` + +Example output: +``` text +$ python my_app.py hydra.verbose=[__main__,hydra] +[2019-09-29 13:06:00,880] - Installed Hydra Plugins +[2019-09-29 13:06:00,880] - *********************** +... +[2019-09-29 13:06:00,896][__main__][INFO] - Info level message +[2019-09-29 13:06:00,896][__main__][DEBUG] - Debug level message +``` + +You can disable the logging output by setting `hydra/job_logging` to `disabled` +```commandline +$ python my_app.py hydra/job_logging=disabled + +``` + +You can also set `hydra/job_logging=none` and `hydra/hydra_logging=none` if you do not want Hydra to configure the logging. + +Logging can be [customized](/configure_hydra/logging.md). diff --git a/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/5_debugging.md b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/5_debugging.md new file mode 100644 index 00000000000..aefd1d0efed --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/5_debugging.md @@ -0,0 +1,54 @@ +--- +id: debugging +title: Debugging +sidebar_label: Debugging +--- +Hydra provides a few options to improve debuggability. + +### Printing the configuration +Print the config for your app without running your function by adding `--cfg` or `-c` to the command line. + +The `--cfg` option takes one argument indicating which part of the config to print: +* `job`: Your config +* `hydra`: Hydra's config +* `all`: The full config, which is a union of `job` and `hydra`. + +```yaml +# A normal run: +$ python my_app.py +MySQL connecting to localhost with user=root and password=1234 + +# just show the config without running your function: +$ python my_app.py --cfg job +db: + host: localhost + user: root + password: 1234 +``` +The printed config includes any modifications done via the command line: +```yaml {3} +$ python my_app.py db.host=10.0.0.1 --cfg job +db: + host: 10.0.0.1 + user: root + password: 1234 +``` + +You can use --package or -p to display a subset of the configuration: +```yaml +python my_app.py --cfg hydra --package hydra.job +# @package hydra.job +name: my_app +config_name: config +... +``` + +By default, config interpolations are not resolved. To print resolved config use the `--resolve` flag in addition to the `--cfg` flag +### Info +The `--info` flag can provide information about various aspects of Hydra and your application: + - `--info all`: Default behavior, prints everything + - `--info config`: Prints information useful to understanding the config composition: + Config Search Path, Defaults Tree, Defaults List and the final config. + - `--info defaults`: Prints the Final Defaults List + - `--info defaults-tree`: Prints the Defaults Tree + - `--info plugins`: Prints information about installed plugins diff --git a/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/6_tab_completion.md b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/6_tab_completion.md new file mode 100644 index 00000000000..9ee7f680dfa --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/running_your_app/6_tab_completion.md @@ -0,0 +1,39 @@ +--- +id: tab_completion +title: Tab completion +sidebar_label: Tab completion +--- + +Tab completion can complete config groups, config nodes and values. +To complete paths, start them with `/` or `./`. + +See this short video demonstration of tab completion: + +import Script from '@site/src/components/Script.jsx'; + + + + +### Install tab completion +Get the exact command to install the completion from `--hydra-help`. +Currently, Bash, zsh and Fish are supported. +We are relying on the community to implement tab completion plugins for additional shells. + +#### Fish instructions +Fish support requires version >= 3.1.2. +Previous versions will work but add an extra space after `.`. + +Because the fish shell implements special behavior for expanding words prefixed +with a tilde character '~', command-line completion does not work for +[tilde deletions](/advanced/override_grammar/basic.md#modifying-the-defaults-list). + +#### Zsh instructions +Zsh is compatible with the existing Bash shell completion by appending +``` +autoload -Uz bashcompinit && bashcompinit +``` +to the `.zshrc` file after `compinit`, restarting the shell and then using the commands provided for Bash. + +Because the zsh shell implements special behavior for expanding words prefixed +with a tilde character '~', command-line completion does not work for +[tilde deletions](/advanced/override_grammar/basic.md#modifying-the-defaults-list). \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/1_simple_cli.md b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/1_simple_cli.md new file mode 100644 index 00000000000..4134e0cfcb5 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/1_simple_cli.md @@ -0,0 +1,44 @@ +--- +id: simple_cli +title: A simple command-line application +--- + +import GithubLink,{ExampleGithubLink} from "@site/src/components/GithubLink" + + + +This is a simple Hydra application that prints your configuration. +The `my_app` function is a placeholder for your code. +We will slowly evolve this example to showcase more Hydra features. + +The examples in this tutorial are available here. + +```python title="my_app.py" +from omegaconf import DictConfig, OmegaConf +import hydra + +@hydra.main(version_base=None) +def my_app(cfg: DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) + +if __name__ == "__main__": + my_app() +``` +In this example, Hydra creates an empty `cfg` object and passes it to the function annotated with `@hydra.main`. + +You can add config values via the command line. The `+` indicates that the field is new. + +```yaml +$ python my_app.py +db.driver=mysql +db.user=omry +db.password=secret +db: + driver: mysql + user: omry + password: secret +``` + +:::info +See the [version_base page](../../../upgrades/version_base.md) for details on the version_base parameter. +::: + +See [Hydra's command line flags](advanced/hydra-command-line-flags.md) and +[Basic Override Syntax](advanced/override_grammar/basic.md) for more information about the command line. diff --git a/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/2_config_file.md b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/2_config_file.md new file mode 100644 index 00000000000..f5a09fcda55 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/2_config_file.md @@ -0,0 +1,66 @@ +--- +id: config_file +title: Specifying a config file +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +It can get tedious to type all those command line arguments. +You can solve it by creating a configuration file next to my_app.py. +Hydra configuration files are yaml files and should have the .yaml file extension. + +```yaml title="config.yaml" +db: + driver: mysql + user: omry + password: secret +``` + +Specify the config name by passing a `config_name` parameter to the **@hydra.main()** decorator. +Note that you should omit the **.yaml** extension. +Hydra also needs to know where to find your config. Specify the directory containing it relative to the application by passing `config_path`: +```python title="my_app.py" {4} +from omegaconf import DictConfig, OmegaConf +import hydra + +@hydra.main(version_base=None, config_path=".", config_name="config") +def my_app(cfg): + print(OmegaConf.to_yaml(cfg)) + +if __name__ == "__main__": + my_app() +``` + +`config.yaml` is loaded automatically when you run your application. +```yaml +$ python my_app.py +db: + driver: mysql + user: omry + password: secret +``` + +You can override values in the loaded config from the command line. +Note the lack of the `+` prefix. +```yaml {4-5} +$ python my_app.py db.user=root db.password=1234 +db: + driver: mysql + user: root + password: 1234 +``` + + +Use `++` to override a config value if it's already in the config, or add it otherwise. +e.g: +```shell +# Override an existing item +$ python my_app.py ++db.password=1234 + +# Add a new item +$ python my_app.py ++db.timeout=5 +``` + +You can enable [tab completion](/tutorials/basic/running_your_app/6_tab_completion.md) for your Hydra applications. \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/3_using_config.md b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/3_using_config.md new file mode 100644 index 00000000000..5479f7daa16 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/3_using_config.md @@ -0,0 +1,50 @@ +--- +id: using_config +title: Using the config object +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +Here are some basic features of the Hydra Configuration Object: + +```yaml title="config.yaml" +node: # Config is hierarchical + loompa: 10 # Simple value + zippity: ${node.loompa} # Value interpolation + do: "oompa ${node.loompa}" # String interpolation + waldo: ??? # Missing value, must be populated prior to access +``` + +```python title="main.py" +from omegaconf import DictConfig, OmegaConf +import hydra + +@hydra.main(version_base=None, config_path=".", config_name="config") +def my_app(cfg: DictConfig): + assert cfg.node.loompa == 10 # attribute style access + assert cfg["node"]["loompa"] == 10 # dictionary style access + + assert cfg.node.zippity == 10 # Value interpolation + assert isinstance(cfg.node.zippity, int) # Value interpolation type + assert cfg.node.do == "oompa 10" # string interpolation + + cfg.node.waldo # raises an exception + +if __name__ == "__main__": + my_app() + ``` +Outputs: +``` +$ python my_app.py +Traceback (most recent call last): + File "my_app.py", line 32, in my_app + cfg.node.waldo +omegaconf.errors.MissingMandatoryValue: Missing mandatory value: node.waldo + full_key: node.waldo + object_type=dict +``` + +Hydra's configuration object is an instance of OmegaConf's DictConfig. +You can learn more about OmegaConf here. diff --git a/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/4_config_groups.md b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/4_config_groups.md new file mode 100644 index 00000000000..4bc24fe1cf9 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/4_config_groups.md @@ -0,0 +1,105 @@ +--- +id: config_groups +title: Grouping config files +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +Suppose you want to benchmark your application on each of PostgreSQL and MySQL. To do this, use config groups. + +A _**Config Group**_ is a named group with a set of valid options. +Selecting a non-existent config option generates an error message with the valid options. + +### Creating config groups +To create a config group, create a directory, e.g. `db`, to hold a file for each database configuration option. +Since we are expecting to have multiple config groups, we will proactively move all the configuration files +into a `conf` directory. + +
+
+ +``` text title="Directory layout" +├─ conf +│ └─ db +│ ├─ mysql.yaml +│ └─ postgresql.yaml +└── my_app.py +``` +
+ +
+ +```yaml title="db/mysql.yaml" +driver: mysql +user: omry +password: secret + + +``` +
+ +```yaml title="db/postgresql.yaml" +driver: postgresql +user: postgres_user +password: drowssap +timeout: 10 + +``` + +
+
+ +### Using config groups +Since we moved all the configs into the `conf` directory, we need to tell Hydra where to find them using the `config_path` parameter. +**`config_path` is a directory relative to `my_app.py`**. +```python title="my_app.py" {4} +from omegaconf import DictConfig, OmegaConf +import hydra + +@hydra.main(version_base=None, config_path="conf") +def my_app(cfg: DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) + +if __name__ == "__main__": + my_app() +``` + +Running `my_app.py` without requesting a configuration will print an empty config. +```yaml +$ python my_app.py +{} +``` + +Select an item from a config group with `+GROUP=OPTION`, e.g: +```yaml {2} +$ python my_app.py +db=postgresql +db: + driver: postgresql + pass: drowssap + timeout: 10 + user: postgres_user +``` + +By default, the config group determines where the config content is placed inside the final config object. +In Hydra, the path to the config content is referred to as the config `package`. +The package of `db/postgresql.yaml` is `db`: + + +Like before, you can still override individual values in the resulting config: +```yaml +$ python my_app.py +db=postgresql db.timeout=20 +db: + driver: postgresql + pass: drowssap + timeout: 20 + user: postgres_user +``` + +### Advanced topics + - Config content can be relocated via package overrides. See [Reference Manual/Packages](advanced/overriding_packages.md). + - Multiple options can be selected from the same Config Group by specifying them as a list. + See [Common Patterns/Selecting multiple configs from a Config Group](patterns/select_multiple_configs_from_config_group.md) + + diff --git a/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/5_defaults.md b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/5_defaults.md new file mode 100644 index 00000000000..21f7827b697 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/5_defaults.md @@ -0,0 +1,156 @@ +--- +id: defaults +title: Selecting default configs +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +After office politics, you decide that you want to use MySQL by default. +You no longer want to type `+db=mysql` every time you run your application. + +You can add a **Default List** to your config file. +A **Defaults List** is a list telling Hydra how to compose the final config object. +By convention, it is the first item in the config. + + +### Config group defaults + +```yaml title="config.yaml" +defaults: + - db: mysql +``` + +Remember to specify the `config_name`: +```python +from omegaconf import DictConfig, OmegaConf +import hydra + +@hydra.main(version_base=None, config_path="conf", config_name="config") +def my_app(cfg: DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) + +if __name__ == "__main__": + my_app() +``` + +When you run the updated application, MySQL is loaded by default. +```yaml +$ python my_app.py +db: + driver: mysql + pass: secret + user: omry +``` + +You can have multiple items in the defaults list, e.g. +```yaml +defaults: + - db: mysql + - db/mysql/engine: innodb +``` + +The defaults are ordered: + * If multiple configs define the same value, the last one wins. + * If multiple configs contribute to the same dictionary, the result is the combined dictionary. + + +#### Overriding a config group default + +You can still load PostgreSQL, and override individual values. +```yaml +$ python my_app.py db=postgresql db.timeout=20 +db: + driver: postgresql + pass: drowssap + timeout: 20 + user: postgres_user +``` + +You can remove a default entry from the defaults list by prefixing it with ~: +```yaml +$ python my_app.py ~db +{} +``` + +### Composition order of primary config +Your primary config can contain both config values and a Defaults List. +In such cases, you should add the `_self_` keyword to your defaults list to specify the composition order of the config file relative to the items in the defaults list. + +* If you want your primary config to override the values of configs from the Defaults List, append `_self_` to the end of the Defaults List. +* If you want the configs from the Defaults List to override the values in your primary config, insert `_self_` as the first item in your Defaults List. + + +
+ +
+ +```yaml title="config.yaml" {3} +defaults: + - db: mysql + - _self_ + +db: + user: root +``` +
+
+ +```yaml title="Result config: db.user from config.yaml" {4} +db: + driver: mysql # db/mysql.yaml + pass: secret # db/mysql.yaml + user: root # config.yaml + + +``` +
+
+ +```yaml title="config.yaml" {2} +defaults: + - _self_ + - db: mysql + +db: + user: root +``` +
+
+ +```yaml title="Result config: All values from db/mysql" {4} +db: + driver: mysql # db/mysql.yaml + pass: secret # db/mysql.yaml + user: omry # db/mysql.yaml + + +``` +
+
+ +See [Composition Order](advanced/defaults_list.md#composition-order) for more information. + +:::info +The default composition order changed between Hydra 1.0 and Hydra 1.1. +- **Hydra 1.0**: Configs from the defaults list are overriding the primary config +- **Hydra 1.1**: A config is overriding the configs from the defaults list. + +To mitigate confusion, Hydra 1.1 issue a warning if the primary config contains both Default List and Config values, and `_self_` is not specified in the Defaults List. + The warning will disappear if you add `_self_` to the Defaults List based on the desired behavior. +::: + +### Non-config group defaults +Sometimes a config file does not belong in any config group. +You can still load it by default. Here is an example for `some_file.yaml`. +```yaml +defaults: + - some_file +``` +Config files that are not part of a config group will always be loaded. They cannot be overridden. +Prefer using a config group. + +:::info +For more information about the Defaults List see [Reference Manual/The Defaults List](../../../advanced/defaults_list.md). +::: diff --git a/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/6_composition.md b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/6_composition.md new file mode 100644 index 00000000000..19467634b98 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/basic/your_first_app/6_composition.md @@ -0,0 +1,82 @@ +--- +id: composition +title: Putting it all together +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +As software gets more complex, we resort to modularity and composition to keep it manageable. +We can do the same with configs. Suppose we want our working example to support multiple databases, with +multiple schemas per database, and different UIs. We wouldn't write a separate class +for each permutation of db, schema and UI, so we shouldn't write separate configs either. We use +the same solution in configuration as in writing the underlying software: composition. + +To do this in Hydra, we first add a `schema` and a `ui` config group: + +``` text title="Directory layout" +├── conf +│ ├── config.yaml +│ ├── db +│ │ ├── mysql.yaml +│ │ └── postgresql.yaml +│ ├── schema +│ │ ├── school.yaml +│ │ ├── support.yaml +│ │ └── warehouse.yaml +│ └── ui +│ ├── full.yaml +│ └── view.yaml +└── my_app.py +``` + +With these configs, we already have 12 possible combinations. Without composition, we would need 12 separate configs. +A single change, such as renaming `db.user` to `db.username`, requires editing all 12 of them. +This is a maintenance nightmare. + +Composition can come to the rescue. +Instead of creating 12 different config files, that fully specify each +config, create a single config that specifies the different configuration dimensions, and the default for each. + +```yaml title="config.yaml" +defaults: + - db: mysql + - ui: full + - schema: school +``` + +The resulting configuration is a composition of the *mysql* database, the *full* ui, and the *school* schema +(which we are seeing for the first time here): + +```yaml +$ python my_app.py +db: + driver: mysql + user: omry + pass: secret +ui: + windows: + create_db: true + view: true +schema: + database: school + tables: + - name: students + fields: + - name: string + - class: int + - name: exams + fields: + - profession: string + - time: data + - class: int +``` + +Stay tuned to see how to run all of the combinations automatically ([Multi-run](/tutorials/basic/running_your_app/2_multirun.md)). + +### Summary + - The addition of each new db, schema, or ui only requires a single file. + - Each config group can have a default specified in the Defaults List. + - Any combination can be composed by selecting the desired option from each config group in the + Defaults List or the command line. diff --git a/website/versioned_docs/version-1.3/tutorials/intro.md b/website/versioned_docs/version-1.3/tutorials/intro.md new file mode 100644 index 00000000000..df9f76eaadb --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/intro.md @@ -0,0 +1,11 @@ +--- +id: intro +title: Tutorials intro +--- + +### Basic Tutorial +The [Basic Tutorial](/tutorials/basic/your_first_app/1_simple_cli.md) covers basic Hydra concepts. + +### Structured configs +The [Structured Configs Tutorial](/tutorials/structured_config/0_intro.md) shows how to create strongly typed configurations. + diff --git a/website/versioned_docs/version-1.3/tutorials/structured_config/0_intro.md b/website/versioned_docs/version-1.3/tutorials/structured_config/0_intro.md new file mode 100644 index 00000000000..533c8226cb1 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/structured_config/0_intro.md @@ -0,0 +1,40 @@ +--- +id: intro +title: Introduction to Structured Configs +sidebar_label: Introduction to Structured Configs +--- + +import GithubLink from "@site/src/components/GithubLink" + +This is an advanced tutorial that assumes that you are comfortable with the concepts introduced in the [Basic Tutorial](/tutorials/basic/your_first_app/1_simple_cli.md). +The examples in this tutorial are available here. + +Structured Configs use Python [dataclasses](https://docs.python.org/3.7/library/dataclasses.html) to +describe your configuration structure and types. They enable: + +* **Runtime type checking** as you compose or mutate your config +* **Static type checking** when using static type checkers (mypy, PyCharm, etc.) + +#### Structured Configs supports: +- Primitive types (`int`, `bool`, `float`, `str`, `Enums`, `bytes`, `pathlib.Path`) +- Nesting of Structured Configs +- Containers (List and Dict) containing primitives, Structured Configs, or other lists/dicts +- Optional fields + +#### Structured Configs Limitations: +- `Union` types are only partially supported (see [OmegaConf docs on unions](https://omegaconf.readthedocs.io/en/latest/structured_config.html#union-types)) +- User methods are not supported + +See the [OmegaConf docs on Structured Configs](https://omegaconf.readthedocs.io/en/latest/structured_config.html) for more details. + +#### There are two primary patterns for using Structured configs with Hydra + +- As a [config](/tutorials/structured_config/1_minimal_example.md), in place of configuration files (often a starting place) +- As a [config schema](/tutorials/structured_config/5_schema.md) validating configuration files (better for complex use cases) + +With both patterns, you still get everything Hydra has to offer (config composition, Command line overrides etc). +This tutorial covers both. \***Read it in order**\*. + +Hydra supports OmegaConf's Structured Configs via the `ConfigStore` API. +This tutorial does not assume any knowledge of them. +It is recommended that you visit the OmegaConf Structured Configs page to learn more later. diff --git a/website/versioned_docs/version-1.3/tutorials/structured_config/10_config_store.md b/website/versioned_docs/version-1.3/tutorials/structured_config/10_config_store.md new file mode 100644 index 00000000000..a7de100805b --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/structured_config/10_config_store.md @@ -0,0 +1,159 @@ +--- +id: config_store +title: Config Store API +--- + +Throughout the rest of tutorials, we will be using `ConfigStore` to register dataclasses as input configs in Hydra. +`ConfigStore` is a singleton storing configs in memory. +The primary API for interacting with the `ConfigStore` is the store method described below. + +### API +```python +class ConfigStore(metaclass=Singleton): + def store( + self, + name: str, + node: Any, + group: Optional[str] = None, + package: Optional[str] = "_group_", + provider: Optional[str] = None, + ) -> None: + """ + Stores a config node into the repository + :param name: config name + :param node: config node, can be DictConfig, ListConfig, + Structured configs and even dict and list + :param group: config group, subgroup separator is '/', + for example hydra/launcher + :param package: Config node parent hierarchy. + Child separator is '.', for example foo.bar.baz + :param provider: the name of the module/app providing this config. + Helps debugging. + """ + ... +``` + +### ConfigStore and YAML input configs + +`ConfigStore` has feature parity with YAML input configs. On top of that, it also provides typing validation. +`ConfigStore` can be used alone or together with YAML. We will see more examples later in this series of tutorials. +For now, let's see how the `ConfigStore` API translates into the YAML input configs, which we've become more familiar +with after the basic tutorials. + +Say we have a simple application and a `db` config group with a `mysql` option: + +
+ +
+ +```python title="my_app.py" +@hydra.main(version_base=None, config_path="conf") +def my_app(cfg: DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) + + +if __name__ == "__main__": + my_app() +``` +
+
+ +```text title="Directory layout" +├─ conf +│ └─ db +│ └─ mysql.yaml +└── my_app.py + + + +``` +
+
+ +```yaml title="db/mysql.yaml" +driver: mysql +user: omry +password: secret + + + + +``` +
+
+ +What if we want to add an `postgresql` option now? Yes, we can easily add a `db/postgresql.yaml` config group option. But +that is not the only way! We can also use `ConfigStore` to make another config group option for `db` available to Hydra. + +To achieve this, we add a few lines (highlighted) in the above `my_app.py` file: + + +```python title="my_app.py" {1-9} +@dataclass +class PostgresSQLConfig: + driver: str = "postgresql" + user: str = "jieru" + password: str = "secret" + +cs = ConfigStore.instance() +# Registering the Config class with the name `postgresql` with the config group `db` +cs.store(name="postgresql", group="db", node=PostgresSQLConfig) + +@hydra.main(version_base=None, config_path="conf") +def my_app(cfg: DictConfig) -> None: + print(OmegaConf.to_yaml(cfg)) + + +if __name__ == "__main__": + my_app() +``` + + +Now that our application has access to both `db` config group options, let's run the application to verify: + +
+ +
+ +```commandline title="python my_app.py +db=mysql" +db: + driver: mysql + user: omry + password: secret + +``` +
+
+ +```commandline title="python my_app.py +db=postgresql" +db: + driver: postgresql + user: jieru + password: secret + +``` +
+
+ + +### Example node values +A few examples of supported node values parameters: +```python +from dataclasses import dataclass + +from hydra.core.config_store import ConfigStore + +@dataclass +class MySQLConfig: + host: str = "localhost" + port: int = 3306 + +cs = ConfigStore.instance() + +# Using the type +cs.store(name="config1", node=MySQLConfig) +# Using an instance, overriding some default values +cs.store(name="config2", node=MySQLConfig(host="test.db", port=3307)) +# Using a dictionary, forfeiting runtime type safety +cs.store(name="config3", node={"host": "localhost", "port": 3308}) +``` \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/tutorials/structured_config/1_minimal_example.md b/website/versioned_docs/version-1.3/tutorials/structured_config/1_minimal_example.md new file mode 100644 index 00000000000..668cae8d672 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/structured_config/1_minimal_example.md @@ -0,0 +1,87 @@ +--- +id: minimal_example +title: Minimal example +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +There are four key elements in this example: +- A `@dataclass` describes the application's configuration +- `ConfigStore` manages the Structured Config +- `cfg` is `duck typed` as a `MySQLConfig` instead of a `DictConfig` +- There is a subtle typo in the code below, can you spot it? + +In this example, the config node stored in the `ConfigStore` replaces the traditional `config.yaml` file. + +```python title="my_app_type_error.py" {18} +from dataclasses import dataclass + +import hydra +from hydra.core.config_store import ConfigStore + +@dataclass +class MySQLConfig: + host: str = "localhost" + port: int = 3306 + +cs = ConfigStore.instance() +# Registering the Config class with the name 'config'. +cs.store(name="config", node=MySQLConfig) + +@hydra.main(version_base=None, config_name="config") +def my_app(cfg: MySQLConfig) -> None: + # pork should be port! + if cfg.pork == 80: + print("Is this a webserver?!") + +if __name__ == "__main__": + my_app() +``` + +### Duck-typing enables static type checking + +Duck-typing the config object as `MySQLConfig` enables static type checkers like `mypy` to catch +type errors before you run your code: +```text title="$ mypy my_app_type_error.py" +my_app_type_error.py:22: error: "MySQLConfig" has no attribute "pork" +Found 1 error in 1 file (checked 1 source file) +``` + +### Structured Configs enable Hydra to catch type errors at runtime +If you forget to run `mypy`, Hydra will report the error at runtime: +``` text title="$ python my_app_type_error.py" +Traceback (most recent call last): + File "my_app_type_error.py", line 22, in my_app + if cfg.pork == 80: +omegaconf.errors.ConfigAttributeError: Key 'pork' not in 'MySQLConfig' + full_key: pork + object_type=MySQLConfig + +Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace. +``` + +Hydra will also catch typos, or type errors in the command line: +``` +$ python my_app_type_error.py port=fail +Error merging override port=fail +Value 'fail' could not be converted to Integer + full_key: port + object_type=MySQLConfig +``` +We will see additional types of runtime errors that Hydra can catch later in this tutorial. Such as: +- Trying to read or write a non existent field in your config object +- Assigning a value that is incompatible with the declared type +- Attempting to modify a [frozen config](https://omegaconf.readthedocs.io/en/latest/structured_config.html#frozen) + +## Duck typing + +In the example above `cfg` is duck typed as `MySQLConfig`. +It is actually an instance of `DictConfig`. The duck typing enables static type checking by tools like Mypy or PyCharm. +This reduces development time by catching coding errors before you run your application. + +The name [Duck typing](https://en.wikipedia.org/wiki/Duck_typing) comes from the phrase "If it walks like a duck, swims like a duck, and quacks like a duck, then it probably is a duck". +It can be useful when you care about the methods or attributes of an object, not the actual type of the object. + + diff --git a/website/versioned_docs/version-1.3/tutorials/structured_config/2_hierarchical_static_config.md b/website/versioned_docs/version-1.3/tutorials/structured_config/2_hierarchical_static_config.md new file mode 100644 index 00000000000..c680a6d3d21 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/structured_config/2_hierarchical_static_config.md @@ -0,0 +1,44 @@ +--- +id: hierarchical_static_config +title: A hierarchical static configuration +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + + +Dataclasses can be nested and then accessed via a common root. The entire tree is type checked. + +```python +from dataclasses import dataclass + +import hydra +from hydra.core.config_store import ConfigStore + +@dataclass +class MySQLConfig: + host: str = "localhost" + port: int = 3306 + +@dataclass +class UserInterface: + title: str = "My app" + width: int = 1024 + height: int = 768 + +@dataclass +class MyConfig: + db: MySQLConfig = field(default_factory=MySQLConfig) + ui: UserInterface = field(default_factory=UserInterface) + +cs = ConfigStore.instance() +cs.store(name="config", node=MyConfig) + +@hydra.main(version_base=None, config_name="config") +def my_app(cfg: MyConfig) -> None: + print(f"Title={cfg.ui.title}, size={cfg.ui.width}x{cfg.ui.height} pixels") + +if __name__ == "__main__": + my_app() +``` \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/tutorials/structured_config/3_config_groups.md b/website/versioned_docs/version-1.3/tutorials/structured_config/3_config_groups.md new file mode 100644 index 00000000000..fe4b8d39310 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/structured_config/3_config_groups.md @@ -0,0 +1,112 @@ +--- +id: config_groups +title: Config Groups +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +Structured Configs can be used to implement config groups. Special care needs to be taken when specifying a +default value for fields populated by a config group. We will look at why below. + +```python title="Defining a config group for database" {16-17,22-23} +from dataclasses import dataclass + +import hydra +from hydra.core.config_store import ConfigStore + +@dataclass +class MySQLConfig: + driver: str = "mysql" + host: str = "localhost" + port: int = 3306 + +@dataclass +class PostGreSQLConfig: + driver: str = "postgresql" + host: str = "localhost" + port: int = 5432 + timeout: int = 10 + +@dataclass +class Config: + # We will populate db using composition. + db: Any + +# Create config group `db` with options 'mysql' and 'postgreqsl' +cs = ConfigStore.instance() +cs.store(name="config", node=Config) +cs.store(group="db", name="mysql", node=MySQLConfig) +cs.store(group="db", name="postgresql", node=PostGreSQLConfig) + +@hydra.main(version_base=None, config_name="config") +def my_app(cfg: Config) -> None: + print(OmegaConf.to_yaml(cfg)) + +if __name__ == "__main__": + my_app() +``` + +:::caution +The *Config* class is **NOT** the Defaults list. We will see the Defaults list in the next page. +::: + +You can select the database from the command line: +```yaml +$ python my_app.py +db=postgresql +db: + driver: postgresql + host: localhost + password: drowssap + port: 5432 + timeout: 10 + user: postgres_user +``` + +The `+` above is required because there is no default choice for the config group `db`. +The next page will reintroduce the Defaults List, eliminating the need for the `+`. + +### Config inheritance + + + +Standard Python inheritance can be used to get improved type safety, and to move common fields to the parent class. + +```python title="Defining a config group for database using inheritance" +from omegaconf import MISSING + +@dataclass +class DBConfig: + host: str = "localhost" + port: int = MISSING + driver: str = MISSING + +@dataclass +class MySQLConfig(DBConfig): + driver: str = "mysql" + port: int = 3306 + +@dataclass +class PostGreSQLConfig(DBConfig): + driver: str = "postgresql" + port: int = 5432 + timeout: int = 10 + +@dataclass +class Config: + # We can now annotate db as DBConfig which + # improves both static and dynamic type safety. + db: DBConfig +``` + +### Missing fields +Assign *MISSING* to a field to indicates that it does not have a default value. This is equivalent to +the `???` literal we have seen in OmegaConf configs before. + +Omitting a default value is equivalent to assigning *MISSING* to it, although it is sometimes +convenient to be able to assign MISSING it to a field. + +:::caution +Do not confuse **omegaconf.MISSING** with **dataclass.MISSING**. +::: diff --git a/website/versioned_docs/version-1.3/tutorials/structured_config/4_defaults.md b/website/versioned_docs/version-1.3/tutorials/structured_config/4_defaults.md new file mode 100644 index 00000000000..3f47f86f379 --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/structured_config/4_defaults.md @@ -0,0 +1,134 @@ +--- +id: defaults +title: Defaults List +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + + + +You can define a defaults list in your primary Structured Config just like you can in your primary `config.yaml` file. +The example below extends the previous example by adding a defaults list that will load `db=mysql` by default. + +
+ +```python {11-14,19,25} +from dataclasses import dataclass + +import hydra +from hydra.core.config_store import ConfigStore +from omegaconf import MISSING, OmegaConf + +@dataclass +class MySQLConfig: + ... + +@dataclass +class PostGreSQLConfig: + ... + +defaults = [ + # Load the config "mysql" from the config group "db" + {"db": "mysql"} +] + +@dataclass +class Config: + # this is unfortunately verbose due to @dataclass limitations + defaults: List[Any] = field(default_factory=lambda: defaults) + + # Hydra will populate this field based on the defaults list + db: Any = MISSING + +cs = ConfigStore.instance() +cs.store(group="db", name="mysql", node=MySQLConfig) +cs.store(group="db", name="postgresql", node=PostGreSQLConfig) +cs.store(name="config", node=Config) + + +@hydra.main(version_base=None, config_name="config") +def my_app(cfg: Config) -> None: + print(OmegaConf.to_yaml(cfg)) + + +if __name__ == "__main__": + my_app() +``` +Running `my_app.py` loads the mysql config option by default: +```yaml +$ python my_app.py +db: + driver: mysql + ... +``` + +You can override the default option via the command line: +```yaml +$ python my_app.py db=postgresql +db: + driver: postgresql + ... +``` + +### A Note about composition order +The default composition order in Hydra is that values defined in a config are merged into values introduced from configs in the Defaults List - or in other words - overriding them. +This behavior can be unintuitive when your primary config is a Structured Config, like in the example above. +For example, if the primary config is: +```python {6} +@dataclass +class Config: + defaults: List[Any] = field(default_factory=lambda: [ + "debug/activate", + # If you do not specify _self_, it will be appended to the end of the defaults list by default. + "_self_" + ]) + + debug: bool = False +``` +And `debug/activate.yaml` is overriding the `debug` flag to `True`, the composition order would be such that debug ends up being `False`. +To get `debug/activate.yaml` to override this config, explicitly specify `_self_` before `debug/activate.yaml`: +```python {4} +@dataclass +class Config: + defaults: List[Any] = field(default_factory=lambda: [ + "_self_", + "debug/activate", + ]) + + debug: bool = False +``` + +See [Composition Order](advanced/defaults_list.md#composition-order) for more information. + +### Requiring users to specify a default list value + +Set `db` as `MISSING` to require the user to specify a value on the command line. + +
+
+ +```python title="Defaults list with a missing db" +defaults = [ + {"db": MISSING} +] + + +``` + +
+ +
+ +```text title="Output" +$ python my_app.py +You must specify 'db', e.g, db=
+
diff --git a/website/versioned_docs/version-1.3/tutorials/structured_config/5_schema.md b/website/versioned_docs/version-1.3/tutorials/structured_config/5_schema.md new file mode 100644 index 00000000000..fb340f025ee --- /dev/null +++ b/website/versioned_docs/version-1.3/tutorials/structured_config/5_schema.md @@ -0,0 +1,305 @@ +--- +id: schema +title: Structured Config schema +--- + +import {ExampleGithubLink} from "@site/src/components/GithubLink" + +We have seen how to use Structured Configs as configuration, but they can also be used as a schema (i.e. validating configuration files). +To achieve this, we will follow the common pattern of [Extending Configs](../../patterns/extending_configs.md) - but instead of extending another config file, +we will extend a Structured Config. + +This page shows how to validate the config files `config.yaml`, `db/mysql.yaml` and `db/postgresql.yaml` +against a Structured Config schema. + +### Validating against a schema in the same config group + + + +Given the config directory structure: +```text +conf/ +├── config.yaml +└── db + ├── mysql.yaml + └── postgresql.yaml +``` + +We will add Structured Config schema for each of the config files above and store in the +Config Store as `base_config`, `db/base_mysql` and `db/base_postgresql`. + +Then, we will use the Defaults List in each config to specify its base config as follows: + +
+
+ +```yaml title="config.yaml" {2} +defaults: + - base_config + - db: mysql + # See composition order note + - _self_ + +debug: true +``` + +
+
+ +```yaml title="db/mysql.yaml" {2} +defaults: + - base_mysql + +user: omry +password: secret + + +``` +
+
+ +```yaml title="db/postgresql.yaml" {2} +defaults: + - base_postgresql + +user: postgres_user +password: drowssap + + +``` +
+
+ +One difference in the source code is that we have removed the Defaults List from the `Config` dataclass. +The primary Defaults List will come from `config.yaml`. +
my_app.py (Click to expand) + +```python {28-30} +from dataclasses import dataclass + +import hydra +from hydra.core.config_store import ConfigStore + +@dataclass +class DBConfig: + driver: str = MISSING + host: str = "localhost" + port: int = MISSING + +@dataclass +class MySQLConfig(DBConfig): + driver: str = "mysql" + port: int = 3306 + user: str = MISSING + password: str = MISSING + +@dataclass +class PostGreSQLConfig(DBConfig): + driver: str = "postgresql" + user: str = MISSING + port: int = 5432 + password: str = MISSING + timeout: int = 10 + +@dataclass +class Config: + db: DBConfig = MISSING + debug: bool = False + +cs = ConfigStore.instance() +cs.store(name="base_config", node=Config) +cs.store(group="db", name="base_mysql", node=MySQLConfig) +cs.store(group="db", name="base_postgresql", node=PostGreSQLConfig) + +@hydra.main(version_base=None, config_path="conf", config_name="config") +def my_app(cfg: Config) -> None: + print(OmegaConf.to_yaml(cfg)) + +if __name__ == "__main__": + my_app() +``` +
+
+ +When Hydra composes the final config object it will use the config schemas as specified in the Default Lists. +Like before, Hydra will catch user errors in the command line: + +```yaml +$ python my_app.py db.port=fail +Error merging override db.port=fail +Value 'fail' could not be converted to Integer + full_key: db.port + object_type=MySQLConfig +``` + +
Use --info commands to see how a config was composed (Expand) + +```text +$ python my_app.py --info defaults-tree + +Defaults Tree +************* +: + hydra/config: + hydra/output: default + hydra/launcher: basic + hydra/sweeper: basic + hydra/help: default + hydra/hydra_help: default + hydra/hydra_logging: default + hydra/job_logging: default + _self_ + config: + base_config + db: mysql: + db/base_mysql + _self_ + _self_ + +$ python my_app.py --info defaults + +Defaults List +************* +| Config path | Package | _self_ | Parent | +------------------------------------------------------------------------------ +| hydra/output/default | hydra | False | hydra/config | +| hydra/launcher/basic | hydra.launcher | False | hydra/config | +| hydra/sweeper/basic | hydra.sweeper | False | hydra/config | +| hydra/help/default | hydra.help | False | hydra/config | +| hydra/hydra_help/default | hydra.hydra_help | False | hydra/config | +| hydra/hydra_logging/default | hydra.hydra_logging | False | hydra/config | +| hydra/job_logging/default | hydra.job_logging | False | hydra/config | +| hydra/config | hydra | True | | +| base_config | | False | config | +| db/base_mysql | db | False | db/mysql | +| db/mysql | db | True | config | +| config | | True | | +------------------------------------------------------------------------------ +``` + +
+ + +### Validating against a schema from a different config group + + + +In the above example, the schema we used was stored in the same config group. +This is not always the case, for example - A library might provide schemas in its own config group. + +Here is a modified version of the example above, where a mock database_lib is providing the schemas +we want to validate against. + + +
+
+ +```python title="my_app.py" +from dataclasses import dataclass + +import hydra +from hydra.core.config_store import ConfigStore + +import database_lib + + +@dataclass +class Config: + db: database_lib.DBConfig = MISSING + debug: bool = False + +cs = ConfigStore.instance() +cs.store(name="base_config", node=Config) + +# database_lib registers its configs +# in database_lib/db +database_lib.register_configs() + + +@hydra.main( + version_base=None, + config_path="conf", + config_name="config", +) +def my_app(cfg: Config) -> None: + print(OmegaConf.to_yaml(cfg)) + + +if __name__ == "__main__": + my_app() +``` +
+
+ +```python title="database_lib.py" {17,22} +from dataclasses import dataclass + +from hydra.core.config_store import ConfigStore + +@dataclass +class DBConfig: + ... + +@dataclass +class MySQLConfig(DBConfig): + ... + +@dataclass +class PostGreSQLConfig(DBConfig): + ... + + +def register_configs() -> None: + cs = ConfigStore.instance() + cs.store( + group="database_lib/db", + name="mysql", + node=MySQLConfig, + ) + cs.store( + group="database_lib/db", + name="postgresql", + node=PostGreSQLConfig, + ) + +``` +
+
+ +The Defaults List entry for the base config is slightly different: +
+
+ +```yaml title="db/mysql.yaml" {2} +defaults: + - /database_lib/db/mysql@_here_ + +user: omry +password: secret +``` +
+
+ +```yaml title="db/postgresql.yaml" {2} +defaults: + - /database_lib/db/postgresql@_here_ + # See composition order note + - _self_ + +user: postgres_user +password: drowssap +``` +
+
+ +- We refer to the config with an absolute path because it is outside the subtree of the db config group. +- we override the package to `_here_` to ensure that the package of the schema is the same as the package + of the config it's validating. + +### A Note about composition order + By default, Hydra 1.1 appends `_self_` to the end of the Defaults List. +This behavior is new in Hydra 1.1 and different from previous Hydra versions. As such Hydra 1.1 issues a warning if `_self_` is not specified **in the primary config**, asking you to add `_self_` and thus indicate the desired composition order. +To address the warning while maintaining the new behavior, append `_self_` to the end of the Defaults List. Note that in some cases it may instead be desirable to add `_self_` directly after the schema and before other Defaults List elements. + + +See [Composition Order](advanced/defaults_list.md#composition-order) for more information. diff --git a/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/adding_a_package_directive.md b/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/adding_a_package_directive.md new file mode 100644 index 00000000000..991e858df4c --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/adding_a_package_directive.md @@ -0,0 +1,61 @@ +--- +id: adding_a_package_directive +title: Adding an @package directive +hide_title: true +--- +## Adding an @package directive +Hydra 1.0 introduces the concept of a config `package`. A `package` is the common parent +path of all nodes in the config file. + + - In Hydra 0.11, there was an implicit default of `_global_` ("") + - In Hydra 1.1 the default will be `_group_` (the name of the config group). + - Hydra 1.0 maintains the implicit default of `_global_` and issues a warning for +any config group file without a `@package` directive. + +By adding an explicit `@package` to these configs now, you guarantee that your configs +will not break when you upgrade to Hydra 1.1. + +The `@package` directive is described in details [here](/advanced/overriding_packages.md). + +## Upgrade instructions: +### Recommended (~10 seconds per config file): +`Case 1`: For config files where the common parent path matches the config group name: + - Add `# @package _group_` to the top of every config group file + - Remove the common parent path config file like in the example below. + +`Case 2`: For files without a common parent path: + - Add `# @package _global_`. + +### Alternative (not recommended): + - If you do not want to restructure the config at this time use `Case 2` for all your config files. + +### Example for `case 1`: + +#### Before +```yaml title="db/mysql.yaml" +db: + driver: mysql + host: localhost + port: 3306 +``` +#### After +```yaml title="db/mysql.yaml" +# @package _group_ +driver: mysql +host: localhost +port: 3306 +``` +The interpretations of the before and after files are identical. + +### Example for `case 2`: +```yaml title="env/prod.yaml" +# @package _global_ +db: + driver: mysql + host: 10.0.0.11 + port: 3306 + +webserver: + host: 10.0.0.11 + port: 443 +``` diff --git a/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/config_path_changes.md b/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/config_path_changes.md new file mode 100644 index 00000000000..4217824d32f --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/config_path_changes.md @@ -0,0 +1,39 @@ +--- +id: config_path_changes +title: Config path changes +hide_title: true +--- + +## Config path changes +Hydra 1.0 adds a new `config_name` parameter to `@hydra.main()` and changes the meaning of the `config_path`. +Previously, `config_path` encapsulated two things: +- Search path relative to the declaring python file. +- Optional config file (.yaml) to load. + +Having both of those things in the same parameter does not work well if you consider configs that are not files +such as Structured Configs. In addition, it prevents overriding just the `config_name` or just the `config_path` + +A second change is that the `config_name` no longer requires a file extension. For configs files the `.yaml` extension +will be added automatically when the file is loaded. + +This change is backward compatible, but support for the old style is deprecated and will be removed in the next major Hydra version. + +## Migration examples + +```python title="Hydra 0.11" +@hydra.main(config_path="config.yaml") +``` +Becomes: +```python title="Hydra 1.0" +@hydra.main(config_name="config") +``` + +And +```python title="Hydra 0.11" +@hydra.main(config_path="conf/config.yaml") +``` + +Becomes: +```python title="Hydra 1.0" +@hydra.main(config_path="conf", config_name="config") +``` diff --git a/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/object_instantiation_changes.md b/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/object_instantiation_changes.md new file mode 100644 index 00000000000..71de03a4844 --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/object_instantiation_changes.md @@ -0,0 +1,62 @@ +--- +id: object_instantiation_changes +title: Object instantiation changes +hide_title: true +--- + + +## Object instantiation changes +Hydra 1.0.0 is deprecating ObjectConf and the corresponding config structure to a simpler one without the params node. +This removes a level of nesting from command line and configs overrides. + +
+
+ +```yaml title="Hydra 0.11" +class: my_app.MySQLConnection +params: + host: localhost + user: root + password: 1234 +``` + +
+ +
+ +```yaml title="Hydra 1.0" +_target_: my_app.MySQLConnection +host: localhost +user: root +password: 1234 + +``` + + +
+
+ +## Hydra configuration +Hydra plugins are configured using the same mechanism. +This means that this change will effect how all plugins are configured and overridden. +This is a breaking change for code overriding configs in such plugins, but luckily it's easy to fix. + +As an example, a Sweeper plugin override will change as follows: + +
+
+ +```shell script title="Hydra 0.11" +hydra.sweeper.params.max_batch_size=10 +``` + +
+ +
+ +```shell script title="Hydra 1.0" +hydra.sweeper.max_batch_size=10 +``` + +
+
\ No newline at end of file diff --git a/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/strict_mode_flag_deprecated.md b/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/strict_mode_flag_deprecated.md new file mode 100644 index 00000000000..ca01be13513 --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/0.11_to_1.0/strict_mode_flag_deprecated.md @@ -0,0 +1,55 @@ +--- +id: strict_mode_flag_deprecated +title: strict flag mode deprecation +hide_title: true +--- +## strict flag mode deprecation +The strict mode is a flag added to `@hydra.main()` to enable two features: +- Command line error detection (overriding a field not in the config) +- Runtime config access error detection (accessing/setting a field not in the config) + +This flag is now deprecated, and the ability to turn it off will be removed in Hydra 1.1. + +## Alternatives to `strict=False` +Below are a few common reasons for people disabling strict mode along with recommended alternatives. + +### Adding config fields through the command line +With strict mode disabled; you can add fields not specified in config file through the command line. +Hydra 1.0 introduces the + prefix to command line overrides, enabling the addition of fields not in the config file. + +```yaml title="config.yaml" +db: + driver: mysql +``` + +```yaml {1,6} +$ python my_app.py +db.host=localhost +db: + driver: mysql + host: localhost +``` + +### Adding fields at runtime +When strict mode is disabled, you can add fields to your config at runtime. +Strict mode is implemented by setting the OmegaConf `struct` flag to True on the root of the config. +- You can disable the struct flag a specific context using `open_dict`. +- You can disable the struct flag permanently for your config using `OmegaConf.set_struct(cfg, False)`. + +Learn more about OmegaConf struct flag here. + + +### Field existence check +With strict mode disabled, you can check if a field is in the config by comparing it to None: +```python +if cfg.host is None: + # host is not in the config +``` +This will no longer work because an exception will be thrown when the `host` field is accessed. +Use `in` or `hasattr` instead: +```python +if "host" not in cfg: + # host is not in the config + +if not hasattr(cfg, "host"): + # host is not in the config +``` \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/automatic_schema_matching.md b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/automatic_schema_matching.md new file mode 100644 index 00000000000..dffd33cecbf --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/automatic_schema_matching.md @@ -0,0 +1,184 @@ +--- +id: automatic_schema_matching +title: Automatic schema-matching +hide_title: true +--- + +In Hydra 1.0, when a config file is loaded, if a config with a matching name and group is present in the `ConfigStore`, +it is used as the schema for the newly loaded config. + +There are several problems with this approach: + +- **Inflexible**: This approach can only be used when a schema should validate a single config file. +It does not work if you want to use the same schema to validate multiple config files. +- **Unexpected**: This behavior can be unexpected. There is no way to tell this is going to happen when looking at a given +config file. + +Hydra 1.1 deprecates this behavior in favor of an explicit config extension via the Defaults List. +This upgrade page aims to provide a summary of the required changes. It is highly recommended that you read the following pages: +- [Background: The Defaults List](../../advanced/defaults_list.md) +- [Background: Extending configs](../../patterns/extending_configs.md) +- [Tutorial: Structured config schema](../../tutorials/structured_config/5_schema.md) + + +## Migration +Before the upgrade, you have two different configs with the same name (a config file, and a Structured Config in the `ConfigStore`). +You need to rename one of them. Depending on the circumstances and your preference you may rename one or the other. +- If you control both configs, you can rename either of them. +- If you only control the config file, rename it. + +### Option 1: rename the Structured Config +This option is less disruptive. Use it if you control the Structured Config. +1. Use a different name when storing the schema into the Config Store. Common choices: + - `base_` prefix, e.g. `base_mysql`. + - `_schema` suffix, e.g. `mysql_schema`. +2. Add the schema to the Defaults List of the extending config file. + +
Click to show an example + +#### Hydra 1.0 +
+
+ +```yaml title="db/mysql.yaml" +# @package _group_ +host: localhost +port: 3306 + + + + + + +``` +
+
+ +```python title="db/mysql schema in the ConfigStore" +@dataclass +class MySQLConfig: + host: str + port: int + +cs = ConfigStore.instance() +cs.store(group="db", + name="mysql", + node=MySQLConfig) +``` +
+
+ +#### Hydra 1.1 +
+
+ +```yaml title="db/mysql.yaml" {1,2} +defaults: + - base_mysql + +host: localhost +port: 3306 + + + + +``` +
+
+ +```python title="db/mysql schema in the ConfigStore" {8} +@dataclass +class MySQLConfig: + host: str + port: int + +cs = ConfigStore.instance() +cs.store(group="db", + name="base_mysql", + node=MySQLConfig) +``` +
+
+ +
+ +### Option 2: rename the config file +This option is a bit more disruptive. Use it if you only control the config file. +1. Rename the config file. Common choices are `custom_` or `my_` prefix, e.g. `custom_mysql.yaml`. You can also use a domain specific name like `prod_mysql.yaml`. +2. Add the schema to the Defaults List of the extending config file. +3. Update references to the config name accordingly, e.g. on the command-line `db=mysql` would become `db=custom_mysql`, and in a defaults list `db: mysql` would become `db: custom_mysql`. + + +
Click to show an example + +#### Hydra 1.0 +
+
+ +```yaml title="db/mysql.yaml" +# @package _group_ +host: localhost +port: 3306 +``` +```yaml title="config.yaml" +defaults: + - db: mysql +``` +
+
+ +```python title="db/mysql schema in the ConfigStore" +@dataclass +class MySQLConfig: + host: str + port: int + +cs = ConfigStore.instance() +cs.store(group="db", + name="mysql", + node=MySQLConfig) + +``` +
+
+ +#### Hydra 1.1 +Rename `db/mysql.yaml` to `db/custom_mysql.yaml` and explicitly add the schema to the Defaults List. +
+ +
+ +```yaml title="db/custom_mysql.yaml" {1,2} +defaults: + - mysql + +host: localhost +port: 3306 +``` +```yaml title="config.yaml" {2} +defaults: + - db: custom_mysql +``` + +
+
+ +```python title="db/mysql schema in the ConfigStore" + + + + + + NO CHANGES + + + + + + +``` +
+
+ +Don't forget to also update your command line overrides from `db=mysql` to `db=custom_mysql`. +
diff --git a/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/changes_to_default_composition_order.md b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/changes_to_default_composition_order.md new file mode 100644 index 00000000000..8a51ad2633a --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/changes_to_default_composition_order.md @@ -0,0 +1,176 @@ +--- +id: default_composition_order +title: Changes to default composition order +--- +Default composition order is changing in Hydra 1.1. + +For this example, let's assume the following two configs: +
+
+ +```yaml title="config.yaml" +defaults: + - foo: bar + +foo: + x: 10 +``` + +
+ +
+ +```yaml title="foo/bar.yaml" +# @package _group_ +x: 20 + + + +``` +
+
+ + +
+
+ +In **Hydra 1.0**, configs from the Defaults List are overriding *config.yaml*, resulting in the following output: +
+
+ +```yaml {2} +foo: + x: 20 +``` +
+
+ + + +
+
+ +As of **Hydra 1.1**, *config.yaml* is overriding configs from the Defaults List, resulting in the following output: +
+
+ +```yaml {2} +foo: + x: 10 +``` +
+
+ + + +## Migration +If your application uses `hydra.main`, the best way to verify that updating Hydra versions does not change your job configurations is to compare the output of `python my_app.py --cfg job` on both the new and old Hydra versions. If your application uses the Compose API, please make sure you have comprehensive unit tests on the composed configuration. + +### Primary config is a YAML file +To ensure this change is not missed by people migrating from Hydra 1.0, Hydra 1.1 issues a warning if the Defaults List in the primary config is missing `_self_`, and there are config values in addition to the Defaults List. +To address the warning, add `_self_` to the Defaults List of the primary config. + +- If the new behavior works for your application, append `_self_` to the end of the Defaults List. +- If your application requires the previous behavior, insert `_self_` as the first item in your Defaults List. + + +
+
+ +```yaml title="config.yaml" {2} +defaults: + - _self_ + - foo: bar + +foo: + x: 10 +``` +
+ +
+ +```yaml title="Output config" +foo: + x: 20 + + + + +``` +
+
+ + +The Defaults List is described [here](/advanced/defaults_list.md). + +### Primary config is a Structured Config +Structured Configs used as primary config may see changes as well. +You should add `_self_` to the defaults list to indicate the composition order. In such cases you will typically want `_self_` to be the first item in the defaults list. + +```python {3,14} +defaults = [ + "_self_", + {"db": "mysql"} +] + +@dataclass +class Config: + # this is unfortunately verbose due to @dataclass limitations + defaults: List[Any] = field(default_factory=lambda: defaults) + + # Hydra will populate this field based on the defaults list + db: Any = MISSING +``` + +### Primary config is a config file with a Structured Config schema +If you use Structured Config as a schema for your primary config, be sure to add `_self_` after the schema in the Defaults List, otherwise the schema will override the config instead of the other way around. + +
+
+ +```python title="my_app.py" +@dataclass +class Config: + host: str = "localhost" + port: int = 8080 + +cs.store(name="base_config", + node=Config) +``` + +
+ +
+ +```yaml {3,5} title="config.yaml" +defaults: + - base_config # schema + - _self_ # after schema + +port: 3306 + + +``` + +
+ +
+ +```yaml {2} title="Output config" +host: localhost # schema +port: 3306 # config.yaml + + + + + +``` + +
+ +
+ + +### Compatibility with both Hydra 1.0 and 1.1 +If your config must be compatible with both Hydra 1.0 and 1.1, Insert `_self_` as the first item in the Defaults List. +Hydra 1.0.7 (or newer releases in Hydra 1.0) ignores `_self_` in the Defaults List and Hydra 1.1 will compose the same config as Hydra 1.0 if `_self_` is the first item. diff --git a/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/changes_to_package_header.md b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/changes_to_package_header.md new file mode 100644 index 00000000000..0706b5f3c0c --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/changes_to_package_header.md @@ -0,0 +1,85 @@ +--- +id: changes_to_package_header +title: Changes to Package Header +--- + +Hydra 1.0 introduced the package header and required everyone to specify it in their configs. +This was done to facilitate a transition from a model where the packages are global +to a model where - by default - package are derived from the config group. + +e.g: Change of the default package for `server/db/mysql.yaml` from `_global_` to `server.db`. + +Hydra 1.1 completes this transition. +- If a package header is not specified, the config will have the default package as described above. +- \_group\_ and \_name\_ in package header are deprecated (You can still use a literal package header). + + +:::info +Another important change in Hydra 1.1 is the +[Changes to default composition order](./changes_to_default_composition_order.md). +::: + +### Migration + +If your header is `# @package _group_`, remove the header. +
+
+ +```yaml title="db/mysql.yaml in Hydra 1.0" +# @package _group_ +host: localhost +``` + +
+ +
+ +```yaml title="db/mysql.yaml in Hydra 1.1" +host: localhost + +``` +
+
+ +If your header is using `_group_` or `_name_` to specify a package other than the default package, +Specify the literal package: + +
+
+ +```yaml title="db/mysql.yaml in Hydra 1.0" +# @package _group_._name_ +host: localhost +``` + +
+ +
+ +```yaml title="db/mysql.yaml in Hydra 1.1" +# @package db.mysql +host: localhost +``` +
+
+ +### Compatibility with both Hydra 1.0 and 1.1 +If your configs should be compatible with both Hydra 1.0 and Hydra 1.1, use literal package headers. +
+
+ +```yaml title="db/mysql.yaml in Hydra 1.0" +# @package _group_ +host: localhost +``` + +
+ +
+ +```yaml title="db/mysql.yaml in Hydra 1.1" +# @package db +host: localhost +``` +
+
diff --git a/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/defaults_list_interpolation_changes.md b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/defaults_list_interpolation_changes.md new file mode 100644 index 00000000000..20d975c49ae --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/defaults_list_interpolation_changes.md @@ -0,0 +1,41 @@ +--- +id: defaults_list_interpolation +title: Defaults List interpolation +--- + +The defaults lists are used to compose the final config object. +Hydra supports a limited form of interpolation in the defaults list. +The interpolation style described there is deprecated in favor of a cleaner style more +appropriate to recursive default lists. + +## Migration examples + +For example, the following snippet from Hydra 1.0 or older: +```yaml +defaults: + - dataset: imagenet + - model: alexnet + - dataset_model: ${defaults.0.dataset}_${defaults.1.model} +``` + +Changes to this in Hydra 1.1 or newer: +```yaml +defaults: + - dataset: imagenet + - model: alexnet + - dataset_model: ${dataset}_${model} +``` + +The new style is more compact and does not require specifying the exact index of the element in the defaults list. +This is enables interpolating using config group values that are coming from recursive defaults. + +Note that: + - This is non-standard interpolation support that is unique to the defaults list + - interpolation keys in the defaults list can not access values from the composed config because it does not yet + exist when Hydra is processing the defaults list + +The Defaults List is described [here](/advanced/defaults_list.md). + +:::warning +Support for the old style will be removed in Hydra 1.2. +::: \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/defaults_list_override.md b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/defaults_list_override.md new file mode 100644 index 00000000000..f5e1c164fbe --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/defaults_list_override.md @@ -0,0 +1,22 @@ +--- +id: defaults_list_override +title: Defaults List Overrides +--- +Hydra versions prior to 1.1 supported overriding of Hydra config groups via the Defaults List in this manner: +```yaml {3} +defaults: + - model: resnet50 + - hydra/launcher: submitit +``` +As of Hydra 1.1, Config group overrides must be marked explicitly with the `override` keyword: +```yaml {3} +defaults: + - model: resnet50 + - override hydra/launcher: submitit +``` + +The Defaults List is described [here](/advanced/defaults_list.md). + +:::warning +Omitting the `override` keyword when overriding Hydra's config groups will result in an error in Hydra 1.2 +::: \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/hydra_main_config_path.md b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/hydra_main_config_path.md new file mode 100644 index 00000000000..d2d3d0bd6cb --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.0_to_1.1/hydra_main_config_path.md @@ -0,0 +1,42 @@ +--- +id: changes_to_hydra_main_config_path +title: Changes to @hydra.main() and hydra.initialize() +--- + +Prior to Hydra 1.1, **@hydra.main()** and **hydra.initialize()** default `config path` was the directory containing the Python app (calling **@hydra.main()** or **hydra.initialize()**). + +This can cause unexpected behavior: +- Sibling directories are interpreted as config groups, which can lead to surprising results (See [#1533](https://github.com/facebookresearch/hydra/issues/1533)). +- The subtree added automatically can have many files/directories - which will cause **--help** to be very slow as it's scanning for all config groups/config files (See [#759](https://github.com/facebookresearch/hydra/issues/759)). + +To address these issues, Hydra 1.1 issues a warning if the config_path is not specified. +Your options are as follows: + +### Dedicated config directory +For applications with config files, specify a directory like "conf" to use a dedicated config directory relative to the application. +```python +@hydra.main(config_path="conf") +# or: +hydra.initialize(config_path="conf") +``` + +### No config directory +For applications that do not define config files next to the Python script (typically applications using only Structured Configs), it is recommended that +you pass `None` as the config_path, indicating that no directory should be added to the config search path. +This will become the default with [version_base](../version_base.md) >= "1.2" +```python +@hydra.main(config_path=None) +# or: +hydra.initialize(config_path=None) +``` + +### Using the application directory +Use the directory/module of the Python script. +This was the default behavior up to Hydra 1.0. +This is not recommended as it can cause the surprising behavior outlined above. + +```python +@hydra.main(config_path=".") +# or: +hydra.initialize(config_path=".") +``` diff --git a/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/changes_to_job_working_dir.md b/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/changes_to_job_working_dir.md new file mode 100644 index 00000000000..f41573e3ae3 --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/changes_to_job_working_dir.md @@ -0,0 +1,16 @@ +--- +id: changes_to_job_working_dir +title: Changes to job's runtime working directory +hide_title: true +--- + +Hydra 1.2 introduces `hydra.job.chdir`. This config allows users to specify whether Hydra should change the runtime working +directory to the job's output directory. +`hydra.job.chdir` will default to `False` if version_base is set to >= "1.2" (or None), +or otherwise will use the old behavior and default to `True`, with a warning being issued if `hydra.job.chdir` is not set. + +If you want to keep the old Hydra behavior, please set `hydra.job.chdir=True` explicitly for your application. + +For more information about `hydra.job.chdir`, +see [Output/Working directory](/tutorials/basic/running_your_app/3_working_directory.md#disable-changing-current-working-dir-to-jobs-output-dir) +and [Job Configuration - hydra.job.chdir](/configure_hydra/job.md#hydrajobchdir). diff --git a/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/changes_to_sweeper_config.md b/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/changes_to_sweeper_config.md new file mode 100644 index 00000000000..99128cae295 --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/changes_to_sweeper_config.md @@ -0,0 +1,58 @@ +--- +id: changes_to_sweeper_config +title: Changes to configuring sweeper's search space +hide_title: true +--- + +Hydra 1.2 introduces `hydra.sweeper.params`. All Hydra Sweepers (BasicSweeper and HPOs) search +space will be defined under this config node. + + +### Optuna +For migration, move search space definition from `hydra.sweeper.search_space` to `hydra.sweeper.params`. Change the search space +definition to be consistent with how you'd override a value from commandline. For example: + +
+
+ +```yaml title="Hydra 1.1" +hydra: + sweeper: + search_space: + search_space: + x: + type: float + low: -5.5 + high: 5.5 + step: 0.5 + 'y': + type: categorical + choices: + - -5 + - 0 + - 5 +``` +
+
+ +```bash title="Hydra 1.2" +hydra: + sweeper: + params: + x: range(-5.5, 5.5, step=0.5) + y: choice(-5, 0, 5) + + + + + + + + + + +``` +
+
+ +Check out [Optuna Sweeper](/plugins/optuna_sweeper.md) for more info. \ No newline at end of file diff --git a/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/hydra_main_config_path.md b/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/hydra_main_config_path.md new file mode 100644 index 00000000000..6f61bb8b91c --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/1.1_to_1.2/hydra_main_config_path.md @@ -0,0 +1,8 @@ +--- +id: changes_to_hydra_main_config_path +title: Changes to @hydra.main() and hydra.initialize() +--- + +Prior to Hydra 1.2, **@hydra.main()** and **hydra.initialize()** default `config path` was the directory containing the Python app (calling **@hydra.main()** or **hydra.initialize()**). +Starting with Hydra 1.1 we give [control over the default config path](../1.0_to_1.1/hydra_main_config_path.md), +and starting with Hydra 1.2, with [version_base](../version_base.md) >= "1.2", we choose a default config_path=None, indicating that no directory should be added to the config search path. diff --git a/website/versioned_docs/version-1.3/upgrades/intro.md b/website/versioned_docs/version-1.3/upgrades/intro.md new file mode 100644 index 00000000000..139d4a3b473 --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/intro.md @@ -0,0 +1,37 @@ +--- +id: intro +title: Introduction +sidebar_label: Introduction +--- + +Upgrading to a new Hydra version is usually an easy process. +Also since Hydra version 1.2, backwards compatibility is improved +by giving the user more control over appropriate defaults +through the use of the [version_base parameter](version_base.md). + +:::info NOTE +Hydra versioning has only major versions and patch versions. A bump of the first two version digits is considered a major release. +A major release may have multiple followup patch releases that will fix bugs without introducing new functionality. +::: + + +## Major version upgrades +Hydra will typically provide helpful warnings about required changes, sometimes pointing to an upgrade page that provides more details about the required changes. + +For a smooth upgrade experience, please follow these simple rules: +- **Upgrade to the latest patch version first**. + e.g: If you are upgrading from 1.0 to 1.1, be sure to upgrade to the latest 1.0 version first (1.0.6). +- **Address ALL runtime warnings issued by Hydra.** + A warning in one version is likely to become a far less friendly error in the next major version. +- **Do not skip major versions**. + e.g: If you are upgrading from Hydra 1.0 to Hydra 1.2 - Do it by + - Upgrading from 1.0 to 1.1, addressing all the warnings. + - Upgrading from 1.1 to 1.2, addressing all the warnings. + +## Patch version upgrades +Patch releases normally contains only bug fixes and are thus safe and easy to upgrade (e.g. **1.0.3** to **1.0.6**). +In rare cases, patch releases will introduce new warnings. Be sure to address them before upgrading to the next major version. + +## Dev release upgrades +Development releases are subject to breaking changes without notice. Please be aware that upgrading to a new development release +is more likely to introduce some breakage. No attempt will be made to make upgrading between development releases easy. diff --git a/website/versioned_docs/version-1.3/upgrades/version_base.md b/website/versioned_docs/version-1.3/upgrades/version_base.md new file mode 100644 index 00000000000..246e9889c7a --- /dev/null +++ b/website/versioned_docs/version-1.3/upgrades/version_base.md @@ -0,0 +1,19 @@ +--- +id: version_base +title: version_base +--- + +Hydra since version 1.2 supports backwards compatible upgrades by default +through the use of the `version_base` parameter to **@hydra.main()** and **hydra.initialize()**. + +There are three classes of values that the `version_base` parameter supports, +given new and existing users greater control of the default behaviors to use. + +1. If the `version_base` parameter is **not specified**, Hydra 1.x will use defaults compatible with version 1.1. +Also in this case, a warning is issued to indicate an explicit `version_base` is preferred. + +2. If the `version_base` parameter is **None**, then the defaults are chosen for the current minor Hydra version. +For example for Hydra 1.2, then would imply `config_path=None` and `hydra.job.chdir=False`. + +3. If the `version_base` parameter is an **explicit version string** like "1.1", +then the defaults appropriate to that version are used. diff --git a/website/versioned_sidebars/version-1.3-sidebars.json b/website/versioned_sidebars/version-1.3-sidebars.json new file mode 100644 index 00000000000..a204a8c33c0 --- /dev/null +++ b/website/versioned_sidebars/version-1.3-sidebars.json @@ -0,0 +1,170 @@ +{ + "docs": { + "About": [ + "intro" + ], + "Tutorials": [ + "tutorials/intro", + { + "type": "category", + "label": "Basic Tutorial", + "items": [ + { + "type": "category", + "label": "Your first Hydra app", + "items": [ + "tutorials/basic/your_first_app/simple_cli", + "tutorials/basic/your_first_app/config_file", + "tutorials/basic/your_first_app/using_config", + "tutorials/basic/your_first_app/config_groups", + "tutorials/basic/your_first_app/defaults", + "tutorials/basic/your_first_app/composition" + ] + }, + { + "type": "category", + "label": "Running your Hydra app", + "items": [ + "tutorials/basic/running_your_app/multi-run", + "tutorials/basic/running_your_app/working_directory", + "tutorials/basic/running_your_app/logging", + "tutorials/basic/running_your_app/debugging", + "tutorials/basic/running_your_app/tab_completion" + ] + } + ] + }, + { + "type": "category", + "label": "Structured Configs Tutorial", + "items": [ + "tutorials/structured_config/intro", + "tutorials/structured_config/config_store", + "tutorials/structured_config/minimal_example", + "tutorials/structured_config/hierarchical_static_config", + "tutorials/structured_config/config_groups", + "tutorials/structured_config/defaults", + "tutorials/structured_config/schema" + ] + } + ], + "Common Patterns": [ + "patterns/extending_configs", + "patterns/configuring_experiments", + "patterns/configuring_plugins", + "patterns/select_multiple_configs_from_config_group", + "patterns/specializing_config", + "patterns/write_protect_config_node" + ], + "Configuring Hydra": [ + "configure_hydra/intro", + "configure_hydra/job", + "configure_hydra/logging", + "configure_hydra/workdir", + "configure_hydra/app_help" + ], + "Available Plugins": [ + "plugins/colorlog", + { + "Launchers": [ + "plugins/joblib_launcher", + "plugins/ray_launcher", + "plugins/rq_launcher", + "plugins/submitit_launcher" + ] + }, + { + "Sweepers": [ + "plugins/ax_sweeper", + "plugins/nevergrad_sweeper", + "plugins/optuna_sweeper" + ] + } + ], + "Reference manual": [ + "advanced/terminology", + "advanced/hydra-command-line-flags", + { + "type": "category", + "label": "Override grammar", + "items": [ + "advanced/override_grammar/basic", + "advanced/override_grammar/extended" + ] + }, + "advanced/defaults_list", + "advanced/overriding_packages", + { + "type": "category", + "label": "Instantiating Objects", + "items": [ + "advanced/instantiate_objects/overview", + "advanced/instantiate_objects/config_files", + "advanced/instantiate_objects/structured_config" + ] + }, + "advanced/compose_api", + "advanced/search_path", + { + "type": "category", + "label": "Plugins", + "items": [ + "advanced/plugins/overview", + "advanced/plugins/develop" + ] + }, + "advanced/app_packaging", + "advanced/decorating_main", + "advanced/jupyter_notebooks", + "advanced/unit_testing" + ], + "Experimental": [ + "experimental/intro", + "experimental/callbacks", + "experimental/rerun" + ], + "Developer Guide": [ + "development/overview", + "development/testing", + "development/style_guide", + "development/documentation", + "development/release" + ], + "Upgrade Guide": [ + "upgrades/intro", + "upgrades/version_base", + { + "type": "category", + "label": "1.1 to 1.2", + "items": [ + "upgrades/1.1_to_1.2/changes_to_hydra_main_config_path", + "upgrades/1.1_to_1.2/changes_to_job_working_dir", + "upgrades/1.1_to_1.2/changes_to_sweeper_config" + ] + }, + { + "type": "category", + "label": "1.0 to 1.1", + "items": [ + "upgrades/1.0_to_1.1/changes_to_hydra_main_config_path", + "upgrades/1.0_to_1.1/default_composition_order", + "upgrades/1.0_to_1.1/defaults_list_override", + "upgrades/1.0_to_1.1/defaults_list_interpolation", + "upgrades/1.0_to_1.1/changes_to_package_header", + "upgrades/1.0_to_1.1/automatic_schema_matching" + ] + }, + { + "type": "category", + "label": "0.11 to 1.0", + "items": [ + "upgrades/0.11_to_1.0/config_path_changes", + "upgrades/0.11_to_1.0/adding_a_package_directive", + "upgrades/0.11_to_1.0/strict_mode_flag_deprecated", + "upgrades/0.11_to_1.0/object_instantiation_changes" + ] + } + ], + "FB Only": [] + } +} diff --git a/website/versions.json b/website/versions.json index 33ee681a811..0bc70603352 100644 --- a/website/versions.json +++ b/website/versions.json @@ -1,4 +1,5 @@ [ + "1.3", "1.2", "1.1", "1.0", diff --git a/website/yarn.lock b/website/yarn.lock index ea314949987..6892a60f790 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -2161,13 +2161,13 @@ resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d" integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ== -accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7: - version "1.3.7" - resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.7.tgz#531bc726517a3b2b41f850021c6cc15eaab507cd" - integrity sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA== +accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: + version "1.3.8" + resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" + integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== dependencies: - mime-types "~2.1.24" - negotiator "0.6.2" + mime-types "~2.1.34" + negotiator "0.6.3" acorn-dynamic-import@^4.0.0: version "4.0.0" @@ -2342,7 +2342,7 @@ argparse@^2.0.1: array-flatten@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2" - integrity sha1-ml9pkFGx5wczKPKgCJaLZOopVdI= + integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== array-flatten@^2.1.0: version "2.1.2" @@ -2513,21 +2513,23 @@ bluebird@^3.7.1: resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" integrity sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg== -body-parser@1.19.1: - version "1.19.1" - resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.19.1.tgz#1499abbaa9274af3ecc9f6f10396c995943e31d4" - integrity sha512-8ljfQi5eBk8EJfECMrgqNGWPEY5jWP+1IzkzkGdFFEwFQZZyaZ21UqdaHktgiMlH0xLHqIFtE/u2OYE5dOtViA== +body-parser@1.20.1: + version "1.20.1" + resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.1.tgz#b1812a8912c195cd371a3ee5e66faa2338a5c668" + integrity sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw== dependencies: - bytes "3.1.1" + bytes "3.1.2" content-type "~1.0.4" debug "2.6.9" - depd "~1.1.2" - http-errors "1.8.1" + depd "2.0.0" + destroy "1.2.0" + http-errors "2.0.0" iconv-lite "0.4.24" - on-finished "~2.3.0" - qs "6.9.6" - raw-body "2.4.2" + on-finished "2.4.1" + qs "6.11.0" + raw-body "2.5.1" type-is "~1.6.18" + unpipe "1.0.0" bonjour@^3.5.0: version "3.5.0" @@ -2634,10 +2636,10 @@ bytes@3.0.0: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" integrity sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg= -bytes@3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.1.tgz#3f018291cb4cbad9accb6e6970bca9c8889e879a" - integrity sha512-dWe4nWO/ruEOY7HkUJ5gFt1DCFV9zPRoJr8pV0/ASQermOZjtq8jMjOprC0Kd10GLN+l7xaUPvxzJFWtxGu8Fg== +bytes@3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" + integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== cacheable-request@^6.0.0: version "6.1.0" @@ -2988,12 +2990,12 @@ convert-source-map@^1.7.0: cookie-signature@1.0.6: version "1.0.6" resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c" - integrity sha1-4wOogrNCzD7oylE6eZmXNNqzriw= + integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ== -cookie@0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.1.tgz#afd713fe26ebd21ba95ceb61f9a8116e50a537d1" - integrity sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA== +cookie@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" + integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== copy-text-to-clipboard@^3.0.1: version "3.0.1" @@ -3318,15 +3320,20 @@ del@^6.0.0: rimraf "^3.0.2" slash "^3.0.0" +depd@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== + depd@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.2.tgz#9bcd52e14c097763e749b274c4346ed2e560b5a9" integrity sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak= -destroy@~1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" - integrity sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA= +destroy@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.2.0.tgz#4803735509ad8be552934c67df614f94e66fa015" + integrity sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg== detab@2.0.4, detab@^2.0.0: version "2.0.4" @@ -3519,7 +3526,7 @@ duplexer@^0.1.1, duplexer@^0.1.2: ee-first@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" - integrity sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0= + integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow== electron-to-chromium@^1.4.17: version "1.4.52" @@ -3544,7 +3551,7 @@ emoticon@^3.2.0: encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" - integrity sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k= + integrity sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w== end-of-stream@^1.1.0: version "1.4.4" @@ -3716,7 +3723,7 @@ eta@^1.12.3: etag@~1.8.1: version "1.8.1" resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887" - integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc= + integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg== eval@^0.1.4: version "0.1.6" @@ -3751,37 +3758,38 @@ execa@^5.0.0: strip-final-newline "^2.0.0" express@^4.17.1: - version "4.17.2" - resolved "https://registry.yarnpkg.com/express/-/express-4.17.2.tgz#c18369f265297319beed4e5558753cc8c1364cb3" - integrity sha512-oxlxJxcQlYwqPWKVJJtvQiwHgosH/LrLSPA+H4UxpyvSS6jC5aH+5MoHFM+KABgTOt0APue4w66Ha8jCUo9QGg== + version "4.18.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" + integrity sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ== dependencies: - accepts "~1.3.7" + accepts "~1.3.8" array-flatten "1.1.1" - body-parser "1.19.1" + body-parser "1.20.1" content-disposition "0.5.4" content-type "~1.0.4" - cookie "0.4.1" + cookie "0.5.0" cookie-signature "1.0.6" debug "2.6.9" - depd "~1.1.2" + depd "2.0.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" - finalhandler "~1.1.2" + finalhandler "1.2.0" fresh "0.5.2" + http-errors "2.0.0" merge-descriptors "1.0.1" methods "~1.1.2" - on-finished "~2.3.0" + on-finished "2.4.1" parseurl "~1.3.3" path-to-regexp "0.1.7" proxy-addr "~2.0.7" - qs "6.9.6" + qs "6.11.0" range-parser "~1.2.1" safe-buffer "5.2.1" - send "0.17.2" - serve-static "1.14.2" + send "0.18.0" + serve-static "1.15.0" setprototypeof "1.2.0" - statuses "~1.5.0" + statuses "2.0.1" type-is "~1.6.18" utils-merge "1.0.1" vary "~1.1.2" @@ -3899,17 +3907,17 @@ fill-range@^7.0.1: dependencies: to-regex-range "^5.0.1" -finalhandler@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.1.2.tgz#b7e7d000ffd11938d0fdb053506f6ebabe9f587d" - integrity sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA== +finalhandler@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-1.2.0.tgz#7d23fe5731b207b4640e4fcd00aec1f9207a7b32" + integrity sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg== dependencies: debug "2.6.9" encodeurl "~1.0.2" escape-html "~1.0.3" - on-finished "~2.3.0" + on-finished "2.4.1" parseurl "~1.3.3" - statuses "~1.5.0" + statuses "2.0.1" unpipe "~1.0.0" find-cache-dir@^3.3.1: @@ -3994,7 +4002,7 @@ fraction.js@^4.1.2: fresh@0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/fresh/-/fresh-0.5.2.tgz#3d8cadd90d976569fa835ab1f8e4b23a105605a7" - integrity sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac= + integrity sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q== fs-extra@^10.0.0: version "10.0.0" @@ -4040,7 +4048,16 @@ gensync@^1.0.0-beta.1, gensync@^1.0.0-beta.2: resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== -get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1: +get-intrinsic@^1.0.2: + version "1.1.3" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.3.tgz#063c84329ad93e83893c7f4f243ef63ffa351385" + integrity sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A== + dependencies: + function-bind "^1.1.1" + has "^1.0.3" + has-symbols "^1.0.3" + +get-intrinsic@^1.1.0, get-intrinsic@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q== @@ -4229,6 +4246,11 @@ has-symbols@^1.0.1, has-symbols@^1.0.2: resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.2.tgz#165d3070c00309752a1236a479331e3ac56f1423" integrity sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw== +has-symbols@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.3.tgz#bb7b2c4349251dce87b125f7bdf874aa7c8b39f8" + integrity sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A== + has-tostringtag@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/has-tostringtag/-/has-tostringtag-1.0.0.tgz#7e133818a7d394734f941e73c3d3f9291e658b25" @@ -4465,24 +4487,24 @@ htmlparser2@^6.1.0: entities "^2.0.0" http-cache-semantics@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz#49e91c5cbf36c9b94bcfcd71c23d5249ec74e390" - integrity sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ== + version "4.1.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" + integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== http-deceiver@^1.2.7: version "1.2.7" resolved "https://registry.yarnpkg.com/http-deceiver/-/http-deceiver-1.2.7.tgz#fa7168944ab9a519d337cb0bec7284dc3e723d87" integrity sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc= -http-errors@1.8.1: - version "1.8.1" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.1.tgz#7c3f28577cbc8a207388455dbd62295ed07bd68c" - integrity sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g== +http-errors@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-2.0.0.tgz#b7774a1486ef73cf7667ac9ae0858c012c57b9d3" + integrity sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ== dependencies: - depd "~1.1.2" + depd "2.0.0" inherits "2.0.4" setprototypeof "1.2.0" - statuses ">= 1.5.0 < 2" + statuses "2.0.1" toidentifier "1.0.1" http-errors@~1.6.2: @@ -5042,9 +5064,9 @@ json-schema-traverse@^1.0.0: integrity sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug== json5@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" - integrity sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow== + version "1.0.2" + resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" + integrity sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA== dependencies: minimist "^1.2.0" @@ -5415,7 +5437,7 @@ mdurl@^1.0.0: media-typer@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" - integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= + integrity sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ== memfs@^3.1.2, memfs@^3.2.2: version "3.4.1" @@ -5427,7 +5449,7 @@ memfs@^3.1.2, memfs@^3.2.2: merge-descriptors@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" - integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= + integrity sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w== merge-stream@^2.0.0: version "2.0.0" @@ -5442,7 +5464,7 @@ merge2@^1.3.0, merge2@^1.4.1: methods@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee" - integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4= + integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== micromark-extension-mdx-expression@^0.3.0, micromark-extension-mdx-expression@^0.3.2, micromark-extension-mdx-expression@~0.3.0: version "0.3.2" @@ -5520,6 +5542,11 @@ mime-db@1.51.0, "mime-db@>= 1.43.0 < 2": resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.51.0.tgz#d9ff62451859b18342d960850dc3cfb77e63fb0c" integrity sha512-5y8A56jg7XVQx2mbv1lu49NR4dokRnhZYTtL+KGfaa27uq4pSTXkwQkFJl4pkRMyNFz/EtYDSkiiEHx3F7UN6g== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + mime-db@~1.33.0: version "1.33.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.33.0.tgz#a3492050a5cb9b63450541e39d9788d2272783db" @@ -5532,13 +5559,20 @@ mime-types@2.1.18: dependencies: mime-db "~1.33.0" -mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.17, mime-types@~2.1.24: +mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.17: version "2.1.34" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.34.tgz#5a712f9ec1503511a945803640fafe09d3793c24" integrity sha512-6cP692WwGIs9XXdOO4++N+7qjqv0rqxxVvJ3VHPh/Sc9mVZcQP+ZGhkKiTvWMQRr2tbHkJP/Yn7Y0npb3ZBs4A== dependencies: mime-db "1.51.0" +mime-types@~2.1.24, mime-types@~2.1.34: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mime@1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" @@ -5588,16 +5622,11 @@ minimatch@3.0.4, minimatch@^3.0.4: dependencies: brace-expansion "^1.1.7" -minimist@^1.2.0: +minimist@^1.2.0, minimist@^1.2.5: version "1.2.7" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== -minimist@^1.2.5: - version "1.2.6" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44" - integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q== - mkdirp@^0.5.5: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" @@ -5613,7 +5642,7 @@ mrmime@^1.0.0: ms@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + integrity sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A== ms@2.1.2: version "2.1.2" @@ -5643,10 +5672,10 @@ nanoid@^3.1.30: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.2.0.tgz#62667522da6673971cca916a6d3eff3f415ff80c" integrity sha512-fmsZYa9lpn69Ad5eDn7FMcnnSR+8R34W9qJEijxYhTbfOWzr22n1QxCMzXLK+ODyW2973V3Fux959iQoUxzUIA== -negotiator@0.6.2: - version "0.6.2" - resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.2.tgz#feacf7ccf525a77ae9634436a64883ffeca346fb" - integrity sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw== +negotiator@0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" + integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== neo-async@^2.6.2: version "2.6.2" @@ -5736,11 +5765,16 @@ object-assign@^4.1.0, object-assign@^4.1.1: resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= -object-inspect@^1.11.0, object-inspect@^1.9.0: +object-inspect@^1.11.0: version "1.12.0" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.0.tgz#6e2c120e868fd1fd18cb4f18c31741d0d6e776f0" integrity sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g== +object-inspect@^1.9.0: + version "1.12.2" + resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea" + integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ== + object-is@^1.0.1: version "1.1.5" resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac" @@ -5769,10 +5803,10 @@ obuf@^1.0.0, obuf@^1.1.2: resolved "https://registry.yarnpkg.com/obuf/-/obuf-1.1.2.tgz#09bea3343d41859ebd446292d11c9d4db619084e" integrity sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg== -on-finished@~2.3.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.3.0.tgz#20f1336481b083cd75337992a16971aa2d906947" - integrity sha1-IPEzZIGwg811M3mSoWlxqi2QaUc= +on-finished@2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" + integrity sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg== dependencies: ee-first "1.1.1" @@ -6007,7 +6041,7 @@ path-root@^0.1.1: path-to-regexp@0.1.7: version "0.1.7" resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c" - integrity sha1-32BBeABfUi8V60SQ5yR6G/qmf4w= + integrity sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ== path-to-regexp@2.2.1: version "2.2.1" @@ -6473,10 +6507,12 @@ pure-color@^1.2.0: resolved "https://registry.yarnpkg.com/pure-color/-/pure-color-1.3.0.tgz#1fe064fb0ac851f0de61320a8bf796836422f33e" integrity sha1-H+Bk+wrIUfDeYTIKi/eWg2Qi8z4= -qs@6.9.6: - version "6.9.6" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.6.tgz#26ed3c8243a431b2924aca84cc90471f35d5a0ee" - integrity sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ== +qs@6.11.0: + version "6.11.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" + integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== + dependencies: + side-channel "^1.0.4" querystring@0.2.0: version "0.2.0" @@ -6505,13 +6541,13 @@ range-parser@^1.2.1, range-parser@~1.2.1: resolved "https://registry.yarnpkg.com/range-parser/-/range-parser-1.2.1.tgz#3cf37023d199e1c24d1a55b84800c2f3e6468031" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -raw-body@2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.4.2.tgz#baf3e9c21eebced59dd6533ac872b71f7b61cb32" - integrity sha512-RPMAFUJP19WIet/99ngh6Iv8fzAbqum4Li7AD6DtGaW2RpMB/11xDoalPiJMTbu6I3hkbMVkATvZrqb9EEqeeQ== +raw-body@2.5.1: + version "2.5.1" + resolved "https://registry.yarnpkg.com/raw-body/-/raw-body-2.5.1.tgz#fe1b1628b181b700215e5fd42389f98b71392857" + integrity sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig== dependencies: - bytes "3.1.1" - http-errors "1.8.1" + bytes "3.1.2" + http-errors "2.0.0" iconv-lite "0.4.24" unpipe "1.0.0" @@ -7178,24 +7214,24 @@ semver@^7.3.2, semver@^7.3.4, semver@^7.3.5: dependencies: lru-cache "^6.0.0" -send@0.17.2: - version "0.17.2" - resolved "https://registry.yarnpkg.com/send/-/send-0.17.2.tgz#926622f76601c41808012c8bf1688fe3906f7820" - integrity sha512-UJYB6wFSJE3G00nEivR5rgWp8c2xXvJ3OPWPhmuteU0IKj8nKbG3DrjiOmLwpnHGYWAVwA69zmTm++YG0Hmwww== +send@0.18.0: + version "0.18.0" + resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" + integrity sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg== dependencies: debug "2.6.9" - depd "~1.1.2" - destroy "~1.0.4" + depd "2.0.0" + destroy "1.2.0" encodeurl "~1.0.2" escape-html "~1.0.3" etag "~1.8.1" fresh "0.5.2" - http-errors "1.8.1" + http-errors "2.0.0" mime "1.6.0" ms "2.1.3" - on-finished "~2.3.0" + on-finished "2.4.1" range-parser "~1.2.1" - statuses "~1.5.0" + statuses "2.0.1" serialize-javascript@^6.0.0: version "6.0.0" @@ -7231,15 +7267,15 @@ serve-index@^1.9.1: mime-types "~2.1.17" parseurl "~1.3.2" -serve-static@1.14.2: - version "1.14.2" - resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.14.2.tgz#722d6294b1d62626d41b43a013ece4598d292bfa" - integrity sha512-+TMNA9AFxUEGuC0z2mevogSnn9MXKb4fa7ngeRMJaaGv8vTwnIEkKi+QGvPt33HSnf8pRS+WGM0EbMtCJLKMBQ== +serve-static@1.15.0: + version "1.15.0" + resolved "https://registry.yarnpkg.com/serve-static/-/serve-static-1.15.0.tgz#faaef08cffe0a1a62f60cad0c4e513cff0ac9540" + integrity sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g== dependencies: encodeurl "~1.0.2" escape-html "~1.0.3" parseurl "~1.3.3" - send "0.17.2" + send "0.18.0" setimmediate@^1.0.5: version "1.0.5" @@ -7422,7 +7458,12 @@ state-toggle@^1.0.0: resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.3.tgz#e123b16a88e143139b09c6852221bc9815917dfe" integrity sha512-d/5Z4/2iiCnHw6Xzghyhb+GcmF89bxwgXG60wjIiZaxnymbyOmI8Hk4VqHXiVVp6u2ysaskFfXg3ekCj4WNftQ== -"statuses@>= 1.4.0 < 2", "statuses@>= 1.5.0 < 2", statuses@~1.5.0: +statuses@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" + integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== + +"statuses@>= 1.4.0 < 2": version "1.5.0" resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= @@ -7723,9 +7764,9 @@ typedarray-to-buffer@^3.1.5: is-typedarray "^1.0.0" ua-parser-js@^0.7.30: - version "0.7.31" - resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.31.tgz#649a656b191dffab4f21d5e053e27ca17cbff5c6" - integrity sha512-qLK/Xe9E2uzmYI3qLeOmI0tEOt+TBBQyUIAh4aAgU05FVYzeZrKUdkAZfBNVGRaHVgV0TDkdEngJSw/SyQchkQ== + version "0.7.33" + resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.33.tgz#1d04acb4ccef9293df6f70f2c3d22f3030d8b532" + integrity sha512-s8ax/CeZdK9R/56Sui0WM6y9OFREJarMRHqLB2EwkovemBxNQ+Bqu8GAsUnVcXKgphb++ghr/B2BZx4mahujPw== unbox-primitive@^1.0.1: version "1.0.1" @@ -7919,7 +7960,7 @@ universalify@^2.0.0: unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" - integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw= + integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== update-notifier@^5.1.0: version "5.1.0" @@ -8019,7 +8060,7 @@ utility-types@^3.10.0: utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" - integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= + integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== uuid@^8.3.2: version "8.3.2" @@ -8042,7 +8083,7 @@ value-equal@^1.0.1: vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" - integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw= + integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== vfile-location@^3.0.0, vfile-location@^3.2.0: version "3.2.0"