diff --git a/.github/workflows/ci_test-acts.yml b/.github/workflows/ci_test-acts.yml index 423d1e4a..d3798cd3 100644 --- a/.github/workflows/ci_test-acts.yml +++ b/.github/workflows/ci_test-acts.yml @@ -25,15 +25,12 @@ jobs: with: python-version: ${{ matrix.python }} - # Note: This uses an internal pip API and may not always work - # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - - name: Get pip cache + - name: Get pip cache dir id: pip-cache - run: | - python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" + run: echo "::set-output name=dir::$(pip cache dir)" - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-py${{ matrix.python }}-pip-${{ hashFiles('requirements.txt') }} diff --git a/.github/workflows/ci_testing.yml b/.github/workflows/ci_testing.yml index 504cac11..4f95d81f 100644 --- a/.github/workflows/ci_testing.yml +++ b/.github/workflows/ci_testing.yml @@ -35,7 +35,9 @@ jobs: run: | pip install -q -r requirements.txt python actions/assistant.py list_runtimes $PR_NUMBER - echo "::set-output name=runtimes::{include: $(python actions/assistant.py list_runtimes $PR_NUMBER 2>&1)}" + pr_runtimes=$(python actions/assistant.py list_runtimes $PR_NUMBER 2>&1) + echo $pr_runtimes + echo "runtimes=$pr_runtimes" >> $GITHUB_OUTPUT pytest-project: @@ -55,18 +57,15 @@ jobs: python-version: ${{ matrix.python }} - name: Cache period - run: echo "::set-output name=period::$(python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print(int(days / 7))' 2>&1)" + run: python -c 'import time ; days = time.time() / 60 / 60 / 24 ; print("period=" + str(int(days / 7)))' >> $GITHUB_OUTPUT id: times - # Note: This uses an internal pip API and may not always work - # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow - - name: Get pip cache + - name: Get pip cache dir id: pip-cache - run: | - python -c "from pip._internal.locations import USER_CACHE_DIR; print('::set-output name=dir::' + USER_CACHE_DIR)" + run: echo "::set-output name=dir::$(pip cache dir)" - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: $COMMON-td${{ steps.times.outputs.period }}-pip-$HASH_FILES @@ -84,9 +83,9 @@ jobs: - name: Some outputs run: | - echo "::set-output name=args::$(python actions/assistant.py specify_tests --config_file=${{ matrix.config }} 2>&1)" - echo "::set-output name=env::$(python actions/assistant.py dict_env --config_file=${{ matrix.config }} 2>&1)" - echo "::set-output name=contacts::$(python actions/assistant.py contacts --config_file=${{ matrix.config }} 2>&1)" + echo "args=$(python actions/assistant.py specify_tests --config_file=${{ matrix.config }})" >> $GITHUB_OUTPUT + echo "env=$(python actions/assistant.py dict_env --config_file=${{ matrix.config }})" >> $GITHUB_OUTPUT + echo "contacts=$(python actions/assistant.py contacts --config_file=${{ matrix.config }})" >> $GITHUB_OUTPUT id: extras - name: Prepare environment diff --git a/actions/_config.yaml b/actions/_config.yaml index e2a98097..25b71e4f 100644 --- a/actions/_config.yaml +++ b/actions/_config.yaml @@ -11,7 +11,9 @@ target_repository: # OPTIONAL, define installing package extras install_extras: all # OPTIONAL, install additional requirements from a file - requirements_file: requirements.txt + # requirements_file: requirements.txt + # passing additional `pip install` flags + install_flags: "--use-pep517" # copy some test from the target repository copy_tests: - integrations @@ -30,17 +32,16 @@ before_install: # OPTIONAL, if any installation require some env. variables env: - HOROVOD_BUILD_ARCH_FLAGS: "-mfma" - # HOROVOD_WITHOUT_MXNET: 1 - # HOROVOD_WITHOUT_TENSORFLOW: 1 + PACKAGE_NAME: "pytorch" dependencies: - name: pytorch-lightning HTTPS: https://github.com/Lightning-AI/lightning.git - checkout: release/1.5.x + checkout: release/stable # install_extras: all - name: Cython checkout: 0.29.24 + install_flags: "--use-pep517" # OPTIONAL, running before installing your project # WARNING: this is custom for given OS, we do not manage any inter OS compatibility diff --git a/actions/assistant.py b/actions/assistant.py index 3afe14c4..2242d4be 100644 --- a/actions/assistant.py +++ b/actions/assistant.py @@ -2,7 +2,7 @@ import json import os import traceback -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union import fire import requests @@ -127,10 +127,21 @@ def _extras(extras: Union[str, list, tuple] = "") -> str: extras = ",".join(extras) if isinstance(extras, (tuple, list, set)) else extras return extras + @staticmethod + def _get_flags(repo: dict, defaults: Tuple[str] = ("--quiet",)) -> List[str]: + """Extract the install's flags with some defaults.""" + flags = repo.get("install_flags", []) + flags = [flags] if isinstance(flags, str) else flags + return list(set(flags + list(defaults))) + @staticmethod def _install_pip(repo: Dict[str, str]) -> str: """Create command for installing a project from source (if HTTPS is given) or from PyPI (if at least name is - given).""" + given). + + Args: + repo: it is package or repository with additional key fields + """ assert any(k in repo for k in ["HTTPS", "name"]), f"Missing key `HTTPS` or `name` among {repo.keys()}" # pip install -q 'https://github.com/...#egg=lightning-flash[tabular] name = repo.get("name") @@ -146,20 +157,22 @@ def _install_pip(repo: Dict[str, str]) -> str: password=repo.get("password"), ) - cmd = f"git+{url}" + pkg = f"git+{url}" if "checkout" in repo: assert isinstance(repo["checkout"], str) - cmd += f"@{repo['checkout']}" + pkg += f"@{repo['checkout']}" if "install_extras" in repo: - cmd += f"#egg={name}[{AssistantCLI._extras(repo['install_extras'])}]" + pkg += f"#egg={name}[{AssistantCLI._extras(repo['install_extras'])}]" else: # make installation from pypi package - cmd = name + pkg = name if "install_extras" in repo: - cmd += f"[{repo['install_extras']}]" + pkg += f"[{repo['install_extras']}]" if "checkout" in repo: - cmd += f"=={repo['checkout']}" - return "pip install --quiet " + cmd + pkg += f"=={repo['checkout']}" + flags = AssistantCLI._get_flags(repo) + cmd = " ".join(["pip install", pkg, " ".join(flags)]) + return cmd @staticmethod def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]: @@ -178,13 +191,18 @@ def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]: if "requirements_file" in repo: reqs = repo["requirements_file"] reqs = [reqs] if isinstance(reqs, str) else reqs - cmds.append(f"pip install --quiet --upgrade {' '.join([f'-r {req}' for req in reqs])}") + args = [f"-r {req}" for req in reqs] + ["--quiet", "--upgrade"] + cmds.append("pip install " + " ".join(args)) pip_install = "." if "install_extras" in repo: pip_install += f"[{AssistantCLI._extras(repo['install_extras'])}]" - cmds.append(f"pip install --quiet {pip_install}") + + flags = AssistantCLI._get_flags(repo) + cmds.append("pip install " + " ".join([pip_install] + flags)) + cmds.append("pip list") cmds.append("cd ..") + if remove_dir: cmds.append(f"rm -rf {repo_name}") return cmds @@ -265,6 +283,7 @@ def prepare_env(config_file: str = "config.yaml", path_root: str = _PATH_ROOT) - reqs = config.get("dependencies", []) for req in reqs: script.append(AssistantCLI._install_pip(req)) + script.append("pip list") script += AssistantCLI.before_commands(config_file, stage="test", as_append=True) return os.linesep.join(script)