diff --git a/.github/workflows/root-ci-config/build_root.py b/.github/workflows/root-ci-config/build_root.py index ed9303effd695..3987ba92e2a8a 100755 --- a/.github/workflows/root-ci-config/build_root.py +++ b/.github/workflows/root-ci-config/build_root.py @@ -24,13 +24,11 @@ import time import build_utils -import openstack from build_utils import ( calc_options_hash, die, github_log_group, is_macos, - load_config, subprocess_with_capture, subprocess_with_log, upload_file, @@ -39,12 +37,6 @@ S3CONTAINER = 'ROOT-build-artifacts' # Used for uploads S3URL = 'https://s3.cern.ch/swift/v1/' + S3CONTAINER # Used for downloads -try: - CONNECTION = openstack.connect(cloud='envvars') -except Exception as exc: - print("Failed to open the S3 connection:", exc, file=sys.stderr) - CONNECTION = None - WINDOWS = (os.name == 'nt') WORKDIR = (os.environ['HOME'] + '/ROOT-CI') if not WINDOWS else 'C:/ROOT-CI' COMPRESSIONLEVEL = 6 if not WINDOWS else 1 @@ -67,16 +59,29 @@ def main(): cleanup_previous_build() - # Load CMake options from .github/workflows/root-ci-config/buildconfig/[platform].txt this_script_dir = os.path.dirname(os.path.abspath(__file__)) - options_dict = { - **load_config(f'{this_script_dir}/buildconfig/global.txt'), - # file below overwrites values from above - **load_config(f'{this_script_dir}/buildconfig/{args.platform}.txt') - } + # Compute CMake build options: + # - Get global options + # - Override with options from .github/workflows/root-ci-config/buildconfig/[platform].txt + # - Apply overrides from command line if necessary + options_dict = build_utils.load_config(f"{this_script_dir}/buildconfig/global.txt") + last_options = dict(options_dict) + + options_dict.update(build_utils.load_config(f"{this_script_dir}/buildconfig/{args.platform}.txt")) + print(f"Build option overrides for {args.platform}:") + build_utils.print_options_diff(options_dict, last_options) + + if args.overrides is not None: + print("Build option overrides from command line:") + last_options = dict(options_dict) + options_dict.update((arg.split("=", maxsplit=1) for arg in args.overrides)) + build_utils.print_options_diff(options_dict, last_options) options = build_utils.cmake_options_from_dict(options_dict) + print("Full build options") + for key, val in sorted(options_dict.items()): + print(f"\t{key: <30}{val}") if WINDOWS: options = "-Thost=x64 " + options @@ -193,6 +198,7 @@ def parse_args(): parser.add_argument("--architecture", default=None, help="Windows only, target arch") parser.add_argument("--repository", default="https://github.com/root-project/root.git", help="url to repository") + parser.add_argument("--overrides", default=None, help="Override build options using a syntax like 'A=1 B=2'", nargs="*") args = parser.parse_args() @@ -333,6 +339,14 @@ def archive_and_upload(archive_name, prefix): targz.add("src") targz.add("build") + try: + import openstack + CONNECTION = openstack.connect(cloud='envvars') + except Exception as exc: + print("Failed to open the S3 connection:", exc, file=sys.stderr) + CONNECTION = None + + upload_file( connection=CONNECTION, container=S3CONTAINER, diff --git a/.github/workflows/root-ci-config/build_utils.py b/.github/workflows/root-ci-config/build_utils.py index 68b9bb4968399..9a6f3606083ac 100755 --- a/.github/workflows/root-ci-config/build_utils.py +++ b/.github/workflows/root-ci-config/build_utils.py @@ -14,9 +14,6 @@ from shutil import which from typing import Callable, Dict -from openstack.connection import Connection -from requests import get - def is_macos(): return 'Darwin' == platform.system() @@ -108,6 +105,16 @@ def print_warning(*values, **kwargs): def print_error(*values, **kwargs): print_fancy("Fatal error: ", *values, sgr=31, **kwargs) +def print_options_diff(new, old): + """Print difference between build option dicts""" + + for key in sorted(new): + try: + if new[key] != old[key]: + print(f"\t{key: <30}{old[key]} --> {new[key]}") + except KeyError: + print(f"\t{key: <30}None --> {new[key]}") + def subprocess_with_log(command: str) -> int: """Runs in shell and appends to log""" @@ -228,7 +235,7 @@ def calc_options_hash(options: str) -> str: options_and_defines += sp_result.stdout return sha1(options_and_defines.encode('utf-8')).hexdigest() -def upload_file(connection: Connection, container: str, dest_object: str, src_file: str) -> None: +def upload_file(connection, container: str, dest_object: str, src_file: str) -> None: print(f"Attempting to upload {src_file} to {dest_object}") if not os.path.exists(src_file): @@ -272,6 +279,8 @@ def create_object_local(): def download_file(url: str, dest: str) -> None: + from requests import get + print(f"\nAttempting to download {url} to {dest}") parent_dir = os.path.dirname(dest) @@ -287,6 +296,7 @@ def download_latest(url: str, prefix: str, destination: str) -> str: """Downloads latest build artifact starting with , and returns the file path to the downloaded file and shell_log.""" + from requests import get # https://docs.openstack.org/api-ref/object-store/#show-container-details-and-list-objects with get(f"{url}/?prefix={prefix}&format=json", timeout=20) as req: if req.status_code == HTTPStatus.NO_CONTENT or req.content == b'[]': @@ -316,7 +326,7 @@ def remove_file_match_ext(directory: str, extension: str) -> str: extension (str): The regular expression pattern to match filenames against. """ print_fancy(f"Removing gcda files from {directory}") - log.add(f"\nfind {directory} -name \*.gcda -exec rm {{}} \;") + log.add(f"\nfind {directory} -name \\*.gcda -exec rm {{}} \\;") pattern = "." + extension count = 0 for currentdir, _, files in os.walk(directory): diff --git a/.github/workflows/root-ci.yml b/.github/workflows/root-ci.yml index b50bc971b85eb..a0e27e702d20f 100644 --- a/.github/workflows/root-ci.yml +++ b/.github/workflows/root-ci.yml @@ -128,31 +128,6 @@ jobs: with: ref: ${{ inputs.ref_name }} - - name: Apply option overrides from matrix for this job for non-release builds - if: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && matrix.overrides != NaN }} - env: - OVERRIDES: ${{ join( matrix.overrides, ' ') }} - CONFIGFILE: '.github/workflows/root-ci-config/buildconfig/${{ matrix.platform }}.txt' - shell: bash - run: | - set -x - - echo '' >> "$CONFIGFILE" - - for ENTRY in $GLOBAL_OVERRIDES $OVERRIDES; do - KEY=$( echo "$ENTRY" | cut -d '=' -f 1 ) - - # Add entry to file if not exists, otherwise replace - - if grep -q "$KEY=" "$CONFIGFILE"; then - sed -i "s/$KEY=.*\$/$ENTRY/" "$CONFIGFILE" - else - echo "$ENTRY" >> "$CONFIGFILE" - fi - done - - cat "$CONFIGFILE" || true - - uses: root-project/gcc-problem-matcher-improved@main with: build-directory: /Users/sftnight/ROOT-CI/src/ @@ -167,6 +142,7 @@ jobs: HOME: /Users/sftnight INCREMENTAL: ${{ !contains(github.event.pull_request.labels.*.name, 'clean build') && !matrix.platform == 'mac15' && !matrix.platform == 'mac26'}} GITHUB_PR_ORIGIN: ${{ github.event.pull_request.head.repo.clone_url }} + OVERRIDES: ${{ join( matrix.overrides, ' ') }} run: ".github/workflows/root-ci-config/build_root.py --buildtype RelWithDebInfo --incremental $INCREMENTAL @@ -176,7 +152,8 @@ jobs: --head_ref refs/pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }} --head_sha ${{ github.event.pull_request.head.sha }} --repository ${{ github.server_url }}/${{ github.repository }} - --platform ${{ matrix.platform }}" + --platform ${{ matrix.platform }} + --overrides ${GLOBAL_OVERRIDES} ${OVERRIDES}" - name: Workflow dispatch shell: bash -leo pipefail {0} @@ -204,13 +181,17 @@ jobs: - name: Update build cache after push to release branch shell: bash -leo pipefail {0} if: github.event_name == 'push' + env: + OVERRIDES: ${{ join( matrix.overrides, ' ') }} run: ".github/workflows/root-ci-config/build_root.py --buildtype RelWithDebInfo --platform ${{ matrix.platform }} --incremental false --base_ref ${{ github.ref_name }} --binaries ${{ startsWith(github.ref, 'refs/tags/') }} - --repository ${{ github.server_url }}/${{ github.repository }}" + --repository ${{ github.server_url }}/${{ github.repository }} + --overrides ${GLOBAL_OVERRIDES} ${OVERRIDES} + " - name: Upload test results if: ${{ !cancelled() }} @@ -491,31 +472,6 @@ jobs: ls -la ' - - name: Apply option overrides from matrix for this job for all builds except non-special nightlies and releases - if: ${{ ( github.event_name != 'schedule' || matrix.is_special) && github.event_name != 'workflow_dispatch' && matrix.overrides != NaN }} - env: - OVERRIDES: ${{ join( matrix.overrides, ' ') }} - CONFIGFILE: '.github/workflows/root-ci-config/buildconfig/${{ matrix.image }}.txt' - shell: bash - run: | - set -x - - echo '' >> "$CONFIGFILE" - - for ENTRY in $GLOBAL_OVERRIDES $OVERRIDES; do - KEY=$( echo "$ENTRY" | cut -d '=' -f 1 ) - - # Add entry to file if not exists, otherwise replace - - if grep -q "$KEY=" "$CONFIGFILE"; then - sed -i "s/$KEY=.*\$/$ENTRY/" "$CONFIGFILE" - else - echo "$ENTRY" >> "$CONFIGFILE" - fi - done - - cat "$CONFIGFILE" || true - - uses: root-project/gcc-problem-matcher-improved@main with: build-directory: /github/home/ROOT-CI/src/ @@ -526,6 +482,7 @@ jobs: INCREMENTAL: ${{ !contains(github.event.pull_request.labels.*.name, 'clean build') }} GITHUB_PR_ORIGIN: ${{ github.event.pull_request.head.repo.clone_url }} CMAKE_GENERATOR: ${{ matrix.cmake_generator }} + OVERRIDES: ${{ join( matrix.overrides, ' ') }} run: ".github/workflows/root-ci-config/build_root.py --buildtype RelWithDebInfo --platform ${{ matrix.image }} @@ -537,6 +494,7 @@ jobs: --head_ref refs/pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }} --head_sha ${{ github.event.pull_request.head.sha }} --repository ${{ github.server_url }}/${{ github.repository }} + --overrides ${GLOBAL_OVERRIDES} ${OVERRIDES} " - name: Workflow dispatch @@ -564,6 +522,8 @@ jobs: - name: Update build cache after push to release branch if: github.event_name == 'push' + env: + OVERRIDES: ${{ join( matrix.overrides, ' ') }} run: ".github/workflows/root-ci-config/build_root.py --buildtype RelWithDebInfo --platform ${{ matrix.image }} @@ -571,6 +531,7 @@ jobs: --base_ref ${{ github.ref_name }} --binaries ${{ startsWith(github.ref, 'refs/tags/') }} --repository ${{ github.server_url }}/${{ github.repository }} + --overrides ${GLOBAL_OVERRIDES} ${OVERRIDES} " - name: Upload test results