[CI] Simplify passing build options to build_root.py

.github/workflows/root-ci-config/build_root.py

@@ -24,13 +24,11 @@
 import time
 
 import build_utils
-import openstack
 from build_utils import (
     calc_options_hash,
     die,
     github_log_group,
     is_macos,
-    load_config,
     subprocess_with_capture,
     subprocess_with_log,
     upload_file,
@@ -39,12 +37,6 @@
 S3CONTAINER = 'ROOT-build-artifacts'  # Used for uploads
 S3URL = 'https://s3.cern.ch/swift/v1/' + S3CONTAINER  # Used for downloads
 
-try:
-    CONNECTION = openstack.connect(cloud='envvars')
-except Exception as exc:
-    print("Failed to open the S3 connection:", exc, file=sys.stderr)
-    CONNECTION = None
-
 WINDOWS = (os.name == 'nt')
 WORKDIR = (os.environ['HOME'] + '/ROOT-CI') if not WINDOWS else 'C:/ROOT-CI'
 COMPRESSIONLEVEL = 6 if not WINDOWS else 1
@@ -67,16 +59,29 @@ def main():
 
     cleanup_previous_build()
 
-    # Load CMake options from .github/workflows/root-ci-config/buildconfig/[platform].txt
     this_script_dir = os.path.dirname(os.path.abspath(__file__))
 
-    options_dict = {
-        **load_config(f'{this_script_dir}/buildconfig/global.txt'),
-        # file below overwrites values from above
-        **load_config(f'{this_script_dir}/buildconfig/{args.platform}.txt')
-    }
+    # Compute CMake build options:
+    # - Get global options
+    # - Override with options from .github/workflows/root-ci-config/buildconfig/[platform].txt
+    # - Apply overrides from command line if necessary
+    options_dict = build_utils.load_config(f"{this_script_dir}/buildconfig/global.txt")
+    last_options = dict(options_dict)
+
+    options_dict.update(build_utils.load_config(f"{this_script_dir}/buildconfig/{args.platform}.txt"))
+    print(f"Build option overrides for {args.platform}:")
+    build_utils.print_options_diff(options_dict, last_options)
+
+    if args.overrides is not None:
+        print("Build option overrides from command line:")
+        last_options = dict(options_dict)
+        options_dict.update((arg.split("=", maxsplit=1) for arg in args.overrides))
+        build_utils.print_options_diff(options_dict, last_options)
 
     options = build_utils.cmake_options_from_dict(options_dict)
+    print("Full build options")
+    for key, val in sorted(options_dict.items()):
+        print(f"\t{key: <30}{val}")
 
     if WINDOWS:
         options = "-Thost=x64 " + options
@@ -193,6 +198,7 @@ def parse_args():
     parser.add_argument("--architecture",    default=None,      help="Windows only, target arch")
     parser.add_argument("--repository",      default="https://github.com/root-project/root.git",
                         help="url to repository")
+    parser.add_argument("--overrides",       default=None,      help="Override build options using a syntax like 'A=1 B=2'", nargs="*")
 
     args = parser.parse_args()
 
@@ -333,6 +339,14 @@ def archive_and_upload(archive_name, prefix):
         targz.add("src")
         targz.add("build")
 
+    try:
+        import openstack
+        CONNECTION = openstack.connect(cloud='envvars')
+    except Exception as exc:
+        print("Failed to open the S3 connection:", exc, file=sys.stderr)
+        CONNECTION = None
+
+
     upload_file(
         connection=CONNECTION,
         container=S3CONTAINER,

.github/workflows/root-ci-config/build_utils.py

@@ -14,9 +14,6 @@
 from shutil import which
 from typing import Callable, Dict
 
-from openstack.connection import Connection
-from requests import get
-
 
 def is_macos():
     return 'Darwin' == platform.system()
@@ -108,6 +105,16 @@ def print_warning(*values, **kwargs):
 def print_error(*values, **kwargs):
     print_fancy("Fatal error: ", *values, sgr=31, **kwargs)
 
+def print_options_diff(new, old):
+    """Print difference between build option dicts"""
+
+    for key in sorted(new):
+        try:
+            if new[key] != old[key]:
+                print(f"\t{key: <30}{old[key]} --> {new[key]}")
+        except KeyError:
+            print(f"\t{key: <30}None --> {new[key]}")
+
 
 def subprocess_with_log(command: str) -> int:
     """Runs <command> in shell and appends <command> to log"""
@@ -228,7 +235,7 @@ def calc_options_hash(options: str) -> str:
         options_and_defines += sp_result.stdout
     return sha1(options_and_defines.encode('utf-8')).hexdigest()
 
-def upload_file(connection: Connection, container: str, dest_object: str, src_file: str) -> None:
+def upload_file(connection, container: str, dest_object: str, src_file: str) -> None:
     print(f"Attempting to upload {src_file} to {dest_object}")
 
     if not os.path.exists(src_file):
@@ -272,6 +279,8 @@ def create_object_local():
 
 
 def download_file(url: str, dest: str) -> None:
+    from requests import get
+
     print(f"\nAttempting to download {url} to {dest}")
 
     parent_dir = os.path.dirname(dest)
@@ -287,6 +296,7 @@ def download_latest(url: str, prefix: str, destination: str) -> str:
     """Downloads latest build artifact starting with <prefix>,
        and returns the file path to the downloaded file and shell_log."""
 
+    from requests import get
     # https://docs.openstack.org/api-ref/object-store/#show-container-details-and-list-objects
     with get(f"{url}/?prefix={prefix}&format=json", timeout=20) as req:
         if req.status_code == HTTPStatus.NO_CONTENT or req.content == b'[]':
@@ -316,7 +326,7 @@ def remove_file_match_ext(directory: str, extension: str) -> str:
         extension (str): The regular expression pattern to match filenames against.
     """
     print_fancy(f"Removing gcda files from {directory}")
-    log.add(f"\nfind {directory} -name \*.gcda -exec rm {{}} \;")
+    log.add(f"\nfind {directory} -name \\*.gcda -exec rm {{}} \\;")
     pattern = "." + extension
     count = 0
     for currentdir, _, files in os.walk(directory):

.github/workflows/root-ci.yml

@@ -128,31 +128,6 @@ jobs:
         with:
           ref: ${{ inputs.ref_name }}
 
-      - name: Apply option overrides from matrix for this job for non-release builds
-        if: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && matrix.overrides != NaN }}
-        env:
-          OVERRIDES: ${{ join( matrix.overrides, ' ') }}
-          CONFIGFILE: '.github/workflows/root-ci-config/buildconfig/${{ matrix.platform }}.txt'
-        shell: bash
-        run: |
-          set -x
-
-          echo '' >> "$CONFIGFILE"
-
-          for ENTRY in $GLOBAL_OVERRIDES $OVERRIDES; do
-              KEY=$( echo "$ENTRY" | cut -d '=' -f 1 )
-
-              # Add entry to file if not exists, otherwise replace
-
-              if grep -q "$KEY=" "$CONFIGFILE"; then
-                  sed -i "s/$KEY=.*\$/$ENTRY/" "$CONFIGFILE"
-              else
-                  echo "$ENTRY" >> "$CONFIGFILE"
-              fi
-          done
-
-          cat "$CONFIGFILE" || true
-
       - uses: root-project/gcc-problem-matcher-improved@main
         with:
           build-directory: /Users/sftnight/ROOT-CI/src/
@@ -167,6 +142,7 @@ jobs:
           HOME: /Users/sftnight
           INCREMENTAL: ${{ !contains(github.event.pull_request.labels.*.name, 'clean build') && !matrix.platform == 'mac15' && !matrix.platform == 'mac26'}}
           GITHUB_PR_ORIGIN: ${{ github.event.pull_request.head.repo.clone_url }}
+          OVERRIDES: ${{ join( matrix.overrides, ' ') }}
         run: ".github/workflows/root-ci-config/build_root.py 
                     --buildtype       RelWithDebInfo
                     --incremental     $INCREMENTAL
@@ -176,7 +152,8 @@ jobs:
                     --head_ref        refs/pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }}
                     --head_sha        ${{ github.event.pull_request.head.sha }}
                     --repository      ${{ github.server_url }}/${{ github.repository }}
-                    --platform        ${{ matrix.platform }}"
+                    --platform        ${{ matrix.platform }}
+                    --overrides       ${GLOBAL_OVERRIDES} ${OVERRIDES}"
 
       - name: Workflow dispatch
         shell: bash -leo pipefail {0}
@@ -204,13 +181,17 @@ jobs:
       - name: Update build cache after push to release branch
         shell: bash -leo pipefail {0}
         if:   github.event_name == 'push'
+        env:
+          OVERRIDES: ${{ join( matrix.overrides, ' ') }}
         run: ".github/workflows/root-ci-config/build_root.py 
                     --buildtype      RelWithDebInfo
                     --platform       ${{ matrix.platform }}
                     --incremental    false
                     --base_ref       ${{ github.ref_name }}
                     --binaries       ${{ startsWith(github.ref, 'refs/tags/') }}
-                    --repository     ${{ github.server_url }}/${{ github.repository }}"
+                    --repository     ${{ github.server_url }}/${{ github.repository }}
+                    --overrides       ${GLOBAL_OVERRIDES} ${OVERRIDES}
+             "
 
       - name: Upload test results
         if:   ${{ !cancelled() }}
@@ -491,31 +472,6 @@ jobs:
                ls -la
               '
 
-      - name: Apply option overrides from matrix for this job for all builds except non-special nightlies and releases
-        if: ${{ ( github.event_name != 'schedule' || matrix.is_special) && github.event_name != 'workflow_dispatch' && matrix.overrides != NaN }}
-        env:
-          OVERRIDES: ${{ join( matrix.overrides, ' ') }}
-          CONFIGFILE: '.github/workflows/root-ci-config/buildconfig/${{ matrix.image }}.txt'
-        shell: bash
-        run: |
-          set -x
-
-          echo '' >> "$CONFIGFILE"
-
-          for ENTRY in $GLOBAL_OVERRIDES $OVERRIDES; do
-              KEY=$( echo "$ENTRY" | cut -d '=' -f 1 )
-
-              # Add entry to file if not exists, otherwise replace
-
-              if grep -q "$KEY=" "$CONFIGFILE"; then
-                  sed -i "s/$KEY=.*\$/$ENTRY/" "$CONFIGFILE"
-              else
-                  echo "$ENTRY" >> "$CONFIGFILE"
-              fi
-          done
-
-          cat "$CONFIGFILE" || true
-
       - uses: root-project/gcc-problem-matcher-improved@main
         with:
           build-directory: /github/home/ROOT-CI/src/
@@ -526,6 +482,7 @@ jobs:
           INCREMENTAL: ${{ !contains(github.event.pull_request.labels.*.name, 'clean build') }}
           GITHUB_PR_ORIGIN: ${{ github.event.pull_request.head.repo.clone_url }}
           CMAKE_GENERATOR: ${{ matrix.cmake_generator }}
+          OVERRIDES: ${{ join( matrix.overrides, ' ') }}
         run: ".github/workflows/root-ci-config/build_root.py
                     --buildtype       RelWithDebInfo
                     --platform        ${{ matrix.image }}
@@ -537,6 +494,7 @@ jobs:
                     --head_ref        refs/pull/${{ github.event.pull_request.number }}/head:${{ github.event.pull_request.head.ref }}
                     --head_sha        ${{ github.event.pull_request.head.sha }}
                     --repository      ${{ github.server_url }}/${{ github.repository }}
+                    --overrides       ${GLOBAL_OVERRIDES} ${OVERRIDES}
               "
 
       - name: Workflow dispatch
@@ -564,13 +522,16 @@ jobs:
 
       - name: Update build cache after push to release branch
         if:   github.event_name == 'push'
+        env:
+          OVERRIDES: ${{ join( matrix.overrides, ' ') }}
         run: ".github/workflows/root-ci-config/build_root.py
                     --buildtype      RelWithDebInfo
                     --platform       ${{ matrix.image }}
                     --incremental    false
                     --base_ref       ${{ github.ref_name }}
                     --binaries       ${{ startsWith(github.ref, 'refs/tags/') }}
                     --repository     ${{ github.server_url }}/${{ github.repository }}
+                    --overrides       ${GLOBAL_OVERRIDES} ${OVERRIDES}
               "
 
       - name: Upload test results