Add switch for running Sonobuoy tests in parallel

Tests/README.md

@@ -49,6 +49,8 @@ Install KaaS-specific requirements:
 pip install -r kaas/requirements.txt
 ```
 
+Additionally, Sonobuoy must be installed and in your PATH environment variable.
+
 Given a kubeconfig file `path/to/kubeconfig.yaml`, run
 
 ```shell
@@ -58,6 +60,10 @@ Given a kubeconfig file `path/to/kubeconfig.yaml`, run
 Replace `SUBJECT` with an arbitrary, but meaningful subject name. Also, please note that the check
 will always use the `current-context` of the kubeconfig and will fail if it isn't set.
 
+Note: Sonobuoy checks (such as CNCF k8s conformance) are known to run for multiple hours (during which
+you won't see feedback). In order to save wall-time, the sonobuoy checks will be run in parallel. You
+disable that by simply adding `-a execution_mode=serial` to the command-line.
+
 A report in YAML format will be created.
 
 Additionally, the directory `sono-results` will be generated. It contains a JUnit XML file:

Tests/kaas/sonobuoy_handler/run_sonobuoy.py

@@ -21,8 +21,13 @@ def cli():
 @click.option("-r", "--result_dir_name", "result_dir_name", type=str, default="sonobuoy_results", help="directory name to store results at",)
 @click.option("-c", "--check", "check_name", type=str, default="sonobuoy_executor", help="this MUST be the same name as the id in 'scs-compatible-kaas.yaml'",)
 @click.option("--scs-sonobuoy-config", "scs_sonobuoy_config_yaml", type=click.Path(exists=True), default="kaas/sonobuoy-config.yaml", help="Configuration for Sonobuoy (yaml format)")
+@click.option('--execution-mode', 'mode', type=click.Choice(('serial', 'parallel')), default='serial')
 @click.option("-a", "--arg", "args", multiple=True)
-def sonobuoy_run(kubeconfig, result_dir_name, check_name, scs_sonobuoy_config_yaml, args):
+def sonobuoy_run(kubeconfig, result_dir_name, check_name, scs_sonobuoy_config_yaml, mode, args):
+    if mode == 'parallel':
+        # This is merely a shortcut to simplify commandline in scs-compatible-kaas.yaml
+        # For more on parallel execution, see https://github.com/vmware-tanzu/sonobuoy/issues/1435
+        args += ('--e2e-parallel=true', )
     sonobuoy_handler = SonobuoyHandler(check_name, kubeconfig, result_dir_name, scs_sonobuoy_config_yaml, args)
     sys.exit(sonobuoy_handler.run())
 

Tests/scs-compatible-kaas.yaml

@@ -9,9 +9,10 @@ variables:
     # working directory for the subject under test
     # (note that we consider each kubernetes branch a test subject of its own)
   - kubeconfig
+  - execution_mode: parallel  # alternatively: serial
 scripts:
 - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py
-  args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-e2e -c 'cncf-k8s-conformance'  -a '--mode=certified-conformance'
+  args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-e2e -c 'cncf-k8s-conformance'  -a '--mode=certified-conformance' --execution-mode {execution_mode}
   #~ args: -k {kubeconfig} -r {subject_root}/sono-results -c 'cncf-k8s-conformance'  -a '--plugin-env e2e.E2E_DRYRUN=true'
   testcases:
   - id: cncf-k8s-conformance
@@ -31,7 +32,7 @@ scripts:
     description: Must fulfill all requirements of scs-0214-v2.
     url: https://docs.scs.community/standards/scs-0214-v2-k8s-node-distribution#decision
 - executable: ./kaas/sonobuoy_handler/run_sonobuoy.py
-  args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-0219 -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"'
+  args: run -k {kubeconfig} --scs-sonobuoy-config kaas/scs-sonobuoy-config.yaml -r {subject_root}/sono-results-0219 -c 'kaas-networking-check' -a '--e2e-focus "NetworkPolicy"' --execution-mode {execution_mode}
   testcases:
   - id: kaas-networking-check
     description: Must fulfill all requirements of scs-0219-v1.

Tests/scs-compliance-check.py

@@ -302,7 +302,8 @@ def main(argv):
         raise RuntimeError("You need pass --subject=SUBJECT.")
     with open(config.arg0, "r", encoding="UTF-8") as specfile:
         spec = load_spec(yaml.load(specfile, Loader=yaml.SafeLoader))
-    missing_vars = [v for v in spec.get("variables", ()) if v not in config.assignment]
+    assignment = {**spec['var_defaults'], **config.assignment}
+    missing_vars = [v for v in spec.get("variables", ()) if v not in assignment]
     if missing_vars:
         raise RuntimeError(f"Missing variable assignments (via -a) for: {', '.join(missing_vars)}")
     if "prerequisite" in spec:
@@ -317,7 +318,7 @@ def main(argv):
     if not versions:
         raise RuntimeError(f"No valid version found for {config.checkdate}")
     check_cwd = os.path.dirname(config.arg0) or os.getcwd()
-    runner = CheckRunner(check_cwd, config.assignment, verbosity=config.verbose and 2 or not config.quiet)
+    runner = CheckRunner(check_cwd, assignment, verbosity=config.verbose and 2 or not config.quiet)
     title, partial = spec['name'], False
     if config.sections:
         title += f" [sections: {', '.join(config.sections)}]"

Tests/scs_cert_lib.py

@@ -108,6 +108,17 @@ def _resolve_spec(spec: dict):
         for vname in entry['versions']:
             # trigger KeyError
             _ = version_lookup[vname]
+    # step 5. unify variables declaration (the list may contain strings as well as singleton dicts)
+    variables = []
+    defaults = {}
+    for var_decl in spec.get('variables', ()):
+        if isinstance(var_decl, dict):
+            defaults.update(var_decl)
+        if isinstance(var_decl, str):
+            variables.append(var_decl)
+    variables.extend(defaults)
+    spec['variables'] = variables
+    spec['var_defaults'] = defaults
 
 
 def load_spec(document: dict) -> dict: