diff --git a/.github/workflows/cache_xmss_and_xmssmt_keys.yml b/.github/workflows/cache_xmss_and_xmssmt_keys.yml new file mode 100644 index 0000000..b8bd805 --- /dev/null +++ b/.github/workflows/cache_xmss_and_xmssmt_keys.yml @@ -0,0 +1,157 @@ +# ============================================================================ +# XMSS/XMSS^MT Key Generation and Caching Pipeline +# ============================================================================ +# +# Purpose: +# -------- +# XMSS and XMSS^MT are post-quantum stateful signature schemes that are very +# expensive to generate (can take several minutes (more than 30 Minutes) for larger parameters). +# This workflow pre-generates these keys and caches them in GitHub Actions +# so that tests can run quickly without regenerating keys every time. +# +# Caching Strategy: +# ----------------- +# - "complete" cache: All required keys have been generated (final state). +# - "progress" cache: Partial key generation (incremental saves during generation). +# +# The workflow will: +# 1. Try to restore a complete cache first. +# 2. Fall back to the latest progress cache if the complete cache doesn't exist. +# 3. Generate any missing keys. +# 4. Save either a new progress cache or complete cache depending on status. +# +# Cache Key Versioning: +# --------------------- +# KEY_GEN_VERSION allows invalidating old caches when the key generation +# logic changes or when we need to regenerate all keys from scratch. +# Increment this version to force a fresh generation. +# ============================================================================ + +name: "Generate & Cache XMSS Keys" + +# Trigger this workflow on any push, manual dispatch, or repository dispatch +on: + push: + branches: [ "**" ] + workflow_dispatch: + repository_dispatch: {} + +env: + # Version of key generation logic - increment to invalidate all caches. + KEY_GEN_VERSION: 0 + # Directory where generated keys will be stored. + KEY_DIR: data/xmss_xmssmt_keys + +# Prevent multiple instances of this workflow from running concurrently +# to avoid cache conflicts and wasted compute time. +concurrency: + group: gen-cache-stfl-keys-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build-cache: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + cache-dependency-glob: "**/pyproject.toml" + + - name: Set up Python 3.9 + run: uv python install 3.9 + + - name: Install dependencies + run: uv sync --extra dev + + # Generate today's date for use in progress cache keys + # This ensures we get a new cache entry each day for incremental progress + - name: Get current date for cache key + id: date + run: echo "date=$(date +%Y-%m-%d)" >> $GITHUB_OUTPUT + + # Try to restore previously cached keys + # Priority order: + # 1. xmss-v{VERSION}-complete (all keys generated - exact match) + # 2. xmss-v{VERSION}-progress- (partial keys - most recent match) + # 3. xmss-v{VERSION}- (any cache for this version) + - name: Restore XMSS/XMSSMT key cache + id: cache-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.KEY_DIR }} + key: xmss-v${{ env.KEY_GEN_VERSION }}-complete + restore-keys: | + xmss-v${{ env.KEY_GEN_VERSION }}-progress- + xmss-v${{ env.KEY_GEN_VERSION }}- + enableCrossOsArchive: true + + # Display what keys were restored from cache (if any) + # Helps with debugging cache issues + - name: Show tree of cache directory + run: | + echo "Listing cache directory:" + mkdir -p data/xmss_xmssmt_keys + ls -R data/xmss_xmssmt_keys + + # Generate any keys that are still missing + # The script skips keys that already exist (from cache restore) + # This is used, to safe the keys across multiple runs. + - name: Generate keys when missing + id: generate + env: + PYTHONUNBUFFERED: "1" # Enable unbuffered output for real-time logs + run: | + set -x # Enable command echoing for transparency + mkdir -p "${KEY_DIR}" + + # Run the key generation script + # It will skip keys that already exist and only generate missing ones + uv run scripts/pipeline_gen_stfl_keys.py "${KEY_DIR}" + + # Show what keys we have now + ls -lah "${KEY_DIR}" + + # Count total keys and export for later steps + KEY_COUNT=$(find "${KEY_DIR}" -type f -name "*.der" | wc -l) + echo "key_count=${KEY_COUNT}" >> $GITHUB_OUTPUT + echo "Generated/found ${KEY_COUNT} keys" + + # Check if we have successfully generated ALL required keys + # Uses the --check_keys_dir flag which returns: + # - exit code 0 if all keys present → complete=true + # - exit code 1 if any keys missing → complete=false + - name: Check if all keys are generated + id: check-complete + run: | + # Use the --check_keys_dir flag to verify all keys are present + if uv run scripts/pipeline_gen_stfl_keys.py "${KEY_DIR}" --check_keys_dir; then + echo "complete=true" >> $GITHUB_OUTPUT + else + echo "complete=false" >> $GITHUB_OUTPUT + fi + + # Save a "progress" cache if key generation is not yet complete + # This allows us to resume from this point in the next workflow run + # Uses a unique key with date and run number to avoid conflicts + # Condition: always() ensures we save even if generation times out or fails + - name: Save progress cache + if: always() && steps.check-complete.outputs.complete != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ env.KEY_DIR }} + key: xmss-v${{ env.KEY_GEN_VERSION }}-progress-${{ steps.date.outputs.date }}-${{ github.run_number }} + enableCrossOsArchive: true + + # Save a "complete" cache once all keys are generated + # This is the final cache that will be restored by future workflow runs + # Uses a fixed key name so it's always found first during cache restore + - name: Save complete cache + if: steps.check-complete.outputs.complete == 'true' + uses: actions/cache/save@v4 + with: + path: ${{ env.KEY_DIR }} + key: xmss-v${{ env.KEY_GEN_VERSION }}-complete + enableCrossOsArchive: true diff --git a/.github/workflows/python_detailed.yml b/.github/workflows/python_detailed.yml index 95ab765..6dc748f 100644 --- a/.github/workflows/python_detailed.yml +++ b/.github/workflows/python_detailed.yml @@ -17,6 +17,8 @@ env: WIN_LIBOQS_INSTALL_PATH: C:\liboqs VERSION: 0.14.0 PYOQS_ENABLE_FAULTHANDLER: "1" + KEY_GEN_VERSION: 0 + KEY_DIR: data/xmss_xmssmt_keys concurrency: group: test-python-detailed-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} @@ -45,6 +47,25 @@ jobs: - name: Install dependencies run: uv sync --extra dev + # Restore the XMSS/XMSSMT key cache. + # Still allows the incomplete cache to be used to avoid regenerating keys. + # But if the complete cache is present, all keys are already generated and used. + - name: Restore XMSS/XMSSMT key cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.KEY_DIR }} + key: xmss-v${{ env.KEY_GEN_VERSION }}-complete + restore-keys: | + xmss-v${{ env.KEY_GEN_VERSION }}-progress- + xmss-v${{ env.KEY_GEN_VERSION }}- + enableCrossOsArchive: true + + # Display the contents of the cache directory to verify what was restored. + - name: Show tree of cache directory + run: | + echo "Listing cache directory:" + ls -R data/xmss_xmssmt_keys + - name: Install liboqs POSIX if: matrix.os != 'windows-latest' run: | diff --git a/.github/workflows/python_simplified.yml b/.github/workflows/python_simplified.yml index ad691cf..4e4571f 100644 --- a/.github/workflows/python_simplified.yml +++ b/.github/workflows/python_simplified.yml @@ -17,6 +17,8 @@ concurrency: env: PYOQS_ENABLE_FAULTHANDLER: "1" + KEY_GEN_VERSION: 0 + KEY_DIR: data/xmss_xmssmt_keys jobs: build: @@ -38,6 +40,25 @@ jobs: - name: Set up Python 3.9 run: uv python install 3.9 + # Restore the XMSS/XMSSMT key cache. + # Still allows the incomplete cache to be used to avoid regenerating keys. + # But if the complete cache is present, all keys are already generated and used. + - name: Restore XMSS/XMSSMT key cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.KEY_DIR }} + key: xmss-v${{ env.KEY_GEN_VERSION }}-complete + restore-keys: | + xmss-v${{ env.KEY_GEN_VERSION }}-progress- + xmss-v${{ env.KEY_GEN_VERSION }}- + enableCrossOsArchive: true + + # Display the contents of the cache directory to verify what was restored. + - name: Show tree of cache directory + run: | + echo "Listing cache directory:" + ls -R data/xmss_xmssmt_keys + - name: Run examples run: | uv sync --extra dev diff --git a/data/xmss_xmssmt_keys/xmss-sha2_16_512.der b/data/xmss_xmssmt_keys/xmss-sha2_16_512.der deleted file mode 100644 index bb0b1f8..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-sha2_16_512.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-sha2_20_192.der b/data/xmss_xmssmt_keys/xmss-sha2_20_192.der deleted file mode 100644 index e84aac4..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-sha2_20_192.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-sha2_20_256.der b/data/xmss_xmssmt_keys/xmss-sha2_20_256.der deleted file mode 100644 index 3a1a8d4..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-sha2_20_256.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-sha2_20_512.der b/data/xmss_xmssmt_keys/xmss-sha2_20_512.der deleted file mode 100644 index e218cef..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-sha2_20_512.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-shake256_20_192.der b/data/xmss_xmssmt_keys/xmss-shake256_20_192.der deleted file mode 100644 index 7dbac18..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-shake256_20_192.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-shake256_20_256.der b/data/xmss_xmssmt_keys/xmss-shake256_20_256.der deleted file mode 100644 index 5612728..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-shake256_20_256.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-shake_16_256.der b/data/xmss_xmssmt_keys/xmss-shake_16_256.der deleted file mode 100644 index 01e03be..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-shake_16_256.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-shake_16_512.der b/data/xmss_xmssmt_keys/xmss-shake_16_512.der deleted file mode 100644 index 7613b0d..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-shake_16_512.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-shake_20_256.der b/data/xmss_xmssmt_keys/xmss-shake_20_256.der deleted file mode 100644 index fa11516..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-shake_20_256.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmss-shake_20_512.der b/data/xmss_xmssmt_keys/xmss-shake_20_512.der deleted file mode 100644 index 19581ea..0000000 Binary files a/data/xmss_xmssmt_keys/xmss-shake_20_512.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmssmt-sha2_40_layers_2_256.der b/data/xmss_xmssmt_keys/xmssmt-sha2_40_layers_2_256.der deleted file mode 100644 index 2c86596..0000000 Binary files a/data/xmss_xmssmt_keys/xmssmt-sha2_40_layers_2_256.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmssmt-sha2_60_layers_3_256.der b/data/xmss_xmssmt_keys/xmssmt-sha2_60_layers_3_256.der deleted file mode 100644 index e73474f..0000000 Binary files a/data/xmss_xmssmt_keys/xmssmt-sha2_60_layers_3_256.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmssmt-shake_40_layers_2_256.der b/data/xmss_xmssmt_keys/xmssmt-shake_40_layers_2_256.der deleted file mode 100644 index fc56808..0000000 Binary files a/data/xmss_xmssmt_keys/xmssmt-shake_40_layers_2_256.der and /dev/null differ diff --git a/data/xmss_xmssmt_keys/xmssmt-shake_60_layers_3_256.der b/data/xmss_xmssmt_keys/xmssmt-shake_60_layers_3_256.der deleted file mode 100644 index 21cca70..0000000 Binary files a/data/xmss_xmssmt_keys/xmssmt-shake_60_layers_3_256.der and /dev/null differ diff --git a/oqs/serialize.py b/oqs/serialize.py index 14652ab..bfc520e 100644 --- a/oqs/serialize.py +++ b/oqs/serialize.py @@ -36,7 +36,7 @@ def _get_oid_from_name(name: str) -> str: def serialize_stateful_signature_key( - stateful_sig: oqs.StatefulSignature, public_key: bytes, fpath: str + stateful_sig: oqs.StatefulSignature, public_key: bytes, fpath: Union[Path, str] ) -> None: """ Serialize the stateful signature key to a `OneAsymmetricKey` structure. diff --git a/pyproject.toml b/pyproject.toml index b50d645..77f2f02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "liboqs-python" requires-python = ">=3.9" -version = "0.14.0" +version = "0.15.0" description = "Python bindings for liboqs, providing post-quantum public key cryptography algorithms" authors = [ { name = "Open Quantum Safe project", email = "contact@openquantumsafe.org" }, diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/pipeline_gen_stfl_keys.py b/scripts/pipeline_gen_stfl_keys.py new file mode 100644 index 0000000..4a93fdc --- /dev/null +++ b/scripts/pipeline_gen_stfl_keys.py @@ -0,0 +1,215 @@ +""" +XMSS/XMSS^MT Stateful Signature Key Pre-Generation Script. + +This module provides functionality to pre-generate expensive XMSS and XMSS^MT +stateful signature keys for use in CI/CD pipelines and testing environments. + +Background +---------- +XMSS (eXtended Merkle Signature Scheme) and XMSS^MT (XMSS Multi-Tree) are +post-quantum stateful signature schemes that can be computationally expensive +to generate, especially for larger tree heights. Pre-generating these keys +significantly reduces test execution time in CI pipelines. +""" + +from __future__ import annotations + +from pathlib import Path +import argparse +import logging +import os +from sys import stdout +from typing import Any, Iterable + +import oqs +import oqs.serialize + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.StreamHandler(stdout)) + + +def _mech_to_filename(name: str) -> str: + """ + Map mechanism name to key filename (keep in sync with CI pipeline). + + Example: + "XMSS^MT-SHA2_20/4_256" -> "xmssmt-sha2_20_layers_4_256.der" + "XMSS-SHA2_10_256" -> "xmss-sha2_10_256.der" + + """ + return f"{name.replace('/', '_layers_', 1).lower()}.der" + + +def _collect_mechanism_names() -> list[str]: + """Return all enabled XMSS/XMSS^MT stateful signature mechanisms.""" + return [ + name + for name in oqs.get_enabled_stateful_sig_mechanisms() + if name.startswith(("XMSS-", "XMSSMT-")) + ] + + +def _check_is_expensive(name: str) -> bool: + """ + Check if the given XMSS/XMSS^MT mechanism is considered expensive to generate. + + Currently, we consider mechanisms with height > 16 as expensive. + """ + if name.startswith("XMSS-"): + parts = name.split("-")[1].split("_") + height = int(parts[1]) + output = int(parts[2]) + return height > 16 or output == 512 + if name.startswith("XMSSMT-"): + parts = name.split("-")[1].split("_") + height = int(parts[1].split("/")[0]) + layers = int(parts[1].split("/")[1]) + return (height == 40 and layers == 2) or (height == 60 and layers == 3) + return False + + +def get_all_keys_to_generate() -> list[str]: + """Get a list of all XMSS/XMSS^MT keys that are considered expensive to generate.""" + all_keys: list[str] = _collect_mechanism_names() + return [name for name in all_keys if _check_is_expensive(name)] + + +def check_generated_all_keys(out_dir: Path) -> bool: + """Check if all XMSS/XMSS^MT keys are present in *out_dir*.""" + all_keys: list[str] = get_all_keys_to_generate() + + for name in all_keys: + key_filename = _mech_to_filename(name) + key_path = out_dir / key_filename + if not key_path.exists(): + return False + return True + + +def generate_keys(out_dir: Path) -> dict[str, Any]: + """ + Generate all XMSS/XMSS^MT keys into *out_dir* if they are missing. + + Returns a small stats dict useful for tests: + {"generated": int, "skipped": int, "total": int, "missing": list[str]} + """ + out_dir.mkdir(parents=True, exist_ok=True) + + all_keys: list[str] = _collect_mechanism_names() + + # Track existing keys by stem for informational purposes + existing_keys: set[str] = {p.stem for p in out_dir.glob("*.der")} + + generated = 0 + skipped = 0 + + for name in all_keys: + if not _check_is_expensive(name): + logger.debug("Skipping %s (does not need to be pre-generated.)", name) + continue + + key_filename = _mech_to_filename(name) + key_path = out_dir / key_filename + + if key_path.exists(): + logger.debug("Skipping %s (already exists)", name) + skipped += 1 + continue + + logger.debug("Generating %s...", name) + with oqs.StatefulSignature(name) as sig: + pub = sig.generate_keypair() + oqs.serialize.serialize_stateful_signature_key(sig, pub, key_path) + logger.debug("Generated %s", name) + generated += 1 + + total = len(all_keys) + logger.debug( + "\n=== Summary ===\nGenerated: %d\nSkipped: %d\nTotal: %d", generated, skipped, total + ) + + missing: list[str] = [] + for name in all_keys: + key_filename = _mech_to_filename(name) + key_path = out_dir / key_filename + if not key_path.exists(): + missing.append(name) + + if missing: + logger.debug("\nERROR: The following keys could not be generated:") + for name in missing: + logger.debug(" - %s", name) + + logger.debug("\nAll %d XMSS/XMSS^MT keys are available in %s.", total, out_dir) + logger.debug("\nFiles in %s:", out_dir) + + return { + "generated": generated, + "skipped": skipped, + "total": total, + "missing": missing, + "existing": sorted(existing_keys), + } + + +def _resolve_out_dir(cli_dir: str | None) -> Path: + """ + Resolve the output directory from CLI argument or KEY_DIR env. + + Precedence: + 1. Explicit CLI argument (if provided). + 2. $KEY_DIR environment variable (if set). + 3. Default "data/xmss_xmssmt_keys" relative to repo root. + """ + if cli_dir: + return Path(cli_dir) + + env_dir = os.environ.get("KEY_DIR") + if env_dir: + return Path(env_dir) + + return Path("data/xmss_xmssmt_keys") + + +def main(argv: Iterable[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Generate XMSS/XMSS^MT stateful signature keys.", + ) + parser.add_argument( + "key_dir", + nargs="?", + help=( + "Output directory for keys. " + "Defaults to $KEY_DIR if set, otherwise data/xmss_xmssmt_keys." + ), + ) + parser.add_argument( + "--check_keys_dir", + action="store_true", + help=( + "If set, do not generate keys; only check whether all required " + "XMSS/XMSS^MT keys exist in the output directory. Returns 0 if all " + "keys are present, 1 if any are missing." + ), + ) + args = parser.parse_args(list(argv) if argv is not None else None) + + out_dir = _resolve_out_dir(args.key_dir) + + if args.check_keys_dir: + # Only check whether all required keys are present; do not generate. + all_present = check_generated_all_keys(out_dir) + if all_present: + logger.debug("All required XMSS/XMSS^MT keys are present in %s", out_dir) + return 0 + else: + logger.debug("Some required XMSS/XMSS^MT keys are missing in %s", out_dir) + return 1 + + _ = generate_keys(out_dir) + return 0 + + +if __name__ == "__main__": + main()