Skip to content

Commit c48c530

Browse files
SbozzoloSbozzolo
committed
Add GPU, large ARM runners, and setup-palace-ci
This commit runs the spack workflow on larger/GPU-accelerated runners. I added a full test matrix, but I could not get everything to work. In particular, the static builds have several linking errors, which indicates some issue in the build system. I tried quick fixes, but I could not get the builds to work, so I am leaving this for future work.
1 parent c60bc85 commit c48c530

9 files changed

Lines changed: 892 additions & 80 deletions

File tree

‎.github/actions/setup-palace-ci/action.yml‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
name: 'Setup Palace CI'
2+
description: 'Common steps to prepare the environment to run Palace tests'
3+
inputs:
4+
julia-version:
5+
description: 'Version of Julia to use (as understood by juliaup)'
6+
required: false
7+
default: 'release'
8+
spack-version:
9+
description: 'Spack git tag to use'
10+
required: false
11+
default: 'develop'
12+
llvm-version:
13+
description: 'Version of LLVM to use (only major version supported, e.g. "19")'
14+
required: false
15+
default: 'dont-install'
16+
setup-intel:
17+
description: 'Install Intel OneAPI'
18+
required: false
19+
default: 'false'
20+
outputs:
21+
julia-version:
22+
description: 'Julia version that was installed'
23+
value: ${{ steps.setup-julia.outputs.julia-version }}
24+
runs:
25+
using: 'composite'
26+
steps:
27+
28+
# This can sometimes interfere with sudo apt operations below. We don't
29+
# really care about these updates anyway, as we spin up new instances all
30+
# the time.
31+
- name: Disable unattended upgrades
32+
shell: bash
33+
run: |
34+
# Kill any running unattended upgrade processes
35+
sudo pkill -9 -f unattended-upgr || true
36+
sudo pkill -9 -f apt.systemd.daily || true
37+
38+
# Stop and disable systemd services
39+
sudo systemctl stop apt-daily.timer || true
40+
sudo systemctl disable apt-daily.timer || true
41+
sudo systemctl stop apt-daily-upgrade.timer || true
42+
sudo systemctl disable apt-daily-upgrade.timer || true
43+
sudo systemctl stop apt-daily.service || true
44+
sudo systemctl stop apt-daily-upgrade.service || true
45+
46+
# Remove lock files if they exist
47+
sudo rm -f /var/lib/dpkg/lock-frontend
48+
sudo rm -f /var/lib/dpkg/lock
49+
sudo dpkg --configure -a || true
50+
51+
# Disable in config
52+
echo 'APT::Periodic::Update-Package-Lists "0";' | sudo tee /etc/apt/apt.conf.d/20auto-upgrades
53+
echo 'APT::Periodic::Unattended-Upgrade "0";' | sudo tee -a /etc/apt/apt.conf.d/20auto-upgrades
54+
55+
- name: Setup repositories
56+
shell: bash
57+
run: |
58+
# Intel
59+
echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
60+
wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
61+
| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
62+
63+
# LLVM
64+
CODENAME=$(lsb_release -cs) # Ubuntu 24 is "noble"
65+
echo "deb http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME} main" | sudo tee /etc/apt/sources.list.d/llvm.list
66+
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
67+
68+
sudo apt update
69+
70+
- name: Setup Intel OneAPI
71+
if: inputs.setup-intel == 'true'
72+
shell: bash
73+
run: |
74+
sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp intel-oneapi-compiler-fortran
75+
76+
- name: Setup LLVM
77+
if: inputs.llvm-version != 'dont-install'
78+
shell: bash
79+
env:
80+
LLVM_VERSION: ${{ inputs.llvm-version }}
81+
run: |
82+
sudo apt-get install -y clang-${LLVM_VERSION} llvm-${LLVM_VERSION} lld-${LLVM_VERSION} lldb-${LLVM_VERSION} libomp5-${LLVM_VERSION} libomp-${LLVM_VERSION}-dev flang-${LLVM_VERSION}
83+
84+
- name: Setup GNU
85+
shell: bash
86+
run: |
87+
sudo apt-get install -y gfortran
88+
89+
- name: Setup GNU OpenMP
90+
shell: bash
91+
if: inputs.llvm-version == 'dont-install'
92+
run: |
93+
sudo apt-get install -y libgomp1
94+
95+
- name: Install Spack
96+
uses: actions/checkout@v6
97+
with:
98+
repository: spack/spack
99+
path: spack
100+
ref: ${{ inputs.spack-version }}
101+
102+
- name: Setup Spack environment
103+
shell: bash
104+
run: |
105+
echo "$(realpath spack/bin)" >> "$GITHUB_PATH"
106+
echo "SPACK_DISABLE_LOCAL_CONFIG=1" >> "$GITHUB_ENV"
107+
108+
- name: Install Julia
109+
id: setup-julia
110+
shell: bash
111+
env:
112+
JULIA_VERSION: ${{ inputs.julia-version }}
113+
run: |
114+
# Install juliaup (installs release by default)
115+
curl -fsSL https://install.julialang.org | sh -s -- --yes
116+
117+
# Add Julia to PATH
118+
echo "$HOME/.juliaup/bin" >> $GITHUB_PATH
119+
120+
# Install and set the specified Julia version as default
121+
if [ "${JULIA_VERSION}" != "release" ]; then
122+
~/.juliaup/bin/juliaup add "${JULIA_VERSION}"
123+
~/.juliaup/bin/juliaup default "${JULIA_VERSION}"
124+
fi
125+
126+
# Output the installed version
127+
echo "julia-version=$JULIA_VERSION" >> $GITHUB_OUTPUT
128+
echo "Installed Julia version: $JULIA_VERSION"

‎.github/workflows/generate_test_matrix.py‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
#!/usr/bin/env python3
2+
3+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
4+
# SPDX-License-Identifier: Apache-2.0
5+
6+
"""Test Matrix Generator
7+
8+
Generates pairwise test combinations used in spack.yml using
9+
[allpairspy](https://github.com/thombashi/allpairspy).
10+
11+
## Usage
12+
13+
```bash
14+
pip install -U pyyaml allpairspy
15+
python generate_test_matrix.py
16+
```
17+
18+
It enforces certain constrains.
19+
20+
"""
21+
from allpairspy import AllPairs
22+
import yaml
23+
24+
parameters = [
25+
["x86", "x86", "arm"], # Favor x86
26+
["gcc", "llvm", "intel-oneapi-compilers"],
27+
["openmpi", "mpich", "intel-oneapi-mpi"],
28+
["openblas", "amdblis", "armpl-gcc", "intel-oneapi-mkl"],
29+
["+shared", "~shared"],
30+
["+int64", "~int64"],
31+
["~openmp", "+openmp"],
32+
["+arpack", "+slepc"],
33+
["+mumps", "+superlu-dist", "+strumpack"],
34+
["~cuda", "+cuda"],
35+
]
36+
37+
def is_valid(combo):
38+
if len(combo) < 4:
39+
# AllPairs creates partial combinations
40+
return True
41+
42+
arch, compiler, mpi, math_libs = combo[0], combo[1], combo[2], combo[3]
43+
44+
# AllPairs creates partial combinations, so we don't always have 9 entries
45+
openmp = combo[6] if len(combo) > 6 else "~openmp"
46+
cuda = combo[9] if len(combo) > 9 else "~cuda"
47+
48+
# Intel packages must all be together and only on x86
49+
50+
# We also do not allow the CUDA+Intel combination because the "g" instances
51+
# are all AMD
52+
intel_packages = [compiler == "intel-oneapi-compilers",
53+
mpi == "intel-oneapi-mpi",
54+
math_libs == "intel-oneapi-mkl"]
55+
if any(intel_packages):
56+
if arch != "x86" or not all(intel_packages) or cuda == "+cuda":
57+
return False
58+
59+
if cuda == "+cuda":
60+
# We only have GPUs on x86 machines
61+
if arch != "x86":
62+
return False
63+
# We don't want to mix GPU with OpenMP
64+
if openmp == "+openmp":
65+
return False
66+
67+
# Use ARMPL only with GCC and ARM
68+
if math_libs == "armpl-gcc" and (compiler != "gcc" or arch != "arm"):
69+
return False
70+
71+
# On ARM, use armpl-gcc or openblas
72+
if arch == "arm" and math_libs not in ("armpl-gcc", "openblas"):
73+
return False
74+
75+
return True
76+
77+
matrix = []
78+
for combo in AllPairs(parameters, filter_func=is_valid):
79+
matrix.append({
80+
"arch": combo[0],
81+
"compiler": combo[1],
82+
"mpi": combo[2],
83+
"math-libs": combo[3],
84+
"shared": combo[4],
85+
"int": combo[5],
86+
"openmp": combo[6],
87+
"eigensolver": combo[7],
88+
"solver": combo[8],
89+
"cuda": combo[9],
90+
})
91+
92+
# Add more CUDA test cases with different solver/eigensolver combinations
93+
# (AllPairs is highly constrained with CUDA, so we manually add a couple of
94+
# extra cases)
95+
desired_cuda_cases = [
96+
{
97+
"arch": "x86", "compiler": "gcc", "mpi": "openmpi", "math-libs": "openblas",
98+
"shared": "+shared", "int": "+int64", "openmp": "~openmp",
99+
"eigensolver": "+slepc", "solver": "+superlu-dist", "cuda": "+cuda"
100+
},
101+
{
102+
"arch": "x86", "compiler": "llvm", "mpi": "openmpi", "math-libs": "amdblis",
103+
"shared": "~shared", "int": "~int64", "openmp": "~openmp",
104+
"eigensolver": "+arpack", "solver": "+strumpack", "cuda": "+cuda"
105+
},
106+
107+
]
108+
109+
# Add one case where we turn a multiple solvers (to check that there's no
110+
# problem with compiling multiple solvers)
111+
for cuda in ("~cuda", "+cuda"):
112+
matrix.append({
113+
"arch": "x86",
114+
"compiler": "gcc",
115+
"mpi": "openmpi",
116+
"math-libs": "openblas",
117+
"shared": "+shared",
118+
"int": "~int64",
119+
"openmp": "~openmp",
120+
"eigensolver": "+slepc+arpack",
121+
"solver": "+superlu-dist+mumps+sundials+strumpack",
122+
"cuda": cuda
123+
})
124+
125+
for cuda_case in desired_cuda_cases:
126+
exists = any(
127+
all(existing.get(k) == v for k, v in cuda_case.items())
128+
for existing in matrix
129+
)
130+
if not exists:
131+
matrix.append(cuda_case)
132+
133+
134+
# Custom list class for flow-style output, so that we can print it as [self-hosted, gpu]
135+
# instead of
136+
# - self-hosted
137+
# - gpu
138+
class FlowList(list):
139+
pass
140+
141+
def represent_flow_list(dumper, data):
142+
return dumper.represent_sequence('tag:yaml.org,2002:seq', data, flow_style=True)
143+
144+
yaml.add_representer(FlowList, represent_flow_list)
145+
146+
# Add runners
147+
for entry in matrix:
148+
if entry["arch"] == "arm":
149+
entry["runner"] = FlowList(["self-hosted", "arm64"])
150+
elif entry["cuda"] == "+cuda":
151+
entry["runner"] = FlowList(["self-hosted", "gpu"])
152+
else:
153+
entry["runner"] = "palace_ubuntu-latest_16-core"
154+
155+
print(yaml.dump(matrix, default_flow_style=False, sort_keys=False))

0 commit comments

Comments
 (0)