Skip to content

Commit 5ef1da4

Browse files
giovtorresgiovtorres
committed
test: update test suite
1 parent 1455c7b commit 5ef1da4

File tree

10 files changed

+221
-92
lines changed

10 files changed

+221
-92
lines changed

‎.github/workflows/pyslurm.yml‎

Lines changed: 116 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,126 @@
11
name: PySlurm
2-
on: [push, pull_request]
2+
3+
on:
4+
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
8+
workflow_dispatch:
9+
310
jobs:
4-
Build:
5-
runs-on: ubuntu-20.04
11+
unit-tests:
12+
runs-on: ubuntu-latest
613
strategy:
714
matrix:
8-
python-version:
9-
- "3.6"
10-
- "3.7"
11-
- "3.8"
12-
- "3.9"
15+
python-version: ["3.12"]
1316
fail-fast: false
1417
steps:
15-
- name: Checkout repository code
16-
uses: actions/checkout@v2
17-
- name: Set up Python ${{ matrix.python-version }}
18-
uses: actions/setup-python@v2
19-
with:
20-
python-version: ${{ matrix.python-version }}
21-
- name: Install Dependencies
18+
- name: Checkout repository
19+
uses: actions/checkout@v4
20+
21+
- name: Pull Slurm container
22+
run: docker pull giovtorres/slurm-docker:25.11.2
23+
24+
- name: Start Slurm container
25+
run: docker compose -f docker-compose-github.yml up -d
26+
27+
- name: Wait for Slurm services
2228
run: |
23-
which python
24-
which python${{ matrix.python-version }}
25-
python${{ matrix.python-version }} -m pip install --upgrade pip
26-
python${{ matrix.python-verison }} -m pip install pipenv
27-
pipenv --python ${{matrix.python-version }} install --dev --skip-lock
28-
- name: Start Slurm Container
29-
uses: sudo-bot/action-docker-compose@latest
30-
with:
31-
cli-args: "-f docker-compose-github.yml up -d"
32-
env:
33-
PYTHON: ${{ matrix.python-version }}
34-
- name: Debug
29+
echo "Waiting for Slurm controller..."
30+
for i in $(seq 1 30); do
31+
if docker exec slurmctl scontrol ping 2>/dev/null | grep -q "UP"; then
32+
echo "Slurm controller is ready"
33+
break
34+
fi
35+
echo "Attempt $i/30..."
36+
sleep 2
37+
done
38+
39+
echo "Waiting for compute nodes..."
40+
for i in $(seq 1 30); do
41+
if docker exec slurmctl sinfo -h -o "%T %D" 2>/dev/null | grep -q "idle"; then
42+
echo "Compute nodes are ready"
43+
break
44+
fi
45+
echo "Attempt $i/30..."
46+
sleep 2
47+
done
48+
docker exec slurmctl sinfo
49+
50+
- name: Build and install PySlurm
3551
run: |
36-
docker ps -a
37-
docker version
38-
pwd
39-
ls -al
40-
docker inspect slurmctl
41-
env | sort
42-
- name: Build/Install/Test PySlurm
52+
docker exec slurmctl bash -c "
53+
python3 -m venv /opt/pyslurm-venv
54+
source /opt/pyslurm-venv/bin/activate
55+
pip install -r /pyslurm/test_requirements.txt
56+
cd /pyslurm
57+
python setup.py build -j4
58+
python setup.py install
59+
"
60+
61+
- name: Run unit tests
4362
run: |
44-
pipenv run pytest -sv scripts/run_tests_in_container.py
45-
env:
46-
PYTHON: ${{ matrix.python-version }}
63+
docker exec slurmctl bash -c "
64+
source /opt/pyslurm-venv/bin/activate
65+
cd /pyslurm
66+
pytest tests/unit -v
67+
"
4768
69+
integration-tests:
70+
runs-on: ubuntu-latest
71+
needs: unit-tests
72+
strategy:
73+
matrix:
74+
python-version: ["3.12"]
75+
fail-fast: false
76+
steps:
77+
- name: Checkout repository
78+
uses: actions/checkout@v4
79+
80+
- name: Pull Slurm container
81+
run: docker pull giovtorres/slurm-docker:25.11.2
82+
83+
- name: Start Slurm container
84+
run: docker compose -f docker-compose-github.yml up -d
85+
86+
- name: Wait for Slurm services
87+
run: |
88+
echo "Waiting for Slurm controller..."
89+
for i in $(seq 1 30); do
90+
if docker exec slurmctl scontrol ping 2>/dev/null | grep -q "UP"; then
91+
echo "Slurm controller is ready"
92+
break
93+
fi
94+
echo "Attempt $i/30..."
95+
sleep 2
96+
done
97+
98+
echo "Waiting for compute nodes..."
99+
for i in $(seq 1 30); do
100+
if docker exec slurmctl sinfo -h -o "%T %D" 2>/dev/null | grep -q "idle"; then
101+
echo "Compute nodes are ready"
102+
break
103+
fi
104+
echo "Attempt $i/30..."
105+
sleep 2
106+
done
107+
docker exec slurmctl sinfo
108+
109+
- name: Build and install PySlurm
110+
run: |
111+
docker exec slurmctl bash -c "
112+
python3 -m venv /opt/pyslurm-venv
113+
source /opt/pyslurm-venv/bin/activate
114+
pip install -r /pyslurm/test_requirements.txt
115+
cd /pyslurm
116+
python setup.py build -j4
117+
python setup.py install
118+
"
119+
120+
- name: Run integration tests
121+
run: |
122+
docker exec slurmctl bash -c "
123+
source /opt/pyslurm-venv/bin/activate
124+
cd /pyslurm
125+
pytest tests/integration -v
126+
"

‎docker-compose-github.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
version: "3.8"
2-
31
services:
42
slurm:
5-
image: giovtorres/docker-centos7-slurm:21.08.6
3+
image: giovtorres/slurm-docker:25.11.2
64
hostname: slurmctl
75
container_name: slurmctl
6+
privileged: true
87
stdin_open: true
98
tty: true
109
working_dir: /pyslurm
1110
environment:
12-
PYTHON: "3.9"
11+
MYSQL_USER: slurm
12+
MYSQL_PASSWORD: password
1313
volumes:
14-
- ${RUNNER_WORKSPACE}/pyslurm:/pyslurm
14+
- ${GITHUB_WORKSPACE}:/pyslurm:z

‎docker-compose.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
1-
version: "3.8"
2-
31
services:
42
slurm:
5-
image: giovtorres/docker-centos7-slurm:21.08.0
3+
image: giovtorres/slurm-docker:25.11.2
64
hostname: slurmctl
75
container_name: slurmctl
6+
privileged: true
87
stdin_open: true
98
tty: true
109
working_dir: /pyslurm
1110
environment:
12-
PYTHON: "3.9"
11+
MYSQL_USER: slurm
12+
MYSQL_PASSWORD: password
1313
volumes:
14-
- ${PWD}:/pyslurm
14+
- ${PWD}:/pyslurm:z

‎scripts/run_integration_tests.sh‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/bin/bash
2+
set -e
3+
4+
CONTAINER_NAME="${CONTAINER_NAME:-slurmctl}"
5+
VENV_PATH="/opt/pyslurm-venv"
6+
BUILD_JOBS="${PYSLURM_BUILD_JOBS:-4}"
7+
8+
usage() {
9+
echo "Usage: $0 [-c container_name] [-j build_jobs] [-s]"
10+
echo " -c Container name (default: slurmctl)"
11+
echo " -j Parallel build jobs (default: 4)"
12+
echo " -s Skip build step (reuse existing install)"
13+
exit 1
14+
}
15+
16+
SKIP_BUILD=false
17+
18+
while getopts ":c:j:sh" o; do
19+
case "${o}" in
20+
c) CONTAINER_NAME="${OPTARG}" ;;
21+
j) BUILD_JOBS="${OPTARG}" ;;
22+
s) SKIP_BUILD=true ;;
23+
h) usage ;;
24+
*) usage ;;
25+
esac
26+
done
27+
28+
# Check that the container is running
29+
if ! docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" &>/dev/null; then
30+
echo "Error: Container '$CONTAINER_NAME' is not running."
31+
echo "Start it with: docker compose up -d"
32+
exit 1
33+
fi
34+
35+
# Wait for slurmctld to be ready
36+
echo "Waiting for Slurm controller..."
37+
for i in $(seq 1 30); do
38+
if docker exec "$CONTAINER_NAME" scontrol ping 2>/dev/null | grep -q "UP"; then
39+
echo "Slurm controller is ready."
40+
break
41+
fi
42+
if [ "$i" -eq 30 ]; then
43+
echo "Error: Slurm controller did not become ready in time."
44+
exit 1
45+
fi
46+
sleep 2
47+
done
48+
49+
# Wait for compute nodes to register and become idle
50+
echo "Waiting for compute nodes..."
51+
for i in $(seq 1 30); do
52+
node_info=$(docker exec "$CONTAINER_NAME" sinfo -h -o "%T %D" 2>/dev/null)
53+
if echo "$node_info" | grep -q "idle"; then
54+
idle_count=$(echo "$node_info" | awk '/idle/{print $2}')
55+
echo "Compute nodes ready ($idle_count idle)."
56+
break
57+
fi
58+
if [ "$i" -eq 30 ]; then
59+
echo "Error: Compute nodes did not become ready in time."
60+
docker exec "$CONTAINER_NAME" sinfo
61+
exit 1
62+
fi
63+
sleep 2
64+
done
65+
66+
if [ "$SKIP_BUILD" = false ]; then
67+
echo "Building and installing PySlurm..."
68+
docker exec "$CONTAINER_NAME" bash -c "
69+
python3 -m venv $VENV_PATH 2>/dev/null || true
70+
source $VENV_PATH/bin/activate
71+
pip install -q -r /pyslurm/test_requirements.txt
72+
cd /pyslurm
73+
python setup.py build -j$BUILD_JOBS
74+
python setup.py install
75+
"
76+
fi
77+
78+
echo "Running integration tests..."
79+
docker exec "$CONTAINER_NAME" bash -c "
80+
source $VENV_PATH/bin/activate
81+
cd /pyslurm
82+
pytest tests/integration -v \"\$@\"
83+
" -- "$@"

‎test_requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
pytest==7.1.2
1+
pytest>=8.0
22
setuptools>=59.2.0
33
wheel>=0.37.0
44
Cython>=0.29.37

‎tests/integration/test_job.py‎

Lines changed: 3 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
JobSubmitDescription,
3333
RPCError,
3434
)
35-
from pyslurm.db import JobStatistics, JobStepStatistics
3635

3736

3837
def test_parse_all(submit_job):
@@ -55,7 +54,7 @@ def test_load(submit_job):
5554

5655
assert job.id == jid
5756
assert job.ntasks == 2
58-
assert job.cpus_per_task == 3
57+
assert job.cpus_per_task == 1
5958
assert job.time_limit == 1440
6059

6160
with pytest.raises(RPCError):
@@ -158,8 +157,8 @@ def test_get_job_queue(submit_job):
158157

159158

160159
def test_load_steps(submit_job):
161-
job_list = [submit_job() for i in range(3)]
162-
util.wait()
160+
job_list = [submit_job() for i in range(2)]
161+
util.wait(10)
163162

164163
jobs = Jobs.load()
165164
jobs.load_steps()
@@ -172,30 +171,6 @@ def test_load_steps(submit_job):
172171
assert job.steps.get("batch")
173172

174173

175-
def test_load_stats(submit_job):
176-
job = submit_job()
177-
util.wait(100)
178-
179-
job = Job.load(job.id)
180-
job.load_stats()
181-
182-
assert job.state == "RUNNING"
183-
assert job.stats
184-
assert isinstance(job.stats, JobStatistics)
185-
assert job.stats.elapsed_cpu_time > 0
186-
assert job.stats.resident_memory > 0
187-
assert job.stats.disk_read > 0
188-
assert job.stats.disk_write > 0
189-
190-
for step in job.steps.values():
191-
assert step.stats
192-
assert step.state == "RUNNING"
193-
assert isinstance(step.stats, JobStepStatistics)
194-
assert step.stats.avg_resident_memory > 0
195-
assert step.stats.avg_disk_read > 0
196-
assert step.stats.avg_disk_write > 0
197-
assert step.stats.elapsed_cpu_time > 0
198-
199174

200175
def test_to_json(submit_job):
201176
job_list = [submit_job() for i in range(3)]

‎tests/integration/test_job_steps.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@
3232

3333
def create_job_script_multi_step(steps=None):
3434
default = f"""
35-
srun -n1 -N1 -c2 \
35+
srun -n1 -N1 -c1 \
3636
-J step_zero --distribution=block:cyclic:block,Pack \
3737
sleep 300 &
38-
srun -n1 -N1 -c3 \
38+
srun -n1 -N1 -c1 \
3939
-t 10 -J step_one --distribution=block:cyclic:block,Pack \
4040
sleep 300 &"""
4141

@@ -87,14 +87,14 @@ def test_load(submit_job):
8787
assert step.job_id == job.id
8888
assert step.name == "step_zero"
8989
assert step.ntasks == 1
90-
assert step.cpus == 2
90+
assert step.cpus == 1
9191
assert step.time_limit is None
9292

9393
step = step_one
9494
assert step.job_id == job.id
9595
assert step.name == "step_one"
9696
assert step.ntasks == 1
97-
assert step.cpus == 3
97+
assert step.cpus == 1
9898
assert step.time_limit == 10
9999

100100

‎tests/integration/test_job_submit.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@ def test_submit_example1():
4242
desc.standard_output = "/tmp/test1.out"
4343
desc.standard_error = "/tmp/test1.err"
4444
desc.ntasks = 2
45-
desc.cpus_per_task = 2
45+
desc.cpus_per_task = 1
4646
desc.resource_sharing = "yes"
47-
desc.memory_per_cpu = "2G"
47+
desc.memory_per_cpu = "100M"
4848
desc.time_limit = 10
4949
desc.nice = 500
5050
desc.distribution = "block:block:cyclic"

0 commit comments

Comments
 (0)