Skip to content

Commit 9735354

Browse files
zhangxinyuehfadzhangxinyuehfad
committed
[Test] Refactor accuracy test to nightly test
Signed-off-by: hfadzxy <[email protected]>
1 parent 4312a92 commit 9735354

File tree

4 files changed

+232
-77
lines changed

4 files changed

+232
-77
lines changed

‎.github/workflows/_e2e_nightly_single_node_accuracy.yaml‎

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
#
2+
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# This file is a part of the vllm-ascend project.
16+
#
17+
18+
name: 'e2e nightly accuracy test'
19+
20+
on:
21+
workflow_call:
22+
inputs:
23+
vllm:
24+
required: true
25+
type: string
26+
runner:
27+
required: true
28+
type: string
29+
image:
30+
required: false
31+
type: string
32+
default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-910b-ubuntu22.04-py3.11"
33+
model:
34+
required: true
35+
type: string
36+
37+
# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
38+
# declared as "shell: bash -el {0}" on steps that need to be properly activated.
39+
# It's used to activate ascend-toolkit environment variables.
40+
defaults:
41+
run:
42+
shell: bash -el {0}
43+
44+
# only cancel in-progress runs of the same workflow
45+
# and ignore the lint / 1 card / 4 cards test type
46+
concurrency:
47+
group: ${{ github.workflow }}-${{ github.ref }}
48+
cancel-in-progress: true
49+
50+
jobs:
51+
e2e-nightly:
52+
name: e2e-nightly
53+
runs-on: ${{ inputs.runner }}
54+
container:
55+
image: ${{ inputs.image }}
56+
env:
57+
VLLM_USE_MODELSCOPE: True
58+
steps:
59+
- name: Check npu and CANN info
60+
run: |
61+
npu-smi info
62+
cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
63+
64+
- name: Config mirrors
65+
run: |
66+
sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' /etc/apt/sources.list
67+
pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple
68+
apt-get update -y
69+
apt install git -y
70+
git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf https://github.com/
71+
72+
- name: Checkout vllm-project/vllm-ascend repo
73+
uses: actions/checkout@v4
74+
75+
- name: Install system dependencies
76+
run: |
77+
apt-get -y install `cat packages.txt`
78+
apt-get -y install gcc g++ cmake libnuma-dev
79+
80+
- name: Checkout vllm-project/vllm repo
81+
uses: actions/checkout@v4
82+
with:
83+
repository: vllm-project/vllm
84+
ref: ${{ inputs.vllm }}
85+
path: ./vllm-empty
86+
87+
- name: Install vllm-project/vllm from source
88+
working-directory: ./vllm-empty
89+
run: |
90+
VLLM_TARGET_DEVICE=empty pip install -e .
91+
92+
- name: Install vllm-project/vllm-ascend
93+
env:
94+
PIP_EXTRA_INDEX_URL: https://mirrors.huaweicloud.com/ascend/repos/pypi
95+
run: |
96+
pip install -r requirements-dev.txt
97+
pip install -v -e .
98+
99+
- name: Install Ascend toolkit & triton_ascend (for Qwen3-Next-80B-A3B-Instruct)
100+
if: ${{ inputs.model_name == 'Qwen3-Next-80B-A3B-Instruct' }}
101+
shell: bash -l {0}
102+
run: |
103+
wget -q https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/Ascend-BiSheng-toolkit_aarch64.run -O /tmp/Ascend-BiSheng-toolkit_aarch64.run
104+
chmod a+x /tmp/Ascend-BiSheng-toolkit_aarch64.run
105+
/tmp/Ascend-BiSheng-toolkit_aarch64.run --install
106+
. /usr/local/Ascend/8.3.RC1/bisheng_toolkit/set_env.sh
107+
python3 -m pip install "https://vllm-ascend.obs.cn-north-4.myhuaweicloud.com/vllm-ascend/triton_ascend-3.2.0.dev20250914-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl"
108+
109+
- name: Resolve vllm-ascend version
110+
run: |
111+
VERSION_INPUT="${{ inputs.vllm-ascend }}"
112+
113+
if [[ "$VERSION_INPUT" == "latest" ]]; then
114+
TAGS=$(git ls-remote --tags --sort=-v:refname https://github.com/vllm-project/vllm-ascend "v*" | cut -f2 | sed 's|refs/tags/||')
115+
LATEST_TAG=$(echo "$TAGS" | head -n1)
116+
if [[ -z "$LATEST_TAG" ]]; then
117+
RESOLVED_VERSION="main"
118+
else
119+
RESOLVED_VERSION="$LATEST_TAG"
120+
fi
121+
else
122+
RESOLVED_VERSION="$VERSION_INPUT"
123+
fi
124+
echo "GHA_VLLM_ASCEND_VERSION=$RESOLVED_VERSION" >> $GITHUB_ENV
125+
126+
- name: Checkout vllm-project/vllm-ascend repo
127+
uses: actions/checkout@v4
128+
with:
129+
repository: vllm-project/vllm-ascend
130+
path: ./vllm-ascend
131+
ref: ${{ env.GHA_VLLM_ASCEND_VERSION }}
132+
133+
- name: Install vllm-project/vllm-ascend
134+
working-directory: ./vllm-ascend
135+
env:
136+
PIP_EXTRA_INDEX_URL: https://mirrors.huaweicloud.com/ascend/repos/pypi
137+
run: |
138+
pip install -r requirements-dev.txt
139+
pip install -v -e .
140+
141+
- name: Get vLLM commit hash and URL
142+
working-directory: ./vllm-empty
143+
run: |
144+
VLLM_COMMIT=$(git rev-parse --short=7 HEAD)
145+
echo "VLLM_COMMIT=$VLLM_COMMIT" >> $GITHUB_ENV
146+
147+
- name: Get vLLM-Ascend commit hash and URL
148+
working-directory: ./vllm-ascend
149+
run: |
150+
VLLM_ASCEND_COMMIT=$(git rev-parse --short=7 HEAD)
151+
echo "VLLM_ASCEND_COMMIT=$VLLM_ASCEND_COMMIT" >> $GITHUB_ENV
152+
153+
- name: Collect version info
154+
run: |
155+
for dir in /usr/local/Ascend/ascend-toolkit/*; do
156+
dname=$(basename "$dir")
157+
if [ "$dname" != "latest" ]; then
158+
TOOLKIT_DIR="$dname"
159+
break
160+
fi
161+
done
162+
INFO_FILE="/usr/local/Ascend/ascend-toolkit/${TOOLKIT_DIR}/$(uname -i)-linux/ascend_toolkit_install.info"
163+
GHA_CANN_VERSION=$(grep "version=" "$INFO_FILE" \
164+
| head -n1 \
165+
| cut -d'=' -f2 \
166+
| tr -d '"')
167+
{
168+
echo "GHA_CANN_VERSION=$GHA_CANN_VERSION"
169+
pip show torch | grep "Version:" | awk '{print "GHA_TORCH_VERSION="$2}'
170+
pip show torch_npu | grep "Version:" | awk '{print "GHA_TORCH_NPU_VERSION="$2}'
171+
pip show vllm | grep "Version:" | awk '{print "GHA_VLLM_VERSION="$2}' | sed 's/+.*//'
172+
} >> "$GITHUB_ENV"
173+
174+
- name: Run vllm-project/vllm-ascend accuracy test
175+
env:
176+
VLLM_WORKER_MULTIPROC_METHOD: spawn
177+
VLLM_USE_MODELSCOPE: True
178+
VLLM_CI_RUNNER: ${{ inputs.runner }}
179+
VLLM_VERSION: ${{ env.GHA_VLLM_VERSION }}
180+
VLLM_COMMIT: ${{ env.VLLM_COMMIT }}
181+
VLLM_ASCEND_VERSION: ${{ env.GHA_VLLM_ASCEND_VERSION || github.ref }}
182+
VLLM_ASCEND_COMMIT: ${{ env.VLLM_ASCEND_COMMIT }}
183+
CANN_VERSION: ${{ env.GHA_CANN_VERSION }}
184+
TORCH_VERSION: ${{ env.GHA_TORCH_VERSION }}
185+
TORCH_NPU_VERSION: ${{ env.GHA_TORCH_NPU_VERSION }}
186+
run: |
187+
model_base_name=$(basename ${{ inputs.model_name }})
188+
markdown_name="${model_base_name}"
189+
echo "markdown_name=$markdown_name" >> $GITHUB_OUTPUT
190+
mkdir -p ./benchmarks/accuracy
191+
pytest -sv ./tests/e2e/models/test_lm_eval_correctness.py \
192+
--config ./tests/e2e/models/configs/${{ inputs.model }}.yaml
193+
194+
- name: Generate step summary
195+
if: ${{ always() }}
196+
run: |
197+
cat ./benchmarks/accuracy/${{ steps.report.outputs.markdown_name }}.md >> $GITHUB_STEP_SUMMARY
198+
199+
- name: Upload Report
200+
if: ${{ inputs.upload == true }}
201+
uses: actions/upload-artifact@v4
202+
with:
203+
name: "report-${{ env.GHA_VLLM_ASCEND_VERSION }}-${{ steps.report.outputs.markdown_name }}"
204+
path: ./benchmarks/accuracy/${{ steps.report.outputs.markdown_name }}.md
205+
if-no-files-found: warn
206+
retention-days: 90
207+
overwrite: true

‎.github/workflows/accuracy_test.yaml‎

Lines changed: 0 additions & 74 deletions
This file was deleted.

‎.github/workflows/vllm_ascend_test_models.yaml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,6 @@
2323
name: ascend test / models
2424

2525
on:
26-
schedule:
27-
# Runs every 6 hours
28-
- cron: '0 */6 * * *'
2926
pull_request:
3027
branches:
3128
- 'main'

‎.github/workflows/vllm_ascend_test_nightly_a2.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,28 @@ jobs:
5858
vllm: v0.11.0
5959
runner: ${{ matrix.test_config.os }}
6060
tests: ${{ matrix.test_config.tests }}
61+
62+
single-node-accuracy-tests:
63+
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
64+
strategy:
65+
fail-fast: false
66+
matrix:
67+
- name: Qwen3-8B
68+
os: linux-aarch64-a2-1
69+
- name: Qwen2.5-VL-7B-Instruct
70+
os: linux-aarch64-a2-1
71+
- name: Qwen2-Audio-7B-Instruct
72+
os: linux-aarch64-a2-1
73+
- name: Qwen3-30B-A3B
74+
os: linux-aarch64-a2-2
75+
- name: Qwen3-VL-30B-A3B-Instruct
76+
os: linux-aarch64-a2-2
77+
- name: DeepSeek-V2-Lite
78+
os: linux-aarch64-a2-2
79+
- name: Qwen3-Next-80B-A3B-Instruct
80+
os: linux-aarch64-a2-4
81+
uses: ./.github/workflows/_e2e_nightly_single_node.yaml
82+
with:
83+
vllm: v0.11.0
84+
runner: ${{ matrix.test_config.os }}
85+
model: ${{ matrix.test_config.name }}

0 commit comments

Comments
 (0)