nightly_ci_ascend #275

Workflow file for this run

.github/workflows/nightly_ascend.yml at 1dda039

	# # Tests layout

	# Each folder under tests/ corresponds to a test category for a sub-namespace in verl. For instance:
	# - `tests/trainer` for testing functionality related to `verl/trainer`
	# - `tests/models` for testing functionality related to `verl/models`
	# - ...

	# There are a few folders with `special_` prefix, created for special purposes:
	# - `special_distributed`: unit tests that must run with multiple GPUs
	# - `special_e2e`: end-to-end tests with training/generation scripts
	# - `special_npu`: tests for NPUs
	# - `special_sanity`: a suite of quick sanity tests
	# - `special_standalone`: a set of test that are designed to run in dedicated environments

	# Accelerators for tests
	# - By default tests are run with GPU available, except for the ones under `special_npu`, and any test script whose name ends with `on_cpu.py`.
	# - For test scripts with `on_cpu.py` name suffix would be tested on CPU resources in linux environment.

	# # Workflow layout

	# All CI tests are configured by yaml files in `.github/workflows/`. Here's an overview of all test configs:
	# 1. A list of always triggered CPU sanity tests: `check-pr-title.yml`, `secrets_scan.yml`, `check-pr-title,yml`, `pre-commit.yml`, `doc.yml`
	# 2. Some heavy multi-GPU unit tests, such as `model.yml`, `vllm.yml`, `sgl.yml`
	# 3. End-to-end tests: `e2e_*.yml`
	# 4. Unit tests
	# - `cpu_unit_tests.yml`, run pytest on all scripts with file name pattern `tests/*/test__on_cpu.py`
	# - `gpu_unit_tests.yml`, run pytest on all scripts with file without the `on_cpu.py` suffix.
	# - Since cpu/gpu unit tests by default runs all tests under `tests`, please make sure tests are manually excluded in them when
	# - new workflow yaml is added to `.github/workflows`
	# - new tests are added to workflow mentioned in 2.

	name: nightly_ci_ascend

	on:
	# Trigger the workflow on push or pull request,
	# but only for the main branch
	# For push, for now only anti-patterns are specified so it is more conservative
	# and achieves higher coverage.
	schedule:
	- cron: "0 17 * * *"
	- cron: "0 18 * * *"
	- cron: "0 19 * * *"

	# Declare permissions just read content.
	permissions:
	contents: read

	jobs:
	# Test ppo qwen3-8b fsdp vllm
	nightlyCI_ppo-qwen3-8b-fsdp-vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 17 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:latest-vllm-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Install the current repository
	run: \|
	pip install --no-deps -e .
	- name: Check final pip list
	run: \|
	pip list
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Prepare GSM8K dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: Running nightlyCI_ppo-qwen3-8b-fsdp-vllm_ascend
	run: \|
	ray stop --force
	bash tests/special_npu/nightly_ci_ascend/run_ppo_qwen3-8b_fsdp_npu.sh
	- name: Running nightlyCI_ppo-qwen3-8b-fsdp-vllm_ascend checking script
	run: \|
	cd /root/.cache/nightly_log/
	python check_npu.py --log run_ppo_qwen3-8b_fsdp_npu/run_ppo_qwen3-8b_fsdp_npu.log --base run_ppo_qwen3-8b_fsdp_npu/baseline_ppo_qwen3-8b_fsdp_npu.txt

	# Test grpo qwen3_vl_8b_Instruct fsdp2 vllm
	nightlyCI_grpo_qwen3_vl_8b_Instruct_fsdp2_vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 17 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:latest-vllm-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Install the current repository
	run: \|
	pip install --no-deps -e .
	- name: Check final pip list
	run: \|
	pip list
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Preprocess geo3k dataset
	run: \|
	python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k
	- name: Running nightlyCI_grpo_qwen3_vl_8b_Instruct_fsdp2_vllm_ascend
	run: \|
	ray stop --force
	bash tests/special_npu/run_qwen3_vl_8b_Instruct_fsdp2_npu.sh
	- name: Running nightlyCI_grpo_qwen3_vl_8b_Instruct_fsdp2_vllm_ascend checking script
	run: \|
	cd /root/.cache/nightly_log/
	python check_npu.py --log run_qwen3_vl_8b_Instruct_fsdp2_npu/run_qwen3_vl_8b_Instruct_fsdp2_npu.log --base run_qwen3_vl_8b_Instruct_fsdp2_npu/baseline_qwen3_vl_8b_Instruct_fsdp2_npu.txt

	# Test dapo moonlight-16b megatron vllm
	nightlyCI_dapo-moonlight-16b-megatron-vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 17 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/verl/verl:latest-vllm-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	submodules: recursive
	clean: true
	- name: Install the current repository
	run: \|
	pip install --no-deps -e .
	- name: Check final pip list
	run: \|
	pip list
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Preprocess geo3k dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: update mbridge
	run: \|
	# get mbridge path
	MBRIDGE_PATH=$(pip show mbridge \| grep Location \| awk '{print $2}')
	# cuda to npu
	TARGET_FILE="${MBRIDGE_PATH}/mbridge/models/ext/deepseek_v3/dequant_fp8_safetensor_io.py"
	sed -i '34s/cuda/npu/;51s/cuda/npu/' "$TARGET_FILE"
	- name: Running nightlyCI_dapo-moonlight-16b-megatron-vllm_ascend
	run: \|
	ray stop --force
	cd recipe
	git checkout main
	cd ..
	export HCCL_OP_EXPANSION_MODE="AIV"
	bash tests/special_npu/nightly_ci_ascend/run_dapo_moonlight-16b_megatron_npu.sh
	- name: Running nightlyCI_dapo-moonlight-16b-megatron-vllm_ascend checking script
	run: \|
	cd /root/.cache/nightly_log/
	python check_npu.py --log run_dapo_moonlight-16b_megatron_npu/run_dapo_moonlight-16b_megatron_npu.log --base run_dapo_moonlight-16b_megatron_npu/baseline_dapo_moonlight-16b_megatron_npu.txt


	# Test gspo qwen3-30b megatron vllm
	nightlyCI_gspo-qwen3-30b-megatron-vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 18 * * *' }}
	runs-on: linux-aarch64-a3-16
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/verl/verl:latest-vllm-a3-ubuntu
	options: >-
	--shm-size 60g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	UV_EXTRA_INDEX_URL: "https://repo.huaweicloud.com/ascend/repos/pypi"
	UV_INDEX_STRATEGY: unsafe-best-match
	UV_INSECURE_HOST: cache-service.nginx-pypi-cache.svc.cluster.local
	UV_HTTP_TIMEOUT: 120
	UV_NO_CACHE: "1"
	UV_SYSTEM_PYTHON: "1"
	UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
	PIP_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
	PIP_TRUSTED_HOST: "cache-service.nginx-pypi-cache.svc.cluster.local"
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Install uv
	run: pip install uv
	- name: Check initial pip list from image
	run: \|
	uv pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	submodules: recursive
	clean: true
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Preprocess GSM8K dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: Running nightlyCI_gspo-qwen3-30b-megatron-vllm_ascend
	run: \|
	ray stop --force
	bash tests/special_npu/nightly_ci_ascend/run_gspo_qwen3_30b_megatron_npu.sh
	- name: Running nightlyCI_gspo-qwen3-30b-megatron-vllm_ascend checking script
	run: \|
	cd /root/.cache/nightly_log/
	python check_npu.py --log run_gspo_qwen3_30b_megatron_npu/run_gspo_qwen3_30b_megatron_npu.log --base run_gspo_qwen3_30b_megatron_npu/baseline_gspo_qwen3_30b_megatron_npu.txt

	# Test grpo qwen3-30b megatron sglang
	nightlyCI_grpo-qwen3-30b-megatron-sglang_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 17 * * *' }}
	runs-on: linux-aarch64-a3-16
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/verl/verl:latest-sglang-a3-ubuntu
	options: >-
	--shm-size 60g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	UV_EXTRA_INDEX_URL: "https://repo.huaweicloud.com/ascend/repos/pypi"
	UV_INDEX_STRATEGY: unsafe-best-match
	UV_INSECURE_HOST: cache-service.nginx-pypi-cache.svc.cluster.local
	UV_HTTP_TIMEOUT: 120
	UV_NO_CACHE: "1"
	UV_SYSTEM_PYTHON: "1"
	UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
	PIP_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
	PIP_TRUSTED_HOST: "cache-service.nginx-pypi-cache.svc.cluster.local"
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Install uv
	run: pip install uv
	- name: Check initial pip list from image
	run: \|
	uv pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	submodules: recursive
	clean: true
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Running nightlyCI_grpo-qwen3-30b-megatron-sglang_ascend
	run: \|
	ray stop --force
	bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen3_30b_megatron_sglang_npu.sh
	- name: Running nightlyCI_gspo-qwen3-30b-megatron-vllm_ascend checking script
	run: \|
	cd /root/.cache/nightly_log/
	python check_npu.py --log run_grpo_qwen3_30b_megatron_sglang_npu/run_grpo_qwen3_30b_megatron_sglang_npu.log --base run_grpo_qwen3_30b_megatron_sglang_npu/baseline_grpo_qwen3_30b_megatron_sglang_npu.txt

	# Test grpo qwen3_5_2b fsdp2 vllm
	nightlyCI_grpo_qwen3_5_2b_fsdp2_vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 18 * * *' }}
	runs-on: linux-aarch64-a3-16
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:latest-vllm-a3-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Install the current repository
	run: \|
	pip install -v -e .
	- name: Check final pip list
	run: \|
	pip list
	- name: Clone Megatron Bridge
	run: \|
	git clone --depth 1 https://github.com/NVIDIA-NeMo/Megatron-Bridge.git /Megatron-Bridge
	cd /Megatron-Bridge
	git fetch --depth 1 origin de93536e9028ecf1e4dc28608dc80f336dcdfe59
	git checkout de93536e9028ecf1e4dc28608dc80f336dcdfe59
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Preprocess geo3k dataset
	run: \|
	python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k
	- name: Running nightlyCI_grpo_qwen3_5_2b_fsdp2_vllm_ascend
	run: \|
	ray stop --force
	export PYTHONPATH=/Megatron-Bridge/src:$PYTHONPATH
	bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen3_5_2b_fsdp2_npu.sh

	# Test grpo qwen3_5_2b fsdp_turbo vllm
	nightlyCI_grpo_qwen3_5_2b_fsdp_turbo_vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 19 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:latest-vllm-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Install the current repository
	run: \|
	pip install --no-deps -e .
	- name: Clone FSDPTurbo
	run: \|
	git clone https://gitcode.com/Ascend/FSDPTurbo.git /FSDPTurbo
	- name: Check final pip list
	run: \|
	pip list
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Preprocess geo3k dataset
	run: \|
	python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k
	- name: Running nightlyCI_grpo_qwen3_5_2b_fsdp_turbo_vllm_ascend
	run: \|
	ray stop --force
	export PYTHONPATH=/FSDPTurbo:$PYTHONPATH
	bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen3_5_2b_fsdp_turbo_npu.sh

	# Test quick start vllm fsdp2
	quick_start_qwen3_0_6b_fsdp2_vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 18 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:latest-vllm-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Install the current repository
	run: \|
	pip install --no-deps -e .
	- name: Check final pip list
	run: \|
	pip list
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Prepare GSM8K dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: Running quick_start_qwen3_0_6b_fsdp2_vllm_ascend
	run: \|
	ray stop --force
	TOTAL_TRAINING_STEPS=1 bash tests/special_npu/quick_start/run_qwen3_0_6b_fsdp2_vllm_ascend.sh

	# Test quick start vllm megatron
	quick_start_qwen3_0_6b_megatron_vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 18 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:latest-vllm-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Prepare GSM8K dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: Running quick_start_qwen3_0_6b_megatron_vllm_ascend
	run: \|
	ray stop --force
	TOTAL_TRAINING_STEPS=1 bash tests/special_npu/quick_start/run_qwen3_0_6b_megatron_vllm_ascend.sh

	# Test quick start sglang megatron
	quick_start_qwen3_0_6b_megatron_sglang_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 18 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/verl/verl:latest-sglang-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Prepare GSM8K dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: Running quick_start_qwen3_0_6b_megatron_sglang_ascend
	run: \|
	ray stop --force
	TOTAL_TRAINING_STEPS=1 bash tests/special_npu/quick_start/run_qwen3_0_6b_megatron_sglang_ascend.sh

	# Test quick start sglang fsdp2
	quick_start_qwen3_0_6b_fsdp2_sglang_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 18 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/verl/verl:latest-sglang-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Prepare GSM8K dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: Running quick_start_qwen3_0_6b_fsdp2_sglang_ascend
	run: \|
	ray stop --force
	TOTAL_TRAINING_STEPS=1 bash tests/special_npu/quick_start/run_qwen3_0_6b_fsdp2_sglang_ascend.sh

	# Test grpo qwen3-30b veomni fsdp
	nightlyCI_grpo-qwen3-30b-veomni-fsdp_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 19 * * *' }}
	runs-on: linux-aarch64-a3-16
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/verl/verl:latest-vllm-a3-ubuntu
	options: >-
	--shm-size 60g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	UV_EXTRA_INDEX_URL: "https://repo.huaweicloud.com/ascend/repos/pypi"
	UV_INDEX_STRATEGY: unsafe-best-match
	UV_INSECURE_HOST: cache-service.nginx-pypi-cache.svc.cluster.local
	UV_HTTP_TIMEOUT: 120
	UV_NO_CACHE: "1"
	UV_SYSTEM_PYTHON: "1"
	UV_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
	PIP_INDEX_URL: "http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple"
	PIP_TRUSTED_HOST: "cache-service.nginx-pypi-cache.svc.cluster.local"
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Install uv
	run: pip install uv
	- name: Install the current repository
	run: \|
	uv pip install -v -e .
	pip install veomni==0.1.11 --ignore-requires-python --no-deps --index-url https://pypi.org/simple/
	uv pip install transformers==5.3.0
	- name: Check initial pip list from image
	run: \|
	uv pip list
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Prepare GSM8K dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: Running nightlyCI_grpo-qwen3-30b-veomni-fsdp_ascend
	run: \|
	ray stop --force
	bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen3_30b_veomni_fsdp.sh

	# Test gspo qwen3-8b fsdp vllm
	nightlyCI_gspo-qwen3-8b-fsdp2-vllm_ascend:
	if: ${{ github.repository_owner == 'verl-project' && github.event.schedule == '0 19 * * *' }}
	runs-on: linux-aarch64-a2b3-8
	timeout-minutes: 180 # Increase this timeout value as needed
	container:
	image: swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/ascend-ci/verl/verl:latest-vllm-910b-ubuntu
	options: >-
	--shm-size 16g
	env:
	HF_ENDPOINT: "https://hf-mirror.com"
	HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	TRANSFORMERS_VERBOSITY: 'error'
	steps:
	- name: Check npu and CANN info
	run: \|
	cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	npu-smi info
	- name: Check initial pip list from image
	run: \|
	pip list
	- name: Checkout verl-project/verl repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	clean: true
	- name: Check final pip list
	run: \|
	pip list
	- name: Prepare weights
	run: \|
	ln -s /root/.cache/models ~/models
	- name: Prepare GSM8K dataset
	run: \|
	python examples/data_preprocess/gsm8k.py --local_dataset_path ${HOME}/.cache/datasets/openai/gsm8k
	- name: Running nightlyCI_grpo-qwen3-8b-fsdp2-vllm_ascend
	run: \|
	ray stop --force
	bash tests/special_npu/nightly_ci_ascend/run_gspo_qwen3_8b_fsdp2_npu.sh
	- name: Running nightlyCI_grpo-qwen3-8b-fsdp2-vllm_ascend checking script
	run: \|
	cd /root/.cache/nightly_log/
	python check_npu.py --log run_gspo_qwen3_8b_fsdp2_npu/run_gspo_qwen3_8b_fsdp2_npu.log --base run_gspo_qwen3_8b_fsdp2_npu/baseline_gspo_qwen3_8b_fsdp2_npu.txt

	# Test grpo qwen33.5-35b megatron vllm
	# nightlyCI_grpo-qwen3_5-35b-megatron-vllm_ascend:
	# if: ${{ github.repository_owner == 'verl-project'}}
	# runs-on: linux-aarch64-a3-16
	# timeout-minutes: 180 # Increase this timeout value as needed
	# container:
	# image: swr.ap-southeast-1.myhuaweicloud.com/base_image/ascend-ci/verl/verl:latest-vllm-a3-ubuntu
	# options: >-
	# --shm-size 16g
	# env:
	# HF_ENDPOINT: "https://hf-mirror.com"
	# HF_HUB_ENABLE_HF_TRANSFER: "0" # This is more stable
	# steps:
	# - name: Check npu and CANN info
	# run: \|
	# cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
	# npu-smi info
	# - name: Check initial pip list from image
	# run: \|
	# pip list
	# - name: Checkout verl-project/verl repo
	# uses: actions/checkout@v4
	# with:
	# fetch-depth: 0
	# clean: true
	# - name: Check final pip list
	# run: \|
	# pip list
	# - name: Clone Megatron Bridge
	# run: \|
	# git clone --depth 1 https://github.com/NVIDIA-NeMo/Megatron-Bridge.git /Megatron-Bridge
	# cd /Megatron-Bridge
	# git fetch --depth 1 origin de93536e9028ecf1e4dc28608dc80f336dcdfe59
	# git checkout de93536e9028ecf1e4dc28608dc80f336dcdfe59
	# - name: Prepare weights
	# run: \|
	# ln -s /root/.cache/models ~/models
	# - name: Preprocess geo3k dataset
	# run: \|
	# python examples/data_preprocess/geo3k.py --local_dataset_path ${HOME}/.cache/datasets/hiyouga/geometry3k
	# - name: Running nightlyCI_grpo-qwen3_5-35b-megatron-vllm_ascend
	# run: \|
	# ray stop --force
	# export PYTHONPATH=/Megatron-Bridge/src:$PYTHONPATH
	# bash tests/special_npu/nightly_ci_ascend/run_grpo_qwen3_5_35b_megatron_npu.sh

Navigation Menu

Uh oh!

nightly_ci_ascend #275

Workflow file

nightly_ci_ascend #275

Uh oh!

Workflow file for this run