Unit Tests #2097
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Unit Tests | |
| run-name: "${{ github.event.inputs.title }}" | |
| defaults: | |
| run: | |
| shell: bash -le {0} | |
| on: | |
| repository_dispatch: | |
| workflow_dispatch: | |
| inputs: | |
| title: | |
| description: 'set a title for this run' | |
| required: false | |
| default: '' | |
| repo: | |
| description: 'GitHub repo {owner}/{repo}' | |
| required: false | |
| default: '' | |
| ref: | |
| description: 'GitHub ref: Branch, Tag or Commit SHA' | |
| required: false | |
| default: '' | |
| pr_number: | |
| description: 'PR Number' | |
| required: false | |
| type: number | |
| test_regex: | |
| description: 'Regex to filter test files' | |
| required: false | |
| default: '' | |
| artifact_id: | |
| description: 'Run id for artifact to be downloaded' | |
| required: false | |
| default: '' | |
| max-parallel: | |
| description: 'Parallel jobs' | |
| required: false | |
| default: '4' | |
| model-test-mode: | |
| description: 'Model compat test mode' | |
| type: choice | |
| required: false | |
| default: 'fast' | |
| options: | |
| - 'fast' | |
| - 'slow' | |
| server: | |
| description: 'Wheel Build Server' | |
| type: choice | |
| default: '["self-hosted", "xeon5"]' | |
| options: | |
| - '["self-hosted", "xeon5"]' | |
| - '["self-hosted", "zen4"]' | |
| env: | |
| CUDA_DEVICE_ORDER: PCI_BUS_ID | |
| CUDA_VISIBLE_DEVICES: 0 | |
| # Force fresh torch.ops JIT builds in CI so heterogeneous GPUs never reuse | |
| # cached kernels compiled by earlier jobs/tests. | |
| GPTQMODEL_KERNEL_REBUILD: '1' | |
| # CI can allocate A100-class devices (sm_80), so the shared JIT arch list | |
| # must include 8.0 in addition to the newer targets. | |
| TORCH_CUDA_ARCH_LIST: '8.0 8.6 8.9 9.0 12.0' | |
| PYTORCH_ALLOC_CONF: 'expandable_segments:True' | |
| RUNNER: 10.0.13.31 | |
| XEON5: 10.0.14.249 | |
| GPU_ALLOCATOR_URL: http://10.0.13.31/gpu | |
| LOGBAR_ANIMATION: '0' | |
| CUDA_VERSION: 130 | |
| UV_TORCH_BACKEND: cu130 | |
| TORCH_VERSION: 2.11.0 | |
| # vllm doesn't support 3.14 | |
| PYTHON_VERSION: 3.13 | |
| UV_PYTHON: python3.13 | |
| # PYTHON_GIL: 0 // test libs don't support yet | |
| IGNORED_TEST_FILES: "test_tgi.py,test_gptneox.py,models/test_mixtral.py,models/test_phi_3_moe.py,test_bits_new.py,models/test_internlm.py,models/test_internlm2_5.py,models/test_xverse.py" | |
| MODEL_TEST_MODE: ${{ github.event.inputs['model-test-mode'] || 'fast' }} | |
| MODEL_TEST_GPU_COUNT: '1' | |
| DEFUSER_GIT_URL: git+https://github.com/modelcloud/Defuser.git | |
| PYPCRE_GIT_URL: git+https://github.com/modelcloud/PyPcre.git | |
| TOKENICER_GIT_URL: git+https://github.com/modelcloud/Tokenicer.git | |
| LOGBAR_GIT_URL: git+https://github.com/modelcloud/LogBar.git | |
| EVALUTION_GIT_URL: git+https://github.com/modelcloud/Evalution.git | |
| repo: ${{ github.event.inputs.repo || github.repository }} | |
| ref: ${{ github.event.inputs.ref || github.ref }} | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| concurrency: | |
| group: ${{ github.event.inputs.ref || github.ref }}-workflow-unit-tests #-${{ github.event.inputs.test_names }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| check-vm: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| ip: ${{ steps.get_ip.outputs.ip }} | |
| run_id: ${{ steps.get_ip.outputs.run_id }} | |
| max-parallel: ${{ steps.get_ip.outputs['max-parallel'] }} | |
| cuda_version: ${{ env.CUDA_VERSION }} | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ env.repo }} | |
| ref: ${{ env.ref }} | |
| - name: Print env | |
| run: | | |
| echo "repo: ${{ env.repo }}" | |
| echo "ref: ${{ env.ref }}" | |
| echo "artifact_id: ${{ github.event.inputs.artifact_id }}" | |
| echo "selected server: ${{ github.event.inputs.server }}" | |
| - name: Set server | |
| id: get_ip | |
| run: | | |
| bash .github/scripts/ci_write_runner_outputs.sh \ | |
| "$RUNNER" \ | |
| "${{ github.run_id }}" \ | |
| "${{ github.event.inputs.artifact_id }}" \ | |
| "${{ github.event.inputs['max-parallel'] }}" | |
| list-test-files: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| torch-files: ${{ steps.files.outputs.torch-files }} | |
| model-files: ${{ steps.files.outputs.model-files }} | |
| m4-files: ${{ steps.files.outputs.m4-files }} | |
| test-matrix: ${{ steps.files.outputs.test-matrix }} | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ env.repo }} | |
| ref: ${{ env.ref }} | |
| - name: Prepare checkout | |
| run: | | |
| .github/scripts/ci_prepare_checkout.sh "${{ github.event.inputs.pr_number }}" | |
| - name: List files | |
| id: files | |
| run: | | |
| test_plan=$(python3 .github/scripts/ci_workflow.py list-tests \ | |
| --ignored-test-files "$IGNORED_TEST_FILES" \ | |
| --test-regex "${{ github.event.inputs.test_regex }}") | |
| echo "Test plan: $test_plan" | |
| TEST_PLAN="$test_plan" python3 - <<'PY' >> "$GITHUB_OUTPUT" | |
| import json | |
| import os | |
| plan = json.loads(os.environ["TEST_PLAN"]) | |
| print(f"torch-files={json.dumps(plan['torch_files'])}") | |
| print(f"model-files={json.dumps(plan['model_files'])}") | |
| print(f"m4-files={json.dumps(plan['mlx_files'])}") | |
| print(f"test-matrix={json.dumps(plan['test_matrix'])}") | |
| PY | |
| TEST_PLAN="$test_plan" python3 - <<'PY' | |
| import json | |
| import os | |
| plan = json.loads(os.environ["TEST_PLAN"]) | |
| print(f"Torch Test files: {json.dumps(plan['torch_files'])}") | |
| print(f"Model Compat Test files: {json.dumps(plan['model_files'])}") | |
| print(f"MLX Test files: {json.dumps(plan['mlx_files'])}") | |
| print(f"Combined Test Matrix: {json.dumps(plan['test_matrix'])}") | |
| PY | |
| echo "Ignored Test files: $IGNORED_TEST_FILES" | |
| test: | |
| needs: | |
| - list-test-files | |
| - check-vm | |
| runs-on: [ self-hosted, xeon5 ] | |
| container: | |
| image: ${{ needs.check-vm.outputs.ip }}:5000/nvidia/cuda:${{ needs.check-vm.outputs.cuda_version }}-ubuntu24.04_0325 | |
| options: --device /dev/dri --ipc=host --runtime=nvidia --gpus all | |
| volumes: | |
| - /monster/ci/env/entrypoint.sh:/entrypoint.sh | |
| - /monster/ci/env/entrypoint.sh:/etc/profile.d/01-entrypoint.sh | |
| - /dev/dri/by-path:/dev/dri/by-path | |
| - /monster/ci/models:/monster/data/model | |
| - /monster/ci/dataset:/monster/data/model/dataset | |
| - /monster/ci/huggingface:/github/home/.cache/huggingface | |
| - /github/workspace/uv:/opt/uv | |
| - /github/workspace/tmp:/opt/uv/tmp | |
| - /monster/ci/uv/python:/opt/uv/python | |
| - /monster/ci/uv/cache/python:/opt/uv/cache/python | |
| - /monster/ci/uv/setup_uv_venv.sh:/opt/uv/setup_uv_venv.sh | |
| - /monster/ci/uv/uv:/opt/uv/uv | |
| - /monster/ci/uv/uvx:/opt/uv/uvx | |
| - /monster/ci/uv/env:/opt/uv/env | |
| - /monster/ci/uv/uv.toml:/opt/uv/uv.toml | |
| - /monster/ci/env:/opt/env | |
| - /monster/ci/dist:/opt/dist | |
| strategy: | |
| fail-fast: false | |
| max-parallel: ${{ fromJson(needs.check-vm.outputs['max-parallel']).size || 20 }} | |
| matrix: | |
| include: ${{ fromJSON(needs.list-test-files.outputs.test-matrix) }} | |
| if: always() && !cancelled() && needs.list-test-files.outputs.test-matrix != '[]' # || github.event.inputs.artifact_id != '' | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ env.repo }} | |
| ref: ${{ env.ref }} | |
| - name: Prepare checkout | |
| run: | | |
| .github/scripts/ci_prepare_checkout.sh "${{ github.event.inputs.pr_number }}" | |
| - name: Restore uv cache | |
| continue-on-error: true | |
| run: | | |
| .github/scripts/ci_restore_uv_cache.sh | |
| - name: Activate uv env | |
| run: | | |
| bash .github/scripts/ci_unit_activate_uv_env.sh \ | |
| "${{ matrix.test_script }}" \ | |
| "${{ needs.check-vm.outputs.cuda_version }}" \ | |
| "${{ env.TORCH_VERSION }}" \ | |
| "${{ github.run_id }}" \ | |
| "${{ github.run_attempt }}" | |
| - name: Setup uv env | |
| run: | | |
| bash .github/scripts/ci_unit_setup_uv_env.sh \ | |
| "${{ matrix.test_group }}" \ | |
| "${{ matrix.test_script }}" \ | |
| "${{ needs.check-vm.outputs.cuda_version }}" \ | |
| "${{ env.TORCH_VERSION }}" \ | |
| "${{ env.PYTHON_VERSION }}" | |
| - name: Install package from source | |
| run: | | |
| uv pip uninstall gptqmodel || true | |
| /opt/env/install_torch.sh ${{ needs.check-vm.outputs.cuda_version }} ${{ env.TORCH_VERSION }} | |
| uv pip install -r requirements.txt -i http://$RUNNER/simple/ --trusted-host $RUNNER --extra-index-url https://pypi.org/simple | |
| uv pip install . -i http://$RUNNER/simple/ --trusted-host $RUNNER --extra-index-url https://pypi.org/simple | |
| .github/scripts/ci_install_modelcloud_git_deps.sh | |
| /opt/env/install_torch.sh ${{ needs.check-vm.outputs.cuda_version }} ${{ env.TORCH_VERSION }} | |
| - name: Print uv env | |
| run: | | |
| echo "::group::uv python list" | |
| uv python list | |
| echo "::endgroup::" | |
| which python | |
| which pip || true | |
| python -V | |
| echo "::group::uv python list" | |
| uv python list | |
| echo "::endgroup::" | |
| echo "== nvcc ==" | |
| nvcc --version | |
| echo "::group::project files" | |
| ls -ahl | |
| echo "::endgroup::" | |
| echo "== torch ==" | |
| uv pip show torch | |
| echo "::group::pip list" | |
| uv pip list | |
| echo "::endgroup::" | |
| - name: Find suitable GPU | |
| if: ${{ !cancelled() && env.SKIP_GPU_ALLOCATION != 'true' }} | |
| run: | | |
| alloc_gpu_count="$GPU_COUNT" | |
| if [[ "${{ matrix.alloc_gpu_count }}" != "resolved" ]]; then | |
| alloc_gpu_count="${{ matrix.alloc_gpu_count }}" | |
| fi | |
| alloc_args=( | |
| --base-url "${GPU_ALLOCATOR_URL}" | |
| --run-id "${{ github.run_id }}" | |
| --test "${{ matrix.test_script }}" | |
| --runner "${RUNNER_NAME:-unknown}" | |
| --count "$alloc_gpu_count" | |
| --sm "${SM}" | |
| ) | |
| if [[ "${{ matrix.require_single_gpu }}" == "true" ]]; then | |
| alloc_args+=(--require-single) | |
| fi | |
| python .github/scripts/ci_gpu.py allocate \ | |
| "${alloc_args[@]}" | |
| - name: Run tests | |
| run: | | |
| extra_args=() | |
| if [[ "${XPU_MODE}" == "true" ]]; then | |
| extra_args+=(--clear-cuda --xpu-mode) | |
| fi | |
| if [[ "${{ matrix.include_model_test_mode }}" == "true" ]]; then | |
| extra_args+=(--model-test-mode "${MODEL_TEST_MODE}") | |
| fi | |
| python .github/scripts/ci_tests.py run \ | |
| --base-url "${GPU_ALLOCATOR_URL}" \ | |
| --run-id "${{ github.run_id }}" \ | |
| --test-script "${{ matrix.test_script }}" \ | |
| --runner "${RUNNER_NAME}" \ | |
| --gpu-id "${{ env.CUDA_VISIBLE_DEVICES }}" \ | |
| "${extra_args[@]}" | |
| - name: Check log | |
| if: ${{ !cancelled() && failure() }} | |
| continue-on-error: true | |
| run: | | |
| python .github/scripts/ci_tests.py check-log \ | |
| --run-id "${{ github.run_id }}" \ | |
| --test-script "${{ matrix.test_script }}" | |
| - name: Release GPU | |
| if: always() && env.SKIP_GPU_ALLOCATION != 'true' | |
| run: | | |
| python .github/scripts/ci_gpu.py release \ | |
| --base-url "${GPU_ALLOCATOR_URL}" \ | |
| --run-id "${{ github.run_id }}" \ | |
| --gpu-id "${{ env.CUDA_VISIBLE_DEVICES }}" \ | |
| --test "${{ matrix.test_script }}" \ | |
| --runner "${RUNNER_NAME}" | |
| - name: Clean cache | |
| if: always() | |
| run: | | |
| echo "Cleaning workspace: $PWD" | |
| rm -rf ./* .[^.] .??* || true | |
| echo "cleaning venv: ${{ env.VIRTUAL_ENV }}" | |
| rm -rf "${{ env.VIRTUAL_ENV }}" | |
| check-torch: | |
| runs-on: [ self-hosted, xeon5 ] | |
| container: | |
| image: 10.0.13.31:5000/nvidia/cuda:130-ubuntu24.04_0325 | |
| options: --device /dev/dri --ipc=host --runtime=nvidia --gpus all | |
| volumes: | |
| - /monster/ci/env/entrypoint.sh:/entrypoint.sh | |
| - /monster/ci/env/entrypoint.sh:/etc/profile.d/01-entrypoint.sh | |
| - /dev/dri/by-path:/dev/dri/by-path | |
| - /monster/ci/uv:/opt/uv | |
| - /monster/ci/env:/opt/env | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Test pypi pip | |
| env_name: pypi_pip_env | |
| use_pip: 'true' | |
| install_cmd: pip install gptqmodel torch -U | |
| - name: Test pypi uv | |
| env_name: pypi_uv_env | |
| use_pip: 'false' | |
| install_cmd: uv pip install gptqmodel torch -U | |
| - name: Test local pip | |
| env_name: local_pip_env | |
| use_pip: 'true' | |
| install_cmd: pip install . torch -U | |
| - name: Test local uv | |
| env_name: local_uv_env | |
| use_pip: 'false' | |
| install_cmd: uv pip install . torch -U | |
| steps: | |
| - name: Checkout Codes | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ env.repo }} | |
| ref: ${{ env.ref }} | |
| - name: Prepare checkout | |
| run: | | |
| .github/scripts/ci_prepare_checkout.sh "${{ github.event.inputs.pr_number }}" | |
| - name: ${{ matrix.name }} | |
| run: | | |
| uv venv "${{ matrix.env_name }}" | |
| source "${{ matrix.env_name }}/bin/activate" | |
| if [[ "${{ matrix.use_pip }}" == "true" ]]; then | |
| uv pip install pip -U | |
| fi | |
| ${{ matrix.install_cmd }} | |
| setuptools-compatibility: | |
| uses: ./.github/workflows/setuptools_compatibility_reusable.yml | |
| with: | |
| repo: ${{ github.event.inputs.repo || github.repository }} | |
| ref: ${{ github.event.inputs.ref || github.ref }} | |
| pr_number: ${{ fromJSON(github.event.inputs.pr_number || '0') }} |