Auto-generated baselines by 1ES Pipeline Templates #8620
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CUDA Plugin Windows CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - rel-* | |
| pull_request: | |
| branches: | |
| - main | |
| - rel-* | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| build: | |
| name: Windows CUDA Plugin EP Build | |
| runs-on: [ | |
| "self-hosted", | |
| "1ES.Pool=onnxruntime-github-vs2022-latest", | |
| "JobId=windows-cuda-plugin-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" | |
| ] | |
| timeout-minutes: 210 | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| with: | |
| fetch-depth: 0 | |
| submodules: 'none' | |
| - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: '3.14' | |
| architecture: x64 | |
| - name: Locate vcvarsall and Setup Env | |
| uses: ./.github/actions/locate-vcvarsall-and-setup-env | |
| with: | |
| architecture: x64 | |
| - name: Install Visual Studio Spectre-mitigated libs | |
| uses: ./.github/actions/install-vs-spectre-libs | |
| with: | |
| architecture: x64 | |
| - name: Install python modules | |
| run: python -m pip install -r .\tools\ci_build\github\windows\python\requirements.txt | |
| working-directory: ${{ github.workspace }} | |
| shell: cmd | |
| - name: Download CUDA SDK v12.8 | |
| working-directory: ${{ runner.temp }} | |
| run: | | |
| azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" . | |
| dir | |
| shell: pwsh | |
| - name: Add CUDA to PATH | |
| shell: powershell | |
| run: | | |
| Write-Host "Adding CUDA to PATH" | |
| Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.8\bin" | |
| Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\bin" | |
| Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\extras\CUPTI\lib64" | |
| - name: Set OnnxRuntimeBuildDirectory | |
| shell: pwsh | |
| run: | | |
| $buildDir = Join-Path ${{ runner.temp }} "build" | |
| echo "OnnxRuntimeBuildDirectory=$buildDir" >> $env:GITHUB_ENV | |
| - name: Build ONNX Runtime with CUDA Plugin EP | |
| working-directory: ${{ runner.temp }} | |
| run: | | |
| python.exe ${{ github.workspace }}\tools\ci_build\build.py ` | |
| --update --build --config Release ` | |
| --build_dir build ` | |
| --skip_submodule_sync ` | |
| --parallel ` | |
| --nvcc_threads 4 ` | |
| --flash_nvcc_threads 4 ` | |
| --use_binskim_compliant_compile_flags ` | |
| --cmake_generator "Visual Studio 17 2022" ` | |
| --build_shared_lib ` | |
| --build_wheel ` | |
| --use_cuda ` | |
| --cuda_home="$env:RUNNER_TEMP\v12.8" ` | |
| --skip_tests ` | |
| --use_vcpkg ` | |
| --use_vcpkg_ms_internal_asset_cache ` | |
| --enable_cuda_profiling ` | |
| --cmake_extra_defines onnxruntime_QUICK_BUILD=ON ` | |
| --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 ` | |
| --cmake_extra_defines onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=ON | |
| if ($lastExitCode -ne 0) { | |
| exit $lastExitCode | |
| } | |
| # Clean up intermediate files before uploading artifacts | |
| $outputDir = "${{ runner.temp }}\build\Release" | |
| Write-Host "Cleaning up files from $outputDir..." | |
| Remove-Item -Path "$outputDir\onnxruntime" -Recurse -Force -ErrorAction SilentlyContinue | |
| Remove-Item -Path "$outputDir\pybind11" -Recurse -Force -ErrorAction SilentlyContinue | |
| Remove-Item -Path "$outputDir\models" -Recurse -Force -ErrorAction SilentlyContinue | |
| Remove-Item -Path "$outputDir\vcpkg_installed" -Recurse -Force -ErrorAction SilentlyContinue | |
| Remove-Item -Path "$outputDir\_deps" -Recurse -Force -ErrorAction SilentlyContinue | |
| Remove-Item -Path "$outputDir\CMakeCache.txt" -Force -ErrorAction SilentlyContinue | |
| Remove-Item -Path "$outputDir\CMakeFiles" -Recurse -Force -ErrorAction SilentlyContinue | |
| Remove-Item -Path $outputDir -Include "*.obj" -Recurse | |
| shell: pwsh | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: cuda-plugin-build-artifacts | |
| path: ${{ runner.temp }}\build | |
| env: | |
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true | |
| setVcvars: true | |
| ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' | |
| ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' | |
| AZCOPY_AUTO_LOGIN_TYPE: MSI | |
| AZCOPY_MSI_CLIENT_ID: d712a4c7-a0cf-4e87-af75-31510eba0a8e | |
| test: | |
| name: Windows CUDA Plugin EP Test | |
| needs: build | |
| timeout-minutes: 120 | |
| runs-on: [ | |
| "self-hosted", | |
| "1ES.Pool=onnxruntime-github-Win2022-GPU-A10", | |
| "JobId=windows-cuda-plugin-test-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" | |
| ] | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 | |
| with: | |
| fetch-depth: 0 | |
| submodules: 'none' | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: cuda-plugin-build-artifacts | |
| path: ${{ runner.temp }}\build | |
| - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: '3.14' | |
| architecture: x64 | |
| - name: Locate vcvarsall and Setup Env | |
| uses: ./.github/actions/locate-vcvarsall-and-setup-env | |
| with: | |
| architecture: x64 | |
| - name: Install Visual Studio Spectre-mitigated libs | |
| uses: ./.github/actions/install-vs-spectre-libs | |
| with: | |
| architecture: x64 | |
| - name: Install python modules | |
| run: python -m pip install -r .\tools\ci_build\github\windows\python\requirements.txt | |
| working-directory: ${{ github.workspace }} | |
| shell: cmd | |
| - name: Install torch for CPU only | |
| run: python -m pip install torch --index-url https://download.pytorch.org/whl/cpu | |
| working-directory: ${{ github.workspace }} | |
| shell: cmd | |
| - name: Download CUDA SDK v12.8 | |
| working-directory: ${{ runner.temp }} | |
| run: | | |
| azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.8" . | |
| dir | |
| shell: pwsh | |
| - name: Add CUDA to PATH | |
| shell: powershell | |
| run: | | |
| Write-Host "Adding CUDA to PATH" | |
| Write-Host "CUDA Path: $env:RUNNER_TEMP\v12.8\bin" | |
| Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\bin" | |
| Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v12.8\extras\CUPTI\lib64" | |
| - name: Set OnnxRuntimeBuildDirectory | |
| shell: pwsh | |
| run: | | |
| $buildDir = Join-Path ${{ runner.temp }} "build" | |
| echo "OnnxRuntimeBuildDirectory=$buildDir" >> $env:GITHUB_ENV | |
| - name: Install ONNX Runtime Wheel | |
| uses: ./.github/actions/install-onnxruntime-wheel | |
| with: | |
| whl-directory: ${{ runner.temp }}\build\Release\Release\dist | |
| # Verify the GPU is accessible before running the full test suite. | |
| # If the NVIDIA driver is not available, tests will fail with | |
| # "CUDA failure 100" and waste significant time. | |
| - name: Verify GPU access | |
| shell: pwsh | |
| run: nvidia-smi | |
| - name: Run Tests | |
| working-directory: ${{ runner.temp }}\build\Release\Release | |
| shell: pwsh | |
| run: | | |
| $env:ORT_UNIT_TEST_MAIN_DYNAMIC_PLUGIN_EP_CONFIG_JSON = '{"ep_library_registration_name":"CUDAExecutionProvider","ep_library_path":"onnxruntime_providers_cuda.dll","selected_ep_name":"CUDAExecutionProvider","tests_to_skip":["BeamSearchTest.*","ContribOpAttentionTest.AttentionPastState_dynamic","ContribOpAttentionTest.Attention_Mask1D_Fp32_B2_S64","ContribOpAttentionTest.Attention_Mask2D_Fp32_B2_S32","ConvTest.Conv2D_MatMul_*","ConvTest.Conv3D_2","CudaPluginArenaTest.DeviceAllocator_ArenaReusesMemory","CudaPluginArenaTest.DeviceAllocator_CudaMemoryIsValid","CudaPluginArenaTest.DeviceAllocator_IsArena","CudaPluginArenaTest.DeviceAllocator_MultipleAllocations","DeformConvTest.Group1OffsetGroup2","GRUTest.*","GreedySearchTest.*","InferenceSessionTests.Test2LayerNestedSubgraph","InferenceSessionTests.Test3LayerNestedSubgraph","InferenceSessionTests.TestArenaShrinkageAfterRun","LSTMTest.*","LogSoftmaxOperator.*","LongformerAttentionTest.*","LoraAdapterTest.VerifyDeviceCopy","PackedAttentionTest.*","PlannerTest.MultiStream","PlannerTest.MultiStream1StreamWaitFor2Streams","PlannerTest.MultiStream2NodesSameStreamConsumedBy1NodeInDifferentStream","PlannerTest.MultiStreamCudaEPNodeCPUOutput","PlannerTest.MultiStreamMultiOutput","RNNTest.*","ReductionOpTest.ArgMax2D_select_last","ReductionOpTest.ArgMin_do_not_keepdims_2_select_last","SamplingTest.*","Scan.MixedExecutionProvidersUnknownDimInSubgraphOutput","Scan8.*","SessionStateTest.TestLayeringPartitioning","SessionStateTest.TestResourceAwarePartitioning_LargeLimit","SessionStateTest.TestResourceAwarePartitioning_NoLimit","SkipGroupNormTest.*","TopKOperator.NthElementBFloat16_NegativeVals"]}' | |
| .\onnxruntime_test_all.exe | |
| if ($lastExitCode -ne 0) { | |
| exit $lastExitCode | |
| } | |
| .\onnxruntime_provider_test.exe | |
| if ($lastExitCode -ne 0) { | |
| exit $lastExitCode | |
| } | |
| - name: Run CUDA Plugin EP Python Tests | |
| working-directory: ${{ github.workspace }}\onnxruntime\test\python\transformers | |
| shell: pwsh | |
| run: | | |
| $env:ORT_TEST_CUDA_PLUGIN_EP = "1" | |
| python test_cuda_plugin_ep.py | |
| if ($lastExitCode -ne 0) { | |
| exit $lastExitCode | |
| } | |
| env: | |
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true | |
| setVcvars: true | |
| ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' | |
| ONNXRUNTIME_TEST_GPU_DEVICE_ID: '0' | |
| AZCOPY_AUTO_LOGIN_TYPE: MSI | |
| AZCOPY_MSI_CLIENT_ID: d712a4c7-a0cf-4e87-af75-31510eba0a8e |