From 50893f11ac533ada0cdbe1a984c19ff1554ef731 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 12 Aug 2025 19:24:49 +0000 Subject: [PATCH 01/14] conglomerated changes to simplify how we use our venvs --- eng/pipelines/templates/steps/analyze.yml | 2 +- .../steps/build-extended-artifacts.yml | 3 ++- .../steps/build-package-artifacts.yml | 24 +++++++++-------- eng/pipelines/templates/steps/build-test.yml | 26 +++---------------- eng/pipelines/templates/steps/install-uv.yml | 26 +++++++++++++++++++ .../steps/release-candidate-steps.yml | 10 +++---- .../steps/seed-virtualenv-wheels.yml | 2 +- eng/pipelines/templates/steps/use-venv.yml | 2 ++ eng/scripts/Language-Settings.ps1 | 21 ++++++++++++--- eng/scripts/activate-venv.ps1 | 13 +++++++--- eng/scripts/create-venv.ps1 | 21 ++++++++++----- 11 files changed, 91 insertions(+), 59 deletions(-) create mode 100644 eng/pipelines/templates/steps/install-uv.yml diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index 0e670721a1a4..cf2d9c86a4a5 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -48,7 +48,7 @@ steps: Condition: succeededOrFailed() - script: | - python -m pip install "./tools/azure-sdk-tools[build]" -q -I + uv pip install "./tools/azure-sdk-tools[build]" sdk_find_invalid_versions --always-succeed --service=${{parameters.ServiceDirectory}} displayName: Find Invalid Versions condition: succeededOrFailed() diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 0505e7561997..6d5474a932d4 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -56,7 +56,8 @@ steps: - template: /eng/pipelines/templates/steps/use-venv.yml - script: | - python -m pip install -r eng/ci_tools.txt + which python + uv pip install -r eng/ci_tools.txt displayName: 'Prep Environment' - template: set-dev-build.yml diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 84c002a911b1..1a948006e3aa 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -88,18 +88,19 @@ steps: - template: /eng/pipelines/templates/steps/use-venv.yml parameters: VirtualEnvironmentName: "venv" - Activate: false Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - pwsh: | - $(VENV_ACTIVATION_SCRIPT) + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true which python - python -m pip install --force -r eng/ci_tools.txt + uv pip install -r eng/ci_tools.txt + if ($env:AGENT_OS -eq "Linux") { Write-Host "Installing release reqs" - python -m pip install -r eng/release_requirements.txt + uv pip install -r eng/release_requirements.txt } - python -m pip freeze --all + uv pip freeze displayName: 'Prep Environment' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) @@ -111,7 +112,6 @@ steps: condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - pwsh: | - $(VENV_ACTIVATION_SCRIPT) which python sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive displayName: 'Generate Packages' @@ -121,11 +121,13 @@ steps: CIBW_BUILD_VERBOSITY: 3 - pwsh: | - $(VENV_ACTIVATION_SCRIPT) - twine check $(Build.ArtifactStagingDirectory)/**/*.whl - twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz - displayName: 'Verify Readme' - condition: and(succeededOrFailed(), eq(variables['Agent.OS'], 'Linux')) + which python + sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive + displayName: 'Generate Packages' + condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + timeoutInMinutes: 80 + env: + CIBW_BUILD_VERBOSITY: 3 - ${{ parameters.BeforePublishSteps }} diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 69f42f70c165..649320f59df8 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -26,23 +26,15 @@ steps: versionSpec: '${{ parameters.PythonVersion }}' - template: /eng/pipelines/templates/steps/use-venv.yml - parameters: - Activate: false - template: set-dev-build.yml - pwsh: | - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } + Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - python -m pip install --force -r eng/ci_tools.txt - python -m pip freeze --all - Write-Host (Get-Command python).Source + uv pip install --force -r eng/ci_tools.txt + uv pip freeze displayName: 'Prep Environment' - template: /eng/common/testproxy/test-proxy-tool.yml @@ -75,12 +67,6 @@ steps: $env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id; $env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants; - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } Write-Host (Get-Command python).Source if ($env:TESTMARKARGUMENT) { @@ -104,12 +90,6 @@ steps: - ${{ else }}: - pwsh: | - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } Write-Host (Get-Command python).Source if ($env:TESTMARKARGUMENT) { diff --git a/eng/pipelines/templates/steps/install-uv.yml b/eng/pipelines/templates/steps/install-uv.yml new file mode 100644 index 000000000000..94089094eb44 --- /dev/null +++ b/eng/pipelines/templates/steps/install-uv.yml @@ -0,0 +1,26 @@ + +steps: + - task: Bash@3 + displayName: 'Install uv (Linux/macOS)' + inputs: + targetType: inline + script: | + curl -LsSf https://astral.sh/uv/install.sh | sh + condition: or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')) + + - task: Bash@3 + inputs: + targetType: inline + script: | + echo "##vso[task.prependpath]$HOME/.local/bin" + displayName: 'Prepend path for MacOS' + condition: eq(variables['Agent.OS'], 'Darwin') + + - task: PowerShell@2 + displayName: 'Install uv (Windows)' + inputs: + targetType: inline + script: | + iex (irm https://astral.sh/uv/install.ps1) + Write-Host "##vso[task.prependpath]C:\Users\cloudtest\.local\bin" + condition: eq(variables['Agent.OS'], 'Windows_NT') diff --git a/eng/pipelines/templates/steps/release-candidate-steps.yml b/eng/pipelines/templates/steps/release-candidate-steps.yml index 720058f3b888..c6d1f9a10d26 100644 --- a/eng/pipelines/templates/steps/release-candidate-steps.yml +++ b/eng/pipelines/templates/steps/release-candidate-steps.yml @@ -4,16 +4,13 @@ steps: versionSpec: $(PythonVersion) - template: /eng/pipelines/templates/steps/use-venv.yml - parameters: - Activate: false - pwsh: | - $(VENV_ACTIVATION_SCRIPT) + Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - python -m pip install -r $(Build.SourcesDirectory)/eng/ci_tools.txt - python -m pip freeze --all - Write-Host (Get-Command python).Source + uv pip install -r $(Build.SourcesDirectory)/eng/ci_tools.txt + uv pip freeze displayName: 'Install Dependencies' - template: /eng/common/testproxy/test-proxy-tool.yml @@ -21,7 +18,6 @@ steps: runProxy: false - pwsh: | - $(VENV_ACTIVATION_SCRIPT) python ./scripts/devops_tasks/dispatch_tox.py "$(TargetedPackages)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="Build" displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)' env: diff --git a/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml b/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml index a3a178ca7a9f..daf184b718c3 100644 --- a/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml +++ b/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml @@ -5,7 +5,7 @@ parameters: steps: - pwsh: | - python -m pip install virtualenv + uv pip install virtualenv displayName: Ensure virtualenv installed - pwsh: | diff --git a/eng/pipelines/templates/steps/use-venv.yml b/eng/pipelines/templates/steps/use-venv.yml index a25459c7a156..fc15efe6996a 100644 --- a/eng/pipelines/templates/steps/use-venv.yml +++ b/eng/pipelines/templates/steps/use-venv.yml @@ -10,6 +10,8 @@ parameters: default: succeeded() steps: + - template: install-uv.yml + - pwsh: | $(Build.SourcesDirectory)/eng/scripts/create-venv.ps1 ` -VenvName "${{ parameters.VirtualEnvironmentName }}" ` diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 70a59b6fdb1e..30af170f3d45 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -161,9 +161,18 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) $allPkgPropLines = $null try { - Push-Location $RepoRoot - $null = python -m pip install "./tools/azure-sdk-tools[build]" -q -I - $allPkgPropLines = python (Join-path eng scripts get_package_properties.py) -s $searchPath + # Use ‘uv pip install’ if uv is on PATH, otherwise fall back to python -m pip + if (Get-Command uv -ErrorAction SilentlyContinue) { + Write-Host "Using uv pip install" + $null = uv pip install "./tools/azure-sdk-tools[build]" + $freezeOutput = uv pip freeze + Write-Host "Pip freeze output: $freezeOutput" + } else { + Write-Host "Using python -m pip install" + $null = python -m pip install "./tools/azure-sdk-tools[build]" -q -I + } + + Write-Host "Running get_package_properties.py to retrieve package properties" } catch { @@ -412,7 +421,11 @@ function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseD { $ReleaseDate = Get-Date -Format "yyyy-MM-dd" } - python -m pip install "$RepoRoot/tools/azure-sdk-tools[build]" -q -I + if (Get-Command uv -ErrorAction SilentlyContinue) { + uv pip install "$RepoRoot/tools/azure-sdk-tools[build]" + } else { + python -m pip install "$RepoRoot/tools/azure-sdk-tools[build]" -q -I + } sdk_set_version --package-name $PackageName --new-version $Version ` --service $ServiceDirectory --release-date $ReleaseDate --replace-latest-entry-title $ReplaceLatestEntryTitle } diff --git a/eng/scripts/activate-venv.ps1 b/eng/scripts/activate-venv.ps1 index 9d192212052e..82a73b37731c 100644 --- a/eng/scripts/activate-venv.ps1 +++ b/eng/scripts/activate-venv.ps1 @@ -1,4 +1,4 @@ -<#! +<#! .SYNOPSIS Activates a virtual environment for a CI machine. Any further usages of "python" will utilize this virtual environment. @@ -9,7 +9,7 @@ When activating a virtual environment, only a few things are actually functional # 2. VIRTUAL_ENV = path to root of the virtual env # 3. VIRTUAL_ENV_PROMPT = the prompt that is displayed next to the CLI cursor when the virtual env is active # within a CI machine, we only need the PATH and VIRTUAL_ENV variables to be set. -# 4. (optional and inconsistently) _OLD_VIRTUAL_PATH = the PATH before the virtual env was activated. This is not set in this script. +# 4. (optional and inconsistently) _OLD_VIRTUAL_PATH = the PATH before the virtual env was activated. This is not set in this script. .PARAMETER VenvName The name of the virtual environment to activate. @@ -44,6 +44,11 @@ else { $env:PATH = "$venvBinPath`:$($env:PATH)" } -Write-Host "Activating virtual environment '$VenvName' at $venvPath via AzDO to the value '$($env:PATH)'" +Write-Host "Activating virtual environment '$VenvName' via VIRTUAL_ENV variable at $venvPath.'" Write-Host "##vso[task.setvariable variable=VIRTUAL_ENV]$($env:VIRTUAL_ENV)" -Write-Host "##vso[task.prependpath]$($env:PATH)" \ No newline at end of file + +Write-Host "Prepending path with $venvBinPath" +Write-Host "##vso[task.prependpath]$venvBinPath" + +Write-Host "Unset of PYTHONHOME" +Write-Host "##vso[task.setvariable variable=PYTHONHOME]" \ No newline at end of file diff --git a/eng/scripts/create-venv.ps1 b/eng/scripts/create-venv.ps1 index 2824f6bc161d..82def49a18c2 100644 --- a/eng/scripts/create-venv.ps1 +++ b/eng/scripts/create-venv.ps1 @@ -21,15 +21,22 @@ param( ) $venvPath = Join-Path $RepoRoot $VenvName + if (!(Test-Path $venvPath)) { $invokingPython = (Get-Command "python").Source - Write-Host "Creating virtual environment '$VenvName' using python located at '$invokingPython'." - python -m pip install virtualenv==20.25.1 - python -m virtualenv "$venvPath" - $pythonVersion = python --version - Write-Host "Virtual environment '$VenvName' created at directory path '$venvPath' utilizing python version $pythonVersion." - Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath" - Write-Host "##vso[task.setvariable variable=$($VenvName)_ACTIVATION_SCRIPT]if(`$IsWindows){. $venvPath/Scripts/Activate.ps1;}else {. $venvPath/bin/activate.ps1}" + + if (Get-Command uv -ErrorAction SilentlyContinue) { + Write-Host "Creating virtual environment '$VenvName' using uv." + uv venv $venvPath --verbose + } + else { + Write-Host "Creating virtual environment '$VenvName' using virtualenv and python located at '$invokingPython'." + python -m pip install virtualenv==20.25.1 + python -m virtualenv "$venvPath" + $pythonVersion = python --version + Write-Host "Virtual environment '$VenvName' created at directory path '$venvPath' utilizing python version $pythonVersion." + Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath" + } } else { Write-Host "Virtual environment '$VenvName' already exists. Skipping creation." From 3e3cbe82dbdc546ce7e483960be1a84f900b0054 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 12 Aug 2025 19:30:54 +0000 Subject: [PATCH 02/14] remove accidental change to Verify REadme --- .../templates/steps/build-package-artifacts.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 1a948006e3aa..2b5a8d74f94a 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -122,12 +122,10 @@ steps: - pwsh: | which python - sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive - displayName: 'Generate Packages' - condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - timeoutInMinutes: 80 - env: - CIBW_BUILD_VERBOSITY: 3 + twine check $(Build.ArtifactStagingDirectory)/**/*.whl + twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz + displayName: 'Verify Readme' + condition: and(succeededOrFailed(), eq(variables['Agent.OS'], 'Linux')) - ${{ parameters.BeforePublishSteps }} From 83021b0c60d8d387b17d50413512b94320033df4 Mon Sep 17 00:00:00 2001 From: Scott Beddall <45376673+scbedd@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:32:31 -0700 Subject: [PATCH 03/14] Update eng/pipelines/templates/steps/install-uv.yml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- eng/pipelines/templates/steps/install-uv.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/install-uv.yml b/eng/pipelines/templates/steps/install-uv.yml index 94089094eb44..9d2a070ddc8f 100644 --- a/eng/pipelines/templates/steps/install-uv.yml +++ b/eng/pipelines/templates/steps/install-uv.yml @@ -22,5 +22,5 @@ steps: targetType: inline script: | iex (irm https://astral.sh/uv/install.ps1) - Write-Host "##vso[task.prependpath]C:\Users\cloudtest\.local\bin" + Write-Host "##vso[task.prependpath]$env:USERPROFILE\.local\bin" condition: eq(variables['Agent.OS'], 'Windows_NT') From 1cc9be94e0bb96d82c0e95ecc2cd1d1e35c748ab Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 12 Aug 2025 19:34:19 +0000 Subject: [PATCH 04/14] fix bug where I accidentally removed code --- eng/scripts/Language-Settings.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 30af170f3d45..a1fd500691ca 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -173,6 +173,7 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) } Write-Host "Running get_package_properties.py to retrieve package properties" + $allPkgPropLines = python (Join-path eng scripts get_package_properties.py) -s $searchPath } catch { From af1804a239cf658dd7d41bae1fec660984f68528 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 12 Aug 2025 19:37:35 +0000 Subject: [PATCH 05/14] ensure we always set this variable --- eng/scripts/create-venv.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/scripts/create-venv.ps1 b/eng/scripts/create-venv.ps1 index 82def49a18c2..1faee2584194 100644 --- a/eng/scripts/create-venv.ps1 +++ b/eng/scripts/create-venv.ps1 @@ -33,10 +33,10 @@ if (!(Test-Path $venvPath)) { Write-Host "Creating virtual environment '$VenvName' using virtualenv and python located at '$invokingPython'." python -m pip install virtualenv==20.25.1 python -m virtualenv "$venvPath" - $pythonVersion = python --version - Write-Host "Virtual environment '$VenvName' created at directory path '$venvPath' utilizing python version $pythonVersion." - Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath" } + $pythonVersion = python --version + Write-Host "Virtual environment '$VenvName' created at directory path '$venvPath' utilizing python version $pythonVersion." + Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath" } else { Write-Host "Virtual environment '$VenvName' already exists. Skipping creation." From 7414d0fb55adba6560d7176ed938246436831a53 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 12 Aug 2025 19:44:53 +0000 Subject: [PATCH 06/14] more pipeline optimizations --- eng/pipelines/templates/jobs/ci.yml | 1 + .../templates/steps/build-package-artifacts.yml | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 8716a14f30de..adef8a50fd54 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -275,6 +275,7 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '3.12' + - template: /eng/pipelines/templates/steps/use-venv.yml - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 2b5a8d74f94a..e12b8a77ff29 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -31,6 +31,11 @@ steps: inputs: versionSpec: $(PythonVersion) + - template: /eng/pipelines/templates/steps/use-venv.yml + parameters: + VirtualEnvironmentName: "venv" + Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml@self parameters: PackageName: "azure-template" @@ -85,11 +90,6 @@ steps: parameters: PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - - template: /eng/pipelines/templates/steps/use-venv.yml - parameters: - VirtualEnvironmentName: "venv" - Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - - pwsh: | $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true From bd7188bd8523bfc1ece1a5ac07865f0ee174abc2 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Tue, 12 Aug 2025 20:59:15 +0000 Subject: [PATCH 07/14] ensure we always activate the venv --- eng/pipelines/templates/steps/build-package-artifacts.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index e12b8a77ff29..f93e1440ab51 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -34,7 +34,6 @@ steps: - template: /eng/pipelines/templates/steps/use-venv.yml parameters: VirtualEnvironmentName: "venv" - Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml@self parameters: From b6f7bafc3aa311d02bf41bed45c1ef339b248b1c Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 13 Aug 2025 00:54:14 +0000 Subject: [PATCH 08/14] pip -> uv pip where applicable. move use-venv template --- eng/pipelines/templates/jobs/ci.yml | 5 ++++- eng/pipelines/templates/steps/analyze.yml | 2 ++ eng/pipelines/templates/steps/build-extended-artifacts.yml | 4 ++-- eng/pipelines/templates/steps/build-test.yml | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index adef8a50fd54..171f0f59d299 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -311,8 +311,11 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '3.12' + - template: /eng/pipelines/templates/steps/use-venv.yml - pwsh: | - python -m pip install "./tools/azure-sdk-tools[build]" + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + uv pip install "./tools/azure-sdk-tools[build]" displayName: 'Prep Environment' - task: PythonScript@0 displayName: 'Ensure service coverage' diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index cf2d9c86a4a5..cd75298ae08b 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -10,6 +10,8 @@ parameters: # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: + - template: /eng/pipelines/templates/steps/use-venv.yml + - template: /eng/pipelines/templates/steps/analyze_dependency.yml parameters: ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 6d5474a932d4..b62cc337aa2c 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -28,6 +28,8 @@ steps: inputs: versionSpec: '3.13' + - template: /eng/pipelines/templates/steps/use-venv.yml + - pwsh: | if ("${{ parameters.ServiceDirectory }}" -ne "auto") { @@ -53,8 +55,6 @@ steps: displayName: 'Tag scheduled builds' condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule')) - - template: /eng/pipelines/templates/steps/use-venv.yml - - script: | which python uv pip install -r eng/ci_tools.txt diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 649320f59df8..b279750fac7e 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -33,7 +33,7 @@ steps: Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - uv pip install --force -r eng/ci_tools.txt + uv pip install -r eng/ci_tools.txt uv pip freeze displayName: 'Prep Environment' From 241c18456635a92cacdb7ebd750a04674bbc86e4 Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 13 Aug 2025 01:23:22 +0000 Subject: [PATCH 09/14] further small adjustments --- eng/pipelines/templates/jobs/ci.yml | 10 ++++++++- eng/pipelines/templates/steps/analyze.yml | 4 +--- .../templates/steps/analyze_dependency.yml | 8 +------ .../steps/build-extended-artifacts.yml | 2 +- .../steps/build-package-artifacts.yml | 8 +++---- eng/pipelines/templates/steps/build-test.yml | 16 ++------------ .../steps/release-candidate-steps.yml | 4 ++-- eng/pipelines/templates/steps/run_apistub.yml | 10 +++++++-- .../steps/seed-virtualenv-wheels.yml | 2 +- .../templates/steps/set-dev-build.yml | 2 +- eng/pipelines/templates/steps/use-venv.yml | 21 +++++++++++++++++-- 11 files changed, 48 insertions(+), 39 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 171f0f59d299..d4bf2a521872 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -209,6 +209,14 @@ jobs: os: linux steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + condition: succeededOrFailed() + inputs: + versionSpec: '$(PythonVersion)' + + - template: /eng/pipelines/templates/steps/use-venv.yml + - template: /eng/common/pipelines/templates/steps/check-spelling.yml parameters: ContinueOnError: false @@ -315,7 +323,7 @@ jobs: - pwsh: | $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - uv pip install "./tools/azure-sdk-tools[build]" + $(PIP_EXE) install "./tools/azure-sdk-tools[build]" displayName: 'Prep Environment' - task: PythonScript@0 displayName: 'Ensure service coverage' diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index cd75298ae08b..6ab18778c379 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -10,8 +10,6 @@ parameters: # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. steps: - - template: /eng/pipelines/templates/steps/use-venv.yml - - template: /eng/pipelines/templates/steps/analyze_dependency.yml parameters: ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} @@ -50,7 +48,7 @@ steps: Condition: succeededOrFailed() - script: | - uv pip install "./tools/azure-sdk-tools[build]" + $(PIP_EXE) install "./tools/azure-sdk-tools[build]" sdk_find_invalid_versions --always-succeed --service=${{parameters.ServiceDirectory}} displayName: Find Invalid Versions condition: succeededOrFailed() diff --git a/eng/pipelines/templates/steps/analyze_dependency.yml b/eng/pipelines/templates/steps/analyze_dependency.yml index 37da2dce5aef..4ed795ab7f7a 100644 --- a/eng/pipelines/templates/steps/analyze_dependency.yml +++ b/eng/pipelines/templates/steps/analyze_dependency.yml @@ -2,14 +2,8 @@ parameters: ScanPath: '' steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' - condition: succeededOrFailed() - inputs: - versionSpec: '$(PythonVersion)' - - pwsh: | - python -m pip install -r eng/ci_tools.txt + $(PIP_EXE) install -r eng/ci_tools.txt displayName: 'Install Python Tools' condition: succeededOrFailed() diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index b62cc337aa2c..05d923b93030 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -57,7 +57,7 @@ steps: - script: | which python - uv pip install -r eng/ci_tools.txt + $(PIP_EXE) install -r eng/ci_tools.txt displayName: 'Prep Environment' - template: set-dev-build.yml diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index f93e1440ab51..cbe172654914 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -32,8 +32,6 @@ steps: versionSpec: $(PythonVersion) - template: /eng/pipelines/templates/steps/use-venv.yml - parameters: - VirtualEnvironmentName: "venv" - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml@self parameters: @@ -93,13 +91,13 @@ steps: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true which python - uv pip install -r eng/ci_tools.txt + $(PIP_EXE) install -r eng/ci_tools.txt if ($env:AGENT_OS -eq "Linux") { Write-Host "Installing release reqs" - uv pip install -r eng/release_requirements.txt + $(PIP_EXE) install -r eng/release_requirements.txt } - uv pip freeze + $(PIP_EXE) freeze displayName: 'Prep Environment' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index b279750fac7e..b005841838f9 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -33,8 +33,8 @@ steps: Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - uv pip install -r eng/ci_tools.txt - uv pip freeze + $(PIP_EXE) install -r eng/ci_tools.txt + $(PIP_EXE) freeze displayName: 'Prep Environment' - template: /eng/common/testproxy/test-proxy-tool.yml @@ -125,12 +125,6 @@ steps: - ${{ parameters.AfterTestSteps }} - pwsh: | - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } Write-Host (Get-Command python).Source python scripts/devops_tasks/create_coverage.py @@ -156,12 +150,6 @@ steps: $env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id; $env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants; - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } Write-Host (Get-Command python).Source python scripts/devops_tasks/dispatch_tox.py "$(TargetingString)" ` diff --git a/eng/pipelines/templates/steps/release-candidate-steps.yml b/eng/pipelines/templates/steps/release-candidate-steps.yml index c6d1f9a10d26..b5c2632a04ae 100644 --- a/eng/pipelines/templates/steps/release-candidate-steps.yml +++ b/eng/pipelines/templates/steps/release-candidate-steps.yml @@ -9,8 +9,8 @@ steps: Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - uv pip install -r $(Build.SourcesDirectory)/eng/ci_tools.txt - uv pip freeze + $(PIP_EXE) install -r $(Build.SourcesDirectory)/eng/ci_tools.txt + $(PIP_EXE) freeze displayName: 'Install Dependencies' - template: /eng/common/testproxy/test-proxy-tool.yml diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index 7928b0d22f74..ea84514f3ba0 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -16,8 +16,14 @@ steps: versionSpec: '3.10' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) - - script: | - python -m pip install -r eng/ci_tools.txt + - template: /eng/pipelines/templates/steps/use-venv.yml + parameters: + VirtualEnvironmentName: "venv-api-stub" + + - pwsh: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $(PIP_EXE) install -r eng/ci_tools.txt displayName: 'Prep Environment' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) diff --git a/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml b/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml index daf184b718c3..6aefa40ad4a5 100644 --- a/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml +++ b/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml @@ -5,7 +5,7 @@ parameters: steps: - pwsh: | - uv pip install virtualenv + $(PIP_EXE) install virtualenv displayName: Ensure virtualenv installed - pwsh: | diff --git a/eng/pipelines/templates/steps/set-dev-build.yml b/eng/pipelines/templates/steps/set-dev-build.yml index 26233110f3c8..74203bc0a263 100644 --- a/eng/pipelines/templates/steps/set-dev-build.yml +++ b/eng/pipelines/templates/steps/set-dev-build.yml @@ -9,7 +9,7 @@ steps: - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml - pwsh: | - python -m pip install "tools/azure-sdk-tools[build]" + $(PIP_EXE) install "tools/azure-sdk-tools[build]" sdk_set_dev_version "*" --build-id="$(Build.BuildNumber)" displayName: "Update package versions for dev build" condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), ${{ parameters.Condition }}) diff --git a/eng/pipelines/templates/steps/use-venv.yml b/eng/pipelines/templates/steps/use-venv.yml index fc15efe6996a..e8fff095a645 100644 --- a/eng/pipelines/templates/steps/use-venv.yml +++ b/eng/pipelines/templates/steps/use-venv.yml @@ -8,9 +8,26 @@ parameters: - name: Condition type: string default: succeeded() + - name: UseUv + type: boolean + default: true steps: - - template: install-uv.yml + - ${{ if eq(parameters.UseUv, true) }}: + - template: install-uv.yml + + - pwsh: | + if (Get-Command 'uv' -ErrorAction=SilentlyContinue) + Write-Host "##vso[task.setvariable variable=PIP_EXE]uv pip" + } else { + Write-Error "Expected 'uv' command to be available, but it was not. Exiting with error." + exit 1 + } + displayName: Setting PIP_EXE to 'uv pip' + - ${{ else }}: + - pwsh: | + Write-Host "##vso[task.setvariable variable=PIP_EXE]python -m pip" + displayName: Setting PIP_EXE to 'python -m pip' - pwsh: | $(Build.SourcesDirectory)/eng/scripts/create-venv.ps1 ` @@ -25,4 +42,4 @@ steps: -VenvName "${{ parameters.VirtualEnvironmentName }}" ` -RepoRoot "$(Build.SourcesDirectory)" displayName: Use ${{ parameters.VirtualEnvironmentName }} Virtual Environment - condition: ${{ parameters.Condition }} + condition: ${{ parameters.Condition }} \ No newline at end of file From b5ec51237279e1d876b677fab78e791e2bc533ff Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 13 Aug 2025 01:32:01 +0000 Subject: [PATCH 10/14] fix the template --- eng/pipelines/templates/jobs/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index d4bf2a521872..8d49bb9903ad 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -213,7 +213,7 @@ jobs: displayName: 'Use Python $(PythonVersion)' condition: succeededOrFailed() inputs: - versionSpec: '$(PythonVersion)' + versionSpec: '$(PythonVersion)' - template: /eng/pipelines/templates/steps/use-venv.yml From a4c208f35cdbd219e68504c1e702fac1725d5fdf Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 13 Aug 2025 01:59:40 +0000 Subject: [PATCH 11/14] ensure we have a trailing } --- eng/pipelines/templates/steps/use-venv.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/use-venv.yml b/eng/pipelines/templates/steps/use-venv.yml index e8fff095a645..a5cafbff33a7 100644 --- a/eng/pipelines/templates/steps/use-venv.yml +++ b/eng/pipelines/templates/steps/use-venv.yml @@ -17,7 +17,7 @@ steps: - template: install-uv.yml - pwsh: | - if (Get-Command 'uv' -ErrorAction=SilentlyContinue) + if (Get-Command 'uv' -ErrorAction=SilentlyContinue) { Write-Host "##vso[task.setvariable variable=PIP_EXE]uv pip" } else { Write-Error "Expected 'uv' command to be available, but it was not. Exiting with error." From 436ef1b77f50c2dd9392f1cbeed29176de76f1de Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 13 Aug 2025 02:06:58 +0000 Subject: [PATCH 12/14] update how we set ERrorAction --- eng/pipelines/templates/steps/use-venv.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/use-venv.yml b/eng/pipelines/templates/steps/use-venv.yml index a5cafbff33a7..158a7e4a6e4e 100644 --- a/eng/pipelines/templates/steps/use-venv.yml +++ b/eng/pipelines/templates/steps/use-venv.yml @@ -17,7 +17,7 @@ steps: - template: install-uv.yml - pwsh: | - if (Get-Command 'uv' -ErrorAction=SilentlyContinue) { + if (Get-Command uv -ErrorAction SilentlyContinue) { Write-Host "##vso[task.setvariable variable=PIP_EXE]uv pip" } else { Write-Error "Expected 'uv' command to be available, but it was not. Exiting with error." From 0bc2d891e3ec706cd85fd515702e620c5ff794aa Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 13 Aug 2025 17:25:36 +0000 Subject: [PATCH 13/14] going to go with moving SetTestPipeline into build-test so I can totally remove a UsePython3.10 + install build tooling + SetTestPipeline + SetToxTargets. none of those are python 3.10 specific and can be moved to run on the same python version that is running in the build --- eng/pipelines/templates/jobs/ci.tests.yml | 23 -------------------- eng/pipelines/templates/steps/build-test.yml | 16 ++++++++++++++ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 8d7460e08ca5..487ab6745965 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -90,29 +90,6 @@ jobs: parameters: AgentImage: ${{ parameters.OSName }} - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: "azure-template" - ServiceDirectory: "template" - TestPipeline: ${{ parameters.TestPipeline }} - - - pwsh: | - python -m pip install "tools/azure-sdk-tools[build]" - displayName: Install build tooling - - - task: PythonScript@0 - displayName: 'Set Tox Environment' - inputs: - scriptPath: 'scripts/devops_tasks/set_tox_environment.py' - arguments: >- - --unsupported="$(UnsupportedToxEnvironments)" - --override="$(Run.ToxCustomEnvs)" - --team-project="$(System.TeamProject)" - - template: ../steps/build-test.yml parameters: TestMarkArgument: ${{ parameters.TestMarkArgument }} diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index b005841838f9..ea98e9b5676b 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -17,6 +17,7 @@ parameters: TestProxy: false UseFederatedAuth: false ServiceConnection: '' + TestPipeline: false # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. @@ -27,6 +28,21 @@ steps: - template: /eng/pipelines/templates/steps/use-venv.yml + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} + + - task: PythonScript@0 + displayName: 'Set Tox Environment' + inputs: + scriptPath: 'scripts/devops_tasks/set_tox_environment.py' + arguments: >- + --unsupported="$(UnsupportedToxEnvironments)" + --override="$(Run.ToxCustomEnvs)" + --team-project="$(System.TeamProject)" + - template: set-dev-build.yml - pwsh: | From f1e1997e6783bda142a5b3bf57bf89572f2a4c4b Mon Sep 17 00:00:00 2001 From: Scott Beddall Date: Wed, 13 Aug 2025 17:44:47 +0000 Subject: [PATCH 14/14] move ci_tools down --- eng/pipelines/templates/steps/build-test.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index ea98e9b5676b..8364a124cb4e 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -34,15 +34,6 @@ steps: ServiceDirectory: "template" TestPipeline: ${{ parameters.TestPipeline }} - - task: PythonScript@0 - displayName: 'Set Tox Environment' - inputs: - scriptPath: 'scripts/devops_tasks/set_tox_environment.py' - arguments: >- - --unsupported="$(UnsupportedToxEnvironments)" - --override="$(Run.ToxCustomEnvs)" - --team-project="$(System.TeamProject)" - - template: set-dev-build.yml - pwsh: | @@ -53,6 +44,15 @@ steps: $(PIP_EXE) freeze displayName: 'Prep Environment' + - task: PythonScript@0 + displayName: 'Set Tox Environment' + inputs: + scriptPath: 'scripts/devops_tasks/set_tox_environment.py' + arguments: >- + --unsupported="$(UnsupportedToxEnvironments)" + --override="$(Run.ToxCustomEnvs)" + --team-project="$(System.TeamProject)" + - template: /eng/common/testproxy/test-proxy-tool.yml parameters: runProxy: false