Skip to content

Commit da404d2

Browse files
scbeddscbedd
authored
Conglomerated changes to optimize venv usage in pipeline (#42486)
* conglomerated changes to venv usage within the CI pipeline. The path prepend + VIRTUAL_ENVIRONMENT setting is definitely enough to make the virtualenv available to the rest of the system cross-task * Add eng/pipelines/templates/steps/install-uv.yml * Add install-uv to use-venv -> also making $(PIP_EXE) available to the rest of the pipeline thereafter and add parameter InstallUv * utilize $(PIP_EXE) everywhere that `python -m pip` used to be used. InstallUv step now sets this value properly to reflect which package manager is selected for the rest of the pipeline' * adjustments to the flow of build-test to eliminate a usage of python 3.10 that was unnecessary
1 parent 47338a5 commit da404d2

16 files changed

+142
-104
lines changed

‎eng/pipelines/templates/jobs/ci.tests.yml

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -90,29 +90,6 @@ jobs:
9090
parameters:
9191
AgentImage: ${{ parameters.OSName }}
9292

93-
- task: UsePythonVersion@0
94-
inputs:
95-
versionSpec: '3.10'
96-
97-
- template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
98-
parameters:
99-
PackageName: "azure-template"
100-
ServiceDirectory: "template"
101-
TestPipeline: ${{ parameters.TestPipeline }}
102-
103-
- pwsh: |
104-
python -m pip install "tools/azure-sdk-tools[build]"
105-
displayName: Install build tooling
106-
107-
- task: PythonScript@0
108-
displayName: 'Set Tox Environment'
109-
inputs:
110-
scriptPath: 'scripts/devops_tasks/set_tox_environment.py'
111-
arguments: >-
112-
--unsupported="$(UnsupportedToxEnvironments)"
113-
--override="$(Run.ToxCustomEnvs)"
114-
--team-project="$(System.TeamProject)"
115-
11693
- template: ../steps/build-test.yml
11794
parameters:
11895
TestMarkArgument: ${{ parameters.TestMarkArgument }}

‎eng/pipelines/templates/jobs/ci.yml

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,6 +209,14 @@ jobs:
209209
os: linux
210210

211211
steps:
212+
- task: UsePythonVersion@0
213+
displayName: 'Use Python $(PythonVersion)'
214+
condition: succeededOrFailed()
215+
inputs:
216+
versionSpec: '$(PythonVersion)'
217+
218+
- template: /eng/pipelines/templates/steps/use-venv.yml
219+
212220
- template: /eng/common/pipelines/templates/steps/check-spelling.yml
213221
parameters:
214222
ContinueOnError: false
@@ -275,6 +283,7 @@ jobs:
275283
- task: UsePythonVersion@0
276284
inputs:
277285
versionSpec: '3.12'
286+
- template: /eng/pipelines/templates/steps/use-venv.yml
278287
- template: /eng/common/pipelines/templates/steps/save-package-properties.yml
279288
parameters:
280289
ServiceDirectory: ${{parameters.ServiceDirectory}}
@@ -310,8 +319,11 @@ jobs:
310319
- task: UsePythonVersion@0
311320
inputs:
312321
versionSpec: '3.12'
322+
- template: /eng/pipelines/templates/steps/use-venv.yml
313323
- pwsh: |
314-
python -m pip install "./tools/azure-sdk-tools[build]"
324+
$ErrorActionPreference = 'Stop'
325+
$PSNativeCommandUseErrorActionPreference = $true
326+
$(PIP_EXE) install "./tools/azure-sdk-tools[build]"
315327
displayName: 'Prep Environment'
316328
- task: PythonScript@0
317329
displayName: 'Ensure service coverage'

‎eng/pipelines/templates/steps/analyze.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ steps:
4848
Condition: succeededOrFailed()
4949

5050
- script: |
51-
python -m pip install "./tools/azure-sdk-tools[build]" -q -I
51+
$(PIP_EXE) install "./tools/azure-sdk-tools[build]"
5252
sdk_find_invalid_versions --always-succeed --service=${{parameters.ServiceDirectory}}
5353
displayName: Find Invalid Versions
5454
condition: succeededOrFailed()

‎eng/pipelines/templates/steps/analyze_dependency.yml

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,8 @@ parameters:
22
ScanPath: ''
33

44
steps:
5-
- task: UsePythonVersion@0
6-
displayName: 'Use Python $(PythonVersion)'
7-
condition: succeededOrFailed()
8-
inputs:
9-
versionSpec: '$(PythonVersion)'
10-
115
- pwsh: |
12-
python -m pip install -r eng/ci_tools.txt
6+
$(PIP_EXE) install -r eng/ci_tools.txt
137
displayName: 'Install Python Tools'
148
condition: succeededOrFailed()
159

‎eng/pipelines/templates/steps/build-extended-artifacts.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ steps:
2828
inputs:
2929
versionSpec: '3.13'
3030

31+
- template: /eng/pipelines/templates/steps/use-venv.yml
32+
3133
- pwsh: |
3234
if ("${{ parameters.ServiceDirectory }}" -ne "auto")
3335
{
@@ -53,10 +55,9 @@ steps:
5355
displayName: 'Tag scheduled builds'
5456
condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule'))
5557
56-
- template: /eng/pipelines/templates/steps/use-venv.yml
57-
5858
- script: |
59-
python -m pip install -r eng/ci_tools.txt
59+
which python
60+
$(PIP_EXE) install -r eng/ci_tools.txt
6061
displayName: 'Prep Environment'
6162
6263
- template: set-dev-build.yml

‎eng/pipelines/templates/steps/build-package-artifacts.yml

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ steps:
3131
inputs:
3232
versionSpec: $(PythonVersion)
3333

34+
- template: /eng/pipelines/templates/steps/use-venv.yml
35+
3436
- template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml@self
3537
parameters:
3638
PackageName: "azure-template"
@@ -85,21 +87,17 @@ steps:
8587
parameters:
8688
PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
8789

88-
- template: /eng/pipelines/templates/steps/use-venv.yml
89-
parameters:
90-
VirtualEnvironmentName: "venv"
91-
Activate: false
92-
Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
93-
9490
- pwsh: |
95-
$(VENV_ACTIVATION_SCRIPT)
91+
$ErrorActionPreference = 'Stop'
92+
$PSNativeCommandUseErrorActionPreference = $true
9693
which python
97-
python -m pip install --force -r eng/ci_tools.txt
94+
$(PIP_EXE) install -r eng/ci_tools.txt
95+
9896
if ($env:AGENT_OS -eq "Linux") {
9997
Write-Host "Installing release reqs"
100-
python -m pip install -r eng/release_requirements.txt
98+
$(PIP_EXE) install -r eng/release_requirements.txt
10199
}
102-
python -m pip freeze --all
100+
$(PIP_EXE) freeze
103101
displayName: 'Prep Environment'
104102
condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
105103
@@ -111,7 +109,6 @@ steps:
111109
condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
112110
113111
- pwsh: |
114-
$(VENV_ACTIVATION_SCRIPT)
115112
which python
116113
sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive
117114
displayName: 'Generate Packages'
@@ -121,7 +118,7 @@ steps:
121118
CIBW_BUILD_VERBOSITY: 3
122119
123120
- pwsh: |
124-
$(VENV_ACTIVATION_SCRIPT)
121+
which python
125122
twine check $(Build.ArtifactStagingDirectory)/**/*.whl
126123
twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz
127124
displayName: 'Verify Readme'

‎eng/pipelines/templates/steps/build-test.yml

Lines changed: 18 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ parameters:
1717
TestProxy: false
1818
UseFederatedAuth: false
1919
ServiceConnection: ''
20+
TestPipeline: false
2021

2122

2223
# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
@@ -26,25 +27,32 @@ steps:
2627
versionSpec: '${{ parameters.PythonVersion }}'
2728

2829
- template: /eng/pipelines/templates/steps/use-venv.yml
30+
31+
- template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml
2932
parameters:
30-
Activate: false
33+
PackageName: "azure-template"
34+
ServiceDirectory: "template"
35+
TestPipeline: ${{ parameters.TestPipeline }}
3136

3237
- template: set-dev-build.yml
3338

3439
- pwsh: |
35-
if ($IsWindows) {
36-
. $(VENV_LOCATION)/Scripts/Activate.ps1
37-
}
38-
else {
39-
. $(VENV_LOCATION)/bin/activate.ps1
40-
}
40+
Write-Host (Get-Command python).Source
4141
$ErrorActionPreference = 'Stop'
4242
$PSNativeCommandUseErrorActionPreference = $true
43-
python -m pip install --force -r eng/ci_tools.txt
44-
python -m pip freeze --all
45-
Write-Host (Get-Command python).Source
43+
$(PIP_EXE) install -r eng/ci_tools.txt
44+
$(PIP_EXE) freeze
4645
displayName: 'Prep Environment'
4746
47+
- task: PythonScript@0
48+
displayName: 'Set Tox Environment'
49+
inputs:
50+
scriptPath: 'scripts/devops_tasks/set_tox_environment.py'
51+
arguments: >-
52+
--unsupported="$(UnsupportedToxEnvironments)"
53+
--override="$(Run.ToxCustomEnvs)"
54+
--team-project="$(System.TeamProject)"
55+
4856
- template: /eng/common/testproxy/test-proxy-tool.yml
4957
parameters:
5058
runProxy: false
@@ -75,12 +83,6 @@ steps:
7583
$env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id;
7684
$env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants;
7785
78-
if ($IsWindows) {
79-
. $(VENV_LOCATION)/Scripts/Activate.ps1
80-
}
81-
else {
82-
. $(VENV_LOCATION)/bin/activate.ps1
83-
}
8486
Write-Host (Get-Command python).Source
8587
8688
if ($env:TESTMARKARGUMENT) {
@@ -104,12 +106,6 @@ steps:
104106
105107
- ${{ else }}:
106108
- pwsh: |
107-
if ($IsWindows) {
108-
. $(VENV_LOCATION)/Scripts/Activate.ps1
109-
}
110-
else {
111-
. $(VENV_LOCATION)/bin/activate.ps1
112-
}
113109
Write-Host (Get-Command python).Source
114110
115111
if ($env:TESTMARKARGUMENT) {
@@ -145,12 +141,6 @@ steps:
145141
- ${{ parameters.AfterTestSteps }}
146142

147143
- pwsh: |
148-
if ($IsWindows) {
149-
. $(VENV_LOCATION)/Scripts/Activate.ps1
150-
}
151-
else {
152-
. $(VENV_LOCATION)/bin/activate.ps1
153-
}
154144
Write-Host (Get-Command python).Source
155145
156146
python scripts/devops_tasks/create_coverage.py
@@ -176,12 +166,6 @@ steps:
176166
$env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id;
177167
$env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants;
178168
179-
if ($IsWindows) {
180-
. $(VENV_LOCATION)/Scripts/Activate.ps1
181-
}
182-
else {
183-
. $(VENV_LOCATION)/bin/activate.ps1
184-
}
185169
Write-Host (Get-Command python).Source
186170
187171
python scripts/devops_tasks/dispatch_tox.py "$(TargetingString)" `

‎eng/pipelines/templates/steps/install-uv.yml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
2+
steps:
3+
- task: Bash@3
4+
displayName: 'Install uv (Linux/macOS)'
5+
inputs:
6+
targetType: inline
7+
script: |
8+
curl -LsSf https://astral.sh/uv/install.sh | sh
9+
condition: or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin'))
10+
11+
- task: Bash@3
12+
inputs:
13+
targetType: inline
14+
script: |
15+
echo "##vso[task.prependpath]$HOME/.local/bin"
16+
displayName: 'Prepend path for MacOS'
17+
condition: eq(variables['Agent.OS'], 'Darwin')
18+
19+
- task: PowerShell@2
20+
displayName: 'Install uv (Windows)'
21+
inputs:
22+
targetType: inline
23+
script: |
24+
iex (irm https://astral.sh/uv/install.ps1)
25+
Write-Host "##vso[task.prependpath]$env:USERPROFILE\.local\bin"
26+
condition: eq(variables['Agent.OS'], 'Windows_NT')

‎eng/pipelines/templates/steps/release-candidate-steps.yml

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,20 @@ steps:
44
versionSpec: $(PythonVersion)
55

66
- template: /eng/pipelines/templates/steps/use-venv.yml
7-
parameters:
8-
Activate: false
97

108
- pwsh: |
11-
$(VENV_ACTIVATION_SCRIPT)
9+
Write-Host (Get-Command python).Source
1210
$ErrorActionPreference = 'Stop'
1311
$PSNativeCommandUseErrorActionPreference = $true
14-
python -m pip install -r $(Build.SourcesDirectory)/eng/ci_tools.txt
15-
python -m pip freeze --all
16-
Write-Host (Get-Command python).Source
12+
$(PIP_EXE) install -r $(Build.SourcesDirectory)/eng/ci_tools.txt
13+
$(PIP_EXE) freeze
1714
displayName: 'Install Dependencies'
1815
1916
- template: /eng/common/testproxy/test-proxy-tool.yml
2017
parameters:
2118
runProxy: false
2219

2320
- pwsh: |
24-
$(VENV_ACTIVATION_SCRIPT)
2521
python ./scripts/devops_tasks/dispatch_tox.py "$(TargetedPackages)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="Build"
2622
displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)'
2723
env:

‎eng/pipelines/templates/steps/run_apistub.yml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,14 @@ steps:
1616
versionSpec: '3.10'
1717
condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true'))
1818

19-
- script: |
20-
python -m pip install -r eng/ci_tools.txt
19+
- template: /eng/pipelines/templates/steps/use-venv.yml
20+
parameters:
21+
VirtualEnvironmentName: "venv-api-stub"
22+
23+
- pwsh: |
24+
$ErrorActionPreference = 'Stop'
25+
$PSNativeCommandUseErrorActionPreference = $true
26+
$(PIP_EXE) install -r eng/ci_tools.txt
2127
displayName: 'Prep Environment'
2228
condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true'))
2329

0 commit comments

Comments
 (0)