Skip to content

Commit f3b963a

Browse files
archerzzarcherzz
authored
build(publish): parallel code gen on azure-sdk-for-net (#3153)
- split job `Update_azure_sdk_for_net` into 3 stages: - `Update_Generator_Versions`: change the generator versions only, create the first commit to new branch `auto-update-autorest-${Version}` - `Update_Codes_X_Y` and `Update_Samples_X_Y`: update SDK codes or sample codes from service of which name starts with `X` to `Y`. - `Create_PR`: create PR from branch `auto-update-autorest-${Version}` to `main` - add retry for `Update_Codes_X_Y` and `Update_Samples_X_Y` resolve #3067 #3068
1 parent fd48b64 commit f3b963a

9 files changed

+335
-39
lines changed

‎eng/UpdateAzureSdkCodes.ps1‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
param(
2+
[Parameter(Mandatory)]
3+
[string]$SdkRepoRoot,
4+
5+
[string[]]$ServiceDirectoryFilters = @("*"),
6+
7+
[switch]$ShowSummary
8+
)
9+
10+
$ErrorActionPreference = 'Stop'
11+
12+
Write-Host "Generating Azure SDK Codes..."
13+
foreach ($filter in $ServiceDirectoryFilters) {
14+
Write-Host 'Generating projects under service directory ' -ForegroundColor Green -NoNewline
15+
Write-Host "$filter" -ForegroundColor Yellow
16+
if ($ShowSummary)
17+
{
18+
dotnet msbuild /restore /t:GenerateCode /p:ServiceDirectory=$filter /v:n /ds "$SdkRepoRoot\eng\service.proj"
19+
}
20+
else
21+
{
22+
dotnet msbuild /restore /t:GenerateCode /p:ServiceDirectory=$filter "$SdkRepoRoot\eng\service.proj"
23+
}
24+
if ($LastExitCode -ne 0) {
25+
Write-Error 'Generation error'
26+
exit 1
27+
}
28+
}

‎eng/UpdateAzureSdkForNet.ps1‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
1-
param($AutorestCSharpVersion, $CadlEmitterVersion, $SdkRepoRoot)
1+
param(
2+
[Parameter(Mandatory)]
3+
[string]$AutorestCSharpVersion,
24

3-
$SdkRepoRoot = Resolve-Path $SdkRepoRoot
5+
[Parameter(Mandatory)]
6+
[string]$CadlEmitterVersion,
47

5-
Write-Host "Running Autorest.CSharp($AutorestCSharpVersion) and Cadl Emitter($CadlEmitterVersion) under $SdkRepoRoot"
8+
[Parameter(Mandatory)]
9+
[string]$SdkRepoRoot,
10+
11+
[string[]]$ServiceDirectoryFilters = @("*"),
612

7-
$PackagesProps = "$SdkRepoRoot\eng\Packages.Data.props"
8-
(Get-Content -Raw $PackagesProps) -replace`
9-
'<PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version=".*?" />',
10-
"<PackageReference Update=`"Microsoft.Azure.AutoRest.CSharp`" Version=`"$AutorestCSharpVersion`" PrivateAssets=`"All`" />" | `
11-
Set-Content $PackagesProps -NoNewline
13+
[switch]$ShowSummary
14+
)
1215

13-
$CadlEmitterProps = "$SdkRepoRoot\eng\emitter-package.json"
14-
(Get-Content -Raw $CadlEmitterProps) -replace`
15-
'"@azure-tools/cadl-csharp": ".*?"',
16-
"`"@azure-tools/cadl-csharp`": `"$CadlEmitterVersion`"" | `
17-
Set-Content $CadlEmitterProps -NoNewline
16+
$ErrorActionPreference = 'Stop'
1817

19-
dotnet msbuild /restore /t:GenerateCode "$SdkRepoRoot\eng\service.proj"
20-
dotnet msbuild /restore /t:GenerateTests "$SdkRepoRoot\eng\service.proj"
18+
Invoke-Expression "$PSScriptRoot\UpdateGeneratorMetadata.ps1 -AutorestCSharpVersion $AutorestCSharpVersion -CadlEmitterVersion $CadlEmitterVersion -SdkRepoRoot $SdkRepoRoot"
19+
20+
Invoke-Expression "$PSScriptRoot\UpdateAzureSdkCodes.ps1 -SdkRepoRoot $SdkRepoRoot -ServiceDirectoryFilters $($ServiceDirectoryFilters -Join ',') $(if ($ShowSummary) {'-ShowSummary'})"

‎eng/UpdateAzureSdkForNetSamples.ps1‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
param(
2+
[Parameter(Mandatory)]
3+
[string]$AutorestCSharpVersion,
4+
5+
[Parameter(Mandatory)]
6+
[string]$CadlEmitterVersion,
7+
8+
[Parameter(Mandatory)]
9+
[string]$SdkRepoRoot,
10+
11+
[string[]]$ServiceDirectoryFilters = @("*"),
12+
13+
[switch]$ShowSummary
14+
)
15+
16+
$ErrorActionPreference = 'Stop'
17+
18+
Invoke-Expression "$PSScriptRoot\UpdateGeneratorMetadata.ps1 -AutorestCSharpVersion $AutorestCSharpVersion -CadlEmitterVersion $CadlEmitterVersion -SdkRepoRoot $SdkRepoRoot"
19+
20+
Invoke-Expression "$PSScriptRoot\UpdateAzureSdkSamples.ps1 -SdkRepoRoot $SdkRepoRoot -ServiceDirectoryFilters $($ServiceDirectoryFilters -Join ',') $(if ($ShowSummary) {'-ShowSummary'})"

‎eng/UpdateAzureSdkSamples.ps1‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
param(
2+
[Parameter(Mandatory)]
3+
[string]$SdkRepoRoot,
4+
5+
[string[]]$ServiceDirectoryFilters = @("*"),
6+
7+
[switch]$ShowSummary
8+
)
9+
10+
$ErrorActionPreference = 'Stop'
11+
12+
Write-Host "Generating Azure SDK Samples..."
13+
foreach ($filter in $ServiceDirectoryFilters) {
14+
Write-Host 'Generating projects under service directory ' -ForegroundColor Green -NoNewline
15+
Write-Host "$filter" -ForegroundColor Yellow
16+
if ($ShowSummary) {
17+
dotnet msbuild /restore /t:GenerateTests /p:ServiceDirectory=$filter /v:n /ds "$SdkRepoRoot\eng\service.proj"
18+
}
19+
else {
20+
dotnet msbuild /restore /t:GenerateTests /p:ServiceDirectory=$filter "$SdkRepoRoot\eng\service.proj"
21+
}
22+
if ($LastExitCode -ne 0) {
23+
Write-Error 'Generation error'
24+
exit 1
25+
}
26+
}

‎eng/UpdateGeneratorMetadata.ps1‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
param(
2+
[Parameter(Mandatory)]
3+
[string]$AutorestCSharpVersion,
4+
5+
[Parameter(Mandatory)]
6+
[string]$CadlEmitterVersion,
7+
8+
[Parameter(Mandatory)]
9+
[string]$SdkRepoRoot)
10+
11+
12+
$ErrorActionPreference = 'Stop'
13+
14+
$SdkRepoRoot = Resolve-Path $SdkRepoRoot
15+
16+
Write-Host "Updating Autorest.CSharp($AutorestCSharpVersion) and Cadl Emitter($CadlEmitterVersion) under $SdkRepoRoot"
17+
18+
$PackagesProps = "$SdkRepoRoot\eng\Packages.Data.props"
19+
(Get-Content -Raw $PackagesProps) -replace `
20+
'<PackageReference Update="Microsoft.Azure.AutoRest.CSharp" Version=".*?" />',
21+
"<PackageReference Update=`"Microsoft.Azure.AutoRest.CSharp`" Version=`"$AutorestCSharpVersion`" PrivateAssets=`"All`" />" | `
22+
Set-Content $PackagesProps -NoNewline
23+
24+
$CadlEmitterProps = "$SdkRepoRoot\eng\emitter-package.json"
25+
(Get-Content -Raw $CadlEmitterProps) -replace `
26+
'"@azure-tools/cadl-csharp": ".*?"',
27+
"`"@azure-tools/cadl-csharp`": `"$CadlEmitterVersion`"" | `
28+
Set-Content $CadlEmitterProps -NoNewline

‎eng/pipelines/build.yml‎

Lines changed: 76 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ resources:
2424
type: github
2525
name: Azure/azure-sdk-for-net
2626
endpoint: azure
27+
ref: main
2728
- repository: azure-sdk-build-tools
2829
type: git
2930
name: internal/azure-sdk-build-tools
@@ -39,7 +40,9 @@ stages:
3940
vmImage: windows-2022
4041
steps:
4142
- checkout: self
43+
fetchDepth: 1
4244
- checkout: azure-sdk-tools
45+
fetchDepth: 1
4346
- task: UseDotNet@2
4447
displayName: 'Use .NET Core SDK'
4548
retryCountOnTaskFailure: 3
@@ -163,34 +166,83 @@ stages:
163166
- Build_and_Test
164167
- Release
165168
jobs:
166-
- job: Update
167-
timeoutInMinutes: 180
168-
pool:
169-
vmImage: ubuntu-20.04
169+
- template: update-generator-versions.yml
170+
# update codes
171+
- template: update-azure-sdk-for-net-codes.yml
172+
parameters:
173+
name: Update_Codes_A_C
174+
filter: "a*,b*,c*"
175+
- template: update-azure-sdk-for-net-codes.yml
176+
parameters:
177+
name: Update_Codes_D_E
178+
filter: "d*,e*"
179+
- template: update-azure-sdk-for-net-codes.yml
180+
parameters:
181+
name: Update_Codes_F_L
182+
filter: "f*,g*,h*,i*,j*,k*,l*"
183+
- template: update-azure-sdk-for-net-codes.yml
184+
parameters:
185+
name: Update_Codes_M_O
186+
filter: "m*,n*,o*"
187+
- template: update-azure-sdk-for-net-codes.yml
188+
parameters:
189+
name: Update_Codes_P_R
190+
filter: "p*,q*,r*"
191+
- template: update-azure-sdk-for-net-codes.yml
192+
parameters:
193+
name: Update_Codes_S_U
194+
filter: "s*,t*,u*"
195+
- template: update-azure-sdk-for-net-codes.yml
196+
parameters:
197+
name: Update_Codes_V_Z
198+
filter: "v*,w*,x*,y*,z*"
199+
# update samples
200+
- template: update-azure-sdk-for-net-samples.yml
201+
parameters:
202+
name: Update_Samples_A_F
203+
filter: "a*,b*,c*,d*,e*,f*"
204+
- template: update-azure-sdk-for-net-samples.yml
205+
parameters:
206+
name: Update_Samples_G_L
207+
filter: "g*,h*,i*,j*,k*,l*"
208+
- template: update-azure-sdk-for-net-samples.yml
209+
parameters:
210+
name: Update_Samples_M_R
211+
filter: "m*,n*,o*,p*,q*,r*"
212+
- template: update-azure-sdk-for-net-samples.yml
213+
parameters:
214+
name: Update_Samples_S_Z
215+
filter: "s*,t*,u*,v*,w*,x*,y*,z*"
216+
- job: Create_PR
217+
dependsOn:
218+
- Update_Codes_A_C
219+
- Update_Codes_D_E
220+
- Update_Codes_F_L
221+
- Update_Codes_M_O
222+
- Update_Codes_P_R
223+
- Update_Codes_S_U
224+
- Update_Codes_V_Z
225+
- Update_Samples_A_F
226+
- Update_Samples_G_L
227+
- Update_Samples_M_R
228+
- Update_Samples_S_Z
170229
variables:
171230
AutorestCSharpVersion: $[stageDependencies.Build_and_Test.Build.outputs['Package.AutorestCSharpVersion']]
172231
CadlEmitterVersion: $[stageDependencies.Build_and_Test.Build.outputs['Publish.CadlEmitterVersion']]
173232
steps:
174233
- checkout: self
175-
- checkout: azure-sdk-for-net
176234
- checkout: azure-sdk-tools
177-
- task: UseDotNet@2
178-
displayName: 'Use .NET Core SDK'
235+
- task: PowerShell@2
236+
displayName: Create pull request
179237
inputs:
180-
useGlobalJson: true
181-
performMultiLevelLookup: true
182-
- pwsh: ./eng/UpdateAzureSdkForNet.ps1 -AutorestCSharpVersion $(AutorestCSharpVersion) -CadlEmitterVersion $(CadlEmitterVersion) -SdkRepoRoot $(Build.SourcesDirectory)/azure-sdk-for-net
183-
failOnStderr: false
184-
workingDirectory: $(Build.SourcesDirectory)/autorest.csharp
185-
displayName: 'Update generator version in Azure SDK repo'
186-
- template: /eng/common/pipelines/templates/steps/create-pull-request.yml@azure-sdk-tools
187-
parameters:
188-
BaseBranchName: main
189-
RepoName: azure-sdk-for-net
190-
PRBranchName: auto-update-autorest
191-
CommitMsg: Update AutoRest C# version to $(AutorestCSharpVersion)
192-
PRBody: Update AutoRest C# version to $(AutorestCSharpVersion)
193-
PRTitle: Update AutoRest C# version
194-
PushArgs: -f
195-
WorkingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-net
196-
ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-tools/eng/common/scripts
238+
pwsh: true
239+
filePath: $(Build.SourcesDirectory)/azure-sdk-tools/eng/common/scripts/Submit-PullRequest.ps1
240+
arguments: >
241+
-RepoOwner "Azure"
242+
-RepoName "azure-sdk-for-net"
243+
-BaseBranch "main"
244+
-PROwner "azure-sdk"
245+
-PRBranch "auto-update-autorest-$(AutorestCSharpVersion)"
246+
-AuthToken "$(azuresdk-github-pat)"
247+
-PRTitle "Update AutoRest C# version to $(AutorestCSharpVersion)"
248+
-PRBody "Update AutoRest C# version to $(AutorestCSharpVersion)"

‎eng/pipelines/update-azure-sdk-for-net-codes.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Generate SDK codes
2+
3+
parameters:
4+
name: ''
5+
filter: ''
6+
7+
jobs:
8+
- job: ${{ parameters.name }}
9+
timeoutInMinutes: 60
10+
pool:
11+
vmImage: ubuntu-20.04
12+
variables:
13+
AutorestCSharpVersion: $[stageDependencies.Build_and_Test.Build.outputs['Package.AutorestCSharpVersion']]
14+
CadlEmitterVersion: $[stageDependencies.Build_and_Test.Build.outputs['Publish.CadlEmitterVersion']]
15+
dependsOn: Update_Generator_Versions
16+
steps:
17+
- checkout: self
18+
fetchDepth: 1
19+
- checkout: azure-sdk-for-net
20+
- checkout: azure-sdk-tools
21+
fetchDepth: 1
22+
- task: UseDotNet@2
23+
displayName: 'Use .NET Core SDK'
24+
inputs:
25+
useGlobalJson: true
26+
performMultiLevelLookup: true
27+
- task: Powershell@2
28+
displayName: Update SDK codes ${{ parameters.filter }}
29+
inputs:
30+
pwsh: true
31+
filePath: $(Build.SourcesDirectory)/autorest.csharp/eng/UpdateAzureSdkForNet.ps1
32+
arguments: >
33+
-AutorestCSharpVersion $(AutorestCSharpVersion)
34+
-CadlEmitterVersion $(CadlEmitterVersion)
35+
-SdkRepoRoot $(Build.SourcesDirectory)/azure-sdk-for-net
36+
-ServiceDirectoryFilters ${{ parameters.filter}}
37+
-ShowSummary
38+
failOnStderr: false
39+
workingDirectory: $(Build.SourcesDirectory)/autorest.csharp
40+
retryCountOnTaskFailure: 3
41+
- pwsh: |
42+
git restore eng/Packages.Data.props
43+
git restore eng/emitter-package.json
44+
displayName: Discard versions change
45+
workingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-net
46+
- template: /eng/common/pipelines/templates/steps/git-push-changes.yml@azure-sdk-tools
47+
parameters:
48+
BaseRepoBranch: auto-update-autorest-$(AutorestCSharpVersion)
49+
BaseRepoOwner: azure-sdk
50+
CommitMsg: Update SDK codes ${{ parameters.filter }}
51+
TargetRepoOwner: Azure
52+
TargetRepoName: azure-sdk-for-net
53+
WorkingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-net
54+
ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-tools/eng/common/scripts

‎eng/pipelines/update-azure-sdk-for-net-samples.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
# Generate SDK samples
2+
3+
parameters:
4+
name: ''
5+
filter: ''
6+
7+
jobs:
8+
- job: ${{ parameters.name }}
9+
timeoutInMinutes: 60
10+
pool:
11+
vmImage: ubuntu-20.04
12+
variables:
13+
AutorestCSharpVersion: $[stageDependencies.Build_and_Test.Build.outputs['Package.AutorestCSharpVersion']]
14+
CadlEmitterVersion: $[stageDependencies.Build_and_Test.Build.outputs['Publish.CadlEmitterVersion']]
15+
dependsOn: Update_Generator_Versions
16+
steps:
17+
- checkout: self
18+
fetchDepth: 1
19+
- checkout: azure-sdk-for-net
20+
- checkout: azure-sdk-tools
21+
fetchDepth: 1
22+
- task: UseDotNet@2
23+
displayName: 'Use .NET Core SDK'
24+
inputs:
25+
useGlobalJson: true
26+
performMultiLevelLookup: true
27+
- task: Powershell@2
28+
displayName: Update SDK samples ${{ parameters.filter }}
29+
inputs:
30+
pwsh: true
31+
filePath: $(Build.SourcesDirectory)/autorest.csharp/eng/UpdateAzureSdkForNetSamples.ps1
32+
arguments: >
33+
-AutorestCSharpVersion $(AutorestCSharpVersion)
34+
-CadlEmitterVersion $(CadlEmitterVersion)
35+
-SdkRepoRoot $(Build.SourcesDirectory)/azure-sdk-for-net
36+
-ServiceDirectoryFilters ${{ parameters.filter}}
37+
-ShowSummary
38+
failOnStderr: false
39+
workingDirectory: $(Build.SourcesDirectory)/autorest.csharp
40+
retryCountOnTaskFailure: 3
41+
- pwsh: |
42+
git restore eng/Packages.Data.props
43+
git restore eng/emitter-package.json
44+
displayName: Discard versions change
45+
workingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-net
46+
- template: /eng/common/pipelines/templates/steps/git-push-changes.yml@azure-sdk-tools
47+
parameters:
48+
BaseRepoBranch: auto-update-autorest-$(AutorestCSharpVersion)
49+
BaseRepoOwner: azure-sdk
50+
CommitMsg: Update SDK samples ${{ parameters.filter }}
51+
TargetRepoOwner: Azure
52+
TargetRepoName: azure-sdk-for-net
53+
WorkingDirectory: $(Build.SourcesDirectory)/azure-sdk-for-net
54+
ScriptDirectory: $(Build.SourcesDirectory)/azure-sdk-tools/eng/common/scripts

0 commit comments

Comments
 (0)