diff --git a/eng/common/templates/1es.yml b/eng/common/templates/1es.yml new file mode 100644 index 000000000..19f8c1f11 --- /dev/null +++ b/eng/common/templates/1es.yml @@ -0,0 +1,70 @@ +# When extending this template, pipelines using a repository resource containing versions files for image caching must +# do the following: +# +# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines +# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans +# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans +# +# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in +# `$(Build.SourcesDirectory)/versions`, as it will not be scanned. + +parameters: +- name: stages + type: stageList + default: [] +# List of repositories that will be excluded from SDL scanning. This should +# only be used when including other repos without building their source code. +# E.g. for the dotnet/versions repo. +- name: reposToExcludeFromScanning + type: object + default: [] +# The pool that will be used for initializing service connections. +- name: pool + type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux +# The pool that will be used for SDL jobs. +- name: sourceAnalysisPool + type: object + default: + name: $(defaultSourceAnalysisPoolName) + image: $(defaultSourceAnalysisPoolImage) + os: windows + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: /eng/common/templates/task-prefix-decorator.yml@self + parameters: + baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates + templateParameters: + pool: ${{ parameters.pool }} + sdl: + # Required for unofficial pipelines because we rely on the ManifestGeneratorTask that is + # automatically installed by 1ES pipeline templates + sbom: + enabled: true + binskim: + enabled: true + componentgovernance: + ignoreDirectories: $(Build.SourcesDirectory)/versions + showAlertLink: true + policheck: + enabled: true + ${{ if ne(length(parameters.reposToExcludeFromScanning), 0) }}: + sourceRepositoriesToScan: + exclude: + - ${{ each repo in parameters.reposToExcludeFromScanning }}: + - repository: ${{ repo }} + sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} + tsa: + enabled: true + stages: + - ${{ parameters.stages }} diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index 420120d25..e769c6a53 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -6,6 +6,7 @@ parameters: buildJobTimeout: 60 commonInitStepsForMatrixAndBuild: [] customInitSteps: [] + publishConfig: null noCache: false internalProjectName: null publicProjectName: null @@ -48,13 +49,13 @@ jobs: # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates # the environment variable for us. - $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" + $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)" if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") { - $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push" + $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} --push" } # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache - if ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true") { + if ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true") { $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)" } @@ -66,10 +67,12 @@ jobs: name: BuildImages displayName: Build Images serviceConnections: + # "name" here refers to the argument name, not the service connection name. + # It should probably be changed to "argName". - name: acr - id: $(build.serviceConnection.id) - tenantId: $(build.serviceConnection.tenantId) - clientId: $(build.serviceConnection.clientId) + id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }} - ${{ if eq(parameters.isInternalServicingValidation, true) }}: - name: storage id: $(dotnetstaging.serviceConnection.id) @@ -86,8 +89,8 @@ jobs: --architecture $(architecture) --retry --digests-out-var 'builtImages' - --acr-subscription '$(acr-staging.subscription)' - --acr-resource-group '$(acr-staging.resourceGroup)' + --acr-subscription '${{ parameters.publishConfig.buildAcr.subscription }}' + --acr-resource-group '${{ parameters.publishConfig.buildAcr.resourceGroup }}' $(manifestVariables) $(imageBuilderBuildArgs) - template: /eng/common/templates/steps/publish-artifact.yml@self @@ -134,7 +137,7 @@ jobs: # Manifest tool docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/custom-sbom-generation-workflows $images -Split ',' | ForEach-Object { echo "Generating SBOM for $_"; - $formattedImageName = $_.Replace('$(acr-staging.server)/$(stagingRepoPrefix)', "").Replace('/', '_').Replace(':', '_'); + $formattedImageName = $_.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_'); $sbomChildDir = "$(sbomDirectory)/$formattedImageName"; New-Item -Type Directory -Path $sbomChildDir > $null; & $dotnetPath "$manifestToolDllPath" ` diff --git a/eng/common/templates/jobs/copy-base-images-staging.yml b/eng/common/templates/jobs/copy-base-images-staging.yml index 71ddb4a56..17f4d359c 100644 --- a/eng/common/templates/jobs/copy-base-images-staging.yml +++ b/eng/common/templates/jobs/copy-base-images-staging.yml @@ -5,6 +5,9 @@ parameters: - name: pool type: object default: {} +- name: publishConfig + type: object + default: null - name: customInitSteps type: stepList default: [] @@ -22,12 +25,5 @@ jobs: pool: ${{ parameters.pool }} customInitSteps: ${{ parameters.customInitSteps }} additionalOptions: ${{ parameters.additionalOptions }} - acr: - server: $(acr-staging.server) - serviceConnection: - tenantId: $(internal-mirror.serviceConnection.tenantId) - clientId: $(internal-mirror.serviceConnection.clientId) - id: $(internal-mirror.serviceConnection.id) - subscription: $(acr-staging.subscription) - resourceGroup: $(acr-staging.resourceGroup) - repoPrefix: $(mirrorRepoPrefix) + acr: ${{ parameters.publishConfig.internalMirrorAcr }} + repoPrefix: ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} diff --git a/eng/common/templates/jobs/generate-matrix.yml b/eng/common/templates/jobs/generate-matrix.yml index 8037aca56..663aeb01f 100644 --- a/eng/common/templates/jobs/generate-matrix.yml +++ b/eng/common/templates/jobs/generate-matrix.yml @@ -6,6 +6,7 @@ parameters: isTestStage: false internalProjectName: null noCache: false + publishConfig: null customInitSteps: [] commonInitStepsForMatrixAndBuild: [] sourceBuildPipelineRunId: "" @@ -20,6 +21,7 @@ jobs: - ${{ parameters.customInitSteps }} - template: /eng/common/templates/steps/validate-branch.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} - template: /eng/common/templates/steps/set-image-info-path-var.yml parameters: @@ -36,7 +38,7 @@ jobs: if ("${{ parameters.isTestStage}}" -eq "true") { $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json" } - elseif ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { + elseif ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") { # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images" } diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml index 5295ad960..68cfb4fee 100644 --- a/eng/common/templates/jobs/publish.yml +++ b/eng/common/templates/jobs/publish.yml @@ -1,6 +1,7 @@ parameters: pool: {} internalProjectName: null + publishConfig: null customInitSteps: [] customPublishVariables: [] sourceBuildPipelineDefinitionId: "" @@ -20,7 +21,7 @@ jobs: - name: imageBuilder.commonCmdArgs value: >- --manifest '$(manifest)' - --registry-override '$(acr.server)' + --registry-override '${{ parameters.publishConfig.publishAcr.server }}' $(manifestVariables) $(imageBuilder.queueArgs) - name: publishNotificationRepoName @@ -46,7 +47,7 @@ jobs: steps: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: - cloneVersionsRepo: ${{ variables.publishImageInfo }} + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} - template: /eng/common/templates/steps/retain-build.yml@self @@ -80,6 +81,8 @@ jobs: publicSourceBranch: $(publicSourceBranch) - template: /eng/common/templates/steps/set-dry-run.yml@self + parameters: + publishConfig: ${{ parameters.publishConfig }} - script: echo "##vso[task.setvariable variable=imageQueueTime]$(date --rfc-2822)" displayName: Set Publish Variables @@ -94,19 +97,19 @@ jobs: displayName: Copy Images serviceConnections: - name: acr - id: $(publish.serviceConnection.id) - tenantId: $(publish.serviceConnection.tenantId) - clientId: $(publish.serviceConnection.clientId) + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} internalProjectName: ${{ parameters.internalProjectName }} args: >- copyAcrImages - '$(acr.subscription)' - '$(acr.resourceGroup)' - '$(stagingRepoPrefix)' - '$(acr-staging.server)' + '${{ parameters.publishConfig.buildAcr.subscription }}' + '${{ parameters.publishConfig.buildAcr.resourceGroup }}' + '${{ parameters.publishConfig.buildAcr.repoPrefix }}' + '${{ parameters.publishConfig.buildAcr.server }}' --os-type '*' --architecture '*' - --repo-prefix '$(publishRepoPrefix)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' --image-info '$(imageInfoContainerDir)/image-info.json' $(dryRunArg) $(imageBuilder.pathArgs) @@ -117,15 +120,15 @@ jobs: displayName: Publish Manifest serviceConnections: - name: acr - id: $(publish.serviceConnection.id) - tenantId: $(publish.serviceConnection.tenantId) - clientId: $(publish.serviceConnection.clientId) + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} internalProjectName: ${{ parameters.internalProjectName }} dockerClientOS: ${{ parameters.dockerClientOS }} args: >- publishManifest '$(imageInfoContainerDir)/image-info.json' - --repo-prefix '$(publishRepoPrefix)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' --os-type '*' --architecture '*' $(dryRunArg) @@ -142,6 +145,7 @@ jobs: - template: /eng/common/templates/steps/wait-for-mcr-image-ingestion.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} imageInfoPath: '$(imageinfoContainerDir)/image-info.json' minQueueTime: $(imageQueueTime) dryRunArg: $(dryRunArg) @@ -206,18 +210,18 @@ jobs: displayName: Generate EOL Annotation Data serviceConnections: - name: acr - id: $(publish.serviceConnection.id) - tenantId: $(publish.serviceConnection.tenantId) - clientId: $(publish.serviceConnection.clientId) + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} internalProjectName: internal condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) args: >- - generateEolAnnotationData + generateEolAnnotationDataForPublish + '${{ parameters.publishConfig.publishAcr.server }}' + '${{ parameters.publishConfig.publishAcr.repoPrefix }}' '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' '$(imageInfoContainerDir)/full-image-info-orig.json' '$(imageInfoContainerDir)/full-image-info-new.json' - '$(acr.server)' - '$(publishRepoPrefix)' $(generateEolAnnotationDataExtraOptions) $(dryRunArg) @@ -232,7 +236,7 @@ jobs: - template: /eng/common/templates/steps/annotate-eol-digests.yml@self parameters: - internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json - script: > @@ -271,7 +275,7 @@ jobs: $(gitHubNotificationsRepoInfo.authArgs) '$(gitHubNotificationsRepoInfo.org)' '$(gitHubNotificationsRepoInfo.repo)' - --repo-prefix '$(publishRepoPrefix)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' --task "🟪 Copy Images" --task "🟪 Publish Manifest" --task "🟪 Wait for Image Ingestion" diff --git a/eng/common/templates/jobs/test-images-linux-client.yml b/eng/common/templates/jobs/test-images-linux-client.yml index 2e9319f2c..9aa30e659 100644 --- a/eng/common/templates/jobs/test-images-linux-client.yml +++ b/eng/common/templates/jobs/test-images-linux-client.yml @@ -5,6 +5,7 @@ parameters: testJobTimeout: 60 preBuildValidation: false internalProjectName: null + publishConfig: null customInitSteps: [] sourceBuildPipelineRunId: "" @@ -24,5 +25,6 @@ jobs: parameters: preBuildValidation: ${{ parameters.preBuildValidation }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/jobs/test-images-windows-client.yml b/eng/common/templates/jobs/test-images-windows-client.yml index 498fee6cb..94965f4cf 100644 --- a/eng/common/templates/jobs/test-images-windows-client.yml +++ b/eng/common/templates/jobs/test-images-windows-client.yml @@ -4,6 +4,7 @@ parameters: matrix: {} testJobTimeout: 60 internalProjectName: null + publishConfig: null customInitSteps: [] sourceBuildPipelineRunId: "" @@ -19,5 +20,6 @@ jobs: - template: /eng/common/templates/steps/test-images-windows-client.yml@self parameters: internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/stages/build-and-test.yml b/eng/common/templates/stages/build-and-test.yml index d1930a2b5..9bc288754 100644 --- a/eng/common/templates/stages/build-and-test.yml +++ b/eng/common/templates/stages/build-and-test.yml @@ -18,6 +18,7 @@ parameters: windowsAmdTestJobTimeout: 60 noCache: false + publishConfig: null internalProjectName: null publicProjectName: null @@ -58,21 +59,23 @@ stages: testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} preBuildValidation: true internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: - - ${{ parameters.customTestInitSteps }} - # These variables are normally set by the matrix. Since this test job is not generated - # by a matrix, we need to set them manually. They can be set to empty values since their - # values aren't actually used for the pre-build tests. - - powershell: | - echo "##vso[task.setvariable variable=productVersion]" - echo "##vso[task.setvariable variable=imageBuilderPaths]" - echo "##vso[task.setvariable variable=osVersions]" - echo "##vso[task.setvariable variable=architecture]" - displayName: Initialize Test Variables + - ${{ parameters.customTestInitSteps }} + # These variables are normally set by the matrix. Since this test job is not generated + # by a matrix, we need to set them manually. They can be set to empty values since their + # values aren't actually used for the pre-build tests. + - powershell: | + echo "##vso[task.setvariable variable=productVersion]" + echo "##vso[task.setvariable variable=imageBuilderPaths]" + echo "##vso[task.setvariable variable=osVersions]" + echo "##vso[task.setvariable variable=architecture]" + displayName: Initialize Test Variables - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self parameters: name: CopyBaseImages + publishConfig: ${{ parameters.publishConfig }} pool: ${{ parameters.linuxAmd64Pool }} additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)" customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} @@ -85,10 +88,12 @@ stages: customBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} internalProjectName: ${{ parameters.internalProjectName }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }} commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} - template: /eng/common/templates/jobs/build-images.yml@self @@ -101,9 +106,11 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} @@ -117,9 +124,11 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} @@ -133,9 +142,11 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} @@ -149,9 +160,11 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} @@ -165,9 +178,11 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} @@ -181,9 +196,11 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} versionsRepoRef: ${{ parameters.versionsRepoRef }} @@ -198,9 +215,11 @@ stages: commonInitStepsForMatrixAndBuild: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} versionsRepoRef: ${{ parameters.versionsRepoRef }} customInitSteps: ${{ parameters.customBuildInitSteps }} noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} @@ -251,6 +270,7 @@ stages: - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self parameters: versionsRepoRef: ${{ parameters.versionsRepoRef }} + publishConfig: ${{ parameters.publishConfig }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: name: Linux_amd64 @@ -258,6 +278,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxAmd64'] testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self @@ -267,6 +288,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm64'] testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-linux-client.yml@self @@ -276,6 +298,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.LinuxArm32'] testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self @@ -285,6 +308,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.Windows1809Amd64'] testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self @@ -294,6 +318,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2022Amd64'] testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self @@ -303,6 +328,7 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2025Amd64'] testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/jobs/test-images-windows-client.yml@self @@ -312,5 +338,6 @@ stages: matrix: dependencies.GenerateTestMatrix.outputs['matrix.WindowsLtsc2016Amd64'] testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customInitSteps: ${{ parameters.customTestInitSteps }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} diff --git a/eng/common/templates/stages/dotnet/build-and-test.yml b/eng/common/templates/stages/dotnet/build-and-test.yml index 3e6c80d89..3b57c18bf 100644 --- a/eng/common/templates/stages/dotnet/build-and-test.yml +++ b/eng/common/templates/stages/dotnet/build-and-test.yml @@ -11,6 +11,7 @@ parameters: # Build parameters noCache: false + publishConfig: null buildMatrixType: platformDependencyGraph buildMatrixCustomBuildLegGroupArgs: "" linuxAmdBuildJobTimeout: 60 @@ -36,6 +37,7 @@ stages: - template: /eng/common/templates/stages/build-and-test.yml@self parameters: noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} diff --git a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml index f7f54fce8..e5648784f 100644 --- a/eng/common/templates/stages/dotnet/build-test-publish-repo.yml +++ b/eng/common/templates/stages/dotnet/build-test-publish-repo.yml @@ -10,6 +10,7 @@ parameters: # Build parameters noCache: false + publishConfig: null buildMatrixType: platformDependencyGraph buildMatrixCustomBuildLegGroupArgs: "" linuxAmdBuildJobTimeout: 60 @@ -32,8 +33,7 @@ parameters: # Other common parameters internalProjectName: null publicProjectName: null - versionsRepoRef: null - + versionsRepoRef: "" stages: - template: /eng/common/templates/stages/dotnet/build-and-test.yml@self @@ -45,6 +45,7 @@ stages: customCopyBaseImagesInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }} # Build noCache: ${{ parameters.noCache }} + publishConfig: ${{ parameters.publishConfig }} buildMatrixType: ${{ parameters.buildMatrixType }} buildMatrixCustomBuildLegGroupArgs: ${{ parameters.buildMatrixCustomBuildLegGroupArgs }} linuxAmdBuildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }} @@ -71,5 +72,6 @@ stages: customPublishInitSteps: ${{ parameters.customPublishInitSteps }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} versionsRepoRef: ${{ parameters.versionsRepoRef }} diff --git a/eng/common/templates/stages/dotnet/publish-config-nonprod.yml b/eng/common/templates/stages/dotnet/publish-config-nonprod.yml new file mode 100644 index 000000000..bc236962b --- /dev/null +++ b/eng/common/templates/stages/dotnet/publish-config-nonprod.yml @@ -0,0 +1,91 @@ +# This pipeline template injects the publish config for the dotnet-docker +# non-production (unofficial) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-prod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + publishConfig: + internalMirrorAcr: + server: $(acr-staging-test.server) + repoPrefix: $(mirrorRepoPrefix) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + serviceConnection: + name: $(internal-mirror-test.serviceConnectionName) + id: $(internal-mirror-test.serviceConnection.id) + clientId: $(internal-mirror-test.serviceConnection.clientId) + tenantId: $(testTenant) + + publicMirrorAcr: + server: $(public-mirror.server) + + buildAcr: + server: $(acr-staging-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + serviceConnection: + name: $(build-test.serviceConnectionName) + id: $(build-test.serviceConnection.id) + clientId: $(build-test.serviceConnection.clientId) + tenantId: $(testTenant) + + testServiceConnection: + name: $(test-nonprod.serviceConnectionName) + id: $(test-nonprod.serviceConnection.id) + clientId: $(test-nonprod.serviceConnection.clientId) + tenantId: $(testTenant) + + publishAcr: + server: $(acr-test.server) + resourceGroup: $(testResourceGroup) + subscription: $(testSubscription) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + serviceConnection: + name: $(publish-test.serviceConnectionName) + id: $(publish-test.serviceConnection.id) + clientId: $(publish-test.serviceConnection.clientId) + tenantId: $(testTenant) diff --git a/eng/common/templates/stages/dotnet/publish-config-prod.yml b/eng/common/templates/stages/dotnet/publish-config-prod.yml new file mode 100644 index 000000000..7ac47d690 --- /dev/null +++ b/eng/common/templates/stages/dotnet/publish-config-prod.yml @@ -0,0 +1,91 @@ +# This pipeline template injects the publish config for the dotnet-docker +# production (official) environment. +# The overall structure of this file should stay mostly in-sync with the +# publish-config-nonprod.yml template. + +parameters: +# By default, images are staged in repos that are prefixed with this pipeline +# build ID. This is makes it easy to look up which pipeline images were built +# from and vice versa. +- name: sourceBuildPipelineRunId + type: string + default: $(Build.BuildId) + +# This prefix is added to the staging repo when pushing images. If the trailing +# slash is omitted, it will not be added automatically. +- name: stagingRepoPrefix + type: string + default: "build-staging/" + +# Images will have this prefix added to their repo name when pushed to the +# publishing ACR. If the trailing slash is omitted, it will not be added +# automatically. +- name: publishRepoPrefix + type: string + default: "public/" + +# This template will have the publishConfig, internalProjectName, and +# publicProjectName parameters passed to it automatically. +- name: stagesTemplate + type: string + +# These parameters will be passed to the template referred to by the +# stagesTemplate parameter. +# Note: publishConfig, internalProjectName, and publicProjectName are passed +# automatically by this template. Don't define them in this parameter - they +# will get overwritten. +- name: stagesTemplateParameters + type: object + default: {} + + +stages: +- template: ${{ parameters.stagesTemplate }} + parameters: + ${{ insert }}: ${{ parameters.stagesTemplateParameters }} + + internalProjectName: "internal" + publicProjectName: "public" + + publishConfig: + internalMirrorAcr: + server: $(acr-staging.server) + repoPrefix: $(mirrorRepoPrefix) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + serviceConnection: + name: $(internal-mirror.serviceConnectionName) + id: $(internal-mirror.serviceConnection.id) + clientId: $(internal-mirror.serviceConnection.clientId) + tenantId: $(internal-mirror.serviceConnection.tenantId) + + publicMirrorAcr: + server: $(public-mirror.server) + + buildAcr: + server: $(acr-staging.server) + resourceGroup: $(acr-staging.resourceGroup) + subscription: $(acr-staging.subscription) + repoPrefix: "${{ parameters.stagingRepoPrefix }}${{ parameters.sourceBuildPipelineRunId }}/" + serviceConnection: + name: $(build.serviceConnectionName) + id: $(build.serviceConnection.id) + clientId: $(build.serviceConnection.clientId) + tenantId: $(build.serviceConnection.tenantId) + + testServiceConnection: + name: $(test.serviceConnectionName) + id: $(test.serviceConnection.id) + clientId: $(test.serviceConnection.clientId) + tenantId: $(test.serviceConnection.tenantId) + + publishAcr: + server: $(acr.server) + resourceGroup: $(acr.resourceGroup) + subscription: $(acr.subscription) + repoPrefix: "${{ parameters.publishRepoPrefix }}" + serviceConnection: + name: $(publish.serviceConnectionName) + id: $(publish.serviceConnection.id) + clientId: $(publish.serviceConnection.clientId) + tenantId: $(publish.serviceConnection.tenantId) diff --git a/eng/common/templates/stages/dotnet/publish.yml b/eng/common/templates/stages/dotnet/publish.yml index d396174cb..4e1745fda 100644 --- a/eng/common/templates/stages/dotnet/publish.yml +++ b/eng/common/templates/stages/dotnet/publish.yml @@ -4,6 +4,7 @@ parameters: internalProjectName: null publicProjectName: null + publishConfig: null pool: "" isInternalServicingValidation: false isStandalonePublish: false @@ -18,6 +19,7 @@ stages: parameters: internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }} + publishConfig: ${{ parameters.publishConfig }} isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }} isStandalonePublish: ${{ parameters.isStandalonePublish }} sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} @@ -45,10 +47,6 @@ stages: displayName: "Set Custom Repo Name Var" - ${{ parameters.customPublishInitSteps }} - ${{ if eq(variables['System.TeamProject'], parameters.internalProjectName) }}: - customPublishVariables: - - group: DotNet-AllOrgs-Darc-Pats - pool: ${{ if ne(parameters.pool, '') }}: ${{ parameters.pool }} diff --git a/eng/common/templates/stages/publish.yml b/eng/common/templates/stages/publish.yml index 18e7c492a..bd294e1c3 100644 --- a/eng/common/templates/stages/publish.yml +++ b/eng/common/templates/stages/publish.yml @@ -5,6 +5,8 @@ parameters: internalProjectName: null publicProjectName: null + publishConfig: null + isInternalServicingValidation: false isStandalonePublish: false @@ -70,6 +72,7 @@ stages: parameters: pool: ${{ parameters.pool }} internalProjectName: ${{ parameters.internalProjectName }} + publishConfig: ${{ parameters.publishConfig }} customPublishVariables: ${{ parameters.customPublishVariables }} customInitSteps: ${{ parameters.customPublishInitSteps }} sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }} diff --git a/eng/common/templates/stages/setup-service-connections.yml b/eng/common/templates/stages/setup-service-connections.yml index f4c74ead6..2ef74e90c 100644 --- a/eng/common/templates/stages/setup-service-connections.yml +++ b/eng/common/templates/stages/setup-service-connections.yml @@ -6,6 +6,10 @@ parameters: - name: pool type: object + default: + name: $(default1ESInternalPoolName) + image: $(default1ESInternalPoolImage) + os: linux # serviceConnections object shape: # - name: string - name: serviceConnections diff --git a/eng/common/templates/steps/annotate-eol-digests.yml b/eng/common/templates/steps/annotate-eol-digests.yml index e6306bc57..0e7d5c32f 100644 --- a/eng/common/templates/steps/annotate-eol-digests.yml +++ b/eng/common/templates/steps/annotate-eol-digests.yml @@ -1,5 +1,10 @@ parameters: - dataFile: null +- name: publishConfig + type: object +# Path to EOL annotation data JSON file generated by 'generateEolAnnotationData*' command +- name: dataFile + type: string + steps: - script: mkdir -p $(Build.ArtifactStagingDirectory)/annotation-digests displayName: Create Annotation Digests Directory @@ -9,16 +14,16 @@ steps: displayName: Annotate EOL Images serviceConnections: - name: acr - id: $(publish.serviceConnection.id) - tenantId: $(publish.serviceConnection.tenantId) - clientId: $(publish.serviceConnection.clientId) + id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }} + tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }} + clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }} internalProjectName: internal condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) args: >- annotateEolDigests ${{ parameters.dataFile }} - $(acr.server) - $(publishRepoPrefix) + ${{ parameters.publishConfig.publishAcr.server }} + ${{ parameters.publishConfig.publishAcr.repoPrefix }} $(artifactsPath)/annotation-digests/annotation-digests.txt $(dryRunArg) - template: /eng/common/templates/steps/publish-artifact.yml@self diff --git a/eng/common/templates/steps/init-matrix-build-publish.yml b/eng/common/templates/steps/init-matrix-build-publish.yml index 95408c66d..5ba018fee 100644 --- a/eng/common/templates/steps/init-matrix-build-publish.yml +++ b/eng/common/templates/steps/init-matrix-build-publish.yml @@ -5,6 +5,7 @@ # - Publishing images parameters: + publishConfig: null versionsRepoRef: "" versionsRepoPath: "versions" @@ -15,11 +16,11 @@ steps: path: s/${{ parameters.versionsRepoPath }} persistCredentials: true fetchDepth: 1 - condition: and(succeeded(), eq(variables['publishImageInfo'], 'true')) + condition: succeeded() - powershell: | $commonMatrixAndBuildOptions = "--source-repo $(publicGitRepoUri)" if ("$(System.TeamProject)" -eq "internal" -and "$(Build.Reason)" -ne "PullRequest") { - $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix $(mirrorRepoPrefix) --registry-override $(acr-staging.server)" + $commonMatrixAndBuildOptions = "$commonMatrixAndBuildOptions --source-repo-prefix ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }} --registry-override ${{ parameters.publishConfig.buildAcr.server }}" } if ("$(System.TeamProject)" -eq "public" -and "$(public-mirror.server)" -ne "") { diff --git a/eng/common/templates/steps/publish-readmes.yml b/eng/common/templates/steps/publish-readmes.yml index 4f7f77dd0..4cf80ecf7 100644 --- a/eng/common/templates/steps/publish-readmes.yml +++ b/eng/common/templates/steps/publish-readmes.yml @@ -6,7 +6,7 @@ steps: - script: > $(runImageBuilderCmd) publishMcrDocs --manifest '$(manifest)' - --registry-override '$(acr.server)' + --registry-override '${{ parameters.publishConfig.publishAcr.server }}' '$(mcrDocsRepoInfo.userName)' '$(mcrDocsRepoInfo.email)' $(mcrDocsRepoInfo.authArgs) diff --git a/eng/common/templates/steps/set-dry-run.yml b/eng/common/templates/steps/set-dry-run.yml index d8e3aeaaf..44e3062b0 100644 --- a/eng/common/templates/steps/set-dry-run.yml +++ b/eng/common/templates/steps/set-dry-run.yml @@ -1,11 +1,30 @@ +parameters: + name: publishConfig + type: object + steps: - powershell: | - # Use dry-run option for certain publish operations if this is not a production build - $dryRunArg="" - if (-not "$(officialRepoPrefixes)".Split(',').Contains("$(publishRepoPrefix)") ` - -or "$(System.TeamProject)" -eq "$(publicProjectName)") + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") { + # Don't use dry-run mode for unofficial builds, since they publish to a + # non-production environment + $dryRunArg="" + } + elseif ("$(System.TeamProject)" -eq "$(publicProjectName)") + { + # Public builds need to use dry-run mode since they don't publish anywhere. $dryRunArg="--dry-run" } + elseif (-not "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) + { + # If we're running an internal build on an official pipeline but not + # publishing to an official repo prefix, then use dry run mode. + $dryRunArg="--dry-run" + } + else + { + $dryRunArg="" + } + echo "##vso[task.setvariable variable=dryRunArg]$dryRunArg" displayName: Set dry-run arg for non-prod diff --git a/eng/common/templates/steps/test-images-linux-client.yml b/eng/common/templates/steps/test-images-linux-client.yml index 1af54d0c1..e97f015c7 100644 --- a/eng/common/templates/steps/test-images-linux-client.yml +++ b/eng/common/templates/steps/test-images-linux-client.yml @@ -1,6 +1,7 @@ parameters: preBuildValidation: false internalProjectName: null + publishConfig: null condition: true customInitSteps: [] sourceBuildPipelineRunId: "" @@ -23,7 +24,7 @@ steps: additionalTestArgs="$additionalTestArgs -TestCategories pre-build" else if [ "${{ variables['System.TeamProject'] }}" == "${{ parameters.internalProjectName }}" ] && [ "${{ variables['Build.Reason'] }}" != "PullRequest" ]; then - additionalTestArgs="$additionalTestArgs -PullImages -Registry $(acr-staging.server) -RepoPrefix $(stagingRepoPrefix) -ImageInfoPath $(artifactsPath)/image-info.json" + additionalTestArgs="$additionalTestArgs -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" if [ "$TESTCATEGORIESOVERRIDE" != "" ]; then additionalTestArgs="$additionalTestArgs -TestCategories $TESTCATEGORIESOVERRIDE" fi @@ -46,13 +47,13 @@ steps: - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self parameters: displayName: Docker login - serviceConnection: $(test.serviceConnectionName) + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} condition: and(succeeded(), ${{ parameters.condition }}) command: >- $azLoginArgs = '--service-principal --tenant $env:AZURE_TENANT_ID -u $env:AZURE_CLIENT_ID --federated-token $env:AZURE_FEDERATED_TOKEN'; docker exec -e AZURE_TENANT_ID=$env:tenantId -e AZURE_CLIENT_ID=$env:servicePrincipalId -e AZURE_FEDERATED_TOKEN=$env:idToken $(testRunner.container) pwsh -File $(engCommonRelativePath)/Invoke-WithRetry.ps1 - "az login $azLoginArgs; az acr login -n $(acr-staging.server)" + "az login $azLoginArgs; az acr login -n ${{ parameters.publishConfig.buildAcr.server }}" - ${{ if eq(parameters.preBuildValidation, 'false') }}: - template: /eng/common/templates/steps/download-build-artifact.yml@self parameters: @@ -75,7 +76,7 @@ steps: displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker exec $(testRunner.container) docker logout $(acr-staging.server) + - script: docker exec $(testRunner.container) docker logout ${{ parameters.publishConfig.buildAcr.server }} displayName: Docker logout condition: and(always(), ${{ parameters.condition }}) continueOnError: true diff --git a/eng/common/templates/steps/test-images-windows-client.yml b/eng/common/templates/steps/test-images-windows-client.yml index d53a42b66..00fa062fd 100644 --- a/eng/common/templates/steps/test-images-windows-client.yml +++ b/eng/common/templates/steps/test-images-windows-client.yml @@ -1,5 +1,6 @@ parameters: internalProjectName: null + publishConfig: null condition: true customInitSteps: [] sourceBuildPipelineRunId: "" @@ -14,17 +15,17 @@ steps: - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self parameters: displayName: Docker login - serviceConnection: $(test.serviceConnectionName) + serviceConnection: ${{ parameters.publishConfig.testServiceConnection.name }} dockerClientOS: windows condition: and(succeeded(), ${{ parameters.condition }}) command: >- az login --service-principal --tenant $env:tenantId -u $env:servicePrincipalId --federated-token $env:idToken; - $accessToken = $(az acr login -n $(acr-staging.server) --expose-token --query accessToken --output tsv); - docker login $(acr-staging.server) -u 00000000-0000-0000-0000-000000000000 -p $accessToken + $accessToken = $(az acr login -n ${{ parameters.publishConfig.buildAcr.server }} --expose-token --query accessToken --output tsv); + docker login ${{ parameters.publishConfig.buildAcr.server }} -u 00000000-0000-0000-0000-000000000000 -p $accessToken - ${{ parameters.customInitSteps }} - powershell: | if ("${{ variables['System.TeamProject'] }}" -eq "${{ parameters.internalProjectName }}" -and "${{ variables['Build.Reason'] }}" -ne "PullRequest") { - $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${env:ACR-STAGING_SERVER} -RepoPrefix $env:STAGINGREPOPREFIX -ImageInfoPath $(artifactsPath)/image-info.json" + $additionalTestArgs="$env:ADDITIONALTESTARGS -PullImages -Registry ${{ parameters.publishConfig.buildAcr.server }} -RepoPrefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} -ImageInfoPath $(artifactsPath)/image-info.json" } echo "##vso[task.setvariable variable=additionalTestArgs]$additionalTestArgs" displayName: Set Test Variables @@ -49,7 +50,7 @@ steps: displayName: Test Images condition: and(succeeded(), ${{ parameters.condition }}) - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - - script: docker logout $(acr-staging.server) + - script: docker logout ${{ parameters.publishConfig.buildAcr.server }} displayName: Docker logout condition: and(always(), ${{ parameters.condition }}) continueOnError: true diff --git a/eng/common/templates/steps/validate-branch.yml b/eng/common/templates/steps/validate-branch.yml index 03ca58da7..0bfcf9c9f 100644 --- a/eng/common/templates/steps/validate-branch.yml +++ b/eng/common/templates/steps/validate-branch.yml @@ -1,28 +1,39 @@ parameters: + publishConfig: null internalProjectName: null steps: - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: - powershell: | + if ("$env:ONEESPT_BUILDTYPE" -eq "Unofficial") + { + echo "Build is from an unofficial pipeline, continuing..." + exit 0 + } + if ("$(officialBranches)".Split(',').Contains("$(sourceBranch)") ` - -and "$(officialRepoPrefixes)".Split(',').Contains("$(publishRepoPrefix)")) + -and "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) { echo "Conditions met for official build, continuing..." exit 0 } - if (-not "$(officialRepoPrefixes)".Split(',').Contains("$(publishRepoPrefix)")) + if (-not "$(officialRepoPrefixes)".Split(',').Contains("${{ parameters.publishConfig.publishAcr.repoPrefix }}")) { echo "This build is a test build, continuing..." exit 0 } - if ("$(overrideOfficialBranchValidation)" -eq "true") + if ("${{ variables['overrideOfficialBranchValidation'] }}" -eq "true") { echo "Variable overrideOfficialBranchValidation is set to true, continuing..." exit 0 } echo "##vso[task.logissue type=error]Official builds must be done from an official branch ($(officialBranches)) and repo prefix ($(officialRepoPrefixes))." + echo "Build definition: $(Build.DefinitionName)" + echo "1ESPT build type: $(OneESPT.BuildType)" + echo "Current branch: $(sourceBranch)" + echo "Publish repo prefix: ${{ parameters.publishConfig.publishAcr.repoPrefix }}" exit 1 displayName: Validate Branch diff --git a/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml b/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml index a3c84ddfe..f8ad85d57 100644 --- a/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml +++ b/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml @@ -1,8 +1,19 @@ parameters: - imageInfoPath: null - minQueueTime: null - condition: true - dryRunArg: "" +- name: publishConfig + type: object + +- name: imageInfoPath + type: string + +- name: minQueueTime + type: string + +- name: dryRunArg + type: string + +- name: condition + type: string + default: "true" steps: - template: /eng/common/templates/steps/run-imagebuilder.yml@self @@ -19,7 +30,7 @@ steps: waitForMcrImageIngestion '${{ parameters.imageInfoPath }}' --manifest '$(manifest)' - --repo-prefix '$(publishRepoPrefix)' + --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}' --min-queue-time '${{ parameters.minQueueTime }}' --timeout '$(mcrImageIngestionTimeout)' $(manifestVariables) diff --git a/eng/common/templates/task-prefix-decorator.yml b/eng/common/templates/task-prefix-decorator.yml index 4f1a86ce6..598bfbe24 100644 --- a/eng/common/templates/task-prefix-decorator.yml +++ b/eng/common/templates/task-prefix-decorator.yml @@ -1,4 +1,4 @@ -# This Azure Pipelines template is adds a prefix to the display name of each +# This Azure Pipelines template adds a prefix to the display name of each # task passed through the `stages` parameter. When used in conjunction with # an "extends" template which injects a lot of tasks into the pipeline, the # added prefix helps to identify which tasks were passed through this template @@ -20,7 +20,7 @@ parameters: default: null # These stages will be modified and passed to the `baseTemplate` as the -# `stages` parameter. The +# `stages` parameter. - name: stages type: stageList default: [] diff --git a/eng/common/templates/variables/common.yml b/eng/common/templates/variables/common.yml index 65d4ce9d8..525709f45 100644 --- a/eng/common/templates/variables/common.yml +++ b/eng/common/templates/variables/common.yml @@ -1,24 +1,7 @@ -parameters: -# sourceBuildPipelineRunId should be overridden when skipping a build to run -# tests or publish images that were produced in a different pipeline run. -# Defaults to $(Build.BuildId) which refers to the current pipeline run. -- name: sourceBuildPipelineRunId - type: string - default: "" - variables: - template: /eng/common/templates/variables/docker-images.yml@self - template: /eng/common/templates/variables/common-paths.yml@self -# stagingRepoPrefix defines where images will be pushed to in the staging repo, -# which is defined in the $(acr-staging.server) variable. -- ${{ if ne(parameters.sourceBuildPipelineRunId, '') }}: - - name: stagingRepoPrefix - value: build-staging/${{ parameters.sourceBuildPipelineRunId }}/ -- ${{ else }}: - - name: stagingRepoPrefix - value: build-staging/$(Build.BuildId)/ - - name: publishReadme value: true - name: publishImageInfo diff --git a/eng/common/templates/variables/docker-images.yml b/eng/common/templates/variables/docker-images.yml index 0d25f8821..5907ae4a2 100644 --- a/eng/common/templates/variables/docker-images.yml +++ b/eng/common/templates/variables/docker-images.yml @@ -1,5 +1,5 @@ variables: - imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2770436 + imageNames.imageBuilderName: mcr.microsoft.com/dotnet-buildtools/image-builder:2781076 imageNames.imageBuilder: $(imageNames.imageBuilderName) imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId) imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux3.0-docker-testrunner diff --git a/eng/common/templates/variables/dotnet/build-test-publish.yml b/eng/common/templates/variables/dotnet/build-test-publish.yml index 005018a0e..94732039b 100644 --- a/eng/common/templates/variables/dotnet/build-test-publish.yml +++ b/eng/common/templates/variables/dotnet/build-test-publish.yml @@ -1,16 +1,7 @@ # Common variables for building/testing/publishing in the .NET team's pipelines -parameters: -# sourceBuildPipelineRunId should be overridden when skipping a build to run -# tests or publish images that were produced in a different pipeline run. -# Defaults to $(Build.BuildId) which refers to the current pipeline run. -- name: sourceBuildPipelineRunId - type: string - default: "" variables: - template: /eng/common/templates/variables/dotnet/common.yml@self - parameters: - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - name: commonVersionsImageInfoPath value: build-info/docker @@ -20,8 +11,10 @@ variables: value: ./tests/run-tests.ps1 - name: testResultsDirectory value: tests/Microsoft.DotNet.Docker.Tests/TestResults/ + - name: officialRepoPrefixes value: public/,internal/private/,unlisted/ + readonly: true - name: mcrDocsRepoInfo.userName value: $(gitHubApp.marDocsUpdater.userName) diff --git a/eng/common/templates/variables/dotnet/common.yml b/eng/common/templates/variables/dotnet/common.yml index 69665b241..bb39d82c2 100644 --- a/eng/common/templates/variables/dotnet/common.yml +++ b/eng/common/templates/variables/dotnet/common.yml @@ -1,15 +1,5 @@ -parameters: -# sourceBuildPipelineRunId should be overridden when skipping a build to run -# tests or publish images that were produced in a different pipeline run. -# Defaults to $(Build.BuildId) which refers to the current pipeline run. -- name: sourceBuildPipelineRunId - type: string - default: "" - variables: - template: /eng/common/templates/variables/common.yml@self - parameters: - sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - name: publicProjectName value: public diff --git a/eng/common/templates/variables/dotnet/secrets-unofficial.yml b/eng/common/templates/variables/dotnet/secrets-unofficial.yml new file mode 100644 index 000000000..d55d9318d --- /dev/null +++ b/eng/common/templates/variables/dotnet/secrets-unofficial.yml @@ -0,0 +1,5 @@ +variables: +- group: DotNet-Docker-Secrets-Unofficial + +- name: dockerHubRegistryCreds + value: --registry-creds 'docker.io=$(dotnetDockerHubBot.userName);$(BotAccount-dotnet-dockerhub-bot-PAT)' diff --git a/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml b/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml index a2cd49c63..4e25e5093 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-all-pr.yml @@ -19,15 +19,16 @@ resources: ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} stages: -- template: stages/build-test-publish-repo.yml +- template: /eng/common/templates/stages/dotnet/publish-config-nonprod.yml@self parameters: - internalProjectName: ${{ variables.internalProjectName }} - publicProjectName: ${{ variables.publicProjectName }} - versionsRepoRef: VersionsRepo - linuxAmdBuildJobTimeout: 480 - linuxArmBuildJobTimeout: 300 - linuxAmd64Pool: - name: NetCore-Public-XL - demands: ImageOverride -equals build.Ubuntu.2204.amd64.open - customBuildInitSteps: - - template: /eng/pipelines/steps/install-cross-build-prereqs.yml + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + stagesTemplate: /eng/pipelines/stages/build-test-publish-repo.yml@self + stagesTemplateParameters: + versionsRepoRef: VersionsRepo + linuxAmdBuildJobTimeout: 480 + linuxArmBuildJobTimeout: 300 + linuxAmd64Pool: + name: NetCore-Public-XL + demands: ImageOverride -equals build.Ubuntu.2204.amd64.open + customBuildInitSteps: + - template: /eng/pipelines/steps/install-cross-build-prereqs.yml diff --git a/eng/pipelines/dotnet-buildtools-prereqs-all.yml b/eng/pipelines/dotnet-buildtools-prereqs-all.yml index 43b9efe7e..ed16c900e 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-all.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-all.yml @@ -32,7 +32,7 @@ schedules: variables: - template: /eng/pipelines/variables/common.yml@self - parameters: + parameters: disableMatrixTrimming: ${{ parameters.disableMatrixTrimming }} sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} - template: /eng/common/templates/variables/dotnet/secrets.yml@self @@ -48,41 +48,41 @@ resources: ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} extends: - template: /eng/common/templates/1es-official.yml@self + template: /eng/common/templates/1es.yml@self parameters: - serviceConnections: - - name: $(internal-mirror.serviceConnectionName) - - name: $(build.serviceConnectionName) - - name: $(publish.serviceConnectionName) - - name: $(marStatus.serviceConnectionName) + reposToExcludeFromScanning: + - VersionsRepo stages: - - template: /eng/pipelines/stages/build-test-publish-repo.yml@self + - template: /eng/common/templates/stages/dotnet/publish-config-prod.yml@self parameters: - internalProjectName: ${{ variables.internalProjectName }} - publicProjectName: ${{ variables.publicProjectName }} - versionsRepoRef: VersionsRepo - linuxAmdBuildJobTimeout: 480 - linuxArmBuildJobTimeout: 300 - linuxAmd64Pool: - name: NetCore1ESPool-Internal-XL - image: 1es-ubuntu-2204 - os: linux - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - customCopyBaseImagesInitSteps: - - template: /eng/pipelines/steps/set-base-image-override-options.yml@self - parameters: - variableName: customCopyBaseImagesArgs - dockerfileOs: (centos|debian) - baseOverrideRegistry: $(overrideRegistry) # Comes from DotNet-Docker-Common variable group - customBuildInitSteps: - - template: /eng/pipelines/steps/install-cross-build-prereqs.yml@self - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - template: /eng/pipelines/steps/set-base-image-override-options.yml@self - parameters: - variableName: imageBuilderBuildArgs - dockerfileOs: (centos|debian) - baseOverrideRegistry: $(overrideRegistry) # Comes from DotNet-Docker-Common variable group - # Force all images to be rebuilt when triggered by the schedule - # See https://github.com/dotnet/dotnet-buildtools-prereqs-docker/issues/1224 - ${{ if eq(variables['Build.Reason'], 'Schedule') }}: - noCache: true + sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }} + stagesTemplate: /eng/pipelines/stages/build-test-publish-repo.yml@self + stagesTemplateParameters: + versionsRepoRef: VersionsRepo + linuxAmdBuildJobTimeout: 480 + linuxArmBuildJobTimeout: 300 + linuxAmd64Pool: + name: NetCore1ESPool-Internal-XL + image: 1es-ubuntu-2204 + os: linux + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + customCopyBaseImagesInitSteps: + - template: /eng/pipelines/steps/set-base-image-override-options.yml@self + parameters: + variableName: customCopyBaseImagesArgs + dockerfileOs: (centos|debian) + baseOverrideRegistry: $(overrideRegistry) # Comes from DotNet-Docker-Common variable group + customBuildInitSteps: + - template: /eng/pipelines/steps/install-cross-build-prereqs.yml@self + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - template: /eng/pipelines/steps/set-base-image-override-options.yml@self + parameters: + variableName: imageBuilderBuildArgs + dockerfileOs: (centos|debian) + baseOverrideRegistry: $(overrideRegistry) # Comes from DotNet-Docker-Common variable group + # Force all images to be rebuilt when triggered by the schedule + # See https://github.com/dotnet/dotnet-buildtools-prereqs-docker/issues/1224 + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + noCache: true + additionalServiceConnections: + - name: $(marStatus.serviceConnectionName) diff --git a/eng/pipelines/dotnet-buildtools-prereqs-eng.yml b/eng/pipelines/dotnet-buildtools-prereqs-eng.yml index 8a1109877..02efd41a3 100644 --- a/eng/pipelines/dotnet-buildtools-prereqs-eng.yml +++ b/eng/pipelines/dotnet-buildtools-prereqs-eng.yml @@ -29,12 +29,12 @@ resources: ref: ${{ variables['gitHubVersionsRepoInfo.branch'] }} stages: -- template: /eng/pipelines/stages/build-test-publish-repo.yml@self +- template: /eng/common/templates/stages/dotnet/publish-config-nonprod.yml@self parameters: - internalProjectName: ${{ variables.internalProjectName }} - publicProjectName: ${{ variables.publicProjectName }} - versionsRepoRef: VersionsRepo - linuxAmdBuildJobTimeout: 360 - linuxArmBuildJobTimeout: 300 - customBuildInitSteps: - - template: /eng/pipelines/steps/install-cross-build-prereqs.yml + stagesTemplate: /eng/pipelines/stages/build-test-publish-repo.yml@self + stagesTemplateParameters: + versionsRepoRef: VersionsRepo + linuxAmdBuildJobTimeout: 360 + linuxArmBuildJobTimeout: 300 + customBuildInitSteps: + - template: /eng/pipelines/steps/install-cross-build-prereqs.yml diff --git a/eng/pipelines/stages/build-test-publish-repo.yml b/eng/pipelines/stages/build-test-publish-repo.yml index fe78c377d..05618f10e 100644 --- a/eng/pipelines/stages/build-test-publish-repo.yml +++ b/eng/pipelines/stages/build-test-publish-repo.yml @@ -1,4 +1,5 @@ parameters: + publishConfig: null noCache: false internalProjectName: null publicProjectName: null @@ -8,10 +9,22 @@ parameters: customCopyBaseImagesInitSteps: [] linuxAmd64Pool: '' versionsRepoRef: null + additionalServiceConnections: [] stages: +- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/common/templates/stages/setup-service-connections.yml@self + parameters: + serviceConnections: + - name: ${{ parameters.publishConfig.internalMirrorAcr.serviceConnection.name }} + - name: ${{ parameters.publishConfig.buildAcr.serviceConnection.name }} + - name: ${{ parameters.publishConfig.publishAcr.serviceConnection.name }} + - ${{ each serviceConnection in parameters.additionalServiceConnections }}: + - name: ${{ serviceConnection.name }} + - template: /eng/common/templates/stages/dotnet/build-test-publish-repo.yml@self parameters: + publishConfig: ${{ parameters.publishConfig }} noCache: ${{ parameters.noCache }} internalProjectName: ${{ parameters.internalProjectName }} publicProjectName: ${{ parameters.publicProjectName }}