[nightly] Update common Docker engineering infrastructure with latest

eng/common/templates/1es.yml

@@ -0,0 +1,70 @@
+# When extending this template, pipelines using a repository resource containing versions files for image caching must
+# do the following:
+#
+# - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines
+# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans
+# - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans
+#
+# If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in
+# `$(Build.SourcesDirectory)/versions`, as it will not be scanned.
+
+parameters:
+- name: stages
+  type: stageList
+  default: []
+# List of repositories that will be excluded from SDL scanning. This should
+# only be used when including other repos without building their source code.
+# E.g. for the dotnet/versions repo.
+- name: reposToExcludeFromScanning
+  type: object
+  default: []
+# The pool that will be used for initializing service connections.
+- name: pool
+  type: object
+  default:
+    name: $(default1ESInternalPoolName)
+    image: $(default1ESInternalPoolImage)
+    os: linux
+# The pool that will be used for SDL jobs.
+- name: sourceAnalysisPool
+  type: object
+  default:
+    name: $(defaultSourceAnalysisPoolName)
+    image: $(defaultSourceAnalysisPoolImage)
+    os: windows
+
+resources:
+  repositories:
+  - repository: 1ESPipelineTemplates
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+
+extends:
+  template: /eng/common/templates/task-prefix-decorator.yml@self
+  parameters:
+    baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates
+    templateParameters:
+      pool: ${{ parameters.pool }}
+      sdl:
+        # Required for unofficial pipelines because we rely on the ManifestGeneratorTask that is
+        # automatically installed by 1ES pipeline templates
+        sbom:
+          enabled: true
+        binskim:
+          enabled: true
+        componentgovernance:
+          ignoreDirectories: $(Build.SourcesDirectory)/versions
+          showAlertLink: true
+        policheck:
+          enabled: true
+        ${{ if ne(length(parameters.reposToExcludeFromScanning), 0) }}:
+          sourceRepositoriesToScan:
+            exclude:
+            - ${{ each repo in parameters.reposToExcludeFromScanning }}:
+              - repository: ${{ repo }}
+        sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
+        tsa:
+          enabled: true
+    stages:
+    - ${{ parameters.stages }}

eng/common/templates/jobs/build-images.yml

@@ -6,6 +6,7 @@ parameters:
   buildJobTimeout: 60
   commonInitStepsForMatrixAndBuild: []
   customInitSteps: []
+  publishConfig: null
   noCache: false
   internalProjectName: null
   publicProjectName: null
@@ -48,13 +49,13 @@ jobs:
       # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
-      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
+      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
       if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
-        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push"
+        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} --push"
       }
 
       # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
-      if ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true") {
+      if ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true") {
         $imageBuilderBuildArgs = "$imageBuilderBuildArgs --image-info-source-path $(versionsBasePath)$(imageInfoVersionsPath)"
       }
 
@@ -66,10 +67,12 @@ jobs:
       name: BuildImages
       displayName: Build Images
       serviceConnections:
+      # "name" here refers to the argument name, not the service connection name.
+      # It should probably be changed to "argName".
       - name: acr
-        id: $(build.serviceConnection.id)
-        tenantId: $(build.serviceConnection.tenantId)
-        clientId: $(build.serviceConnection.clientId)
+        id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }}
+        tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }}
+        clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }}
       - ${{ if eq(parameters.isInternalServicingValidation, true) }}:
         - name: storage
           id: $(dotnetstaging.serviceConnection.id)
@@ -86,8 +89,8 @@ jobs:
         --architecture $(architecture)
         --retry
         --digests-out-var 'builtImages'
-        --acr-subscription '$(acr-staging.subscription)'
-        --acr-resource-group '$(acr-staging.resourceGroup)'
+        --acr-subscription '${{ parameters.publishConfig.buildAcr.subscription }}'
+        --acr-resource-group '${{ parameters.publishConfig.buildAcr.resourceGroup }}'
         $(manifestVariables)
         $(imageBuilderBuildArgs)
   - template: /eng/common/templates/steps/publish-artifact.yml@self
@@ -134,7 +137,7 @@ jobs:
         # Manifest tool docs: https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/secure-supply-chain/custom-sbom-generation-workflows
         $images -Split ',' | ForEach-Object {
           echo "Generating SBOM for $_";
-          $formattedImageName = $_.Replace('$(acr-staging.server)/$(stagingRepoPrefix)', "").Replace('/', '_').Replace(':', '_');
+          $formattedImageName = $_.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_');
           $sbomChildDir = "$(sbomDirectory)/$formattedImageName";
           New-Item -Type Directory -Path $sbomChildDir > $null;
           & $dotnetPath "$manifestToolDllPath" `

eng/common/templates/jobs/copy-base-images-staging.yml

@@ -5,6 +5,9 @@ parameters:
 - name: pool
   type: object
   default: {}
+- name: publishConfig
+  type: object
+  default: null
 - name: customInitSteps
   type: stepList
   default: []
@@ -22,12 +25,5 @@ jobs:
     pool: ${{ parameters.pool }}
     customInitSteps: ${{ parameters.customInitSteps }}
     additionalOptions: ${{ parameters.additionalOptions }}
-    acr:
-      server: $(acr-staging.server)
-      serviceConnection:
-        tenantId: $(internal-mirror.serviceConnection.tenantId)
-        clientId: $(internal-mirror.serviceConnection.clientId)
-        id: $(internal-mirror.serviceConnection.id)
-      subscription: $(acr-staging.subscription)
-      resourceGroup: $(acr-staging.resourceGroup)
-    repoPrefix: $(mirrorRepoPrefix)
+    acr: ${{ parameters.publishConfig.internalMirrorAcr }}
+    repoPrefix: ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }}

eng/common/templates/jobs/generate-matrix.yml

@@ -6,6 +6,7 @@ parameters:
   isTestStage: false
   internalProjectName: null
   noCache: false
+  publishConfig: null
   customInitSteps: []
   commonInitStepsForMatrixAndBuild: []
   sourceBuildPipelineRunId: ""
@@ -20,6 +21,7 @@ jobs:
   - ${{ parameters.customInitSteps }}
   - template: /eng/common/templates/steps/validate-branch.yml@self
     parameters:
+      publishConfig: ${{ parameters.publishConfig }}
       internalProjectName: ${{ parameters.internalProjectName }}
   - template: /eng/common/templates/steps/set-image-info-path-var.yml
     parameters:
@@ -36,7 +38,7 @@ jobs:
       if ("${{ parameters.isTestStage}}" -eq "true") {
         $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json"
       }
-      elseif ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") {
+      elseif ("$(pipelineDisabledCache)" -ne "true" -and "${{ parameters.noCache }}" -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") {
         # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
         $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images"
       }

eng/common/templates/jobs/publish.yml

@@ -1,6 +1,7 @@
 parameters:
   pool: {}
   internalProjectName: null
+  publishConfig: null
   customInitSteps: []
   customPublishVariables: []
   sourceBuildPipelineDefinitionId: ""
@@ -20,7 +21,7 @@ jobs:
   - name: imageBuilder.commonCmdArgs
     value: >-
       --manifest '$(manifest)'
-      --registry-override '$(acr.server)'
+      --registry-override '${{ parameters.publishConfig.publishAcr.server }}'
       $(manifestVariables)
       $(imageBuilder.queueArgs)
   - name: publishNotificationRepoName
@@ -46,7 +47,7 @@ jobs:
   steps:
   - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
     parameters:
-      cloneVersionsRepo: ${{ variables.publishImageInfo }}
+      publishConfig: ${{ parameters.publishConfig }}
       versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
   - template: /eng/common/templates/steps/retain-build.yml@self
@@ -94,19 +95,19 @@ jobs:
       displayName: Copy Images
       serviceConnections:
       - name: acr
-        id: $(publish.serviceConnection.id)
-        tenantId: $(publish.serviceConnection.tenantId)
-        clientId: $(publish.serviceConnection.clientId)
+        id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }}
+        tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }}
+        clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }}
       internalProjectName: ${{ parameters.internalProjectName }}
       args: >-
         copyAcrImages
-        '$(acr.subscription)'
-        '$(acr.resourceGroup)'
-        '$(stagingRepoPrefix)'
-        '$(acr-staging.server)'
+        '${{ parameters.publishConfig.buildAcr.subscription }}'
+        '${{ parameters.publishConfig.buildAcr.resourceGroup }}'
+        '${{ parameters.publishConfig.buildAcr.repoPrefix }}'
+        '${{ parameters.publishConfig.buildAcr.server }}'
         --os-type '*'
         --architecture '*'
-        --repo-prefix '$(publishRepoPrefix)'
+        --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}'
         --image-info '$(imageInfoContainerDir)/image-info.json'
         $(dryRunArg)
         $(imageBuilder.pathArgs)
@@ -117,15 +118,15 @@ jobs:
       displayName: Publish Manifest
       serviceConnections:
       - name: acr
-        id: $(publish.serviceConnection.id)
-        tenantId: $(publish.serviceConnection.tenantId)
-        clientId: $(publish.serviceConnection.clientId)
+        id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }}
+        tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }}
+        clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }}
       internalProjectName: ${{ parameters.internalProjectName }}
       dockerClientOS: ${{ parameters.dockerClientOS }}
       args: >-
         publishManifest
         '$(imageInfoContainerDir)/image-info.json'
-        --repo-prefix '$(publishRepoPrefix)'
+        --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}'
         --os-type '*'
         --architecture '*'
         $(dryRunArg)
@@ -206,18 +207,18 @@ jobs:
       displayName: Generate EOL Annotation Data
       serviceConnections:
       - name: acr
-        id: $(publish.serviceConnection.id)
-        tenantId: $(publish.serviceConnection.tenantId)
-        clientId: $(publish.serviceConnection.clientId)
+        id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }}
+        tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }}
+        clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }}
       internalProjectName: internal
       condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
       args: >-
         generateEolAnnotationData
         '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json'
         '$(imageInfoContainerDir)/full-image-info-orig.json'
         '$(imageInfoContainerDir)/full-image-info-new.json'
-        '$(acr.server)'
-        '$(publishRepoPrefix)'
+        '${{ parameters.publishConfig.publishAcr.server }}'
+        '${{ parameters.publishConfig.publishAcr.repoPrefix }}'
         $(generateEolAnnotationDataExtraOptions)
         $(dryRunArg)
 
@@ -233,6 +234,7 @@ jobs:
   - template: /eng/common/templates/steps/annotate-eol-digests.yml@self
     parameters:
       internalProjectName: ${{ parameters.internalProjectName }}
+      publishConfig: ${{ parameters.publishConfig }}
       dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json
 
   - script: >
@@ -271,7 +273,7 @@ jobs:
       $(gitHubNotificationsRepoInfo.authArgs)
       '$(gitHubNotificationsRepoInfo.org)'
       '$(gitHubNotificationsRepoInfo.repo)'
-      --repo-prefix '$(publishRepoPrefix)'
+      --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}'
       --task "🟪 Copy Images"
       --task "🟪 Publish Manifest"
       --task "🟪 Wait for Image Ingestion"

eng/common/templates/jobs/test-images-linux-client.yml

@@ -5,6 +5,7 @@ parameters:
   testJobTimeout: 60
   preBuildValidation: false
   internalProjectName: null
+  publishConfig: null
   customInitSteps: []
   sourceBuildPipelineRunId: ""
 
@@ -24,5 +25,6 @@ jobs:
     parameters:
       preBuildValidation: ${{ parameters.preBuildValidation }}
       internalProjectName: ${{ parameters.internalProjectName }}
+      publishConfig: ${{ parameters.publishConfig }}
       customInitSteps: ${{ parameters.customInitSteps }}
       sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}

eng/common/templates/jobs/test-images-windows-client.yml

@@ -4,6 +4,7 @@ parameters:
   matrix: {}
   testJobTimeout: 60
   internalProjectName: null
+  publishConfig: null
   customInitSteps: []
   sourceBuildPipelineRunId: ""
 
@@ -19,5 +20,6 @@ jobs:
   - template: /eng/common/templates/steps/test-images-windows-client.yml@self
     parameters:
       internalProjectName: ${{ parameters.internalProjectName }}
+      publishConfig: ${{ parameters.publishConfig }}
       customInitSteps: ${{ parameters.customInitSteps }}
       sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}