[main] Update common Docker engineering infrastructure with latest

eng/common/templates/1es-official.yml

@@ -2,7 +2,7 @@
 # do the following:
 #
 # - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines
-# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans
+# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans
 # - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans
 #
 # If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in
@@ -57,14 +57,14 @@ extends:
           enabled: true
         sourceRepositoriesToScan:
           exclude:
-          - repository: InternalVersionsRepo
-          - repository: PublicVersionsRepo
+          - repository: VersionsRepo
         sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
         tsa:
           enabled: true
     stages:
-    - template: /eng/common/templates/stages/setup-service-connections.yml@self
-      parameters:
-        pool: ${{ parameters.pool }}
-        serviceConnections: ${{ parameters.serviceConnections }}
+    - ${{ if gt(length(parameters.serviceConnections), 0) }}:
+      - template: /eng/common/templates/stages/setup-service-connections.yml@self
+        parameters:
+          pool: ${{ parameters.pool }}
+          serviceConnections: ${{ parameters.serviceConnections }}
     - ${{ parameters.stages }}

eng/common/templates/1es-unofficial.yml

@@ -71,8 +71,9 @@ extends:
         tsa:
           enabled: true
     stages:
-    - template: /eng/common/templates/stages/setup-service-connections.yml@self
-      parameters:
-        pool: ${{ parameters.pool }}
-        serviceConnections: ${{ parameters.serviceConnections }}
+    - ${{ if gt(length(parameters.serviceConnections), 0) }}:
+      - template: /eng/common/templates/stages/setup-service-connections.yml@self
+        parameters:
+          pool: ${{ parameters.pool }}
+          serviceConnections: ${{ parameters.serviceConnections }}
     - ${{ parameters.stages }}

eng/common/templates/jobs/build-images.yml

@@ -49,7 +49,7 @@ jobs:
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
       $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
-      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") {
+      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
         $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push"
       }
 
@@ -70,6 +70,11 @@ jobs:
         id: $(build.serviceConnection.id)
         tenantId: $(build.serviceConnection.tenantId)
         clientId: $(build.serviceConnection.clientId)
+      - ${{ if eq(parameters.isInternalServicingValidation, true) }}:
+        - name: storage
+          id: $(dotnetstaging.serviceConnection.id)
+          tenantId: $(dotnetstaging.serviceConnection.tenantId)
+          clientId: $(dotnetstaging.serviceConnection.clientId)
       internalProjectName: ${{ parameters.internalProjectName }}
       dockerClientOS: ${{ parameters.dockerClientOS }}
       args: >-
@@ -92,7 +97,7 @@ jobs:
       displayName: Publish Image Info File Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
     # The following task depends on the SBOM Manifest Generator task installed on the agent.
     # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves.
     - powershell: |
@@ -144,11 +149,11 @@ jobs:
         }
       displayName: Generate SBOMs
       condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
-  - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}:
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
     - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
       parameters:
         condition: ne(variables.testScriptPath, '')
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
     - template: /eng/common/templates/steps/publish-artifact.yml@self
       parameters:
         path: $(sbomDirectory)

eng/common/templates/jobs/publish.yml

@@ -5,6 +5,11 @@ parameters:
   customPublishVariables: []
   sourceBuildPipelineDefinitionId: ""
   sourceBuildPipelineRunId: ""
+  versionsRepoRef: null
+  versionsRepoPath: ""
+  # When true, overrides the commit SHA in merged image info files to use the current repository commit.
+  # This ensures that updated images reference the correct commit in their commitUrl properties.
+  overrideImageInfoCommit: false
 
 jobs:
 - job: Publish
@@ -31,16 +36,28 @@ jobs:
     value: $(artifactsPath)/imageInfo
   - name: sourceBuildIdOutputDir
     value: $(Build.ArtifactStagingDirectory)/sourceBuildId
+  - name: commitOverrideArg
+    ${{ if eq(parameters.overrideImageInfoCommit, true) }}:
+      value: --commit-override $(Build.SourceVersion)
+    ${{ else }}:
+      value: ''
   - ${{ parameters.customPublishVariables }}
 
   steps:
+  - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
+    parameters:
+      cloneVersionsRepo: ${{ variables.publishImageInfo }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+
   - template: /eng/common/templates/steps/retain-build.yml@self
 
   - template: /eng/common/templates/steps/init-docker-linux.yml@self
 
   - pwsh: |
       $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI
       echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName"
+      $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}"
+      echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot"
     displayName: Set Publish Variables
 
   - ${{ parameters.customInitSteps }}
@@ -138,13 +155,16 @@ jobs:
   - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
     displayName: Create EOL Annotation Data Directory
 
-  - powershell: >-
-      $(engCommonPath)/Invoke-WithRetry.ps1
-      "curl -fSL
-      --output $(imageInfoHostDir)/full-image-info-orig.json
-      https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)"
+  - script: |-
+      cd $(versionsRepoRoot)
+      git pull origin $(gitHubVersionsRepoInfo.branch)
+    condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
+    displayName: Pull Latest Changes from Versions Repo
+
+  - script: >-
+      cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
-    displayName: Download Latest Image Info
+    displayName: Copy Latest Image Info from Versions Repo
 
   - script: >
       $(runImageBuilderCmd) mergeImageInfo
@@ -155,6 +175,7 @@ jobs:
       --manifest $(manifest)
       --publish
       --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json
+      $(commitOverrideArg)
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
     displayName: Merge Image Info
 

eng/common/templates/stages/build-and-test.yml

@@ -22,8 +22,7 @@ parameters:
   internalProjectName: null
   publicProjectName: null
 
-  internalVersionsRepoRef: null
-  publicVersionsRepoRef: null
+  versionsRepoRef: ""
 
   isInternalServicingValidation: false
 
@@ -51,6 +50,7 @@ stages:
   condition: and(succeeded(), contains(variables['stages'], 'build'))
   dependsOn: []
   jobs:
+
   - template: /eng/common/templates/jobs/test-images-linux-client.yml@self
     parameters:
       name: PreBuildValidation
@@ -69,12 +69,14 @@ stages:
             echo "##vso[task.setvariable variable=osVersions]"
             echo "##vso[task.setvariable variable=architecture]"
           displayName: Initialize Test Variables
+
   - template: /eng/common/templates/jobs/copy-base-images-staging.yml@self
     parameters:
       name: CopyBaseImages
       pool: ${{ parameters.linuxAmd64Pool }}
       additionalOptions: "--manifest '$(manifest)' $(imageBuilder.pathArgs) $(manifestVariables)"
       customInitSteps: ${{ parameters.customCopyBaseImagesInitSteps }}
+
   - template: /eng/common/templates/jobs/generate-matrix.yml@self
     parameters:
       matrixType: ${{ parameters.buildMatrixType }}
@@ -85,12 +87,10 @@ stages:
       noCache: ${{ parameters.noCache }}
       customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
+
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
       name: Linux_amd64
@@ -99,12 +99,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -118,12 +115,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -137,12 +131,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -156,12 +147,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -175,12 +163,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -194,18 +179,14 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
       isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
@@ -215,12 +196,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
-          isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -243,7 +221,7 @@ stages:
 ################################################################################
 # Test Images
 ################################################################################
-- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
+- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
   - stage: Test
     dependsOn: Post_Build
     condition: "
@@ -270,11 +248,9 @@ stages:
         customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
         sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
         commonInitStepsForMatrixAndBuild:
-        - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
           parameters:
-            noCache: ${{ parameters.noCache }}
-            internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-            publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+            versionsRepoRef: ${{ parameters.versionsRepoRef }}
     - template: /eng/common/templates/jobs/test-images-linux-client.yml@self
       parameters:
         name: Linux_amd64

eng/common/templates/stages/dotnet/build-and-test.yml

@@ -30,6 +30,8 @@ parameters:
   internalProjectName: null
   publicProjectName: null
 
+  versionsRepoRef: null
+
 stages:
 - template: /eng/common/templates/stages/build-and-test.yml@self
   parameters:
@@ -51,8 +53,9 @@ stages:
     testMatrixType: ${{ parameters.testMatrixType }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
 
-    internalVersionsRepoRef: InternalVersionsRepo
-    publicVersionsRepoRef: PublicVersionsRepo
+    # Only clone versions repo if we need to reference it during the build in order to cache images.
+    ${{ if eq(parameters.noCache, false) }}:
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
     # Linux AMD64
     linuxAmd64Pool:

eng/common/templates/stages/dotnet/build-test-publish-repo.yml

@@ -32,6 +32,7 @@ parameters:
   # Other common parameters
   internalProjectName: null
   publicProjectName: null
+  versionsRepoRef: null
 
 
 stages:
@@ -61,6 +62,7 @@ stages:
     # Other
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
 - template: /eng/common/templates/stages/dotnet/publish.yml@self
   parameters:
@@ -70,3 +72,4 @@ stages:
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}