[main] Update common Docker engineering infrastructure with latest

eng/common/templates/1es-official.yml

@@ -2,7 +2,7 @@
 # do the following:
 #
 # - Do not rely on any source code from the versions repo so as to not circumvent SDL and CG guidelines
-# - The versions repo resource must be named `InternalVersionsRepo` or `PublicVersionsRepo` to avoid SDL scans
+# - The versions repo resource must be named `VersionsRepo` to avoid SDL scans
 # - The versions repo must be checked out to `$(Build.SourcesDirectory)/versions` to avoid CG scans
 #
 # If the pipeline is not using a separate repository resource, ensure that there is no source code checked out in
@@ -57,14 +57,14 @@ extends:
           enabled: true
         sourceRepositoriesToScan:
           exclude:
-          - repository: InternalVersionsRepo
-          - repository: PublicVersionsRepo
+          - repository: VersionsRepo
         sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
         tsa:
           enabled: true
     stages:
-    - template: /eng/common/templates/stages/setup-service-connections.yml@self
-      parameters:
-        pool: ${{ parameters.pool }}
-        serviceConnections: ${{ parameters.serviceConnections }}
+    - ${{ if gt(length(parameters.serviceConnections), 0) }}:
+      - template: /eng/common/templates/stages/setup-service-connections.yml@self
+        parameters:
+          pool: ${{ parameters.pool }}
+          serviceConnections: ${{ parameters.serviceConnections }}
     - ${{ parameters.stages }}

eng/common/templates/1es-unofficial.yml

@@ -71,8 +71,9 @@ extends:
         tsa:
           enabled: true
     stages:
-    - template: /eng/common/templates/stages/setup-service-connections.yml@self
-      parameters:
-        pool: ${{ parameters.pool }}
-        serviceConnections: ${{ parameters.serviceConnections }}
+    - ${{ if gt(length(parameters.serviceConnections), 0) }}:
+      - template: /eng/common/templates/stages/setup-service-connections.yml@self
+        parameters:
+          pool: ${{ parameters.pool }}
+          serviceConnections: ${{ parameters.serviceConnections }}
     - ${{ parameters.stages }}

eng/common/templates/jobs/publish.yml

@@ -5,6 +5,11 @@ parameters:
   customPublishVariables: []
   sourceBuildPipelineDefinitionId: ""
   sourceBuildPipelineRunId: ""
+  versionsRepoRef: null
+  versionsRepoPath: ""
+  # When true, overrides the commit SHA in merged image info files to use the current repository commit.
+  # This ensures that updated images reference the correct commit in their commitUrl properties.
+  overrideImageInfoCommit: false
 
 jobs:
 - job: Publish
@@ -31,16 +36,28 @@ jobs:
     value: $(artifactsPath)/imageInfo
   - name: sourceBuildIdOutputDir
     value: $(Build.ArtifactStagingDirectory)/sourceBuildId
+  - name: commitOverrideArg
+    ${{ if eq(parameters.overrideImageInfoCommit, true) }}:
+      value: --commit-override $(Build.SourceVersion)
+    ${{ else }}:
+      value: ''
   - ${{ parameters.customPublishVariables }}
 
   steps:
+  - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
+    parameters:
+      cloneVersionsRepo: ${{ variables.publishImageInfo }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
+
   - template: /eng/common/templates/steps/retain-build.yml@self
 
   - template: /eng/common/templates/steps/init-docker-linux.yml@self
 
   - pwsh: |
       $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI
       echo "##vso[task.setvariable variable=azdoOrgName]$azdoOrgName"
+      $versionsRepoRoot = "$(Pipeline.Workspace)/s/${{ parameters.versionsRepoPath }}"
+      echo "##vso[task.setvariable variable=versionsRepoRoot]$versionsRepoRoot"
     displayName: Set Publish Variables
 
   - ${{ parameters.customInitSteps }}
@@ -133,18 +150,21 @@ jobs:
   - template: /eng/common/templates/steps/publish-readmes.yml@self
     parameters:
       dryRunArg: $(dryRunArg)
-      condition: and(succeeded(), eq(variables['publishReadme'], 'true'))
+      condition: false
 
   - script: mkdir -p $(Build.ArtifactStagingDirectory)/eol-annotation-data
     displayName: Create EOL Annotation Data Directory
 
-  - powershell: >-
-      $(engCommonPath)/Invoke-WithRetry.ps1
-      "curl -fSL
-      --output $(imageInfoHostDir)/full-image-info-orig.json
-      https://raw.githubusercontent.com/$(gitHubVersionsRepoInfo.org)/$(gitHubVersionsRepoInfo.repo)/refs/heads/$(gitHubVersionsRepoInfo.branch)/$(gitHubImageInfoVersionsPath)"
+  - script: |-
+      cd $(versionsRepoRoot)
+      git pull origin $(gitHubVersionsRepoInfo.branch)
+    condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
+    displayName: Pull Latest Changes from Versions Repo
+
+  - script: >-
+      cp $(versionsRepoRoot)/$(gitHubImageInfoVersionsPath) $(imageInfoHostDir)/full-image-info-orig.json
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
-    displayName: Download Latest Image Info
+    displayName: Copy Latest Image Info from Versions Repo
 
   - script: >
       $(runImageBuilderCmd) mergeImageInfo
@@ -155,6 +175,7 @@ jobs:
       --manifest $(manifest)
       --publish
       --initial-image-info-path $(imageInfoContainerDir)/full-image-info-orig.json
+      $(commitOverrideArg)
     condition: and(succeeded(), eq(variables['publishImageInfo'], 'true'))
     displayName: Merge Image Info
 
@@ -254,8 +275,6 @@ jobs:
       --task "🟪 Copy Images"
       --task "🟪 Publish Manifest"
       --task "🟪 Wait for Image Ingestion"
-      --task "🟪 Publish Readmes"
-      --task "🟪 Wait for MCR Doc Ingestion"
       --task "🟪 Publish Image Info"
       --task "🟪 Ingest Kusto Image Info"
       --task "🟪 Generate EOL Annotation Data"

eng/common/templates/stages/build-and-test.yml

@@ -22,8 +22,7 @@ parameters:
   internalProjectName: null
   publicProjectName: null
 
-  internalVersionsRepoRef: null
-  publicVersionsRepoRef: null
+  versionsRepoRef: ""
 
   isInternalServicingValidation: false
 
@@ -88,11 +87,9 @@ stages:
       noCache: ${{ parameters.noCache }}
       customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
@@ -102,11 +99,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -120,11 +115,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -138,11 +131,9 @@ stages:
       dockerClientOS: linux
       buildJobTimeout: ${{ parameters.linuxArmBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -156,11 +147,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -174,11 +163,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -192,17 +179,14 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-      internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-      publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
       isInternalServicingValidation: ${{ parameters.isInternalServicingValidation }}
   - template: /eng/common/templates/jobs/build-images.yml@self
     parameters:
@@ -212,11 +196,9 @@ stages:
       dockerClientOS: windows
       buildJobTimeout: ${{ parameters.windowsAmdBuildJobTimeout }}
       commonInitStepsForMatrixAndBuild:
-      - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+      - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
         parameters:
-          noCache: ${{ parameters.noCache }}
-          internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-          publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+          versionsRepoRef: ${{ parameters.versionsRepoRef }}
       customInitSteps: ${{ parameters.customBuildInitSteps }}
       noCache: ${{ parameters.noCache }}
       internalProjectName: ${{ parameters.internalProjectName }}
@@ -266,11 +248,9 @@ stages:
         customInitSteps: ${{ parameters.customGenerateMatrixInitSteps }}
         sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
         commonInitStepsForMatrixAndBuild:
-        - template: /eng/common/templates/steps/common-init-for-matrix-and-build.yml@self
+        - template: /eng/common/templates/steps/init-matrix-build-publish.yml@self
           parameters:
-            noCache: ${{ parameters.noCache }}
-            internalVersionsRepoRef: ${{ parameters.internalVersionsRepoRef }}
-            publicVersionsRepoRef: ${{ parameters.publicVersionsRepoRef }}
+            versionsRepoRef: ${{ parameters.versionsRepoRef }}
     - template: /eng/common/templates/jobs/test-images-linux-client.yml@self
       parameters:
         name: Linux_amd64

eng/common/templates/stages/dotnet/build-and-test.yml

@@ -30,6 +30,8 @@ parameters:
   internalProjectName: null
   publicProjectName: null
 
+  versionsRepoRef: null
+
 stages:
 - template: /eng/common/templates/stages/build-and-test.yml@self
   parameters:
@@ -51,8 +53,9 @@ stages:
     testMatrixType: ${{ parameters.testMatrixType }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
 
-    internalVersionsRepoRef: InternalVersionsRepo
-    publicVersionsRepoRef: PublicVersionsRepo
+    # Only clone versions repo if we need to reference it during the build in order to cache images.
+    ${{ if eq(parameters.noCache, false) }}:
+      versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
     # Linux AMD64
     linuxAmd64Pool:

eng/common/templates/stages/dotnet/build-test-publish-repo.yml

@@ -32,6 +32,7 @@ parameters:
   # Other common parameters
   internalProjectName: null
   publicProjectName: null
+  versionsRepoRef: null
 
 
 stages:
@@ -61,6 +62,7 @@ stages:
     # Other
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}
 
 - template: /eng/common/templates/stages/dotnet/publish.yml@self
   parameters:
@@ -70,3 +72,4 @@ stages:
     internalProjectName: ${{ parameters.internalProjectName }}
     publicProjectName: ${{ parameters.publicProjectName }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}

eng/common/templates/stages/dotnet/publish.yml

@@ -10,6 +10,8 @@ parameters:
   customPublishInitSteps: []
   sourceBuildPipelineDefinitionId: ''
   sourceBuildPipelineRunId: ''
+  versionsRepoRef: null
+  overrideImageInfoCommit: false
 
 stages:
 - template: /eng/common/templates/stages/publish.yml@self
@@ -20,6 +22,8 @@ stages:
     isStandalonePublish: ${{ parameters.isStandalonePublish }}
     sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
     sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+    versionsRepoRef: ${{ parameters.versionsRepoRef }}
+    overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }}
 
     customPublishInitSteps:
     - pwsh: |

eng/common/templates/stages/publish.yml

@@ -14,6 +14,15 @@ parameters:
   sourceBuildPipelineDefinitionId: ''
   sourceBuildPipelineRunId: ''
 
+  versionsRepoRef: null
+  versionsRepoPath: "versions"
+
+  # When true, any updated images will have the SHA in their commit URL updated
+  # to the commit that this pipeline is running on, instead of the commit they
+  # were built from. Use in combination with isStandalonePublish to ensure that
+  # internally built images still reference public Dockerfiles.
+  overrideImageInfoCommit: false
+
 ################################################################################
 # Publish Images
 ################################################################################
@@ -65,3 +74,6 @@ stages:
         customInitSteps: ${{ parameters.customPublishInitSteps }}
         sourceBuildPipelineDefinitionId: ${{ parameters.sourceBuildPipelineDefinitionId }}
         sourceBuildPipelineRunId: ${{ parameters.sourceBuildPipelineRunId }}
+        versionsRepoRef: ${{ parameters.versionsRepoRef }}
+        versionsRepoPath: ${{ parameters.versionsRepoPath }}
+        overrideImageInfoCommit: ${{ parameters.overrideImageInfoCommit }}

eng/common/templates/stages/setup-service-connections.yml

@@ -22,7 +22,7 @@ stages:
     displayName: Setup service connections
     pool: ${{ parameters.pool }}
     steps:
-
+    - checkout: none
     - ${{ each serviceConnection in parameters.serviceConnections }}:
       - task: AzureCLI@2
         displayName: Setup ${{ serviceConnection.name }}