Add support for custom templates + customizations (#853)

Changes included in this PR:
- Support custom template repository = Point to an existing AL-Go
repository as a template, like:


![image](https://github.com/microsoft/AL-Go/assets/10775043/683a394b-3a9e-42b7-acce-8ab39b76a74f)

In this case, org/myptetemplate is a standard AL-Go repository, which
gets updated as any other AL-Go repository using Update AL-Go System
Files. This is a way of standardizing your repository based on type
specific AL-Go templates (which can be private or public).

Normally, Update AL-Go System Files follows this mechanism:

| File Type | Repository file(s) | AL-Go Template |
| :-- | :-- | :-: |
| Workflows | .github/workflows/*.yaml | <- |
| Repo Scripts | .github/*.ps1 | <- |
| Project Scripts | .AL-Go/*.ps1 | <- |

Workflows and scripts are read from the template repository and modified
based on settings and saved to the repository.

Using a custom template repository, the picture looks like this:

| File Type | Repository file(s) | Custom Template | AL-Go Template |
| :-- | :-- | :-: | :-: |
| Workflows | .github/workflows/*.yaml | <- Customizations | <- |
| Workflows | .github/workflows/*.yaml | <- New files | |
| Repo Scripts | .github/*.ps1 | | <- |
| Repo Scripts | .github/*.ps1 | <- New files | |
| Repo Settings | .github/templateRepoSettings.doNotEdit.json | <-
.github/AL-Go-Settings.json | |
| Project Scripts | .AL-Go/*.ps1 | | <- |
| Project Scripts | .AL-Go/*.ps1 | <- New files | |
| Project Settings | .github/templateProjectSettings.doNotEdit.json | <-
.AL-Go/settings.json | |

Workflows and scripts are read from the "real" template repository and
modified based on settings. Customizations for workflows in the custom
template repository are applied.

New workflows and scripts from the custom template repository are copied
over.

Repo and Project settings from the custom template repository are copied
over to new settings files, which are applied before the final repos
repo and project settings in order to allow the indirect template to add
new defaults to settings to be applied to all repositories.

The final repository can also contain customizations to workflows and
the winner here is the indirect template repository (if a custom job
exists in both places).

This PR: freddydk/customized#17 - is an example
of a repository with customizations of CI/CD and _BuildALGoProject,
which runs Update AL-Go System Files and gets customizations from the
indirect template repository (both settings, scripts and workflow
customizations)

Documentation on customization capabilities is here:
https://github.com/freddydk/AL-Go/blob/customize/Scenarios/CustomizingALGoForGitHub.md

---------

Co-authored-by: freddydk <[email protected]>
Co-authored-by: Maria Zhelezova <[email protected]>
Co-authored-by: Patrick Schiefer <[email protected]>
Co-authored-by: Philipp Stöhr <[email protected]>
Co-authored-by: Johannes Wikman <[email protected]>
Co-authored-by: Alexander Holstrup <[email protected]>

Author: 6 people

.github/workflows/CI.yaml

@@ -19,6 +19,7 @@ jobs:
     runs-on: [ ubuntu-latest ]
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit

.github/workflows/CleanupTempRepos.yaml

@@ -26,6 +26,7 @@ jobs:
     name: Cleanup Temp Repos
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit

.github/workflows/Deploy.yaml

@@ -53,6 +53,7 @@ jobs:
       defaultBcContainerHelperVersion: ${{ steps.CreateInputs.outputs.defaultBcContainerHelperVersion }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -85,6 +86,7 @@ jobs:
     needs: [ Inputs ]
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -126,6 +128,7 @@ jobs:
       contents: write
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit

.github/workflows/E2E.yaml

@@ -52,6 +52,7 @@ jobs:
       githubOwner: ${{ steps.check.outputs.githubOwner }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -113,6 +114,7 @@ jobs:
       appSourceAppRepo: ${{ steps.setup.outputs.appSourceAppRepo }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -147,6 +149,7 @@ jobs:
       scenarios: ${{ steps.Analyze.outputs.scenarios }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -236,6 +239,7 @@ jobs:
     strategy: ${{ fromJson(needs.Analyze.outputs.scenarios) }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -272,6 +276,7 @@ jobs:
     strategy: ${{ fromJson(needs.Analyze.outputs.scenarios) }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -308,6 +313,7 @@ jobs:
     strategy: ${{ fromJson(needs.Analyze.outputs.publictestruns) }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -356,6 +362,7 @@ jobs:
     strategy: ${{ fromJson(needs.Analyze.outputs.privatetestruns) }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit
@@ -404,6 +411,7 @@ jobs:
     strategy: ${{ fromJson(needs.Analyze.outputs.releases) }}
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit

.github/workflows/powershell.yaml

@@ -21,6 +21,7 @@ jobs:
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit

.github/workflows/pre-commit.yml

@@ -14,6 +14,7 @@ jobs:
     runs-on: windows-latest
     steps:
     - name: Harden Runner
+      if: github.repository_owner == 'microsoft'
       uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
       with:
         egress-policy: audit

.github/workflows/scorecard-analysis.yml

@@ -18,6 +18,7 @@ jobs:
 
     steps:
       - name: Harden Runner
+        if: github.repository_owner == 'microsoft'
         uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1
         with:
           egress-policy: audit

Actions/AL-Go-Helper.ps1

@@ -13,13 +13,15 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S
 $ALGoFolderName = '.AL-Go'
 $ALGoSettingsFile = Join-Path '.AL-Go' 'settings.json'
 $RepoSettingsFile = Join-Path '.github' 'AL-Go-Settings.json'
+$CustomTemplateRepoSettingsFile = Join-Path '.github' 'AL-Go-TemplateRepoSettings.doNotEdit.json'
+$CustomTemplateProjectSettingsFile = Join-Path '.github' 'AL-Go-TemplateProjectSettings.doNotEdit.json'
 [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPushBranches', Justification = 'False positive.')]
 $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' )
 [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')]
 $defaultCICDPullRequestBranches = @( 'main' )
 $runningLocal = $local.IsPresent
 $defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip"
-$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","GitHubAppClientId")
+$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","GitHubAppClientId","EnvironmentName")
 
 $runAlPipelineOverrides = @(
     "DockerPull"
@@ -38,6 +40,8 @@ $runAlPipelineOverrides = @(
     "InstallMissingDependencies"
     "PreCompileApp"
     "PostCompileApp"
+    "PipelineInitialize"
+    "PipelineFinalize"
 )
 
 # Well known AppIds
@@ -676,13 +680,15 @@ function GetDefaultSettings
 
 # Read settings from the settings files
 # Settings are read from the following files:
-# - ALGoOrgSettings (github Variable)                 = Organization settings variable
-# - .github/AL-Go-Settings.json                       = Repository Settings file
-# - ALGoRepoSettings (github Variable)                = Repository settings variable
-# - <project>/.AL-Go/settings.json                    = Project settings file
-# - .github/<workflowName>.settings.json              = Workflow settings file
-# - <project>/.AL-Go/<workflowName>.settings.json     = Project workflow settings file
-# - <project>/.AL-Go/<userName>.settings.json         = User settings file
+# - ALGoOrgSettings (github Variable)                    = Organization settings variable
+# - .github/AL-Go-TemplateRepoSettings.doNotEdit.json    = Repository settings from custom template
+# - .github/AL-Go-Settings.json                          = Repository Settings file
+# - ALGoRepoSettings (github Variable)                   = Repository settings variable
+# - .github/AL-Go-TemplateProjectSettings.doNotEdit.json = Project settings from custom template
+# - <project>/.AL-Go/settings.json                       = Project settings file
+# - .github/<workflowName>.settings.json                 = Workflow settings file
+# - <project>/.AL-Go/<workflowName>.settings.json        = Project workflow settings file
+# - <project>/.AL-Go/<userName>.settings.json            = User settings file
 function ReadSettings {
     Param(
         [string] $baseFolder = "$ENV:GITHUB_WORKSPACE",
@@ -740,20 +746,31 @@ function ReadSettings {
         $orgSettingsVariableObject = $orgSettingsVariableValue | ConvertFrom-Json
         $settingsObjects += @($orgSettingsVariableObject)
     }
+
+    # Read settings from repository settings file
+    $customTemplateRepoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $CustomTemplateRepoSettingsFile)
+    $settingsObjects += @($customTemplateRepoSettingsObject)
+
     # Read settings from repository settings file
     $repoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $RepoSettingsFile)
     $settingsObjects += @($repoSettingsObject)
+
     # Read settings from repository settings variable (parameter)
     if ($repoSettingsVariableValue) {
         $repoSettingsVariableObject = $repoSettingsVariableValue | ConvertFrom-Json
         $settingsObjects += @($repoSettingsVariableObject)
     }
+
     if ($project) {
+        # Read settings from repository settings file
+        $customTemplateProjectSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $CustomTemplateProjectSettingsFile)
+        $settingsObjects += @($customTemplateProjectSettingsObject)
         # Read settings from project settings file
         $projectFolder = Join-Path $baseFolder $project -Resolve
         $projectSettingsObject = GetSettingsObject -Path (Join-Path $projectFolder $ALGoSettingsFile)
         $settingsObjects += @($projectSettingsObject)
     }
+
     if ($workflowName) {
         # Read settings from workflow settings file
         $workflowSettingsObject = GetSettingsObject -Path (Join-Path $gitHubFolder "$workflowName.settings.json")
@@ -766,6 +783,7 @@ function ReadSettings {
             $settingsObjects += @($projectWorkflowSettingsObject, $userSettingsObject)
         }
     }
+
     foreach($settingsJson in $settingsObjects) {
         if ($settingsJson) {
             MergeCustomObjectIntoOrderedDictionary -dst $settings -src $settingsJson

Actions/AL-Go-TestRepoHelper.ps1

@@ -1,4 +1,6 @@
-function Test-Property {
+. (Join-Path $PSScriptRoot "AL-Go-Helper.ps1")
+
+function Test-Property {
     Param(
         [HashTable] $json,
         [string] $settingsDescription,
@@ -169,13 +171,16 @@ function TestALGoRepository {
     # Test .json files are formatted correctly
     # Get-ChildItem needs -force to include folders starting with . (e.x. .github / .AL-Go) on Linux
     Get-ChildItem -Path $baseFolder -Filter '*.json' -Recurse -Force | ForEach-Object {
-        if ($_.Directory.Name -eq '.AL-Go' -and $_.BaseName -eq 'settings') {
+        if ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($ALGoSettingsFile)) -and $_.Name -eq ([System.IO.Path]::GetFileName($ALGoSettingsFile))) {
             Test-JsonFile -jsonFile $_.FullName -baseFolder $baseFolder -type 'Project'
         }
-        elseif ($_.Directory.Name -eq '.github' -and $_.BaseName -like '*ettings') {
-            if ($_.BaseName -eq 'AL-Go-Settings') {
+        elseif ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($RepoSettingsFile)) -and $_.BaseName -like '*ettings') {
+            if ($_.Name -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.Name -eq ([System.IO.Path]::GetFileName($CustomTemplateRepoSettingsFile))) {
                 $type = 'Repo'
             }
+            elseif ($_.Name -eq ([System.IO.Path]::GetFileName($CustomTemplateProjectSettingsFile))) {
+                $type = 'Project'
+            }
             else {
                 $type = 'Workflow'
             }

Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1

@@ -1,8 +1,8 @@
 <#
 .SYNOPSIS
 Downloads a template repository and returns the path to the downloaded folder
-.PARAMETER headers
-The headers to use when calling the GitHub API
+.PARAMETER token
+The GitHub token / PAT to use for authentication (if the template repository is private/internal)
 .PARAMETER templateUrl
 The URL to the template repository
 .PARAMETER templateSha
@@ -12,12 +12,31 @@ If true, the latest SHA of the template repository will be downloaded
 #>
 function DownloadTemplateRepository {
     Param(
-        [hashtable] $headers,
+        [string] $token,
         [string] $templateUrl,
         [ref] $templateSha,
         [bool] $downloadLatest
     )
 
+    $templateRepositoryUrl = $templateUrl.Split('@')[0]
+    $templateRepository = $templateRepositoryUrl.Split('/')[-2..-1] -join '/'
+
+    # Use Authenticated API request if possible to avoid the 60 API calls per hour limit
+    $headers = GetHeaders -token $env:GITHUB_TOKEN -repository $templateRepository
+    try {
+        $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl
+    }
+    catch {
+        # Ignore error
+        $response = $null
+    }
+    if (-not $response -or $response.StatusCode -ne 200) {
+        # GITHUB_TOKEN doesn't have access to template repository, must be private/internal
+        # Get token with read permissions for the template repository
+        # NOTE that the GitHub app needs to be installed in the template repository for this to work
+        $headers = GetHeaders -token $token -repository $templateRepository
+    }
+
     # Construct API URL
     $apiUrl = $templateUrl.Split('@')[0] -replace "^(https:\/\/github\.com\/)(.*)$", "$ENV:GITHUB_API_URL/repos/`$2"
 
@@ -427,7 +446,7 @@ function IsDirectALGo {
 
 function GetSrcFolder {
     Param(
-        [hashtable] $repoSettings,
+        [string] $repoType,
         [string] $templateUrl,
         [string] $templateFolder,
         [string] $srcPath
@@ -439,7 +458,7 @@ function GetSrcFolder {
         return ''
     }
     if (IsDirectALGo -templateUrl $templateUrl) {
-        switch ($repoSettings.type) {
+        switch ($repoType) {
             "PTE" {
                 $typePath = "Per Tenant Extension"
             }
@@ -486,48 +505,50 @@ function GetModifiedSettingsContent {
     else {
         # Change the $schema property to be the same as the source settings file (add it if it doesn't exist)
         $schemaKey = '$schema'
-        $schemaValue = $srcSettings."$schemaKey"
+        if ($srcSettings.PSObject.Properties.Name -eq $schemaKey) {
+            $schemaValue = $srcSettings."$schemaKey"
 
-        $dstSettings | Add-Member -MemberType NoteProperty -Name "$schemaKey" -Value $schemaValue -Force
+            $dstSettings | Add-Member -MemberType NoteProperty -Name "$schemaKey" -Value $schemaValue -Force
 
-        # Make sure the $schema property is the first property in the object
-        $dstSettings = $dstSettings | Select-Object @{ Name = '$schema'; Expression = { $_.'$schema' } }, * -ExcludeProperty '$schema'
+            # Make sure the $schema property is the first property in the object
+            $dstSettings = $dstSettings | Select-Object @{ Name = '$schema'; Expression = { $_.'$schema' } }, * -ExcludeProperty '$schema'
+        }
     }
 
-
     return $dstSettings | ConvertTo-JsonLF
 }
 
 function UpdateSettingsFile {
     Param(
         [string] $settingsFile,
-        [hashtable] $updateSettings,
-        [hashtable] $additionalSettings = @{}
+        [hashtable] $updateSettings
     )
 
+    $modified = $false
     # Update Repo Settings file with the template URL
     if (Test-Path $settingsFile) {
         $settings = Get-Content $settingsFile -Encoding UTF8 | ConvertFrom-Json
     }
     else {
         $settings = [PSCustomObject]@{}
+        $modified = $true
     }
     foreach($key in $updateSettings.Keys) {
         if ($settings.PSObject.Properties.Name -eq $key) {
-            $settings."$key" = $updateSettings."$key"
+            if ($settings."$key" -ne $updateSettings."$key") {
+                $settings."$key" = $updateSettings."$key"
+                $modified = $true
+            }
         }
         else {
             # Add the property if it doesn't exist
             $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $updateSettings."$key"
+            $modified = $true
         }
     }
-    # Grab settings from additionalSettings if they are not already in settings
-    foreach($key in $additionalSettings.Keys) {
-        if (!($settings.PSObject.Properties.Name -eq $key)) {
-            # Add the property if it doesn't exist
-            $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $additionalSettings."$key"
-        }
+    if ($modified) {
+        # Save the file with LF line endings and UTF8 encoding
+        $settings | Set-JsonContentLF -path $settingsFile
     }
-    # Save the file with LF line endings and UTF8 encoding
-    $settings | Set-JsonContentLF -path $settingsFile
+    return $modified
 }