diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 085cbd6d4..05f24de87 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -19,6 +19,7 @@ jobs: runs-on: [ ubuntu-latest ] steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit diff --git a/.github/workflows/CleanupTempRepos.yaml b/.github/workflows/CleanupTempRepos.yaml index 8302832d1..7c9ce7573 100644 --- a/.github/workflows/CleanupTempRepos.yaml +++ b/.github/workflows/CleanupTempRepos.yaml @@ -26,6 +26,7 @@ jobs: name: Cleanup Temp Repos steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit diff --git a/.github/workflows/Deploy.yaml b/.github/workflows/Deploy.yaml index 70a067564..650554d1d 100644 --- a/.github/workflows/Deploy.yaml +++ b/.github/workflows/Deploy.yaml @@ -53,6 +53,7 @@ jobs: defaultBcContainerHelperVersion: ${{ steps.CreateInputs.outputs.defaultBcContainerHelperVersion }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -85,6 +86,7 @@ jobs: needs: [ Inputs ] steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -126,6 +128,7 @@ jobs: contents: write steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit diff --git a/.github/workflows/E2E.yaml b/.github/workflows/E2E.yaml index 2df74d855..b2acf16bf 100644 --- a/.github/workflows/E2E.yaml +++ b/.github/workflows/E2E.yaml @@ -52,6 +52,7 @@ jobs: githubOwner: ${{ steps.check.outputs.githubOwner }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -113,6 +114,7 @@ jobs: appSourceAppRepo: ${{ steps.setup.outputs.appSourceAppRepo }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -147,6 +149,7 @@ jobs: scenarios: ${{ steps.Analyze.outputs.scenarios }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -236,6 +239,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.scenarios) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -272,6 +276,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.scenarios) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -308,6 +313,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.publictestruns) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -356,6 +362,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.privatetestruns) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit @@ -404,6 +411,7 @@ jobs: strategy: ${{ fromJson(needs.Analyze.outputs.releases) }} steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit diff --git a/.github/workflows/powershell.yaml b/.github/workflows/powershell.yaml index 04b73ed0c..2b71a42dd 100644 --- a/.github/workflows/powershell.yaml +++ b/.github/workflows/powershell.yaml @@ -21,6 +21,7 @@ jobs: security-events: write # for github/codeql-action/upload-sarif to upload SARIF results steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index c5ae698e4..3f4abcd73 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,6 +14,7 @@ jobs: runs-on: windows-latest steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit diff --git a/.github/workflows/scorecard-analysis.yml b/.github/workflows/scorecard-analysis.yml index c29330c4e..4efdc3b23 100644 --- a/.github/workflows/scorecard-analysis.yml +++ b/.github/workflows/scorecard-analysis.yml @@ -18,6 +18,7 @@ jobs: steps: - name: Harden Runner + if: github.repository_owner == 'microsoft' uses: step-security/harden-runner@002fdce3c6a235733a90a27c80493a3241e56863 # v2.12.1 with: egress-policy: audit diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 7711cc705..a9982e024 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -13,13 +13,15 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S $ALGoFolderName = '.AL-Go' $ALGoSettingsFile = Join-Path '.AL-Go' 'settings.json' $RepoSettingsFile = Join-Path '.github' 'AL-Go-Settings.json' +$CustomTemplateRepoSettingsFile = Join-Path '.github' 'AL-Go-TemplateRepoSettings.doNotEdit.json' +$CustomTemplateProjectSettingsFile = Join-Path '.github' 'AL-Go-TemplateProjectSettings.doNotEdit.json' [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPushBranches', Justification = 'False positive.')] $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent $defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step - ex. "https://github.com/organization/navcontainerhelper/archive/refs/heads/branch.zip" -$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","GitHubAppClientId") +$notSecretProperties = @("Scopes","TenantId","BlobName","ContainerName","StorageAccountName","ServerUrl","ppUserName","GitHubAppClientId","EnvironmentName") $runAlPipelineOverrides = @( "DockerPull" @@ -38,6 +40,8 @@ $runAlPipelineOverrides = @( "InstallMissingDependencies" "PreCompileApp" "PostCompileApp" + "PipelineInitialize" + "PipelineFinalize" ) # Well known AppIds @@ -676,13 +680,15 @@ function GetDefaultSettings # Read settings from the settings files # Settings are read from the following files: -# - ALGoOrgSettings (github Variable) = Organization settings variable -# - .github/AL-Go-Settings.json = Repository Settings file -# - ALGoRepoSettings (github Variable) = Repository settings variable -# - /.AL-Go/settings.json = Project settings file -# - .github/.settings.json = Workflow settings file -# - /.AL-Go/.settings.json = Project workflow settings file -# - /.AL-Go/.settings.json = User settings file +# - ALGoOrgSettings (github Variable) = Organization settings variable +# - .github/AL-Go-TemplateRepoSettings.doNotEdit.json = Repository settings from custom template +# - .github/AL-Go-Settings.json = Repository Settings file +# - ALGoRepoSettings (github Variable) = Repository settings variable +# - .github/AL-Go-TemplateProjectSettings.doNotEdit.json = Project settings from custom template +# - /.AL-Go/settings.json = Project settings file +# - .github/.settings.json = Workflow settings file +# - /.AL-Go/.settings.json = Project workflow settings file +# - /.AL-Go/.settings.json = User settings file function ReadSettings { Param( [string] $baseFolder = "$ENV:GITHUB_WORKSPACE", @@ -740,20 +746,31 @@ function ReadSettings { $orgSettingsVariableObject = $orgSettingsVariableValue | ConvertFrom-Json $settingsObjects += @($orgSettingsVariableObject) } + + # Read settings from repository settings file + $customTemplateRepoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $CustomTemplateRepoSettingsFile) + $settingsObjects += @($customTemplateRepoSettingsObject) + # Read settings from repository settings file $repoSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $RepoSettingsFile) $settingsObjects += @($repoSettingsObject) + # Read settings from repository settings variable (parameter) if ($repoSettingsVariableValue) { $repoSettingsVariableObject = $repoSettingsVariableValue | ConvertFrom-Json $settingsObjects += @($repoSettingsVariableObject) } + if ($project) { + # Read settings from repository settings file + $customTemplateProjectSettingsObject = GetSettingsObject -Path (Join-Path $baseFolder $CustomTemplateProjectSettingsFile) + $settingsObjects += @($customTemplateProjectSettingsObject) # Read settings from project settings file $projectFolder = Join-Path $baseFolder $project -Resolve $projectSettingsObject = GetSettingsObject -Path (Join-Path $projectFolder $ALGoSettingsFile) $settingsObjects += @($projectSettingsObject) } + if ($workflowName) { # Read settings from workflow settings file $workflowSettingsObject = GetSettingsObject -Path (Join-Path $gitHubFolder "$workflowName.settings.json") @@ -766,6 +783,7 @@ function ReadSettings { $settingsObjects += @($projectWorkflowSettingsObject, $userSettingsObject) } } + foreach($settingsJson in $settingsObjects) { if ($settingsJson) { MergeCustomObjectIntoOrderedDictionary -dst $settings -src $settingsJson diff --git a/Actions/AL-Go-TestRepoHelper.ps1 b/Actions/AL-Go-TestRepoHelper.ps1 index 7ca708e7f..f8187bd02 100644 --- a/Actions/AL-Go-TestRepoHelper.ps1 +++ b/Actions/AL-Go-TestRepoHelper.ps1 @@ -1,4 +1,6 @@ -function Test-Property { +. (Join-Path $PSScriptRoot "AL-Go-Helper.ps1") + +function Test-Property { Param( [HashTable] $json, [string] $settingsDescription, @@ -169,13 +171,16 @@ function TestALGoRepository { # Test .json files are formatted correctly # Get-ChildItem needs -force to include folders starting with . (e.x. .github / .AL-Go) on Linux Get-ChildItem -Path $baseFolder -Filter '*.json' -Recurse -Force | ForEach-Object { - if ($_.Directory.Name -eq '.AL-Go' -and $_.BaseName -eq 'settings') { + if ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($ALGoSettingsFile)) -and $_.Name -eq ([System.IO.Path]::GetFileName($ALGoSettingsFile))) { Test-JsonFile -jsonFile $_.FullName -baseFolder $baseFolder -type 'Project' } - elseif ($_.Directory.Name -eq '.github' -and $_.BaseName -like '*ettings') { - if ($_.BaseName -eq 'AL-Go-Settings') { + elseif ($_.Directory.Name -eq ([System.IO.Path]::GetDirectoryName($RepoSettingsFile)) -and $_.BaseName -like '*ettings') { + if ($_.Name -eq ([System.IO.Path]::GetFileName($RepoSettingsFile)) -or $_.Name -eq ([System.IO.Path]::GetFileName($CustomTemplateRepoSettingsFile))) { $type = 'Repo' } + elseif ($_.Name -eq ([System.IO.Path]::GetFileName($CustomTemplateProjectSettingsFile))) { + $type = 'Project' + } else { $type = 'Workflow' } diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 3b0937812..e11dc6b30 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -1,8 +1,8 @@ <# .SYNOPSIS Downloads a template repository and returns the path to the downloaded folder -.PARAMETER headers -The headers to use when calling the GitHub API +.PARAMETER token +The GitHub token / PAT to use for authentication (if the template repository is private/internal) .PARAMETER templateUrl The URL to the template repository .PARAMETER templateSha @@ -12,12 +12,31 @@ If true, the latest SHA of the template repository will be downloaded #> function DownloadTemplateRepository { Param( - [hashtable] $headers, + [string] $token, [string] $templateUrl, [ref] $templateSha, [bool] $downloadLatest ) + $templateRepositoryUrl = $templateUrl.Split('@')[0] + $templateRepository = $templateRepositoryUrl.Split('/')[-2..-1] -join '/' + + # Use Authenticated API request if possible to avoid the 60 API calls per hour limit + $headers = GetHeaders -token $env:GITHUB_TOKEN -repository $templateRepository + try { + $response = Invoke-WebRequest -Headers $headers -Method Head -Uri $templateRepositoryUrl + } + catch { + # Ignore error + $response = $null + } + if (-not $response -or $response.StatusCode -ne 200) { + # GITHUB_TOKEN doesn't have access to template repository, must be private/internal + # Get token with read permissions for the template repository + # NOTE that the GitHub app needs to be installed in the template repository for this to work + $headers = GetHeaders -token $token -repository $templateRepository + } + # Construct API URL $apiUrl = $templateUrl.Split('@')[0] -replace "^(https:\/\/github\.com\/)(.*)$", "$ENV:GITHUB_API_URL/repos/`$2" @@ -427,7 +446,7 @@ function IsDirectALGo { function GetSrcFolder { Param( - [hashtable] $repoSettings, + [string] $repoType, [string] $templateUrl, [string] $templateFolder, [string] $srcPath @@ -439,7 +458,7 @@ function GetSrcFolder { return '' } if (IsDirectALGo -templateUrl $templateUrl) { - switch ($repoSettings.type) { + switch ($repoType) { "PTE" { $typePath = "Per Tenant Extension" } @@ -486,48 +505,50 @@ function GetModifiedSettingsContent { else { # Change the $schema property to be the same as the source settings file (add it if it doesn't exist) $schemaKey = '$schema' - $schemaValue = $srcSettings."$schemaKey" + if ($srcSettings.PSObject.Properties.Name -eq $schemaKey) { + $schemaValue = $srcSettings."$schemaKey" - $dstSettings | Add-Member -MemberType NoteProperty -Name "$schemaKey" -Value $schemaValue -Force + $dstSettings | Add-Member -MemberType NoteProperty -Name "$schemaKey" -Value $schemaValue -Force - # Make sure the $schema property is the first property in the object - $dstSettings = $dstSettings | Select-Object @{ Name = '$schema'; Expression = { $_.'$schema' } }, * -ExcludeProperty '$schema' + # Make sure the $schema property is the first property in the object + $dstSettings = $dstSettings | Select-Object @{ Name = '$schema'; Expression = { $_.'$schema' } }, * -ExcludeProperty '$schema' + } } - return $dstSettings | ConvertTo-JsonLF } function UpdateSettingsFile { Param( [string] $settingsFile, - [hashtable] $updateSettings, - [hashtable] $additionalSettings = @{} + [hashtable] $updateSettings ) + $modified = $false # Update Repo Settings file with the template URL if (Test-Path $settingsFile) { $settings = Get-Content $settingsFile -Encoding UTF8 | ConvertFrom-Json } else { $settings = [PSCustomObject]@{} + $modified = $true } foreach($key in $updateSettings.Keys) { if ($settings.PSObject.Properties.Name -eq $key) { - $settings."$key" = $updateSettings."$key" + if ($settings."$key" -ne $updateSettings."$key") { + $settings."$key" = $updateSettings."$key" + $modified = $true + } } else { # Add the property if it doesn't exist $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $updateSettings."$key" + $modified = $true } } - # Grab settings from additionalSettings if they are not already in settings - foreach($key in $additionalSettings.Keys) { - if (!($settings.PSObject.Properties.Name -eq $key)) { - # Add the property if it doesn't exist - $settings | Add-Member -MemberType NoteProperty -Name "$key" -Value $additionalSettings."$key" - } + if ($modified) { + # Save the file with LF line endings and UTF8 encoding + $settings | Set-JsonContentLF -path $settingsFile } - # Save the file with LF line endings and UTF8 encoding - $settings | Set-JsonContentLF -path $settingsFile + return $modified } diff --git a/Actions/CheckForUpdates/CheckForUpdates.ps1 b/Actions/CheckForUpdates/CheckForUpdates.ps1 index 088cd9b62..01335d0ef 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.ps1 @@ -1,7 +1,7 @@ Param( [Parameter(HelpMessage = "The GitHub actor running the action", Mandatory = $false)] [string] $actor, - [Parameter(HelpMessage = "The GitHub token running the action", Mandatory = $false)] + [Parameter(HelpMessage = "Base64 encoded GhTokenWorkflow secret", Mandatory = $false)] [string] $token, [Parameter(HelpMessage = "URL of the template repository (default is the template repository used to create the repository)", Mandatory = $false)] [string] $templateUrl = "", @@ -45,21 +45,6 @@ if ($token) { $token = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($token)) } -# Use Authenticated API request if possible to avoid the 60 API calls per hour limit -$headers = GetHeaders -token $ENV:GITHUB_TOKEN -$templateRepositoryUrl = $templateUrl.Split('@')[0] -$response = Invoke-WebRequest -UseBasicParsing -Headers $headers -Method Head -Uri $templateRepositoryUrl -ErrorAction SilentlyContinue -if (-not $response -or $response.StatusCode -ne 200) { - # GITHUB_TOKEN doesn't have access to template repository, must be is private/internal - # Get token with read permissions for the template repository - # NOTE that the GitHub app needs to be installed in the template repository for this to work - $templateRepository = $templateRepositoryUrl.Split('/')[-2..-1] -join '/' - $templateReadToken = GetAccessToken -token $token -permissions @{"actions"="read";"contents"="read";"metadata"="read"} -repository $templateRepository - - # Use read token for authenticated API request - $headers = GetHeaders -token $templateReadToken -} - # CheckForUpdates will read all AL-Go System files from the Template repository and compare them to the ones in the current repository # CheckForUpdates will apply changes to the AL-Go System files based on AL-Go repo settings, such as "runs-on" etc. # if $update is set to Y, CheckForUpdates will also update the AL-Go System files in the current repository using a PR or a direct commit (if $directCommit is set to true) @@ -81,7 +66,8 @@ if ($repoSettings.templateUrl -ne $templateUrl -or $templateSha -eq '') { $downloadLatest = $true } -$templateFolder = DownloadTemplateRepository -headers $headers -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest +$originalTemplateFolder = $null +$templateFolder = DownloadTemplateRepository -token $token -templateUrl $templateUrl -templateSha ([ref]$templateSha) -downloadLatest $downloadLatest Write-Host "Template Folder: $templateFolder" $templateBranch = $templateUrl.Split('@')[1] @@ -90,11 +76,40 @@ $templateInfo = "$templateOwner/$($templateUrl.Split('/')[4])" $isDirectALGo = IsDirectALGo -templateUrl $templateUrl if (-not $isDirectALGo) { - $ALGoSettingsFile = Join-Path $templateFolder "*/$repoSettingsFile" - if (Test-Path -Path $ALGoSettingsFile -PathType Leaf) { - $templateRepoSettings = Get-Content $ALGoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse + $templateRepoSettingsFile = Join-Path $templateFolder "*/$RepoSettingsFile" + if (Test-Path -Path $templateRepoSettingsFile -PathType Leaf) { + $templateRepoSettings = Get-Content $templateRepoSettingsFile -Encoding UTF8 | ConvertFrom-Json | ConvertTo-HashTable -Recurse if ($templateRepoSettings.Keys -contains "templateUrl" -and $templateRepoSettings.templateUrl -ne $templateUrl) { - throw "The specified template repository is not a template repository, but instead another AL-Go repository. This is not supported." + # The template repository is a url to another AL-Go repository (a custom template repository) + Trace-Information -Message "Using custom AL-Go template repository" + + # TemplateUrl and TemplateSha from .github/AL-Go-Settings.json in the custom template repository points to the "original" template repository + # Copy files and folders from the custom template repository, but grab the unmodified file from the "original" template repository if it exists and apply customizations + # Copy .github/AL-Go-Settings.json to .github/templateRepoSettings.doNotEdit.json (will be read before .github/AL-Go-Settings.json in the final repo) + # Copy .AL-Go/settings.json to .github/templateProjectSettings.doNotEdit.json (will be read before .AL-Go/settings.json in the final repo) + + Write-Host "Custom AL-Go template repository detected, downloading the 'original' template repository" + $originalTemplateUrl = $templateRepoSettings.templateUrl + if ($templateRepoSettings.Keys -contains "templateSha") { + $originalTemplateSha = $templateRepoSettings.templateSha + } + else { + $originalTemplateSha = "" + } + + # Download the "original" template repository - use downloadLatest if no TemplateSha is specified in the custom template repository + $originalTemplateFolder = DownloadTemplateRepository -token $token -templateUrl $originalTemplateUrl -templateSha ([ref]$originalTemplateSha) -downloadLatest ($originalTemplateSha -eq '') + Write-Host "Original Template Folder: $originalTemplateFolder" + + # Set TemplateBranch and TemplateOwner + # Keep TemplateUrl and TemplateSha pointing to the custom template repository + $templateBranch = $originalTemplateUrl.Split('@')[1] + $templateOwner = $originalTemplateUrl.Split('/')[3] + + # If the custom template contains unusedALGoSystemFiles, we need to remove them from the current repository + if ($templateRepoSettings.ContainsKey('unusedALGoSystemFiles')) { + $unusedALGoSystemFiles += $templateRepoSettings.unusedALGoSystemFiles + } } } } @@ -109,12 +124,20 @@ if (-not $isDirectALGo) { # - All files in .github that ends with .copy.md # - All PowerShell scripts in .AL-Go folders (all projects) $checkfiles = @( - @{ 'dstPath' = Join-Path '.github' 'workflows'; 'srcPath' = Join-Path '.github' 'workflows'; 'pattern' = '*'; 'type' = 'workflow' }, - @{ 'dstPath' = '.github'; 'srcPath' = '.github'; 'pattern' = '*.copy.md'; 'type' = 'releasenotes' }, - @{ 'dstPath' = '.github'; 'srcPath' = '.github'; 'pattern' = 'AL-Go-Settings.json'; 'type' = 'settings' }, - @{ 'dstPath' = '.github'; 'srcPath' = '.github'; 'pattern' = '*.settings.json'; 'type' = 'settings' } + @{ 'dstPath' = (Join-Path '.github' 'workflows'); 'dstName' = ''; 'srcPath' = (Join-Path '.github' 'workflows'); 'pattern' = '*'; 'type' = 'workflow' }, + @{ 'dstPath' = '.github'; 'dstName' = ''; 'srcPath' = '.github'; 'pattern' = '*.copy.md'; 'type' = 'releasenotes' } + @{ 'dstPath' = '.github'; 'dstName' = ''; 'srcPath' = '.github'; 'pattern' = '*.ps1'; 'type' = 'script' } + @{ 'dstPath' = '.github'; 'dstName' = ''; 'srcPath' = '.github'; 'pattern' = 'AL-Go-Settings.json'; 'type' = 'settings' }, + @{ 'dstPath' = '.github'; 'dstName' = ''; 'srcPath' = '.github'; 'pattern' = '*.settings.json'; 'type' = 'settings' } ) +if ($originalTemplateFolder) { + $checkfiles += @( + @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateRepoSettingsFile)); 'dstName' = ([system.IO.Path]::GetFileName($CustomTemplateRepoSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($RepoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($RepoSettingsFile)); 'type' = 'template repo settings' } + @{ 'dstPath' = ([system.IO.Path]::GetDirectoryName($CustomTemplateProjectSettingsFile)); 'dstName' = ([system.IO.Path]::GetFileName($CustomTemplateProjectSettingsFile)); 'SrcPath' = ([system.IO.Path]::GetDirectoryName($ALGoSettingsFile)); 'pattern' = ([system.IO.Path]::GetFileName($ALGoSettingsFile)); ; 'type' = 'template project settings' } + ) +} + # Get the list of projects in the current repository $baseFolder = $ENV:GITHUB_WORKSPACE $projects = @(GetProjectsFromRepository -baseFolder $baseFolder -projectsFromSettings $repoSettings.projects) @@ -122,8 +145,8 @@ Write-Host "Projects found: $($projects.Count)" foreach($project in $projects) { Write-Host "- $project" $checkfiles += @( - @{ 'dstPath' = Join-Path $project '.AL-Go'; 'srcPath' = '.AL-Go'; 'pattern' = '*.ps1'; 'type' = 'script' }, - @{ 'dstPath' = Join-Path $project '.AL-Go'; 'srcPath' = '.AL-Go'; 'pattern' = 'settings.json'; 'type' = 'settings' } + @{ 'dstPath' = Join-Path $project '.AL-Go'; 'dstName' = ''; 'srcPath' = '.AL-Go'; 'pattern' = '*.ps1'; 'type' = 'script' }, + @{ 'dstPath' = Join-Path $project '.AL-Go'; 'dstName' = ''; 'srcPath' = '.AL-Go'; 'pattern' = 'settings.json'; 'type' = 'settings' } ) } @@ -149,20 +172,48 @@ foreach($checkfile in $checkfiles) { $srcPath = $checkfile.srcPath $dstPath = $checkfile.dstPath $dstFolder = Join-Path $baseFolder $dstPath - $srcFolder = GetSrcFolder -repoSettings $repoSettings -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath + $srcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $templateUrl -templateFolder $templateFolder -srcPath $srcPath + $originalSrcFolder = $null + if ($originalTemplateFolder -and $type -notlike 'template*settings') { + # Get Original source folder except for template settings - these are applied from the custom template´repository + $originalSrcFolder = GetSrcFolder -repoType $repoSettings.type -templateUrl $originalTemplateUrl -templateFolder $originalTemplateFolder -srcPath $srcPath + } if ($srcFolder) { Push-Location -Path $srcFolder try { + # Remove unused AL-Go system files + $unusedALGoSystemFiles | ForEach-Object { + if (Test-Path -Path (Join-Path $dstFolder $_) -PathType Leaf) { + Write-Host "Remove unused AL-Go system file: $_" + $removeFiles += @(Join-Path $dstPath $_) + } + } + # Loop through all files in the template repository matching the pattern Get-ChildItem -Path $srcFolder -Filter $checkfile.pattern | ForEach-Object { # Read the template file and modify it based on the settings # Compare the modified file with the file in the current repository - $fileName = $_.Name + if ($checkfile.dstName) { + $filename = $checkfile.dstName + } + else { + $filename = $_.Name + } Write-Host "- $filename" - $dstFile = Join-Path $dstFolder $fileName + $dstFile = Join-Path $dstFolder $filename $srcFile = $_.FullName + $originalSrcFile = $srcFile + $isFileDirectALGo = $isDirectALGo Write-Host "SrcFolder: $srcFolder" - + if ($originalSrcFolder) { + # if SrcFile is a custom template repository, we need to find the file in the "original" template repository + $fname = Join-Path $originalSrcFolder (Resolve-Path $srcFile -Relative) + if (Test-Path -Path $fname -PathType Leaf) { + Write-Host "File is available in the 'original' template repository" + $originalSrcFile = $fname + $isFileDirectALGo = IsDirectALGo -templateUrl $originalTemplateUrl + } + } $dstFileExists = Test-Path -Path $dstFile -PathType Leaf if ($unusedALGoSystemFiles -contains $fileName) { # File is not used by AL-Go, remove it if it exists @@ -173,34 +224,44 @@ foreach($checkfile in $checkfiles) { } return } - switch ($type) { "workflow" { # For workflow files, we might need to modify the file based on the settings - $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $srcFile -repoSettings $repoSettings -depth $depth -includeBuildPP $includeBuildPP + $srcContent = GetWorkflowContentWithChangesFromSettings -srcFile $originalsrcFile -repoSettings $repoSettings -depth $depth -includeBuildPP $includeBuildPP } "settings" { # For settings files, we need to modify the file based on the settings - $srcContent = GetModifiedSettingsContent -srcSettingsFile $srcFile -dstSettingsFile $dstFile + $srcContent = GetModifiedSettingsContent -srcSettingsFile $originalSrcFile -dstSettingsFile $dstFile } Default { # For non-workflow files, just read the file content - $srcContent = Get-ContentLF -Path $srcFile + $srcContent = Get-ContentLF -Path $originalSrcFile } } # Replace static placeholders $srcContent = $srcContent.Replace('{TEMPLATEURL}', $templateUrl) - if ($isDirectALGo) { + if ($isFileDirectALGo) { # If we are using direct AL-Go repo, we need to change the owner to the templateOwner, the repo names to AL-Go and AL-Go/Actions and the branch to templateBranch ReplaceOwnerRepoAndBranch -srcContent ([ref]$srcContent) -templateOwner $templateOwner -templateBranch $templateBranch } + if ($type -eq 'workflow' -and $originalSrcFile -ne $srcFile) { + # Apply customizations from custom template repository + Write-Host "Apply customizations from custom template repository: $srcFile" + [Yaml]::ApplyCustomizations([ref] $srcContent, $srcFile) + } + if ($dstFileExists) { + if ($type -eq 'workflow') { + Write-Host "Apply customizations from current repository: $dstFile" + [Yaml]::ApplyCustomizations([ref] $srcContent, $dstFile) + } + # file exists, compare and add to $updateFiles if different $dstContent = Get-ContentLF -Path $dstFile if ($dstContent -cne $srcContent) { - Write-Host "Updates in $type ($(Join-Path $dstPath $filename)) available" + Write-Host "Updated $type ($(Join-Path $dstPath $filename)) available" $updateFiles += @{ "DstFile" = Join-Path $dstPath $filename; "content" = $srcContent } } else { @@ -219,10 +280,20 @@ foreach($checkfile in $checkfiles) { } } } +$removeFiles = @($removeFiles | Select-Object -Unique) if ($update -ne 'Y') { # $update not set, just issue a warning in the CI/CD workflow that updates are available if (($updateFiles) -or ($removeFiles)) { + if ($updateFiles) { + Write-Host "Updated files:" + $updateFiles | ForEach-Object { Write-Host "- $($_.DstFile)" } + + } + if ($removeFiles) { + Write-Host "Removed files:" + $removeFiles | ForEach-Object { Write-Host "- $_" } + } OutputWarning -message "There are updates for your AL-Go system, run 'Update AL-Go System Files' workflow to download the latest version of AL-Go." } else { diff --git a/Actions/CheckForUpdates/README.md b/Actions/CheckForUpdates/README.md index 554ca9db5..c7208f425 100644 --- a/Actions/CheckForUpdates/README.md +++ b/Actions/CheckForUpdates/README.md @@ -14,7 +14,7 @@ none | :-- | :-: | :-- | :-- | | shell | | The shell (powershell or pwsh) in which the PowerShell script in this action should run | powershell | | actor | | The GitHub actor running the action | github.actor | -| token | | The GitHub token running the action | github.token | +| token | | Base64 encoded GhTokenWorkflow secret | | | templateUrl | | URL of the template repository (default is the template repository used to create the repository) | default | | downloadLatest | Yes | Set this input to true in order to download latest version of the template repository (else it will reuse the SHA from last update) | | | update | | Set this input to Y in order to update AL-Go System Files if needed | N | diff --git a/Actions/CheckForUpdates/action.yaml b/Actions/CheckForUpdates/action.yaml index 96d94d1bd..38c936c9b 100644 --- a/Actions/CheckForUpdates/action.yaml +++ b/Actions/CheckForUpdates/action.yaml @@ -10,9 +10,9 @@ inputs: required: false default: ${{ github.actor }} token: - description: The GitHub token running the action + description: Base64 encoded GhTokenWorkflow secret required: false - default: ${{ github.token }} + default: '' templateUrl: description: URL of the template repository (default is the template repository used to create the repository) required: false diff --git a/Actions/CheckForUpdates/yamlclass.ps1 b/Actions/CheckForUpdates/yamlclass.ps1 index 7b0eb850c..49087e379 100644 --- a/Actions/CheckForUpdates/yamlclass.ps1 +++ b/Actions/CheckForUpdates/yamlclass.ps1 @@ -20,7 +20,7 @@ class Yaml { # Save the Yaml file with LF line endings using UTF8 encoding Save([string] $filename) { - $this.content | Set-ContentLF -Path $filename + $this.content -join "`n" | Set-ContentLF -Path $filename } # Find the lines for the specified Yaml path, given by $line @@ -126,6 +126,45 @@ class Yaml { return $this.Get($line, [ref] $start, [ref] $count) } + # Locate all lines in the next level of a yaml path + # if $line is empty, you get all first level lines + # Example: + # GetNextLevel("jobs:/") returns @("Initialization:","CheckForUpdates:","Build:","Deploy:",...) + [string[]] GetNextLevel([string] $line) { + [int]$start = 0 + [int]$count = 0 + [Yaml] $yaml = $this + if ($line) { + $yaml = $this.Get($line, [ref] $start, [ref] $count) + } + return $yaml.content | Where-Object { $_ -and -not $_.StartsWith(' ') } + } + + # Get the value of a property as a string + # Example: + # GetProperty("jobs:/Build:/needs:") returns "[ Initialization, Build1 ]" + [string] GetProperty([string] $line) { + [int]$start = 0 + [int]$count = 0 + [Yaml] $yaml = $this.Get($line, [ref] $start, [ref] $count) + if ($yaml -and $yaml.content.Count -eq 1) { + return $yaml.content[0].SubString($yaml.content[0].IndexOf(':')+1).Trim() + } + return $null + } + + # Get the value of a property as a string array + # Example: + # GetPropertyArray("jobs:/Build:/needs:") returns @("Initialization", "Build") + [string[]] GetPropertyArray([string] $line) { + $prop = $this.GetProperty($line) + if ($prop) { + # "needs: [ Initialization, Build ]" becomes @("Initialization", "Build") + return $prop.TrimStart('[').TrimEnd(']').Split(',').Trim() + } + return $null + } + # Replace the lines for the specified Yaml path, given by $line with the lines in $content # If $line ends with '/', then the lines for the section are replaced only # If $line doesn't end with '/', then the line + the lines for the section are replaced @@ -194,4 +233,99 @@ class Yaml { $this.content = $this.content[0..($index-1)] + $yamlContent + $this.content[$index..($this.content.Count-1)] } } + + # Add lines to Yaml content + [void] Add([string[]] $yamlContent) { + if (!$yamlContent) { + return + } + $this.Insert($this.content.Count, $yamlContent) + } + + # Locate jobs in YAML based on a name pattern + # Example: + # GetCustomJobsFromYaml() returns @("CustomJob1", "CustomJob2") + # GetCustomJobsFromYaml("Build*") returns @("Build1","Build2","Build") + [hashtable[]] GetCustomJobsFromYaml([string] $name) { + $result = @() + $allJobs = $this.GetNextLevel('jobs:/').Trim(':') + $customJobs = @($allJobs | Where-Object { $_ -like $name }) + if ($customJobs) { + $nativeJobs = @($allJobs | Where-Object { $customJobs -notcontains $_ }) + Write-Host "Native Jobs:" + foreach($nativeJob in $nativeJobs) { + Write-Host "- $nativeJob" + } + Write-Host "Custom Jobs:" + foreach($customJob in $customJobs) { + Write-Host "- $customJob" + $jobsWithDependency = @($nativeJobs | Where-Object { $this.GetPropertyArray("jobs:/$($_):/needs:") | Where-Object { $_ -eq $customJob } }) + # If any Build Job has a dependency on this CustomJob, add will be added to all build jobs later + if ($jobsWithDependency | Where-Object { $_ -like 'Build*' }) { + $jobsWithDependency = @($jobsWithDependency | Where-Object { $_ -notlike 'Build*' }) + @('Build') + } + if ($jobsWithDependency) { + Write-Host " - Jobs with dependency: $($jobsWithDependency -join ', ')" + } + else { + Write-Host " - No jobs with dependency on this" + } + $result += @(@{ "Name" = $customJob; "Content" = @($this.Get("jobs:/$($customJob):").content); "NeedsThis" = $jobsWithDependency }) + } + } + return $result + } + + # Add jobs to Yaml and update Needs section from native jobs which needs this custom Job + # $customJobs is an array of hashtables with Name, Content and NeedsThis + # Example: + # $customJobs = @(@{ "Name" = "CustomJob1"; "Content" = @(" - pwsh"," -File Build1"); "NeedsThis" = @("Initialization", "Build") }) + # AddCustomJobsToYaml($customJobs) + # The function will add the job CustomJob1 to the Yaml file and update the Needs section of Initialization and Build + # The function will not add the job CustomJob1 if it already exists + [void] AddCustomJobsToYaml([hashtable[]] $customJobs) { + $existingJobs = $this.GetNextLevel('jobs:/').Trim(':') + Write-Host "Adding New Jobs" + foreach($customJob in $customJobs) { + Write-Host "$($customJob.Name) has dependencies from $($customJob.NeedsThis -join ',')" + foreach($needsthis in $customJob.NeedsThis) { + if ($needsthis -eq 'Build') { + $existingJobs | Where-Object { $_ -like 'Build*'} | ForEach-Object { + # Add dependency to all build jobs + $this.Replace("jobs:/$($_):/needs:","needs: [ $(@($this.GetPropertyArray("jobs:/$($_):/needs:"))+@($customJob.Name) -join ', ') ]") + } + } + elseif ($existingJobs -contains $needsthis) { + # Add dependency to job + $needs = @(@($this.GetPropertyArray("jobs:/$($needsthis):/needs:"))+@($customJob.Name) | Where-Object { $_ } | Select-Object -Unique) -join ', ' + $this.Replace("jobs:/$($needsthis):/needs:","needs: [ $needs ]") + } + } + if ($existingJobs -contains $customJob.Name) { + Write-Host "Job $($customJob.Name) already exists" + continue + } + $this.content += @('') + @($customJob.content | ForEach-Object { " $_" }) + } + } + + static [void] ApplyCustomizations([ref] $srcContent, [string] $yamlFile) { + $srcYaml = [Yaml]::new($srcContent.Value.Split("`n")) + try { + $yaml = [Yaml]::Load($yamlFile) + } + catch { + Write-Host "Unable to read YAML file $yamlFile. Skipping custom jobs." + return + } + + # Locate custom jobs in destination YAML + Write-Host "Apply custom jobs" + $customJobs = @($yaml.GetCustomJobsFromYaml('CustomJob*')) + if ($customJobs) { + # Add custom jobs to template YAML + $srcYaml.AddCustomJobsToYaml($customJobs) + } + $srcContent.Value = $srcYaml.content -join "`n" + } } diff --git a/Actions/Github-Helper.psm1 b/Actions/Github-Helper.psm1 index bbaea7853..96653675b 100644 --- a/Actions/Github-Helper.psm1 +++ b/Actions/Github-Helper.psm1 @@ -656,14 +656,15 @@ function GetHeaders { [string] $accept = "application/vnd.github+json", [string] $apiVersion = "2022-11-28", [string] $api_url = $ENV:GITHUB_API_URL, - [string] $repository = $ENV:GITHUB_REPOSITORY + [string] $repository = $ENV:GITHUB_REPOSITORY, + [hashtable] $permissions = @{"contents"="read";"metadata"="read";"actions"="read"} ) $headers = @{ "Accept" = $accept "X-GitHub-Api-Version" = $apiVersion } if (![string]::IsNullOrEmpty($token)) { - $accessToken = GetAccessToken -token $token -api_url $api_url -repository $repository -permissions @{"contents"="read";"metadata"="read";"actions"="read"} + $accessToken = GetAccessToken -token $token -api_url $api_url -repository $repository -permissions $permissions $headers["Authorization"] = "token $accessToken" } return $headers @@ -788,17 +789,12 @@ function Set-ContentLF { [parameter(Mandatory = $true, ValueFromPipeline = $false)] [string] $path, [parameter(Mandatory = $true, ValueFromPipeline = $true)] - $content + [string] $content ) Process { $path = $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($path) - if ($content -is [array]) { - $content = $content -join "`n" - } - else { - $content = "$content".Replace("`r", "") - } + $content = "$content".Replace("`r", "").TrimEnd("`n") [System.IO.File]::WriteAllText($path, "$content`n") } } diff --git a/Actions/ReadSecrets/ReadSecrets.ps1 b/Actions/ReadSecrets/ReadSecrets.ps1 index 818357669..76be8b09b 100644 --- a/Actions/ReadSecrets/ReadSecrets.ps1 +++ b/Actions/ReadSecrets/ReadSecrets.ps1 @@ -77,6 +77,7 @@ try { } } } + # Look through installApps and installTestApps for secrets and add them to the collection of secrets to get foreach($installSettingsKey in @('installApps','installTestApps')) { if ($settings.Keys -contains $installSettingsKey) { diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 41f3b1713..84ca0b8f8 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -105,7 +105,7 @@ try { "appSourceAppRepo" = "$($config.githubOwner)/$($config.appSourceAppRepo)" } - if ($config.branch -eq 'preview') { + if ($config.branch -eq 'preview' -or $config.githubOwner -ne 'microsoft') { # When deploying to preview, we are NOT going to deploy to a branch in the AL-Go-Actions repository # Instead, we are going to have AL-Go-PTE and AL-Go-AppSource point directly to the SHA in AL-Go $dstOwnerAndRepo += @{ diff --git a/README.md b/README.md index 5d6b5e982..c350fa568 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ Try out the [AL-Go workshop](https://aka.ms/algoworkshop) for an in-depth worksh 1. [Connect your GitHub repository to Power Platform](Scenarios/SetupPowerPlatform.md) 1. [How to set up Service Principal for Power Platform](Scenarios/SetupServicePrincipalForPowerPlatform.md) 1. [Try one of the Business Central and Power Platform samples](Scenarios/TryPowerPlatformSamples.md) +1. [Customizing AL-Go for GitHub](Scenarios/CustomizingALGoForGitHub.md) ## Migration scenarios diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 1c9a091cc..0daebe627 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -4,9 +4,22 @@ AL-Go now offers a dataexplorer dashboard to get started with AL-Go telemetry. A ### Issues -- Issue 1770: Wrong type of _projects_ setting in settings schema +- Issue 1770 Wrong type of _projects_ setting in settings schema - Issue 1787: Publish to Environment from PR fails in private repos +### Add custom jobs to AL-Go workflows + +It is now possible to add custom jobs to AL-Go workflows. The Custom Job needs to be named `CustomJob` and should be placed after all other jobs in the .yaml file. The order of which jobs are executed is determined by the Needs statements. Your custom job will be executed after all jobs specified in the Needs clause in your job and if you need the job to be executed before other jobs, you should add the job name in the Needs clause of that job. See [https://aka.ms/algosettings#customjobs](https://aka.ms/algosettings#customjobs) for details. + +Note that custom jobs might break by future changes to AL-Go for GitHub workflows. If you have customizations to AL-Go for GitHub workflows, you should always doublecheck the pull request generated by Update AL-Go System Files. + +### Support for Custom AL-Go template repositories + +Create an AL-Go for GitHub repository based on [https://aka.ms/algopte](https://aka.ms/algopte) or [https://aka.ms/algoappsource](https://aka.ms/algoappsource), add custom workflows, custom jobs and/or settings to this repository and then use that repository as the template repository for other repositories. Using custom template repositories allows you to create and use highly customized template repositories and control the uptake of this in all repositories. See [https://aka.ms/algosettings#customtemplate](https://aka.ms/algosettings#customtemplate) for details. + +> [!NOTE] +> Customized repositories might break by future changes to AL-Go for GitHub. If you are customizing AL-Go for GitHub, you should always double-check the pull request when updating AL-Go system files in your custom template repositories. + ## v7.2 ### Removed functionality diff --git a/Scenarios/Contribute.md b/Scenarios/Contribute.md index 6dd1d4f60..522b88034 100644 --- a/Scenarios/Contribute.md +++ b/Scenarios/Contribute.md @@ -5,16 +5,16 @@ This section describes how to contribute to AL-Go. How to set up your own enviro You can do this in two ways: - Use a fork of AL-Go for GitHub in your own **personal GitHub account** in development mode -- Use 3 public repositories in your own **personal GitHub account** (AL-Go-PTE, AL-Go-AppSource and AL-Go-Actions, much like in production) +- Use 2 public repositories in your own **personal GitHub account** (AL-Go-PTE and AL-Go-AppSource, much like in production) ## Use a fork of AL-Go for GitHub in "development mode" 1. Fork the [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go) repository to your **personal GitHub account**. 1. You can optionally also create a branch in the AL-Go fork for the feature you are working on. -**https://github.com//AL-Go@** can now be used as a template in your AL-Go project when running _Update AL-Go System Files_ to use the actions/workflows from this fork. +**/AL-Go@** can now be used as a template in your AL-Go project when running _Update AL-Go System Files_ to use the actions/workflows from this fork. -## Use 3 public repositories in "production mode" +## Use 2 public repositories in "production mode" 1. Fork the [https://github.com/microsoft/AL-Go](https://github.com/microsoft/AL-Go) repository to your **personal GitHub account**. 1. Navigate to [https://github.com/settings/tokens/new](https://github.com/settings/tokens/new) and create a new personal access token with **Full control of private repositories** and **workflow** permissions. @@ -22,9 +22,8 @@ You can do this in two ways: 1. In your personal fork of AL-Go, navigate to **Actions**, select the **Deploy** workflow and choose **Run Workflow**. 1. Using the default settings press **Run workflow**. Select the AL-Go branch to run from and the branch to deploy to. -Now you should have 3 new public repositories: +Now you should have 2 new public repositories: -- [https://github.com/yourGitHubUserName/AL-Go-Actions](https://github.com/yourGitHubUserName/AL-Go-Actions) - [https://github.com/yourGitHubUserName/AL-Go-AppSource](https://github.com/yourGitHubUserName/AL-Go-AppSource) - [https://github.com/yourGitHubUserName/AL-Go-PTE](https://github.com/yourGitHubUserName/AL-Go-PTE) @@ -57,9 +56,8 @@ In the e2eTests folder, in the AL-Go repository, there are 3 types of end to end - The scenarios folder contains a set of AL-Go scenarios, which tests specific functionality end to end. Every folder under e2eTests/scenarios, which contains a runtests.ps1 will be run as a scenario test, like: - UseProjectDependencies - create a repo with multiple projects and set **UseProjectDependencies** to modify CI/CD and other build workflows to build projects in the right order - GitHubPackages - create 3 repositories using GitHub Packages as dependency resolver and check that artifacts are built properly - - BuildModes - create a repository, set buildModes and test that generated artifacts are as expected. - - ReleaseBranches - testing that create release works, release branches are create and subsequently found correctly as previous build - SpecialCharacters - testing that various settings (+ publisher name and app name) can contain special national characters + - and more... In your personal fork, you can now run the end to end tests, if the following pre-requisites are available: diff --git a/Scenarios/CustomizingALGoForGitHub.md b/Scenarios/CustomizingALGoForGitHub.md new file mode 100644 index 000000000..beb2029ed --- /dev/null +++ b/Scenarios/CustomizingALGoForGitHub.md @@ -0,0 +1,225 @@ +# Customizing AL-Go for GitHub + +AL-Go for GitHub is a plug-and-play DevOps solution, intended to support 100% of the functionality needed by 90% of the people developing applications for Microsoft Dynamics 365 Business Central out-of-the-box. + +If AL-Go functionality out-of-the-box doesn't match your needs, you should always start by creating a feature suggestion [here](https://github.com/microsoft/AL-Go/discussions) and see whether your needs are met by other mechanisms or required by other partners and should be part of AL-Go for GitHub. + +If your feature should be part of AL-Go for GitHub, you can select to [contribute](Contribute.md) to AL-Go for GitHub yourself or wait for Microsoft or other partners to pickup your feature suggestion. + +If your feature suggestion isn't accepted, you really have three options: + +1. Customize AL-Go for GitHub to fit your needs +1. Select another managed DevOps solution +1. Create your own DevOps solution from scratch (not recommended) + +Creating your own DevOps solution from scratch requires dedicated resources to develop and maintain workflows, processes etc. **This is not a small task**. There are many moving parts in a DevOps solution, which might require you to make changes to workflows and scripts over time and stay secure and having to maintain many repositories is tedious and time consuming, even when using templates and other advanced features. + +Microsoft will continuously develop and maintain AL-Go for GitHub and ensure that we always use the latest versions of GitHub actions, which are under our control. Microsoft will never add dependencies to any third party GitHub action, which are not under our control. + +Keeping your repositories up-to-date can be done manually or on a schedule (like Windows update really). You will be notified when an update is available and we recommend that you keep your repositories up-to-date at all time. If you make modifications to the AL-Go System Files (scripts and workflows) in your repository, in other ways than described in this document, these changes will be removed with the next AL-Go update. + +> [!TIP] +> If for some reason the updated version of AL-Go for GitHub doesn't work for you, we recommend that you file an issue [here](https://github.com/microsoft/AL-Go/issues) with a detailed description of the problem and full logs of the failing workflows. You can then revert back to the prior version of AL-Go for GitHub until the issue is resolved. +> +> It is important to get back to the mainstream version of AL-Go for GitHub as soon as the issue is resolved. + +There are three ways you can customize AL-Go for GitHub to fit your needs. You can + +1. customize the repository with custom scripts, workflows or jobs following the guidelines below +1. create a customized repository and use this as your custom template repository +1. fork the AL-Go for GitHub and create your "own" version (not recommended) + +> [!CAUTION] +> The more you customize AL-Go for GitHub, the more likely you are to be broken by future updates to AL-Go for GitHub, meaning that you will have to update your customizations to match the changes in AL-Go for GitHub. + +## Customizing your repository + +There are several ways you can customize your AL-Go repository and ensure that the changes you make, will survive an update of AL-Go for GitHub. + +### Hide/Remove unused workflows + +By adding a setting called [`unusedALGoSystemFiles`](https://aka.ms/algosettings#unusedalgosystemfiles) in your [repo settings](https://aka.ms/algosettings#settings), you can tell AL-Go for GitHub that these system files are not used. Example: + +``` + "unusedALGoSystemFiles": [ + "AddExistingAppOrTestApp.yaml", + "CreateApp.yaml", + "CreatePerformanceTestApp.yaml", + "CreateTestApp.yaml", + "cloudDevEnv.ps1" + ] +``` + +This setting will cause AL-Go for GitHub to remove these files during the next update. Note that if you remove files like `_BuildALGoProject.yaml`, AL-Go will obviously stop working as intended - so please use with care. + +### Custom delivery + +You can setup [custom delivery](https://aka.ms/algosettings#customdelivery) in order to deliver your apps to locations not supported by AL-Go for GitHub out-of-the-box, by adding a custom delivery powershell script (named `.github/DeliverTo.ps1`) and a context secret (called `Context`) formatted as compressed json, you can define the delivery functionality as you like. Example: + +```powershell +Param([Hashtable] $parameters) + +Get-ChildItem -Path $parameters.appsFolder | Out-Host +$context = $parameters.context | ConvertFrom-Json +Write-Host "Token Length: $($context.Token.Length)" +``` + +In this example the context secret is assumed to contain a Token property. Read [this](https://aka.ms/algosettings#customdelivery) for more information. + +### Custom deployment + +You can setup [custom deployment](https://aka.ms/algosettings#customdeployment) to environment types not supported by AL-Go for GitHub out-of-the-box. You can also override deployment functionality to environment Type `SaaS` if you like. You can add an environment called `` and a `DeployTo` setting, defining which environment Type should be used. Example: + +```json + "Environments": [ + "" + ], + "DeployTo": { + "EnvironmentType": "" + } +``` + +You also need to create an AuthContext secret (called `_AuthContext`) and a powershell script (named `.github/DeployTo.ps1`), which defines the deployment functionality. Example: + +```powershell +Param([Hashtable] $parameters) + +$parameters | ConvertTo-Json -Depth 99 | Out-Host +$tempPath = Join-Path ([System.IO.Path]::GetTempPath()) ([GUID]::NewGuid().ToString()) +New-Item -ItemType Directory -Path $tempPath | Out-Null +Copy-AppFilesToFolder -appFiles $parameters.apps -folder $tempPath | Out-Null +Get-ChildItem -Path $tempPath -Filter *.app | Out-Host +$authContext = $parameters.authContext | ConvertFrom-Json +Write-Host "Token Length: $($authContext.Token.Length)" +``` + +In this example the AuthContext secret is assumed to contain a Token property. Read [this](https://aka.ms/algosettings#customdeployment) for more information. + +### Adding custom workflows + +If you add new workflows to the `.github/workflows` folder, which is unknown to AL-Go for GitHub, AL-Go will leave them untouched. These workflows need to follow standard GitHub Actions schema (yaml) and can be triggered as any other workflows. Example: + +```yaml +name: 'Create Build Tag' + +on: + workflow_run: + workflows: [' CI/CD','CI/CD'] + types: [completed] + branches: [ 'main' ] + +run-name: "[${{ github.ref_name }}] Create build tag" + +permissions: read-all + +jobs: + CreateTag: + if: github.event.workflow_run.conclusion == 'success' + runs-on: windows-latest + steps: + - name: mystep + run: | + Write-Host "Create tag" +``` + +It is recommended to prefix your workflows with `my`, `our`, your name or your organization name in order to avoid that the workflow suddenly gets overridden by a new workflow in AL-Go for GitHub. The above workflow is a real example from [here](https://github.com/microsoft/BCApps/blob/main/.github/workflows/CreateBuildTag.yaml). + +> [!CAUTION] +> This workflow gets triggered when the CI/CD workflow has completed. Note that the name of the CI/CD workflow currently is prefixed with a space, this space will very likely be removed in the future, which is why we specify both names in this example. Obviously this workflow would break if we decide to rename the CI/CD workflow to something different. + +### Adding custom scripts + +You can add custom powershell scripts under the .github folder for repository scoped scripts or in the .AL-Go folder for project scoped scripts. Specially named scripts in the .AL-Go folder can override standard functionality in AL-Go for GitHub workflows. A list of these script overrides can be found [here](https://aka.ms/algosettings#scriptoverrides). Scripts under the .github folder can be used in custom workflows instead of using inline scripts inside the workflow. + +One example of a script override is the NewBcContainer override used in the System Application project in BCApps (can be found [here](https://github.com/microsoft/BCApps/blob/647efdacac0c0d13d726e14c89180a32cbb55cf2/build/projects/System%20Application/.AL-Go/NewBcContainer.ps1)). This override looks like: + +```powershell +Param([Hashtable] $parameters) + +$script = Join-Path $PSScriptRoot "../../../scripts/NewBcContainer.ps1" -Resolve +. $script -parameters $parameters +``` + +Which basically launches a script located in the script folder in the repository for creating the build container needed for building and testing the System Application. That script can be found [here](https://github.com/microsoft/BCApps/blob/647efdacac0c0d13d726e14c89180a32cbb55cf2/build/scripts/NewBcContainer.ps1). + +> [!CAUTION] +> Script overrides will almost certainly be broken in the future. The current script overrides is very much tied to the current implementation of the `Run-AlPipeline` function in BcContainerHelper. In the future, we will move this functionality to GitHub actions and no longer depend on BcContainerHelper and Run-AlPipeline. At that time, these script overrides will have to be changed to follow the new implementation. + +### Adding custom jobs + +You can also add custom jobs to any of the existing AL-Go for GitHub workflows. Custom jobs can depend on other jobs and other jobs can be made to depend on custom jobs. Custom jobs need to be named `CustomJob`, but can specify another name to be shown in the UI. Example: + +```yaml + CustomJob-CreateBuildTag: + name: Create Build Tag + needs: [ Initialization, Build ] + if: (!cancelled()) && (needs.Build.result == 'success') + runs-on: [ ubuntu-latest ] + steps: + - name: Create Tag + run: | + Write-Host "Create Tag" + + PostProcess: + needs: [ Initialization, Build2, Build1, Build, Deploy, Deliver, DeployALDoc, CustomJob-CreateBuildTag ] + if: (!cancelled()) + runs-on: [ windows-latest ] + steps: + ... +``` + +Adding a custom job like this, will cause this job to run simultaneously with the deploy and the deliver jobs. + +> [!NOTE] +> All custom jobs will be moved to the tail of the yaml file when running Update AL-Go System Files, but dependencies to/from the custom jobs will be maintained. + +> [!CAUTION] +> Custom jobs might be broken if the customized AL-Go for GitHub workflow has been refactored and the referenced jobs have been renamed. Therefore, please make sure to review the changes in AL-Go workflows when running Update AL-Go System Files. + +### Custom job permissions + +If any of your custom jobs require permissions, which exceeds the permissions already assigned in the workflow, then these permissions can be specified directly on the custom job. + +## Using custom template repositories + +If you have have customizations you want to apply to multiple repositories, you might want to consider using a custom template. A custom template is really just an AL-Go repository (which can be customized), which you use as a template repository for your repositories. This way, you can control your scripts, jobs or steps in a central location, potentially for specific purposes. + +> [!NOTE] +> Custom templates can be public or private. If you are using a private custom template repository, AL-Go for GitHub will use the GhTokenWorkflow secret for downloading the template during Update AL-Go System Files and check for updates. + +> [!TIP] +> The recommended way to create a new repository based on your custom AL-Go template is to create a new repository based on [AL-Go-PTE](https://github.com/microsoft/AL-Go-PTE) or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource), create a **GhTokenWorkflow** secret and then run the `Update AL-Go System Files` workflow with your custom template specified. + +> [!NOTE] +> If you use the custom template as a GitHub template for creating the repository, by clicking use this template in your custom template - then you need to re-specify the custom Template the first time you run Update `AL-Go System Files` as the repository will be a copy of the template repository and by default point to the template repository of the custom template as it's template repository. + +Repositories based on your custom template will notify you that changes are available for your AL-Go System Files when you update the custom template only. You will not be notified when new versions of AL-Go for GitHub is released in every repository - only in the custom template repository. + +> [!WARNING] +> You should ensure that your custom template repository is kept up-to-date with the latest changes in AL-Go for GitHub. + +> [!TIP] +> You can setup the Update AL-Go System Files workflow to run on a schedule to uptake new releases of AL-Go for GitHub regularly. + +## Forking AL-Go for GitHub and making your "own" **public** version + +Using a fork of AL-Go for GitHub to have your "own" public version of AL-Go for GitHub gives you the maximum customization capabilities. It does however also come with the most work. + +> [!NOTE] +> When customizing AL-Go for GitHub using a fork, your customizations are public and will be visible to everyone. For more information, [read this](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-permissions-and-visibility-of-forks). + +There are two ways of forking AL-Go for GitHub. You can fork the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository and develop AL-Go for GitHub like we do in Microsoft, or you can fork the template repositories [AL-Go PTE](https://github.com/microsoft/AL-Go-PTE) and/or [AL-Go-AppSource](https://github.com/microsoft/AL-Go-AppSource). + +While we don't recommend forking the template repositories, we realize that it is possible for simple changes to the templates. You can fork the template repositories and make the changes directly in your fork. Note that we do not accept any pull requests to the template repositories as they are deployed from the main AL-Go repository. We do not actually develop anything in the template repositories ourself. In the template repositories you will find a branch for every version of AL-Go we have shipped. The main branch is the latest version and the preview branch is the next version. You can customize the preview branch and/or the main branch and then use your fork as the template repository when running Update AL-Go System Files from your app repositories. + +> [!TIP] +> When forking the template repositories, you should include all branches in order to be able to use either the latest version of AL-Go or the preview version of AL-Go. + +When forking the main [AL-Go for GitHub](https://github.com/microsoft/AL-Go) repository, you are basically developing AL-Go in the same way as we are doing in Microsoft. Please follow the guidelines [here](Contribute.md) on how to develop. This gives you maximum customization capabilities, but if your changes are not being contributed to AL-Go, then you will have to merge our changes all the time. + +> [!CAUTION] +> We strongly suggest that you keep your changes to a minimum and that you keep your fork up-to-date with the latest changes of AL-Go for GitHub at all time. + +______________________________________________________________________ + +[back](../README.md) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 21429a9b4..564ba37fe 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -4,6 +4,8 @@ The behavior of AL-Go for GitHub is very much controlled by settings and secrets To learn more about the secrets used by AL-Go for GitHub, please navigate to [Secrets](secrets.md). + + ## Where are the settings located Settings can be defined in GitHub variables or in various settings file. An AL-Go repository can consist of a single project (with multiple apps) or multiple projects (each with multiple apps). Settings can be applied on the project level or on the repository level. Multiple projects in a single repository are comparable to multiple repositories; they are built, deployed, and tested separately. All apps in each project (single or multiple) are built together in the same pipeline, published and tested together. If a repository is multiple projects, each project is stored in a separate folder in the root of the repository. @@ -12,10 +14,14 @@ When running a workflow or a local script, the settings are applied by reading s 1. `ALGoOrgSettings` is a **GitHub variable**, which can be defined on an **organizational level** and will apply to **all AL-Go repositories** in this organization. +1. `.github/AL-Go-TemplateRepoSettings.doNotEdit.json` is the repository settings from a custom template repository (if applicable) + 1. `.github/AL-Go-settings.json` is the **repository settings file**. This settings file contains settings that are relevant for all projects in the repository. **Special note:** The repository settings file can also contains `BcContainerHelper` settings, which will be applied when loading `BcContainerHelper` in a workflow - the GitHub variables are not considered for BcContainerHelper settings. (see expert section). 1. `ALGoRepoSettings` is a **GitHub variable**, which can be defined on an **repository level** and can contain settings that are relevant for **all projects** in the repository. +1. `.github/AL-Go-TemplateProjectSettings.doNotEdit.json` is the project settings from a custom template repository (if applicable) + 1. `.AL-Go/settings.json` is the **project settings file**. If the repository is a single project, the .AL-Go folder is in the root folder of the repository. If the repository contains multiple projects, there will be a .AL-Go folder in each project folder (like `project/.AL-Go/settings.json`) 1. `.github/.settings.json` is the **workflow-specific settings file** for **all projects**. This option is used for the Current, NextMinor and NextMajor workflows to determine artifacts and build numbers when running these workflows. @@ -24,6 +30,8 @@ When running a workflow or a local script, the settings are applied by reading s 1. `.AL-Go/.settings.json` is the **user-specific settings file**. This option is rarely used, but if you have special settings, which should only be used for one specific user (potentially in the local scripts), these settings can be added to a settings file with the name of the user followed by `.settings.json`. + + ## Basic Project settings | Name | Description | Default value | @@ -76,6 +84,8 @@ The repository settings are only read from the repository settings file (.github | commitOptions | If you want more control over how AL-Go creates pull requests or commits changes to the repository you can define `commitOptions`. It is a structure defining how you want AL-Go to handle automated commits or pull requests coming from AL-Go (e.g. for Update AL-Go System Files). The structure contains the following properties:
**messageSuffix** = A string you want to append to the end of commits/pull requests created by AL-Go. This can be useful if you are using the Azure Boards integration (or similar integration) to link commits to work items.
`createPullRequest` : A boolean defining whether AL-Go should create a pull request or attempt to push directly in the branch.
**pullRequestAutoMerge** = A boolean defining whether you want AL-Go pull requests to be set to auto-complete. This will auto-complete the pull requests once all checks are green and all required reviewers have approved.
**pullRequestLabels** = A list of labels to add to the pull request. The labels need to be created in the repository before they can be applied.
If you want different behavior in different AL-Go workflows you can add the `commitOptions` setting to your [workflow-specific settings files](https://github.com/microsoft/AL-Go/blob/main/Scenarios/settings.md#where-are-the-settings-located). | | incrementalBuilds | A structure defining how you want AL-Go to handle incremental builds. When using incremental builds for a build, AL-Go will look for the latest successful CI/CD build, newer than the defined `retentionDays` and only rebuild projects or apps (based on `mode`) which needs to be rebuilt. The structure supports the following properties:
**onPush** = Determines whether incremental builds is enabled in CI/CD triggered by a merge/push event. Default is **false**.
**onPull_Request** = Determines whether incremental builds is enabled in Pull Requests. Default is **true**.
**onSchedule** = Determines whether incremental builds is enabled in CI/CD when running on a schedule. Default is **false**.
**retentionDays** = Number of days a successful build is good (and can be used for incremental builds). Default is **30**.
**mode** = Specifies the mode for incremental builds. Currently, two values are supported. Use **modifiedProjects** when you want to rebuild all apps in all modified projects and depending projects or **modifiedApps** if you want to rebuild modified apps and all apps with dependencies to this app.
**NOTE:** when running incremental builds, it is recommended to also set `workflowConcurrency` for the CI/CD workflow, as defined [here](https://aka.ms/algosettings#workflowConcurrency). | + + ## Advanced settings | Name | Description | Default value | @@ -139,6 +149,8 @@ The following settings are only allowed in workflow specific settings files or i | appSourceContextSecretName | This setting specifies the name (**NOT the secret**) of a secret containing a json string with ClientID, TenantID and ClientSecret or RefreshToken. If this secret exists, AL-Go will can upload builds to AppSource validation. | AppSourceContext | | keyVaultCertificateUrlSecretName
keyVaultCertificatePasswordSecretName
keyVaultClientIdSecretName | If you want to enable KeyVault access for your AppSource App, you need to provide 3 secrets as GitHub Secrets or in the Azure KeyVault. The names of those secrets (**NOT the secrets**) should be specified in the settings file with these 3 settings. Default is to not have KeyVault access from your AppSource App. Read [this](EnableKeyVaultForAppSourceApp.md) for more information. | | + + ## Conditional Settings In any of the settings files, you can add conditional settings by using the ConditionalSettings setting. @@ -191,6 +203,14 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza > [!NOTE] > You can have conditional settings on any level and all conditional settings which has all conditions met will be applied in the order of settings file + appearance. + + +# Expert level + +The settings and functionality in the expert section might require knowledge about GitHub Workflows/Actions, YAML, docker and PowerShell. Please only change these settings and use this functionality after careful consideration as these things might change in the future and will require you to modify the functionality you added based on this. + +Please read the release notes carefully when installing new versions of AL-Go for GitHub. + ## Expert settings (rarely used) | Name | Description | Default value | @@ -211,11 +231,11 @@ Which will ensure that for all repositories named `bcsamples-*` in this organiza | BcContainerHelperVersion | This setting can be set to a specific version (ex. 3.0.8) of BcContainerHelper to force AL-Go to use this version. **latest** means that AL-Go will use the latest released version. **preview** means that AL-Go will use the latest preview version. **dev** means that AL-Go will use the dev branch of containerhelper. | latest (or preview for AL-Go preview) | | unusedALGoSystemFiles | An array of AL-Go System Files, which won't be updated during Update AL-Go System Files. They will instead be removed.
Use this setting with care, as this can break the AL-Go for GitHub functionality and potentially leave your repo no longer functional. | [ ] | -# Expert level + ## Custom Delivery -You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named DeliverTo\*.ps1 in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. +You can override existing AL-Go Delivery functionality or you can define your own custom delivery mechanism for AL-Go for GitHub, by specifying a PowerShell script named `DeliverTo.ps1` in the .github folder. The following example will spin up a delivery job to SharePoint on CI/CD and Release. Beside the script, you also need to create a secret called `Context`, formatted as compressed json, containing delivery information for your delivery target. ### DeliverToSharePoint.ps1 @@ -227,6 +247,7 @@ Param( Write-Host "Current project path: $($parameters.project)" Write-Host "Current project name: $($parameters.projectName)" Write-Host "Delivery Type (CD or Release): $($parameters.type)" +Write-Host "Delivery Context: $($parameters.context)" Write-Host "Folder containing apps: $($parameters.appsFolder)" Write-Host "Folder containing test apps: $($parameters.testAppsFolder)" Write-Host "Folder containing dependencies (requires generateDependencyArtifact set to true): $($parameters.dependenciesFolder)" @@ -254,6 +275,8 @@ Here are the parameters to use in your custom script: | `$parameters.testAppsFolders` | The folders that contain the build artifacts from all builds (from different build modes) of the test apps in the AL-Go project | AllProjects_MyProject-main-TestApps-1.0.0.0, AllProjects_MyProject-main-CleanTestApps-1.0.0.0 | | `$parameters.dependenciesFolders` | The folders that contain the dependencies of the AL-Go project for all builds (from different build modes) | AllProjects_MyProject-main-Dependencies-1.0.0.0, AllProjects_MyProject-main-CleanDependencies-1.0.0.0 | + + ## Custom Deployment You can override existing AL-Go Deployment functionality or you can define your own custom deployment mechanism for AL-Go for GitHub. By specifying a PowerShell script named `DeployTo.ps1` in the .github folder. Default Environment Type is SaaS, but you can define your own type by specifying EnvironmentType in the `DeployTo` setting. The following example will create a script, which would be called by CI/CD and Publish To Environment, when EnvironmentType is set to OnPrem. @@ -304,14 +327,19 @@ Here are the parameters to use in your custom script: | `$parameters."runs-on"` | GitHub runner to be used to run the deployment script | windows-latest | | `$parameters."shell"` | Shell used to run the deployment script, pwsh or powershell | powershell | + + ## Run-AlPipeline script override AL-Go for GitHub utilizes the Run-AlPipeline function from BcContainerHelper to perform the actual build (compile, publish, test etc). The Run-AlPipeline function supports overriding functions for creating containers, compiling apps and a lot of other things. This functionality is also available in AL-Go for GitHub, by adding a file to the .AL-Go folder, you automatically override the function. +Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the future might break the usage of these overrides. + | Override | Description | | :-- | :-- | +| PipelineInitialize.ps1 | Initialize the pipeline | | DockerPull.ps1 | Pull the image specified by the parameter $imageName | | NewBcContainer.ps1 | Create the container using the parameters transferred in the $parameters hashtable | | ImportTestToolkitToBcContainer.ps1 | Import the test toolkit apps specified by the $parameters hashtable | @@ -330,6 +358,8 @@ This functionality is also available in AL-Go for GitHub, by adding a file to th | RestoreDatabasesInBcContainer | Restore Databases in container | | PreCompileApp | Custom script to run _before_ compiling an app. The script should accept the type of the app (`[string] $appType`) and a reference to the compilation parameters (`[ref] $compilationParams`).
Possible values for `$appType` are: _app_, _testApp_, _bcptApp_. | PostCompileApp | Custom script to run _after_ compiling an app. The script should accept the file path of the produced .app file (`[string] $appFilePath`), the type of the app (`[string] $appType`), and a hashtable of the compilation parameters (`[hashtable] $compilationParams`).
Possible values for `$appType` are: _app_, _testApp_, _bcptApp_. +| InstallMissingDependencies | Install missing dependencies | +| PipelineFinalize.ps1 | Finalize the pipeline | ## BcContainerHelper settings @@ -337,6 +367,8 @@ The repo settings file (.github\\AL-Go-Settings.json) can contain BcContainerHel Settings, which might be relevant to set in the settings file includes +Note that changes to AL-Go for GitHub or Run-AlPipeline functionality in the future might break the usage of these overrides. + | Setting | Description | Default | | :-- | :-- | :-- | | baseUrl | The Base Url for the online Business Central Web Client. This should be changed when targetting embed apps. | [https://businesscentral.dynamics.com](https://businesscentral.dynamics.com) | @@ -347,12 +379,66 @@ Settings, which might be relevant to set in the settings file includes | TreatWarningsAsErrors | A list of AL warning codes, which should be treated as errors | [ ] | | DefaultNewContainerParameters | A list of parameters to be added to all container creations in this repo | { } | + + +## Custom jobs in AL-Go for GitHub workflows + +Adding a custom job to any AL-Go for GitHub workflow is done by adding a job with the name `CustomJob` to the end of an AL-Go for GitHub workflow, like this: + +``` + CustomJob-PrepareDeploy: + name: My Job + needs: [ Build ] + runs-on: [ ubuntu-latest ] + defaults: + run: + shell: pwsh + steps: + - name: This is my job + run: | + Write-Host "This is my job" +``` + +In the `needs` property, you specify which jobs should be complete before this job is run. If you require this job to run before other AL-Go for GitHub jobs are complete, you can add the name of this job in the `needs` property of that job, like: + +``` + Deploy: + needs: [ Initialization, Build, CustomJob-PrepareDeploy ] + if: always() && needs.Build.result == 'Success' && needs.Initialization.outputs.environmentCount > 0 + strategy: ${{ fromJson(needs.Initialization.outputs.environmentsMatrixJson) }} +``` + +Custom jobs will be preserved when running Update AL-Go System Files. + +**Note** that installing [apps from the GitHub marketplace](https://github.com/marketplace?type=apps) might require you to add custom jobs or steps to some of the workflows to get the right integration. In custom jobs, you can use any [actions from the GitHub marketplace](https://github.com/marketplace?type=actions). + + + +## Custom template repositories + +If you are utilizing script overrides, custom jobs, custom delivery or like in many repositories, you might want to take advantage of the custom template repository feature. + +A custom template repository is an AL-Go for GitHub repository (without any apps), which is used as a template for the remaining AL-Go for GitHub repositories. As an example, if you are using a custom delivery script, which you want to have in all your repositories, you can create an empty AL-Go for GitHub repository, place the delivery script in the .github folder and use that repository as a template when running Update AL-Go system files in your other repositories. + +This would make sure that all repositories would have this script (and updated versions of the script) in the future. + +The items, which are currently supported from custom template repositories are: + +- Repository script overrides in the .github folder +- Project script overrides in the .AL-Go folder +- Custom workflows in the .github/workflows folder +- Custom jobs in any AL-Go for GitHub workflow +- Changes to repository settings in .github/AL-Go-settings.json +- Changes to project settings in .AL-Go/settings.json + +**Note** that an AL-Go for GitHub custom template repository can be private or public. + ## Your own version of AL-Go for GitHub For experts only, following the description [here](Contribute.md) you can setup a local fork of **AL-Go for GitHub** and use that as your templates. You can fetch upstream changes from Microsoft regularly to incorporate these changes into your version and this way have your modified version of AL-Go for GitHub. > [!NOTE] -> Our goal is to never break repositories, which are using AL-Go for GitHub as their template. We almost certainly will break you if you create local modifications to scripts and pipelines. +> Our goal is to never break repositories, which are using standard AL-Go for GitHub as their template. We almost certainly will break you at some point in time if you create local modifications to scripts and pipelines. ______________________________________________________________________ diff --git a/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 b/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 index 70e8eb4f5..39e13e4ea 100644 --- a/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 +++ b/Templates/AppSource App/.AL-Go/cloudDevEnv.ps1 @@ -15,14 +15,23 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S function DownloadHelperFile { param( [string] $url, - [string] $folder + [string] $folder, + [switch] $notifyAuthenticatedAttempt ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' $name = [System.IO.Path]::GetFileName($url) Write-Host "Downloading $name from $url" $path = Join-Path $folder $name - Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + try { + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + } + catch { + if ($notifyAuthenticatedAttempt) { + Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" + } + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } + } $ProgressPreference = $prevProgressPreference return $path } @@ -42,7 +51,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -notifyAuthenticatedAttempt $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/AppSource App/.AL-Go/localDevEnv.ps1 b/Templates/AppSource App/.AL-Go/localDevEnv.ps1 index ce06cb2ca..aeaaedb73 100644 --- a/Templates/AppSource App/.AL-Go/localDevEnv.ps1 +++ b/Templates/AppSource App/.AL-Go/localDevEnv.ps1 @@ -19,14 +19,23 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S function DownloadHelperFile { param( [string] $url, - [string] $folder + [string] $folder, + [switch] $notifyAuthenticatedAttempt ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' $name = [System.IO.Path]::GetFileName($url) Write-Host "Downloading $name from $url" $path = Join-Path $folder $name - Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + try { + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + } + catch { + if ($notifyAuthenticatedAttempt) { + Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" + } + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } + } $ProgressPreference = $prevProgressPreference return $path } @@ -46,7 +55,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -notifyAuthenticatedAttempt $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/AppSource App/.github/workflows/CICD.yaml b/Templates/AppSource App/.github/workflows/CICD.yaml index ce527aaab..1a5e2d96f 100644 --- a/Templates/AppSource App/.github/workflows/CICD.yaml +++ b/Templates/AppSource App/.github/workflows/CICD.yaml @@ -195,10 +195,11 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' signArtifacts: true useArtifactCache: true + needsContext: ${{ toJson(needs) }} DeployALDoc: needs: [ Initialization, Build ] - if: (!cancelled()) && needs.Build.result == 'Success' && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' + if: (!cancelled()) && (needs.Build.result == 'success' || needs.Build.result == 'skipped') && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' runs-on: [ windows-latest ] name: Deploy Reference Documentation permissions: diff --git a/Templates/AppSource App/.github/workflows/CreateRelease.yaml b/Templates/AppSource App/.github/workflows/CreateRelease.yaml index a7e1ff6c7..e538db408 100644 --- a/Templates/AppSource App/.github/workflows/CreateRelease.yaml +++ b/Templates/AppSource App/.github/workflows/CreateRelease.yaml @@ -121,6 +121,8 @@ jobs: - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main + env: + GITHUB_TOKEN: ${{ github.token }} with: shell: powershell templateUrl: ${{ env.templateUrl }} diff --git a/Templates/AppSource App/.github/workflows/Current.yaml b/Templates/AppSource App/.github/workflows/Current.yaml index e7f69dd20..e398ab102 100644 --- a/Templates/AppSource App/.github/workflows/Current.yaml +++ b/Templates/AppSource App/.github/workflows/Current.yaml @@ -102,6 +102,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' artifactsRetentionDays: ${{ fromJson(needs.Initialization.outputs.artifactsRetentionDays) }} artifactsNameSuffix: 'Current' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/NextMajor.yaml b/Templates/AppSource App/.github/workflows/NextMajor.yaml index 51aed2800..d430e0a56 100644 --- a/Templates/AppSource App/.github/workflows/NextMajor.yaml +++ b/Templates/AppSource App/.github/workflows/NextMajor.yaml @@ -102,6 +102,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' artifactsRetentionDays: ${{ fromJson(needs.Initialization.outputs.artifactsRetentionDays) }} artifactsNameSuffix: 'NextMajor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/NextMinor.yaml b/Templates/AppSource App/.github/workflows/NextMinor.yaml index dafa8cef2..1634dc718 100644 --- a/Templates/AppSource App/.github/workflows/NextMinor.yaml +++ b/Templates/AppSource App/.github/workflows/NextMinor.yaml @@ -102,6 +102,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' artifactsRetentionDays: ${{ fromJson(needs.Initialization.outputs.artifactsRetentionDays) }} artifactsNameSuffix: 'NextMinor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml b/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml index 6b035d436..a1780c43f 100644 --- a/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml +++ b/Templates/AppSource App/.github/workflows/PullRequestHandler.yaml @@ -104,6 +104,7 @@ jobs: secrets: 'licenseFileUrl,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' artifactsRetentionDays: ${{ fromJson(needs.Initialization.outputs.artifactsRetentionDays) }} artifactsNameSuffix: 'PR${{ github.event.number }}' + needsContext: ${{ toJson(needs) }} useArtifactCache: true StatusCheck: diff --git a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml index 770af62fc..03b43fcca 100644 --- a/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/AppSource App/.github/workflows/_BuildALGoProject.yaml @@ -73,6 +73,10 @@ on: description: Flag determining whether to use the Artifacts Cache type: boolean default: false + needsContext: + description: JSON formatted needs context + type: string + default: '{}' permissions: actions: read @@ -167,6 +171,7 @@ jobs: env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} + NeedsContext: ${{ inputs.needsContext }} with: shell: ${{ inputs.shell }} artifact: ${{ env.artifact }} diff --git a/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 b/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 index 70e8eb4f5..39e13e4ea 100644 --- a/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 +++ b/Templates/Per Tenant Extension/.AL-Go/cloudDevEnv.ps1 @@ -15,14 +15,23 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S function DownloadHelperFile { param( [string] $url, - [string] $folder + [string] $folder, + [switch] $notifyAuthenticatedAttempt ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' $name = [System.IO.Path]::GetFileName($url) Write-Host "Downloading $name from $url" $path = Join-Path $folder $name - Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + try { + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + } + catch { + if ($notifyAuthenticatedAttempt) { + Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" + } + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } + } $ProgressPreference = $prevProgressPreference return $path } @@ -42,7 +51,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -notifyAuthenticatedAttempt $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 b/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 index ce06cb2ca..aeaaedb73 100644 --- a/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 +++ b/Templates/Per Tenant Extension/.AL-Go/localDevEnv.ps1 @@ -19,14 +19,23 @@ $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-S function DownloadHelperFile { param( [string] $url, - [string] $folder + [string] $folder, + [switch] $notifyAuthenticatedAttempt ) $prevProgressPreference = $ProgressPreference; $ProgressPreference = 'SilentlyContinue' $name = [System.IO.Path]::GetFileName($url) Write-Host "Downloading $name from $url" $path = Join-Path $folder $name - Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + try { + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path + } + catch { + if ($notifyAuthenticatedAttempt) { + Write-Host -ForegroundColor Red "Failed to download $name, trying authenticated download" + } + Invoke-WebRequest -UseBasicParsing -uri $url -OutFile $path -Headers @{ "Authorization" = "token $(gh auth token)" } + } $ProgressPreference = $prevProgressPreference return $path } @@ -46,7 +55,7 @@ Write-Host -ForegroundColor Yellow @' $tmpFolder = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString())" New-Item -Path $tmpFolder -ItemType Directory -Force | Out-Null -$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder +$GitHubHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Github-Helper.psm1' -folder $tmpFolder -notifyAuthenticatedAttempt $ALGoHelperPath = DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/AL-Go-Helper.ps1' -folder $tmpFolder DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/settings.schema.json' -folder $tmpFolder | Out-Null DownloadHelperFile -url 'https://raw.githubusercontent.com/microsoft/AL-Go-Actions/main/Packages.json' -folder $tmpFolder | Out-Null diff --git a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml index 1e48e4a51..d28aefcf7 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CICD.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CICD.yaml @@ -195,6 +195,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' signArtifacts: true useArtifactCache: true + needsContext: ${{ toJson(needs) }} BuildPP: needs: [ Initialization ] @@ -212,7 +213,7 @@ jobs: DeployALDoc: needs: [ Initialization, Build ] - if: (!cancelled()) && needs.Build.result == 'Success' && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' + if: (!cancelled()) && (needs.Build.result == 'success' || needs.Build.result == 'skipped') && needs.Initialization.outputs.generateALDocArtifact == 1 && github.ref_name == 'main' runs-on: [ windows-latest ] name: Deploy Reference Documentation permissions: diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml index a7e1ff6c7..e538db408 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateRelease.yaml @@ -121,6 +121,8 @@ jobs: - name: Check for updates to AL-Go system files uses: microsoft/AL-Go-Actions/CheckForUpdates@main + env: + GITHUB_TOKEN: ${{ github.token }} with: shell: powershell templateUrl: ${{ env.templateUrl }} diff --git a/Templates/Per Tenant Extension/.github/workflows/Current.yaml b/Templates/Per Tenant Extension/.github/workflows/Current.yaml index e7f69dd20..e398ab102 100644 --- a/Templates/Per Tenant Extension/.github/workflows/Current.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/Current.yaml @@ -102,6 +102,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' artifactsRetentionDays: ${{ fromJson(needs.Initialization.outputs.artifactsRetentionDays) }} artifactsNameSuffix: 'Current' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml b/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml index 51aed2800..d430e0a56 100644 --- a/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/NextMajor.yaml @@ -102,6 +102,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' artifactsRetentionDays: ${{ fromJson(needs.Initialization.outputs.artifactsRetentionDays) }} artifactsNameSuffix: 'NextMajor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml b/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml index dafa8cef2..1634dc718 100644 --- a/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/NextMinor.yaml @@ -102,6 +102,7 @@ jobs: secrets: 'licenseFileUrl,codeSignCertificateUrl,*codeSignCertificatePassword,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' artifactsRetentionDays: ${{ fromJson(needs.Initialization.outputs.artifactsRetentionDays) }} artifactsNameSuffix: 'NextMinor' + needsContext: ${{ toJson(needs) }} PostProcess: needs: [ Initialization, Build ] diff --git a/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml b/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml index 6b035d436..a1780c43f 100644 --- a/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/PullRequestHandler.yaml @@ -104,6 +104,7 @@ jobs: secrets: 'licenseFileUrl,keyVaultCertificateUrl,*keyVaultCertificatePassword,keyVaultClientId,gitHubPackagesContext,applicationInsightsConnectionString' artifactsRetentionDays: ${{ fromJson(needs.Initialization.outputs.artifactsRetentionDays) }} artifactsNameSuffix: 'PR${{ github.event.number }}' + needsContext: ${{ toJson(needs) }} useArtifactCache: true StatusCheck: diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml index 770af62fc..03b43fcca 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildALGoProject.yaml @@ -73,6 +73,10 @@ on: description: Flag determining whether to use the Artifacts Cache type: boolean default: false + needsContext: + description: JSON formatted needs context + type: string + default: '{}' permissions: actions: read @@ -167,6 +171,7 @@ jobs: env: Secrets: '${{ steps.ReadSecrets.outputs.Secrets }}' BuildMode: ${{ inputs.buildMode }} + NeedsContext: ${{ inputs.needsContext }} with: shell: ${{ inputs.shell }} artifact: ${{ env.artifact }} diff --git a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml index b7555c8ef..ad5ac0a0a 100644 --- a/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/_BuildPowerPlatformSolution.yaml @@ -41,6 +41,10 @@ on: required: false type: string +permissions: + contents: read + actions: read + env: ALGoOrgSettings: ${{ vars.ALGoOrgSettings }} ALGoRepoSettings: ${{ vars.ALGoRepoSettings }} diff --git a/Tests/CheckForUpdates.Action.Test.ps1 b/Tests/CheckForUpdates.Action.Test.ps1 index b6b82d82a..3b8cd8379 100644 --- a/Tests/CheckForUpdates.Action.Test.ps1 +++ b/Tests/CheckForUpdates.Action.Test.ps1 @@ -116,6 +116,24 @@ Describe "CheckForUpdates Action Tests" { $permissionsContent.content[1].Trim() | Should -be 'actions: read' } + It 'Test YamlClass Customizations' { + . (Join-Path $scriptRoot "yamlclass.ps1") + + $customizedYaml = [Yaml]::load((Join-Path $PSScriptRoot 'CustomizedYamlSnippet.txt')) + $yaml = [Yaml]::load((Join-Path $PSScriptRoot 'YamlSnippet.txt')) + + # Get Custom jobs from yaml + $customJobs = $customizedYaml.GetCustomJobsFromYaml('CustomJob*') + $customJobs | Should -Not -BeNullOrEmpty + $customJobs.Count | Should -be 1 + + # Apply Custom jobs and steps to yaml + $yaml.AddCustomJobsToYaml($customJobs) + + # Check if new yaml content is equal to customized yaml content + ($yaml.content -join "`r`n") | Should -be ($customizedYaml.content -join "`r`n") + } + It 'Test that Update AL-Go System Files uses fixes runs-on' { . (Join-Path $scriptRoot "yamlclass.ps1") @@ -125,8 +143,6 @@ Describe "CheckForUpdates Action Tests" { $_.Trim() | Should -Be 'runs-on: windows-latest' -Because "Expected 'runs-on: windows-latest', in order to hardcode runner to windows-latest, but got $_" } } - - # Call action } Describe "CheckForUpdates Action: CheckForUpdates.HelperFunctions.ps1" { diff --git a/Tests/CustomizedYamlSnippet.txt b/Tests/CustomizedYamlSnippet.txt new file mode 100644 index 000000000..efee10311 --- /dev/null +++ b/Tests/CustomizedYamlSnippet.txt @@ -0,0 +1,81 @@ +name: 'CI/CD' + +on: + workflow_dispatch: + workflow_run: + workflows: ["Pull Request Handler"] + types: + - completed + push: + paths-ignore: + - '**.md' + - '.github/workflows/*.yaml' + - '!.github/workflows/CICD.yaml' + branches: [ 'main', 'release/*', 'feature/*' ] + +run-name: ${{ fromJson(format('["","Check pull request from {1}/{2}{0} {3}"]',':',github.event.workflow_run.head_repository.owner.login,github.event.workflow_run.head_branch,github.event.workflow_run.display_title))[github.event_name == 'workflow_run'] }} + +permissions: + contents: read + actions: read + pull-requests: write + checks: write + +defaults: + run: + shell: powershell + +env: + workflowDepth: 1 + +jobs: + Initialization: + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: [ windows-latest ] + outputs: + telemetryScopeJson: ${{ steps.init.outputs.telemetryScopeJson }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: true + + - name: Initialize the workflow + id: init + uses: microsoft/AL-Go-Actions/WorkflowInitialize@main + with: + shell: powershell + + - name: Read settings + id: ReadSettings + uses: microsoft/AL-Go-Actions/ReadSettings@main + with: + shell: powershell + + CheckForUpdates: + runs-on: [ windows-latest ] + needs: [ Initialization, CustomJob-MyJob ] + if: github.event_name != 'workflow_run' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read settings + uses: microsoft/AL-Go-Actions/ReadSettings@main + with: + shell: powershell + get: templateUrl + + - name: Check for updates to AL-Go system files + uses: microsoft/AL-Go-Actions/CheckForUpdates@main + with: + shell: powershell + templateUrl: ${{ env.templateUrl }} + + CustomJob-MyJob: + needs: [ Initialization ] + runs-on: [ windows-latest ] + steps: + - name: MyStep + run: | + Write-Host 'My own job!' diff --git a/Workshop/Index.md b/Workshop/Index.md index b716ca688..638afe540 100644 --- a/Workshop/Index.md +++ b/Workshop/Index.md @@ -22,6 +22,13 @@ This workshop shows you how to take advantage of the functionality, which is pro 1. [The Development Process](TheDevelopmentProcess.md) - *FUTURE TOPIC: The recommended way to work with feature branches, pull requests, code reviews and branch protection rules.* 1. [Keeping your Repository Up-to-date](KeepUpToDate.md) - *FUTURE TOPIC: Updating AL-Go for GitHub to the latest version by running a workflow.* +## Expert level + +1. [Custom Delivery](CustomDelivery.md) - *FUTURE TOPIC: Setting up custom delivery to f.ex. a Teams channel.* +1. [Custom Deployment](CustomDeployment.md) - *FUTURE TOPIC: Setting up custom deployment to f.ex. an on-premises environment.* +1. [Custom Jobs](CustomJobs.md) - *FUTURE TOPIC: Adding a custom job to an AL-Go for GitHub workflows.* +1. [Using Custom Templates](CustomTemplates.md) - *FUTURE TOPIC: Using custom templates to ensure that all repositories are using the same customizations* + ## Additional Future topics 1. Dependencies to other apps diff --git a/e2eTests/scenarios/CustomTemplate/runtest.ps1 b/e2eTests/scenarios/CustomTemplate/runtest.ps1 new file mode 100644 index 000000000..7968a81bb --- /dev/null +++ b/e2eTests/scenarios/CustomTemplate/runtest.ps1 @@ -0,0 +1,238 @@ +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidGlobalVars', '', Justification = 'Global vars used for local test execution only.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSReviewUnusedParameter', '', Justification = 'All scenario tests have equal parameter set.')] +[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSAvoidUsingPlainTextForPassword', '', Justification = 'Secrets are transferred as plain text.')] +Param( + [switch] $github, + [switch] $linux, + [string] $githubOwner = $global:E2EgithubOwner, + [string] $repoName = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName()), + [string] $e2eAppId, + [string] $e2eAppKey, + [string] $algoauthapp = ($global:SecureALGOAUTHAPP | Get-PlainText), + [string] $pteTemplate = $global:pteTemplate, + [string] $appSourceTemplate = $global:appSourceTemplate, + [string] $adminCenterApiCredentials = ($global:SecureadminCenterApiCredentials | Get-PlainText), + [string] $azureCredentials = ($global:SecureAzureCredentials | Get-PlainText), + [string] $githubPackagesToken = ($global:SecureGitHubPackagesToken | Get-PlainText) +) + +Write-Host -ForegroundColor Yellow @' +# _____ _ _______ _ _ +# / ____| | | |__ __| | | | | +# | | _ _ ___| |_ ___ _ __ ___ | | ___ _ __ ___ _ __ | | __ _| |_ ___ +# | | | | | / __| __/ _ \| '_ ` _ \ | |/ _ \ '_ ` _ \| '_ \| |/ _` | __/ _ \ +# | |___| |_| \__ \ || (_) | | | | | | | | __/ | | | | | |_) | | (_| | || __/ +# \_____\__,_|___/\__\___/|_| |_| |_| |_|\___|_| |_| |_| .__/|_|\__,_|\__\___| +# | | +# |_| +# This test tests the following scenario: +# +# - Create a new repository based on the PTE template with no apps (this will be the custom template repository) +# - Create a new repository based on the PTE template with 1 app, using compilerfolder and donotpublishapps (this will be the "final" template repository) +# - Run Update AL-Go System Files in final repo (using custom template repository as template) +# - Run Update AL-Go System files in custom template repository +# - Validate that custom job is present in custom template repository +# - Run Update AL-Go System files in final repo +# - Validate that custom job is present in final repo +# +'@ + +$errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 +$prevLocation = Get-Location + +Remove-Module e2eTestHelper -ErrorAction SilentlyContinue +Import-Module (Join-Path $PSScriptRoot "..\..\e2eTestHelper.psm1") -DisableNameChecking +. (Join-Path -Path $PSScriptRoot -ChildPath '..\..\..\Actions\AL-Go-Helper.ps1') +. (Join-Path -Path $PSScriptRoot -ChildPath '..\..\..\Actions\CheckForUpdates\yamlclass.ps1') +. (Join-Path -Path $PSScriptRoot -ChildPath "..\..\..\Actions\CheckForUpdates\CheckForUpdates.HelperFunctions.ps1") + +$templateRepository = "$githubOwner/$repoName-template" +$repository = "$githubOwner/$repoName" +$branch = "main" + +$template = "https://github.com/$pteTemplate" + +# Login +SetTokenAndRepository -github:$github -githubOwner $githubOwner -appId $e2eAppId -appKey $e2eAppKey -repository $repository + +# Create tempolate repository +CreateAlGoRepository ` + -github:$github ` + -linux:$linux ` + -template $template ` + -repository $templateRepository ` + -branch $branch +$templateRepoPath = (Get-Location).Path + +Set-Location $prevLocation + +$appName = 'MyApp' +$publisherName = 'Contoso' + +# Create final repository +CreateAlGoRepository ` + -github:$github ` + -linux:$linux ` + -template $template ` + -repository $repository ` + -branch $branch ` + -contentScript { + Param([string] $path) + $null = CreateNewAppInFolder -folder $path -name $appName -publisher $publisherName + } +$finalRepoPath = (Get-Location).Path + +# Update AL-Go System Files to use template repository +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $algoauthapp -repository $repository -branch $branch | Out-Null + +Set-Location $templateRepoPath + +Pull + +# Make modifications to the template repository + +# Add Custom Jobs to CICD.yaml +$cicdWorkflow = Join-Path $templateRepoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty + +$customJobs = @( + @{ + "Name" = "CustomJob-TemplateInit" + "Content" = @( + "CustomJob-TemplateInit:" + " runs-on: [ windows-latest ]" + " steps:" + " - name: Init" + " run: |" + " Write-Host 'CustomJob-TemplateInit was here!'" + ) + "NeedsThis" = @( 'Initialization' ) + } + @{ + "Name" = "CustomJob-TemplateDeploy" + "Content" = @( + "CustomJob-TemplateDeploy:" + " needs: [ Initialization, Build ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: Deploy" + " run: |" + " Write-Host 'CustomJob-TemplateDeploy was here!'" + ) + "NeedsThis" = @( 'PostProcess' ) + } + @{ + "Name" = "JustSomeTemplateJob" + "Content" = @( + "JustSomeTemplateJob:" + " needs: [ PostProcess ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: JustSomeTemplateStep" + " run: |" + " Write-Host 'JustSomeTemplateJob was here!'" + ) + "NeedsThis" = @( ) + } +) +# Add custom Jobs +$cicdYaml.AddCustomJobsToYaml($customJobs) +$cicdYaml.Save($cicdWorkflow) + +# Push +CommitAndPush -commitMessage 'Add template customizations' + +# Do not run workflows on template repository +CancelAllWorkflows -repository $templateRepository + +# Add local customizations to the final repository +Set-Location $finalRepoPath +Pull + +# Make modifications to the final repository + +# Add Custom Jobs to CICD.yaml +$cicdWorkflow = Join-Path $finalRepoPath '.github/workflows/CICD.yaml' +$cicdYaml = [yaml]::Load($cicdWorkflow) +$cicdYaml | Should -Not -BeNullOrEmpty + +$customJobs = @( + @{ + "Name" = "JustSomeJob" + "Content" = @( + "JustSomeJob:" + " needs: [ Initialization ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: JustSomeStep" + " run: |" + " Write-Host 'JustSomeJob was here!'" + ) + "NeedsThis" = @( 'Build' ) + } + @{ + "Name" = "CustomJob-PreDeploy" + "Content" = @( + "CustomJob-PreDeploy:" + " needs: [ Initialization, Build ]" + " runs-on: [ windows-latest ]" + " steps:" + " - name: PreDeploy" + " run: |" + " Write-Host 'CustomJob-PreDeploy was here!'" + ) + "NeedsThis" = @( 'Deploy' ) + } + @{ + "Name" = "CustomJob-PostDeploy" + "Content" = @( + "CustomJob-PostDeploy:" + " needs: [ Initialization, Build, Deploy ]" + " if: (!cancelled())" + " runs-on: [ windows-latest ]" + " steps:" + " - name: PostDeploy" + " run: |" + " Write-Host 'CustomJob-PostDeploy was here!'" + ) + "NeedsThis" = @( 'PostProcess' ) + } +) +# Add custom Jobs +$cicdYaml.AddCustomJobsToYaml($customJobs) + +# save +$cicdYaml.Save($cicdWorkflow) + + +# Push +CommitAndPush -commitMessage 'Add final repo customizations' + +# Update AL-Go System Files to uptake UseProjectDependencies setting +RunUpdateAlGoSystemFiles -directCommit -wait -templateUrl $templateRepository -ghTokenWorkflow $algoauthapp -repository $repository -branch $branch | Out-Null + +# Stop all currently running workflows and run a new CI/CD workflow +CancelAllWorkflows -repository $repository + +# Pull changes +Pull + +(Join-Path (Get-Location) $CustomTemplateRepoSettingsFile) | Should -Exist +(Join-Path (Get-Location) $CustomTemplateProjectSettingsFile) | Should -Exist + +# Run CICD +$run = RunCICD -repository $repository -branch $branch -wait + +# Check Custom Jobs +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateInit' -stepName 'Init' -expectedText 'CustomJob-TemplateInit was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-TemplateDeploy' -stepName 'Deploy' -expectedText 'CustomJob-TemplateDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PreDeploy' -stepName 'PreDeploy' -expectedText 'CustomJob-PreDeploy was here!' +Test-LogContainsFromRun -runid $run.id -jobName 'CustomJob-PostDeploy' -stepName 'PostDeploy' -expectedText 'CustomJob-PostDeploy was here!' +{ Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeJob' -stepName 'JustSomeStep' -expectedText 'JustSomeJob was here!' } | Should -Throw +{ Test-LogContainsFromRun -runid $run.id -jobName 'JustSomeTemplateJob' -stepName 'JustSomeTemplateStep' -expectedText 'JustSomeTemplateJob was here!' } | Should -Throw + +Set-Location $prevLocation + +RemoveRepository -repository $repository -path $finalRepoPath +RemoveRepository -repository $templateRepository -path $templateRepoPath