From 78e6dfbc47be045425f23caf89a5f4735a45c3b8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 18 Jul 2023 08:14:23 +0200 Subject: [PATCH 01/90] test deploy failure --- Internal/Deploy.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Internal/Deploy.ps1 b/Internal/Deploy.ps1 index 7718e059e..c6a47d27b 100644 --- a/Internal/Deploy.ps1 +++ b/Internal/Deploy.ps1 @@ -321,7 +321,7 @@ try { } if ($_.Name -eq "AL-Go-Helper.ps1" -and ($config.PSObject.Properties.Name -eq "defaultBcContainerHelperVersion") -and ($config.defaultBcContainerHelperVersion)) { # replace defaultBcContainerHelperVersion (even if a version is set) - $lines = $lines | ForEach-Object { $_ -replace '^(\s*)\$defaultBcContainerHelperVersion(\s*)=(\s*)"(.*)" # (.*)$', "`$1`$defaultBcContainerHelperVersion`$2=`$3""$defaultBcContainerHelperVersion"" # `$5" } + $lines = $lines | ForEach-Object { $_ -replace '^(\s*)\$defaultBcContainerHelperVersion(\s*)=(\s*)"(.*)" # (.*)$', "`$1`$defaultBcContainerHelperVersion`$2=`$3""$($config.defaultBcContainerHelperVersion)"" # `$5" } } [System.IO.File]::WriteAllText($dstFile, "$($lines -join "`n")`n") } From a3161353e109a41b1d8987fac3bc80f99b1a1902 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 18 Jul 2023 11:09:34 +0200 Subject: [PATCH 02/90] add bcptline --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index f51bfaec0..39fd0547f 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -41,10 +41,10 @@ try { $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\BCPTTestResults.json" if (Test-Path $bcptTestResultsFile) { - # TODO Display BCPT Test Results + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "## BCPT test results`n`n" } else { - #Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "*BCPT test results not found*`n`n" + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "*BCPT test results not found*`n`n" } TrackTrace -telemetryScope $telemetryScope From e0fddc5488e08e0710458bc0de8dca942727ba72 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 20 Jul 2023 00:27:24 +0200 Subject: [PATCH 03/90] bcpt --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 39 ++-- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 189 +++++++++++--------- Actions/AnalyzeTests/action.yaml | 4 - 3 files changed, 126 insertions(+), 106 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index 39fd0547f..62bfb4e15 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -26,26 +26,33 @@ try { . (Join-Path -Path $PSScriptRoot 'TestResultAnalyzer.ps1') $testResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\TestResults.xml" - if (Test-Path $testResultsFile) { - $testResults = [xml](Get-Content "$project\TestResults.xml") - $testResultSummary = GetTestResultSummary -testResults $testResults -includeFailures 50 - - Add-Content -Path $env:GITHUB_OUTPUT -Value "TestResultMD=$testResultSummary" - Write-Host "TestResultMD=$testResultSummary" - - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultSummary.Replace("\n","`n"))`n" - } - else { - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "*Test results not found*`n`n" - } + $testResultsSummaryMD, $testResultsfailuresMD, $failuresSummaryMD = GetTestResultSummaryMD -path $testResultsFile $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\BCPTTestResults.json" - if (Test-Path $bcptTestResultsFile) { - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "## BCPT test results`n`n" + $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\BCPTBaseLine.json" + $bcptSummaryMD = GetBcptSummaryMD -path $bcptTestResultsFile -baseLinePath $bcptBaseLineFile + + # If summary fits, we will display it in the GitHub summary + if ($summaryMD.Length -gt 65000) { + # If Test results summary is too long, we will not display it in the GitHub summary, instead we will display a message to download the test results + $summaryMD = "Test results summary is too long to be displayed in the GitHub summary. Download test results to see details." } - else { - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "*BCPT test results not found*`n`n" + # If summary AND BCPT summary fits, we will display both in the GitHub summary + if ($summaryMD.Length+$bcptSummaryMD.Length -gt 65000) { + # If Combined Test Results and BCPT summary exceeds GitHub summary capacity, we will not display the BCPT summary + $bcptSummaryMD = "Performance test results summary is too long to be displayed in the GitHub summary. Download BCPT Test results to see details." } + # If summary AND BCPT summary AND failures summary fits, we will display all in the GitHub summary + if ($summaryMD.Length+$failuresMD.Length+$bcptSummaryMD.Length -gt 65000) { + # If Combined Test Results, failures and BCPT summary exceeds GitHub summary capacity, we will not display the failures details, only the failures summary + $failuresMD = $failuresSummaryMD + } + + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "## Test results`n`n" + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($summaryMD.Replace("\n","`n"))`n`n" + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($failuresMD.Replace("\n","`n"))`n`n" + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "## Performance test results`n`n" + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($bcptSummaryMD.Replace("\n","`n"))`n`n" TrackTrace -telemetryScope $telemetryScope } diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index e999de7f2..8bc24747b 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -1,108 +1,125 @@ -function GetTestResultSummary { +function ReadBcptFile { Param( - [xml] $testResults, - [int] $includeFailures + [string] $path + ) + + +} + + +function GetBcptSummaryMD { + Param( + [string] $path, + [string] $baseLinePath + ) + + $summarySb = [System.Text.StringBuilder]::new() + + + $summarySb.ToString() +} + +# Build MarkDown of TestResults file +# This function will not fail if the file does not exist or if any test errors are found +# TestResults is in JUnit format +# Returns both a summary part and a failures part +function GetTestResultSummaryMD { + Param( + [string] $path ) - $totalTests = 0 - $totalTime = 0.0 - $totalFailed = 0 - $totalSkipped = 0 - $failuresIncluded = 0 $summarySb = [System.Text.StringBuilder]::new() $failuresSb = [System.Text.StringBuilder]::new() - if ($testResults.testsuites) { - $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "appName" } | ForEach-Object { $_.Value } } | Select-Object -Unique) - if (-not $appNames) { - $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "extensionId" } | ForEach-Object { $_.Value } } | Select-Object -Unique) - } - $testResults.testsuites.testsuite | ForEach-Object { - $totalTests += $_.Tests - $totalTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) - $totalFailed += $_.failures - $totalSkipped += $_.skipped - } - Write-Host "$($appNames.Count) TestApps, $totalTests tests, $totalFailed failed, $totalSkipped skipped, $totalTime seconds" - $summarySb.Append('|Test app|Tests|Passed|Failed|Skipped|Time|\n|:---|---:|---:|---:|---:|---:|\n') | Out-Null - $appNames | ForEach-Object { - $appName = $_ - $appTests = 0 - $appTime = 0.0 - $appFailed = 0 - $appSkipped = 0 - $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Value -eq $appName } } - $suites | ForEach-Object { - $appTests += [int]$_.tests - $appFailed += [int]$_.failures - $appSkipped += [int]$_.skipped - $appTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) - } - $appPassed = $appTests-$appFailed-$appSkipped - Write-Host "- $appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds" - $summarySb.Append("|$appName|$appTests|") | Out-Null - if ($appPassed -gt 0) { - $summarySb.Append("$($appPassed):white_check_mark:") | Out-Null - } - $summarySb.Append("|") | Out-Null - if ($appFailed -gt 0) { - $summarySb.Append("$($appFailed):x:") | Out-Null + if (Test-Path -Path $path -PathType Leaf) { + $testResults = [xml](Get-Content "$project\TestResults.xml" -Encoding UTF8) + $totalTests = 0 + $totalTime = 0.0 + $totalFailed = 0 + $totalSkipped = 0 + if ($testResults.testsuites) { + $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "appName" } | ForEach-Object { $_.Value } } | Select-Object -Unique) + if (-not $appNames) { + $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "extensionId" } | ForEach-Object { $_.Value } } | Select-Object -Unique) } - $summarySb.Append("|") | Out-Null - if ($appSkipped -gt 0) { - $summarySb.Append("$($appSkipped):white_circle:") | Out-Null + $testResults.testsuites.testsuite | ForEach-Object { + $totalTests += $_.Tests + $totalTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) + $totalFailed += $_.failures + $totalSkipped += $_.skipped } - $summarySb.Append("|$($appTime)s|\n") | Out-Null - if ($appFailed -gt 0) { - $failuresSb.Append("
$appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds\n") | Out-Null + Write-Host "$($appNames.Count) TestApps, $totalTests tests, $totalFailed failed, $totalSkipped skipped, $totalTime seconds" + $summarySb.Append('|Test app|Tests|Passed|Failed|Skipped|Time|\n|:---|---:|---:|---:|---:|---:|\n') | Out-Null + $appNames | ForEach-Object { + $appName = $_ + $appTests = 0 + $appTime = 0.0 + $appFailed = 0 + $appSkipped = 0 + $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Value -eq $appName } } $suites | ForEach-Object { - Write-Host " - $($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds" - if ($_.failures -gt 0 -and $failuresSb.Length -lt 32000) { - $failuresSb.Append("
$($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds") | Out-Null - $_.testcase | ForEach-Object { - if ($_.ChildNodes.Count -gt 0) { - Write-Host " - $($_.name), Failure, $($_.time) seconds" - $failuresSb.Append("
$($_.name), Failure") | Out-Null - $_.ChildNodes | ForEach-Object { - Write-Host " - Error: $($_.message)" - Write-Host " Stacktrace:" - Write-Host " $($_."#text".Trim().Replace("`n","`n "))" - $failuresSb.Append("      Error: $($_.message)
") | Out-Null - $failuresSb.Append("      Stack trace
") | Out-Null - $failuresSb.Append("      $($_."#text".Trim().Replace("`n","
      "))

") | Out-Null + $appTests += [int]$_.tests + $appFailed += [int]$_.failures + $appSkipped += [int]$_.skipped + $appTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) + } + $appPassed = $appTests-$appFailed-$appSkipped + Write-Host "- $appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds" + $summarySb.Append("|$appName|$appTests|") | Out-Null + if ($appPassed -gt 0) { + $summarySb.Append("$($appPassed):white_check_mark:") | Out-Null + } + $summarySb.Append("|") | Out-Null + if ($appFailed -gt 0) { + $summarySb.Append("$($appFailed):x:") | Out-Null + } + $summarySb.Append("|") | Out-Null + if ($appSkipped -gt 0) { + $summarySb.Append("$($appSkipped):white_circle:") | Out-Null + } + $summarySb.Append("|$($appTime)s|\n") | Out-Null + if ($appFailed -gt 0) { + $failuresSb.Append("
$appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds\n") | Out-Null + $suites | ForEach-Object { + Write-Host " - $($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds" + if ($_.failures -gt 0 -and $failuresSb.Length -lt 32000) { + $failuresSb.Append("
$($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds") | Out-Null + $_.testcase | ForEach-Object { + if ($_.ChildNodes.Count -gt 0) { + Write-Host " - $($_.name), Failure, $($_.time) seconds" + $failuresSb.Append("
$($_.name), Failure") | Out-Null + $_.ChildNodes | ForEach-Object { + Write-Host " - Error: $($_.message)" + Write-Host " Stacktrace:" + Write-Host " $($_."#text".Trim().Replace("`n","`n "))" + $failuresSb.Append("      Error: $($_.message)
") | Out-Null + $failuresSb.Append("      Stack trace
") | Out-Null + $failuresSb.Append("      $($_."#text".Trim().Replace("`n","
      "))

") | Out-Null + } + $failuresSb.Append("
") | Out-Null } - $failuresSb.Append("
") | Out-Null } + $failuresSb.Append("
") | Out-Null } - $failuresSb.Append("
") | Out-Null - $failuresIncluded++ } + $failuresSb.Append("
") | Out-Null } - $failuresSb.Append("
") | Out-Null } } - } - if ($totalFailed -gt 0) { - if ($totalFailed -gt $failuresIncluded) { - $failuresSb.Insert(0,"
$totalFailed failing tests (showing the first $failuresIncluded here, download test results to see all)") | Out-Null - } - else { - $failuresSb.Insert(0,"
$totalFailed failing tests") | Out-Null - } - $failuresSb.Append("
") | Out-Null - if (($summarySb.Length + $failuresSb.Length) -lt 65000) { - $summarySb.Append("\n\n$($failuresSb.ToString())") | Out-Null + $failuresSummaryMD = "" + if ($totalFailed -gt 0) { + $failuresSummaryMD = "$totalFailed failing tests, download test results to see details" + $failuresSb.Insert(0,"
$failuresSummaryMD") | Out-Null + $failuresSb.Append("
") | Out-Null } else { - $summarySb.Append("\n\n$totalFailed failing tests. Download test results to see all") | Out-Null + $failuresSummaryMD = "No test failures" + $failuresDb.Append($failuresSummaryMD) | Out-Null } } else { - $summarySb.Append("\n\nNo test failures") | Out-Null - } - if ($summarySb.Length -lt 65500) { - $summarySb.ToString() - } - else { - "$totalFailed failing tests. Download test results to see all" + $summarySb.Append("No test results found") | Out-Null } + $summarySb.ToString() + $failuresSb.ToString() + $failuresSummaryMD } diff --git a/Actions/AnalyzeTests/action.yaml b/Actions/AnalyzeTests/action.yaml index a76cb0439..215f68675 100644 --- a/Actions/AnalyzeTests/action.yaml +++ b/Actions/AnalyzeTests/action.yaml @@ -20,10 +20,6 @@ inputs: project: description: Project to analyze required: true -outputs: - TestResultMD: - description: MarkDown of test result - value: ${{ steps.AnalyzeTests.outputs.TestResultMD }} runs: using: composite steps: From d51bee09b03059362141bd7904d27a93141a2a6e Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 20 Jul 2023 06:50:39 +0200 Subject: [PATCH 04/90] test settings --- Actions/AL-Go-Helper.ps1 | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 3b514167e..ab017d485 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -627,6 +627,13 @@ function ReadSettings { if ($settings.githubRunnerShell -eq "") { $settings.githubRunnerShell = $settings.shell } + # Check that gitHubRunnerShell and Shell is valid + if ($settings.githubRunnerShell -ne "powershell" -and $settings.githubRunnerShell -ne "pwsh") { + throw "Invalid value for setting: gitHubRunnerShell: $($settings.githubRunnerShell)" + } + if ($settings.shell -ne "powershell" -and $settings.shell -ne "pwsh") { + throw "Invalid value for setting: shell: $($settings.githubRunnerShell)" + } $settings } From 415c1f34f4ebe6ef43f8846c8e3f96face7c28b6 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Jul 2023 20:59:23 +0200 Subject: [PATCH 05/90] Add bcpt test analyzer --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 157 +++++++++++++++++++- 1 file changed, 155 insertions(+), 2 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 8bc24747b..e14c853cf 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -1,21 +1,174 @@ +$ErrorActionPreference = "stop" function ReadBcptFile { Param( [string] $path ) + if (-not (Test-Path -Path $path -PathType Leaf)) { + return $null + } -} + # Read BCPT file + $bcptResult = Get-Content -Path $path -Encoding UTF8 | ConvertFrom-Json + $suites = @{} + # Sort by bcptCode, codeunitID, operation + $bcptResult | ForEach-Object { + $bcptCode = $_.bcptCode + $codeunitID = $_.codeunitID + $codeunitName = $_.codeunitName + $operation = $_.operation + + # Create Suite if it doesn't exist + if(-not $suites.containsKey($bcptCode)) { + $suites."$bcptCode" = @{} + } + # Create Codeunit under Suite if it doesn't exist + if (-not $suites."$bcptCode".ContainsKey("$codeunitID")) { + $suites."$bcptCode"."$codeunitID" = @{ + "codeunitName" = $codeunitName + "operations" = @{} + } + } + # Create Operation under Codeunit if it doesn't exist + if (-not $suites."$bcptCode"."$codeunitID"."operations".ContainsKey($operation)) { + $suites."$bcptCode"."$codeunitID"."operations"."$operation" = @{ + "measurements" = @() + } + } + # Add measurement to measurements under operation + $suites."$bcptCode"."$codeunitID"."operations"."$operation".measurements += @(@{ + "sessionId" = $_.sessionId + "durationMin" = $_.durationMin + "numberOfSQLStmts" = $_.numberOfSQLStmts + "status" = $_.status + }) + } + # calculate statistics on measurements, skipping the $skipMeasurements longest measurements +# $suites.Keys | ForEach-Object { +# $suite = $suites."$_" +# $suite.Keys | ForEach-Object { +# $codeunit = $suite."$_" +# $codeunit."operations".Keys | ForEach-Object { +# $operation = $codeunit."operations"."$_" +# # Get measurements to use for statistics +# $measurements = @($operation."measurements" | Sort-Object -Descending -Property 'durationMin' | Select-Object -Skip $skipMeasurements) +# # Calculate statistics and store them in the operation +# $operation."durationMin" = $measurements | Measure-Object -property 'durationMin' -AllStats +# $operation."numberOfSQLStmts" = $measurements | Measure-Object -property 'numberOfSQLStmts' -AllStats +# } +# } +# } + + $suites +} function GetBcptSummaryMD { Param( [string] $path, - [string] $baseLinePath + [string] $baseLinePath, + [int] $skipMeasurements = 1, + [int] $warningThreasHold = 5, + [int] $errorThreasHold = 10 ) + $bcpt = ReadBcptFile -path $path + $baseLine = ReadBcptFile -path $baseLinePath + $summarySb = [System.Text.StringBuilder]::new() + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|$(if ($baseLine){'Status|'})Duration|$(if ($baseLine){'Duration (BaseLine)|'})SQL Stmts|$(if ($baseLine){'SQL Stmts (BaseLine)|'})\n|:---|:---|:---|:---|$(if ($baseLine){'---:|'}):--:|$(if ($baseLine){'---:|'})---:|$(if ($baseLine){'---:|'})\n") | Out-Null + $lastSuiteName = '' + $lastCodeunitID = '' + $lastCodeunitName = '' + $lastOperationName = '' + # calculate statistics on measurements, skipping the $skipMeasurements longest measurements + $bcpt.Keys | ForEach-Object { + $suiteName = $_ + Write-Host $suiteName + $suite = $bcpt."$suiteName" + $suite.Keys | ForEach-Object { + $codeUnitID = $_ + $codeunit = $suite."$codeunitID" + $codeUnitName = $codeunit.codeunitName + $codeunit."operations".Keys | ForEach-Object { + $operationName = $_ + Write-Host $operationName + $operation = $codeunit."operations"."$operationName" + # Get measurements to use for statistics + $measurements = @($operation."measurements" | Sort-Object -Descending -Property 'durationMin' | Select-Object -Skip $skipMeasurements) + # Calculate statistics and store them in the operation + $durationMin = ($measurements | Measure-Object -property 'durationMin' -AllStats).Average + $numberOfSQLStmts = ($measurements | Measure-Object -property 'numberOfSQLStmts' -AllStats).Average + + try { + $baseLineMeasurements = @($baseLine."$suiteName"."$codeUnitID"."operations"."$operationName"."measurements" | Sort-Object -Descending -Property 'durationMin' | Select-Object -Skip $skipMeasurements) + if ($baseLineMeasurements.Count -eq 0) { + throw "No base line measurements" + } + $baseLineDurationMin = ($baseLineMeasurements | Measure-Object -property 'durationMin' -AllStats).Average + $baseLineNumberOfSQLStmts = ($baseLineMeasurements | Measure-Object -property 'numberOfSQLStmts' -AllStats).Average + } + catch { + $baseLineDurationMin = $durationMin + $baseLineNumberOfSQLStmts = $numberOfSQLStmts + } + + $pctDurationMin = ($durationMin-$baseLineDurationMin)*100/$baseLineDurationMin + if ($pctDurationMin -le 0) { + $durationMinStr = "**$($durationMin.ToString("N2"))**|" + $baseLineDurationMinStr = "$($baseLineDurationMin.ToString("N2"))|" + } + else { + $durationMinStr = "$($durationMin.ToString("N2"))|" + $baseLineDurationMinStr = "**$($baseLineDurationMin.ToString("N2"))**|" + } + + $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseLineNumberOfSQLStmts)*100/$baseLineNumberOfSQLStmts + if ($pctNumberOfSQLStmts -le 0) { + $numberOfSQLStmtsStr = "**$($numberOfSQLStmts.ToString("N0"))**|" + $baseLineNumberOfSQLStmtsStr = "$($baseLineNumberOfSQLStmts.ToString("N0"))|" + } + else { + $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" + $baseLineNumberOfSQLStmtsStr = "**$($baseLineNumberOfSQLStmts.ToString("N0"))**|" + } + + if (-not $baseLine) { + $statusStr = '' + $baselinedurationMinStr = '' + $baseLineNumberOfSQLStmtsStr = '' + } + elseif ($pctDurationMin -gt $errorThreasHold -or $pctNumberOfSQLStmts -gt $errorThreasHold) { + $statusStr = ":x:|" + } + elseif ($pctDurationMin -gt $warningThreasHold -or $pctNumberOfSQLStmts -gt $warningThreasHold) { + $statusStr = ":warning:|" + } + else { + $statusStr = ":heavy_check_mark:|" + } + + $thisSuiteName = ''; if ($suiteName -ne $lastSuiteName) { $thisSuiteName = $suiteName } + $thisCodeunitID = ''; if ($codeunitID -ne $lastCodeunitID) { $thisCodeunitID = $codeunitID } + $thisCodeunitName = ''; if ($codeunitName -ne $lastCodeunitName) { $thisCodeunitName = $codeunitName } + $thisOperationName = ''; if ($operationName -ne $lastOperationName) { $thisOperationName = $operationName } + + $summarySb.Append("|$thisSuiteName|$thisCodeunitID|$thisCodeunitName|$thisOperationName|$statusStr$durationMinStr$baseLineDurationMinStr$numberOfSQLStmtsStr$baseLineNumberOfSQLStmtsStr\n") | Out-Null + + $lastSuiteName = $suiteName + $lastCodeunitID = $codeUnitID + $lastCodeunitName = $codeUnitName + $lastOperationName = $operationName + } + } + } + + if (-not $baseLine) { + $summarySb.Append('\n> No baseline provided. Copy a set of BCPT results to $baseLinePath in order to establish a baseline.') | Out-Null + } + $summarySb.ToString() } From 98966e0800a11dacd201367a0a68b1051dd51c9a Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Jul 2023 21:00:50 +0200 Subject: [PATCH 06/90] sb --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index e14c853cf..a94d7854e 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -266,7 +266,7 @@ function GetTestResultSummaryMD { } else { $failuresSummaryMD = "No test failures" - $failuresDb.Append($failuresSummaryMD) | Out-Null + $failuresSb.Append($failuresSummaryMD) | Out-Null } } else { From d8016fcf2da0e3ebe43a0baed507ceabda7fc6bc Mon Sep 17 00:00:00 2001 From: freddydk Date: Sun, 23 Jul 2023 21:33:29 +0200 Subject: [PATCH 07/90] ave --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 24 ++++----------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index a94d7854e..37b303ad6 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -44,22 +44,6 @@ function ReadBcptFile { }) } - # calculate statistics on measurements, skipping the $skipMeasurements longest measurements -# $suites.Keys | ForEach-Object { -# $suite = $suites."$_" -# $suite.Keys | ForEach-Object { -# $codeunit = $suite."$_" -# $codeunit."operations".Keys | ForEach-Object { -# $operation = $codeunit."operations"."$_" -# # Get measurements to use for statistics -# $measurements = @($operation."measurements" | Sort-Object -Descending -Property 'durationMin' | Select-Object -Skip $skipMeasurements) -# # Calculate statistics and store them in the operation -# $operation."durationMin" = $measurements | Measure-Object -property 'durationMin' -AllStats -# $operation."numberOfSQLStmts" = $measurements | Measure-Object -property 'numberOfSQLStmts' -AllStats -# } -# } -# } - $suites } @@ -99,16 +83,16 @@ function GetBcptSummaryMD { # Get measurements to use for statistics $measurements = @($operation."measurements" | Sort-Object -Descending -Property 'durationMin' | Select-Object -Skip $skipMeasurements) # Calculate statistics and store them in the operation - $durationMin = ($measurements | Measure-Object -property 'durationMin' -AllStats).Average - $numberOfSQLStmts = ($measurements | Measure-Object -property 'numberOfSQLStmts' -AllStats).Average + $durationMin = ($measurements | Measure-Object -property 'durationMin' -Average).Average + $numberOfSQLStmts = ($measurements | Measure-Object -property 'numberOfSQLStmts' -Average).Average try { $baseLineMeasurements = @($baseLine."$suiteName"."$codeUnitID"."operations"."$operationName"."measurements" | Sort-Object -Descending -Property 'durationMin' | Select-Object -Skip $skipMeasurements) if ($baseLineMeasurements.Count -eq 0) { throw "No base line measurements" } - $baseLineDurationMin = ($baseLineMeasurements | Measure-Object -property 'durationMin' -AllStats).Average - $baseLineNumberOfSQLStmts = ($baseLineMeasurements | Measure-Object -property 'numberOfSQLStmts' -AllStats).Average + $baseLineDurationMin = ($baseLineMeasurements | Measure-Object -property 'durationMin' -Average).Average + $baseLineNumberOfSQLStmts = ($baseLineMeasurements | Measure-Object -property 'numberOfSQLStmts' -Average).Average } catch { $baseLineDurationMin = $durationMin From 99de5d3867447559bb3707e28d84831d3bec1ca7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Jul 2023 01:02:56 +0200 Subject: [PATCH 08/90] ps5 support --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 37b303ad6..272ac0315 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -4,6 +4,10 @@ function ReadBcptFile { [string] $path ) + if (-not $path) { + return $null + } + if (-not (Test-Path -Path $path -PathType Leaf)) { return $null } @@ -50,7 +54,7 @@ function ReadBcptFile { function GetBcptSummaryMD { Param( [string] $path, - [string] $baseLinePath, + [string] $baseLinePath = '', [int] $skipMeasurements = 1, [int] $warningThreasHold = 5, [int] $errorThreasHold = 10 @@ -70,7 +74,6 @@ function GetBcptSummaryMD { # calculate statistics on measurements, skipping the $skipMeasurements longest measurements $bcpt.Keys | ForEach-Object { $suiteName = $_ - Write-Host $suiteName $suite = $bcpt."$suiteName" $suite.Keys | ForEach-Object { $codeUnitID = $_ @@ -78,21 +81,20 @@ function GetBcptSummaryMD { $codeUnitName = $codeunit.codeunitName $codeunit."operations".Keys | ForEach-Object { $operationName = $_ - Write-Host $operationName $operation = $codeunit."operations"."$operationName" # Get measurements to use for statistics - $measurements = @($operation."measurements" | Sort-Object -Descending -Property 'durationMin' | Select-Object -Skip $skipMeasurements) + $measurements = @($operation."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) # Calculate statistics and store them in the operation - $durationMin = ($measurements | Measure-Object -property 'durationMin' -Average).Average - $numberOfSQLStmts = ($measurements | Measure-Object -property 'numberOfSQLStmts' -Average).Average + $durationMin = ($measurements | ForEach-Object { $_.durationMin } | Measure-Object -Average).Average + $numberOfSQLStmts = ($measurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Average).Average try { - $baseLineMeasurements = @($baseLine."$suiteName"."$codeUnitID"."operations"."$operationName"."measurements" | Sort-Object -Descending -Property 'durationMin' | Select-Object -Skip $skipMeasurements) + $baseLineMeasurements = @($baseLine."$suiteName"."$codeUnitID"."operations"."$operationName"."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) if ($baseLineMeasurements.Count -eq 0) { throw "No base line measurements" } - $baseLineDurationMin = ($baseLineMeasurements | Measure-Object -property 'durationMin' -Average).Average - $baseLineNumberOfSQLStmts = ($baseLineMeasurements | Measure-Object -property 'numberOfSQLStmts' -Average).Average + $baseLineDurationMin = ($baseLineMeasurements | ForEach-Object { $_.durationMin } | Measure-Object -Average).Average + $baseLineNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Average).Average } catch { $baseLineDurationMin = $durationMin From 8816ff5b65c37ab388147cdc640360e4700d91bc Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Jul 2023 07:13:38 +0200 Subject: [PATCH 09/90] vars --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 18 +++++++++--------- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index 62bfb4e15..4085f09a9 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -26,31 +26,31 @@ try { . (Join-Path -Path $PSScriptRoot 'TestResultAnalyzer.ps1') $testResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\TestResults.xml" - $testResultsSummaryMD, $testResultsfailuresMD, $failuresSummaryMD = GetTestResultSummaryMD -path $testResultsFile + $testResultsSummaryMD, $testResultsfailuresMD, $testResultsFailuresSummaryMD = GetTestResultSummaryMD -path $testResultsFile $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\BCPTTestResults.json" $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\BCPTBaseLine.json" $bcptSummaryMD = GetBcptSummaryMD -path $bcptTestResultsFile -baseLinePath $bcptBaseLineFile # If summary fits, we will display it in the GitHub summary - if ($summaryMD.Length -gt 65000) { + if ($testResultsSummaryMD.Length -gt 65000) { # If Test results summary is too long, we will not display it in the GitHub summary, instead we will display a message to download the test results - $summaryMD = "Test results summary is too long to be displayed in the GitHub summary. Download test results to see details." + $testResultsSummaryMD = "Test results summary size exceeds GitHub summary capacity. Download **TestResults** artifact to see details." } # If summary AND BCPT summary fits, we will display both in the GitHub summary - if ($summaryMD.Length+$bcptSummaryMD.Length -gt 65000) { + if ($testResultsSummaryMD.Length + $bcptSummaryMD.Length -gt 65000) { # If Combined Test Results and BCPT summary exceeds GitHub summary capacity, we will not display the BCPT summary - $bcptSummaryMD = "Performance test results summary is too long to be displayed in the GitHub summary. Download BCPT Test results to see details." + $bcptSummaryMD = "Performance test results summary size exceeds GitHub summary capacity. Download **BcptTestResults** artifact to see details." } # If summary AND BCPT summary AND failures summary fits, we will display all in the GitHub summary - if ($summaryMD.Length+$failuresMD.Length+$bcptSummaryMD.Length -gt 65000) { + if ($testResultsSummaryMD.Length + $testResultsfailuresMD.Length + $bcptSummaryMD.Length -gt 65000) { # If Combined Test Results, failures and BCPT summary exceeds GitHub summary capacity, we will not display the failures details, only the failures summary - $failuresMD = $failuresSummaryMD + $testResultsfailuresMD = $testResultsFailuresSummaryMD } Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "## Test results`n`n" - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($summaryMD.Replace("\n","`n"))`n`n" - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($failuresMD.Replace("\n","`n"))`n`n" + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsSummaryMD.Replace("\n","`n"))`n`n" + Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsfailuresMD.Replace("\n","`n"))`n`n" Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "## Performance test results`n`n" Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($bcptSummaryMD.Replace("\n","`n"))`n`n" diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 272ac0315..9a32cdcee 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -152,7 +152,7 @@ function GetBcptSummaryMD { } if (-not $baseLine) { - $summarySb.Append('\n> No baseline provided. Copy a set of BCPT results to $baseLinePath in order to establish a baseline.') | Out-Null + $summarySb.Append("\nNo baseline provided. Copy a set of BCPT results to $([System.IO.Path]::GetFileName($baseLinePath)) in the project folder in order to establish a baseline.") | Out-Null } $summarySb.ToString() From 3a84ebc76b14f5afbe735ddeff9a3bc40e0f5461 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Jul 2023 07:52:21 +0200 Subject: [PATCH 10/90] TODOs --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 4 ++-- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index 4085f09a9..039d41a8b 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -28,8 +28,8 @@ try { $testResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\TestResults.xml" $testResultsSummaryMD, $testResultsfailuresMD, $testResultsFailuresSummaryMD = GetTestResultSummaryMD -path $testResultsFile - $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\BCPTTestResults.json" - $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\BCPTBaseLine.json" + $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptTestResults.json" + $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptBaseLine.json" $bcptSummaryMD = GetBcptSummaryMD -path $bcptTestResultsFile -baseLinePath $bcptBaseLineFile # If summary fits, we will display it in the GitHub summary diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 9a32cdcee..ae2ee7815 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -60,6 +60,8 @@ function GetBcptSummaryMD { [int] $errorThreasHold = 10 ) + # TODO: grab skipMeasurements and thresholds from settings + $bcpt = ReadBcptFile -path $path $baseLine = ReadBcptFile -path $baseLinePath @@ -128,9 +130,11 @@ function GetBcptSummaryMD { } elseif ($pctDurationMin -gt $errorThreasHold -or $pctNumberOfSQLStmts -gt $errorThreasHold) { $statusStr = ":x:|" + # TODO: issue error } elseif ($pctDurationMin -gt $warningThreasHold -or $pctNumberOfSQLStmts -gt $warningThreasHold) { $statusStr = ":warning:|" + # TODO: issue warning } else { $statusStr = ":heavy_check_mark:|" From 0818399ed6a25995c35cc89294ed31d66c7c6e72 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Jul 2023 08:17:45 +0200 Subject: [PATCH 11/90] collect --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index ae2ee7815..394f52b78 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -1,14 +1,9 @@ -$ErrorActionPreference = "stop" function ReadBcptFile { Param( [string] $path ) - if (-not $path) { - return $null - } - - if (-not (Test-Path -Path $path -PathType Leaf)) { + if ((-not $path) -or (-not (Test-Path -Path $path -PathType Leaf))) { return $null } @@ -47,7 +42,6 @@ function ReadBcptFile { "status" = $_.status }) } - $suites } From 9be367e1bd2ade57832f8b683f9b2a8ad29079d7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Jul 2023 08:36:31 +0200 Subject: [PATCH 12/90] display operation name if codeunit or suite change --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 394f52b78..408c5508f 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -134,10 +134,10 @@ function GetBcptSummaryMD { $statusStr = ":heavy_check_mark:|" } - $thisSuiteName = ''; if ($suiteName -ne $lastSuiteName) { $thisSuiteName = $suiteName } - $thisCodeunitID = ''; if ($codeunitID -ne $lastCodeunitID) { $thisCodeunitID = $codeunitID } - $thisCodeunitName = ''; if ($codeunitName -ne $lastCodeunitName) { $thisCodeunitName = $codeunitName } $thisOperationName = ''; if ($operationName -ne $lastOperationName) { $thisOperationName = $operationName } + $thisCodeunitName = ''; if ($codeunitName -ne $lastCodeunitName) { $thisCodeunitName = $codeunitName; $thisOperationName = $operationName } + $thisCodeunitID = ''; if ($codeunitID -ne $lastCodeunitID) { $thisCodeunitID = $codeunitID; $thisOperationName = $operationName } + $thisSuiteName = ''; if ($suiteName -ne $lastSuiteName) { $thisSuiteName = $suiteName; $thisOperationName = $operationName } $summarySb.Append("|$thisSuiteName|$thisCodeunitID|$thisCodeunitName|$thisOperationName|$statusStr$durationMinStr$baseLineDurationMinStr$numberOfSQLStmtsStr$baseLineNumberOfSQLStmtsStr\n") | Out-Null From effdfa0d5ad6af8c7149f24136a62009d4b28798 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 24 Jul 2023 15:20:51 +0200 Subject: [PATCH 13/90] add tests --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 2 +- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 4 +- Actions/AnalyzeTests/action.yaml | 4 +- Tests/AnalyzeTests.Test.ps1 | 63 +++++++++++++++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 Tests/AnalyzeTests.Test.ps1 diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index 039d41a8b..44d825597 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -6,7 +6,7 @@ Param( [Parameter(HelpMessage = "Specifies the parent telemetry scope for the telemetry signal", Mandatory = $false)] [string] $parentTelemetryScopeJson = '7b7d', [Parameter(HelpMessage = "Project to analyze", Mandatory = $false)] - [string] $project + [string] $project = '.' ) $ErrorActionPreference = "Stop" diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 408c5508f..105c96d92 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -36,10 +36,8 @@ function ReadBcptFile { } # Add measurement to measurements under operation $suites."$bcptCode"."$codeunitID"."operations"."$operation".measurements += @(@{ - "sessionId" = $_.sessionId "durationMin" = $_.durationMin "numberOfSQLStmts" = $_.numberOfSQLStmts - "status" = $_.status }) } $suites @@ -168,7 +166,7 @@ function GetTestResultSummaryMD { $summarySb = [System.Text.StringBuilder]::new() $failuresSb = [System.Text.StringBuilder]::new() if (Test-Path -Path $path -PathType Leaf) { - $testResults = [xml](Get-Content "$project\TestResults.xml" -Encoding UTF8) + $testResults = [xml](Get-Content -path "$project\TestResults.xml" -Encoding UTF8) $totalTests = 0 $totalTime = 0.0 $totalFailed = 0 diff --git a/Actions/AnalyzeTests/action.yaml b/Actions/AnalyzeTests/action.yaml index 215f68675..9fa4db5b0 100644 --- a/Actions/AnalyzeTests/action.yaml +++ b/Actions/AnalyzeTests/action.yaml @@ -19,13 +19,13 @@ inputs: default: '7b7d' project: description: Project to analyze - required: true + required: false + default: '.' runs: using: composite steps: - name: run shell: ${{ inputs.shell }} - id: AnalyzeTests env: _actor: ${{ inputs.actor }} _token: ${{ inputs.token }} diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 new file mode 100644 index 000000000..575dadc24 --- /dev/null +++ b/Tests/AnalyzeTests.Test.ps1 @@ -0,0 +1,63 @@ +Get-Module TestActionsHelper | Remove-Module -Force +Import-Module (Join-Path $PSScriptRoot 'TestActionsHelper.psm1') + +function GetBcptTestResultFile { + Param( + [int] $noOfSuites = 1, + [int] $noOfCodeunits = 100, + [int] $noOfOperations = 100, + [int] $noOfMeasurements = 100 + ) + + $bcpt = @() + 1..$noOfSuites | ForEach-Object { + $suiteName = "SUITE$_" + 1..$noOfCodeunits | ForEach-Object { + $codeunitID = $_ + $codeunitName = "Codeunit$_" + 1..$noOfOperations | ForEach-Object { + $operationName = "Operation$_" + 1..$noOfMeasurements | ForEach-Object { + $no = $_ + $bcpt += @(@{ + "id" = [GUID]::NewGuid().ToString() + "bcptCode" = $suiteName + "codeunitID" = $codeunitID + "codeunitName" = $codeunitName + "operation" = $operationName + "durationMin" = 1000*$codeunitID+$no + "numberOfSQLStmts" = $codeunitID+$no + }) + } + } + } + } + $bcpt | ConvertTo-Json -Depth 100 | set-content -path c:\temp\bcpt.json -encoding UTF8 +} + +Describe "AnalyzeTests Action Tests" { + BeforeAll { + $actionName = "AnalyzeTests" + $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve + $scriptName = "$actionName.ps1" + $scriptPath = Join-Path $scriptRoot $scriptName + $actionScript = GetActionScript -scriptRoot $scriptRoot -scriptName $scriptName + } + + It 'Compile Action' { + Invoke-Expression $actionScript + } + + It 'Test action.yaml matches script' { + $permissions = [ordered]@{ + } + $outputs = [ordered]@{ + } + YamlTest -scriptRoot $scriptRoot -actionName $actionName -actionScript $actionScript -permissions $permissions -outputs $outputs + } + + It 'Test ReadBcptFile' { + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + Write-Host $test + } +} From 7fa0b49da1ac602d14de5696f49e446df3093401 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 00:30:48 +0200 Subject: [PATCH 14/90] tests --- Tests/AnalyzeTests.Test.ps1 | 45 ++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 575dadc24..e68705636 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -4,9 +4,11 @@ Import-Module (Join-Path $PSScriptRoot 'TestActionsHelper.psm1') function GetBcptTestResultFile { Param( [int] $noOfSuites = 1, - [int] $noOfCodeunits = 100, - [int] $noOfOperations = 100, - [int] $noOfMeasurements = 100 + [int] $noOfCodeunits = 1, + [int] $noOfOperations = 1, + [int] $noOfMeasurements = 1, + [int] $durationOffset = 0, + [int] $numberOfSQLStmtsOffset = 0 ) $bcpt = @() @@ -16,7 +18,8 @@ function GetBcptTestResultFile { $codeunitID = $_ $codeunitName = "Codeunit$_" 1..$noOfOperations | ForEach-Object { - $operationName = "Operation$_" + $operationNo = $_ + $operationName = "Operation$operationNo" 1..$noOfMeasurements | ForEach-Object { $no = $_ $bcpt += @(@{ @@ -25,16 +28,21 @@ function GetBcptTestResultFile { "codeunitID" = $codeunitID "codeunitName" = $codeunitName "operation" = $operationName - "durationMin" = 1000*$codeunitID+$no - "numberOfSQLStmts" = $codeunitID+$no + "durationMin" = $codeunitNo*100+$operationNo*10+$no+$durationOffset + "numberOfSQLStmts" = $operationNo+$numberOfSQLStmtsOffset }) } } } } - $bcpt | ConvertTo-Json -Depth 100 | set-content -path c:\temp\bcpt.json -encoding UTF8 + $filename = Join-Path $ENV:TEMP "$([GUID]::NewGuid().ToString()).json" + $bcpt | ConvertTo-Json -Depth 100 | Set-Content -Path $filename -Encoding UTF8 + return $filename } +$bcptFilename = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 3 -noOfMeasurements 4 +$bcptBaseLine = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 4 -noOfOperations 6 -noOfMeasurements 4 -durationOffset -1 -numberOfSQLStmtsOffset -1 + Describe "AnalyzeTests Action Tests" { BeforeAll { $actionName = "AnalyzeTests" @@ -58,6 +66,27 @@ Describe "AnalyzeTests Action Tests" { It 'Test ReadBcptFile' { . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') - Write-Host $test + $bcpt = ReadBcptFile -path $bcptFilename + $bcpt.Count | should -Be 1 + $bcpt."SUITE1".Count | should -Be 2 + $bcpt."SUITE1"."1".operations.Count | should -Be 3 + $bcpt."SUITE1"."1".operations."operation2".measurements.Count | should -Be 4 } + + It 'Test GetBcptSummaryMD (no baseline)' { + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + $md = GetBcptSummaryMD -path $bcptFilename + $md | should -Match 'No baseline provided' + [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 + [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 6 + [regex]::Matches($md, '\|').Count | should -Be (7*8) + +# -Contain 'No baseline provided' +# $md | should -Contain '|SUITE1|' + +# $md.Replace('\n',"`n") | Set-Content -Path "C:\temp\bcpt.md" -Encoding UTF8 + + } + } From 8a55ea78f48a0780720931c98b63b7acf75e0f37 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 08:18:47 +0200 Subject: [PATCH 15/90] add .md tests --- Tests/AnalyzeTests.Test.ps1 | 52 ++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index e68705636..04c317c7f 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -40,16 +40,16 @@ function GetBcptTestResultFile { return $filename } -$bcptFilename = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 3 -noOfMeasurements 4 -$bcptBaseLine = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 4 -noOfOperations 6 -noOfMeasurements 4 -durationOffset -1 -numberOfSQLStmtsOffset -1 - Describe "AnalyzeTests Action Tests" { BeforeAll { $actionName = "AnalyzeTests" $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve $scriptName = "$actionName.ps1" - $scriptPath = Join-Path $scriptRoot $scriptName $actionScript = GetActionScript -scriptRoot $scriptRoot -scriptName $scriptName + + $bcptFilename = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 3 -noOfMeasurements 4 + $bcptBaseLine1 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 4 -noOfOperations 6 -noOfMeasurements 4 -durationOffset 1 -numberOfSQLStmtsOffset 1 + $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 2 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 1 } It 'Compile Action' { @@ -76,17 +76,51 @@ Describe "AnalyzeTests Action Tests" { It 'Test GetBcptSummaryMD (no baseline)' { . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') $md = GetBcptSummaryMD -path $bcptFilename + Write-Host $md.Replace('\n',"`n") $md | should -Match 'No baseline provided' + $columns = 6 + $rows = 8 [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 [regex]::Matches($md, '\|Operation.\|').Count | should -Be 6 - [regex]::Matches($md, '\|').Count | should -Be (7*8) - -# -Contain 'No baseline provided' -# $md | should -Contain '|SUITE1|' + [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) + } -# $md.Replace('\n',"`n") | Set-Content -Path "C:\temp\bcpt.md" -Encoding UTF8 + It 'Test GetBcptSummaryMD (with worse baseline)' { + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine1 + Write-Host $md.Replace('\n',"`n") + $md | should -Not -Match 'No baseline provided' + $columns = 9 + $rows = 8 + [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 + [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 6 + [regex]::Matches($md, '\|\:heavy_check_mark\:\|').Count | should -Be 6 + [regex]::Matches($md, '\|\:warning\:\|').Count | should -Be 0 + [regex]::Matches($md, '\|\:x\:\|').Count | should -Be 0 + [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) + } + It 'Test GetBcptSummaryMD (with better baseline)' { + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 + Write-Host $md.Replace('\n',"`n") + $md | should -Not -Match 'No baseline provided' + $columns = 9 + $rows = 8 + [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 + [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 6 + [regex]::Matches($md, '\|\:heavy_check_mark\:\|').Count | should -Be 2 + [regex]::Matches($md, '\|\:warning\:\|').Count | should -Be 2 + [regex]::Matches($md, '\|\:x\:\|').Count | should -Be 2 + [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) } + AfterAll { + Remove-Item -Path $bcptFilename -Force -ErrorAction SilentlyContinue + Remove-Item -Path $bcptBaseLine1 -Force -ErrorAction SilentlyContinue + Remove-Item -Path $bcptBaseLine2 -Force -ErrorAction SilentlyContinue + } } From aa79886d007bb003ef4fd05436abd24108d5d494 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 09:20:35 +0200 Subject: [PATCH 16/90] tests + issue errors and warnings --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 57 +++++++++++++++------ Tests/AnalyzeTests.Test.ps1 | 47 +++++++++++------ 2 files changed, 73 insertions(+), 31 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 105c96d92..75246522b 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -1,3 +1,8 @@ +$statusOK = ":white_check_mark:" +$statusWarning = ":warning:" +$statusError = ":x:" +$statusSkipped = ":white_circle:" + function ReadBcptFile { Param( [string] $path @@ -48,8 +53,8 @@ function GetBcptSummaryMD { [string] $path, [string] $baseLinePath = '', [int] $skipMeasurements = 1, - [int] $warningThreasHold = 5, - [int] $errorThreasHold = 10 + [int] $warningThreshold = 5, + [int] $errorThreshold = 10 ) # TODO: grab skipMeasurements and thresholds from settings @@ -82,6 +87,7 @@ function GetBcptSummaryMD { $durationMin = ($measurements | ForEach-Object { $_.durationMin } | Measure-Object -Average).Average $numberOfSQLStmts = ($measurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Average).Average + $baseLineFound = $true try { $baseLineMeasurements = @($baseLine."$suiteName"."$codeUnitID"."operations"."$operationName"."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) if ($baseLineMeasurements.Count -eq 0) { @@ -91,6 +97,7 @@ function GetBcptSummaryMD { $baseLineNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Average).Average } catch { + $baseLineFound = $false $baseLineDurationMin = $durationMin $baseLineNumberOfSQLStmts = $numberOfSQLStmts } @@ -115,21 +122,41 @@ function GetBcptSummaryMD { $baseLineNumberOfSQLStmtsStr = "**$($baseLineNumberOfSQLStmts.ToString("N0"))**|" } - if (-not $baseLine) { + if (!$baseLine) { + # No baseline provided $statusStr = '' $baselinedurationMinStr = '' $baseLineNumberOfSQLStmtsStr = '' } - elseif ($pctDurationMin -gt $errorThreasHold -or $pctNumberOfSQLStmts -gt $errorThreasHold) { - $statusStr = ":x:|" - # TODO: issue error - } - elseif ($pctDurationMin -gt $warningThreasHold -or $pctNumberOfSQLStmts -gt $warningThreasHold) { - $statusStr = ":warning:|" - # TODO: issue warning - } else { - $statusStr = ":heavy_check_mark:|" + if (!$baseLineFound) { + # Baseline provided, but not found for this operation + $statusStr = '' + $baselinedurationMinStr = 'N/A|' + $baseLineNumberOfSQLStmtsStr = 'N/A|' + } + else { + $statusStr = $statusOK + if ($pctDurationMin -ge $errorThreshold) { + $statusStr = $statusError + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($errorThreshold)% for duration" + } + if ($pctNumberOfSQLStmts -ge $errorThreshold) { + $statusStr = $statusError + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($errorThreshold)% for number of SQL statements" + } + if ($statusStr -eq $statusOK) { + if ($pctDurationMin -ge $warningThreshold) { + $statusStr = $statusWarning + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($warningThreshold)% for duration" + } + if ($pctNumberOfSQLStmts -ge $warningThreshold) { + $statusStr = $statusWarning + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($warningThreshold)% for number of SQL statements" + } + } + } + $statusStr += '|' } $thisOperationName = ''; if ($operationName -ne $lastOperationName) { $thisOperationName = $operationName } @@ -201,15 +228,15 @@ function GetTestResultSummaryMD { Write-Host "- $appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds" $summarySb.Append("|$appName|$appTests|") | Out-Null if ($appPassed -gt 0) { - $summarySb.Append("$($appPassed):white_check_mark:") | Out-Null + $summarySb.Append("$($appPassed)$statusOK") | Out-Null } $summarySb.Append("|") | Out-Null if ($appFailed -gt 0) { - $summarySb.Append("$($appFailed):x:") | Out-Null + $summarySb.Append("$($appFailed)$statusError") | Out-Null } $summarySb.Append("|") | Out-Null if ($appSkipped -gt 0) { - $summarySb.Append("$($appSkipped):white_circle:") | Out-Null + $summarySb.Append("$($appSkipped)$statusSkipped") | Out-Null } $summarySb.Append("|$($appTime)s|\n") | Out-Null if ($appFailed -gt 0) { diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 04c317c7f..61485c94f 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -47,9 +47,11 @@ Describe "AnalyzeTests Action Tests" { $scriptName = "$actionName.ps1" $actionScript = GetActionScript -scriptRoot $scriptRoot -scriptName $scriptName - $bcptFilename = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 3 -noOfMeasurements 4 + $bcptFilename = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 5 -noOfMeasurements 4 + # BaseLine1 has overall highter duration and more SQL statements than bcptFilename (+ one more opearion) $bcptBaseLine1 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 4 -noOfOperations 6 -noOfMeasurements 4 -durationOffset 1 -numberOfSQLStmtsOffset 1 - $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 2 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 1 + # BaseLine2 has overall lower duration and less SQL statements than bcptFilename (+ one less opearion) + $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 4 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 1 } It 'Compile Action' { @@ -65,57 +67,70 @@ Describe "AnalyzeTests Action Tests" { } It 'Test ReadBcptFile' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') $bcpt = ReadBcptFile -path $bcptFilename $bcpt.Count | should -Be 1 $bcpt."SUITE1".Count | should -Be 2 - $bcpt."SUITE1"."1".operations.Count | should -Be 3 + $bcpt."SUITE1"."1".operations.Count | should -Be 5 $bcpt."SUITE1"."1".operations."operation2".measurements.Count | should -Be 4 } It 'Test GetBcptSummaryMD (no baseline)' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') $md = GetBcptSummaryMD -path $bcptFilename Write-Host $md.Replace('\n',"`n") $md | should -Match 'No baseline provided' $columns = 6 - $rows = 8 + $rows = 12 [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 - [regex]::Matches($md, '\|Operation.\|').Count | should -Be 6 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) } It 'Test GetBcptSummaryMD (with worse baseline)' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine1 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 9 - $rows = 8 + $rows = 12 [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 - [regex]::Matches($md, '\|Operation.\|').Count | should -Be 6 - [regex]::Matches($md, '\|\:heavy_check_mark\:\|').Count | should -Be 6 - [regex]::Matches($md, '\|\:warning\:\|').Count | should -Be 0 - [regex]::Matches($md, '\|\:x\:\|').Count | should -Be 0 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 + [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 10 + [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 0 + [regex]::Matches($md, "\|$statusError\|").Count | should -Be 0 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) } It 'Test GetBcptSummaryMD (with better baseline)' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') - $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 + + $script:errorCount = 0 + Mock OutputError { Param([string] $message) Write-Host "ERROR: $message"; $script:errorCount++ } + $script:warningCount = 0 + Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } + + $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningThreshold 6 -errorThreshold 12 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 9 - $rows = 8 + $rows = 12 [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 - [regex]::Matches($md, '\|Operation.\|').Count | should -Be 6 - [regex]::Matches($md, '\|\:heavy_check_mark\:\|').Count | should -Be 2 - [regex]::Matches($md, '\|\:warning\:\|').Count | should -Be 2 - [regex]::Matches($md, '\|\:x\:\|').Count | should -Be 2 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 + [regex]::Matches($md, '\|N\/A\|').Count | should -Be 4 + [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 2 + [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 4 + [regex]::Matches($md, "\|$statusError\|").Count | should -Be 2 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) + $script:errorCount | Should -be 2 + $script:warningCount | Should -be 4 } AfterAll { From 32a6cd911e660d7b9b50794f3ac52694e2b7ced2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 09:25:47 +0200 Subject: [PATCH 17/90] move function --- Tests/AnalyzeTests.Test.ps1 | 75 +++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 61485c94f..091ceade1 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -1,47 +1,48 @@ Get-Module TestActionsHelper | Remove-Module -Force Import-Module (Join-Path $PSScriptRoot 'TestActionsHelper.psm1') -function GetBcptTestResultFile { - Param( - [int] $noOfSuites = 1, - [int] $noOfCodeunits = 1, - [int] $noOfOperations = 1, - [int] $noOfMeasurements = 1, - [int] $durationOffset = 0, - [int] $numberOfSQLStmtsOffset = 0 - ) +Describe "AnalyzeTests Action Tests" { + BeforeAll { - $bcpt = @() - 1..$noOfSuites | ForEach-Object { - $suiteName = "SUITE$_" - 1..$noOfCodeunits | ForEach-Object { - $codeunitID = $_ - $codeunitName = "Codeunit$_" - 1..$noOfOperations | ForEach-Object { - $operationNo = $_ - $operationName = "Operation$operationNo" - 1..$noOfMeasurements | ForEach-Object { - $no = $_ - $bcpt += @(@{ - "id" = [GUID]::NewGuid().ToString() - "bcptCode" = $suiteName - "codeunitID" = $codeunitID - "codeunitName" = $codeunitName - "operation" = $operationName - "durationMin" = $codeunitNo*100+$operationNo*10+$no+$durationOffset - "numberOfSQLStmts" = $operationNo+$numberOfSQLStmtsOffset - }) + function GetBcptTestResultFile { + Param( + [int] $noOfSuites = 1, + [int] $noOfCodeunits = 1, + [int] $noOfOperations = 1, + [int] $noOfMeasurements = 1, + [int] $durationOffset = 0, + [int] $numberOfSQLStmtsOffset = 0 + ) + + $bcpt = @() + 1..$noOfSuites | ForEach-Object { + $suiteName = "SUITE$_" + 1..$noOfCodeunits | ForEach-Object { + $codeunitID = $_ + $codeunitName = "Codeunit$_" + 1..$noOfOperations | ForEach-Object { + $operationNo = $_ + $operationName = "Operation$operationNo" + 1..$noOfMeasurements | ForEach-Object { + $no = $_ + $bcpt += @(@{ + "id" = [GUID]::NewGuid().ToString() + "bcptCode" = $suiteName + "codeunitID" = $codeunitID + "codeunitName" = $codeunitName + "operation" = $operationName + "durationMin" = $codeunitNo*100+$operationNo*10+$no+$durationOffset + "numberOfSQLStmts" = $operationNo+$numberOfSQLStmtsOffset + }) + } + } } } + $filename = Join-Path $ENV:TEMP "$([GUID]::NewGuid().ToString()).json" + $bcpt | ConvertTo-Json -Depth 100 | Set-Content -Path $filename -Encoding UTF8 + return $filename } - } - $filename = Join-Path $ENV:TEMP "$([GUID]::NewGuid().ToString()).json" - $bcpt | ConvertTo-Json -Depth 100 | Set-Content -Path $filename -Encoding UTF8 - return $filename -} - -Describe "AnalyzeTests Action Tests" { - BeforeAll { + $actionName = "AnalyzeTests" $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve $scriptName = "$actionName.ps1" From cfff3245cbb21d93ea50fdfb3494357d1b272c6c Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 09:30:55 +0200 Subject: [PATCH 18/90] remove codeunitno --- Tests/AnalyzeTests.Test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 091ceade1..3cb08451f 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -31,7 +31,7 @@ Describe "AnalyzeTests Action Tests" { "codeunitID" = $codeunitID "codeunitName" = $codeunitName "operation" = $operationName - "durationMin" = $codeunitNo*100+$operationNo*10+$no+$durationOffset + "durationMin" = $operationNo*10+$no+$durationOffset "numberOfSQLStmts" = $operationNo+$numberOfSQLStmtsOffset }) } From 0f5d4d2627e4d870fbd9997564db04ab2033d945 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 09:52:09 +0200 Subject: [PATCH 19/90] temp --- Tests/AnalyzeTests.Test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 3cb08451f..1eecb3e10 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -38,7 +38,7 @@ Describe "AnalyzeTests Action Tests" { } } } - $filename = Join-Path $ENV:TEMP "$([GUID]::NewGuid().ToString()).json" + $filename = Join-Path ([System.IO.Path]::GetTempPath()) "$([GUID]::NewGuid().ToString()).json" $bcpt | ConvertTo-Json -Depth 100 | Set-Content -Path $filename -Encoding UTF8 return $filename } From 7a63ec3b6c6e311277347b70919b909021b0983f Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 11:22:30 +0200 Subject: [PATCH 20/90] use heavy checkmark --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 75246522b..b99d6902c 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -1,4 +1,4 @@ -$statusOK = ":white_check_mark:" +$statusOK = ":heavy_check_mark:" $statusWarning = ":warning:" $statusError = ":x:" $statusSkipped = ":white_circle:" From 2301c253b9cb7f9b4c2270ac7ddc29a908f03487 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 11:29:52 +0200 Subject: [PATCH 21/90] use skipped when N/A --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index b99d6902c..81c9bbdb7 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -131,7 +131,7 @@ function GetBcptSummaryMD { else { if (!$baseLineFound) { # Baseline provided, but not found for this operation - $statusStr = '' + $statusStr = $statusSkipped $baselinedurationMinStr = 'N/A|' $baseLineNumberOfSQLStmtsStr = 'N/A|' } From 3914ec4c062657e81cf235d9e2ff97bbfb27fb21 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 11:39:22 +0200 Subject: [PATCH 22/90] use question for skip --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 81c9bbdb7..9632f6867 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -1,7 +1,7 @@ $statusOK = ":heavy_check_mark:" $statusWarning = ":warning:" $statusError = ":x:" -$statusSkipped = ":white_circle:" +$statusSkipped = ":question:" function ReadBcptFile { Param( From a4d7ba6b54ebfad734c4aef9281572dda1702382 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 26 Jul 2023 11:41:52 +0200 Subject: [PATCH 23/90] only for scenarios --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 9632f6867..0da9e4cfb 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -139,20 +139,31 @@ function GetBcptSummaryMD { $statusStr = $statusOK if ($pctDurationMin -ge $errorThreshold) { $statusStr = $statusError - OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($errorThreshold)% for duration" + if ($operationName -eq "Scenario") { + + # TODO: Determine when to give errors and warnings + + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($errorThreshold)% for duration" + } } if ($pctNumberOfSQLStmts -ge $errorThreshold) { $statusStr = $statusError - OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($errorThreshold)% for number of SQL statements" + if ($operationName -eq "Scenario") { + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($errorThreshold)% for number of SQL statements" + } } if ($statusStr -eq $statusOK) { if ($pctDurationMin -ge $warningThreshold) { $statusStr = $statusWarning - OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($warningThreshold)% for duration" + if ($operationName -eq "Scenario") { + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($warningThreshold)% for duration" + } } if ($pctNumberOfSQLStmts -ge $warningThreshold) { $statusStr = $statusWarning - OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($warningThreshold)% for number of SQL statements" + if ($operationName -eq "Scenario") { + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($warningThreshold)% for number of SQL statements" + } } } } From ef467bf582802d772eedc45b2582b549fe9d588f Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 27 Jul 2023 07:08:57 +0200 Subject: [PATCH 24/90] expect no err+warn --- Tests/AnalyzeTests.Test.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 1eecb3e10..cf1f7e8e9 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -130,8 +130,8 @@ Describe "AnalyzeTests Action Tests" { [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 4 [regex]::Matches($md, "\|$statusError\|").Count | should -Be 2 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) - $script:errorCount | Should -be 2 - $script:warningCount | Should -be 4 + $script:errorCount | Should -be 0 + $script:warningCount | Should -be 0 } AfterAll { From a452d24699ecbe43189343837ae575ec4853b35e Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 27 Jul 2023 16:25:02 +0200 Subject: [PATCH 25/90] set failuresSummaryMD --- Actions/AL-Go-Helper.ps1 | 2 +- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index ab017d485..53ebc75bf 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -17,7 +17,7 @@ $RepoSettingsFile = Join-Path '.github' 'AL-Go-Settings.json' $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent -$defaultBcContainerHelperVersion = "" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step +$defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step $microsoftTelemetryConnectionString = "InstrumentationKey=84bd9223-67d4-4378-8590-9e4a46023be2;IngestionEndpoint=https://westeurope-1.in.applicationinsights.azure.com/" $runAlPipelineOverrides = @( diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 0da9e4cfb..e2f25e1fe 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -62,6 +62,8 @@ function GetBcptSummaryMD { $bcpt = ReadBcptFile -path $path $baseLine = ReadBcptFile -path $baseLinePath + $failuresSummaryMD = "" + $summarySb = [System.Text.StringBuilder]::new() $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|$(if ($baseLine){'Status|'})Duration|$(if ($baseLine){'Duration (BaseLine)|'})SQL Stmts|$(if ($baseLine){'SQL Stmts (BaseLine)|'})\n|:---|:---|:---|:---|$(if ($baseLine){'---:|'}):--:|$(if ($baseLine){'---:|'})---:|$(if ($baseLine){'---:|'})\n") | Out-Null @@ -278,7 +280,6 @@ function GetTestResultSummaryMD { } } } - $failuresSummaryMD = "" if ($totalFailed -gt 0) { $failuresSummaryMD = "$totalFailed failing tests, download test results to see details" $failuresSb.Insert(0,"
$failuresSummaryMD") | Out-Null From bb658fe8289e2d5d9d3c625648f0c68f019863a7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 27 Jul 2023 17:32:31 +0200 Subject: [PATCH 26/90] different threshold --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 22 +++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index e2f25e1fe..d90f744a3 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -53,8 +53,10 @@ function GetBcptSummaryMD { [string] $path, [string] $baseLinePath = '', [int] $skipMeasurements = 1, - [int] $warningThreshold = 5, - [int] $errorThreshold = 10 + [int] $warningDurationThreshold = 10, + [int] $errorDurationThreshold = 25, + [int] $warningNumberOfSqlStmtsThreshold = 5, + [int] $errorNumberOfSqlStmtsThreshold = 10 ) # TODO: grab skipMeasurements and thresholds from settings @@ -139,32 +141,32 @@ function GetBcptSummaryMD { } else { $statusStr = $statusOK - if ($pctDurationMin -ge $errorThreshold) { + if ($pctDurationMin -ge $errorDurationThreshold) { $statusStr = $statusError if ($operationName -eq "Scenario") { # TODO: Determine when to give errors and warnings - OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($errorThreshold)% for duration" + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($errorDurationThreshold)% for duration" } } - if ($pctNumberOfSQLStmts -ge $errorThreshold) { + if ($pctNumberOfSQLStmts -ge $errorNumberOfSQLStmtsThreshold) { $statusStr = $statusError if ($operationName -eq "Scenario") { - OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($errorThreshold)% for number of SQL statements" + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($errorNumberOfSQLStmtsThreshold)% for number of SQL statements" } } if ($statusStr -eq $statusOK) { - if ($pctDurationMin -ge $warningThreshold) { + if ($pctDurationMin -ge $warningDurationThreshold) { $statusStr = $statusWarning if ($operationName -eq "Scenario") { - OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($warningThreshold)% for duration" + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($warningDurationThreshold)% for duration" } } - if ($pctNumberOfSQLStmts -ge $warningThreshold) { + if ($pctNumberOfSQLStmts -ge $warningNumberOfSQLStmtsThreshold) { $statusStr = $statusWarning if ($operationName -eq "Scenario") { - OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($warningThreshold)% for number of SQL statements" + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($warningNumberOfSQLStmtsThreshold)% for number of SQL statements" } } } From 6465183e99b7673b8446fb2ccf8fed63167371e1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 27 Jul 2023 18:35:08 +0200 Subject: [PATCH 27/90] fix tests --- Tests/AnalyzeTests.Test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index cf1f7e8e9..4522e88ad 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -117,7 +117,7 @@ Describe "AnalyzeTests Action Tests" { $script:warningCount = 0 Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } - $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningThreshold 6 -errorThreshold 12 + $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningDurationThreshold 6 -errorDurationThreshold 12 -warningNumberOfSqlStmtsThreshold 6 -errorNumberOfSqlStmtsThreshold 12 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 9 From e44a2bc0c9ec9be4f6a40c1ffa5fb91444ddeed5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 31 Jul 2023 07:40:45 +0200 Subject: [PATCH 28/90] Add UTF8 encoding on SUMMARY --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 10 +++++----- .../workflows/CreateOnlineDevelopmentEnvironment.yaml | 4 ++-- .../.github/workflows/PublishToEnvironment.yaml | 4 ++-- .../workflows/CreateOnlineDevelopmentEnvironment.yaml | 4 ++-- .../.github/workflows/PublishToEnvironment.yaml | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index ecdba15dc..3bdc9be8d 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -47,11 +47,11 @@ try { $testResultsfailuresMD = $testResultsFailuresSummaryMD } - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "## Test results`n`n" - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsSummaryMD.Replace("\n","`n"))`n`n" - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsfailuresMD.Replace("\n","`n"))`n`n" - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "## Performance test results`n`n" - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($bcptSummaryMD.Replace("\n","`n"))`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "## Test results`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsSummaryMD.Replace("\n","`n"))`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsfailuresMD.Replace("\n","`n"))`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "## Performance test results`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($bcptSummaryMD.Replace("\n","`n"))`n`n" TrackTrace -telemetryScope $telemetryScope } diff --git a/Templates/AppSource App/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml b/Templates/AppSource App/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml index 315f5fe2a..4eb7a48f6 100644 --- a/Templates/AppSource App/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml +++ b/Templates/AppSource App/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml @@ -79,7 +79,7 @@ jobs: $adminCenterApiCredentials = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($env:adminCenterApiCredentials)) if ($adminCenterApiCredentials) { Write-Host "AdminCenterApiCredentials provided in secret $($ENV:adminCenterApiCredentialsSecretName)!" - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "Admin Center Api Credentials was provided in a secret called $($ENV:adminCenterApiCredentialsSecretName). Using this information for authentication." + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "Admin Center Api Credentials was provided in a secret called $($ENV:adminCenterApiCredentialsSecretName). Using this information for authentication." } else { Write-Host "AdminCenterApiCredentials not provided, initiating Device Code flow" @@ -90,7 +90,7 @@ jobs: $BcContainerHelperPath = DownloadAndImportBcContainerHelper -baseFolder $ENV:GITHUB_WORKSPACE $authContext = New-BcAuthContext -includeDeviceLogin -deviceLoginTimeout ([TimeSpan]::FromSeconds(0)) CleanupAfterBcContainerHelper -bcContainerHelperPath $bcContainerHelperPath - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Admin Center Api and could not locate a secret called $($ENV:adminCenterApiCredentialsSecretName) (https://aka.ms/ALGoSettings#AdminCenterApiCredentialsSecretName)`n`n$($authContext.message)" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Admin Center Api and could not locate a secret called $($ENV:adminCenterApiCredentialsSecretName) (https://aka.ms/ALGoSettings#AdminCenterApiCredentialsSecretName)`n`n$($authContext.message)" Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "deviceCode=$($authContext.deviceCode)" } diff --git a/Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml b/Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml index 4cbb87d61..0884a94e1 100644 --- a/Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml +++ b/Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml @@ -90,7 +90,7 @@ jobs: } if ($authContext) { Write-Host "AuthContext provided in secret $secretName!" - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AuthContext was provided in a secret called $secretName. Using this information for authentication." + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AuthContext was provided in a secret called $secretName. Using this information for authentication." } else { Write-Host "No AuthContext provided for $envName, initiating Device Code flow" @@ -101,7 +101,7 @@ jobs: $BcContainerHelperPath = DownloadAndImportBcContainerHelper -baseFolder $ENV:GITHUB_WORKSPACE $authContext = New-BcAuthContext -includeDeviceLogin -deviceLoginTimeout ([TimeSpan]::FromSeconds(0)) CleanupAfterBcContainerHelper -bcContainerHelperPath $bcContainerHelperPath - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Environment $('${{ steps.envName.outputs.envName }}'.Split(' ')[0]) and could not locate a secret called ${{ steps.envName.outputs.envName }}_AuthContext`n`n$($authContext.message)" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Environment $('${{ steps.envName.outputs.envName }}'.Split(' ')[0]) and could not locate a secret called ${{ steps.envName.outputs.envName }}_AuthContext`n`n$($authContext.message)" Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "deviceCode=$($authContext.deviceCode)" } diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml index 315f5fe2a..4eb7a48f6 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml @@ -79,7 +79,7 @@ jobs: $adminCenterApiCredentials = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($env:adminCenterApiCredentials)) if ($adminCenterApiCredentials) { Write-Host "AdminCenterApiCredentials provided in secret $($ENV:adminCenterApiCredentialsSecretName)!" - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "Admin Center Api Credentials was provided in a secret called $($ENV:adminCenterApiCredentialsSecretName). Using this information for authentication." + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "Admin Center Api Credentials was provided in a secret called $($ENV:adminCenterApiCredentialsSecretName). Using this information for authentication." } else { Write-Host "AdminCenterApiCredentials not provided, initiating Device Code flow" @@ -90,7 +90,7 @@ jobs: $BcContainerHelperPath = DownloadAndImportBcContainerHelper -baseFolder $ENV:GITHUB_WORKSPACE $authContext = New-BcAuthContext -includeDeviceLogin -deviceLoginTimeout ([TimeSpan]::FromSeconds(0)) CleanupAfterBcContainerHelper -bcContainerHelperPath $bcContainerHelperPath - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Admin Center Api and could not locate a secret called $($ENV:adminCenterApiCredentialsSecretName) (https://aka.ms/ALGoSettings#AdminCenterApiCredentialsSecretName)`n`n$($authContext.message)" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Admin Center Api and could not locate a secret called $($ENV:adminCenterApiCredentialsSecretName) (https://aka.ms/ALGoSettings#AdminCenterApiCredentialsSecretName)`n`n$($authContext.message)" Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "deviceCode=$($authContext.deviceCode)" } diff --git a/Templates/Per Tenant Extension/.github/workflows/PublishToEnvironment.yaml b/Templates/Per Tenant Extension/.github/workflows/PublishToEnvironment.yaml index 4cbb87d61..0884a94e1 100644 --- a/Templates/Per Tenant Extension/.github/workflows/PublishToEnvironment.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/PublishToEnvironment.yaml @@ -90,7 +90,7 @@ jobs: } if ($authContext) { Write-Host "AuthContext provided in secret $secretName!" - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AuthContext was provided in a secret called $secretName. Using this information for authentication." + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AuthContext was provided in a secret called $secretName. Using this information for authentication." } else { Write-Host "No AuthContext provided for $envName, initiating Device Code flow" @@ -101,7 +101,7 @@ jobs: $BcContainerHelperPath = DownloadAndImportBcContainerHelper -baseFolder $ENV:GITHUB_WORKSPACE $authContext = New-BcAuthContext -includeDeviceLogin -deviceLoginTimeout ([TimeSpan]::FromSeconds(0)) CleanupAfterBcContainerHelper -bcContainerHelperPath $bcContainerHelperPath - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Environment $('${{ steps.envName.outputs.envName }}'.Split(' ')[0]) and could not locate a secret called ${{ steps.envName.outputs.envName }}_AuthContext`n`n$($authContext.message)" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Environment $('${{ steps.envName.outputs.envName }}'.Split(' ')[0]) and could not locate a secret called ${{ steps.envName.outputs.envName }}_AuthContext`n`n$($authContext.message)" Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "deviceCode=$($authContext.deviceCode)" } From 362ea9bf516b7027de8ed0d674d43b420baf2616 Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 31 Jul 2023 07:47:01 +0200 Subject: [PATCH 29/90] release notes --- RELEASENOTES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 485c0b7be..a7a67f9f2 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -29,6 +29,9 @@ Now, you can set the checkbox called Use GhTokenWorkflow to allowing you to use ### New Actions - `DownloadProjectDependencies`: Downloads the dependency apps for a given project and build mode. +### Business Central Performance Toolkit Test Result Viewer +- In the summary after a Test Run, you now also have the result of performance tests. + ## v3.1 ### Issues From 31942b2ed018736aba1bf95214948071239faf3c Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 9 Aug 2023 19:48:29 +0200 Subject: [PATCH 30/90] move --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 207 ++++++++++---------- 1 file changed, 104 insertions(+), 103 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index d90f744a3..df457981f 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -3,6 +3,110 @@ $statusWarning = ":warning:" $statusError = ":x:" $statusSkipped = ":question:" +# Build MarkDown of TestResults file +# This function will not fail if the file does not exist or if any test errors are found +# TestResults is in JUnit format +# Returns both a summary part and a failures part +function GetTestResultSummaryMD { + Param( + [string] $path + ) + + $summarySb = [System.Text.StringBuilder]::new() + $failuresSb = [System.Text.StringBuilder]::new() + if (Test-Path -Path $path -PathType Leaf) { + $testResults = [xml](Get-Content -path "$project\TestResults.xml" -Encoding UTF8) + $totalTests = 0 + $totalTime = 0.0 + $totalFailed = 0 + $totalSkipped = 0 + if ($testResults.testsuites) { + $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "appName" } | ForEach-Object { $_.Value } } | Select-Object -Unique) + if (-not $appNames) { + $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "extensionId" } | ForEach-Object { $_.Value } } | Select-Object -Unique) + } + $testResults.testsuites.testsuite | ForEach-Object { + $totalTests += $_.Tests + $totalTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) + $totalFailed += $_.failures + $totalSkipped += $_.skipped + } + Write-Host "$($appNames.Count) TestApps, $totalTests tests, $totalFailed failed, $totalSkipped skipped, $totalTime seconds" + $summarySb.Append('|Test app|Tests|Passed|Failed|Skipped|Time|\n|:---|---:|---:|---:|---:|---:|\n') | Out-Null + $appNames | ForEach-Object { + $appName = $_ + $appTests = 0 + $appTime = 0.0 + $appFailed = 0 + $appSkipped = 0 + $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Value -eq $appName } } + $suites | ForEach-Object { + $appTests += [int]$_.tests + $appFailed += [int]$_.failures + $appSkipped += [int]$_.skipped + $appTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) + } + $appPassed = $appTests-$appFailed-$appSkipped + Write-Host "- $appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds" + $summarySb.Append("|$appName|$appTests|") | Out-Null + if ($appPassed -gt 0) { + $summarySb.Append("$($appPassed)$statusOK") | Out-Null + } + $summarySb.Append("|") | Out-Null + if ($appFailed -gt 0) { + $summarySb.Append("$($appFailed)$statusError") | Out-Null + } + $summarySb.Append("|") | Out-Null + if ($appSkipped -gt 0) { + $summarySb.Append("$($appSkipped)$statusSkipped") | Out-Null + } + $summarySb.Append("|$($appTime)s|\n") | Out-Null + if ($appFailed -gt 0) { + $failuresSb.Append("
$appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds\n") | Out-Null + $suites | ForEach-Object { + Write-Host " - $($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds" + if ($_.failures -gt 0 -and $failuresSb.Length -lt 32000) { + $failuresSb.Append("
$($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds") | Out-Null + $_.testcase | ForEach-Object { + if ($_.ChildNodes.Count -gt 0) { + Write-Host " - $($_.name), Failure, $($_.time) seconds" + $failuresSb.Append("
$($_.name), Failure") | Out-Null + $_.ChildNodes | ForEach-Object { + Write-Host " - Error: $($_.message)" + Write-Host " Stacktrace:" + Write-Host " $($_."#text".Trim().Replace("`n","`n "))" + $failuresSb.Append("      Error: $($_.message)
") | Out-Null + $failuresSb.Append("      Stack trace
") | Out-Null + $failuresSb.Append("      $($_."#text".Trim().Replace("`n","
      "))

") | Out-Null + } + $failuresSb.Append("
") | Out-Null + } + } + $failuresSb.Append("
") | Out-Null + } + } + $failuresSb.Append("
") | Out-Null + } + } + } + if ($totalFailed -gt 0) { + $failuresSummaryMD = "$totalFailed failing tests, download test results to see details" + $failuresSb.Insert(0,"
$failuresSummaryMD") | Out-Null + $failuresSb.Append("
") | Out-Null + } + else { + $failuresSummaryMD = "No test failures" + $failuresSb.Append($failuresSummaryMD) | Out-Null + } + } + else { + $summarySb.Append("No test results found") | Out-Null + } + $summarySb.ToString() + $failuresSb.ToString() + $failuresSummaryMD +} + function ReadBcptFile { Param( [string] $path @@ -196,106 +300,3 @@ function GetBcptSummaryMD { $summarySb.ToString() } -# Build MarkDown of TestResults file -# This function will not fail if the file does not exist or if any test errors are found -# TestResults is in JUnit format -# Returns both a summary part and a failures part -function GetTestResultSummaryMD { - Param( - [string] $path - ) - - $summarySb = [System.Text.StringBuilder]::new() - $failuresSb = [System.Text.StringBuilder]::new() - if (Test-Path -Path $path -PathType Leaf) { - $testResults = [xml](Get-Content -path "$project\TestResults.xml" -Encoding UTF8) - $totalTests = 0 - $totalTime = 0.0 - $totalFailed = 0 - $totalSkipped = 0 - if ($testResults.testsuites) { - $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "appName" } | ForEach-Object { $_.Value } } | Select-Object -Unique) - if (-not $appNames) { - $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "extensionId" } | ForEach-Object { $_.Value } } | Select-Object -Unique) - } - $testResults.testsuites.testsuite | ForEach-Object { - $totalTests += $_.Tests - $totalTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) - $totalFailed += $_.failures - $totalSkipped += $_.skipped - } - Write-Host "$($appNames.Count) TestApps, $totalTests tests, $totalFailed failed, $totalSkipped skipped, $totalTime seconds" - $summarySb.Append('|Test app|Tests|Passed|Failed|Skipped|Time|\n|:---|---:|---:|---:|---:|---:|\n') | Out-Null - $appNames | ForEach-Object { - $appName = $_ - $appTests = 0 - $appTime = 0.0 - $appFailed = 0 - $appSkipped = 0 - $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Value -eq $appName } } - $suites | ForEach-Object { - $appTests += [int]$_.tests - $appFailed += [int]$_.failures - $appSkipped += [int]$_.skipped - $appTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) - } - $appPassed = $appTests-$appFailed-$appSkipped - Write-Host "- $appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds" - $summarySb.Append("|$appName|$appTests|") | Out-Null - if ($appPassed -gt 0) { - $summarySb.Append("$($appPassed)$statusOK") | Out-Null - } - $summarySb.Append("|") | Out-Null - if ($appFailed -gt 0) { - $summarySb.Append("$($appFailed)$statusError") | Out-Null - } - $summarySb.Append("|") | Out-Null - if ($appSkipped -gt 0) { - $summarySb.Append("$($appSkipped)$statusSkipped") | Out-Null - } - $summarySb.Append("|$($appTime)s|\n") | Out-Null - if ($appFailed -gt 0) { - $failuresSb.Append("
$appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds\n") | Out-Null - $suites | ForEach-Object { - Write-Host " - $($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds" - if ($_.failures -gt 0 -and $failuresSb.Length -lt 32000) { - $failuresSb.Append("
$($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds") | Out-Null - $_.testcase | ForEach-Object { - if ($_.ChildNodes.Count -gt 0) { - Write-Host " - $($_.name), Failure, $($_.time) seconds" - $failuresSb.Append("
$($_.name), Failure") | Out-Null - $_.ChildNodes | ForEach-Object { - Write-Host " - Error: $($_.message)" - Write-Host " Stacktrace:" - Write-Host " $($_."#text".Trim().Replace("`n","`n "))" - $failuresSb.Append("      Error: $($_.message)
") | Out-Null - $failuresSb.Append("      Stack trace
") | Out-Null - $failuresSb.Append("      $($_."#text".Trim().Replace("`n","
      "))

") | Out-Null - } - $failuresSb.Append("
") | Out-Null - } - } - $failuresSb.Append("
") | Out-Null - } - } - $failuresSb.Append("
") | Out-Null - } - } - } - if ($totalFailed -gt 0) { - $failuresSummaryMD = "$totalFailed failing tests, download test results to see details" - $failuresSb.Insert(0,"
$failuresSummaryMD") | Out-Null - $failuresSb.Append("
") | Out-Null - } - else { - $failuresSummaryMD = "No test failures" - $failuresSb.Append($failuresSummaryMD) | Out-Null - } - } - else { - $summarySb.Append("No test results found") | Out-Null - } - $summarySb.ToString() - $failuresSb.ToString() - $failuresSummaryMD -} From a3a70c6e1d10670f9340fb4cc8cf94ae8ece0027 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 9 Aug 2023 21:20:53 +0200 Subject: [PATCH 31/90] add columns --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 62 ++++++++++----------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index df457981f..5fc17956b 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -168,10 +168,8 @@ function GetBcptSummaryMD { $bcpt = ReadBcptFile -path $path $baseLine = ReadBcptFile -path $baseLinePath - $failuresSummaryMD = "" - $summarySb = [System.Text.StringBuilder]::new() - $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|$(if ($baseLine){'Status|'})Duration|$(if ($baseLine){'Duration (BaseLine)|'})SQL Stmts|$(if ($baseLine){'SQL Stmts (BaseLine)|'})\n|:---|:---|:---|:---|$(if ($baseLine){'---:|'}):--:|$(if ($baseLine){'---:|'})---:|$(if ($baseLine){'---:|'})\n") | Out-Null + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|$(if ($baseLine){'Status|'})Duration|$(if ($baseLine){'Duration (Base)|Duration (Diff)|'})SQL Stmts|$(if ($baseLine){'SQL Stmts (Base)|SQL Stmts (Diff)|'})\n|:---|:---|:---|:---|$(if ($baseLine){'---:|'}):--:|$(if ($baseLine){'---:|'})---:|$(if ($baseLine){'---:|'})\n") | Out-Null $lastSuiteName = '' $lastCodeunitID = '' @@ -192,8 +190,8 @@ function GetBcptSummaryMD { # Get measurements to use for statistics $measurements = @($operation."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) # Calculate statistics and store them in the operation - $durationMin = ($measurements | ForEach-Object { $_.durationMin } | Measure-Object -Average).Average - $numberOfSQLStmts = ($measurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Average).Average + $durationMin = ($measurements | ForEach-Object { $_.durationMin } | Measure-Object -Minimum).Minimum + $numberOfSQLStmts = ($measurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum $baseLineFound = $true try { @@ -201,47 +199,45 @@ function GetBcptSummaryMD { if ($baseLineMeasurements.Count -eq 0) { throw "No base line measurements" } - $baseLineDurationMin = ($baseLineMeasurements | ForEach-Object { $_.durationMin } | Measure-Object -Average).Average - $baseLineNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Average).Average + $baseDurationMin = ($baseLineMeasurements | ForEach-Object { $_.durationMin } | Measure-Object -Minimum).Minimum + $diffDurationMin = $durationMin-$baseDurationMin + $baseNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum + $diffNumberOfSQLStmts = $numberOfSQLStmts-$baseNumberOfSQLStmts } catch { $baseLineFound = $false - $baseLineDurationMin = $durationMin - $baseLineNumberOfSQLStmts = $numberOfSQLStmts + $baseDurationMin = $durationMin + $diffDurationMin = 0 + $baseNumberOfSQLStmts = $numberOfSQLStmts + $diffNumberOfSQLStmts = 0 } - $pctDurationMin = ($durationMin-$baseLineDurationMin)*100/$baseLineDurationMin - if ($pctDurationMin -le 0) { - $durationMinStr = "**$($durationMin.ToString("N2"))**|" - $baseLineDurationMinStr = "$($baseLineDurationMin.ToString("N2"))|" - } - else { - $durationMinStr = "$($durationMin.ToString("N2"))|" - $baseLineDurationMinStr = "**$($baseLineDurationMin.ToString("N2"))**|" - } + $pctDurationMin = ($durationMin-$baseDurationMin)*100/$baseDurationMin + $durationMinStr = "$($durationMin.ToString("N2"))|" + $baseDurationMinStr = "$($baseDurationMin.ToString("N2"))|" + $diffDurationMinStr = "$($diffDurationMin.ToString("N2"))|" - $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseLineNumberOfSQLStmts)*100/$baseLineNumberOfSQLStmts - if ($pctNumberOfSQLStmts -le 0) { - $numberOfSQLStmtsStr = "**$($numberOfSQLStmts.ToString("N0"))**|" - $baseLineNumberOfSQLStmtsStr = "$($baseLineNumberOfSQLStmts.ToString("N0"))|" - } - else { - $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" - $baseLineNumberOfSQLStmtsStr = "**$($baseLineNumberOfSQLStmts.ToString("N0"))**|" - } + $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseNumberOfSQLStmts)*100/$baseNumberOfSQLStmts + $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" + $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("N0"))|" + $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("N0"))|" if (!$baseLine) { # No baseline provided $statusStr = '' - $baselinedurationMinStr = '' - $baseLineNumberOfSQLStmtsStr = '' + $baseDurationMinStr = '' + $diffDurationMinStr = '' + $baseNumberOfSQLStmtsStr = '' + $diffNumberOfSQLStmtsStr = '' } else { if (!$baseLineFound) { # Baseline provided, but not found for this operation $statusStr = $statusSkipped - $baselinedurationMinStr = 'N/A|' - $baseLineNumberOfSQLStmtsStr = 'N/A|' + $baseDurationMinStr = 'N/A|' + $diffDurationMinStr = '|' + $baseNumberOfSQLStmtsStr = 'N/A|' + $diffNumberOfSQLStmtsStr = '|' } else { $statusStr = $statusOK @@ -283,7 +279,7 @@ function GetBcptSummaryMD { $thisCodeunitID = ''; if ($codeunitID -ne $lastCodeunitID) { $thisCodeunitID = $codeunitID; $thisOperationName = $operationName } $thisSuiteName = ''; if ($suiteName -ne $lastSuiteName) { $thisSuiteName = $suiteName; $thisOperationName = $operationName } - $summarySb.Append("|$thisSuiteName|$thisCodeunitID|$thisCodeunitName|$thisOperationName|$statusStr$durationMinStr$baseLineDurationMinStr$numberOfSQLStmtsStr$baseLineNumberOfSQLStmtsStr\n") | Out-Null + $summarySb.Append("|$thisSuiteName|$thisCodeunitID|$thisCodeunitName|$thisOperationName|$statusStr$durationMinStr$baseDurationMinStr$diffDurationMinStr$numberOfSQLStmtsStr$baseNumberOfSQLStmtsStr$diffNumberOfSQLStmtsStr\n") | Out-Null $lastSuiteName = $suiteName $lastCodeunitID = $codeUnitID @@ -296,7 +292,7 @@ function GetBcptSummaryMD { if (-not $baseLine) { $summarySb.Append("\nNo baseline provided. Copy a set of BCPT results to $([System.IO.Path]::GetFileName($baseLinePath)) in the project folder in order to establish a baseline.") | Out-Null } - + $summarySb.ToString() } From ae4492295f1d1f138969cea80dfcc012ae52d4d0 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 9 Aug 2023 22:19:02 +0200 Subject: [PATCH 32/90] format --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 5fc17956b..d4813f2bb 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -169,7 +169,7 @@ function GetBcptSummaryMD { $baseLine = ReadBcptFile -path $baseLinePath $summarySb = [System.Text.StringBuilder]::new() - $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|$(if ($baseLine){'Status|'})Duration|$(if ($baseLine){'Duration (Base)|Duration (Diff)|'})SQL Stmts|$(if ($baseLine){'SQL Stmts (Base)|SQL Stmts (Diff)|'})\n|:---|:---|:---|:---|$(if ($baseLine){'---:|'}):--:|$(if ($baseLine){'---:|'})---:|$(if ($baseLine){'---:|'})\n") | Out-Null + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|$(if ($baseLine){'Status|'})Duration|$(if ($baseLine){'Duration (Base)|Duration (Diff)|'})SQL Stmts|$(if ($baseLine){'SQL Stmts (Base)|SQL Stmts (Diff)|'})\n|:---|:---|:---|:---|$(if ($baseLine){'---:|'}):--:|$(if ($baseLine){'---:|---:|'})---:|$(if ($baseLine){'---:|---:|'})\n") | Out-Null $lastSuiteName = '' $lastCodeunitID = '' @@ -200,9 +200,9 @@ function GetBcptSummaryMD { throw "No base line measurements" } $baseDurationMin = ($baseLineMeasurements | ForEach-Object { $_.durationMin } | Measure-Object -Minimum).Minimum - $diffDurationMin = $durationMin-$baseDurationMin + $diffDurationMin = $baseDurationMin-$durationMin $baseNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum - $diffNumberOfSQLStmts = $numberOfSQLStmts-$baseNumberOfSQLStmts + $diffNumberOfSQLStmts = $baseNumberOfSQLStmts-$numberOfSQLStmts } catch { $baseLineFound = $false @@ -215,12 +215,12 @@ function GetBcptSummaryMD { $pctDurationMin = ($durationMin-$baseDurationMin)*100/$baseDurationMin $durationMinStr = "$($durationMin.ToString("N2"))|" $baseDurationMinStr = "$($baseDurationMin.ToString("N2"))|" - $diffDurationMinStr = "$($diffDurationMin.ToString("N2"))|" + $diffDurationMinStr = "$($diffDurationMin.ToString("#.##;(#.##);"))|" $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseNumberOfSQLStmts)*100/$baseNumberOfSQLStmts $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("N0"))|" - $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("N0"))|" + $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("#;(#);"))|" if (!$baseLine) { # No baseline provided From d676bf8fc10fe41327bf149cddec5cb24d46bb50 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 9 Aug 2023 22:19:27 +0200 Subject: [PATCH 33/90] bold --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index d4813f2bb..437be2c41 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -220,7 +220,7 @@ function GetBcptSummaryMD { $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseNumberOfSQLStmts)*100/$baseNumberOfSQLStmts $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("N0"))|" - $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("#;(#);"))|" + $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("#;**(#)**;"))|" if (!$baseLine) { # No baseline provided From ce816068873db628514a1112ad4b7b08cfb014e0 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 9 Aug 2023 22:53:37 +0200 Subject: [PATCH 34/90] other formatting --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 437be2c41..57c8983c0 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -200,9 +200,9 @@ function GetBcptSummaryMD { throw "No base line measurements" } $baseDurationMin = ($baseLineMeasurements | ForEach-Object { $_.durationMin } | Measure-Object -Minimum).Minimum - $diffDurationMin = $baseDurationMin-$durationMin + $diffDurationMin = $durationMin-$baseDurationMin $baseNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum - $diffNumberOfSQLStmts = $baseNumberOfSQLStmts-$numberOfSQLStmts + $diffNumberOfSQLStmts = $numberOfSQLStmts-$baseNumberOfSQLStmts } catch { $baseLineFound = $false @@ -215,12 +215,12 @@ function GetBcptSummaryMD { $pctDurationMin = ($durationMin-$baseDurationMin)*100/$baseDurationMin $durationMinStr = "$($durationMin.ToString("N2"))|" $baseDurationMinStr = "$($baseDurationMin.ToString("N2"))|" - $diffDurationMinStr = "$($diffDurationMin.ToString("#.##;(#.##);"))|" + $diffDurationMinStr = "$($diffDurationMin.ToString("**#.##**;-#.##;"))|" $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseNumberOfSQLStmts)*100/$baseNumberOfSQLStmts $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("N0"))|" - $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("#;**(#)**;"))|" + $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("**#**;-#;"))|" if (!$baseLine) { # No baseline provided From c992cd117f0c1e7a905ee443776f595edfb75d8a Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 9 Aug 2023 23:24:16 +0200 Subject: [PATCH 35/90] format --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 57c8983c0..516a014e7 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -215,12 +215,12 @@ function GetBcptSummaryMD { $pctDurationMin = ($durationMin-$baseDurationMin)*100/$baseDurationMin $durationMinStr = "$($durationMin.ToString("N2"))|" $baseDurationMinStr = "$($baseDurationMin.ToString("N2"))|" - $diffDurationMinStr = "$($diffDurationMin.ToString("**#.##**;-#.##;"))|" + $diffDurationMinStr = "$($diffDurationMin.ToString("**#**;-#;0"))|" $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseNumberOfSQLStmts)*100/$baseNumberOfSQLStmts $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("N0"))|" - $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("**#**;-#;"))|" + $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("**#**;-#;0"))|" if (!$baseLine) { # No baseline provided From 7af00ede70193c779eb3b1d1244bc9b64d1fb8d5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 9 Aug 2023 23:34:11 +0200 Subject: [PATCH 36/90] settings --- Actions/AL-Go-Helper.ps1 | 4 ++++ Actions/AnalyzeTests/AnalyzeTests.ps1 | 9 ++++++- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 26 ++++++++++----------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 58e44b6d4..28eb9c66e 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -523,6 +523,10 @@ function ReadSettings { "buildModes" = @() "useCompilerFolder" = $false "PullRequestTrigger" = "pull_request_target" + "DurationThresholdWarning" = 10 + "DurationThresholdError" = 25 + "NumberOfSqlStmtsThresholdWarning" = 5 + "NumberOfSqlStmtsThresholdError" = 10 } # Read settings from files and merge them into the settings object diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index 543704519..61b50411e 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -24,9 +24,16 @@ try { $testResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\TestResults.xml" $testResultsSummaryMD, $testResultsfailuresMD, $testResultsFailuresSummaryMD = GetTestResultSummaryMD -path $testResultsFile + $settings = $env:Settings | ConvertFrom-Json $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptTestResults.json" $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptBaseLine.json" - $bcptSummaryMD = GetBcptSummaryMD -path $bcptTestResultsFile -baseLinePath $bcptBaseLineFile + $bcptSummaryMD = GetBcptSummaryMD ` + -path $bcptTestResultsFile ` + -baseLinePath $bcptBaseLineFile ` + -DurationThresholdWarning $settings.DurationThresholdWarning ` + -DurationThresholdError $settings.DurationThresholdError ` + -NumberOfSqlStmtsThresholdWarning $settings.NumberOfSqlStmtsThresholdWarning ` + -NumberOfSqlStmtsThresholdError $settings.NumberOfSqlStmtsThresholdError # If summary fits, we will display it in the GitHub summary if ($testResultsSummaryMD.Length -gt 65000) { diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 516a014e7..aabebe583 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -157,14 +157,12 @@ function GetBcptSummaryMD { [string] $path, [string] $baseLinePath = '', [int] $skipMeasurements = 1, - [int] $warningDurationThreshold = 10, - [int] $errorDurationThreshold = 25, - [int] $warningNumberOfSqlStmtsThreshold = 5, - [int] $errorNumberOfSqlStmtsThreshold = 10 + [int] $DurationThresholdWarning = 10, + [int] $DurationThresholdError = 25, + [int] $NumberOfSqlStmtsThresholdWarning = 5, + [int] $NumberOfSqlStmtsThresholdError = 10 ) - # TODO: grab skipMeasurements and thresholds from settings - $bcpt = ReadBcptFile -path $path $baseLine = ReadBcptFile -path $baseLinePath @@ -241,32 +239,32 @@ function GetBcptSummaryMD { } else { $statusStr = $statusOK - if ($pctDurationMin -ge $errorDurationThreshold) { + if ($pctDurationMin -ge $DurationThresholdError) { $statusStr = $statusError if ($operationName -eq "Scenario") { # TODO: Determine when to give errors and warnings - OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($errorDurationThreshold)% for duration" + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($DurationThresholdError)% for duration" } } - if ($pctNumberOfSQLStmts -ge $errorNumberOfSQLStmtsThreshold) { + if ($pctNumberOfSQLStmts -ge $NumberOfSqlStmtsThresholdError) { $statusStr = $statusError if ($operationName -eq "Scenario") { - OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($errorNumberOfSQLStmtsThreshold)% for number of SQL statements" + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($NumberOfSqlStmtsThresholdError)% for number of SQL statements" } } if ($statusStr -eq $statusOK) { - if ($pctDurationMin -ge $warningDurationThreshold) { + if ($pctDurationMin -ge $DurationThresholdWarning) { $statusStr = $statusWarning if ($operationName -eq "Scenario") { - OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($warningDurationThreshold)% for duration" + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($DurationThresholdWarning)% for duration" } } - if ($pctNumberOfSQLStmts -ge $warningNumberOfSQLStmtsThreshold) { + if ($pctNumberOfSQLStmts -ge $NumberOfSqlStmtsThresholdWarning) { $statusStr = $statusWarning if ($operationName -eq "Scenario") { - OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($warningNumberOfSQLStmtsThreshold)% for number of SQL statements" + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($NumberOfSqlStmtsThresholdWarning)% for number of SQL statements" } } } From 8f5a2d655c183597c6354c78221349e7d7f2da8a Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 10 Aug 2023 06:25:57 +0200 Subject: [PATCH 37/90] fix tests --- Tests/AnalyzeTests.Test.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 4522e88ad..cadbd1e2a 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -97,7 +97,7 @@ Describe "AnalyzeTests Action Tests" { $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine1 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' - $columns = 9 + $columns = 11 $rows = 12 [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 @@ -126,7 +126,7 @@ Describe "AnalyzeTests Action Tests" { [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 [regex]::Matches($md, '\|N\/A\|').Count | should -Be 4 - [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 2 + [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 4 [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 4 [regex]::Matches($md, "\|$statusError\|").Count | should -Be 2 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) From 42c2fe318e9f45de524dba12dcccc9d4b1e7406b Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 10 Aug 2023 06:28:50 +0200 Subject: [PATCH 38/90] test --- Tests/AnalyzeTests.Test.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index cadbd1e2a..52ad7625a 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -117,7 +117,7 @@ Describe "AnalyzeTests Action Tests" { $script:warningCount = 0 Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } - $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningDurationThreshold 6 -errorDurationThreshold 12 -warningNumberOfSqlStmtsThreshold 6 -errorNumberOfSqlStmtsThreshold 12 + $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningDurationThreshold 4 -errorDurationThreshold 8 -warningNumberOfSqlStmtsThreshold 4 -errorNumberOfSqlStmtsThreshold 8 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 9 @@ -126,7 +126,7 @@ Describe "AnalyzeTests Action Tests" { [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 [regex]::Matches($md, '\|N\/A\|').Count | should -Be 4 - [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 4 + [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 2 [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 4 [regex]::Matches($md, "\|$statusError\|").Count | should -Be 2 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) From 897ec588c18baf1f777a4227374a4067ec352b58 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 10 Aug 2023 08:11:45 +0200 Subject: [PATCH 39/90] fix tests --- Tests/AnalyzeTests.Test.ps1 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 52ad7625a..141223d89 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -50,9 +50,9 @@ Describe "AnalyzeTests Action Tests" { $bcptFilename = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 5 -noOfMeasurements 4 # BaseLine1 has overall highter duration and more SQL statements than bcptFilename (+ one more opearion) - $bcptBaseLine1 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 4 -noOfOperations 6 -noOfMeasurements 4 -durationOffset 1 -numberOfSQLStmtsOffset 1 + $bcptBaseLine1 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 4 -noOfOperations 6 -noOfMeasurements 4 -durationOffset 5 -numberOfSQLStmtsOffset 1 # BaseLine2 has overall lower duration and less SQL statements than bcptFilename (+ one less opearion) - $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 4 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 1 + $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 4 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 0 } It 'Compile Action' { @@ -117,18 +117,18 @@ Describe "AnalyzeTests Action Tests" { $script:warningCount = 0 Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } - $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningDurationThreshold 4 -errorDurationThreshold 8 -warningNumberOfSqlStmtsThreshold 4 -errorNumberOfSqlStmtsThreshold 8 + $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningDurationThreshold 1 -errorDurationThreshold 2 -warningNumberOfSqlStmtsThreshold 1 -errorNumberOfSqlStmtsThreshold 2 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' - $columns = 9 + $columns = 11 $rows = 12 [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 [regex]::Matches($md, '\|N\/A\|').Count | should -Be 4 - [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 2 + [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 4 [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 4 - [regex]::Matches($md, "\|$statusError\|").Count | should -Be 2 + [regex]::Matches($md, "\|$statusError\|").Count | should -Be 0 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) $script:errorCount | Should -be 0 $script:warningCount | Should -be 0 From ff0856f097c6cdec96e8e8c516b9d07e3bf5bf2d Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 10 Aug 2023 09:07:42 +0200 Subject: [PATCH 40/90] fix review --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 21 ++++++++++++++------- Tests/AnalyzeTests.Test.ps1 | 4 ++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index aabebe583..1eb088ad9 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -167,7 +167,14 @@ function GetBcptSummaryMD { $baseLine = ReadBcptFile -path $baseLinePath $summarySb = [System.Text.StringBuilder]::new() - $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|$(if ($baseLine){'Status|'})Duration|$(if ($baseLine){'Duration (Base)|Duration (Diff)|'})SQL Stmts|$(if ($baseLine){'SQL Stmts (Base)|SQL Stmts (Diff)|'})\n|:---|:---|:---|:---|$(if ($baseLine){'---:|'}):--:|$(if ($baseLine){'---:|---:|'})---:|$(if ($baseLine){'---:|---:|'})\n") | Out-Null + if ($baseLine) { + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Status|Duration|Duration base|Duration diff|Duration diff|SQL Stmts|SQL Stmts base|SQL Stmts diff|SQL Stmts diff|\n") | Out-Null + $summarySb.Append("|:---------|:----------|:------------|:--------|:----:|-------:|------------:|------------:|------------:|--------:|-------------:|-------------:|-------------:|\n") | Out-Null + } + else { + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Duration|SQL Stmts|\n") | Out-Null + $summarySb.Append("|:---------|:----------|:------------|:--------|-------:|--------:|\n") | Out-Null + } $lastSuiteName = '' $lastCodeunitID = '' @@ -211,14 +218,14 @@ function GetBcptSummaryMD { } $pctDurationMin = ($durationMin-$baseDurationMin)*100/$baseDurationMin - $durationMinStr = "$($durationMin.ToString("N2"))|" - $baseDurationMinStr = "$($baseDurationMin.ToString("N2"))|" - $diffDurationMinStr = "$($diffDurationMin.ToString("**#**;-#;0"))|" + $durationMinStr = "$($durationMin.ToString("N2"))ms|" + $baseDurationMinStr = "$($baseDurationMin.ToString("N2"))ms|" + $diffDurationMinStr = "$($diffDurationMin.ToString("**#**;-#;0"))ms|$($pctDurationMin.ToString('+#.#;-#.#;0'))%|" $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseNumberOfSQLStmts)*100/$baseNumberOfSQLStmts $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("N0"))|" - $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("**#**;-#;0"))|" + $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("**#**;-#;0"))|$($pctNumberOfSQLStmts.ToString('+#.#;-#.#;0'))%|" if (!$baseLine) { # No baseline provided @@ -233,9 +240,9 @@ function GetBcptSummaryMD { # Baseline provided, but not found for this operation $statusStr = $statusSkipped $baseDurationMinStr = 'N/A|' - $diffDurationMinStr = '|' + $diffDurationMinStr = '||' $baseNumberOfSQLStmtsStr = 'N/A|' - $diffNumberOfSQLStmtsStr = '|' + $diffNumberOfSQLStmtsStr = '||' } else { $statusStr = $statusOK diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 141223d89..056456222 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -97,7 +97,7 @@ Describe "AnalyzeTests Action Tests" { $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine1 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' - $columns = 11 + $columns = 13 $rows = 12 [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 @@ -120,7 +120,7 @@ Describe "AnalyzeTests Action Tests" { $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningDurationThreshold 1 -errorDurationThreshold 2 -warningNumberOfSqlStmtsThreshold 1 -errorNumberOfSqlStmtsThreshold 2 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' - $columns = 11 + $columns = 13 $rows = 12 [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 From bbca5d60a84802a263739dc6add6c5998aa87bfe Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 10 Aug 2023 09:14:19 +0200 Subject: [PATCH 41/90] errs and warnings --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 25 +++++++++++---------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 1eb088ad9..2e65dca7c 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -227,6 +227,11 @@ function GetBcptSummaryMD { $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("N0"))|" $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("**#**;-#;0"))|$($pctNumberOfSQLStmts.ToString('+#.#;-#.#;0'))%|" + $thisOperationName = ''; if ($operationName -ne $lastOperationName) { $thisOperationName = $operationName } + $thisCodeunitName = ''; if ($codeunitName -ne $lastCodeunitName) { $thisCodeunitName = $codeunitName; $thisOperationName = $operationName } + $thisCodeunitID = ''; if ($codeunitID -ne $lastCodeunitID) { $thisCodeunitID = $codeunitID; $thisOperationName = $operationName } + $thisSuiteName = ''; if ($suiteName -ne $lastSuiteName) { $thisSuiteName = $suiteName; $thisOperationName = $operationName } + if (!$baseLine) { # No baseline provided $statusStr = '' @@ -248,29 +253,30 @@ function GetBcptSummaryMD { $statusStr = $statusOK if ($pctDurationMin -ge $DurationThresholdError) { $statusStr = $statusError - if ($operationName -eq "Scenario") { - - # TODO: Determine when to give errors and warnings - + if ($thisCodeunitName) { + # Only give errors and warnings on top level operation OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($DurationThresholdError)% for duration" } } if ($pctNumberOfSQLStmts -ge $NumberOfSqlStmtsThresholdError) { $statusStr = $statusError - if ($operationName -eq "Scenario") { + if ($thisCodeunitName) { + # Only give errors and warnings on top level operation OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($NumberOfSqlStmtsThresholdError)% for number of SQL statements" } } if ($statusStr -eq $statusOK) { if ($pctDurationMin -ge $DurationThresholdWarning) { $statusStr = $statusWarning - if ($operationName -eq "Scenario") { + if ($thisCodeunitName) { + # Only give errors and warnings on top level operation OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($DurationThresholdWarning)% for duration" } } if ($pctNumberOfSQLStmts -ge $NumberOfSqlStmtsThresholdWarning) { $statusStr = $statusWarning - if ($operationName -eq "Scenario") { + if ($thisCodeunitName) { + # Only give errors and warnings on top level operation OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($NumberOfSqlStmtsThresholdWarning)% for number of SQL statements" } } @@ -279,11 +285,6 @@ function GetBcptSummaryMD { $statusStr += '|' } - $thisOperationName = ''; if ($operationName -ne $lastOperationName) { $thisOperationName = $operationName } - $thisCodeunitName = ''; if ($codeunitName -ne $lastCodeunitName) { $thisCodeunitName = $codeunitName; $thisOperationName = $operationName } - $thisCodeunitID = ''; if ($codeunitID -ne $lastCodeunitID) { $thisCodeunitID = $codeunitID; $thisOperationName = $operationName } - $thisSuiteName = ''; if ($suiteName -ne $lastSuiteName) { $thisSuiteName = $suiteName; $thisOperationName = $operationName } - $summarySb.Append("|$thisSuiteName|$thisCodeunitID|$thisCodeunitName|$thisOperationName|$statusStr$durationMinStr$baseDurationMinStr$diffDurationMinStr$numberOfSQLStmtsStr$baseNumberOfSQLStmtsStr$diffNumberOfSQLStmtsStr\n") | Out-Null $lastSuiteName = $suiteName From e52c1179dbefc7fada1b591a407b4a9ce6cc5235 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 10 Aug 2023 09:40:47 +0200 Subject: [PATCH 42/90] move ms to header --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 2e65dca7c..6867326a9 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -168,12 +168,12 @@ function GetBcptSummaryMD { $summarySb = [System.Text.StringBuilder]::new() if ($baseLine) { - $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Status|Duration|Duration base|Duration diff|Duration diff|SQL Stmts|SQL Stmts base|SQL Stmts diff|SQL Stmts diff|\n") | Out-Null - $summarySb.Append("|:---------|:----------|:------------|:--------|:----:|-------:|------------:|------------:|------------:|--------:|-------------:|-------------:|-------------:|\n") | Out-Null + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Status|Duration (ms)|Duration base (ms)|Duration diff (ms)|Duration diff|SQL Stmts|SQL Stmts base|SQL Stmts diff|SQL Stmts diff|\n") | Out-Null + $summarySb.Append("|:---------|:----------|:------------|:--------|:----:|------------:|-----------------:|-----------------:|------------:|--------:|-------------:|-------------:|-------------:|\n") | Out-Null } else { - $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Duration|SQL Stmts|\n") | Out-Null - $summarySb.Append("|:---------|:----------|:------------|:--------|-------:|--------:|\n") | Out-Null + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Duration (ms)|SQL Stmts|\n") | Out-Null + $summarySb.Append("|:---------|:----------|:------------|:--------|------------:|--------:|\n") | Out-Null } $lastSuiteName = '' @@ -218,14 +218,14 @@ function GetBcptSummaryMD { } $pctDurationMin = ($durationMin-$baseDurationMin)*100/$baseDurationMin - $durationMinStr = "$($durationMin.ToString("N2"))ms|" - $baseDurationMinStr = "$($baseDurationMin.ToString("N2"))ms|" - $diffDurationMinStr = "$($diffDurationMin.ToString("**#**;-#;0"))ms|$($pctDurationMin.ToString('+#.#;-#.#;0'))%|" + $durationMinStr = "$($durationMin.ToString("#"))|" + $baseDurationMinStr = "$($baseDurationMin.ToString("#"))|" + $diffDurationMinStr = "$($diffDurationMin.ToString("+#;-#;0"))|$($pctDurationMin.ToString('+#;-#;0'))%|" $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseNumberOfSQLStmts)*100/$baseNumberOfSQLStmts - $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("N0"))|" - $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("N0"))|" - $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("**#**;-#;0"))|$($pctNumberOfSQLStmts.ToString('+#.#;-#.#;0'))%|" + $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("#"))|" + $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("#"))|" + $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("+#;-#;0"))|$($pctNumberOfSQLStmts.ToString('+#;-#;0'))%|" $thisOperationName = ''; if ($operationName -ne $lastOperationName) { $thisOperationName = $operationName } $thisCodeunitName = ''; if ($codeunitName -ne $lastCodeunitName) { $thisCodeunitName = $codeunitName; $thisOperationName = $operationName } From 17fc35ddfe2b684ce6e2e699a95e2fb877549af3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 10 Aug 2023 23:41:15 +0200 Subject: [PATCH 43/90] add baseline text --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 6867326a9..c9a213308 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -295,7 +295,10 @@ function GetBcptSummaryMD { } } - if (-not $baseLine) { + if ($baseLine) { + $summarySb.Append("\nUsed baseline provided in $([System.IO.Path]::GetFileName($baseLinePath)).") | Out-Null + } + else { $summarySb.Append("\nNo baseline provided. Copy a set of BCPT results to $([System.IO.Path]::GetFileName($baseLinePath)) in the project folder in order to establish a baseline.") | Out-Null } From f8fc0dd327e025efb80c24dedc1fb3ee1ec50e34 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 7 Sep 2023 11:38:58 +0200 Subject: [PATCH 44/90] fix PSSA --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 72 ++++++++++----------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index ad82aeef3..c30273d3a 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -25,26 +25,25 @@ function GetTestResultSummaryMD { if (-not $appNames) { $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "extensionId" } | ForEach-Object { $_.Value } } | Select-Object -Unique) } - $testResults.testsuites.testsuite | ForEach-Object { - $totalTests += $_.Tests - $totalTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) - $totalFailed += $_.failures - $totalSkipped += $_.skipped + foreach($testsuite in $testResults.testsuites.testsuite) { + $totalTests += $testsuite.Tests + $totalTime += [decimal]::Parse($testsuite.time, [System.Globalization.CultureInfo]::InvariantCulture) + $totalFailed += $testsuite.failures + $totalSkipped += $testsuite.skipped } Write-Host "$($appNames.Count) TestApps, $totalTests tests, $totalFailed failed, $totalSkipped skipped, $totalTime seconds" $summarySb.Append('|Test app|Tests|Passed|Failed|Skipped|Time|\n|:---|---:|---:|---:|---:|---:|\n') | Out-Null - $appNames | ForEach-Object { - $appName = $_ + foreach($appName in $appNames) { $appTests = 0 $appTime = 0.0 $appFailed = 0 $appSkipped = 0 $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Value -eq $appName } } - $suites | ForEach-Object { - $appTests += [int]$_.tests - $appFailed += [int]$_.failures - $appSkipped += [int]$_.skipped - $appTime += [decimal]::Parse($_.time, [System.Globalization.CultureInfo]::InvariantCulture) + foreach($suite in $suites) { + $appTests += [int]$suite.tests + $appFailed += [int]$suite.failures + $appSkipped += [int]$suite.skipped + $appTime += [decimal]::Parse($suite.time, [System.Globalization.CultureInfo]::InvariantCulture) } $appPassed = $appTests-$appFailed-$appSkipped Write-Host "- $appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds" @@ -63,21 +62,21 @@ function GetTestResultSummaryMD { $summarySb.Append("|$($appTime)s|\n") | Out-Null if ($appFailed -gt 0) { $failuresSb.Append("
$appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds\n") | Out-Null - $suites | ForEach-Object { - Write-Host " - $($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds" - if ($_.failures -gt 0 -and $failuresSb.Length -lt 32000) { - $failuresSb.Append("
$($_.name), $($_.tests) tests, $($_.failures) failed, $($_.skipped) skipped, $($_.time) seconds") | Out-Null - $_.testcase | ForEach-Object { - if ($_.ChildNodes.Count -gt 0) { - Write-Host " - $($_.name), Failure, $($_.time) seconds" - $failuresSb.Append("
$($_.name), Failure") | Out-Null - $_.ChildNodes | ForEach-Object { - Write-Host " - Error: $($_.message)" + foreach($suite in $suites) { + Write-Host " - $($suite.name), $($suite.tests) tests, $($suite.failures) failed, $($suite.skipped) skipped, $($suite.time) seconds" + if ($suite.failures -gt 0 -and $failuresSb.Length -lt 32000) { + $failuresSb.Append("
$($suite.name), $($suite.tests) tests, $($suite.failures) failed, $($suite.skipped) skipped, $($suite.time) seconds") | Out-Null + foreach($testcase in $suite.testcase) { + if ($testcase.ChildNodes.Count -gt 0) { + Write-Host " - $($testcase.name), Failure, $($testcase.time) seconds" + $failuresSb.Append("
$($testcase.name), Failure") | Out-Null + foreach($failure in $testcase.ChildNodes) { + Write-Host " - Error: $($failure.message)" Write-Host " Stacktrace:" - Write-Host " $($_."#text".Trim().Replace("`n","`n "))" - $failuresSb.Append("      Error: $($_.message)
") | Out-Null + Write-Host " $($failure."#text".Trim().Replace("`n","`n "))" + $failuresSb.Append("      Error: $($failure.message)
") | Out-Null $failuresSb.Append("      Stack trace
") | Out-Null - $failuresSb.Append("      $($_."#text".Trim().Replace("`n","
      "))

") | Out-Null + $failuresSb.Append("      $($failure."#text".Trim().Replace("`n","
      "))

") | Out-Null } $failuresSb.Append("
") | Out-Null } @@ -120,11 +119,11 @@ function ReadBcptFile { $bcptResult = Get-Content -Path $path -Encoding UTF8 | ConvertFrom-Json $suites = @{} # Sort by bcptCode, codeunitID, operation - $bcptResult | ForEach-Object { - $bcptCode = $_.bcptCode - $codeunitID = $_.codeunitID - $codeunitName = $_.codeunitName - $operation = $_.operation + foreach($measure in $bcptResult) { + $bcptCode = $measure.bcptCode + $codeunitID = $measure.codeunitID + $codeunitName = $measure.codeunitName + $operation = $measure.operation # Create Suite if it doesn't exist if(-not $suites.containsKey($bcptCode)) { @@ -145,8 +144,8 @@ function ReadBcptFile { } # Add measurement to measurements under operation $suites."$bcptCode"."$codeunitID"."operations"."$operation".measurements += @(@{ - "durationMin" = $_.durationMin - "numberOfSQLStmts" = $_.numberOfSQLStmts + "durationMin" = $measure.durationMin + "numberOfSQLStmts" = $measure.numberOfSQLStmts }) } $suites @@ -182,15 +181,12 @@ function GetBcptSummaryMD { $lastOperationName = '' # calculate statistics on measurements, skipping the $skipMeasurements longest measurements - $bcpt.Keys | ForEach-Object { - $suiteName = $_ + foreach($suiteName in $bcpt.Keys) { $suite = $bcpt."$suiteName" - $suite.Keys | ForEach-Object { - $codeUnitID = $_ + foreach($codeUnitID in $suite.Keys) { $codeunit = $suite."$codeunitID" $codeUnitName = $codeunit.codeunitName - $codeunit."operations".Keys | ForEach-Object { - $operationName = $_ + foreach($operationName in $codeunit."operations".Keys) { $operation = $codeunit."operations"."$operationName" # Get measurements to use for statistics $measurements = @($operation."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) From e370a9d5f37e2d10ca9367f3f952c9bc88d794d3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 7 Sep 2023 11:46:29 +0200 Subject: [PATCH 45/90] PSSA --- Tests/AnalyzeTests.Test.ps1 | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 056456222..ab1958aaf 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -15,16 +15,13 @@ Describe "AnalyzeTests Action Tests" { ) $bcpt = @() - 1..$noOfSuites | ForEach-Object { - $suiteName = "SUITE$_" - 1..$noOfCodeunits | ForEach-Object { - $codeunitID = $_ + for($suiteNo = 1; $suiteNo -le $noOfSuites; $suiteNo++) { + $suiteName = "SUITE$suiteNo" + for($codeUnitID = 1; $codeunitID -le $noOfCodeunits; $codeunitID++) { $codeunitName = "Codeunit$_" - 1..$noOfOperations | ForEach-Object { - $operationNo = $_ + for($operationNo = 1; $operationNo -le $noOfOperations; $operationNo++) { $operationName = "Operation$operationNo" - 1..$noOfMeasurements | ForEach-Object { - $no = $_ + for($no = 1; $no -le $noOfMeasurements; $no++) { $bcpt += @(@{ "id" = [GUID]::NewGuid().ToString() "bcptCode" = $suiteName @@ -46,12 +43,16 @@ Describe "AnalyzeTests Action Tests" { $actionName = "AnalyzeTests" $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve $scriptName = "$actionName.ps1" + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'actionScript', Justification = 'False positive.')] $actionScript = GetActionScript -scriptRoot $scriptRoot -scriptName $scriptName + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'bcptFilename', Justification = 'False positive.')] $bcptFilename = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 5 -noOfMeasurements 4 # BaseLine1 has overall highter duration and more SQL statements than bcptFilename (+ one more opearion) + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'bcptBaseLine1', Justification = 'False positive.')] $bcptBaseLine1 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 4 -noOfOperations 6 -noOfMeasurements 4 -durationOffset 5 -numberOfSQLStmtsOffset 1 # BaseLine2 has overall lower duration and less SQL statements than bcptFilename (+ one less opearion) + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'bcptBaseLine2', Justification = 'False positive.')] $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 4 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 0 } From c37c97cc43dbe94f9f0968fde3ded88f8cf37534 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 8 Sep 2023 09:01:16 +0200 Subject: [PATCH 46/90] Test failure --- Tests/AnalyzeTests.Test.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index ab1958aaf..bb5f3d5ad 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -13,12 +13,12 @@ Describe "AnalyzeTests Action Tests" { [int] $durationOffset = 0, [int] $numberOfSQLStmtsOffset = 0 ) - + $bcpt = @() for($suiteNo = 1; $suiteNo -le $noOfSuites; $suiteNo++) { $suiteName = "SUITE$suiteNo" for($codeUnitID = 1; $codeunitID -le $noOfCodeunits; $codeunitID++) { - $codeunitName = "Codeunit$_" + $codeunitName = "Codeunit$codeunitID" for($operationNo = 1; $operationNo -le $noOfOperations; $operationNo++) { $operationName = "Operation$operationNo" for($no = 1; $no -le $noOfMeasurements; $no++) { @@ -39,7 +39,7 @@ Describe "AnalyzeTests Action Tests" { $bcpt | ConvertTo-Json -Depth 100 | Set-Content -Path $filename -Encoding UTF8 return $filename } - + $actionName = "AnalyzeTests" $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve $scriptName = "$actionName.ps1" From 82c2b19e21fab3161486dfb0b9fce475802200d7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Sep 2023 06:01:24 +0200 Subject: [PATCH 47/90] Add thresholds file --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 2 ++ Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 11 ++++++++++- Tests/AnalyzeTests.Test.ps1 | 7 +++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index 1559f1a68..2c621b97e 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -22,9 +22,11 @@ try { $settings = $env:Settings | ConvertFrom-Json $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptTestResults.json" $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptBaseLine.json" + $bcptThresholdsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptThresholds.json" $bcptSummaryMD = GetBcptSummaryMD ` -path $bcptTestResultsFile ` -baseLinePath $bcptBaseLineFile ` + -thresholdsPath $bcptThresholdsFile ` -DurationThresholdWarning $settings.DurationThresholdWarning ` -DurationThresholdError $settings.DurationThresholdError ` -NumberOfSqlStmtsThresholdWarning $settings.NumberOfSqlStmtsThresholdWarning ` diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index c30273d3a..abc6fa682 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -155,6 +155,7 @@ function GetBcptSummaryMD { Param( [string] $path, [string] $baseLinePath = '', + [string] $thresholdsPath = '', [int] $skipMeasurements = 1, [int] $DurationThresholdWarning = 10, [int] $DurationThresholdError = 25, @@ -164,7 +165,15 @@ function GetBcptSummaryMD { $bcpt = ReadBcptFile -path $path $baseLine = ReadBcptFile -path $baseLinePath - + # Override thresholds if thresholds file exists + if ($thresholdsPath -and (Test-Path -path $thresholdsPath)) { + $thresholds = Get-Content -Path $thresholdsPath -Encoding UTF8 | ConvertFrom-Json + foreach($threshold in 'DurationThresholdWarning', 'DurationThresholdError', 'NumberOfSqlStmtsThresholdWarning', 'NumberOfSqlStmtsThresholdError') { + if ($thresholds.PSObject.Properties.Name -eq $threshold) { + Set-Variable -Name $threshold -Value $thresholds."$threshold" -Scope local + } + } + } $summarySb = [System.Text.StringBuilder]::new() if ($baseLine) { $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Status|Duration (ms)|Duration base (ms)|Duration diff (ms)|Duration diff|SQL Stmts|SQL Stmts base|SQL Stmts diff|SQL Stmts diff|\n") | Out-Null diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index bb5f3d5ad..643853293 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -54,6 +54,9 @@ Describe "AnalyzeTests Action Tests" { # BaseLine2 has overall lower duration and less SQL statements than bcptFilename (+ one less opearion) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'bcptBaseLine2', Justification = 'False positive.')] $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 4 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 0 + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'thresholdsFile', Justification = 'False positive.')] + $thresholdsFile = Join-Path ([System.IO.Path]::GetTempPath()) "$([GUID]::NewGuid().ToString()).json" + @{ "WarningNumberOfSqlStmtsThreshold" = 1; "ErrorNumberOfSqlStmtsThreshold" = 2 } | ConvertTo-Json | Set-Content -Path $thresholdsFile -Encoding UTF8 } It 'Compile Action' { @@ -95,7 +98,7 @@ Describe "AnalyzeTests Action Tests" { It 'Test GetBcptSummaryMD (with worse baseline)' { . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') - $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine1 + $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine1 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 13 @@ -118,7 +121,7 @@ Describe "AnalyzeTests Action Tests" { $script:warningCount = 0 Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } - $md = GetBcptSummaryMD -path $bcptFilename -baseline $bcptBaseLine2 -warningDurationThreshold 1 -errorDurationThreshold 2 -warningNumberOfSqlStmtsThreshold 1 -errorNumberOfSqlStmtsThreshold 2 + $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -warningDurationThreshold 1 -errorDurationThreshold 2 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 13 From bed8e4c9ce855bd37ce50746c2bc9c676dcc4552 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Sep 2023 06:03:08 +0200 Subject: [PATCH 48/90] dump using --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index abc6fa682..a09ee664e 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -174,6 +174,12 @@ function GetBcptSummaryMD { } } } + Write-Host "Using thresholds:" + Write-Host "- DurationThresholdWarning: $DurationThresholdWarning" + Write-Host "- DurationThresholdError: $DurationThresholdError" + Write-Host "- NumberOfSqlStmtsThresholdWarning: $NumberOfSqlStmtsThresholdWarning" + Write-Host "- NumberOfSqlStmtsThresholdError: $NumberOfSqlStmtsThresholdError" + $summarySb = [System.Text.StringBuilder]::new() if ($baseLine) { $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Status|Duration (ms)|Duration base (ms)|Duration diff (ms)|Duration diff|SQL Stmts|SQL Stmts base|SQL Stmts diff|SQL Stmts diff|\n") | Out-Null From 1c2345c0ba9b31990df547259c0f7b4c7128170c Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Sep 2023 06:09:04 +0200 Subject: [PATCH 49/90] thresholds --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 3 ++- Tests/AnalyzeTests.Test.ps1 | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index a09ee664e..5ea835f66 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -167,6 +167,7 @@ function GetBcptSummaryMD { $baseLine = ReadBcptFile -path $baseLinePath # Override thresholds if thresholds file exists if ($thresholdsPath -and (Test-Path -path $thresholdsPath)) { + Write-Host "Reading thresholds from $thresholdsPath" $thresholds = Get-Content -Path $thresholdsPath -Encoding UTF8 | ConvertFrom-Json foreach($threshold in 'DurationThresholdWarning', 'DurationThresholdError', 'NumberOfSqlStmtsThresholdWarning', 'NumberOfSqlStmtsThresholdError') { if ($thresholds.PSObject.Properties.Name -eq $threshold) { @@ -179,7 +180,7 @@ function GetBcptSummaryMD { Write-Host "- DurationThresholdError: $DurationThresholdError" Write-Host "- NumberOfSqlStmtsThresholdWarning: $NumberOfSqlStmtsThresholdWarning" Write-Host "- NumberOfSqlStmtsThresholdError: $NumberOfSqlStmtsThresholdError" - + $summarySb = [System.Text.StringBuilder]::new() if ($baseLine) { $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Status|Duration (ms)|Duration base (ms)|Duration diff (ms)|Duration diff|SQL Stmts|SQL Stmts base|SQL Stmts diff|SQL Stmts diff|\n") | Out-Null diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 643853293..342f7a9db 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -56,7 +56,7 @@ Describe "AnalyzeTests Action Tests" { $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 4 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 0 [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'thresholdsFile', Justification = 'False positive.')] $thresholdsFile = Join-Path ([System.IO.Path]::GetTempPath()) "$([GUID]::NewGuid().ToString()).json" - @{ "WarningNumberOfSqlStmtsThreshold" = 1; "ErrorNumberOfSqlStmtsThreshold" = 2 } | ConvertTo-Json | Set-Content -Path $thresholdsFile -Encoding UTF8 + @{ "NumberOfSqlStmtsThresholdWarning" = 1; "NumberOfSqlStmtsThresholdError" = 2 } | ConvertTo-Json | Set-Content -Path $thresholdsFile -Encoding UTF8 } It 'Compile Action' { @@ -121,7 +121,7 @@ Describe "AnalyzeTests Action Tests" { $script:warningCount = 0 Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } - $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -warningDurationThreshold 1 -errorDurationThreshold 2 + $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -durationThresholdWarning 1 -durationThresholdError 2 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 13 @@ -135,7 +135,7 @@ Describe "AnalyzeTests Action Tests" { [regex]::Matches($md, "\|$statusError\|").Count | should -Be 0 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) $script:errorCount | Should -be 0 - $script:warningCount | Should -be 0 + $script:warningCount | Should -be 2 } AfterAll { From cbd32c1f820ebe520e597be568531c75db254ee1 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Sep 2023 06:12:59 +0200 Subject: [PATCH 50/90] 5 and 10 --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- Tests/AnalyzeTests.Test.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 5ea835f66..eceaa323c 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -176,7 +176,7 @@ function GetBcptSummaryMD { } } Write-Host "Using thresholds:" - Write-Host "- DurationThresholdWarning: $DurationThresholdWarning" + Write-Host "- DurationThresholdWarning: $DurationThresholdWarning" Write-Host "- DurationThresholdError: $DurationThresholdError" Write-Host "- NumberOfSqlStmtsThresholdWarning: $NumberOfSqlStmtsThresholdWarning" Write-Host "- NumberOfSqlStmtsThresholdError: $NumberOfSqlStmtsThresholdError" diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 342f7a9db..0f8fe0192 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -121,7 +121,7 @@ Describe "AnalyzeTests Action Tests" { $script:warningCount = 0 Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } - $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -durationThresholdWarning 1 -durationThresholdError 2 + $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -durationThresholdWarning 5 -durationThresholdError 10 Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 13 From e1b850e86aea36f62674f19cc447e5e82e6fcea0 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Sep 2023 06:18:10 +0200 Subject: [PATCH 51/90] fix tests --- Tests/AnalyzeTests.Test.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 0f8fe0192..a06081cf8 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -130,9 +130,9 @@ Describe "AnalyzeTests Action Tests" { [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 [regex]::Matches($md, '\|N\/A\|').Count | should -Be 4 - [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 4 + [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 0 [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 4 - [regex]::Matches($md, "\|$statusError\|").Count | should -Be 0 + [regex]::Matches($md, "\|$statusError\|").Count | should -Be 4 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) $script:errorCount | Should -be 0 $script:warningCount | Should -be 2 From 558d19c7b947208a7ab0bf5d71577e79e1e24254 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Sep 2023 07:43:10 +0200 Subject: [PATCH 52/90] dumps --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index eceaa323c..29d911bad 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -156,7 +156,7 @@ function GetBcptSummaryMD { [string] $path, [string] $baseLinePath = '', [string] $thresholdsPath = '', - [int] $skipMeasurements = 1, + [int] $skipMeasurements = 0, [int] $DurationThresholdWarning = 10, [int] $DurationThresholdError = 25, [int] $NumberOfSqlStmtsThresholdWarning = 5, @@ -210,6 +210,9 @@ function GetBcptSummaryMD { $durationMin = ($measurements | ForEach-Object { $_.durationMin } | Measure-Object -Minimum).Minimum $numberOfSQLStmts = ($measurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum + Write-Host "DurationMin: $durationMin" + Write-Host "NumberOfSQLStmts: $numberOfSQLStmts" + $baseLineFound = $true try { $baseLineMeasurements = @($baseLine."$suiteName"."$codeUnitID"."operations"."$operationName"."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) @@ -220,6 +223,8 @@ function GetBcptSummaryMD { $diffDurationMin = $durationMin-$baseDurationMin $baseNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum $diffNumberOfSQLStmts = $numberOfSQLStmts-$baseNumberOfSQLStmts + Write-Host "BaseDurationMin: $baseDurationMin" + Write-Host "BaseNumberOfSQLStmts: $baseNumberOfSQLStmts" } catch { $baseLineFound = $false From 2dd0b12b08fd4a58e60c1103ea2576de22c36ae3 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Sep 2023 07:49:34 +0200 Subject: [PATCH 53/90] ordered --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 12 ++++++------ Tests/AnalyzeTests.Test.ps1 | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 29d911bad..bb4cd236c 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -117,7 +117,7 @@ function ReadBcptFile { # Read BCPT file $bcptResult = Get-Content -Path $path -Encoding UTF8 | ConvertFrom-Json - $suites = @{} + $suites = [ordered]@{} # Sort by bcptCode, codeunitID, operation foreach($measure in $bcptResult) { $bcptCode = $measure.bcptCode @@ -126,18 +126,18 @@ function ReadBcptFile { $operation = $measure.operation # Create Suite if it doesn't exist - if(-not $suites.containsKey($bcptCode)) { - $suites."$bcptCode" = @{} + if(-not $suites.Contains($bcptCode)) { + $suites."$bcptCode" = [ordered]@{} } # Create Codeunit under Suite if it doesn't exist - if (-not $suites."$bcptCode".ContainsKey("$codeunitID")) { + if (-not $suites."$bcptCode".Contains("$codeunitID")) { $suites."$bcptCode"."$codeunitID" = @{ "codeunitName" = $codeunitName - "operations" = @{} + "operations" = [ordered]@{} } } # Create Operation under Codeunit if it doesn't exist - if (-not $suites."$bcptCode"."$codeunitID"."operations".ContainsKey($operation)) { + if (-not $suites."$bcptCode"."$codeunitID"."operations".Contains($operation)) { $suites."$bcptCode"."$codeunitID"."operations"."$operation" = @{ "measurements" = @() } diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index a06081cf8..4dff4ef63 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -134,8 +134,8 @@ Describe "AnalyzeTests Action Tests" { [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 4 [regex]::Matches($md, "\|$statusError\|").Count | should -Be 4 [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) - $script:errorCount | Should -be 0 - $script:warningCount | Should -be 2 + $script:errorCount | Should -be 2 + $script:warningCount | Should -be 0 } AfterAll { From 6a685f68289329f2af0cb9d9e786d8bcb7fbee18 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 19 Sep 2023 07:50:44 +0200 Subject: [PATCH 54/90] remove dumps --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index bb4cd236c..31eed97cf 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -210,9 +210,6 @@ function GetBcptSummaryMD { $durationMin = ($measurements | ForEach-Object { $_.durationMin } | Measure-Object -Minimum).Minimum $numberOfSQLStmts = ($measurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum - Write-Host "DurationMin: $durationMin" - Write-Host "NumberOfSQLStmts: $numberOfSQLStmts" - $baseLineFound = $true try { $baseLineMeasurements = @($baseLine."$suiteName"."$codeUnitID"."operations"."$operationName"."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) @@ -223,8 +220,6 @@ function GetBcptSummaryMD { $diffDurationMin = $durationMin-$baseDurationMin $baseNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum $diffNumberOfSQLStmts = $numberOfSQLStmts-$baseNumberOfSQLStmts - Write-Host "BaseDurationMin: $baseDurationMin" - Write-Host "BaseNumberOfSQLStmts: $baseNumberOfSQLStmts" } catch { $baseLineFound = $false From 38476cbff3b80662bee001ddc2308fc0cce2f2ef Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 5 Dec 2023 08:12:41 +0100 Subject: [PATCH 55/90] spaces --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 31eed97cf..c675b53b2 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -1,7 +1,7 @@ -$statusOK = ":heavy_check_mark:" -$statusWarning = ":warning:" -$statusError = ":x:" -$statusSkipped = ":question:" +$statusOK = " :heavy_check_mark:" +$statusWarning = " :warning:" +$statusError = " :x:" +$statusSkipped = " :question:" # Build MarkDown of TestResults file # This function will not fail if the file does not exist or if any test errors are found From 800af5b6efabf7141cc2861c2c2460b15b1dc428 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 5 Dec 2023 08:20:23 +0100 Subject: [PATCH 56/90] use local containerhelper --- Actions/AL-Go-Helper.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index a1098c702..511339045 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -18,7 +18,7 @@ $defaultCICDPushBranches = @( 'main', 'release/*', 'feature/*' ) [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'defaultCICDPullRequestBranches', Justification = 'False positive.')] $defaultCICDPullRequestBranches = @( 'main' ) $runningLocal = $local.IsPresent -$defaultBcContainerHelperVersion = "preview" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step +$defaultBcContainerHelperVersion = "https://github.com/freddydk/navcontainerhelper/zipball/renamecache" # Must be double quotes. Will be replaced by BcContainerHelperVersion if necessary in the deploy step $microsoftTelemetryConnectionString = "InstrumentationKey=84bd9223-67d4-4378-8590-9e4a46023be2;IngestionEndpoint=https://westeurope-1.in.applicationinsights.azure.com/" $runAlPipelineOverrides = @( From bacf6c128c5ee2441962c3f12f1730a7efba0637 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 5 Dec 2023 09:45:09 +0100 Subject: [PATCH 57/90] summary --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index c675b53b2..bd45ce63c 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -100,6 +100,7 @@ function GetTestResultSummaryMD { } else { $summarySb.Append("No test results found") | Out-Null + $failuresSummaryMD = '' } $summarySb.ToString() $failuresSb.ToString() From 5df4b8a3a3db1117311b50930826c8cd7fb4e00e Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 5 Dec 2023 09:52:48 +0100 Subject: [PATCH 58/90] return empty --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index bd45ce63c..cf8785b4e 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -165,6 +165,9 @@ function GetBcptSummaryMD { ) $bcpt = ReadBcptFile -path $path + if (-not $bcpt) { + return '' + } $baseLine = ReadBcptFile -path $baseLinePath # Override thresholds if thresholds file exists if ($thresholdsPath -and (Test-Path -path $thresholdsPath)) { From 9e978a341a18b25044c40e48cccbc44873b759fb Mon Sep 17 00:00:00 2001 From: freddydk Date: Wed, 6 Dec 2023 10:58:05 +0100 Subject: [PATCH 59/90] do not add bcpt section --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index 2c621b97e..aa8393e34 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -51,8 +51,10 @@ try { Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "## Test results`n`n" Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsSummaryMD.Replace("\n","`n"))`n`n" Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsfailuresMD.Replace("\n","`n"))`n`n" - Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "## Performance test results`n`n" - Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($bcptSummaryMD.Replace("\n","`n"))`n`n" + if ($bcptSummaryMD) { + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "## Performance test results`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($bcptSummaryMD.Replace("\n","`n"))`n`n" + } TrackTrace -telemetryScope $telemetryScope } From b0789ae70105052c8855caa07e1fd4cf8b9b54a4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 9 Dec 2023 06:36:10 +0100 Subject: [PATCH 60/90] move --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 90 ++++++++++----------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index cf8785b4e..d9e09e510 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -3,6 +3,51 @@ $statusWarning = " :warning:" $statusError = " :x:" $statusSkipped = " :question:" +function ReadBcptFile { + Param( + [string] $path + ) + + if ((-not $path) -or (-not (Test-Path -Path $path -PathType Leaf))) { + return $null + } + + # Read BCPT file + $bcptResult = Get-Content -Path $path -Encoding UTF8 | ConvertFrom-Json + $suites = [ordered]@{} + # Sort by bcptCode, codeunitID, operation + foreach($measure in $bcptResult) { + $bcptCode = $measure.bcptCode + $codeunitID = $measure.codeunitID + $codeunitName = $measure.codeunitName + $operation = $measure.operation + + # Create Suite if it doesn't exist + if(-not $suites.Contains($bcptCode)) { + $suites."$bcptCode" = [ordered]@{} + } + # Create Codeunit under Suite if it doesn't exist + if (-not $suites."$bcptCode".Contains("$codeunitID")) { + $suites."$bcptCode"."$codeunitID" = @{ + "codeunitName" = $codeunitName + "operations" = [ordered]@{} + } + } + # Create Operation under Codeunit if it doesn't exist + if (-not $suites."$bcptCode"."$codeunitID"."operations".Contains($operation)) { + $suites."$bcptCode"."$codeunitID"."operations"."$operation" = @{ + "measurements" = @() + } + } + # Add measurement to measurements under operation + $suites."$bcptCode"."$codeunitID"."operations"."$operation".measurements += @(@{ + "durationMin" = $measure.durationMin + "numberOfSQLStmts" = $measure.numberOfSQLStmts + }) + } + $suites +} + # Build MarkDown of TestResults file # This function will not fail if the file does not exist or if any test errors are found # TestResults is in JUnit format @@ -107,51 +152,6 @@ function GetTestResultSummaryMD { $failuresSummaryMD } -function ReadBcptFile { - Param( - [string] $path - ) - - if ((-not $path) -or (-not (Test-Path -Path $path -PathType Leaf))) { - return $null - } - - # Read BCPT file - $bcptResult = Get-Content -Path $path -Encoding UTF8 | ConvertFrom-Json - $suites = [ordered]@{} - # Sort by bcptCode, codeunitID, operation - foreach($measure in $bcptResult) { - $bcptCode = $measure.bcptCode - $codeunitID = $measure.codeunitID - $codeunitName = $measure.codeunitName - $operation = $measure.operation - - # Create Suite if it doesn't exist - if(-not $suites.Contains($bcptCode)) { - $suites."$bcptCode" = [ordered]@{} - } - # Create Codeunit under Suite if it doesn't exist - if (-not $suites."$bcptCode".Contains("$codeunitID")) { - $suites."$bcptCode"."$codeunitID" = @{ - "codeunitName" = $codeunitName - "operations" = [ordered]@{} - } - } - # Create Operation under Codeunit if it doesn't exist - if (-not $suites."$bcptCode"."$codeunitID"."operations".Contains($operation)) { - $suites."$bcptCode"."$codeunitID"."operations"."$operation" = @{ - "measurements" = @() - } - } - # Add measurement to measurements under operation - $suites."$bcptCode"."$codeunitID"."operations"."$operation".measurements += @(@{ - "durationMin" = $measure.durationMin - "numberOfSQLStmts" = $measure.numberOfSQLStmts - }) - } - $suites -} - function GetBcptSummaryMD { Param( [string] $path, From 10f98f2810aebb6f5ca54180a2d794ee24d7f15f Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 9 Dec 2023 08:01:08 +0100 Subject: [PATCH 61/90] move --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 90 ++++++++++----------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index d9e09e510..cf8785b4e 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -3,51 +3,6 @@ $statusWarning = " :warning:" $statusError = " :x:" $statusSkipped = " :question:" -function ReadBcptFile { - Param( - [string] $path - ) - - if ((-not $path) -or (-not (Test-Path -Path $path -PathType Leaf))) { - return $null - } - - # Read BCPT file - $bcptResult = Get-Content -Path $path -Encoding UTF8 | ConvertFrom-Json - $suites = [ordered]@{} - # Sort by bcptCode, codeunitID, operation - foreach($measure in $bcptResult) { - $bcptCode = $measure.bcptCode - $codeunitID = $measure.codeunitID - $codeunitName = $measure.codeunitName - $operation = $measure.operation - - # Create Suite if it doesn't exist - if(-not $suites.Contains($bcptCode)) { - $suites."$bcptCode" = [ordered]@{} - } - # Create Codeunit under Suite if it doesn't exist - if (-not $suites."$bcptCode".Contains("$codeunitID")) { - $suites."$bcptCode"."$codeunitID" = @{ - "codeunitName" = $codeunitName - "operations" = [ordered]@{} - } - } - # Create Operation under Codeunit if it doesn't exist - if (-not $suites."$bcptCode"."$codeunitID"."operations".Contains($operation)) { - $suites."$bcptCode"."$codeunitID"."operations"."$operation" = @{ - "measurements" = @() - } - } - # Add measurement to measurements under operation - $suites."$bcptCode"."$codeunitID"."operations"."$operation".measurements += @(@{ - "durationMin" = $measure.durationMin - "numberOfSQLStmts" = $measure.numberOfSQLStmts - }) - } - $suites -} - # Build MarkDown of TestResults file # This function will not fail if the file does not exist or if any test errors are found # TestResults is in JUnit format @@ -152,6 +107,51 @@ function GetTestResultSummaryMD { $failuresSummaryMD } +function ReadBcptFile { + Param( + [string] $path + ) + + if ((-not $path) -or (-not (Test-Path -Path $path -PathType Leaf))) { + return $null + } + + # Read BCPT file + $bcptResult = Get-Content -Path $path -Encoding UTF8 | ConvertFrom-Json + $suites = [ordered]@{} + # Sort by bcptCode, codeunitID, operation + foreach($measure in $bcptResult) { + $bcptCode = $measure.bcptCode + $codeunitID = $measure.codeunitID + $codeunitName = $measure.codeunitName + $operation = $measure.operation + + # Create Suite if it doesn't exist + if(-not $suites.Contains($bcptCode)) { + $suites."$bcptCode" = [ordered]@{} + } + # Create Codeunit under Suite if it doesn't exist + if (-not $suites."$bcptCode".Contains("$codeunitID")) { + $suites."$bcptCode"."$codeunitID" = @{ + "codeunitName" = $codeunitName + "operations" = [ordered]@{} + } + } + # Create Operation under Codeunit if it doesn't exist + if (-not $suites."$bcptCode"."$codeunitID"."operations".Contains($operation)) { + $suites."$bcptCode"."$codeunitID"."operations"."$operation" = @{ + "measurements" = @() + } + } + # Add measurement to measurements under operation + $suites."$bcptCode"."$codeunitID"."operations"."$operation".measurements += @(@{ + "durationMin" = $measure.durationMin + "numberOfSQLStmts" = $measure.numberOfSQLStmts + }) + } + $suites +} + function GetBcptSummaryMD { Param( [string] $path, From 1f5be044f023e394d0293648a307012eda571fdf Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 15 Dec 2023 22:29:08 +0100 Subject: [PATCH 62/90] do not use special naming --- Actions/Deliver/Deliver.ps1 | 3 --- 1 file changed, 3 deletions(-) diff --git a/Actions/Deliver/Deliver.ps1 b/Actions/Deliver/Deliver.ps1 index b10aac177..a4ebefddd 100644 --- a/Actions/Deliver/Deliver.ps1 +++ b/Actions/Deliver/Deliver.ps1 @@ -218,9 +218,6 @@ try { Get-Item -Path (Join-Path $folder[0] "*.app") | ForEach-Object { $parameters = @{ "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY" - "includeNuGetDependencies" = $true - "dependencyIdTemplate" = "AL-Go-{id}" - "packageId" = "AL-Go-{id}" } $parameters.appFiles = $_.FullName $package = New-BcNuGetPackage @parameters From c1a27cc10f994c909db5531052bc70d798f6f64a Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 16 Dec 2023 08:50:23 +0100 Subject: [PATCH 63/90] nuget test --- Actions/RunPipeline/RunPipeline.ps1 | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index a60a20265..558e82802 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -301,9 +301,6 @@ try { $version = [System.Version]$version.SubString(0,$version.Length-4) $publishParams = @{ "nuGetServerUrl" = $gitHubPackagesCredential.serverUrl - "nuGetToken" = $gitHubPackagesCredential.token - "packageName" = "AL-Go-$appId" - "version" = $version } if ($parameters.ContainsKey('CopyInstalledAppsToFolder')) { $publishParams += @{ @@ -311,10 +308,10 @@ try { } } if ($parameters.ContainsKey('containerName')) { - Publish-BcNuGetPackageToContainer -containerName $parameters.containerName -tenant $parameters.tenant -skipVerification @publishParams + Publish-BcNuGetPackageToContainer -containerName $parameters.containerName -tenant $parameters.tenant -skipVerification @publishParams -verbose } else { - Copy-BcNuGetPackageToFolder -appSymbolsFolder $parameters.appSymbolsFolder @publishParams + Copy-BcNuGetPackageToFolder -appSymbolsFolder $parameters.appSymbolsFolder @publishParams -verbose } } From cb4fb8db2e33b1695017e3cf8dcf5289247d9805 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 16 Dec 2023 08:58:09 +0100 Subject: [PATCH 64/90] fix --- Actions/RunPipeline/RunPipeline.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index 558e82802..853c28fa3 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -301,6 +301,9 @@ try { $version = [System.Version]$version.SubString(0,$version.Length-4) $publishParams = @{ "nuGetServerUrl" = $gitHubPackagesCredential.serverUrl + "nuGetToken" = $gitHubPackagesCredential.token + "packageName" = $appId + "version" = $version } if ($parameters.ContainsKey('CopyInstalledAppsToFolder')) { $publishParams += @{ From 19fe9d776f81440c4ea4e6fb869cbc372e2922ce Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 16 Dec 2023 09:09:16 +0100 Subject: [PATCH 65/90] remove nuget fix --- Actions/Deliver/Deliver.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Actions/Deliver/Deliver.ps1 b/Actions/Deliver/Deliver.ps1 index a4ebefddd..b10aac177 100644 --- a/Actions/Deliver/Deliver.ps1 +++ b/Actions/Deliver/Deliver.ps1 @@ -218,6 +218,9 @@ try { Get-Item -Path (Join-Path $folder[0] "*.app") | ForEach-Object { $parameters = @{ "gitHubRepository" = "$ENV:GITHUB_SERVER_URL/$ENV:GITHUB_REPOSITORY" + "includeNuGetDependencies" = $true + "dependencyIdTemplate" = "AL-Go-{id}" + "packageId" = "AL-Go-{id}" } $parameters.appFiles = $_.FullName $package = New-BcNuGetPackage @parameters From fc61cdfd5c5adbee060efca3c786ec054844a62b Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 16 Dec 2023 09:09:45 +0100 Subject: [PATCH 66/90] remove nuget fox --- Actions/RunPipeline/RunPipeline.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Actions/RunPipeline/RunPipeline.ps1 b/Actions/RunPipeline/RunPipeline.ps1 index 853c28fa3..a60a20265 100644 --- a/Actions/RunPipeline/RunPipeline.ps1 +++ b/Actions/RunPipeline/RunPipeline.ps1 @@ -302,7 +302,7 @@ try { $publishParams = @{ "nuGetServerUrl" = $gitHubPackagesCredential.serverUrl "nuGetToken" = $gitHubPackagesCredential.token - "packageName" = $appId + "packageName" = "AL-Go-$appId" "version" = $version } if ($parameters.ContainsKey('CopyInstalledAppsToFolder')) { @@ -311,10 +311,10 @@ try { } } if ($parameters.ContainsKey('containerName')) { - Publish-BcNuGetPackageToContainer -containerName $parameters.containerName -tenant $parameters.tenant -skipVerification @publishParams -verbose + Publish-BcNuGetPackageToContainer -containerName $parameters.containerName -tenant $parameters.tenant -skipVerification @publishParams } else { - Copy-BcNuGetPackageToFolder -appSymbolsFolder $parameters.appSymbolsFolder @publishParams -verbose + Copy-BcNuGetPackageToFolder -appSymbolsFolder $parameters.appSymbolsFolder @publishParams } } From eef783592ef372f51be5caba2f5f669d3e7c8922 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 16 Dec 2023 09:23:05 +0100 Subject: [PATCH 67/90] dup --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 492e06735..338f43178 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -28,6 +28,8 @@ function DownloadTemplateRepository { if ($downloadLatest) { # Get Branches from template repository + Write-Host $apiUrl + $headers | out-host $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches" -retry $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch } if (!$branchInfo) { From c951150d2fae37a324353656c4a0df7bf59bfde9 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 16 Dec 2023 09:26:12 +0100 Subject: [PATCH 68/90] get 100 branches for direct AL-Go --- Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 338f43178..d51830b92 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -28,9 +28,7 @@ function DownloadTemplateRepository { if ($downloadLatest) { # Get Branches from template repository - Write-Host $apiUrl - $headers | out-host - $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches" -retry + $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches?per_page=100" -retry $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch } if (!$branchInfo) { throw "$templateUrl doesn't exist" From 274ec66d9cf438b40e8a752e00464ef85df99045 Mon Sep 17 00:00:00 2001 From: freddydk Date: Sat, 13 Apr 2024 23:18:16 +0200 Subject: [PATCH 69/90] release notes --- RELEASENOTES.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index c241ece1d..7a03ab3da 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -26,6 +26,9 @@ Note that when using the preview version of AL-Go for GitHub, we recommend you U - `appSourceMainAppFolder` is moved to the `deliverToAppSource` structure - `appSourceProductId` is moved to the `deliverToAppSource` structure +### Business Central Performance Toolkit Test Result Viewer +- In the summary after a Test Run, you now also have the result of performance tests. + ## v5.0 ### Issues @@ -204,9 +207,6 @@ Now, you can set the checkbox called Use GhTokenWorkflow to allowing you to use ### New Actions - `DownloadProjectDependencies`: Downloads the dependency apps for a given project and build mode. -### Business Central Performance Toolkit Test Result Viewer -- In the summary after a Test Run, you now also have the result of performance tests. - ### Settings and Secrets in AL-Go for GitHub In earlier versions of AL-Go for GitHub, all settings were available as individual environment variables to scripts and overrides, this is no longer the case. Settings were also available as one compressed JSON structure in env:Settings, this is still the case. @@ -219,7 +219,6 @@ All requested secrets were also available (base64 encoded) as one compressed JSO Use `$secrets = $ENV:Secrets | ConvertFrom-Json` to get all requested secrets in PowerShell. You cannot get to any secrets that weren't requested by AL-Go for GitHub. - ## v3.1 ### Issues From 6a3fe1a9891c66958bd2bedb57105908568b759e Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 14 May 2024 12:10:26 +0200 Subject: [PATCH 70/90] release notes --- RELEASENOTES.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index d62f3b488..c5fedaea4 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -29,10 +29,6 @@ > [!NOTE] > PowerPlatform functionality are only available in the PTE template. -### New parameter -clean on localdevenv and clouddevenv - -Adding -clean when running localdevenv or clouddevenv will create a clean development environment without compiling and publishing your apps. - ### Business Central Performance Toolkit Test Result Viewer In the summary after a Test Run, you now also have the result of performance tests. @@ -77,6 +73,10 @@ The artifact setting in your project settings file can now contain a `*` instead - `appSourceMainAppFolder` is moved to the `deliverToAppSource` structure - `appSourceProductId` is moved to the `deliverToAppSource` structure +### New parameter -clean on localdevenv and clouddevenv + +Adding -clean when running localdevenv or clouddevenv will create a clean development environment without compiling and publishing your apps. + ## v5.0 ### Issues From bc4a60dc842b40e1c3921dd4bcb593f9aedaa1f4 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 14 May 2024 12:27:44 +0200 Subject: [PATCH 71/90] bcptThresholds setting --- Actions/AL-Go-Helper.ps1 | 10 ++++++---- Actions/AnalyzeTests/AnalyzeTests.ps1 | 8 ++++---- RELEASENOTES.md | 16 ++++++++++++---- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index 881235b97..010687240 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -638,10 +638,12 @@ function ReadSettings { "buildModes" = @() "useCompilerFolder" = $false "pullRequestTrigger" = "pull_request_target" - "DurationThresholdWarning" = 10 - "DurationThresholdError" = 25 - "NumberOfSqlStmtsThresholdWarning" = 5 - "NumberOfSqlStmtsThresholdError" = 10 + "bcptThresholds" = [ordered]@{ + "DurationWarning" = 10 + "DurationError" = 25 + "NumberOfSqlStmtsWarning" = 5 + "NumberOfSqlStmtsError" = 10 + } "fullBuildPatterns" = @() "excludeEnvironments" = @() "alDoc" = [ordered]@{ diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index aa8393e34..4e56622ca 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -27,10 +27,10 @@ try { -path $bcptTestResultsFile ` -baseLinePath $bcptBaseLineFile ` -thresholdsPath $bcptThresholdsFile ` - -DurationThresholdWarning $settings.DurationThresholdWarning ` - -DurationThresholdError $settings.DurationThresholdError ` - -NumberOfSqlStmtsThresholdWarning $settings.NumberOfSqlStmtsThresholdWarning ` - -NumberOfSqlStmtsThresholdError $settings.NumberOfSqlStmtsThresholdError + -DurationThresholdWarning $settings.bcptThresholds.DurationdWarning ` + -DurationThresholdError $settings.bcptThresholds.DurationError ` + -NumberOfSqlStmtsThresholdWarning $settings.bcptThresholds.NumberOfSqlStmtsWarning ` + -NumberOfSqlStmtsThresholdError $settings.bcptThresholds.NumberOfSqlStmtsError # If summary fits, we will display it in the GitHub summary if ($testResultsSummaryMD.Length -gt 65000) { diff --git a/RELEASENOTES.md b/RELEASENOTES.md index c5fedaea4..2deb24c65 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,7 +1,19 @@ +### Business Central Performance Toolkit Test Result Viewer + +In the summary after a Test Run, you now also have the result of performance tests. + ### New Settings - `PowerPlatformSolutionFolder`: Contains the name of the folder containing a PowerPlatform Solution (only one) - `DeployTo` now has two additional properties `companyId` is the Company Id from Business Central (for PowerPlatform connection) and `ppEnvironmentUrl` is the Url of the PowerPlatform environment to deploy to. +- `bcptThresholds` is a JSON object with properties for the default thresholds for the Business Central Performance Toolkit + - **DurationWarning** - a warning is issued if the duration of a bcpt test degrades more than this percentage (default 10) + - **DurationError** - an error is issued if the duration of a bcpt test degrades more than this percentage (default 25) + - **NumberOfSqlStmtsWarning** - a warning is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 5) + - **NumberOfSqlStmtsError** - an error is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 10) + +> [!NOTE] +> Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. ### New Actions @@ -29,10 +41,6 @@ > [!NOTE] > PowerPlatform functionality are only available in the PTE template. -### Business Central Performance Toolkit Test Result Viewer - -In the summary after a Test Run, you now also have the result of performance tests. - ## v5.1 ### Issues From 90d75e084963690fdc3585e5e0008a37f3bcbef2 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 14 May 2024 13:09:35 +0200 Subject: [PATCH 72/90] use hashtable --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 6 ++-- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 38 +++++++++++---------- Tests/AnalyzeTests.Test.ps1 | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index 4e56622ca..bb7ce69f2 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -20,6 +20,7 @@ try { $testResultsSummaryMD, $testResultsfailuresMD, $testResultsFailuresSummaryMD = GetTestResultSummaryMD -path $testResultsFile $settings = $env:Settings | ConvertFrom-Json + Write-Host "thresholds type: $($settings.bcptThresholds.GetType().FullName)" $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptTestResults.json" $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptBaseLine.json" $bcptThresholdsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptThresholds.json" @@ -27,10 +28,7 @@ try { -path $bcptTestResultsFile ` -baseLinePath $bcptBaseLineFile ` -thresholdsPath $bcptThresholdsFile ` - -DurationThresholdWarning $settings.bcptThresholds.DurationdWarning ` - -DurationThresholdError $settings.bcptThresholds.DurationError ` - -NumberOfSqlStmtsThresholdWarning $settings.bcptThresholds.NumberOfSqlStmtsWarning ` - -NumberOfSqlStmtsThresholdError $settings.bcptThresholds.NumberOfSqlStmtsError + -bcptThresholds ($settings.bcptThresholds | ConvertTo-HashTable) # If summary fits, we will display it in the GitHub summary if ($testResultsSummaryMD.Length -gt 65000) { diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index cf8785b4e..0d7dc347c 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -158,10 +158,12 @@ function GetBcptSummaryMD { [string] $baseLinePath = '', [string] $thresholdsPath = '', [int] $skipMeasurements = 0, - [int] $DurationThresholdWarning = 10, - [int] $DurationThresholdError = 25, - [int] $NumberOfSqlStmtsThresholdWarning = 5, - [int] $NumberOfSqlStmtsThresholdError = 10 + [hashtable] $bcptThresholds = @{ + [int] $durationWarning = 10 + [int] $durationError = 25 + [int] $numberOfSqlStmtsWarning = 5 + [int] $numberOfSqlStmtsError = 10 + } ) $bcpt = ReadBcptFile -path $path @@ -173,17 +175,17 @@ function GetBcptSummaryMD { if ($thresholdsPath -and (Test-Path -path $thresholdsPath)) { Write-Host "Reading thresholds from $thresholdsPath" $thresholds = Get-Content -Path $thresholdsPath -Encoding UTF8 | ConvertFrom-Json - foreach($threshold in 'DurationThresholdWarning', 'DurationThresholdError', 'NumberOfSqlStmtsThresholdWarning', 'NumberOfSqlStmtsThresholdError') { + foreach($threshold in 'durationWarning', 'durationError', 'numberOfSqlStmtsWarning', 'numberOfSqlStmtsError') { if ($thresholds.PSObject.Properties.Name -eq $threshold) { - Set-Variable -Name $threshold -Value $thresholds."$threshold" -Scope local + $bcptThresholds."$threshold" = $thresholds."$threshold" } } } Write-Host "Using thresholds:" - Write-Host "- DurationThresholdWarning: $DurationThresholdWarning" - Write-Host "- DurationThresholdError: $DurationThresholdError" - Write-Host "- NumberOfSqlStmtsThresholdWarning: $NumberOfSqlStmtsThresholdWarning" - Write-Host "- NumberOfSqlStmtsThresholdError: $NumberOfSqlStmtsThresholdError" + Write-Host "- DurationWarning: $($bcptThresholds.durationWarning)" + Write-Host "- DurationError: $($bcptThresholds.durationError)" + Write-Host "- NumberOfSqlStmtsWarning: $($bcptThresholds.numberOfSqlStmtsWarning)" + Write-Host "- NumberOfSqlStmtsError: $($bcptThresholds.numberOfSqlStmtsError)" $summarySb = [System.Text.StringBuilder]::new() if ($baseLine) { @@ -267,33 +269,33 @@ function GetBcptSummaryMD { } else { $statusStr = $statusOK - if ($pctDurationMin -ge $DurationThresholdError) { + if ($pctDurationMin -ge $bcptThresholds.durationError) { $statusStr = $statusError if ($thisCodeunitName) { # Only give errors and warnings on top level operation - OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($DurationThresholdError)% for duration" + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($bcptThresholds.durationError)% for duration" } } - if ($pctNumberOfSQLStmts -ge $NumberOfSqlStmtsThresholdError) { + if ($pctNumberOfSQLStmts -ge $bcptThresholds.numberOfSqlStmtsError) { $statusStr = $statusError if ($thisCodeunitName) { # Only give errors and warnings on top level operation - OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($NumberOfSqlStmtsThresholdError)% for number of SQL statements" + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($bcptThresholds.numberOfSqlStmtsError)% for number of SQL statements" } } if ($statusStr -eq $statusOK) { - if ($pctDurationMin -ge $DurationThresholdWarning) { + if ($pctDurationMin -ge $bcptThresholds.durationWarning) { $statusStr = $statusWarning if ($thisCodeunitName) { # Only give errors and warnings on top level operation - OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($DurationThresholdWarning)% for duration" + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($bcptThresholds.durationWarning)% for duration" } } - if ($pctNumberOfSQLStmts -ge $NumberOfSqlStmtsThresholdWarning) { + if ($pctNumberOfSQLStmts -ge $bcptThresholds.numberOfSqlStmtsWarning) { $statusStr = $statusWarning if ($thisCodeunitName) { # Only give errors and warnings on top level operation - OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($NumberOfSqlStmtsThresholdWarning)% for number of SQL statements" + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($bcptThresholds.numberOfSqlStmtsWarning)% for number of SQL statements" } } } diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 4dff4ef63..821dc0dfe 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -121,7 +121,7 @@ Describe "AnalyzeTests Action Tests" { $script:warningCount = 0 Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } - $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -durationThresholdWarning 5 -durationThresholdError 10 + $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -bcptThresholds @{"durationWarning"=5;"durationError"=10;"numberOfSqlStmtsWarning"=5;"numberOfSqlStmtsError"=10} Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 13 From 2d534b9ae6a65ae9c0ae503b7168e81baadbed7d Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 08:57:56 +0200 Subject: [PATCH 73/90] default --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 0d7dc347c..2f8c52103 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -159,10 +159,10 @@ function GetBcptSummaryMD { [string] $thresholdsPath = '', [int] $skipMeasurements = 0, [hashtable] $bcptThresholds = @{ - [int] $durationWarning = 10 - [int] $durationError = 25 - [int] $numberOfSqlStmtsWarning = 5 - [int] $numberOfSqlStmtsError = 10 + "durationWarning" = 10 + "durationError" = 25 + "numberOfSqlStmtsWarning" = 5 + "numberOfSqlStmtsError" = 10 } ) From 5a4c4b6a149b9b0e57628202ac65e23fbe82daf5 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 10:16:34 +0200 Subject: [PATCH 74/90] doc --- Scenarios/AddAPerformanceTestApp.md | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index d57f04ae8..dce20c892 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -2,15 +2,32 @@ *Prerequisites: A completed [scenario 1](GetStarted.md)* 1. On **github.com**, open **Actions** on your solution, select **Create a new performance test app** and then choose **Run workflow**. Enter values for **name**, **publisher**, and **ID range** and choose **Run workflow** -![Run Workflow](https://github.com/microsoft/AL-Go/assets/10775043/40f9bda7-578b-4844-9c2b-f59200d04584) + + ![Run workflow](https://github.com/microsoft/AL-Go/assets/10775043/d499294e-8c88-4f2d-9bb4-b34bad276a6b) + + > [!NOTE] + > If workflows are not allowed to create pull requests due to GitHub Settings, you can create the PR manually by following the link in the annotation + > + > ![Annotation](https://github.com/microsoft/AL-Go/assets/10775043/d346f0fc-5db4-4ff1-9c76-e93cb03ae504) + 1. When the workflow is done, navigate to **Pull Requests**, **inspect the PR** and **Merge the pull request** -![Pull Request](https://github.com/microsoft/AL-Go/assets/10775043/e97ef897-93f4-4c9f-9ce2-e747d7021003) + + ![Pull Request](https://github.com/microsoft/AL-Go/assets/10775043/d2831620-3bc9-4808-aa7a-997944aaaa33) + 1. Under **Actions**, you will see that a Merge pull request **CI workflow** has been kicked off -![Workflows](https://github.com/microsoft/AL-Go/assets/10775043/90ee2dee-4e2a-4d16-80bc-63d3ce1f53b5) + + ![Workflows](https://github.com/microsoft/AL-Go/assets/10775043/37f6c5b9-aaac-4cdc-b1d0-ef661cd2bfbe) + 1. If you wait for the workflow to complete, you will see that it completes and one of the build artifacts are the **BCPT Test Results** -![Fail](https://github.com/microsoft/AL-Go/assets/10775043/ad154e32-34d4-49f1-a8de-e74ed5a79217) + + ![BCPT Test Results](https://github.com/microsoft/AL-Go/assets/10775043/cb206f91-3b83-4000-987c-39faa9765695) + 1. Opening the **BCPT Test Results** and inspecting the results looks like this -![Test failure](https://github.com/microsoft/AL-Go/assets/10775043/0869601d-55e6-4e1d-9d1e-fb1a2c0c6b05) + + ![Test failure](https://github.com/microsoft/AL-Go/assets/10775043/0869601d-55e6-4e1d-9d1e-fb1a2c0c6b05) + +1. Scrolling down further reveals the Performance Test Results + 1. Currently there isn't a visual viewer of these results. The goal is to have a PowerBI dashboard, which can gather BCPT test results from multiple builds and compare. --- From a576434796f8965009d0638b9bc3753138655b08 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 11:00:40 +0200 Subject: [PATCH 75/90] documentation --- Scenarios/AddAPerformanceTestApp.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index dce20c892..fed9074a8 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -24,11 +24,17 @@ 1. Opening the **BCPT Test Results** and inspecting the results looks like this - ![Test failure](https://github.com/microsoft/AL-Go/assets/10775043/0869601d-55e6-4e1d-9d1e-fb1a2c0c6b05) + ![BCPT Test Results.json](https://github.com/microsoft/AL-Go/assets/10775043/27acb70c-1ead-4832-b22a-b022c578250d) -1. Scrolling down further reveals the Performance Test Results +1. Scrolling down further reveals the Performance Test Results in a table, which also indicates that if we want to set a baseline for comparing future BCPT Test Results, we need to add a `bcptBaseLine.json` file in the project folder. -1. Currently there isn't a visual viewer of these results. The goal is to have a PowerBI dashboard, which can gather BCPT test results from multiple builds and compare. + ![BCPT Test Results viewer](https://github.com/microsoft/AL-Go/assets/10775043/4b263e9e-7ec9-4101-92a7-046e7807e797) + +1. After uploading the `bcptBaseLin.json`, to the project root (which is the repo root in single project repositories), another CI/CD workflow will complete and now compare the results with the baseline: + + ![With BaseLine](https://github.com/microsoft/AL-Go/assets/10775043/c00840d5-4c67-4a72-a4d9-cdebe62e54c0) + + Where negative numbers in the diff fields indicates faster execution or lower number of SQL statements than the baseline. --- [back](../README.md) From 6420e7e88314377790dcc7b81aae043f343176ff Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 11:01:36 +0200 Subject: [PATCH 76/90] line --- Scenarios/AddAPerformanceTestApp.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index fed9074a8..6cec32aec 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -6,6 +6,7 @@ ![Run workflow](https://github.com/microsoft/AL-Go/assets/10775043/d499294e-8c88-4f2d-9bb4-b34bad276a6b) > [!NOTE] + > > If workflows are not allowed to create pull requests due to GitHub Settings, you can create the PR manually by following the link in the annotation > > ![Annotation](https://github.com/microsoft/AL-Go/assets/10775043/d346f0fc-5db4-4ff1-9c76-e93cb03ae504) From 5c3aff1a4f005ce825973cd2c41a4d384a9647ec Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 11:02:31 +0200 Subject: [PATCH 77/90] note --- Scenarios/AddAPerformanceTestApp.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index 6cec32aec..0f7ac2ff8 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -5,11 +5,11 @@ ![Run workflow](https://github.com/microsoft/AL-Go/assets/10775043/d499294e-8c88-4f2d-9bb4-b34bad276a6b) - > [!NOTE] - > - > If workflows are not allowed to create pull requests due to GitHub Settings, you can create the PR manually by following the link in the annotation - > - > ![Annotation](https://github.com/microsoft/AL-Go/assets/10775043/d346f0fc-5db4-4ff1-9c76-e93cb03ae504) +> [!NOTE] +> +> If workflows are not allowed to create pull requests due to GitHub Settings, you can create the PR manually by following the link in the annotation +> +> ![Annotation](https://github.com/microsoft/AL-Go/assets/10775043/d346f0fc-5db4-4ff1-9c76-e93cb03ae504) 1. When the workflow is done, navigate to **Pull Requests**, **inspect the PR** and **Merge the pull request** From e620abedab946bc407311dbffbef03b8e6fed5b8 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 11:03:44 +0200 Subject: [PATCH 78/90] doc --- Scenarios/AddAPerformanceTestApp.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index 0f7ac2ff8..689855e7d 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -5,11 +5,9 @@ ![Run workflow](https://github.com/microsoft/AL-Go/assets/10775043/d499294e-8c88-4f2d-9bb4-b34bad276a6b) -> [!NOTE] -> -> If workflows are not allowed to create pull requests due to GitHub Settings, you can create the PR manually by following the link in the annotation -> -> ![Annotation](https://github.com/microsoft/AL-Go/assets/10775043/d346f0fc-5db4-4ff1-9c76-e93cb03ae504) + Note that if workflows are not allowed to create pull requests due to GitHub Settings, you can create the PR manually by following the link in the annotation + + ![Annotation](https://github.com/microsoft/AL-Go/assets/10775043/d346f0fc-5db4-4ff1-9c76-e93cb03ae504) 1. When the workflow is done, navigate to **Pull Requests**, **inspect the PR** and **Merge the pull request** From 06f5a3adf3eb01c7e97f219728248948990821f7 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 12:51:21 +0200 Subject: [PATCH 79/90] scenario --- Scenarios/AddAPerformanceTestApp.md | 29 +++++++++++++++++++++++------ Scenarios/settings.md | 1 + 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index 689855e7d..537b78c1a 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -9,31 +9,48 @@ ![Annotation](https://github.com/microsoft/AL-Go/assets/10775043/d346f0fc-5db4-4ff1-9c76-e93cb03ae504) -1. When the workflow is done, navigate to **Pull Requests**, **inspect the PR** and **Merge the pull request** +2. When the workflow is done, navigate to **Pull Requests**, **inspect the PR** and **Merge the pull request** ![Pull Request](https://github.com/microsoft/AL-Go/assets/10775043/d2831620-3bc9-4808-aa7a-997944aaaa33) -1. Under **Actions**, you will see that a Merge pull request **CI workflow** has been kicked off +3. Under **Actions**, you will see that a Merge pull request **CI workflow** has been kicked off ![Workflows](https://github.com/microsoft/AL-Go/assets/10775043/37f6c5b9-aaac-4cdc-b1d0-ef661cd2bfbe) -1. If you wait for the workflow to complete, you will see that it completes and one of the build artifacts are the **BCPT Test Results** +4. If you wait for the workflow to complete, you will see that it completes and one of the build artifacts are the **BCPT Test Results** ![BCPT Test Results](https://github.com/microsoft/AL-Go/assets/10775043/cb206f91-3b83-4000-987c-39faa9765695) -1. Opening the **BCPT Test Results** and inspecting the results looks like this +5. Opening the **BCPT Test Results** and inspecting the results looks like this ![BCPT Test Results.json](https://github.com/microsoft/AL-Go/assets/10775043/27acb70c-1ead-4832-b22a-b022c578250d) -1. Scrolling down further reveals the Performance Test Results in a table, which also indicates that if we want to set a baseline for comparing future BCPT Test Results, we need to add a `bcptBaseLine.json` file in the project folder. +6. Scrolling down further reveals the Performance Test Results in a table, which also indicates that if we want to set a baseline for comparing future BCPT Test Results, we need to add a `bcptBaseLine.json` file in the project folder. ![BCPT Test Results viewer](https://github.com/microsoft/AL-Go/assets/10775043/4b263e9e-7ec9-4101-92a7-046e7807e797) -1. After uploading the `bcptBaseLin.json`, to the project root (which is the repo root in single project repositories), another CI/CD workflow will complete and now compare the results with the baseline: +7. After uploading a `bcptBaseLin.json`, to the project root (which is the repo root in single project repositories), another CI/CD workflow will be kicked off, which now compares the results with the baseline: ![With BaseLine](https://github.com/microsoft/AL-Go/assets/10775043/c00840d5-4c67-4a72-a4d9-cdebe62e54c0) Where negative numbers in the diff fields indicates faster execution or lower number of SQL statements than the baseline. +> [!NOTE] +> +> You can specify thresholds for performance testing in project settings (see [https://aka.ms/algosettings#bcptThresholds](https://aka.ms/algosettings#bcptThresholds)) or in a file called `bcptThresholds.json`, which should be located next to the `bcptBaseLine.json` file. + +8. After uploading a `bcptThresholds.json` file with this content: + + ``` + { + "durationWarning": 0, + "durationError": 1 + } + ``` + + The CI/CD workflow now uses these thresholds for the CI/CD run: + + ![Warnings and Error](https://github.com/microsoft/AL-Go/assets/10775043/be85d4c1-c710-410d-aba3-b55de8750396) + --- [back](../README.md) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 8c0851084..68a8c3f73 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -33,6 +33,7 @@ When running a workflow or a local script, the settings are applied by reading s | bcptTestFolders | bcptTestFolders should be an array of folders (relative to project root), which contains performance test apps for this project. Apps in these folders are sorted based on dependencies and built, published and bcpt tests are run in that order.
If bcptTestFolders are not specified, AL-Go for GitHub will try to locate bcptTestFolders in the root of the project. | [ ] | | appDependencyProbingPaths | Array of dependency specifications, from which apps will be downloaded when the CI/CD workflow is starting. Every dependency specification consists of the following properties:
**repo** = repository
**version** = version (default latest)
**release_status** = latestBuild/release/prerelease/draft (default release)
**projects** = projects (default * = all)
**branch** = branch (default main)
**AuthTokenSecret** = Name of secret containing auth token (default none)
| [ ] | | cleanModePreprocessorSymbols | List of clean tags to be used in _Clean_ build mode | [ ] | +| bcptThresholds | Structure with properties for the thresholds when running performance tests using the Business Central Performance Toolkit:
**DurationWarning** = a warning is issued if the duration of a bcpt test degrades more than this percentage (default 10)
**DurationError** - an error is issued if the duration of a bcpt test degrades more than this percentage (default 25)
**NumberOfSqlStmtsWarning** - a warning is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 5)
**NumberOfSqlStmtsError** - an error is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 10)
| ## AppSource specific basic project settings | Name | Description | Default value | From 7f4e00c9b2511ce958f355dd38e12798555bffde Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 16:13:41 +0200 Subject: [PATCH 80/90] review --- Actions/AnalyzeTests/AnalyzeTests.ps1 | 5 +- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 52 ++++++++++----------- Tests/AnalyzeTests.Test.ps1 | 8 ++-- 3 files changed, 32 insertions(+), 33 deletions(-) diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index bb7ce69f2..1b6589cca 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -17,15 +17,14 @@ try { . (Join-Path -Path $PSScriptRoot 'TestResultAnalyzer.ps1') $testResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\TestResults.xml" - $testResultsSummaryMD, $testResultsfailuresMD, $testResultsFailuresSummaryMD = GetTestResultSummaryMD -path $testResultsFile + $testResultsSummaryMD, $testResultsfailuresMD, $testResultsFailuresSummaryMD = GetTestResultSummaryMD -testResultsFile $testResultsFile $settings = $env:Settings | ConvertFrom-Json - Write-Host "thresholds type: $($settings.bcptThresholds.GetType().FullName)" $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptTestResults.json" $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptBaseLine.json" $bcptThresholdsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptThresholds.json" $bcptSummaryMD = GetBcptSummaryMD ` - -path $bcptTestResultsFile ` + -bcptTestResultsFile $bcptTestResultsFile ` -baseLinePath $bcptBaseLineFile ` -thresholdsPath $bcptThresholdsFile ` -bcptThresholds ($settings.bcptThresholds | ConvertTo-HashTable) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index 2f8c52103..c3de7552b 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -9,13 +9,13 @@ $statusSkipped = " :question:" # Returns both a summary part and a failures part function GetTestResultSummaryMD { Param( - [string] $path + [string] $testResultsFile ) $summarySb = [System.Text.StringBuilder]::new() $failuresSb = [System.Text.StringBuilder]::new() - if (Test-Path -Path $path -PathType Leaf) { - $testResults = [xml](Get-Content -path "$project\TestResults.xml" -Encoding UTF8) + if (Test-Path -Path $testResultsFile -PathType Leaf) { + $testResults = [xml](Get-Content -path $testResultsFile -Encoding UTF8) $totalTests = 0 $totalTime = 0.0 $totalFailed = 0 @@ -109,15 +109,15 @@ function GetTestResultSummaryMD { function ReadBcptFile { Param( - [string] $path + [string] $bcptTestResultsFile ) - if ((-not $path) -or (-not (Test-Path -Path $path -PathType Leaf))) { + if ((-not $bcptTestResultsFile) -or (-not (Test-Path -Path $bcptTestResultsFile -PathType Leaf))) { return $null } # Read BCPT file - $bcptResult = Get-Content -Path $path -Encoding UTF8 | ConvertFrom-Json + $bcptResult = Get-Content -Path $bcptTestResultsFile -Encoding UTF8 | ConvertFrom-Json $suites = [ordered]@{} # Sort by bcptCode, codeunitID, operation foreach($measure in $bcptResult) { @@ -154,38 +154,38 @@ function ReadBcptFile { function GetBcptSummaryMD { Param( - [string] $path, + [string] $bcptTestResultsFile, [string] $baseLinePath = '', [string] $thresholdsPath = '', [int] $skipMeasurements = 0, - [hashtable] $bcptThresholds = @{ - "durationWarning" = 10 - "durationError" = 25 - "numberOfSqlStmtsWarning" = 5 - "numberOfSqlStmtsError" = 10 - } + [hashtable] $bcptThresholds = $null ) - $bcpt = ReadBcptFile -path $path + $bcpt = ReadBcptFile -bcptTestResultsFile $bcptTestResultsFile if (-not $bcpt) { return '' } $baseLine = ReadBcptFile -path $baseLinePath - # Override thresholds if thresholds file exists - if ($thresholdsPath -and (Test-Path -path $thresholdsPath)) { - Write-Host "Reading thresholds from $thresholdsPath" - $thresholds = Get-Content -Path $thresholdsPath -Encoding UTF8 | ConvertFrom-Json - foreach($threshold in 'durationWarning', 'durationError', 'numberOfSqlStmtsWarning', 'numberOfSqlStmtsError') { - if ($thresholds.PSObject.Properties.Name -eq $threshold) { - $bcptThresholds."$threshold" = $thresholds."$threshold" + if ($baseLine) { + if ($null -eq $bcptThresholds) { + throw "Thresholds must be provided when comparing to a baseline" + } + # Override thresholds if thresholds file exists + if ($thresholdsPath -and (Test-Path -path $thresholdsPath)) { + Write-Host "Reading thresholds from $thresholdsPath" + $thresholds = Get-Content -Path $thresholdsPath -Encoding UTF8 | ConvertFrom-Json + foreach($threshold in 'durationWarning', 'durationError', 'numberOfSqlStmtsWarning', 'numberOfSqlStmtsError') { + if ($thresholds.PSObject.Properties.Name -eq $threshold) { + $bcptThresholds."$threshold" = $thresholds."$threshold" + } } } + Write-Host "Using thresholds:" + Write-Host "- DurationWarning: $($bcptThresholds.durationWarning)" + Write-Host "- DurationError: $($bcptThresholds.durationError)" + Write-Host "- NumberOfSqlStmtsWarning: $($bcptThresholds.numberOfSqlStmtsWarning)" + Write-Host "- NumberOfSqlStmtsError: $($bcptThresholds.numberOfSqlStmtsError)" } - Write-Host "Using thresholds:" - Write-Host "- DurationWarning: $($bcptThresholds.durationWarning)" - Write-Host "- DurationError: $($bcptThresholds.durationError)" - Write-Host "- NumberOfSqlStmtsWarning: $($bcptThresholds.numberOfSqlStmtsWarning)" - Write-Host "- NumberOfSqlStmtsError: $($bcptThresholds.numberOfSqlStmtsError)" $summarySb = [System.Text.StringBuilder]::new() if ($baseLine) { diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 821dc0dfe..383533e89 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -74,7 +74,7 @@ Describe "AnalyzeTests Action Tests" { It 'Test ReadBcptFile' { . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') - $bcpt = ReadBcptFile -path $bcptFilename + $bcpt = ReadBcptFile -bcptTestResultsFile $bcptFilename $bcpt.Count | should -Be 1 $bcpt."SUITE1".Count | should -Be 2 $bcpt."SUITE1"."1".operations.Count | should -Be 5 @@ -84,7 +84,7 @@ Describe "AnalyzeTests Action Tests" { It 'Test GetBcptSummaryMD (no baseline)' { . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') - $md = GetBcptSummaryMD -path $bcptFilename + $md = GetBcptSummaryMD -bcptTestResultsFile $bcptFilename Write-Host $md.Replace('\n',"`n") $md | should -Match 'No baseline provided' $columns = 6 @@ -98,7 +98,7 @@ Describe "AnalyzeTests Action Tests" { It 'Test GetBcptSummaryMD (with worse baseline)' { . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') - $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine1 + $md = GetBcptSummaryMD -bcptTestResultsFile $bcptFilename -baselinePath $bcptBaseLine1 -bcptThresholds @{"durationWarning"=10;"durationError"=25;"numberOfSqlStmtsWarning"=5;"numberOfSqlStmtsError"=10} Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 13 @@ -121,7 +121,7 @@ Describe "AnalyzeTests Action Tests" { $script:warningCount = 0 Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } - $md = GetBcptSummaryMD -path $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -bcptThresholds @{"durationWarning"=5;"durationError"=10;"numberOfSqlStmtsWarning"=5;"numberOfSqlStmtsError"=10} + $md = GetBcptSummaryMD -bcptTestResultsFile $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -bcptThresholds @{"durationWarning"=5;"durationError"=10;"numberOfSqlStmtsWarning"=5;"numberOfSqlStmtsError"=10} Write-Host $md.Replace('\n',"`n") $md | should -Not -Match 'No baseline provided' $columns = 13 From 63db44784e0d1ff6ae7fdae0d83c15cf00aaaea4 Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Thu, 16 May 2024 16:17:00 +0200 Subject: [PATCH 81/90] Update Scenarios/AddAPerformanceTestApp.md Co-authored-by: Alexander Holstrup <117829001+aholstrup1@users.noreply.github.com> --- Scenarios/AddAPerformanceTestApp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index 537b78c1a..7a2aa6ccc 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -29,7 +29,7 @@ ![BCPT Test Results viewer](https://github.com/microsoft/AL-Go/assets/10775043/4b263e9e-7ec9-4101-92a7-046e7807e797) -7. After uploading a `bcptBaseLin.json`, to the project root (which is the repo root in single project repositories), another CI/CD workflow will be kicked off, which now compares the results with the baseline: +7. After uploading a `bcptBaseLine.json`, to the project root (which is the repo root in single project repositories), another CI/CD workflow will be kicked off, which now compares the results with the baseline: ![With BaseLine](https://github.com/microsoft/AL-Go/assets/10775043/c00840d5-4c67-4a72-a4d9-cdebe62e54c0) From cb10c891c63a7e98da8b365e9db40ef2adb98b54 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 16 May 2024 16:22:43 +0200 Subject: [PATCH 82/90] parameter name --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index c3de7552b..bdf8f47dc 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -165,7 +165,7 @@ function GetBcptSummaryMD { if (-not $bcpt) { return '' } - $baseLine = ReadBcptFile -path $baseLinePath + $baseLine = ReadBcptFile -bcptTestResultsFile $baseLinePath if ($baseLine) { if ($null -eq $bcptThresholds) { throw "Thresholds must be provided when comparing to a baseline" From bc1e51ef796fc907b73e3c91814ee2f2df4f8371 Mon Sep 17 00:00:00 2001 From: freddydk Date: Tue, 28 May 2024 09:35:11 +0200 Subject: [PATCH 83/90] review --- Scenarios/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 68a8c3f73..a62061bc3 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -33,7 +33,7 @@ When running a workflow or a local script, the settings are applied by reading s | bcptTestFolders | bcptTestFolders should be an array of folders (relative to project root), which contains performance test apps for this project. Apps in these folders are sorted based on dependencies and built, published and bcpt tests are run in that order.
If bcptTestFolders are not specified, AL-Go for GitHub will try to locate bcptTestFolders in the root of the project. | [ ] | | appDependencyProbingPaths | Array of dependency specifications, from which apps will be downloaded when the CI/CD workflow is starting. Every dependency specification consists of the following properties:
**repo** = repository
**version** = version (default latest)
**release_status** = latestBuild/release/prerelease/draft (default release)
**projects** = projects (default * = all)
**branch** = branch (default main)
**AuthTokenSecret** = Name of secret containing auth token (default none)
| [ ] | | cleanModePreprocessorSymbols | List of clean tags to be used in _Clean_ build mode | [ ] | -| bcptThresholds | Structure with properties for the thresholds when running performance tests using the Business Central Performance Toolkit:
**DurationWarning** = a warning is issued if the duration of a bcpt test degrades more than this percentage (default 10)
**DurationError** - an error is issued if the duration of a bcpt test degrades more than this percentage (default 25)
**NumberOfSqlStmtsWarning** - a warning is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 5)
**NumberOfSqlStmtsError** - an error is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 10)
| +| bcptThresholds | Structure with properties for the thresholds when running performance tests using the Business Central Performance Toolkit.
**DurationWarning** = a warning is shown if the duration of a bcpt test degrades more than this percentage (default 10)
**DurationError** - an error is shown if the duration of a bcpt test degrades more than this percentage (default 25)
**NumberOfSqlStmtsWarning** - a warning is shown if the number of SQL statements from a bcpt test increases more than this percentage (default 5)
**NumberOfSqlStmtsError** - an error is shown if the number of SQL statements from a bcpt test increases more than this percentage (default 10)
*Note that errors and warnings on the build in GitHub are only issued when a threshold is exceeded on the codeunit level, when an individual operation threshold is exceeded, it is only shown in the test results viewer.* | ## AppSource specific basic project settings | Name | Description | Default value | From e2210c25fb5117803bd164de70372651e7bb03eb Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Mon, 3 Jun 2024 23:21:37 +0200 Subject: [PATCH 84/90] Update Scenarios/AddAPerformanceTestApp.md Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Scenarios/AddAPerformanceTestApp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index 7a2aa6ccc..8c27fc9a7 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -13,7 +13,7 @@ ![Pull Request](https://github.com/microsoft/AL-Go/assets/10775043/d2831620-3bc9-4808-aa7a-997944aaaa33) -3. Under **Actions**, you will see that a Merge pull request **CI workflow** has been kicked off +3. Under **Actions**, you will see that a Merge pull request **CI/CD workflow** has been kicked off ![Workflows](https://github.com/microsoft/AL-Go/assets/10775043/37f6c5b9-aaac-4cdc-b1d0-ef661cd2bfbe) From 61cbaa7e7edaae9de7d87c65096d858e7126a44b Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 7 Jun 2024 10:05:09 +0200 Subject: [PATCH 85/90] move --- RELEASENOTES.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 958b9099d..a2c5f1dce 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,17 +1,11 @@ -## v5.2 - -### Issues -- Issue 1084 Automatic updates for AL-Go are failing when main branch requires Pull Request - ### Business Central Performance Toolkit Test Result Viewer In the summary after a Test Run, you now also have the result of performance tests. ### New Settings -- `PowerPlatformSolutionFolder`: Contains the name of the folder containing a PowerPlatform Solution (only one) -- `DeployTo` now has two additional properties `companyId` is the Company Id from Business Central (for PowerPlatform connection) and `ppEnvironmentUrl` is the Url of the PowerPlatform environment to deploy to. +of the PowerPlatform environment to deploy to. - `bcptThresholds` is a JSON object with properties for the default thresholds for the Business Central Performance Toolkit - **DurationWarning** - a warning is issued if the duration of a bcpt test degrades more than this percentage (default 10) - **DurationError** - an error is issued if the duration of a bcpt test degrades more than this percentage (default 25) @@ -21,6 +15,16 @@ In the summary after a Test Run, you now also have the result of performance tes > [!NOTE] > Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. +## v5.2 + +### Issues +- Issue 1084 Automatic updates for AL-Go are failing when main branch requires Pull Request + +### New Settings + +- `PowerPlatformSolutionFolder`: Contains the name of the folder containing a PowerPlatform Solution (only one) +- `DeployTo` now has two additional properties `companyId` is the Company Id from Business Central (for PowerPlatform connection) and `ppEnvironmentUrl` is the Url of the PowerPlatform environment to deploy to. + ### New Actions - `BuildPowerPlatform`: to build a PowerPlatform Solution From 1627b2ba67a7915f9ef187f17b380a8d0250b3fe Mon Sep 17 00:00:00 2001 From: freddydk Date: Mon, 17 Jun 2024 16:18:18 +0200 Subject: [PATCH 86/90] merge error --- RELEASENOTES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index a2c5f1dce..6ff757800 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -5,7 +5,6 @@ In the summary after a Test Run, you now also have the result of performance tes ### New Settings -of the PowerPlatform environment to deploy to. - `bcptThresholds` is a JSON object with properties for the default thresholds for the Business Central Performance Toolkit - **DurationWarning** - a warning is issued if the duration of a bcpt test degrades more than this percentage (default 10) - **DurationError** - an error is issued if the duration of a bcpt test degrades more than this percentage (default 25) From 954bfda81b40156b67905318a6f4eb5416053e9a Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Thu, 20 Jun 2024 13:24:27 +0200 Subject: [PATCH 87/90] Update Tests/AnalyzeTests.Test.ps1 Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Tests/AnalyzeTests.Test.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 index 383533e89..7f0b542b8 100644 --- a/Tests/AnalyzeTests.Test.ps1 +++ b/Tests/AnalyzeTests.Test.ps1 @@ -3,7 +3,6 @@ Import-Module (Join-Path $PSScriptRoot 'TestActionsHelper.psm1') Describe "AnalyzeTests Action Tests" { BeforeAll { - function GetBcptTestResultFile { Param( [int] $noOfSuites = 1, From 082a94c65b5e422b7316c6ccb1799d6842f1ffbb Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 20 Jun 2024 13:25:49 +0200 Subject: [PATCH 88/90] check appname --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index bdf8f47dc..fd3d298e5 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -38,7 +38,7 @@ function GetTestResultSummaryMD { $appTime = 0.0 $appFailed = 0 $appSkipped = 0 - $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Value -eq $appName } } + $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object $_.Name -eq 'appName' -and $_.Value -eq $appName } } foreach($suite in $suites) { $appTests += [int]$suite.tests $appFailed += [int]$suite.failures From f1dfe508bfa8243f57e30764ca6b2b9c6f6ff049 Mon Sep 17 00:00:00 2001 From: freddydk Date: Thu, 20 Jun 2024 13:34:30 +0200 Subject: [PATCH 89/90] syntax err --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index fd3d298e5..aa0106ff7 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -38,7 +38,7 @@ function GetTestResultSummaryMD { $appTime = 0.0 $appFailed = 0 $appSkipped = 0 - $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object $_.Name -eq 'appName' -and $_.Value -eq $appName } } + $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Name -eq 'appName' -and $_.Value -eq $appName } } foreach($suite in $suites) { $appTests += [int]$suite.tests $appFailed += [int]$suite.failures From 831529ab18c2db80bd1cc974f9002d5bcf927e77 Mon Sep 17 00:00:00 2001 From: freddydk Date: Fri, 21 Jun 2024 10:23:45 +0200 Subject: [PATCH 90/90] review --- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index aa0106ff7..f3efad58f 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -38,7 +38,7 @@ function GetTestResultSummaryMD { $appTime = 0.0 $appFailed = 0 $appSkipped = 0 - $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Name -eq 'appName' -and $_.Value -eq $appName } } + $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { ($_.Name -eq 'appName' -or $_.Name -eq 'extensionId') -and $_.Value -eq $appName } } foreach($suite in $suites) { $appTests += [int]$suite.tests $appFailed += [int]$suite.failures