Merge the 1.0.0-preview.7 release branch back to main

.github/workflows/dotnet.yml

@@ -20,7 +20,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: 8.0.411
+        dotnet-version: 8.0.413
     - name: Build
       shell: pwsh
       run: |
@@ -37,7 +37,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: 8.0.411
+        dotnet-version: 8.0.413
     - name: Build
       shell: pwsh
       run: |
@@ -54,7 +54,7 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: 8.0.411
+        dotnet-version: 8.0.413
     - name: Build
       shell: pwsh
       run: |

.pipelines/APIScan.yml

@@ -0,0 +1,101 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+name: apiscan-$(BUILD.SOURCEBRANCHNAME)-$(Build.BuildId)
+trigger: none
+
+parameters:
+  - name: FORCE_CODEQL
+    displayName: Debugging - Enable CodeQL and set cadence to 1 hour
+    type: boolean
+    default: false
+  - name: SkipVerifyPackages
+    type: boolean
+    default: false
+
+variables:
+  # PAT permissions NOTE: Declare a SymbolServerPAT variable in this group with a 'microsoft' organizanization scoped PAT with 'Symbols' Read permission.
+  # A PAT in the wrong org will give a single Error 203. No PAT will give a single Error 401, and individual pdbs may be missing even if permissions are correct.
+  - group: symbols
+  - name: ob_outputDirectory
+    value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
+  - name: CDP_DEFINITION_BUILD_COUNT
+    value: $[counter('', 0)]
+  # Defines the variables CgPat, CgOrganization, and CgProject
+  - group: 'ComponentGovernance'
+  - group: 'PoolNames'
+  - name: LinuxContainerImage
+    value: mcr.microsoft.com/onebranch/azurelinux/build:3.0
+  - name: WindowsContainerImage
+    value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest
+  - ${{ if eq(parameters['FORCE_CODEQL'],'true') }}:
+    # Cadence is hours before CodeQL will allow a re-upload of the database
+    - name: CodeQL.Cadence
+      value: 0
+  - name: CODEQL_ENABLED
+    ${{ if or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), eq(parameters['FORCE_CODEQL'],'true')) }}:
+      value: true
+    ${{ else }}:
+      value: false
+  - name: Codeql.TSAEnabled
+    value: $(CODEQL_ENABLED)
+  # AnalyzeInPipeline: false = upload results
+  # AnalyzeInPipeline: true = do not upload results
+  - name: Codeql.AnalyzeInPipeline
+    value: $(CODEQL_ENABLED)
+
+resources:
+  repositories:
+  - repository: templates
+    type: git
+    name: OneBranch.Pipelines/GovernedTemplates
+    ref: refs/heads/main
+
+extends:
+  template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
+  parameters:
+    featureFlags:
+      WindowsHostVersion:
+        Version: 2022
+    globalSdl:
+      codeql:
+        compiled:
+          enabled: $(CODEQL_ENABLED)
+        tsaEnabled: $(CODEQL_ENABLED) # This enables TSA bug filing only for CodeQL 3000
+      armory:
+        enabled: false
+      sbom:
+        enabled: false
+      cg:
+        enabled: true
+        ignoreDirectories: 'docs,shell,tools'
+      tsa:
+        enabled: true  # onebranch publish all SDL results to TSA. If TSA is disabled all SDL tools will forced into 'break' build mode.
+      credscan:
+        enabled: true
+        scanFolder:  $(Build.SourcesDirectory)
+      binskim:
+        break: true  # always break the build on binskim issues in addition to TSA upload
+      policheck:
+        break: true  # always break the build on policheck issues. You can disable it by setting to 'false'
+      # APIScan requires a non-Ready-To-Run build
+      apiscan:
+        enabled: true
+        softwareName: 'AIShell' # Default is repo name
+        versionNumber: '1.0' # Default is build number
+        isLargeApp: false # Default: false.
+        symbolsFolder: $(SymbolsServerUrl);$(ob_outputDirectory)
+      tsaOptionsFile: .config\tsaoptions.json
+      psscriptanalyzer:
+          enabled: true
+          policyName: Microsoft
+          break: false
+
+    stages:
+      - stage: APIScan
+        displayName: 'ApiScan'
+        dependsOn: []
+        jobs:
+          - template: /.pipelines/templates/compliance/apiscan-build.yaml@self
+            parameters:
+              parentJobs: []

.pipelines/Build-Official.yml

@@ -28,7 +28,7 @@ variables:
   - name: BUILDSECMON_OPT_IN
     value: true
   - name: LinuxContainerImage
-    value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
+    value: mcr.microsoft.com/onebranch/azurelinux/build:3.0
   - name: WindowsContainerImage
     value: onebranch.azurecr.io/windows/ltsc2022/vse2022:latest
   - name: CDP_DEFINITION_BUILD_COUNT
@@ -68,7 +68,8 @@ extends:
           enabled: true
           scanFolder: $(Build.SourcesDirectory)
         binskim:
-          enabled: false
+          enabled: true
+          exactToolVersion: 4.4.2
         apiscan:
           enabled: false
         tsaOptionsFile: .config\tsaoptions.json

.pipelines/Package-Official.yml

@@ -18,7 +18,7 @@ variables:
   - name: WindowsContainerImage
     value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' # Docker image which is used to build the project
   - name: LinuxContainerImage
-    value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
+    value: mcr.microsoft.com/onebranch/azurelinux/build:3.0
   - group: mscodehub-feed-read-general
   - group: mscodehub-feed-read-akv
   - name: branchCounterKey
@@ -71,7 +71,8 @@ extends:
           enabled: true
           scanFolder:  $(Build.SourcesDirectory)
         binskim:
-          enabled: false
+          enabled: true
+          exactToolVersion: 4.4.2
         apiscan:
           enabled: false
         tsaOptionsFile: .config\tsaoptions.json

.pipelines/Release-Official.yml

@@ -20,7 +20,7 @@ variables:
   - name: WindowsContainerImage
     value: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest'
   - name: LinuxContainerImage
-    value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
+    value: mcr.microsoft.com/onebranch/azurelinux/build:3.0
 
 resources:
   repositories:

.pipelines/templates/compliance/apiscan-build.yaml

@@ -0,0 +1,196 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+jobs:
+- job: APIScan
+  displayName: APIScan with fxdependent build
+  condition: succeeded()
+  pool:
+    type: windows
+  variables:
+  - name: runCodesignValidationInjection
+    value : false
+  - name: NugetSecurityAnalysisWarningLevel
+    value: none
+  # Defines the variables APIScanClient, APIScanTenant and APIScanSecret
+  - group: PS-PS-APIScan
+  - group: 'Azure Blob variable group'
+  - group: mscodehub-feed-read-general
+  - group: mscodehub-feed-read-akv
+  - name: ob_outputDirectory
+    value: '$(Build.ArtifactStagingDirectory)\ONEBRANCH_ARTIFACT'
+  - name: repoRoot
+    value: $(Build.SourcesDirectory)\AIShell
+  - name: ob_sdl_tsa_configFile
+    value: $(repoRoot)\.config\tsaoptions.json
+  - name: Codeql.SourceRoot
+    value: $(repoRoot)
+
+  # APIScan can take a long time
+  timeoutInMinutes: 180
+
+  steps:
+  - checkout: self
+    clean: true
+    fetchTags: true
+    fetchDepth: 1000
+    displayName: Checkout AIShell
+    retryCountOnTaskFailure: 1
+    env:
+      ob_restore_phase: true # This ensures checkout is done at the beginning of the restore phase
+
+  - template: ..\update-nuget-config.yml@self
+    parameters:
+      repoRoot: $(repoRoot)
+
+  - task: UseDotNet@2
+    displayName: 'Use .NET Core sdk'
+    inputs:
+      useGlobalJson: true
+      packageType: 'sdk'
+      workingDirectory: $(Build.SourcesDirectory)"
+
+  # - pwsh: |
+  #     dotnet tool install dotnet-symbol --tool-path $(Agent.ToolsDirectory)\tools\dotnet-symbol
+  #     $symbolToolPath = Get-ChildItem -Path $(Agent.ToolsDirectory)\tools\dotnet-symbol\dotnet-symbol.exe | Select-Object -First 1 -ExpandProperty FullName
+  #     Write-Host "##vso[task.setvariable variable=symbolToolPath]$symbolToolPath"
+  #   displayName: Install dotnet-symbol
+  #   workingDirectory: '$(repoRoot)'
+  #   retryCountOnTaskFailure: 2
+
+  - task: AzurePowerShell@5
+    displayName: Download winverify-private Artifacts
+    inputs:
+      azureSubscription: az-blob-cicd-infra
+      scriptType: inlineScript
+      azurePowerShellVersion: LatestVersion
+      workingDirectory: '$(repoRoot)'
+      pwsh: true
+      inline: |
+        # download smybols for getfilesiginforedist.dll
+        $storageAccountName = "pscoretestdata"
+        $containerName = 'winverify-private'
+        $winverifySymbolsPath = New-Item -ItemType Directory -Path '$(System.ArtifactsDirectory)\winverify-symbols' -Force
+        $dllName = 'getfilesiginforedist.dll'
+        $winverifySymbolsDllPath = Join-Path $winverifySymbolsPath $dllName
+
+        $context = New-AzStorageContext -StorageAccountName $storageAccountName -UseConnectedAccount
+        Get-AzStorageBlobContent -Container $containerName -Blob $dllName -Destination $winverifySymbolsDllPath -Context $context
+
+  - pwsh: |
+      Get-ChildItem -Path '$(System.ArtifactsDirectory)\winverify-symbols'
+    displayName: Capture winverify-private Artifacts
+    workingDirectory: '$(repoRoot)'
+    condition: succeededOrFailed()
+
+  - task: CodeQL3000Init@0 # Add CodeQL Init task right before your 'Build' step.
+    displayName: 🔏 CodeQL 3000 Init
+    condition: eq(variables['CODEQL_ENABLED'], 'true')
+    inputs:
+      Language: csharp
+
+  - pwsh: |
+      Import-Module '$(repoRoot)\build.psm1' -Force
+      Start-Build -Configuration StaticAnalysis -Runtime fxdependent -Clean -Verbose
+
+      $outputJson = '$(repoRoot)\_build_output_.json'
+      if (-not (Test-Path $outputJson)) {
+        throw "'_build_output_.json' was not produced."
+      }
+
+      $result = Get-Content $outputJson | ConvertFrom-Json
+      $OutputFolder = $result.App
+      Write-Verbose "App path: $OutputFolder" -Verbose
+
+      Write-Verbose -Verbose -Message "Deleting the 'ref' folder ..."
+      if (Test-Path $OutputFolder\ref) {
+        Remove-Item -Recurse -Force $OutputFolder\ref
+      }
+
+      # Only keep windows runtimes
+      Write-Verbose -Verbose -Message "Deleting non-win-x64 runtimes ..."
+      Get-ChildItem -Path "$OutputFolder\runtimes\*" | Where-Object {$_.FullName -notmatch '.*\\runtimes\\win'} | Foreach-Object {
+        Write-Verbose -Verbose -Message "Deleting $($_.FullName)"
+        Remove-Item -Path $_.FullName -Recurse -Force
+      }
+
+      # Remove win-x86/arm/arm64 runtimes due to issues with those runtimes
+      Write-Verbose -Verbose -Message "Temporarily deleting win-x86/arm/arm64 runtimes ..."
+      Get-ChildItem -Path "$OutputFolder\runtimes\*" | Where-Object {$_.FullName -match '.*\\runtimes\\win-(x86|arm)'} | Foreach-Object {
+        Write-Verbose -Verbose -Message "Deleting $($_.FullName)"
+        Remove-Item -Path $_.FullName -Recurse -Force
+      }
+
+      Write-Host
+      Write-Verbose -Verbose -Message "Show content in 'runtimes' folder:"
+      Get-ChildItem -Path "$OutputFolder\runtimes"
+      Write-Host
+
+      # Replace 'getfilesiginforedist.dll' from the 'runtimes\win-x64' folder
+      Write-Verbose -Verbose -Message "Replace 'getfilesiginforedist.dll':"
+      $targetFile = Get-ChildItem -Path "$OutputFolder\runtimes\*" -Recurse | Where-Object {$_.Name -eq 'getfilesiginforedist.dll'}
+      Remove-Item -Path $targetFile.FullName -Verbose
+      Copy-Item -Path '$(System.ArtifactsDirectory)\winverify-symbols\getfilesiginforedist.dll' -Destination $targetFile.FullName -Verbose
+
+      Write-Host "##vso[task.setvariable variable=appPath]$OutputFolder"
+    workingDirectory: '$(repoRoot)'
+    displayName: 'Build AIShell Source'
+
+  - pwsh: |
+      $Destination = '$(ob_outputDirectory)'
+      if (-not (Test-Path $Destination)) {
+        Write-Verbose -Verbose -Message "Creating destination folder '$Destination'"
+        $null = mkdir $Destination
+      }
+
+      Write-Verbose -Verbose -Message "Copy AIShell app to '$Destination':"
+      Copy-Item -Path '$(appPath)\*' -Destination $Destination -Recurse -Verbose
+
+      Write-Host
+      Write-Verbose -Verbose -Message "Show content in '$Destination':"
+      Get-ChildItem -Path $Destination | Out-String -width 150
+      Write-Host
+      Write-Verbose -Verbose -Message "Show content in '$Destination\runtimes':"
+      Get-ChildItem -Path "$Destination\runtimes" -Recurse | Out-String -width 150
+    workingDirectory: '$(repoRoot)'
+    displayName: 'Copy AIShell to ob_outputDirectory'
+
+  - task: CodeQL3000Finalize@0 # Add CodeQL Finalize task right after your 'Build' step.
+    displayName: 🔏 CodeQL 3000 Finalize
+    condition: eq(variables['CODEQL_ENABLED'], 'true')
+
+  - pwsh: |
+      Get-ChildItem -Path env: | Out-String -width 150 -Stream | write-Verbose -Verbose
+    workingDirectory: '$(repoRoot)'
+    displayName: Capture Environment
+    condition: succeededOrFailed()
+
+  # Explicitly download symbols for the drop since the SDL image doesn't have http://SymWeb access and APIScan cannot handle https yet.
+  # - pwsh: |
+  #     $pat = '$(SymbolServerPAT)'
+  #     if ($pat -like '*PAT*' -or $pat -eq '')
+  #     {
+  #       throw 'No PAT defined'
+  #     }
+  #     $url = 'https://microsoft.artifacts.visualstudio.com/defaultcollection/_apis/symbol/symsrv'
+  #     $(symbolToolPath) --authenticated-server-path $(SymbolServerPAT) $url --symbols -d "$env:ob_outputDirectory\*" --recurse-subdirectories
+  #   displayName: 'Download Symbols for binaries'
+  #   retryCountOnTaskFailure: 2
+  #   workingDirectory: '$(repoRoot)'
+
+  - pwsh: |
+      Get-ChildItem '$(ob_outputDirectory)' -File -Recurse |
+      Foreach-Object {
+        [pscustomobject]@{
+          Path = $_.FullName
+          Version = $_.VersionInfo.FileVersion
+          Md5Hash = (Get-FileHash -Algorithm MD5 -Path $_.FullName).Hash
+          Sha512Hash = (Get-FileHash -Algorithm SHA512 -Path $_.FullName).Hash
+        }
+      } | Export-Csv -Path '$(Build.SourcesDirectory)\ReleaseFileHash.csv'
+    workingDirectory: '$(repoRoot)'
+    displayName: 'Create release file hash artifact'
+
+  - pwsh: |
+      Copy-Item -Path '$(Build.SourcesDirectory)\ReleaseFileHash.csv' -Destination '$(ob_outputDirectory)' -Verbose
+    displayName: 'Publish Build File Hash artifact'

.pipelines/templates/linux-build.yml

@@ -21,8 +21,6 @@ jobs:
     value: false
   - name: ob_sdl_codeSignValidation_enabled
     value: false
-  - name: ob_sdl_binskim_enabled
-    value: true
   - name: ob_sdl_tsa_configFile
     value: $(repoRoot)\.config\tsaoptions.json
   - name: Architecture

.pipelines/templates/linux-package.yml

@@ -26,8 +26,6 @@ jobs:
       value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
     - name: repoRoot
       value: $(Build.SourcesDirectory)/AIShell
-    - name: ob_sdl_binskim_enabled
-      value: true
     - name: ob_sdl_tsa_configFile
       value: $(repoRoot)/.config/tsaoptions.json
     - name: Architecture

.pipelines/templates/mac-package.yml

@@ -29,8 +29,6 @@ jobs:
       value: false
     - name: ob_outputDirectory
       value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
-    - name: ob_sdl_binskim_enabled
-      value: true
     - name: repoRoot
       value: $(Build.SourcesDirectory)
     - name: Architecture