[WIP] Trying to filter what tests run based on PR contents

.azure/pipelines/ci.yml

@@ -556,6 +556,8 @@ extends:
             agentOs: macOS
             timeoutInMinutes: 90
             isAzDOTestingJob: true
+            # fetchDepth: 0 ensures full git history is available for affected area computation in build.sh
+            fetchDepth: 0
             buildArgs: --all --test --binaryLog /p:RunTemplateTests=false /p:SkipHelixReadyTests=true $(_InternalRuntimeDownloadArgs)
             beforeBuild:
             - bash: "./eng/scripts/install-nginx.sh"
@@ -577,6 +579,8 @@ extends:
             agentOs: Linux
             isAzDOTestingJob: true
             useHostedUbuntu: false
+            # fetchDepth: 0 ensures full git history is available for affected area computation in build.sh
+            fetchDepth: 0
             buildArgs: --all --test --binaryLog /p:RunTemplateTests=false /p:SkipHelixReadyTests=true $(_InternalRuntimeDownloadArgs)
             beforeBuild:
             - bash: "./eng/scripts/install-nginx.sh"
@@ -600,6 +604,8 @@ extends:
             jobDisplayName: 'Tests: Helix x64 Subset 1'
             agentOs: Windows
             timeoutInMinutes: 240
+            # Need full git history for affected area computation
+            fetchDepth: 0
             steps:
             # Build the shared framework
             - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -pack -arch x64
@@ -608,9 +614,11 @@ extends:
                 MSBUILDUSESERVER: "1"
               displayName: Build shared fx
             # -noBuildNative -noBuild to avoid repeating work done in the previous step.
+            # Affected test area filtering is computed automatically by build.cmd for PR builds.
             - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildNative -noBuild -test
                       -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true /p:HelixSubset=1
-                      /p:CrossgenOutput=false /p:RunTemplateTests=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
+                      /p:CrossgenOutput=false /p:RunTemplateTests=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+                      $(_InternalRuntimeDownloadArgs)
               displayName: Run build.cmd helix target
               env:
                 HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
@@ -629,6 +637,8 @@ extends:
             jobDisplayName: 'Tests: Helix x64 Subset 2'
             agentOs: Windows
             timeoutInMinutes: 240
+            # fetchDepth: 0 ensures full git history is available for affected area computation in build.cmd
+            fetchDepth: 0
             steps:
             # Build the shared framework
             - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -pack -arch x64
@@ -637,9 +647,11 @@ extends:
                 MSBUILDUSESERVER: "1"
               displayName: Build shared fx
             # -noBuildNative -noBuild to avoid repeating work done in the previous step.
+            # Affected test area filtering is computed automatically by build.cmd for PR builds.
             - script: ./eng/build.cmd -ci -prepareMachine -nativeToolsOnMachine -all -noBuildNative -noBuild -test
                       -projects eng\helix\helix.proj /p:IsHelixPRCheck=true /p:IsHelixJob=true /p:HelixSubset=2
-                      /p:CrossgenOutput=false /p:RunTemplateTests=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log $(_InternalRuntimeDownloadArgs)
+                      /p:CrossgenOutput=false /p:RunTemplateTests=false /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log
+                      $(_InternalRuntimeDownloadArgs)
               displayName: Run build.cmd helix target
               env:
                 HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues

eng/AffectedProjectAreas.json

@@ -0,0 +1,46 @@
+{
+  "_comment": "This file defines the dependency graph between top-level src/ areas. Each area maps to the list of other areas it directly depends on (forward dependencies). This is used by GetAffectedTestAreas.ps1 to compute which areas need testing when files change, and by VerifyDependencyGraph.ps1 to ensure the graph stays current. Run eng/scripts/GenerateAffectedProjectAreas.ps1 to regenerate.",
+
+  "Analyzers": [],
+  "Antiforgery": ["DataProtection", "Http", "ObjectPool"],
+  "Assets": [],
+  "Azure": ["Hosting", "Logging.AzureAppServices"],
+  "BuildAfterTargetingPack": [],
+  "Caching": [],
+  "Components": ["Antiforgery", "DataProtection", "DefaultBuilder", "Extensions", "FileProviders", "Hosting", "Html.Abstractions", "Http", "JSInterop", "Middleware", "Security", "SignalR", "StaticAssets", "Validation"],
+  "Configuration.KeyPerFile": [],
+  "DataProtection": [],
+  "DefaultBuilder": ["Antiforgery", "Hosting", "Http", "Middleware", "Security", "Servers"],
+  "Extensions": [],
+  "Features": ["Http"],
+  "FileProviders": [],
+  "Framework": [],
+  "Grpc": [],
+  "HealthChecks": [],
+  "Hosting": ["Extensions", "FileProviders", "Http", "Testing"],
+  "Html.Abstractions": [],
+  "Http": ["Extensions", "Hosting", "ObjectPool", "Security", "Servers", "Validation"],
+  "HttpClientFactory": [],
+  "Identity": ["DataProtection", "Extensions", "FileProviders", "Hosting", "Html.Abstractions", "Http", "Middleware", "Mvc", "Razor", "Security", "Servers", "Testing"],
+  "Installers": [],
+  "JSInterop": [],
+  "Localization": [],
+  "Logging.AzureAppServices": [],
+  "Middleware": ["DataProtection", "Extensions", "HealthChecks", "Hosting", "Http", "Localization", "ObjectPool", "Servers", "WebEncoders"],
+  "Mvc": ["Antiforgery", "Components", "DataProtection", "Features", "Hosting", "Html.Abstractions", "Http", "Localization", "Middleware", "ObjectPool", "Razor", "Security", "Servers", "StaticAssets", "WebEncoders"],
+  "ObjectPool": [],
+  "OpenApi": ["DefaultBuilder", "Hosting", "Http", "Mvc"],
+  "ProjectTemplates": [],
+  "Razor": ["Html.Abstractions"],
+  "Security": ["DataProtection", "Hosting", "Http", "Servers", "WebEncoders"],
+  "Servers": ["Extensions", "Hosting", "Http", "Middleware", "ObjectPool"],
+  "Shared": ["Testing"],
+  "SignalR": ["Hosting", "Http", "Middleware", "Security", "Servers"],
+  "SiteExtensions": [],
+  "StaticAssets": ["Hosting", "Http", "Middleware"],
+  "submodules": [],
+  "Testing": [],
+  "Tools": ["DefaultBuilder", "Hosting", "Http"],
+  "Validation": [],
+  "WebEncoders": []
+}

eng/Build.props

@@ -281,7 +281,62 @@
                           $(RepoRoot)**\bin\**\*;
                           $(RepoRoot)**\obj\**\*;"
                         Condition=" '$(BuildMainlyReferenceProviders)' != 'true' " />
+      </ItemGroup>
+
+      <!--
+        When AffectedTestAreas is set (semicolon-delimited), remove projects from areas NOT in the list.
+        This filters tests to only run for areas affected by PR changes.
+        The delimiter-wrapping pattern ';Area;' prevents partial matches (e.g., 'Http' matching 'HttpClientFactory').
+      -->
+      <PropertyGroup Condition=" '$(AffectedTestAreas)' != '' ">
+        <_AffectedTestAreasWrapped>;$(AffectedTestAreas);</_AffectedTestAreasWrapped>
+      </PropertyGroup>
+
+      <ItemGroup Condition=" '$(AffectedTestAreas)' != '' AND '$(BuildMainlyReferenceProviders)' != 'true' ">
+        <!-- Subset 1 area filtering -->
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Framework\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Framework;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Caching\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Caching;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\DefaultBuilder\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';DefaultBuilder;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Features\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Features;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\DataProtection\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';DataProtection;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Antiforgery\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Antiforgery;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Hosting\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Hosting;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Http\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Http;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\HttpClientFactory\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';HttpClientFactory;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Html.Abstractions\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Html.Abstractions;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Identity\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Identity;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Servers\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Servers;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Security\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Security;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\SiteExtensions\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';SiteExtensions;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Shared\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Shared;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Tools\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Tools;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Logging.AzureAppServices\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Logging.AzureAppServices;')) " />
+        <ProjectsWithTestsSubset1 Remove="$(RepoRoot)src\Middleware\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Middleware;')) " />
+
+        <!-- Subset 2 area filtering -->
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Razor\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Razor;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Mvc\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Mvc;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Azure\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Azure;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\SignalR\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';SignalR;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\StaticAssets\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';StaticAssets;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Components\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Components;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Analyzers\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Analyzers;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\FileProviders\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';FileProviders;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Configuration.KeyPerFile\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Configuration.KeyPerFile;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Localization\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Localization;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\ObjectPool\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';ObjectPool;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\JSInterop\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';JSInterop;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\WebEncoders\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';WebEncoders;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\HealthChecks\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';HealthChecks;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Testing\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Testing;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Grpc\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Grpc;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\ProjectTemplates\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';ProjectTemplates;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Extensions\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Extensions;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\OpenApi\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';OpenApi;')) " />
+        <ProjectsWithTestsSubset2 Remove="$(RepoRoot)src\Validation\**\*" Condition=" !$(_AffectedTestAreasWrapped.Contains(';Validation;')) " />
+      </ItemGroup>
 
+      <ItemGroup Condition=" '$(BuildMainlyReferenceProviders)' != 'true' ">
         <DotNetProjects Condition=" '$(HelixSubset)' == '' OR '$(HelixSubset)' == '1'" Include="@(ProjectsWithTestsSubset1)" />
         <DotNetProjects Condition=" '$(HelixSubset)' == '' OR '$(HelixSubset)' == '2'" Include="@(ProjectsWithTestsSubset2)" />
 

eng/build.ps1

@@ -466,6 +466,41 @@ try {
         $global:VerbosePreference = 'Continue'
     }
 
+    # Compute affected test areas for PR builds to filter tests to only affected areas.
+    # Cache the result to a file so subsequent build steps in the same CI job use the same areas.
+    # This is important because the build step may generate files that change the git diff result.
+    if ($CI -and $env:SYSTEM_PULLREQUEST_TARGETBRANCH -and -not $env:AffectedTestAreas) {
+        $affectedAreasCacheFile = Join-Path (Join-Path $ArtifactsDir "tmp") "AffectedTestAreas.txt"
+        if (Test-Path $affectedAreasCacheFile) {
+            $env:AffectedTestAreas = (Get-Content $affectedAreasCacheFile -Raw).Trim()
+            if ($env:AffectedTestAreas) {
+                Write-Host "AffectedTestAreas loaded from cache: $($env:AffectedTestAreas)"
+            } else {
+                Write-Host "No test area filtering applied (cached result was empty)."
+            }
+        } else {
+            $affectedAreasScript = Join-Path $PSScriptRoot "scripts/GetAffectedTestAreas.ps1"
+            if (Test-Path $affectedAreasScript) {
+                try {
+                    Write-Host "Computing affected test areas for PR build..."
+                    $env:AffectedTestAreas = & $affectedAreasScript
+                    if ($env:AffectedTestAreas) {
+                        Write-Host "AffectedTestAreas set to: $($env:AffectedTestAreas)"
+                        $cacheDir = Split-Path $affectedAreasCacheFile -Parent
+                        if (-not (Test-Path $cacheDir)) { New-Item -ItemType Directory -Path $cacheDir -Force | Out-Null }
+                        $env:AffectedTestAreas | Out-File -FilePath $affectedAreasCacheFile -NoNewline -Encoding utf8
+                    } else {
+                        Write-Host "No test area filtering applied (all tests will run)."
+                    }
+                } catch {
+                    Write-Host "Warning: Failed to compute affected test areas: $_"
+                    Write-Host "All tests will run."
+                    $env:AffectedTestAreas = $null
+                }
+            }
+        }
+    }
+
     if ($performDesktopBuild) {
         Write-Host
         Remove-Item variable:global:_BuildTool -ErrorAction Ignore

eng/build.sh

@@ -409,5 +409,42 @@ InitializeToolset
 
 restore=$_tmp_restore=
 
+# Compute affected test areas for PR builds to filter tests to only affected areas
+if [ "$ci" = true ] && [ -n "${SYSTEM_PULLREQUEST_TARGETBRANCH:-}" ] && [ -z "${AffectedTestAreas:-}" ]; then
+    output_file="$artifacts_dir/tmp/AffectedTestAreas.txt"
+    if [ -f "$output_file" ]; then
+        # Reuse cached result from a previous build step in this CI job
+        areas=$(cat "$output_file")
+        if [ -n "$areas" ]; then
+            export AffectedTestAreas="$areas"
+            echo "AffectedTestAreas loaded from cache: $areas"
+        else
+            echo "No test area filtering applied (cached result was empty)."
+        fi
+    else
+        affected_areas_script="$DIR/scripts/GetAffectedTestAreas.ps1"
+        if [ -f "$affected_areas_script" ] && command -v pwsh &>/dev/null; then
+            echo "Computing affected test areas for PR build..."
+            mkdir -p "$(dirname "$output_file")"
+            # Use set +e to prevent script failure if area computation fails
+            set +e
+            pwsh -NoProfile -Command "& '$affected_areas_script' -OutputFile '$output_file'"
+            pwsh_exit=$?
+            set -e
+            if [ $pwsh_exit -eq 0 ] && [ -f "$output_file" ]; then
+                areas=$(cat "$output_file")
+                if [ -n "$areas" ]; then
+                    export AffectedTestAreas="$areas"
+                    echo "AffectedTestAreas set to: $areas"
+                else
+                    echo "No test area filtering applied (all tests will run)."
+                fi
+            else
+                echo "Warning: Failed to compute affected test areas. All tests will run."
+            fi
+        fi
+    fi
+fi
+
 MSBuild $_InitializeToolset -p:RepoRoot="$repo_root" ${msbuild_args[@]+"${msbuild_args[@]}"}
 ExitWithExitCode 0

eng/scripts/CodeCheck.ps1

@@ -149,6 +149,19 @@ try {
         & $PSScriptRoot\GenerateProjectList.ps1 -ci:$ci
     }
 
+    #
+    # Verify the affected project areas dependency graph is up to date
+    #
+
+    Write-Host "Verifying dependency graph (eng/AffectedProjectAreas.json)"
+    try {
+        & $PSScriptRoot\VerifyDependencyGraph.ps1 -ci:$ci
+    }
+    catch {
+        LogError -code 'BUILD005' `
+            "Dependency graph verification failed. Run eng/scripts/GenerateAffectedProjectAreas.ps1 to update eng/AffectedProjectAreas.json."
+    }
+
     Write-Host "Running git diff to check for pending changes"
 
     # Redirect stderr to stdout because PowerShell does not consistently handle output to stderr