Skip to content

Commit 5c57420

Browse files
LoopedBard3Copilot
LoopedBard3
and
Copilot
authored
Fix startup scenario test failures on .NET 11 Preview 2 (#5119)
* Fix startup scenario test failures on .NET 11 Preview 2 - Update console template pre.py scripts to search for "Console.WriteLine" instead of the removed "// See https://aka.ms/new-console-template" comment - Make insert_after() raise an exception when the search string is not found instead of silently succeeding - Handle empty parser results in Reporter.WriteResultTable() to prevent InvalidOperationException on .Max() of empty sequence - Add diagnostic logging and fail with exit code -1 when parser returns no results in Startup.cs * Gate WPF scenarios to internal runs due to CloseFailed on CI VMs WPF Template and WPF SFC fail with CloseFailed on generic Helix CI VMs because CloseMainWindow() doesn't work in headless environments. This is the same issue that caused WinForms scenarios to be gated. Gate both WPF scenarios behind SYSTEM_TEAMPROJECT=internal, with IncludeVMFlakyScenarios override. * Address PR feedback: add empty-results tests, skip report on failure - Add three tests for WriteResultTable() with no counters, empty results, and null results to cover the empty-results guard - Change Reporter message to neutral 'No results in file.' - Skip CreateTestReport() in Startup.cs when parser returns no results Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent f728a51 commit 5c57420

File tree

7 files changed

+101
-10
lines changed

7 files changed

+101
-10
lines changed

‎eng/performance/scenarios.proj‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,15 @@
4343
<ScenarioDirectoryName>windowsformslarge</ScenarioDirectoryName>
4444
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
4545
</UIScenario>
46-
<UIScenario Include="WPF Template" Condition="'$(TargetsWindows)' == 'true' AND '$(OS)' == 'Windows_NT'">
46+
<!-- WPF Template: limited to internal runs due to CloseFailed on generic CI VMs (same issue as WinForms).
47+
Override with /p:IncludeVMFlakyScenarios=true to include on other builds. -->
48+
<UIScenario Include="WPF Template" Condition="'$(TargetsWindows)' == 'true' AND '$(OS)' == 'Windows_NT' AND ( '$(SYSTEM_TEAMPROJECT)' == 'internal' OR '$(IncludeVMFlakyScenarios)' == 'true' )">
4749
<ScenarioDirectoryName>wpf</ScenarioDirectoryName>
4850
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
4951
</UIScenario>
50-
<UIScenario Include="WPF SFC" Condition="'$(TargetsWindows)' == 'true' AND '$(OS)' == 'Windows_NT'">
52+
<!-- WPF SFC: gated for consistency with WPF Template.
53+
Override with /p:IncludeVMFlakyScenarios=true to include on other builds. -->
54+
<UIScenario Include="WPF SFC" Condition="'$(TargetsWindows)' == 'true' AND '$(OS)' == 'Windows_NT' AND ( '$(SYSTEM_TEAMPROJECT)' == 'internal' OR '$(IncludeVMFlakyScenarios)' == 'true' )">
5155
<ScenarioDirectoryName>wpfsfc</ScenarioDirectoryName>
5256
<PayloadDirectory>$(ScenariosDir)%(ScenarioDirectoryName)</PayloadDirectory>
5357
</UIScenario>

‎src/scenarios/emptyconsolenativeaot/pre.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
args = ['/p:PublishAot=true', '/p:StripSymbols=true', '/p:IlcGenerateMstatFile=true']
1818
if not iswin():
1919
args.extend(['/p:ObjCopyName=objcopy'])
20-
precommands.add_onmain_logging("Program.cs", "// See https://aka.ms/new-console-template for more information")
20+
precommands.add_onmain_logging("Program.cs", "Console.WriteLine")
2121
precommands.execute(args)
2222

2323
src = os.path.join(const.APPDIR, 'obj', precommands.configuration, precommands.framework, precommands.runtime_identifier, 'native', f'{EXENAME}.mstat')

‎src/scenarios/emptyconsoletemplate/pre.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,5 @@
1414
bin_dir=const.BINDIR,
1515
exename=EXENAME,
1616
working_directory=sys.path[0])
17-
precommands.add_onmain_logging("Program.cs", "// See https://aka.ms/new-console-template for more information")
17+
precommands.add_onmain_logging("Program.cs", "Console.WriteLine")
1818
precommands.execute()

‎src/scenarios/shared/codefixes.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,14 @@ def writefile(file: str, lines: list[str]):
2222
# insert string after the first occurance of the search string
2323
def insert_after(file: str, search: str, insert: str):
2424
lines = readfile(file)
25+
found = False
2526
for i in range(len(lines)):
2627
if search in lines[i]:
2728
lines.insert(i+1, ("%s\n" % insert))
29+
found = True
2830
break
31+
if not found:
32+
raise Exception(f"insert_after: search string '{search}' not found in {file}")
2933
writefile(file, lines)
3034

3135
def replace_line(file: str, search: str, replace: str):

‎src/tools/Reporting/Reporting.Tests/ReporterTests.cs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,65 @@ public class ReporterTests
3131
CounterName |10000000000000000.000 ns |10000000000000000.000 ns |10000000000000000.000 ns
3232
";
3333

34+
private const string NoResultsTable =
35+
@"TestName
36+
No results in file.
37+
";
38+
39+
[Fact]
40+
public void WriteReportTableWithNoCounters()
41+
{
42+
var reporter = new Reporter(new NonPerfLabEnvironmentProviderMock());
43+
var test = new Test { Name = "TestName" };
44+
reporter.AddTest(test);
45+
var table = reporter.WriteResultTable();
46+
Assert.Equal(NoResultsTable, table);
47+
}
48+
49+
[Fact]
50+
public void WriteReportTableWithEmptyResults()
51+
{
52+
var reporter = new Reporter(new NonPerfLabEnvironmentProviderMock());
53+
var test = new Test
54+
{
55+
Name = "TestName",
56+
Counters = [
57+
new Counter
58+
{
59+
DefaultCounter = true,
60+
MetricName = "ns",
61+
Name = "CounterName",
62+
Results = []
63+
}
64+
]
65+
};
66+
reporter.AddTest(test);
67+
var table = reporter.WriteResultTable();
68+
Assert.Equal(NoResultsTable, table);
69+
}
70+
71+
[Fact]
72+
public void WriteReportTableWithNullResults()
73+
{
74+
var reporter = new Reporter(new NonPerfLabEnvironmentProviderMock());
75+
var test = new Test
76+
{
77+
Name = "TestName",
78+
Counters = [
79+
new Counter
80+
{
81+
DefaultCounter = true,
82+
MetricName = "ns",
83+
Name = "CounterName",
84+
Results = null
85+
}
86+
]
87+
};
88+
reporter.AddTest(test);
89+
var table = reporter.WriteResultTable();
90+
Assert.Equal(NoResultsTable, table);
91+
}
92+
3493
[Fact]
3594
public void ReporterWithUnsetEnvironmentProducesNoJson()
3695
{

‎src/tools/Reporting/Reporting/Reporter.cs‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,16 +72,27 @@ public string WriteResultTable()
7272
var ret = new StringBuilder();
7373
foreach (var test in Tests)
7474
{
75+
if (!test.Counters.Any() || test.Counters.All(c => c.Results == null || c.Results.Count == 0))
76+
{
77+
ret.AppendLine(test.Name);
78+
ret.AppendLine("No results in file.");
79+
continue;
80+
}
81+
82+
var countersWithResults = test.Counters.Where(c => c.Results != null && c.Results.Count > 0);
7583
var counterWidth = Math.Max(test.Counters.Max(c => c.Name.Length) + 1, 15);
76-
var resultWidth = Math.Max(test.Counters.Max(c => c.Results.Max().ToString("F3", _culture).Length + c.MetricName.Length) + 2, 15);
84+
var resultWidth = Math.Max(countersWithResults.Max(c => c.Results.Max().ToString("F3", _culture).Length + c.MetricName.Length) + 2, 15);
7785
ret.AppendLine(test.Name);
7886
ret.AppendLine($"{LeftJustify("Metric", counterWidth)}|{LeftJustify("Average", resultWidth)}|{LeftJustify("Min", resultWidth)}|{LeftJustify("Max", resultWidth)}");
7987
ret.AppendLine($"{new string('-', counterWidth)}|{new string('-', resultWidth)}|{new string('-', resultWidth)}|{new string('-', resultWidth)}");
8088

81-
var defaultCounter = test.Counters.Single(c => c.DefaultCounter);
82-
ret.AppendLine(PrintCounter(defaultCounter, counterWidth, resultWidth));
83-
var topCounters = test.Counters.Where(c => c.TopCounter && !c.DefaultCounter);
84-
var restCounters = test.Counters.Where(c => !c.TopCounter && !c.DefaultCounter);
89+
var defaultCounter = test.Counters.SingleOrDefault(c => c.DefaultCounter);
90+
if (defaultCounter != null && defaultCounter.Results != null && defaultCounter.Results.Count > 0)
91+
{
92+
ret.AppendLine(PrintCounter(defaultCounter, counterWidth, resultWidth));
93+
}
94+
var topCounters = test.Counters.Where(c => c.TopCounter && !c.DefaultCounter && c.Results != null && c.Results.Count > 0);
95+
var restCounters = test.Counters.Where(c => !c.TopCounter && !c.DefaultCounter && c.Results != null && c.Results.Count > 0);
8596
foreach (var counter in topCounters.Concat(restCounters))
8697
{
8798
ret.AppendLine(PrintCounter(counter, counterWidth, resultWidth));

‎src/tools/ScenarioMeasurement/Startup/Startup.cs‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System.Diagnostics;
55
using System.Diagnostics.Tracing;
66
using System.IO;
7+
using System.Linq;
78
using System.Runtime.CompilerServices;
89
using System.Text;
910
using System.Threading;
@@ -339,7 +340,19 @@ static void checkArg(string arg, string name)
339340
try
340341
{
341342
var counters = parser.Parse(traceFilePath, processName, pids, commandLine);
342-
CreateTestReport(scenarioName, counters, reportJsonPath, logger);
343+
if (!counters.Any() || counters.All(c => c.Results == null || c.Results.Count == 0))
344+
{
345+
logger.Log("ERROR: Parser returned no results. The ETL trace may not contain the expected events.");
346+
logger.Log($"{nameof(parser)} = {parser.GetType().FullName}");
347+
logger.Log($"{nameof(processName)} = {processName}");
348+
logger.Log($"{nameof(pids)} = {string.Join(", ", pids)}");
349+
logger.Log($"{nameof(commandLine)} = {commandLine}");
350+
failed = true;
351+
}
352+
if (!failed)
353+
{
354+
CreateTestReport(scenarioName, counters, reportJsonPath, logger);
355+
}
343356
}
344357
catch
345358
{

0 commit comments

Comments
 (0)