From 8c11fdf190f951193506d9289abe9c65cf3ccdf5 Mon Sep 17 00:00:00 2001 From: Johnni Winther Date: Wed, 19 Nov 2025 11:18:40 +0100 Subject: [PATCH 1/3] Add .fromEnvironment flags to the global declarer The global declarer is used by package:test_reflective_loader which has no means of configuring the test output. It is for instance used to run all tests in package:analyzer, and the current output prints a line for each test (of which there is currently >32000) and uses colors in the output. This makes it very hard to output to find failing tests, since the failures are hidden within all the succeeding tests and piping the output to a file for easier search still leaves the color in the output. This change adds .fromEnvironment variables to the creation of the global declarer, such that it can be configured at the command line. This allow for enabling compact output which only displays the failing tests, and whether to use colors in the output. --- pkgs/test_core/lib/src/scaffolding.dart | 32 ++- .../test/scaffolding/fixtures/tester.dart | 32 +++ .../scaffolding/global_declarer_test.dart | 241 ++++++++++++++++++ 3 files changed, 299 insertions(+), 6 deletions(-) create mode 100644 pkgs/test_core/test/scaffolding/fixtures/tester.dart create mode 100644 pkgs/test_core/test/scaffolding/global_declarer_test.dart diff --git a/pkgs/test_core/lib/src/scaffolding.dart b/pkgs/test_core/lib/src/scaffolding.dart index bf6d36177..7fb0ccc8c 100644 --- a/pkgs/test_core/lib/src/scaffolding.dart +++ b/pkgs/test_core/lib/src/scaffolding.dart @@ -13,6 +13,7 @@ import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_im import 'runner/engine.dart'; import 'runner/plugin/environment.dart'; +import 'runner/reporter/compact.dart'; import 'runner/reporter/expanded.dart'; import 'runner/runner_suite.dart'; import 'runner/suite.dart'; @@ -60,13 +61,32 @@ Declarer get _declarer { var engine = Engine(); engine.suiteSink.add(suite); engine.suiteSink.close(); - ExpandedReporter.watch( - engine, - PrintSink(), - color: true, - printPath: false, - printPlatform: false, + var useCompactReporter = const bool.fromEnvironment( + 'test_core.compactReporter', ); + var colors = const bool.fromEnvironment( + 'test_core.colors', + defaultValue: true, + ); + var printPath = const bool.fromEnvironment('test_core.printPath'); + var printPlatform = const bool.fromEnvironment('test_core.printPlatform'); + if (useCompactReporter) { + CompactReporter.watch( + engine, + PrintSink(), + color: colors, + printPath: printPath, + printPlatform: printPlatform, + ); + } else { + ExpandedReporter.watch( + engine, + PrintSink(), + color: colors, + printPath: printPath, + printPlatform: printPlatform, + ); + } var success = await runZoned( () => Invoker.guard(engine.run), diff --git a/pkgs/test_core/test/scaffolding/fixtures/tester.dart b/pkgs/test_core/test/scaffolding/fixtures/tester.dart new file mode 100644 index 000000000..961724b5e --- /dev/null +++ b/pkgs/test_core/test/scaffolding/fixtures/tester.dart @@ -0,0 +1,32 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:test/test.dart' as test_package; + +typedef TestFunction = dynamic Function(); + +class Test { + final String name; + final TestFunction function; + + Test(this.name, this.function); +} + +void main() { + var tests = [ + Test('a', () {}), + Test('b', () => throw test_package.TestFailure('b')), + Test('c', () {}), + Test('d', () => throw test_package.TestFailure('d')), + Test('e', () {}), + ]; + + for (var test in tests) { + test_package.test(test.name, () async { + try { + await test.function(); + } finally {} + }); + } +} diff --git a/pkgs/test_core/test/scaffolding/global_declarer_test.dart b/pkgs/test_core/test/scaffolding/global_declarer_test.dart new file mode 100644 index 000000000..aa3f525fb --- /dev/null +++ b/pkgs/test_core/test/scaffolding/global_declarer_test.dart @@ -0,0 +1,241 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:test/test.dart'; + +Future main() async { + group('environment', () { + test('default', () async { + await _run([], contains: _generateExpectedOutput()); + }); + + group('colors', () { + test('true', () async { + await _run([ + '-Dtest_core.colors=true', + ], contains: _generateExpectedOutput()); + }); + test('false', () async { + await _run([ + '-Dtest_core.colors=false', + ], contains: _generateExpectedOutput(colors: false)); + }); + }); + + group('compactReporter', () { + test('false', () async { + await _run([ + '-Dtest_core.compactReporter=false', + ], contains: _generateExpectedOutput()); + }); + test('true', () async { + await _run([ + '-Dtest_core.compactReporter=true', + ], contains: _generateExpectedOutput(compactReporter: true)); + }); + }); + + group('printPath', () { + test('false', () async { + await _run([ + '-Dtest_core.printPath=false', + ], contains: _generateExpectedOutput()); + }); + test('true', () async { + await _run([ + '-Dtest_core.printPath=true', + ], contains: _generateExpectedOutput(path: ': .')); + }); + }); + + group('printPlatform', () { + test('false', () async { + await _run([ + '-Dtest_core.printPlatform=false', + ], contains: _generateExpectedOutput()); + }); + test('true', () async { + await _run([ + '-Dtest_core.printPlatform=true', + ], contains: _generateExpectedOutput(platform: ' [VM, Kernel]')); + }); + }); + + group('mixed', () { + test('compactReporter, no colors', () async { + await _run( + ['-Dtest_core.compactReporter=true', '-Dtest_core.colors=false'], + contains: _generateExpectedOutput( + compactReporter: true, + colors: false, + ), + ); + }); + }); + }); +} + +const boldCode = '\x1B[1m'; +const cyanCode = '\x1B[36m'; +const debugPrint = false; +const greenCode = '\x1B[32m'; +const noColorCode = '\x1B[0m'; +const redCode = '\x1B[31m'; + +String _decode(dynamic stdout) { + if (stdout is String) { + return stdout; + } else { + return systemEncoding.decoder.convert(stdout as Uint8List); + } +} + +List _generateExpectedOutput({ + bool colors = true, + bool compactReporter = false, + String path = '', + String platform = '', +}) { + String green(String text) => colors ? '$greenCode$text$noColorCode' : text; + + String red(String text) => colors ? '$redCode$text$noColorCode' : text; + + String bold(String text) => colors ? '$boldCode$text$noColorCode' : text; + + String cyan(String text) => colors ? '$cyanCode$text$noColorCode' : text; + + Pattern hidden(String text) => RegExp( + '${RegExp.escape(text)}${colors ? RegExp.escape(noColorCode) : ''} +\\r', + ); + + Pattern line( + int success, + int fail, + String text, { + bool hide = false, + bool failure = false, + bool end = false, + }) { + var sb = StringBuffer(); + sb.write(green('+$success')); + if (fail != 0) { + sb.write(red(' $fail')); + } + if (end) { + sb.write(': ${red(text)}'); + } else { + sb.write('$path:$platform $text'); + if (failure) { + sb.write(' ${bold(red('[E]'))}'); + } + } + var result = sb.toString(); + if (hide) { + return hidden(result); + } else { + return result; + } + } + + return [ + line(0, 0, 'a', hide: compactReporter), + line(1, 0, 'b', hide: compactReporter), + line(1, -1, 'b', failure: true), + if (compactReporter) cyan('To run this test again:'), + line(1, -1, 'c', hide: compactReporter), + line(2, -1, 'd', hide: compactReporter), + line(2, -2, 'd', failure: true), + if (compactReporter) cyan('To run this test again:'), + line(2, -2, 'e', hide: compactReporter), + line(3, -2, 'Some tests failed.', end: true), + ]; +} + +Future _run( + List args, { + List contains = const [], + List? doesNotContain, +}) async { + if (debugPrint) { + print( + '====================================================================', + ); + print('args: $args'); + } + var result = await Process.run(Platform.executable, [ + ...args, + Platform.script + .resolve('fixtures/tester.dart') + .toFilePath(windows: Platform.isWindows), + ]); + + var stdout = _decode(result.stdout); + if (debugPrint) { + print(stdout); + } + + expect(stdout, _stringMatchesInOrder(contains)); + if (doesNotContain != null) { + for (var text in doesNotContain) { + expect(stdout, isNot(stringContainsInOrder([text]))); + } + } +} + +Matcher _stringMatchesInOrder(List substrings) => + _StringMatchesInOrder(substrings); + +class _StringMatchesInOrder implements Matcher { + final List _patterns; + + const _StringMatchesInOrder(this._patterns); + + @override + Description describe(Description description) => + description.addAll('a string containing ', ', ', ' in order', _patterns); + + @override + Description describeMismatch( + dynamic item, + Description mismatchDescription, + Map matchState, + bool verbose, + ) { + var patternIndex = _matches(item, matchState); + var pattern = _patterns[patternIndex!]; + return mismatchDescription.add( + 'Pattern #$patternIndex, $pattern, not found.', + ); + } + + @override + bool matches(dynamic item, Map matchState) { + return _matches(item, matchState) == null; + } + + int? _matches(dynamic item, Map matchState) { + item as String; + var fromIndex = 0; + for ( + var patternIndex = 0; + patternIndex < _patterns.length; + patternIndex++ + ) { + var s = _patterns[patternIndex]; + if (s is String) { + var index = item.indexOf(s, fromIndex); + if (index < 0) return patternIndex; + fromIndex = index + s.length; + } else { + var matches = s.allMatches(item, fromIndex); + if (matches.isEmpty) return patternIndex; + fromIndex = matches.first.end; + } + } + return null; + } +} From 00e3bc700caadf3829787538790587fb75a47423 Mon Sep 17 00:00:00 2001 From: Johnni Winther Date: Wed, 19 Nov 2025 13:45:18 +0100 Subject: [PATCH 2/3] Fix test and add changelog --- pkgs/test_core/CHANGELOG.md | 3 +++ pkgs/test_core/test/scaffolding/global_declarer_test.dart | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md index 9a5fba622..65824c30d 100644 --- a/pkgs/test_core/CHANGELOG.md +++ b/pkgs/test_core/CHANGELOG.md @@ -1,3 +1,6 @@ +## 0.6.14 +* Support environment flags for configuring the global declarer. + ## 0.6.13 * Require Dart 3.7 diff --git a/pkgs/test_core/test/scaffolding/global_declarer_test.dart b/pkgs/test_core/test/scaffolding/global_declarer_test.dart index aa3f525fb..159a0212c 100644 --- a/pkgs/test_core/test/scaffolding/global_declarer_test.dart +++ b/pkgs/test_core/test/scaffolding/global_declarer_test.dart @@ -166,11 +166,10 @@ Future _run( ); print('args: $args'); } + var tester = File('test/scaffolding/fixtures/tester.dart'); var result = await Process.run(Platform.executable, [ ...args, - Platform.script - .resolve('fixtures/tester.dart') - .toFilePath(windows: Platform.isWindows), + tester.absolute.path, ]); var stdout = _decode(result.stdout); From 7aee99b6b9d6677eefaf0d42b7c7a76b92279aee Mon Sep 17 00:00:00 2001 From: Johnni Winther Date: Wed, 19 Nov 2025 14:05:39 +0100 Subject: [PATCH 3/3] Update pubspec.yaml --- pkgs/test_core/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml index 30587cd7f..65f9d41e6 100644 --- a/pkgs/test_core/pubspec.yaml +++ b/pkgs/test_core/pubspec.yaml @@ -1,5 +1,5 @@ name: test_core -version: 0.6.13 +version: 0.6.14 description: A basic library for writing tests and running them on the VM. repository: https://github.com/dart-lang/test/tree/master/pkgs/test_core issue_tracker: https://github.com/dart-lang/test/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Atest