Implemented console-lines mode which prints each test on separate line. (#443)

This is handy for environments with non-standard (buffered) handling of
standard output, for example Github Actions (where a progress of tests
cannot be seen until until end-of-line appears, which in standard `console`
mode happens only when all tests finish) or Docker Compose logging output,
where in standard `console` mode each finished test's dot is printed
alone on separate line.

Or the console-lines mode can be handy just to see a more detailed
progress of tests in all environments, because it outputs something
like this:

```
· 1/85 Framework/Assert.contains.phpt OK in 0.14 s
· 2/85 CodeCoverage/PhpParser.parse.edge.phpt OK in 0.17 s
· 3/85 CodeCoverage/PhpParser.parse.lines-of-code.phpt SKIPPED in 0.18 s
· 4/85 CodeCoverage/PhpParser.parse.lines.phpt FAILED in 0.19 s
...
```

Also, "cider mode" now shows a lemon emoji for skipped tests.

Co-authored-by: David Grudl <[email protected]>

Author: smuuf

phpstan.neon

@@ -4,3 +4,5 @@ parameters:
 
 	paths:
 		- src
+	typeAliases:
+		Alias_TestResultState: 'Tester\Runner\Test::Passed|Tester\Runner\Test::Skipped|Tester\Runner\Test::Failed|Tester\Runner\Test::Prepared'

readme.md

@@ -223,7 +223,8 @@ Options:
     -s                           Show information about skipped tests.
     --stop-on-fail               Stop execution upon the first failure.
     -j <num>                     Run <num> jobs in parallel (default: 8).
-    -o <console|tap|junit|none>  Specify output format.
+    -o <console|console-lines|tap|junit|none>
+                                 Specify output format.
     -w | --watch <path>          Watch directory.
     -i | --info                  Show tests environment info and exit.
     --setup <path>               Script for runner setup.

src/Runner/CliTester.php

@@ -13,7 +13,7 @@
 use Tester\Dumper;
 use Tester\Environment;
 use Tester\Helpers;
-
+use Tester\Runner\Output\ConsolePrinter;
 
 /**
  * CLI Tester.
@@ -112,7 +112,7 @@ private function loadOptions(): CommandLine
 				    -s                           Show information about skipped tests.
 				    --stop-on-fail               Stop execution upon the first failure.
 				    -j <num>                     Run <num> jobs in parallel (default: 8).
-				    -o <console|tap|junit|log|none>  (e.g. -o junit:output.xml)
+				    -o <console|console-lines|tap|junit|log|none>  (e.g. -o junit:output.xml)
 				                                 Specify one or more output formats with optional file name.
 				    -w | --watch <path>          Watch directory.
 				    -i | --info                  Show tests environment info and exit.
@@ -219,18 +219,14 @@ private function createRunner(): Runner
 		$runner->setTempDirectory($this->options['--temp']);
 
 		if ($this->stdoutFormat === null) {
-			$runner->outputHandlers[] = new Output\ConsolePrinter(
-				$runner,
-				(bool) $this->options['-s'],
-				'php://output',
-				(bool) $this->options['--cider'],
-			);
+			$runner->outputHandlers[] = $this->buildConsolePrinter($runner, 'php://output', false);
 		}
 
 		foreach ($this->options['-o'] as $output) {
 			[$format, $file] = $output;
 			match ($format) {
-				'console' => $runner->outputHandlers[] = new Output\ConsolePrinter($runner, (bool) $this->options['-s'], $file, (bool) $this->options['--cider']),
+				'console' => $runner->outputHandlers[] = $this->buildConsolePrinter($runner, $file, false),
+				'console-lines' => $runner->outputHandlers[] = $this->buildConsolePrinter($runner, $file, true),
 				'tap' => $runner->outputHandlers[] = new Output\TapPrinter($file),
 				'junit' => $runner->outputHandlers[] = new Output\JUnitPrinter($file),
 				'log' => $runner->outputHandlers[] = new Output\Logger($runner, $file),
@@ -248,6 +244,25 @@ private function createRunner(): Runner
 		return $runner;
 	}
 
+	/**
+	 * Builds and returns a new `ConsolePrinter`.
+	 * @param bool $lineMode If `true`, reports each finished test on separate line.
+	 */
+	private function buildConsolePrinter(
+		Runner $runner,
+		?string $file,
+		bool $lineMode,
+	): ConsolePrinter
+	{
+		return new Output\ConsolePrinter(
+			$runner,
+			(bool) $this->options['-s'],
+			$file,
+			(bool) $this->options['--cider'],
+			$lineMode,
+		);
+	}
+
 
 	private function prepareCodeCoverage(Runner $runner): string
 	{

src/Runner/Output/ConsolePrinter.php

@@ -20,46 +20,66 @@
  */
 class ConsolePrinter implements Tester\Runner\OutputHandler
 {
-	private Runner $runner;
-
 	/** @var resource */
 	private $file;
-	private bool $displaySkipped = false;
-	private string $buffer;
+
+	/** @var list<string> */
+	private array $buffer;
+
+	/**
+	 * @phpstan-var array<Alias_TestResultState, string>
+	 * @var array<int, string>
+	 */
+	private array $symbols;
+
+	/**
+	 * @phpstan-var array<Alias_TestResultState, int>
+	 * @var array<int, string>
+	 */
+	private array $results = [
+		Test::Passed => 0,
+		Test::Skipped => 0,
+		Test::Failed => 0,
+	];
+
 	private float $time;
 	private int $count;
-	private array $results;
 	private ?string $baseDir;
-	private array $symbols;
-
+	private int $resultsCount = 0;
 
+	/**
+	 * @param bool $lineMode If `true`, reports each finished test on separate line.
+	 */
 	public function __construct(
-		Runner $runner,
-		bool $displaySkipped = false,
+		private Runner $runner,
+		private bool $displaySkipped = false,
 		?string $file = null,
 		bool $ciderMode = false,
+		private bool $lineMode = false,
 	) {
 		$this->runner = $runner;
 		$this->displaySkipped = $displaySkipped;
 		$this->file = fopen($file ?: 'php://output', 'w');
+
 		$this->symbols = [
-			Test::Passed => $ciderMode ? Dumper::color('green', '🍎') : '.',
-			Test::Skipped => 's',
-			Test::Failed => $ciderMode ? Dumper::color('red', '🍎') : Dumper::color('white/red', 'F'),
+			Test::Passed => $this->lineMode ? Dumper::color('lime', 'OK') : '.',
+			Test::Skipped => $this->lineMode ? Dumper::color('yellow', 'SKIP') : 's',
+			Test::Failed => $this->lineMode ? Dumper::color('white/red', 'FAIL') : Dumper::color('white/red', 'F'),
 		];
+
+		if ($ciderMode) {
+			$this->symbols[Test::Passed] = '🍏';
+			$this->symbols[Test::Skipped] = '🍋';
+			$this->symbols[Test::Failed] = '🍎';
+		}
 	}
 
 
 	public function begin(): void
 	{
 		$this->count = 0;
-		$this->buffer = '';
+		$this->buffer = [];
 		$this->baseDir = null;
-		$this->results = [
-			Test::Passed => 0,
-			Test::Skipped => 0,
-			Test::Failed => 0,
-		];
 		$this->time = -microtime(true);
 		fwrite($this->file, $this->runner->getInterpreter()->getShortInfo()
 			. ' | ' . $this->runner->getInterpreter()->getCommandLine()
@@ -94,15 +114,17 @@ public function prepare(Test $test): void
 	public function finish(Test $test): void
 	{
 		$this->results[$test->getResult()]++;
-		fwrite($this->file, $this->symbols[$test->getResult()]);
+		$this->lineMode
+			? $this->printFinishedTestLine($test)
+			: $this->printFinishedTestDot($test);
 
 		$title = ($test->title ? "$test->title | " : '') . substr($test->getSignature(), strlen($this->baseDir));
 		$message = '   ' . str_replace("\n", "\n   ", trim((string) $test->message)) . "\n\n";
 		$message = preg_replace('/^   $/m', '', $message);
 		if ($test->getResult() === Test::Failed) {
-			$this->buffer .= Dumper::color('red', "-- FAILED: $title") . "\n$message";
+			$this->buffer[] = Dumper::color('red', "-- FAILED: $title") . "\n$message";
 		} elseif ($test->getResult() === Test::Skipped && $this->displaySkipped) {
-			$this->buffer .= "-- Skipped: $title\n$message";
+			$this->buffer[] = "-- Skipped: $title\n$message";
 		}
 	}
 
@@ -111,14 +133,73 @@ public function end(): void
 	{
 		$run = array_sum($this->results);
 		fwrite($this->file, !$this->count ? "No tests found\n" :
-			"\n\n" . $this->buffer . "\n"
+			"\n\n" . implode('', $this->buffer) . "\n"
 			. ($this->results[Test::Failed] ? Dumper::color('white/red') . 'FAILURES!' : Dumper::color('white/green') . 'OK')
 			. " ($this->count test" . ($this->count > 1 ? 's' : '') . ', '
 			. ($this->results[Test::Failed] ? $this->results[Test::Failed] . ' failure' . ($this->results[Test::Failed] > 1 ? 's' : '') . ', ' : '')
 			. ($this->results[Test::Skipped] ? $this->results[Test::Skipped] . ' skipped, ' : '')
 			. ($this->count !== $run ? ($this->count - $run) . ' not run, ' : '')
 			. sprintf('%0.1f', $this->time + microtime(true)) . ' seconds)' . Dumper::color() . "\n");
 
-		$this->buffer = '';
+		$this->buffer = [];
+		$this->resultsCount = 0;
+	}
+
+
+	private function printFinishedTestDot(Test $test): void
+	{
+		fwrite($this->file, $this->symbols[$test->getResult()]);
+	}
+
+
+	private function printFinishedTestLine(Test $test): void
+	{
+		$this->resultsCount++;
+		$result = $test->getResult();
+
+		$shortFilePath = str_replace($this->baseDir, '', $test->getFile());
+		$shortDirPath = dirname($shortFilePath) . DIRECTORY_SEPARATOR;
+		$basename = basename($shortFilePath);
+
+		// Filename.
+		if ($result === Test::Failed) {
+			$fileText =	Dumper::color('red', $shortDirPath) . Dumper::color('white/red', $basename);
+		} else {
+			$fileText =	Dumper::color('gray', $shortDirPath) . Dumper::color('silver', $basename);
+		}
+
+		// Line header.
+		$header = "· ";
+		// Test's title.
+		$titleText = $test->title
+			? Dumper::color('fuchsia', " [$test->title]")
+			: '';
+
+		// Print test's message only if it's not failed (those will be printed
+		// after all tests are finished and will contain detailed info about the
+		// failed test).
+		$message = '';
+		if ($result !== Test::Failed && $test->message) {
+			$message = $test->message;
+			$indent = str_repeat(' ', mb_strlen($header));
+
+			if (preg_match('#\n#', $message)) {
+				$message = "\n$indent" . preg_replace('#\r?\n#', '\0' . $indent, $message);
+			} else {
+				$message = Dumper::color('olive', "[$message]");
+			}
+		}
+
+		$output = sprintf(
+			"%s%s %s %s %s %s\n",
+			$header,
+			"{$this->resultsCount}/{$this->count}",
+			"$fileText{$titleText}",
+			$this->symbols[$result],
+			Dumper::color('gray', sprintf("in %.2f s", $test->getDuration())),
+			$message,
+		);
+
+		fwrite($this->file, $output);
 	}
 }

src/Runner/Test.php

@@ -28,14 +28,23 @@ class Test
 		PASSED = self::Passed,
 		SKIPPED = self::Skipped;
 
+	private const PossibleResults = [
+		self::Prepared,
+		self::Failed,
+		self::Passed,
+		self::Skipped,
+	];
+
 	public ?string $title;
 	public ?string $message = null;
 	public string $stdout = '';
 	public string $stderr = '';
 	private string $file;
-	private int $result = self::Prepared;
 	private ?float $duration = null;
 
+	/** @phpstan-var Alias_TestResultState */
+	private int $result = self::Prepared;
+
 	/** @var string[]|string[][] */
 	private $args = [];
 
@@ -70,6 +79,9 @@ public function getSignature(): string
 	}
 
 
+	/**
+	 * @phpstan-return Alias_TestResultState
+	 */
 	public function getResult(): int
 	{
 		return $this->result;
@@ -123,6 +135,7 @@ public function withArguments(array $args): self
 
 
 	/**
+	 * @phpstan-param Alias_TestResultState $result
 	 * @return static
 	 */
 	public function withResult(int $result, ?string $message, ?float $duration = null): self
@@ -131,6 +144,10 @@ public function withResult(int $result, ?string $message, ?float $duration = nul
 			throw new \LogicException("Result of test is already set to $this->result with message '$this->message'.");
 		}
 
+		if (!in_array($result, self::PossibleResults, true)) {
+			throw new \LogicException("Invalid test result $result");
+		}
+
 		$me = clone $this;
 		$me->result = $result;
 		$me->message = $message;

tests/RunnerOutput/OutputHandlers.expect.consoleLines.txt

@@ -0,0 +1,73 @@
+%a% | %a% | 1 thread
+
+· 1/%d% ./01-basic.fail.phptx FAIL in %f% s
+· 2/%d% ./01-basic.pass.phptx OK in %f% s
+· 3/%d% ./01-basic.skip.phptx SKIP in %f% s
+· 4/%d% ./02-title.fail.phptx [Title for output handlers] FAIL in %f% s
+· 5/%d% ./02-title.pass.phptx [Title for output handlers] OK in %f% s
+· 6/%d% ./02-title.skip.phptx [Title for output handlers] SKIP in %f% s
+· 7/%d% ./03-message.fail.phptx FAIL in %f% s
+· 8/%d% ./03-message.skip.phptx SKIP in %f% s
+  Multi
+  line
+  message.
+· 9/%d% ./04-args.fail.phptx FAIL in %f% s
+· 10/%d% ./04-args.pass.phptx OK in %f% s
+· 11/%d% ./04-args.skip.phptx SKIP in %f% s
+  Multi
+  line
+  message.
+
+
+-- FAILED: 01-basic.fail.phptx
+   Multi
+   line
+   stdout.Failed:
+
+   in %a%01-basic.fail.phptx(%d%) Tester\Assert::fail('');
+
+   STDERR:
+   Multi
+   line
+   stderr.
+
+-- FAILED: Title for output handlers | 02-title.fail.phptx
+   Multi
+   line
+   stdout.Failed:
+
+   in %a%02-title.fail.phptx(%d%) Tester\Assert::fail('');
+
+   STDERR:
+   Multi
+   line
+   stderr.
+
+-- FAILED: 03-message.fail.phptx
+   Multi
+   line
+   stdout.Failed: Multi
+   line
+   message.
+
+   in %a%03-message.fail.phptx(%d%) Tester\Assert::fail("Multi\nline\nmessage.");
+
+   STDERR:
+   Multi
+   line
+   stderr.
+
+-- FAILED: 04-args.fail.phptx dataprovider=thisIsAVeryVeryVeryLongArgumentNameToTestHowOutputHandlersDealWithItsLengthInTheirOutputFormatting|%a%provider.ini
+   Multi
+   line
+   stdout.Failed:
+
+   in %a%04-args.fail.phptx(%d%) Tester\Assert::fail('');
+
+   STDERR:
+   Multi
+   line
+   stderr.
+
+
+FAILURES! (11 tests, 4 failures, 4 skipped, %a% seconds)