diff --git a/src/Runner/TestSuiteSorter.php b/src/Runner/TestSuiteSorter.php index b5c0c19d695..077893108bb 100644 --- a/src/Runner/TestSuiteSorter.php +++ b/src/Runner/TestSuiteSorter.php @@ -76,16 +76,6 @@ final class TestSuiteSorter private array $defectSortOrder = []; private readonly ResultCache $cache; - /** - * @psalm-var array A list of normalized names of tests before reordering - */ - private array $originalExecutionOrder = []; - - /** - * @psalm-var array A list of normalized names of tests affected by reordering - */ - private array $executionOrder = []; - public function __construct(?ResultCache $cache = null) { $this->cache = $cache ?? new NullResultCache; @@ -94,7 +84,7 @@ public function __construct(?ResultCache $cache = null) /** * @throws Exception */ - public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects, bool $isRootTestSuite = true): void + public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDependencies, int $orderDefects): void { $allowedOrders = [ self::ORDER_DEFAULT, @@ -117,13 +107,9 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend throw new InvalidOrderException; } - if ($isRootTestSuite) { - $this->originalExecutionOrder = $this->calculateTestExecutionOrder($suite); - } - if ($suite instanceof TestSuite) { foreach ($suite as $_suite) { - $this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects, false); + $this->reorderTestsInSuite($_suite, $order, $resolveDependencies, $orderDefects); } if ($orderDefects === self::ORDER_DEFECTS_FIRST) { @@ -132,20 +118,6 @@ public function reorderTestsInSuite(Test $suite, int $order, bool $resolveDepend $this->sort($suite, $order, $resolveDependencies, $orderDefects); } - - if ($isRootTestSuite) { - $this->executionOrder = $this->calculateTestExecutionOrder($suite); - } - } - - public function getOriginalExecutionOrder(): array - { - return $this->originalExecutionOrder; - } - - public function getExecutionOrder(): array - { - return $this->executionOrder; } private function sort(TestSuite $suite, int $order, bool $resolveDependencies, int $orderDefects): void @@ -324,21 +296,4 @@ private function resolveDependencies(array $tests): array return array_merge($newTestOrder, $tests); } - - private function calculateTestExecutionOrder(Test $suite): array - { - $tests = []; - - if ($suite instanceof TestSuite) { - foreach ($suite->tests() as $test) { - if (!$test instanceof TestSuite && $test instanceof Reorderable) { - $tests[] = $test->sortId(); - } else { - $tests = array_merge($tests, $this->calculateTestExecutionOrder($test)); - } - } - } - - return $tests; - } } diff --git a/tests/_files/TestWithDifferentSizes.php b/tests/_files/TestWithDifferentSizes.php deleted file mode 100644 index 31ff76389ab..00000000000 --- a/tests/_files/TestWithDifferentSizes.php +++ /dev/null @@ -1,76 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\TestFixture; - -use PHPUnit\Framework\TestCase; - -final class TestWithDifferentSizes extends TestCase -{ - public static function provider(): array - { - return [ - [false], - [true], - ]; - } - - public function testWithSizeUnknown(): void - { - $this->assertTrue(true); - } - - /** - * @large - */ - public function testWithSizeLarge(): void - { - $this->assertTrue(true); - } - - /** - * @depends testDataProviderWithSizeMedium - * - * @medium - */ - public function testWithSizeMedium(): void - { - $this->assertTrue(true); - } - - /** - * @depends testWithSizeLarge - * - * @small - */ - public function testWithSizeSmall(): void - { - $this->assertTrue(true); - } - - /** - * @dataProvider provider - * - * @small - */ - public function testDataProviderWithSizeSmall(bool $value): void - { - $this->assertTrue(true); - } - - /** - * @dataProvider provider - * - * @medium - */ - public function testDataProviderWithSizeMedium(bool $value): void - { - $this->assertTrue(true); - } -} diff --git a/tests/end-to-end/execution-order/_files/ClonedDependencyTest.php b/tests/end-to-end/execution-order/_files/ClonedDependencyTest.php deleted file mode 100644 index e557414f091..00000000000 --- a/tests/end-to-end/execution-order/_files/ClonedDependencyTest.php +++ /dev/null @@ -1,70 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\TestFixture; - -use PHPUnit\Framework\TestCase; -use stdClass; - -class ClonedDependencyTest extends TestCase -{ - private static $dependency; - - public static function setUpBeforeClass(): void - { - self::$dependency = new stdClass; - } - - public function testOne() - { - $this->assertTrue(true); - - return self::$dependency; - } - - /** - * @depends testOne - */ - public function testTwo($dependency): void - { - $this->assertSame(self::$dependency, $dependency); - } - - /** - * @depends !clone testOne - */ - public function testThree($dependency): void - { - $this->assertSame(self::$dependency, $dependency); - } - - /** - * @depends clone testOne - */ - public function testFour($dependency): void - { - $this->assertNotSame(self::$dependency, $dependency); - } - - /** - * @depends !shallowClone testOne - */ - public function testFive($dependency): void - { - $this->assertSame(self::$dependency, $dependency); - } - - /** - * @depends shallowClone testOne - */ - public function testSix($dependency): void - { - $this->assertNotSame(self::$dependency, $dependency); - } -} diff --git a/tests/end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt b/tests/end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt deleted file mode 100644 index a9735ef50ec..00000000000 --- a/tests/end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt +++ /dev/null @@ -1 +0,0 @@ -{"version":2,"defects":{"PHPUnit\\TestFixture\\MultiDependencyTest::testFive":1},"times":{"PHPUnit\\TestFixture\\MultiDependencyTest::testOne":0,"PHPUnit\\TestFixture\\MultiDependencyTest::testTwo":0,"PHPUnit\\TestFixture\\MultiDependencyTest::testThree":0,"PHPUnit\\TestFixture\\MultiDependencyTest::testFour":0,"PHPUnit\\TestFixture\\MultiDependencyTest::testFive":0}} diff --git a/tests/end-to-end/execution-order/_files/StackTest.php b/tests/end-to-end/execution-order/_files/StackTest.php deleted file mode 100644 index d11fd67e0b9..00000000000 --- a/tests/end-to-end/execution-order/_files/StackTest.php +++ /dev/null @@ -1,38 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\TestFixture; - -use function array_pop; -use function end; -use PHPUnit\Framework\TestCase; - -class StackTest extends TestCase -{ - public function testPush() - { - $stack = []; - $this->assertCount(0, $stack); - - $stack[] = 'foo'; - $this->assertEquals('foo', end($stack)); - $this->assertCount(1, $stack); - - return $stack; - } - - /** - * @depends testPush - */ - public function testPop(array $stack): void - { - $this->assertEquals('foo', array_pop($stack)); - $this->assertCount(0, $stack); - } -} diff --git a/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.phpunit.result.cache.txt b/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.phpunit.result.cache.txt deleted file mode 100644 index 8fd87ae1f03..00000000000 --- a/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.phpunit.result.cache.txt +++ /dev/null @@ -1 +0,0 @@ -{"version":2,"defects":[],"times":{"PHPUnit\\TestFixture\\TestWithDifferentDurations::testOne":2.006,"PHPUnit\\TestFixture\\TestWithDifferentDurations::testTwo":0,"PHPUnit\\TestFixture\\TestWithDifferentDurations::testThree":3.001,"PHPUnit\\TestFixture\\ExampleTest::testOne":2.006,"PHPUnit\\TestFixture\\ExampleTest::testTwo":3.001,"PHPUnit\\TestFixture\\ExampleTest::testThree":0}} diff --git a/tests/end-to-end/execution-order/_files/order-by-duration.phpunit.xml b/tests/end-to-end/execution-order/_files/order-by-duration.phpunit.xml deleted file mode 100644 index 2ef82a66223..00000000000 --- a/tests/end-to-end/execution-order/_files/order-by-duration.phpunit.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - ./TestWithDifferentDurations.php - - - diff --git a/tests/end-to-end/execution-order/cache-result.phpt b/tests/end-to-end/execution-order/cache-result.phpt deleted file mode 100644 index 32d66e71f90..00000000000 --- a/tests/end-to-end/execution-order/cache-result.phpt +++ /dev/null @@ -1,34 +0,0 @@ ---TEST-- -phpunit --order-by=no-depends,reverse --cache-result --cache-result-file ./tests/_files/MultiDependencyTest.php ---FILE-- -run($_SERVER['argv']); - -print file_get_contents($cacheDirectory . DIRECTORY_SEPARATOR . 'test-results'); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -.SS.. 5 / 5 (100%) - -Time: %s, Memory: %s - -OK, but some tests were skipped! -Tests: 5, Assertions: 3, Skipped: 2. -{"version":2,"defects":{"PHPUnit\\TestFixture\\MultiDependencyTest::testFour":1,"PHPUnit\\TestFixture\\MultiDependencyTest::testThree":1},"times":{"PHPUnit\\TestFixture\\MultiDependencyTest::testFive":%f,"PHPUnit\\TestFixture\\MultiDependencyTest::testFour":%f,"PHPUnit\\TestFixture\\MultiDependencyTest::testThree":%f,"PHPUnit\\TestFixture\\MultiDependencyTest::testTwo":%f,"PHPUnit\\TestFixture\\MultiDependencyTest::testOne":%f}} ---CLEAN-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -...... 6 / 6 (100%) - -Time: %s, Memory: %s - -OK (6 tests, 6 assertions) - diff --git a/tests/end-to-end/execution-order/dependencies-isolation.phpt b/tests/end-to-end/execution-order/dependencies-isolation.phpt deleted file mode 100644 index 4b924f59e57..00000000000 --- a/tests/end-to-end/execution-order/dependencies-isolation.phpt +++ /dev/null @@ -1,54 +0,0 @@ ---TEST-- -phpunit --process-isolation --display-skipped ../../_files/dependencies ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -...FSSSEE 9 / 9 (100%) - -Time: %s, Memory: %s - -There were 2 errors: - -1) PHPUnit\TestFixture\DependencyFailureTest::testHandlesDependsAnnotationForNonexistentTests -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::doesNotExist" which does not exist - -2) PHPUnit\TestFixture\DependencyFailureTest::testHandlesDependsAnnotationWithNoMethodSpecified -This test has an invalid dependency - --- - -There was 1 failure: - -1) PHPUnit\TestFixture\DependencyFailureTest::testOne -Failed asserting that false is true. - -%s:%i - --- - -There were 3 skipped tests: - -1) PHPUnit\TestFixture\DependencyFailureTest::testTwo -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testOne" to pass - -2) PHPUnit\TestFixture\DependencyFailureTest::testThree -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testTwo" to pass - -3) PHPUnit\TestFixture\DependencyFailureTest::testFour -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testOne" to pass - -ERRORS! -Tests: 9, Assertions: 4, Errors: 2, Failures: 1, Skipped: 3. diff --git a/tests/end-to-end/execution-order/depends-as-parameter-with-isolation.phpt b/tests/end-to-end/execution-order/depends-as-parameter-with-isolation.phpt deleted file mode 100644 index 3e1a5ce946f..00000000000 --- a/tests/end-to-end/execution-order/depends-as-parameter-with-isolation.phpt +++ /dev/null @@ -1,22 +0,0 @@ ---TEST-- -phpunit --process-isolation _files/StackTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -.. 2 / 2 (100%) - -Time: %s, Memory: %s - -OK (2 tests, 5 assertions) diff --git a/tests/end-to-end/execution-order/depends-as-parameter.phpt b/tests/end-to-end/execution-order/depends-as-parameter.phpt deleted file mode 100644 index d7461f6eb16..00000000000 --- a/tests/end-to-end/execution-order/depends-as-parameter.phpt +++ /dev/null @@ -1,21 +0,0 @@ ---TEST-- -phpunit _files/StackTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -.. 2 / 2 (100%) - -Time: %s, Memory: %s - -OK (2 tests, 5 assertions) diff --git a/tests/end-to-end/execution-order/depends-multiple-parameter-with-isolation.phpt b/tests/end-to-end/execution-order/depends-multiple-parameter-with-isolation.phpt deleted file mode 100644 index 9f19922e0aa..00000000000 --- a/tests/end-to-end/execution-order/depends-multiple-parameter-with-isolation.phpt +++ /dev/null @@ -1,22 +0,0 @@ ---TEST-- -phpunit --process-isolation _files/MultiDependencyTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -..... 5 / 5 (100%) - -Time: %s, Memory: %s - -OK (5 tests, 6 assertions) diff --git a/tests/end-to-end/execution-order/depends-multiple-parameters.phpt b/tests/end-to-end/execution-order/depends-multiple-parameters.phpt deleted file mode 100644 index 02c6884b487..00000000000 --- a/tests/end-to-end/execution-order/depends-multiple-parameters.phpt +++ /dev/null @@ -1,21 +0,0 @@ ---TEST-- -phpunit _files/MultiDependencyTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s - -..... 5 / 5 (100%) - -Time: %s, Memory: %s - -OK (5 tests, 6 assertions) diff --git a/tests/end-to-end/execution-order/depends-on-class.phpt b/tests/end-to-end/execution-order/depends-on-class.phpt deleted file mode 100644 index ee179155d5a..00000000000 --- a/tests/end-to-end/execution-order/depends-on-class.phpt +++ /dev/null @@ -1,58 +0,0 @@ ---TEST-- -phpunit -c ../../_files/configuration.depends-on-class.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s -Configuration: %s%etests%e_files%econfiguration.depends-on-class.xml - -....SFSSSEE 11 / 11 (100%) - -Time: %s, Memory: %s - -There were 2 errors: - -1) PHPUnit\TestFixture\DependencyFailureTest::testHandlesDependsAnnotationForNonexistentTests -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::doesNotExist" which does not exist - -2) PHPUnit\TestFixture\DependencyFailureTest::testHandlesDependsAnnotationWithNoMethodSpecified -This test has an invalid dependency - --- - -There was 1 failure: - -1) PHPUnit\TestFixture\DependencyFailureTest::testOne -Failed asserting that false is true. - -%s%etests%e_files%edependencies%eDependencyFailureTest.php:18 - --- - -There were 4 skipped tests: - -1) PHPUnit\TestFixture\DependencyOnClassTest::testThatDependsOnAFailingClass -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::class" to pass - -2) PHPUnit\TestFixture\DependencyFailureTest::testTwo -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testOne" to pass - -3) PHPUnit\TestFixture\DependencyFailureTest::testThree -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testTwo" to pass - -4) PHPUnit\TestFixture\DependencyFailureTest::testFour -This test depends on "PHPUnit\TestFixture\DependencyFailureTest::testOne" to pass - -ERRORS! -Tests: 11, Assertions: 5, Errors: 2, Failures: 1, Skipped: 4. diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php new file mode 100644 index 00000000000..d78e1c5d2aa --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BarTest.php @@ -0,0 +1,38 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\Defects; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\TestCase; +use RuntimeException; + +#[CoversNothing] +final class BarTest extends TestCase +{ + public function testOne(): void + { + $this->assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(false); + } + + public function testThree(): void + { + throw new RuntimeException('message'); + } + + public function testFour(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php new file mode 100644 index 00000000000..4325f660cfb --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/BazTest.php @@ -0,0 +1,38 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\Defects; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\TestCase; +use RuntimeException; + +#[CoversNothing] +final class BazTest extends TestCase +{ + public function testOne(): void + { + $this->assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(false); + } + + public function testThree(): void + { + throw new RuntimeException('message'); + } + + public function testFour(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-defects/test-results b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/test-results new file mode 100644 index 00000000000..0939ed3671f --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-defects/test-results @@ -0,0 +1 @@ +{"version":2,"defects":{"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testTwo":7,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testThree":8,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testTwo":7,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testThree":8},"times":{"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testOne":0.002,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testTwo":0.003,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testThree":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BarTest::testFour":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testOne":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testTwo":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testThree":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\BazTest::testFour":0}} \ No newline at end of file diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php new file mode 100644 index 00000000000..e44d7da4196 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/EndToEndTest.php @@ -0,0 +1,24 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\DifferentSizes; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\Attributes\Large; +use PHPUnit\Framework\TestCase; + +#[Large] +#[CoversNothing] +final class EndToEndTest extends TestCase +{ + public function testOne(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php new file mode 100644 index 00000000000..912cceb656c --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/IntegrationTest.php @@ -0,0 +1,24 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\DifferentSizes; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\Attributes\Medium; +use PHPUnit\Framework\TestCase; + +#[Medium] +#[CoversNothing] +final class IntegrationTest extends TestCase +{ + public function testOne(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php new file mode 100644 index 00000000000..04130056b04 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes-with-different-sizes/UnitTest.php @@ -0,0 +1,24 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\DifferentSizes; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\Attributes\Small; +use PHPUnit\Framework\TestCase; + +#[Small] +#[CoversNothing] +final class UnitTest extends TestCase +{ + public function testOne(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php b/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php new file mode 100644 index 00000000000..d0294712f5d --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-classes/BarTest.php @@ -0,0 +1,22 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\TestCase; + +#[CoversNothing] +final class BarTest extends TestCase +{ + public function testOne(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php b/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php similarity index 72% rename from tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php rename to tests/end-to-end/execution-order/fixture/test-classes/FooTest.php index b01a39f23db..e4180f5461f 100644 --- a/tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php +++ b/tests/end-to-end/execution-order/fixture/test-classes/FooTest.php @@ -7,11 +7,13 @@ * For the full copyright and license information, please view the LICENSE * file that was distributed with this source code. */ -namespace PHPUnit\TestFixture; +namespace PHPUnit\TestFixture\ExecutionOrder; +use PHPUnit\Framework\Attributes\CoversNothing; use PHPUnit\Framework\TestCase; -final class TestWithDifferentDurations extends TestCase +#[CoversNothing] +final class FooTest extends TestCase { public function testOne(): void { @@ -22,9 +24,4 @@ public function testTwo(): void { $this->assertTrue(true); } - - public function testThree(): void - { - $this->assertTrue(true); - } } diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php new file mode 100644 index 00000000000..7617de2e0bb --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/FooTest.php @@ -0,0 +1,38 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\Defects; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\TestCase; +use RuntimeException; + +#[CoversNothing] +final class FooTest extends TestCase +{ + public function testOne(): void + { + $this->assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(false); + } + + public function testThree(): void + { + throw new RuntimeException('message'); + } + + public function testFour(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-defects/test-results b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/test-results new file mode 100644 index 00000000000..eb8ac71fc22 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-defects/test-results @@ -0,0 +1 @@ +{"version":2,"defects":{"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testTwo":7,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testThree":8},"times":{"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testOne":0,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testTwo":0.001,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testThree":0.004,"PHPUnit\\TestFixture\\ExecutionOrder\\Defects\\FooTest::testFour":0}} \ No newline at end of file diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php new file mode 100644 index 00000000000..d0111c51d45 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-dependencies/FooTest.php @@ -0,0 +1,29 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\Dependencies; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\Attributes\Depends; +use PHPUnit\Framework\TestCase; + +#[CoversNothing] +final class FooTest extends TestCase +{ + #[Depends('testTwo')] + public function testOne(): void + { + $this->assertTrue(true); + } + + public function testTwo(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php b/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php new file mode 100644 index 00000000000..640689781d7 --- /dev/null +++ b/tests/end-to-end/execution-order/fixture/test-methods-with-different-sizes/SizesTest.php @@ -0,0 +1,41 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ +namespace PHPUnit\TestFixture\ExecutionOrder\DifferentSizes; + +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\TestCase; + +#[CoversNothing] +final class SizesTest extends TestCase +{ + /** + * @medium + */ + public function testOne(): void + { + $this->assertTrue(true); + } + + /** + * @large + */ + public function testTwo(): void + { + $this->assertTrue(true); + } + + /** + * @small + */ + public function testThree(): void + { + $this->assertTrue(true); + } +} diff --git a/tests/end-to-end/execution-order/order-by-default-invalid-via-cli.phpt b/tests/end-to-end/execution-order/order-by-default-invalid-via-cli.phpt deleted file mode 100644 index ad7c5efc69c..00000000000 --- a/tests/end-to-end/execution-order/order-by-default-invalid-via-cli.phpt +++ /dev/null @@ -1,15 +0,0 @@ ---TEST-- -phpunit --order-by=default,foobar ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -unrecognized --order-by option: foobar diff --git a/tests/end-to-end/execution-order/order-by-default-test-classes-with-defects.phpt b/tests/end-to-end/execution-order/order-by-default-test-classes-with-defects.phpt new file mode 100644 index 00000000000..f06b9456a2f --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-classes-with-defects.phpt @@ -0,0 +1,73 @@ +--TEST-- +Default order: Test classes with defects +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (8 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (8 tests) +Test Suite Started (CLI Arguments, 8 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Suite Finished (CLI Arguments, 8 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-default-test-classes-with-different-sizes.phpt b/tests/end-to-end/execution-order/order-by-default-test-classes-with-different-sizes.phpt new file mode 100644 index 00000000000..70798229c1c --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-classes-with-different-sizes.phpt @@ -0,0 +1,48 @@ +--TEST-- +Default order: Suite with test classes that have different sizes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest, 1 test) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-default-test-classes.phpt b/tests/end-to-end/execution-order/order-by-default-test-classes.phpt new file mode 100644 index 00000000000..71f606586ed --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-classes.phpt @@ -0,0 +1,46 @@ +--TEST-- +Default order: Suite with test classes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\BarTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\BarTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-default-test-methods-with-defects.phpt b/tests/end-to-end/execution-order/order-by-default-test-methods-with-defects.phpt new file mode 100644 index 00000000000..c58504764e7 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-methods-with-defects.phpt @@ -0,0 +1,50 @@ +--TEST-- +Default order: Test methods with defects +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (4 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (4 tests) +Test Suite Started (CLI Arguments, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Suite Finished (CLI Arguments, 4 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-default-test-methods-with-dependencies.phpt b/tests/end-to-end/execution-order/order-by-default-test-methods-with-dependencies.phpt new file mode 100644 index 00000000000..83653d503ac --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-methods-with-dependencies.phpt @@ -0,0 +1,37 @@ +--TEST-- +Default order: Test methods with dependencies +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (2 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest, 2 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-default-test-methods-with-different-sizes.phpt b/tests/end-to-end/execution-order/order-by-default-test-methods-with-different-sizes.phpt new file mode 100644 index 00000000000..1a599ea04fb --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-methods-with-different-sizes.phpt @@ -0,0 +1,44 @@ +--TEST-- +Default order: Test class with test methods that have different sizes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest, 3 tests) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-default-test-methods.phpt b/tests/end-to-end/execution-order/order-by-default-test-methods.phpt new file mode 100644 index 00000000000..1a5989eb0c2 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-default-test-methods.phpt @@ -0,0 +1,37 @@ +--TEST-- +Default order: Test methods +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (2 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects-but-result-cache-does-not-exist.phpt b/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects-but-result-cache-does-not-exist.phpt new file mode 100644 index 00000000000..7ef75b33218 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects-but-result-cache-does-not-exist.phpt @@ -0,0 +1,80 @@ +--TEST-- +Order by defects (without result cache): Test classes with defects +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (8 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (8 tests) +Test Suite Started (CLI Arguments, 8 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Suite Finished (CLI Arguments, 8 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects.phpt b/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects.phpt new file mode 100644 index 00000000000..134895a6f71 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-defects-test-classes-with-defects.phpt @@ -0,0 +1,84 @@ +--TEST-- +Order by defects (with result cache): Test classes with defects +--FILE-- +run($_SERVER['argv']); + +unlink($testResultsFile); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (8 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (8 tests) +Test Suite Started (CLI Arguments, 8 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testFour) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BarTest, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\BazTest, 4 tests) +Test Suite Finished (CLI Arguments, 8 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects-but-result-cache-does-not-exist.phpt b/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects-but-result-cache-does-not-exist.phpt new file mode 100644 index 00000000000..75290020c23 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects-but-result-cache-does-not-exist.phpt @@ -0,0 +1,57 @@ +--TEST-- +Order by defects (without result cache): Test methods with defects +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (4 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (4 tests) +Test Suite Started (CLI Arguments, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Suite Finished (CLI Arguments, 4 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects.phpt b/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects.phpt new file mode 100644 index 00000000000..5d2326dcf76 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-defects-test-methods-with-defects.phpt @@ -0,0 +1,61 @@ +--TEST-- +Order by defects (with result cache): Test methods with defects +--FILE-- +run($_SERVER['argv']); + +unlink($testResultsFile); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (4 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (4 tests) +Test Suite Started (CLI Arguments, 4 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Errored (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +message +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Assertion Failed (Constraint: is true, Value: false) +Test Failed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Failed asserting that false is true. +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest::testFour) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Defects\FooTest, 4 tests) +Test Suite Finished (CLI Arguments, 4 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 2) diff --git a/tests/end-to-end/execution-order/order-by-duration-via-cli.phpt b/tests/end-to-end/execution-order/order-by-duration-via-cli.phpt deleted file mode 100644 index 9c9c73aa24a..00000000000 --- a/tests/end-to-end/execution-order/order-by-duration-via-cli.phpt +++ /dev/null @@ -1,50 +0,0 @@ ---TEST-- -phpunit --order-by=duration ./tests/end-to-end/execution-order/_files/TestWithDifferentDurations.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit Started (PHPUnit %s using PHP %s) -Test Runner Configured -Test Suite Loaded (3 tests) -Event Facade Sealed -Test Runner Started -Test Suite Sorted -Test Runner Execution Started (3 tests) -Test Suite Started (PHPUnit\TestFixture\TestWithDifferentDurations, 3 tests) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Suite Finished (PHPUnit\TestFixture\TestWithDifferentDurations, 3 tests) -Test Runner Execution Finished -Test Runner Finished -PHPUnit Finished (Shell Exit Code: 0) ---CLEAN-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit Started (PHPUnit %s using PHP %s) -Test Runner Configured -Test Suite Loaded (3 tests) -Event Facade Sealed -Test Runner Started -Test Suite Sorted -Test Runner Execution Started (3 tests) -Test Suite Started (%sorder-by-duration.phpunit.xml, 3 tests) -Test Suite Started (order-by-duration, 3 tests) -Test Suite Started (PHPUnit\TestFixture\TestWithDifferentDurations, 3 tests) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testTwo) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testOne) -Test Preparation Started (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Prepared (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Assertion Succeeded (Constraint: is true, Value: true) -Test Passed (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Finished (PHPUnit\TestFixture\TestWithDifferentDurations::testThree) -Test Suite Finished (PHPUnit\TestFixture\TestWithDifferentDurations, 3 tests) -Test Suite Finished (order-by-duration, 3 tests) -Test Suite Finished (%sorder-by-duration.phpunit.xml, 3 tests) -Test Runner Execution Finished -Test Runner Finished -PHPUnit Finished (Shell Exit Code: 0) ---CLEAN-- -run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (2 tests) +Event Facade Sealed +Test Runner Started +Test Runner Execution Started (2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest, 2 tests) +Test Skipped (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testOne) +This test depends on "PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo" to pass +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\Dependencies\FooTest, 2 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-reverse-test-classes.phpt b/tests/end-to-end/execution-order/order-by-reverse-test-classes.phpt new file mode 100644 index 00000000000..08c8a856202 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-reverse-test-classes.phpt @@ -0,0 +1,46 @@ +--TEST-- +Reverse order: Suite with test classes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\BarTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\BarTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\BarTest, 1 test) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-reverse-test-methods.phpt b/tests/end-to-end/execution-order/order-by-reverse-test-methods.phpt new file mode 100644 index 00000000000..9d976d17fa6 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-reverse-test-methods.phpt @@ -0,0 +1,37 @@ +--TEST-- +Reverse order: Test methods +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (2 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (2 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testTwo) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\FooTest, 2 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-size-test-classes-with-different-sizes.phpt b/tests/end-to-end/execution-order/order-by-size-test-classes-with-different-sizes.phpt new file mode 100644 index 00000000000..36c08dca4ce --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-size-test-classes-with-different-sizes.phpt @@ -0,0 +1,50 @@ +--TEST-- +Order by test size: Suite with test classes that have different sizes +--XFAIL-- +TestSuiteSorter::cmpSize() does not handle TestSuite objects for TestCase classes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\UnitTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\IntegrationTest, 1 test) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest, 1 test) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest::testOne) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\EndToEndTest, 1 test) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/order-by-size-test-methods-with-different-sizes.phpt b/tests/end-to-end/execution-order/order-by-size-test-methods-with-different-sizes.phpt new file mode 100644 index 00000000000..bcfce3ee358 --- /dev/null +++ b/tests/end-to-end/execution-order/order-by-size-test-methods-with-different-sizes.phpt @@ -0,0 +1,44 @@ +--TEST-- +Order by test size: Test class with test methods that have different sizes +--FILE-- +run($_SERVER['argv']); +--EXPECTF-- +PHPUnit Started (PHPUnit %s using %s) +Test Runner Configured +Test Suite Loaded (3 tests) +Event Facade Sealed +Test Runner Started +Test Suite Sorted +Test Runner Execution Started (3 tests) +Test Suite Started (CLI Arguments, 3 tests) +Test Suite Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest, 3 tests) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testThree) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testOne) +Test Preparation Started (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Prepared (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Assertion Succeeded (Constraint: is true, Value: true) +Test Passed (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest::testTwo) +Test Suite Finished (PHPUnit\TestFixture\ExecutionOrder\DifferentSizes\SizesTest, 3 tests) +Test Suite Finished (CLI Arguments, 3 tests) +Test Runner Execution Finished +Test Runner Finished +PHPUnit Finished (Shell Exit Code: 0) diff --git a/tests/end-to-end/execution-order/test-order-randomized-with-dependency-resolution.phpt b/tests/end-to-end/execution-order/test-order-randomized-with-dependency-resolution.phpt deleted file mode 100644 index 2c81c005efb..00000000000 --- a/tests/end-to-end/execution-order/test-order-randomized-with-dependency-resolution.phpt +++ /dev/null @@ -1,24 +0,0 @@ ---TEST-- -phpunit --order-by=depends,random ../_files/MultiDependencyTest.php ---FILE-- -run($_SERVER['argv']); ---EXPECTF-- -PHPUnit %s by Sebastian Bergmann and contributors. - -Runtime: %s -Random Seed: %d - -..... 5 / 5 (100%) - -Time: %s, Memory: %s - -OK (5 tests, 6 assertions) diff --git a/tests/unit/Runner/ResultCache/DefaultResultCacheTest.php b/tests/unit/Runner/ResultCache/DefaultResultCacheTest.php deleted file mode 100644 index c2c4224b1d6..00000000000 --- a/tests/unit/Runner/ResultCache/DefaultResultCacheTest.php +++ /dev/null @@ -1,99 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\Runner\ResultCache; - -use function sys_get_temp_dir; -use function tempnam; -use function uniqid; -use function unlink; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\Small; -use PHPUnit\Framework\TestCase; -use PHPUnit\Framework\TestStatus\TestStatus; -use PHPUnit\TestFixture\MultiDependencyTest; - -#[CoversClass(DefaultResultCache::class)] -#[Small] -final class DefaultResultCacheTest extends TestCase -{ - public function testGetTimeForNonExistentTestNameReturnsFloatZero(): void - { - $this->assertSame(0.0, (new DefaultResultCache)->time('doesNotExist')); - } - - public function testReadsCacheFromProvidedFilename(): void - { - $cacheFile = TEST_FILES_PATH . '../end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt'; - $cache = new DefaultResultCache($cacheFile); - $cache->load(); - - $this->assertTrue($cache->status(MultiDependencyTest::class . '::testOne')->isUnknown()); - $this->assertTrue($cache->status(MultiDependencyTest::class . '::testFive')->isSkipped()); - } - - public function testDoesClearCacheBeforeLoad(): void - { - $cacheFile = TEST_FILES_PATH . '../end-to-end/execution-order/_files/MultiDependencyTest_result_cache.txt'; - $cache = new DefaultResultCache($cacheFile); - $cache->setStatus('someTest', TestStatus::failure()); - - $this->assertTrue($cache->status(MultiDependencyTest::class . '::testFive')->isUnknown()); - - $cache->load(); - - $this->assertTrue($cache->status(MultiDependencyTest::class . '::someTest')->isUnknown()); - $this->assertTrue($cache->status(MultiDependencyTest::class . '::testFive')->isSkipped()); - } - - public function testCanPersistCacheToFile(): void - { - $cacheFile = tempnam(sys_get_temp_dir(), 'phpunit_'); - $cache = new DefaultResultCache($cacheFile); - $testName = 'test' . uniqid('', true); - - $cache->setStatus($testName, TestStatus::skipped()); - $cache->persist(); - - $cache = new DefaultResultCache($cacheFile); - $cache->load(); - - $this->assertTrue($cache->status($testName)->isSkipped()); - - unlink($cacheFile); - } - - public function testCanBeMerged(): void - { - $cacheSourceOne = new DefaultResultCache; - $cacheSourceOne->setStatus('status.a', TestStatus::skipped()); - $cacheSourceOne->setStatus('status.b', TestStatus::incomplete()); - $cacheSourceOne->setTime('time.a', 1); - $cacheSourceOne->setTime('time.b', 2); - $cacheSourceTwo = new DefaultResultCache; - $cacheSourceTwo->setStatus('status.c', TestStatus::failure()); - $cacheSourceTwo->setTime('time.c', 4); - - $sum = new DefaultResultCache; - $sum->mergeWith($cacheSourceOne); - - $this->assertSame(TestStatus::skipped()->asString(), $sum->status('status.a')->asString()); - $this->assertSame(TestStatus::incomplete()->asString(), $sum->status('status.b')->asString()); - $this->assertNotSame(TestStatus::failure()->asString(), $sum->status('status.c')->asString()); - - $this->assertSame(1.0, $sum->time('time.a')); - $this->assertSame(2.0, $sum->time('time.b')); - $this->assertNotSame(4.0, $sum->time('time.c')); - - $sum->mergeWith($cacheSourceTwo); - - $this->assertSame(TestStatus::failure()->asString(), $sum->status('status.c')->asString()); - $this->assertSame(4.0, $sum->time('time.c')); - } -} diff --git a/tests/unit/Runner/ResultCache/NullTestResultCacheTest.php b/tests/unit/Runner/ResultCache/NullTestResultCacheTest.php deleted file mode 100644 index c421013190f..00000000000 --- a/tests/unit/Runner/ResultCache/NullTestResultCacheTest.php +++ /dev/null @@ -1,29 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\Runner\ResultCache; - -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\Small; -use PHPUnit\Framework\TestCase; - -#[CoversClass(NullResultCache::class)] -#[Small] -final class NullTestResultCacheTest extends TestCase -{ - public function testHasWorkingStubs(): void - { - $cache = new NullResultCache; - $cache->load(); - $cache->persist(); - - $this->assertTrue($cache->status('testName')->isUnknown()); - $this->assertSame(0.0, $cache->time('testName')); - } -} diff --git a/tests/unit/Runner/TestSuiteSorterTest.php b/tests/unit/Runner/TestSuiteSorterTest.php deleted file mode 100644 index acf68db50ce..00000000000 --- a/tests/unit/Runner/TestSuiteSorterTest.php +++ /dev/null @@ -1,647 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ -namespace PHPUnit\Runner; - -use function mt_srand; -use PHPUnit\Framework\Attributes\CoversClass; -use PHPUnit\Framework\Attributes\DataProvider; -use PHPUnit\Framework\Attributes\Small; -use PHPUnit\Framework\TestCase; -use PHPUnit\Framework\TestStatus\TestStatus; -use PHPUnit\Framework\TestSuite; -use PHPUnit\Runner\ResultCache\DefaultResultCache; -use PHPUnit\TestFixture\MultiDependencyTest; -use PHPUnit\TestFixture\TestWithDifferentSizes; -use ReflectionClass; - -#[CoversClass(TestSuiteSorter::class)] -#[Small] -final class TestSuiteSorterTest extends TestCase -{ - private const IGNORE_DEPENDENCIES = false; - private const RESOLVE_DEPENDENCIES = true; - private const MULTI_DEPENDENCY_TEST_EXECUTION_ORDER = [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ]; - - public static function orderDurationWithoutCacheProvider(): array - { - return [ - 'dependency-ignore' => [ - self::IGNORE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - 'dependency-resolve' => [ - self::RESOLVE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - ]; - } - - public static function orderDurationWithCacheProvider(): array - { - return [ - 'duration-same-dependency-ignore' => [ - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => 1, - 'testTwo' => 1, - 'testThree' => 1, - 'testFour' => 1, - 'testFive' => 1, - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - 'duration-same-dependency-resolve' => [ - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => 1, - 'testTwo' => 1, - 'testThree' => 1, - 'testFour' => 1, - 'testFive' => 1, - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - 'duration-different-dependency-ignore' => [ - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => 5, - 'testTwo' => 3, - 'testThree' => 4, - 'testFour' => 1, - 'testFive' => 2, - ], - [ - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testOne', - ], - ], - 'duration-different-dependency-resolve' => [ - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => 5, - 'testTwo' => 3, - 'testThree' => 4, - 'testFour' => 1, - 'testFive' => 2, - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - ]; - } - - /** - * A data provider for basic execution reordering options based on MultiDependencyTest. - * - * This class has the following relevant properties: - * - * - it has five tests testOne, testTwo, testThree, testFour, testFive - * - testThree depends on testOne and testTwo - * - testFour depends on MultiDependencyTest::testThree - * - testFive has no dependencies - */ - public static function commonSorterOptionsProvider(): array - { - return [ - 'default' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Activating dependency resolution should have no effect under normal circumstances - 'resolve default' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::RESOLVE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Reversing without checks should give a simple reverse order - 'reverse' => [ - TestSuiteSorter::ORDER_REVERSED, - self::IGNORE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - ], - ], - - // Reversing with resolution still allows testFive to move to front, testTwo before testOne - 'resolve reverse' => [ - TestSuiteSorter::ORDER_REVERSED, - self::RESOLVE_DEPENDENCIES, - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - ]; - } - - /** - * A data provider for testing defects execution reordering options based on MultiDependencyTest. - * - * This class has the following relevant properties: - * - * - it has five tests testOne, testTwo, testThree, testFour, testFive - * - testThree depends on testOne and testTwo - * - testFour depends on MultiDependencyTest::testThree - * - testFive has no dependencies - */ - public static function defectsSorterOptionsProvider(): array - { - return [ - // The most simple situation should work as normal - 'default, no defects' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::success(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::success(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Running with an empty cache should not spook the TestSuiteSorter - 'default, empty result cache' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - // empty result cache - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // testFive is independent and can be moved to the front - 'default, testFive skipped' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::success(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::skipped(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - - // Defects in testFive and testTwo, but the faster testFive should be run first - 'default, testTwo testFive skipped' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::skipped(), 'time' => 1], - 'testThree' => ['state' => TestStatus::success(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::skipped(), 'time' => 0], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - - // Skipping testThree will move it to the front when ignoring dependencies - 'default, testThree skipped' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::success(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Skipping testThree will move it to the front but behind its dependencies - 'default resolve, testThree skipped' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::success(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - ], - ], - - // Skipping testThree will move it to the front and keep the others reversed - 'reverse, testThree skipped' => [ - TestSuiteSorter::ORDER_REVERSED, - self::IGNORE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::success(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - ], - ], - - // Demonstrate a limit of the dependency resolver: after sorting defects to the front, - // the resolver will mark testFive done before testThree because of dependencies - 'default resolve, testThree skipped, testFive fast' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 0], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::skipped(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - - // Torture test - // - incomplete TestResultCache - // - skipped testThree: will move it to the front as far as possible - // - testOne and testTwo are required before testThree, but can be reversed locally - // - testFive is independent will remain reversed up front - 'reverse resolve, testThree skipped' => [ - TestSuiteSorter::ORDER_REVERSED, - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::skipped(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - - // Make sure the dependency resolver is not confused by failing tests. - // Scenario: Four has a @depends on Three and fails. Result: Three is still run first - // testFive also fails but can be moved around freely and will be up front. - 'depends first, then defects' => [ - TestSuiteSorter::ORDER_DEFAULT, - self::RESOLVE_DEPENDENCIES, - [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::success(), 'time' => 1], - 'testFour' => ['state' => TestStatus::failure(), 'time' => 1], - 'testFive' => ['state' => TestStatus::failure(), 'time' => 1], - ], - [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ], - ], - ]; - } - - public static function suiteSorterOptionPermutationsProvider(): array - { - $orderValues = [TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_REVERSED, TestSuiteSorter::ORDER_RANDOMIZED]; - $resolveValues = [false, true]; - $orderDefectsValues = [TestSuiteSorter::ORDER_DEFAULT, TestSuiteSorter::ORDER_DEFECTS_FIRST]; - - $data = []; - - foreach ($orderValues as $order) { - foreach ($resolveValues as $resolve) { - foreach ($orderDefectsValues as $orderDefects) { - $data[] = [$order, $resolve, $orderDefects]; - } - } - } - - return $data; - } - - public function testThrowsExceptionWhenUsingInvalidOrderOption(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - $this->expectException(InvalidOrderException::class); - - $sorter->reorderTestsInSuite($suite, -1, false, TestSuiteSorter::ORDER_DEFAULT); - } - - public function testThrowsExceptionWhenUsingInvalidOrderDefectsOption(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - $this->expectException(InvalidOrderException::class); - - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_DEFAULT, false, -1); - } - - #[DataProvider('suiteSorterOptionPermutationsProvider')] - public function testShouldNotAffectEmptyTestSuite(int $order, bool $resolveDependencies, int $orderDefects): void - { - $sorter = new TestSuiteSorter; - $suite = TestSuite::empty('test suite name'); - - $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, $orderDefects); - - $this->assertEmpty($suite->tests()); - $this->assertEmpty($sorter->getOriginalExecutionOrder()); - $this->assertEmpty($sorter->getExecutionOrder()); - } - - #[DataProvider('commonSorterOptionsProvider')] - public function testBasicExecutionOrderOptions(int $order, bool $resolveDependencies, array $expectedOrder): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, TestSuiteSorter::ORDER_DEFAULT); - - $this->assertSame(self::MULTI_DEPENDENCY_TEST_EXECUTION_ORDER, $sorter->getOriginalExecutionOrder()); - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } - - public function testCanSetRandomizationWithASeed(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - mt_srand(54321); - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_RANDOMIZED, false, TestSuiteSorter::ORDER_DEFAULT); - - $expectedOrder = [ - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testOne', - ]; - - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } - - public function testCanSetRandomizationWithDefectsFirst(): void - { - $cache = new DefaultResultCache; - - $runState = [ - 'testOne' => ['state' => TestStatus::success(), 'time' => 1], - 'testTwo' => ['state' => TestStatus::success(), 'time' => 1], - 'testThree' => ['state' => TestStatus::error(), 'time' => 1], - 'testFour' => ['state' => TestStatus::success(), 'time' => 1], - 'testFive' => ['state' => TestStatus::error(), 'time' => 1], - ]; - - foreach ($runState as $testName => $data) { - $cache->setStatus(MultiDependencyTest::class . '::' . $testName, $data['state']); - $cache->setTime(MultiDependencyTest::class . '::' . $testName, $data['time']); - } - - $sorter = new TestSuiteSorter($cache); - - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - - mt_srand(54321); - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_RANDOMIZED, false, TestSuiteSorter::ORDER_DEFECTS_FIRST); - - $expectedOrder = [ - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testFour', - MultiDependencyTest::class . '::testOne', - ]; - - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } - - public function testCanSetRandomizationWithASeedAndResolveDependencies(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - $sorter = new TestSuiteSorter; - - mt_srand(54321); - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_RANDOMIZED, true, TestSuiteSorter::ORDER_DEFAULT); - - $expectedOrder = [ - MultiDependencyTest::class . '::testTwo', - MultiDependencyTest::class . '::testFive', - MultiDependencyTest::class . '::testOne', - MultiDependencyTest::class . '::testThree', - MultiDependencyTest::class . '::testFour', - ]; - - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } - - #[DataProvider('orderDurationWithoutCacheProvider')] - public function testOrderDurationWithoutCache(bool $resolveDependencies, array $expected): void - { - $suite = TestSuite::empty('test suite name'); - - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - - $sorter = new TestSuiteSorter; - - $sorter->reorderTestsInSuite( - $suite, - TestSuiteSorter::ORDER_DURATION, - $resolveDependencies, - TestSuiteSorter::ORDER_DEFAULT, - ); - - $this->assertSame($expected, $sorter->getExecutionOrder()); - } - - #[DataProvider('orderDurationWithCacheProvider')] - public function testOrderDurationWithCache(bool $resolveDependencies, array $testTimes, array $expected): void - { - $suite = TestSuite::empty('test suite name'); - - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - - $cache = new DefaultResultCache; - - foreach ($testTimes as $testName => $time) { - $cache->setTime(MultiDependencyTest::class . '::' . $testName, $time); - } - - $sorter = new TestSuiteSorter($cache); - - $sorter->reorderTestsInSuite( - $suite, - TestSuiteSorter::ORDER_DURATION, - $resolveDependencies, - TestSuiteSorter::ORDER_DEFAULT, - ); - - $this->assertSame($expected, $sorter->getExecutionOrder()); - } - - #[DataProvider('defectsSorterOptionsProvider')] - public function testSuiteSorterDefectsOptions(int $order, bool $resolveDependencies, array $runState, array $expected): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(MultiDependencyTest::class)); - - $cache = new DefaultResultCache; - - foreach ($runState as $testName => $data) { - $cache->setStatus(MultiDependencyTest::class . '::' . $testName, $data['state']); - $cache->setTime(MultiDependencyTest::class . '::' . $testName, $data['time']); - } - - $sorter = new TestSuiteSorter($cache); - $sorter->reorderTestsInSuite($suite, $order, $resolveDependencies, TestSuiteSorter::ORDER_DEFECTS_FIRST); - - $this->assertSame($expected, $sorter->getExecutionOrder()); - } - - public function testOrderBySize(): void - { - $suite = TestSuite::empty('test suite name'); - $suite->addTestSuite(new ReflectionClass(TestWithDifferentSizes::class)); - $sorter = new TestSuiteSorter; - - $sorter->reorderTestsInSuite($suite, TestSuiteSorter::ORDER_SIZE, true, TestSuiteSorter::ORDER_DEFAULT); - - $expectedOrder = [ - TestWithDifferentSizes::class . '::testDataProviderWithSizeSmall with data set #0', - TestWithDifferentSizes::class . '::testDataProviderWithSizeSmall with data set #1', - TestWithDifferentSizes::class . '::testDataProviderWithSizeMedium with data set #0', - TestWithDifferentSizes::class . '::testDataProviderWithSizeMedium with data set #1', - TestWithDifferentSizes::class . '::testWithSizeMedium', - TestWithDifferentSizes::class . '::testWithSizeLarge', - TestWithDifferentSizes::class . '::testWithSizeSmall', - TestWithDifferentSizes::class . '::testWithSizeUnknown', - ]; - - $this->assertSame($expectedOrder, $sorter->getExecutionOrder()); - } -}