Block Delimiter: Harden performance test to reduce flakiness

projects/packages/block-delimiter/changelog/update-block-delimiter-performance-comparision-flakiness-HOG-307

@@ -0,0 +1,5 @@
+Significance: patch
+Type: changed
+Comment: Reduce flakiness of Block_Scanner performance comparison unit test
+
+

projects/packages/block-delimiter/composer.json

@@ -16,6 +16,14 @@
 			"src/"
 		]
 	},
+	"autoload-dev": {
+		"classmap": [
+			"tests/php/lib/"
+		],
+		"psr-4": {
+			"Automattic\\Block_Delimiter\\Tests\\": "tests/php/"
+		}
+	},
 	"scripts": {
 		"phpunit": [
 			"phpunit-select-config phpunit.#.xml.dist --colors=always"

projects/packages/block-delimiter/tests/php/Block_Scanner_Test.php

@@ -2,8 +2,10 @@
 
 namespace Automattic;
 
+use Automattic\Block_Delimiter\Tests\Lib\Performance_Benchmark_Utils;
 use PHPUnit\Framework\Attributes\CoversClass;
 use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\Attributes\Group;
 use PHPUnit\Framework\TestCase;
 
 /**
@@ -724,185 +726,55 @@ public function test_malformed_json_handling(): void {
 	}
 
 	/**
-	 * Test performance comparison between parse_blocks and Block_Scanner.
+	 * Test performance regression detection between parse_blocks and Block_Scanner.
 	 *
-	 * Verifies Block_Scanner is faster when finding specific blocks early.
+	 * Ensures Block_Scanner performance doesn't significantly regress compared to parse_blocks.
+	 * Uses CI-friendly thresholds and extensive retry logic to minimize false positives.
+	 *
+	 * @group performance
 	 */
+	#[Group( 'performance' )]
 	public function test_performance_comparison(): void {
-		if ( ! function_exists( 'parse_blocks' ) ) {
+		if ( ! \function_exists( 'parse_blocks' ) ) {
 			$this->markTestSkipped( 'parse_blocks not available. Block editor not available' );
 		}
 
-		if ( ! function_exists( 'serialize_blocks' ) ) {
+		if ( ! \function_exists( 'serialize_blocks' ) ) {
 			$this->markTestSkipped( 'serialize_blocks not available. Block editor not available' );
 		}
 
-		$test_content = $this->generate_test_content_with_target_image();
-
-		// Benchmark both approaches
-		// Warm-up to avoid autoloader/JIT noise in timed runs
-		$this->find_first_image_with_scanner( $test_content );
-		$this->find_first_image_with_parse_blocks( $test_content );
-
-		$scanner_metrics      = $this->benchmark_scanner_search_multiple( $test_content, 5 );
-		$parse_blocks_metrics = $this->benchmark_parse_blocks_search_multiple( $test_content, 5 );
-
-		// Verify correctness
-		$this->assert_same_results( $scanner_metrics['result'], $parse_blocks_metrics['result'] );
-
-		// Verify performance advantage
-		$this->assert_performance_advantage( $scanner_metrics, $parse_blocks_metrics );
-	}
-
-	/**
-	 * Generate test content with target image positioned for early termination test.
-	 *
-	 * @return string Block content with image at ~1/3 position.
-	 */
-	private function generate_test_content_with_target_image(): string {
-		$blocks = array();
-
-		// Add ~130 paragraph blocks before target
-		for ( $i = 0; $i < 130; $i++ ) {
-			$blocks[] = array(
-				'blockName'    => 'core/paragraph',
-				'attrs'        => array(),
-				'innerContent' => array( sprintf( '<p>Content %d</p>', $i + 1 ) ),
-			);
-		}
-
-		// Add target image block
-		$blocks[] = array(
-			'blockName'    => 'core/image',
-			'attrs'        => array(
-				'id'  => 12345,
-				'alt' => 'Test image',
-			),
-			'innerContent' => array( '<figure class="wp-block-image"><img src="test.jpg" alt="Test image"/></figure>' ),
-		);
-
-		// Add more blocks after target (parse_blocks will process these, scanner won't)
-		for ( $i = 0; $i < 270; $i++ ) {
-			$blocks[] = array(
-				'blockName'    => 'core/paragraph',
-				'attrs'        => array(),
-				'innerContent' => array( sprintf( '<p>After %d</p>', $i + 1 ) ),
-			);
+		// Skip test under high system load conditions to reduce CI flakiness
+		if ( Performance_Benchmark_Utils::is_system_under_high_load() ) {
+			$this->markTestSkipped( 'System under high load - skipping performance test for stability' );
 		}
 
-		return serialize_blocks( $blocks );
-	}
+		$test_content = Performance_Benchmark_Utils::generate_test_content_with_target_image();
 
-	/**
-	 * Benchmark Block_Scanner search approach.
-	 *
-	 * @param string $content Content to search.
-	 * @return array Metrics with timing, memory, and result data.
-	 */
-	private function benchmark_scanner_search( string $content ): array {
-		$start_time   = microtime( true );
-		$start_memory = memory_get_usage();
-
-		$result = $this->find_first_image_with_scanner( $content );
-
-		return array(
-			'time'   => microtime( true ) - $start_time,
-			'memory' => memory_get_usage() - $start_memory,
-			'result' => $result,
-		);
-	}
-
-	/**
-	 * Benchmark parse_blocks search approach.
-	 *
-	 * @param string $content Content to search.
-	 * @return array Metrics with timing, memory, and result data.
-	 */
-	private function benchmark_parse_blocks_search( string $content ): array {
-		$start_time   = microtime( true );
-		$start_memory = memory_get_usage();
-
-		$result = $this->find_first_image_with_parse_blocks( $content );
-
-		return array(
-			'time'   => microtime( true ) - $start_time,
-			'memory' => memory_get_usage() - $start_memory,
-			'result' => $result,
-		);
-	}
-
-	/**
-	 * Benchmark Block_Scanner search approach over multiple iterations and aggregate results.
-	 *
-	 * @param string $content    Content to search.
-	 * @param int    $iterations Number of iterations to run.
-	 * @return array Metrics with aggregated timing (median), peak memory delta, and result data.
-	 */
-	private function benchmark_scanner_search_multiple( string $content, int $iterations = 5 ): array {
-		return $this->run_benchmark_multiple(
-			function () use ( $content ) {
-				return $this->find_first_image_with_scanner( $content );
+		// Run competitive benchmark with paired measurements
+		$benchmark_results = Performance_Benchmark_Utils::run_competitive_benchmark(
+			function () use ( $test_content ) {
+				return $this->find_first_image_with_scanner( $test_content );
 			},
-			$iterations
-		);
-	}
-
-	/**
-	 * Benchmark parse_blocks search approach over multiple iterations and aggregate results.
-	 *
-	 * @param string $content    Content to search.
-	 * @param int    $iterations Number of iterations to run.
-	 * @return array Metrics with aggregated timing (median), peak memory delta, and result data.
-	 */
-	private function benchmark_parse_blocks_search_multiple( string $content, int $iterations = 5 ): array {
-		return $this->run_benchmark_multiple(
-			function () use ( $content ) {
-				return $this->find_first_image_with_parse_blocks( $content );
+			function () use ( $test_content ) {
+				return $this->find_first_image_with_parse_blocks( $test_content );
 			},
-			$iterations
+			Performance_Benchmark_Utils::PERF_BENCHMARK_ITERATIONS
 		);
-	}
-
-	/**
-	 * Run a callable multiple times, returning aggregated benchmark metrics.
-	 * Uses median time to reduce noise and the maximum observed memory delta as a conservative proxy for peak.
-	 *
-	 * @param callable $callable   Callable to benchmark.
-	 * @param int      $iterations Number of iterations to run.
-	 * @return array{time: float, memory: int, result: mixed}
-	 */
-	private function run_benchmark_multiple( callable $callable, int $iterations ): array {
-		$times  = array();
-		$mems   = array();
-		$result = null;
-
-		for ( $i = 0; $i < $iterations; $i++ ) {
-			// Encourage collection between runs to reduce cross-iteration interference.
-			if ( function_exists( 'gc_collect_cycles' ) ) {
-				gc_collect_cycles();
-			}
-
-			$start_memory = memory_get_usage();
-			$start_time   = microtime( true );
-
-			$run_result = $callable();
 
-			$times[] = microtime( true ) - $start_time;
-			$mems[]  = max( 0, memory_get_usage() - $start_memory );
+		// Verify correctness - both methods should find the same result
+		$scanner_result      = $this->find_first_image_with_scanner( $test_content );
+		$parse_blocks_result = $this->find_first_image_with_parse_blocks( $test_content );
+		$this->assert_same_results( $scanner_result, $parse_blocks_result );
 
-			if ( null === $result && null !== $run_result ) {
-				$result = $run_result;
+		Performance_Benchmark_Utils::assert_performance_advantage_paired(
+			$this,
+			$benchmark_results,
+			function () use ( $test_content ) {
+				return $this->find_first_image_with_scanner( $test_content );
+			},
+			function () use ( $test_content ) {
+				return $this->find_first_image_with_parse_blocks( $test_content );
 			}
-		}
-
-		sort( $times );
-		$median_time = $times[ (int) floor( count( $times ) / 2 ) ];
-		$peak_memory = max( $mems );
-
-		return array(
-			'time'   => $median_time,
-			'memory' => $peak_memory,
-			'result' => $result,
 		);
 	}
 
@@ -919,26 +791,6 @@ private function assert_same_results( ?array $scanner_result, ?array $parse_bloc
 		$this->assertSame( $parse_blocks_result['attrs'], $scanner_result['attrs'] );
 	}
 
-	/**
-	 * Assert Block_Scanner has performance advantage.
-	 *
-	 * @param array $scanner_metrics Scanner performance data.
-	 * @param array $parse_blocks_metrics parse_blocks performance data.
-	 */
-	private function assert_performance_advantage( array $scanner_metrics, array $parse_blocks_metrics ): void {
-		// Assert time advantage using median across multiple iterations to reduce noise.
-		$time_ratio = $parse_blocks_metrics['time'] / $scanner_metrics['time'];
-		$this->assertGreaterThan( 1.15, $time_ratio, 'Scanner should be measurably faster than parse_blocks' );
-
-		// Assert memory is not worse than parse_blocks beyond a small tolerance for measurement noise.
-		$memory_tolerance = 4096; // 4KB tolerance
-		$this->assertLessThanOrEqual(
-			$parse_blocks_metrics['memory'] + $memory_tolerance,
-			$scanner_metrics['memory'],
-			'Scanner should use comparable or less memory than parse_blocks'
-		);
-	}
-
 	/**
 	 * Find the first image block using Block_Scanner.
 	 *
@@ -969,7 +821,7 @@ private function find_first_image_with_scanner( string $content ): ?array {
 	 * @return array|null Image block data or null if not found.
 	 */
 	private function find_first_image_with_parse_blocks( string $content ): ?array {
-		$blocks = parse_blocks( $content );
+		$blocks = \parse_blocks( $content );
 
 		foreach ( $blocks as $block ) {
 			$result = $this->search_blocks_recursive( array( $block ), 'core/image' );
@@ -1009,4 +861,28 @@ private function search_blocks_recursive( array $blocks, string $target_type ):
 
 		return null;
 	}
+
+	/**
+	 * Test that scanner and parse_blocks find the same first image block.
+	 *
+	 * This is an always-on correctness test that verifies both methods return
+	 * the same result without performance assertions.
+	 */
+	public function test_scanner_and_parse_blocks_find_same_first_image(): void {
+		if ( ! \function_exists( 'parse_blocks' ) ) {
+			$this->markTestSkipped( 'parse_blocks not available. Block editor not available' );
+		}
+
+		if ( ! \function_exists( 'serialize_blocks' ) ) {
+			$this->markTestSkipped( 'serialize_blocks not available. Block editor not available' );
+		}
+
+		$test_content = Performance_Benchmark_Utils::generate_test_content_with_target_image();
+
+		$scanner_result      = $this->find_first_image_with_scanner( $test_content );
+		$parse_blocks_result = $this->find_first_image_with_parse_blocks( $test_content );
+
+		// Verify correctness only
+		$this->assert_same_results( $scanner_result, $parse_blocks_result );
+	}
 }