chore(testRunner): expose pauseAtEnd/pauseOnError (#37923)

Author: dgozman

packages/playwright/src/common/ipc.ts

@@ -86,6 +86,11 @@ export type AttachmentPayload = {
 
 export type TestInfoErrorImpl = TestInfoError;
 
+export type TestPausedPayload = {
+  errors: TestInfoErrorImpl[];
+  extraData: any;
+};
+
 export type TestEndPayload = {
   testId: string;
   duration: number;

packages/playwright/src/index.ts

@@ -22,7 +22,7 @@ import { setBoxedStackPrefixes, createGuid, currentZone, debugMode, jsonStringif
 
 import { currentTestInfo } from './common/globals';
 import { rootTestType } from './common/testType';
-import { runBrowserBackendAtEnd } from './mcp/test/browserBackend';
+import { runBrowserBackendOnTestPause } from './mcp/test/browserBackend';
 
 import type { Fixtures, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions, ScreenshotMode, TestInfo, TestType, VideoMode } from '../types/test';
 import type { ContextReuseMode } from './common/config';
@@ -237,7 +237,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
       (testInfo as TestInfoImpl)._setDebugMode();
 
     playwright._defaultContextOptions = _combinedContextOptions;
-    playwright._defaultContextTimeout = (testInfo as TestInfoImpl)._pauseOnError() ? 5000 : actionTimeout || 0;
+    playwright._defaultContextTimeout = actionTimeout || 0;
     playwright._defaultContextNavigationTimeout = navigationTimeout || 0;
     await use();
     playwright._defaultContextOptions = undefined;
@@ -417,14 +417,14 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
     attachConnectedHeaderIfNeeded(testInfo, browserImpl);
     if (!_reuseContext) {
       const { context, close } = await _contextFactory();
-      (testInfo as TestInfoImpl)._onDidFinishTestFunctions.unshift(() => runBrowserBackendAtEnd(context, testInfo.errors[0]?.message));
+      (testInfo as TestInfoImpl)._onDidPauseTestCallback = () => runBrowserBackendOnTestPause(testInfo, context);
       await use(context);
       await close();
       return;
     }
 
     const context = await browserImpl._wrapApiCall(() => browserImpl._newContextForReuse(), { internal: true });
-    (testInfo as TestInfoImpl)._onDidFinishTestFunctions.unshift(() => runBrowserBackendAtEnd(context, testInfo.errors[0]?.message));
+    (testInfo as TestInfoImpl)._onDidPauseTestCallback = () => runBrowserBackendOnTestPause(testInfo, context);
     await use(context);
     const closeReason = testInfo.status === 'timedOut' ? 'Test timeout of ' + testInfo.timeout + 'ms exceeded.' : 'Test ended.';
     await browserImpl._wrapApiCall(() => browserImpl._disconnectFromReusedContext(closeReason), { internal: true });
@@ -647,7 +647,7 @@ class ArtifactsRecorder {
 
   async willStartTest(testInfo: TestInfoImpl) {
     this._testInfo = testInfo;
-    testInfo._onDidFinishTestFunctions.push(() => this.didFinishTestFunction());
+    testInfo._onDidFinishTestFunctionCallback = () => this.didFinishTestFunction();
 
     this._screenshotRecorder.fixOrdinal();
 

packages/playwright/src/isomorphic/testServerConnection.ts

@@ -17,6 +17,7 @@
 import * as events from './events';
 
 import type { TestServerInterface, TestServerInterfaceEvents } from '@testIsomorphic/testServerInterface';
+import type * as reporterTypes from '../../types/testReporter';
 
 // -- Reuse boundary -- Everything below this line is reused in the vscode extension.
 
@@ -68,12 +69,14 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
   readonly onStdio: events.Event<{ type: 'stderr' | 'stdout'; text?: string | undefined; buffer?: string | undefined; }>;
   readonly onTestFilesChanged: events.Event<{ testFiles: string[] }>;
   readonly onLoadTraceRequested: events.Event<{ traceUrl: string }>;
+  readonly onTestPaused: events.Event<{ errors: reporterTypes.TestError[] }>;
 
   private _onCloseEmitter = new events.EventEmitter<void>();
   private _onReportEmitter = new events.EventEmitter<any>();
   private _onStdioEmitter = new events.EventEmitter<{ type: 'stderr' | 'stdout'; text?: string | undefined; buffer?: string | undefined; }>();
   private _onTestFilesChangedEmitter = new events.EventEmitter<{ testFiles: string[] }>();
   private _onLoadTraceRequestedEmitter = new events.EventEmitter<{ traceUrl: string }>();
+  private _onTestPausedEmitter = new events.EventEmitter<{ errors: reporterTypes.TestError[] }>();
 
   private _lastId = 0;
   private _transport: TestServerTransport;
@@ -87,6 +90,7 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
     this.onStdio = this._onStdioEmitter.event;
     this.onTestFilesChanged = this._onTestFilesChangedEmitter.event;
     this.onLoadTraceRequested = this._onLoadTraceRequestedEmitter.event;
+    this.onTestPaused = this._onTestPausedEmitter.event;
 
     this._transport = transport;
     this._transport.onmessage(data => {
@@ -147,6 +151,8 @@ export class TestServerConnection implements TestServerInterface, TestServerInte
       this._onTestFilesChangedEmitter.fire(params);
     else if (method === 'loadTraceRequested')
       this._onLoadTraceRequestedEmitter.fire(params);
+    else if (method === 'testPaused')
+      this._onTestPausedEmitter.fire(params);
   }
 
   async initialize(params: Parameters<TestServerInterface['initialize']>[0]): ReturnType<TestServerInterface['initialize']> {

packages/playwright/src/isomorphic/testServerInterface.ts

@@ -103,6 +103,8 @@ export interface TestServerInterface {
     reuseContext?: boolean;
     connectWsEndpoint?: string;
     timeout?: number;
+    pauseOnError?: boolean;
+    pauseAtEnd?: boolean;
   }): Promise<{
     status: reporterTypes.FullResult['status'];
   }>;
@@ -121,11 +123,13 @@ export interface TestServerInterfaceEvents {
   onStdio: Event<{ type: 'stdout' | 'stderr', text?: string, buffer?: string }>;
   onTestFilesChanged: Event<{ testFiles: string[] }>;
   onLoadTraceRequested: Event<{ traceUrl: string }>;
+  onTestPaused: Event<{ errors: reporterTypes.TestError[] }>;
 }
 
 export interface TestServerInterfaceEventEmitters {
   dispatchEvent(event: 'report', params: ReportEntry): void;
   dispatchEvent(event: 'stdio', params: { type: 'stdout' | 'stderr', text?: string, buffer?: string }): void;
   dispatchEvent(event: 'testFilesChanged', params: { testFiles: string[] }): void;
   dispatchEvent(event: 'loadTraceRequested', params: { traceUrl: string }): void;
+  dispatchEvent(event: 'testPaused', params: { errors: reporterTypes.TestError[] }): void;
 }

packages/playwright/src/mcp/sdk/mdb.ts

@@ -17,7 +17,6 @@
 import { debug } from 'playwright-core/lib/utilsBundle';
 import { ManualPromise } from 'playwright-core/lib/utils';
 
-import { defineToolSchema } from './tool';
 import * as mcpBundle from './bundle';
 import * as mcpServer from './server';
 import * as mcpHttp from './http';
@@ -27,7 +26,11 @@ import type { Client } from '@modelcontextprotocol/sdk/client/index.js';
 
 const mdbDebug = debug('pw:mcp:mdb');
 const errorsDebug = debug('pw:mcp:errors');
-const z = mcpBundle.z;
+
+export type MDBPushClientCallback = (mcpUrl: string, introMessage?: string) => Promise<void>;
+export type MDBServerBackendFactory = Omit<mcpServer.ServerBackendFactory, 'create'> & {
+  create: (pushClient: MDBPushClientCallback) => mcpServer.ServerBackend;
+};
 
 export class MDBBackend implements mcpServer.ServerBackend {
   private _onPauseClient: { client: Client, tools: mcpServer.Tool[], transport: StreamableHTTPClientTransport } | undefined;
@@ -37,8 +40,8 @@ export class MDBBackend implements mcpServer.ServerBackend {
   private _progress: mcpServer.CallToolResult['content'] = [];
   private _progressCallback: mcpServer.ProgressCallback;
 
-  constructor(mainBackend: mcpServer.ServerBackend) {
-    this._mainBackend = mainBackend;
+  constructor(mainBackendFactory: MDBServerBackendFactory) {
+    this._mainBackend = mainBackendFactory.create(this._createOnPauseClient.bind(this));
     this._progressCallback = (params: mcpServer.ProgressParams) => {
       if (params.message)
         this._progress.push({ type: 'text', text: params.message });
@@ -57,11 +60,6 @@ export class MDBBackend implements mcpServer.ServerBackend {
   }
 
   async callTool(name: string, args: mcpServer.CallToolRequest['params']['arguments']): Promise<mcpServer.CallToolResult> {
-    if (name === pushToolsSchema.name) {
-      await this._createOnPauseClient(pushToolsSchema.inputSchema.parse(args || {}));
-      return { content: [{ type: 'text', text: 'Tools pushed' }] };
-    }
-
     if (this._onPauseClient?.tools.find(tool => tool.name === name)) {
       const result = await this._onPauseClient.client.callTool({
         name,
@@ -95,16 +93,16 @@ export class MDBBackend implements mcpServer.ServerBackend {
     return result;
   }
 
-  private async _createOnPauseClient(params: { mcpUrl: string, introMessage?: string }) {
+  private async _createOnPauseClient(mcpUrl: string, introMessage?: string) {
     if (this._onPauseClient)
       await this._onPauseClient.client.close().catch(errorsDebug);
 
-    this._onPauseClient = await this._createClient(params.mcpUrl);
+    this._onPauseClient = await this._createClient(mcpUrl);
 
     this._interruptPromise?.resolve({
       content: [{
         type: 'text',
-        text: params.introMessage || '',
+        text: introMessage || '',
       }],
     });
     this._interruptPromise = undefined;
@@ -128,100 +126,18 @@ export class MDBBackend implements mcpServer.ServerBackend {
   }
 }
 
-const pushToolsSchema = defineToolSchema({
-  name: 'mdb_push_tools',
-  title: 'Push MCP tools to the tools stack',
-  description: 'Push MCP tools to the tools stack',
-  inputSchema: z.object({
-    mcpUrl: z.string(),
-    introMessage: z.string().optional(),
-  }),
-  type: 'readOnly',
-});
-
-export async function runMainBackend(backendFactory: mcpServer.ServerBackendFactory, options?: { port?: number }): Promise<string | undefined> {
-  const mdbBackend = new MDBBackend(backendFactory.create());
-  // Start HTTP unconditionally.
+// TODO: add all options from mcpHttp.startHttpServer.
+export async function runMainBackend(backendFactory: MDBServerBackendFactory, options?: { port?: number }): Promise<string | undefined> {
+  const mdbBackend = new MDBBackend(backendFactory);
   const factory: mcpServer.ServerBackendFactory = {
     ...backendFactory,
     create: () => mdbBackend
   };
-  const url = await startAsHttp(factory, { port: options?.port || 0 });
-  process.env.PLAYWRIGHT_DEBUGGER_MCP = url;
-
-  if (options?.port !== undefined)
-    return url;
-
-  // Start stdio conditionally.
-  await mcpServer.connect(factory, new mcpBundle.StdioServerTransport(), false);
-}
-
-export async function runOnPauseBackendLoop(backend: mcpServer.ServerBackend, introMessage: string) {
-  const wrappedBackend = new ServerBackendWithCloseListener(backend);
-
-  const factory = {
-    name: 'on-pause-backend',
-    nameInConfig: 'on-pause-backend',
-    version: '0.0.0',
-    create: () => wrappedBackend,
-  };
-
-  const httpServer = await mcpHttp.startHttpServer({ port: 0 });
-  const url = await mcpHttp.installHttpTransport(httpServer, factory, true);
-
-  const client = new mcpBundle.Client({ name: 'Pushing client', version: '0.0.0' });
-  client.setRequestHandler(mcpBundle.PingRequestSchema, () => ({}));
-  const transport = new mcpBundle.StreamableHTTPClientTransport(new URL(process.env.PLAYWRIGHT_DEBUGGER_MCP!));
-  await client.connect(transport);
-
-  const pushToolsResult = await client.callTool({
-    name: pushToolsSchema.name,
-    arguments: {
-      mcpUrl: url,
-      introMessage,
-    },
-  });
-  if (pushToolsResult.isError)
-    errorsDebug('Failed to push tools', pushToolsResult.content);
-  await transport.terminateSession();
-  await client.close();
-
-  await wrappedBackend.waitForClosed();
-  httpServer.close();
-}
-
-async function startAsHttp(backendFactory: mcpServer.ServerBackendFactory, options: { port: number }) {
-  const httpServer = await mcpHttp.startHttpServer(options);
-  return await mcpHttp.installHttpTransport(httpServer, backendFactory, true);
-}
 
-
-class ServerBackendWithCloseListener implements mcpServer.ServerBackend {
-  private _backend: mcpServer.ServerBackend;
-  private _serverClosedPromise = new ManualPromise<void>();
-
-  constructor(backend: mcpServer.ServerBackend) {
-    this._backend = backend;
-  }
-
-  async initialize(server: mcpServer.Server, clientInfo: mcpServer.ClientInfo): Promise<void> {
-    await this._backend.initialize?.(server, clientInfo);
-  }
-
-  async listTools(): Promise<mcpServer.Tool[]> {
-    return this._backend.listTools();
-  }
-
-  async callTool(name: string, args: mcpServer.CallToolRequest['params']['arguments'], progress: mcpServer.ProgressCallback): Promise<mcpServer.CallToolResult> {
-    return this._backend.callTool(name, args, progress);
+  if (options?.port !== undefined) {
+    const httpServer = await mcpHttp.startHttpServer(options);
+    return await mcpHttp.installHttpTransport(httpServer, factory, true);
   }
 
-  serverClosed(server: mcpServer.Server) {
-    this._backend.serverClosed?.(server);
-    this._serverClosedPromise.resolve();
-  }
-
-  async waitForClosed() {
-    await this._serverClosedPromise;
-  }
+  await mcpServer.connect(factory, new mcpBundle.StdioServerTransport(), false);
 }

packages/playwright/src/mcp/test/browserBackend.ts

@@ -15,31 +15,22 @@
  */
 
 import * as mcp from '../sdk/exports';
-import { currentTestInfo } from '../../common/globals';
-import { stripAnsiEscapes } from '../../util';
 import { defaultConfig, FullConfig } from '../browser/config';
 import { BrowserServerBackend } from '../browser/browserServerBackend';
 import { Tab } from '../browser/tab';
 
 import type * as playwright from '../../../index';
 import type { Page } from '../../../../playwright-core/src/client/page';
 import type { BrowserContextFactory } from '../browser/browserContextFactory';
-import type { ClientInfo } from '../sdk/server';
+import type { TestInfo } from '../../../test';
 
-export async function runBrowserBackendAtEnd(context: playwright.BrowserContext, errorMessage?: string) {
-  const testInfo = currentTestInfo();
-  if (!testInfo)
-    return;
-
-  const shouldPause = errorMessage ? testInfo?._pauseOnError() : testInfo?._pauseAtEnd();
-  if (!shouldPause)
-    return;
+export type TestPausedExtraData = {
+  mcpUrl: string;
+  contextState: string;
+};
 
+export async function runBrowserBackendOnTestPause(testInfo: TestInfo, context: playwright.BrowserContext) {
   const lines: string[] = [];
-  if (errorMessage)
-    lines.push(`### Paused on error:`, stripAnsiEscapes(errorMessage));
-  else
-    lines.push(`### Paused at end of test. ready for interaction`);
 
   for (let i = 0; i < context.pages().length; i++) {
     const page = context.pages()[i];
@@ -51,7 +42,7 @@ export async function runBrowserBackendAtEnd(context: playwright.BrowserContext,
         `- Page Title: ${await page.title()}`.trim()
     );
     // Only print console errors when pausing on error, not when everything works as expected.
-    let console = errorMessage ? await Tab.collectConsoleMessages(page) : [];
+    let console = testInfo.errors.length ? await Tab.collectConsoleMessages(page) : [];
     console = console.filter(msg => !msg.type || msg.type === 'error');
     if (console.length) {
       lines.push('- Console Messages:');
@@ -66,21 +57,29 @@ export async function runBrowserBackendAtEnd(context: playwright.BrowserContext,
     );
   }
 
-  lines.push('');
-  if (errorMessage)
-    lines.push(`### Task`, `Try recovering from the error prior to continuing`);
-
   const config: FullConfig = {
     ...defaultConfig,
     capabilities: ['testing'],
   };
 
-  await mcp.runOnPauseBackendLoop(new BrowserServerBackend(config, identityFactory(context)), lines.join('\n'));
+  const factory: mcp.ServerBackendFactory = {
+    name: 'Playwright',
+    nameInConfig: 'playwright',
+    version: '0.0.0',
+    create: () => new BrowserServerBackend(config, identityFactory(context))
+  };
+  const httpServer = await mcp.startHttpServer({ port: 0 });
+  const mcpUrl = await mcp.installHttpTransport(httpServer, factory, true);
+  const dispose = async () => {
+    await new Promise(cb => httpServer.close(cb));
+  };
+  const extraData = { mcpUrl, contextState: lines.join('\n') } as TestPausedExtraData;
+  return { extraData, dispose };
 }
 
 function identityFactory(browserContext: playwright.BrowserContext): BrowserContextFactory {
   return {
-    createContext: async (clientInfo: ClientInfo, abortSignal: AbortSignal, toolName: string | undefined) => {
+    createContext: async (clientInfo: mcp.ClientInfo, abortSignal: AbortSignal, toolName: string | undefined) => {
       return {
         browserContext,
         close: async () => {}

packages/playwright/src/mcp/test/testBackend.ts

@@ -40,8 +40,8 @@ export class TestServerBackend implements mcp.ServerBackend {
   private _context: TestContext;
   private _configOption: string | undefined;
 
-  constructor(configOption: string | undefined, options?: { muteConsole?: boolean, headless?: boolean }) {
-    this._context = new TestContext(options);
+  constructor(configOption: string | undefined, pushClient: mcp.MDBPushClientCallback, options?: { muteConsole?: boolean, headless?: boolean }) {
+    this._context = new TestContext(pushClient, options);
     this._configOption = configOption;
   }