chore: page.agent timeouts sorted (#38839)

Author: dgozman

docs/src/api/class-page.md

@@ -722,6 +722,11 @@ Initialize page agent with the llm provider and cache.
   - `cacheFile` ?<[string]> Cache file to use/generate code for performed actions into. Cache is not used if not specified (default).
   - `cacheOutFile` ?<[string]> When specified, generated entries are written into the `cacheOutFile` instead of updating the `cacheFile`.
 
+### option: Page.agent.expect
+* since: v1.58
+- `expect` <[Object]>
+  - `timeout` ?<[int]> Default timeout for expect calls in milliseconds, defaults to 5000ms.
+
 ### option: Page.agent.limits
 * since: v1.58
 - `limits` <[Object]>

docs/src/api/class-pageagent.md

@@ -35,6 +35,12 @@ await agent.expect('"0 items" to be reported');
 
 Expectation to assert.
 
+### option: PageAgent.expect.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in the config, or by specifying the `expect` property of the [`option: Page.agent.expect`] option. Pass `0` to disable timeout.
+
 ### option: PageAgent.expect.-inline- = %%-page-agent-call-options-v1.58-%%
 * since: v1.58
 
@@ -68,6 +74,13 @@ Task to perform using agentic loop.
 * since: v1.58
 - `schema` <[z.ZodSchema]>
 
+### option: PageAgent.extract.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Extract timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or
+[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout.
+
 ### option: PageAgent.extract.-inline- = %%-page-agent-call-options-v1.58-%%
 * since: v1.58
 
@@ -94,6 +107,13 @@ await agent.perform('Click submit button');
 
 Task to perform using agentic loop.
 
+### option: PageAgent.perform.timeout
+* since: v1.58
+- `timeout` <[float]>
+
+Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in the config, or by using the [`method: BrowserContext.setDefaultTimeout`] or
+[`method: Page.setDefaultTimeout`] methods. Pass `0` to disable timeout.
+
 ### option: PageAgent.perform.-inline- = %%-page-agent-call-options-v1.58-%%
 * since: v1.58
 

docs/src/api/params.md

@@ -396,18 +396,11 @@ Maximum number of agentic actions to generate, defaults to context-wide value sp
 
 Maximum number of retries when generating each action, defaults to context-wide value specified in `agent` property.
 
-## page-agent-timeout
-* since: v1.58
-- `timeout` <[int]>
-
-Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
-
 ## page-agent-call-options-v1.58
 - %%-page-agent-cache-key-%%
 - %%-page-agent-max-tokens-%%
 - %%-page-agent-max-actions-%%
 - %%-page-agent-max-action-retries-%%
-- %%-page-agent-timeout-%%
 
 ## fetch-param-url
 - `url` <[string]>

packages/injected/src/injectedScript.ts

@@ -50,8 +50,7 @@ import type { Builtins } from './utilityScript';
 export type FrameExpectParams = Omit<channels.FrameExpectParams, 'expectedValue' | 'timeout'> & {
   expectedValue?: any;
   timeoutForLogs?: number;
-  explicitTimeout?: number;
-  noPreChecks?: boolean;
+  noAutoWaiting?: boolean;
 };
 
 export type ElementState = 'visible' | 'hidden' | 'enabled' | 'disabled' | 'editable' | 'checked' | 'unchecked' | 'indeterminate' | 'stable';

packages/playwright-client/types/types.d.ts

@@ -2112,6 +2112,13 @@ export interface Page {
       cacheOutFile?: string;
     };
 
+    expect?: {
+      /**
+       * Default timeout for expect calls in milliseconds, defaults to 5000ms.
+       */
+      timeout?: number;
+    };
+
     /**
      * Limits to use for the agentic loop.
      */
@@ -5440,7 +5447,10 @@ export interface PageAgent {
     maxTokens?: number;
 
     /**
-     * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
+     * Expect timeout in milliseconds. Defaults to `5000`. The default value can be changed via `expect.timeout` option in
+     * the config, or by specifying the `expect` property of the
+     * [`expect`](https://playwright.dev/docs/api/class-page#page-agent-option-expect) option. Pass `0` to disable
+     * timeout.
      */
     timeout?: number;
   }): Promise<void>;
@@ -5482,7 +5492,11 @@ export interface PageAgent {
     maxTokens?: number;
 
     /**
-     * Request timeout in milliseconds. Defaults to action timeout. Pass `0` to disable timeout.
+     * Perform timeout in milliseconds. Defaults to `5000`. The default value can be changed via `actionTimeout` option in
+     * the config, or by using the
+     * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+     * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+     * Pass `0` to disable timeout.
      */
     timeout?: number;
   }): Promise<{

packages/playwright-core/src/client/page.ts

@@ -865,7 +865,9 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
       systemPrompt: options.systemPrompt,
     };
     const { agent } = await this._channel.agent(params);
-    return PageAgent.from(agent);
+    const pageAgent = PageAgent.from(agent);
+    pageAgent._expectTimeout = options?.expect?.timeout;
+    return pageAgent;
   }
 
   async _snapshotForAI(options: TimeoutOptions & { track?: string } = {}): Promise<{ full: string, incremental?: string }> {

packages/playwright-core/src/client/pageAgent.ts

@@ -22,14 +22,9 @@ import { Page } from './page';
 import type * as api from '../../types/types';
 import type * as channels from '@protocol/channels';
 
-type PageAgentOptions = {
-  maxTokens?: number;
-  maxTurns?: number;
-  cacheKey?: string;
-};
-
 export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implements api.PageAgent {
   private _page: Page;
+  _expectTimeout?: number;
 
   static from(channel: channels.PageAgentChannel): PageAgent {
     return (channel as any)._object;
@@ -41,17 +36,20 @@ export class PageAgent extends ChannelOwner<channels.PageAgentChannel> implement
     this._channel.on('turn', params => this.emit(Events.Page.AgentTurn, params));
   }
 
-  async expect(expectation: string, options: PageAgentOptions = {}) {
-    await this._channel.expect({ expectation, ...options });
+  async expect(expectation: string, options: channels.PageAgentExpectOptions = {}) {
+    const timeout = options.timeout ?? this._expectTimeout ?? 5000;
+    await this._channel.expect({ expectation, ...options, timeout });
   }
 
-  async perform(task: string, options: PageAgentOptions = {}) {
-    const { usage } = await this._channel.perform({ task, ...options });
+  async perform(task: string, options: channels.PageAgentPerformOptions = {}) {
+    const timeout = this._page._timeoutSettings.timeout(options);
+    const { usage } = await this._channel.perform({ task, ...options, timeout });
     return { usage };
   }
 
-  async extract<Schema extends any>(query: string, schema: Schema, options: PageAgentOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> {
-    const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options });
+  async extract<Schema extends any>(query: string, schema: Schema, options: channels.PageAgentExtractOptions = {}): Promise<{ result: any, usage: channels.AgentUsage }> {
+    const timeout = this._page._timeoutSettings.timeout(options);
+    const { result, usage } = await this._channel.extract({ query, schema: this._page._platform.zodToJsonSchema(schema), ...options, timeout });
     return { result, usage };
   }
 

packages/playwright-core/src/server/agent/actionRunner.ts

@@ -143,14 +143,11 @@ async function innerRunAction(progress: Progress, mode: 'generate' | 'run', page
 }
 
 async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'run', selector: string | undefined, options: FrameExpectParams, expected: string | RegExp, matcherName: string, expectation: string) {
-  // Pass explicit timeout to limit the single expect action inside the overall "agentic expect" multi-step progress.
-  const timeout = expectTimeout(mode);
   const result = await frame.expect(progress, selector, {
     ...options,
-    timeoutForLogs: timeout,
-    explicitTimeout: timeout,
-    // Disable pre-checks to avoid them timing out, model has seen the snapshot anyway.
-    noPreChecks: mode === 'generate',
+    // When generating, we want the expect to pass or fail immediately and give feedback to the model.
+    noAutoWaiting: mode === 'generate',
+    timeoutForLogs: mode === 'generate' ? undefined : progress.timeout,
   });
   if (!result.matches === !options.isNot) {
     const received = matcherName === 'toMatchAriaSnapshot' ? '\n' + result.received.raw : result.received;
@@ -162,7 +159,7 @@ async function runExpect(frame: Frame, progress: Progress, mode: 'generate' | 'r
       expectation,
       locator: selector ? asLocatorDescription('javascript', selector) : undefined,
       timedOut: result.timedOut,
-      timeout,
+      timeout: mode === 'generate' ? undefined : progress.timeout,
       printedExpected: options.isNot ? `Expected${expectedSuffix}: not ${expectedDisplay}` : `Expected${expectedSuffix}: ${expectedDisplay}`,
       printedReceived: result.errorMessage ? '' : `Received: ${received}`,
       errorMessage: result.errorMessage,
@@ -266,7 +263,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
           expression: 'to.have.value',
           expectedText,
           isNot: !!action.isNot,
-          timeout: expectTimeout(mode),
+          timeout,
         };
         return { type: 'Frame', method: 'expect', title: 'Expect Value', params };
       } else if (action.type === 'checkbox' || action.type === 'radio') {
@@ -275,7 +272,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
           selector: action.selector,
           expression: 'to.be.checked',
           isNot: !!action.isNot,
-          timeout: expectTimeout(mode),
+          timeout,
         };
         return { type: 'Frame', method: 'expect', title: 'Expect Checked', params };
       } else {
@@ -287,7 +284,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
         selector: action.selector,
         expression: 'to.be.visible',
         isNot: !!action.isNot,
-        timeout: expectTimeout(mode),
+        timeout,
       };
       return { type: 'Frame', method: 'expect', title: 'Expect Visible', params };
     }
@@ -298,7 +295,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
         expression: 'to.match.snapshot',
         expectedText: [],
         isNot: !!action.isNot,
-        timeout: expectTimeout(mode),
+        timeout,
       };
       return { type: 'Frame', method: 'expect', title: 'Expect Aria Snapshot', params };
     }
@@ -310,7 +307,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
         expression: 'to.have.url',
         expectedText,
         isNot: !!action.isNot,
-        timeout: expectTimeout(mode),
+        timeout,
       };
       return { type: 'Frame', method: 'expect', title: 'Expect URL', params };
     }
@@ -321,7 +318,7 @@ export function traceParamsForAction(progress: Progress, action: actions.Action,
         expression: 'to.have.title',
         expectedText,
         isNot: !!action.isNot,
-        timeout: expectTimeout(mode),
+        timeout,
       };
       return { type: 'Frame', method: 'expect', title: 'Expect Title', params };
     }
@@ -341,7 +338,3 @@ function callMetadataForAction(progress: Progress, frame: Frame, action: actions
   };
   return callMetadata;
 }
-
-function expectTimeout(mode: 'generate' | 'run') {
-  return mode === 'generate' ? 0 : 5000;
-}

packages/playwright-core/src/server/agent/pageAgent.ts

@@ -50,7 +50,7 @@ export async function pageAgentPerform(progress: Progress, context: Context, use
 ### Task
 ${userTask}
 `;
-
+  progress.disableTimeout();
   await runLoop(progress, context, performTools, task, undefined, callParams);
   await updateCache(context, cacheKey);
 }
@@ -68,7 +68,7 @@ export async function pageAgentExpect(progress: Progress, context: Context, expe
 ### Expectation
 ${expectation}
 `;
-
+  progress.disableTimeout();
   await runLoop(progress, context, expectTools, task, undefined, callParams);
   await updateCache(context, cacheKey);
 }
@@ -101,7 +101,7 @@ async function runLoop(progress: Progress, context: Context, toolDefinitions: To
 
   const apiCacheTextBefore = context.agentParams.apiCacheFile ?
     await fs.promises.readFile(context.agentParams.apiCacheFile, 'utf-8').catch(() => '{}') : '{}';
-  const apiCacheBefore = JSON.parse(apiCacheTextBefore);
+  const apiCacheBefore = JSON.parse(apiCacheTextBefore || '{}');
 
   const loop = new Loop({
     api: context.agentParams.api as any,

packages/playwright-core/src/server/frames.ts

@@ -1389,34 +1389,30 @@ export class Frame extends SdkObject<FrameEventMap> {
         progress.metadata.error = { error: { name: 'Expect', message: 'Expect failed' } };
     };
     try {
-      // When explicit timeout is passed, constrain expect by it, in addition to regular progress abort.
-      const timeoutPromise = options.explicitTimeout !== undefined ? progress.wait(options.explicitTimeout).then(() => { throw new TimeoutError(`Timed out after ${options.explicitTimeout}ms`); }) : undefined;
-      timeoutPromise?.catch(() => { /* Prevent unhandled promise rejection */ });
-
       // Step 1: perform locator handlers checkpoint with a specified timeout.
       if (selector)
         progress.log(`waiting for ${this._asLocator(selector)}`);
-      if (!options.noPreChecks)
+      if (!options.noAutoWaiting)
         await this._page.performActionPreChecks(progress);
 
       // Step 2: perform one-shot expect check without a timeout.
       // Supports the case of `expect(locator).toBeVisible({ timeout: 1 })`
       // that should succeed when the locator is already visible.
       try {
         const resultOneShot = await this._expectInternal(progress, selector, options, lastIntermediateResult, true);
-        if (resultOneShot.matches !== options.isNot)
+        if (options.noAutoWaiting || resultOneShot.matches !== options.isNot)
           return resultOneShot;
       } catch (e) {
-        if (this.isNonRetriableError(e))
+        if (options.noAutoWaiting || this.isNonRetriableError(e))
           throw e;
         // Ignore any other errors from one-shot, we'll handle them during retries.
       }
 
       // Step 3: auto-retry expect with increasing timeouts. Bounded by the total remaining time.
       const result = await this.retryWithProgressAndTimeouts(progress, [100, 250, 500, 1000], async continuePolling => {
-        if (!options.noPreChecks)
+        if (!options.noAutoWaiting)
           await this._page.performActionPreChecks(progress);
-        const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false, timeoutPromise);
+        const { matches, received } = await this._expectInternal(progress, selector, options, lastIntermediateResult, false);
         if (matches === options.isNot) {
           // Keep waiting in these cases:
           // expect(locator).conditionThatDoesNotMatch
@@ -1446,9 +1442,9 @@ export class Frame extends SdkObject<FrameEventMap> {
     }
   }
 
-  private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean, timeoutPromise?: Promise<never>) {
+  private async _expectInternal(progress: Progress, selector: string | undefined, options: FrameExpectParams, lastIntermediateResult: { received?: any, isSet: boolean, errorMessage?: string }, noAbort: boolean) {
     // The first expect check, a.k.a. one-shot, always finishes - even when progress is aborted.
-    const race = <T>(p: Promise<T>) => noAbort ? p : (timeoutPromise ? progress.race([p, timeoutPromise]) : progress.race(p));
+    const race = <T>(p: Promise<T>) => noAbort ? p : progress.race(p);
     const selectorInFrame = selector ? await race(this.selectors.resolveFrameForSelector(selector, { strict: true })) : undefined;
 
     const { frame, info } = selectorInFrame || { frame: this, info: undefined };