Merge pull request #213 from PredicateSystems/multistep_form

fix: prioritize form-fill over extraction category to prevent false p…

Author: rcholic

src/agents/planner-executor/category-pruner.ts

@@ -284,6 +284,21 @@ export function detectPruningCategory(
     return PruningTaskCategory.FORM_FILLING;
   }
 
+  // Form-fill keyword detection takes priority over extraction
+  // because "Display name" or "email" in a form task are field labels, not extraction.
+  if (
+    normalizedGoal.includes('form') ||
+    normalizedGoal.includes('fill') ||
+    normalizedGoal.includes('fill out') ||
+    normalizedGoal.includes('submit') ||
+    normalizedGoal.includes('onboarding') ||
+    normalizedGoal.includes('sign up') ||
+    normalizedGoal.includes('signup') ||
+    normalizedGoal.includes('register')
+  ) {
+    return PruningTaskCategory.FORM_FILLING;
+  }
+
   // Extraction keyword detection takes priority over TRANSACTION/SHOPPING
   // because "find the title of X" or "extract Y" on an e-commerce site is
   // still an extraction task, not a shopping task.

src/agents/planner-executor/extraction-keywords.ts

@@ -177,13 +177,37 @@ export const TEXT_EXTRACTION_KEYWORDS: readonly string[] = [
  * @param task - The task or step description to analyse
  * @returns true if this looks like a text extraction task
  */
+const FORM_FILL_SIGNALS: readonly string[] = [
+  'form',
+  'fill',
+  'submit',
+  'onboarding',
+  'sign up',
+  'signup',
+  'register',
+  'checkbox',
+  'dropdown',
+  'radio button',
+  'next button',
+  'click the',
+  'type ',
+  'enter ',
+];
+
 export function isTextExtractionTask(task: string): boolean {
   if (!task) {
     return false;
   }
 
   const taskLower = task.toLowerCase();
 
+  // Form-fill negative signal: if the task clearly involves filling a form,
+  // it's not extraction even if it contains extraction-like keywords
+  // (e.g., "Display name", "email" are field labels, not extraction targets)
+  if (FORM_FILL_SIGNALS.some(signal => taskLower.includes(signal))) {
+    return false;
+  }
+
   // Tier 1: Strong extraction phrases (multi-word substring match)
   for (const phrase of EXTRACTION_PHRASES) {
     if (taskLower.includes(phrase)) {

src/agents/planner-executor/plan-models.ts

@@ -175,6 +175,8 @@ export interface ActionRecord {
   action: string;
   /** Element description or URL */
   target: string | null;
+  /** Planner intent for this action, if provided */
+  intent?: string | null;
   /** Outcome (success, failed) */
   result: string;
   /** URL after action completed */
@@ -213,6 +215,7 @@ export interface StepOutcome {
   urlBefore?: string;
   urlAfter?: string;
   extractedData?: unknown;
+  pageContentPreview?: string;
 }
 
 // ---------------------------------------------------------------------------

src/agents/planner-executor/plan-utils.ts

@@ -182,14 +182,25 @@ function stripThinkingTags(content: string): string {
 function repairJson(text: string): string {
   let repaired = text;
 
+  // Fix backslash-escaped quotes that make JSON unparseable.
+  // Some LLMs output: {"action":"CLICK",...,\"reasoning\":\"text\"}
+  // where the outer quotes are real but inner key/value quotes are escaped.
+  // Heuristic: if the text has unescaped quotes AND backslash-escaped quotes
+  // mixed together, unescape the backslash-escaped ones.
+  const hasUnescapedQuotes = /[^\\]"/.test(repaired);
+  const hasEscapedQuotes = /\\"/.test(repaired);
+  if (hasUnescapedQuotes && hasEscapedQuotes) {
+    repaired = repaired.replace(/\\"/g, '"');
+  }
+
   // Add double quotes around unquoted object keys
   // Matches: word-characters followed by colon (not already inside a string)
   // Pattern: start of object `{` or comma `,`, optional whitespace, then unquoted key, then `:`
   repaired = repaired.replace(/([{,]\s*)([\w$]+)\s*:/g, '$1"$2":');
 
   // Replace single-quoted strings with double-quoted strings
   // This is a simple heuristic — it won't handle escaped single quotes inside strings,
-  // but it handles the common case of LLMs outputting `'text'` instead of `"text"`
+  // but it handles the common case of LLMs outputting 'text' instead of "text"
   repaired = repaired.replace(/'([^']*)'/g, '"$1"');
 
   // Remove trailing commas before } or ]
@@ -338,6 +349,8 @@ const ACTION_ALIASES: Record<string, string> = {
   CLICK_ELEMENT: 'CLICK',
   CLICK_BUTTON: 'CLICK',
   CLICK_LINK: 'CLICK',
+  CLICK_XY: 'CLICK',
+  TYPE_AT: 'TYPE',
   INPUT: 'TYPE_AND_SUBMIT',
   TYPE_TEXT: 'TYPE_AND_SUBMIT',
   ENTER_TEXT: 'TYPE_AND_SUBMIT',
@@ -347,6 +360,8 @@ const ACTION_ALIASES: Record<string, string> = {
   OPEN: 'NAVIGATE',
   SCROLL_DOWN: 'SCROLL',
   SCROLL_UP: 'SCROLL',
+  SCROLL_TO: 'SCROLL',
+  SCROLL_INTO_VIEW: 'SCROLL',
 };
 
 /**
@@ -517,6 +532,20 @@ function normalizeStep(step: Record<string, unknown>): Record<string, unknown> {
   if ('target' in normalizedStep && normalizedStep.target === null) {
     delete normalizedStep.target;
   }
+  if ('input' in normalizedStep && typeof normalizedStep.input !== 'string') {
+    const inputValue = normalizedStep.input;
+    if (inputValue === null || inputValue === undefined) {
+      delete normalizedStep.input;
+    } else if (
+      typeof inputValue === 'number' ||
+      typeof inputValue === 'boolean' ||
+      typeof inputValue === 'bigint'
+    ) {
+      normalizedStep.input = String(inputValue);
+    } else {
+      normalizedStep.input = JSON.stringify(inputValue);
+    }
+  }
 
   if ('id' in normalizedStep && typeof normalizedStep.id === 'string') {
     const parsed = parseInt(normalizedStep.id, 10);