fix: planner intent specificity, vision auto-type on CLICK_XY, exempt CLICK from pre-verification

- Planner prompt: instruct specific intent with label/placeholder context
- Vision TYPE: CLICK_XY returns field coords, executor auto-types input after click
- Pre-verification: exempt CLICK actions from skip (Next/Submit buttons)
- Vision prompt: ask for CLICK_XY on correct field by label, not two-step action

Author: SentienceDEV

src/agents/planner-executor/planner-executor-agent.ts

@@ -1459,7 +1459,8 @@ export class PlannerExecutorAgent {
       this.config.preStepVerification &&
       (plannerAction.verify?.length || 0) > 0 &&
       plannerAction.action !== 'TYPE_AND_SUBMIT' &&
-      plannerAction.action !== 'TYPE'
+      plannerAction.action !== 'TYPE' &&
+      plannerAction.action !== 'CLICK'
     ) {
       const alreadySatisfied = await this.checkPreStepVerification(runtime, plannerAction);
       if (alreadySatisfied) {
@@ -1905,14 +1906,26 @@ export class PlannerExecutorAgent {
             error: 'CLICK_XY not supported by runtime (no clickCoordinate method)',
           };
         }
+        const isVisionTypeAction =
+          plannerAction.action === 'TYPE_AND_SUBMIT' || plannerAction.action === 'TYPE';
+        if (isVisionTypeAction && plannerAction.input && runtime.typeCoordinate) {
+          await new Promise(r => setTimeout(r, 300));
+          await runtime.typeCoordinate(plannerAction.input);
+          if (this.config.verbose) {
+            console.log(`[VISION] AUTO-TYPE after CLICK_XY: "${plannerAction.input}"`);
+          }
+        }
         await new Promise(r => setTimeout(r, 500));
         const urlAfter = await runtime.getCurrentUrl();
         const verificationPassed = await this.verifyStepOutcome(runtime, plannerAction);
         return {
           stepId: stepNum,
           goal: stepGoal,
           status: verificationPassed ? StepStatus.SUCCESS : StepStatus.FAILED,
-          actionTaken: `CLICK_XY(${vx}, ${vy})`,
+          actionTaken:
+            isVisionTypeAction && plannerAction.input
+              ? `CLICK_XY(${vx}, ${vy}) + TYPE_AT("${plannerAction.input}")`
+              : `CLICK_XY(${vx}, ${vy})`,
           llmResponseText: executorResp?.content,
           verificationPassed,
           usedVision: true,

src/agents/planner-executor/prompts.ts

@@ -58,7 +58,7 @@ export function buildStepwisePlannerPrompt(
 
 Actions:
 - NAVIGATE: Go directly to a URL when the next destination is known. Set "target" to the URL.
-- CLICK: Click an element. Set "intent" to element type/role. Set "input" to EXACT text from elements list.
+- CLICK: Click an element. Set "intent" to describe the SPECIFIC element (include label, placeholder, or nearby text, e.g. "email textbox", "display name field", "Next button", NOT just "textbox" or "button"). Set "input" to EXACT text from elements list.
 - TYPE_AND_SUBMIT: Type text into a search box and submit. Set "input" to the SEARCH QUERY from the goal (NOT the element label).
 - SCROLL: Scroll page. Set "direction" to "up" or "down".
 - WAIT: Wait for content to appear when a follow-up verification is needed.
@@ -104,7 +104,9 @@ RULES:
 8. Output ONLY JSON - no <think> tags, no markdown, no prose
 9. Do NOT output <think> or any reasoning
 10. Do NOT return DONE until ALL parts of the goal are complete
-11. Never copy example URLs from these instructions. Only NAVIGATE to a URL from the user's task, the current page, or a visible element.`;
+11. Never copy example URLs from these instructions. Only NAVIGATE to a URL from the user's task, the current page, or a visible element.
+12. For multi-step forms: process each field as a separate step, then CLICK the Next/Submit button as a separate step.
+13. "intent" must be SPECIFIC: describe the element with its label or context (e.g., "email field", "plan dropdown", "Next button on step 2")`;
 
   // Inject extraction-specific guidance when the goal is an extraction task
   const extractionGuidance = isExtractionTask(goal) ? getExtractionDomainGuidance() : '';
@@ -368,11 +370,12 @@ RULES:
 - Be precise with coordinates — aim for the center of the target element
 - Do NOT output any reasoning or prose
 - Do NOT use CLICK(id) — only coordinate-based actions
+- For form fields, match the field by its LABEL or PLACEHOLDER text, not just any input
 - Example: CLICK_XY(450, 320)`;
 
   let actionHint: string;
   if (isTypeAction && inputText) {
-    actionHint = `First CLICK_XY on the input field, then TYPE_AT("${inputText}")`;
+    actionHint = `Return CLICK_XY(x, y) for the INPUT FIELD matching: "${intent || goal}". Look at field labels/placeholders to find the correct field.`;
   } else {
     actionHint = `Return CLICK_XY(x, y) for the element matching: "${intent || goal}"`;
   }