Mock MCP tools during Agent Evals (#9380)

Author: samedson

scripts/agent-evals/README.md

@@ -78,3 +78,40 @@ describe("<prompt-or-tool-name>", function (this: Mocha.Suite) {
   });
 });
 ```
+
+## Adding Mocks for MCP Tools
+
+Mocks applied to MCP tools will completely replace their impelementation with a static output string.
+
+1. Add your mocked tools to the `scripts/agent-evals/src/mock/mocks` folder, eg. `scripts/agent-evals/src/mock/mocks/next-js-with-project-mock.ts`
+
+```
+import { toMockContent } from "../tool-mock-utils.js";
+
+export const environment_nice_day_mock = {
+  firebase_get_environment: toMockContent("Tell the user to have a nice day"),
+} as const;
+```
+
+2. Add the new set of mocked tools to the map:
+
+```
+import { environment_nice_day_mock } from "./mocks/next-js-with-project-mock.js";
+
+const allToolMocks = {
+  // New tool mock
+  environment_nice_day_mock,
+} as const;
+```
+
+3. Start using the mock in your test:
+
+Note: If you apply multiple mocks to the same tool, later values in the list will take precedence.
+
+```
+const run: AgentTestRunner = await startAgentTest(this, {
+  templateName: "next-app-hello-world",
+  // Add the name of your mock here
+  toolMocks: ["environment_nice_day_mock"],
+});
+```

scripts/agent-evals/src/data/index.ts

@@ -0,0 +1,7 @@
+export const DEFAULT_FIREBASE_USER = "[email protected]";
+export const DEFAULT_FIREBASE_PROJECT_NAME = "My Project";
+export const DEFAULT_FIREBASE_PROJECT = "my-project";
+export const DEFAULT_FIREBASE_PROJECT_NUMBER = "1234321098765";
+export const DEFAULT_FIREBASE_WEB_APP_NAME = "My Web App";
+export const DEFAULT_FIREBASE_WEB_APP_ID = `1:${DEFAULT_FIREBASE_PROJECT_NUMBER}:web:84d1de6d7ees1e0be7949c`;
+export const DEFAULT_FIREBASE_WEB_APP_API_KEY = "aaaaaaaa-ffff-4444-bbbb-ffffffffffff";

scripts/agent-evals/src/mock/mock-tools-main.ts

@@ -0,0 +1,82 @@
+import { Module } from "module";
+import path from "path";
+import fs from "fs";
+import os from "os";
+import { getFirebaseCliRoot } from "../runner/paths.js";
+import { getToolMocks } from "./tool-mocks.js";
+
+//
+// This file is run as a node --import parameter before the Firebase CLI to
+// patch the implementation for tools with the mocked implementation
+//
+
+// Path to the built MCP Tools implementation in the Firebase CLI, relative to
+// the repo's root
+const MCP_TOOLS_INDEX_PATH = "lib/mcp/tools/index.js";
+const LOG_FILE_PATH = path.join(os.homedir(), "Desktop", "agent_evals_mock_logs.txt");
+// Enable this to turn on file logging. This can be helpful for debugging
+// because console logs get swallowed
+const ENABLE_FILE_LOGGING = false;
+
+const mocks = getToolMocks();
+
+const originalRequire = Module.prototype.require;
+Module.prototype.require = function (id: string) {
+  const requiredModule = originalRequire.apply(this, [id]);
+  const absolutePath = Module.createRequire(this.filename).resolve(id);
+  const pathRelativeToCliRoot = path.relative(getFirebaseCliRoot(), absolutePath);
+  if (!pathRelativeToCliRoot.endsWith(MCP_TOOLS_INDEX_PATH)) {
+    return requiredModule;
+  }
+
+  logToFile(
+    `Creating proxy implementation for file: ${pathRelativeToCliRoot} with tool mocks: ${JSON.stringify(Object.keys(mocks))}`,
+  );
+
+  return new Proxy(requiredModule, {
+    get(target, prop, receiver) {
+      if (prop !== "availableTools") {
+        return Reflect.get(target, prop, receiver);
+      }
+
+      logToFile(`Intercepting access to 'availableTools'`);
+
+      const originalAvailableTools = Reflect.get(target, prop, receiver);
+      return async (ctx: any, features?: string[]): Promise<any[]> => {
+        const realTools = await originalAvailableTools(ctx, features);
+        if (!Array.isArray(realTools)) {
+          logToFile(`Error: Real tools is not an array: ${JSON.stringify(realTools)}`);
+          return realTools;
+        }
+
+        const finalTools = realTools.map((tool) => {
+          const toolName = tool.mcp.name;
+          if (!mocks[toolName]) {
+            return tool;
+          }
+          logToFile(`Applying mock for tool: ${toolName}`);
+          return {
+            ...tool,
+            fn: async () => mocks[toolName],
+          };
+        });
+
+        return finalTools;
+      };
+    },
+  });
+};
+
+function logToFile(message: string) {
+  if (!ENABLE_FILE_LOGGING) {
+    return;
+  }
+  const timestamp = new Date().toISOString();
+  const logMessage = `[${timestamp}] ${message}\n`;
+  try {
+    fs.appendFileSync(LOG_FILE_PATH, logMessage, "utf8");
+  } catch (err) {
+    console.error(`[AGENT-EVALS-MOCK-ERROR] Failed to write log to ${LOG_FILE_PATH}:`, err);
+    console.error(`[AGENT-EVALS-MOCK-ERROR] Original message: ${message}`);
+  }
+}

scripts/agent-evals/src/mock/mocks/next-js-with-project-mock.ts

@@ -0,0 +1,66 @@
+import {
+  DEFAULT_FIREBASE_PROJECT,
+  DEFAULT_FIREBASE_PROJECT_NAME,
+  DEFAULT_FIREBASE_PROJECT_NUMBER,
+  DEFAULT_FIREBASE_USER,
+  DEFAULT_FIREBASE_WEB_APP_ID,
+  DEFAULT_FIREBASE_WEB_APP_NAME,
+  DEFAULT_FIREBASE_WEB_APP_API_KEY,
+} from "../../data/index.js";
+import { toMockContent } from "../tool-mock-utils.js";
+
+export const nextJsWithProjectMock = {
+  firebase_login: toMockContent(`Successfully logged in as ${DEFAULT_FIREBASE_USER}`),
+
+  firebase_get_environment: toMockContent(`# Environment Information
+
+Project Directory:
+/Users/samedson/Firebase/firebase-tools/scripts/agent-evals/output/2025-10-24_15-36-06-588Z/-firebase-init-backend-app-2c27e75e3e5d809c/repo
+Project Config Path: <NO CONFIG PRESENT>
+Active Project ID: ${DEFAULT_FIREBASE_PROJECT}
+Gemini in Firebase Terms of Service: Accepted
+Authenticated User: ${DEFAULT_FIREBASE_USER}
+Detected App IDs: <NONE>
+Available Project Aliases (format: '[alias]: [projectId]'): <NONE>
+
+No firebase.json file was found.
+
+If this project does not use Firebase services that require a firebase.json file, no action is necessary.
+
+If this project uses Firebase services that require a firebase.json file, the user will most likely want to:
+
+a) Change the project directory using the 'firebase_update_environment' tool to select a directory with a 'firebase.json' file in it, or
+b) Initialize a new Firebase project directory using the 'firebase_init' tool.
+
+Confirm with the user before taking action.`),
+
+  firebase_update_environment: toMockContent(
+    `- Updated active project to '${DEFAULT_FIREBASE_PROJECT}'\n`,
+  ),
+
+  firebase_list_projects: toMockContent(`
+- projectId: ${DEFAULT_FIREBASE_PROJECT}
+  projectNumber: '${DEFAULT_FIREBASE_PROJECT_NUMBER}'
+  displayName: ${DEFAULT_FIREBASE_PROJECT_NAME}
+  name: projects/${DEFAULT_FIREBASE_PROJECT}
+  resources:
+    hostingSite: ${DEFAULT_FIREBASE_PROJECT}
+  state: ACTIVE
+  etag: 1_99999999-7777-4444-8888-dddddddddddd
+`),
+
+  firebase_list_apps: toMockContent(`
+- name: 'projects/${DEFAULT_FIREBASE_PROJECT}/webApps/${DEFAULT_FIREBASE_WEB_APP_ID}'
+  displayName: ${DEFAULT_FIREBASE_WEB_APP_NAME}
+  platform: WEB
+  appId: '${DEFAULT_FIREBASE_WEB_APP_ID}'
+  namespace: 000000000000000000000000000000000000000000000000
+  apiKeyId: ${DEFAULT_FIREBASE_WEB_APP_API_KEY}
+  state: ACTIVE
+  expireTime: '1970-01-01T00:00:00Z'
+`),
+
+  firebase_get_sdk_config: toMockContent(
+    `{"projectId":"${DEFAULT_FIREBASE_PROJECT}","appId":"${DEFAULT_FIREBASE_WEB_APP_ID}","storageBucket":"${DEFAULT_FIREBASE_PROJECT}.firebasestorage.app","apiKey":"${DEFAULT_FIREBASE_WEB_APP_API_KEY}","authDomain":"${DEFAULT_FIREBASE_PROJECT}.firebaseapp.com","messagingSenderId":"${DEFAULT_FIREBASE_PROJECT_NUMBER}","projectNumber":"${DEFAULT_FIREBASE_PROJECT_NUMBER}","version":"2"}`,
+  ),
+} as const;

scripts/agent-evals/src/mock/tool-mock-utils.ts

@@ -0,0 +1,5 @@
+import { ToolMock } from "./tool-mocks.js";
+
+export function toMockContent(data: any): ToolMock {
+  return { content: [{ type: "text", text: data }] };
+}

scripts/agent-evals/src/mock/tool-mocks.ts

@@ -0,0 +1,30 @@
+import { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { nextJsWithProjectMock } from "./mocks/next-js-with-project-mock.js";
+
+export type ToolMock = CallToolResult;
+
+const allToolMocks = {
+  nextJsWithProjectMock,
+} as const;
+
+export type ToolMockName = keyof typeof allToolMocks;
+
+function isToolMockName(name: string): name is ToolMockName {
+  return name in allToolMocks;
+}
+
+export function getToolMocks(): Record<string, ToolMock> {
+  const mockNames = process.env.TOOL_MOCKS;
+  let mocks = {};
+  for (const mockName of mockNames?.split(",") || []) {
+    if (isToolMockName(mockName)) {
+      mocks = {
+        ...mocks,
+        ...allToolMocks[mockName], // No more error!
+      };
+    } else {
+      console.error(`Invalid mock name provided: "${mockName}"`);
+    }
+  }
+  return mocks;
+}

scripts/agent-evals/src/runner/gemini-cli-runner.ts

@@ -10,6 +10,9 @@ import {
 } from "./tool-matcher.js";
 import fs from "fs";
 import { throwFailure } from "./logging.js";
+import { getAgentEvalsRoot } from "./paths.js";
+import { execSync } from "node:child_process";
+import { ToolMockName } from "../mock/tool-mocks.js";
 
 const READY_PROMPT = "Type your message";
 
@@ -43,9 +46,12 @@ export class GeminiCliRunner implements AgentTestRunner {
     private readonly testName: string,
     testDir: string,
     runDir: string,
+    toolMocks: ToolMockName[],
   ) {
     // Create a settings file to point the CLI to a local telemetry log
     this.telemetryPath = path.join(testDir, "telemetry.log");
+    const mockPath = path.resolve(path.join(getAgentEvalsRoot(), "lib/mock/mock-tools-main.js"));
+    const firebasePath = execSync("which firebase").toString().trim();
     const settings = {
       general: {
         disableAutoUpdate: true,
@@ -58,8 +64,11 @@ export class GeminiCliRunner implements AgentTestRunner {
       },
       mcpServers: {
         firebase: {
-          command: "firebase",
-          args: ["experimental:mcp"],
+          command: "node",
+          args: ["--import", mockPath, firebasePath, "experimental:mcp"],
+          env: {
+            TOOL_MOCKS: `${toolMocks?.join(",") || ""}`,
+          },
         },
       },
     };

scripts/agent-evals/src/runner/index.ts

@@ -6,6 +6,7 @@ import { GeminiCliRunner } from "./gemini-cli-runner.js";
 import { buildFirebaseCli, clearUserMcpServers } from "./setup.js";
 import { addCleanup } from "../helpers/cleanup.js";
 import { TemplateName, copyTemplate, buildTemplates } from "../template/index.js";
+import { ToolMockName } from "../mock/tool-mocks.js";
 
 export * from "./agent-test-runner.js";
 
@@ -21,6 +22,9 @@ export interface AgentTestOptions {
   // Name of the template directory to copy into this test run. Leave this empty
   // to run the test in an empty directory
   templateName?: TemplateName;
+  // List of MCP Tool mocks to apply, in order. Later mocks overwrite earlier
+  // mocks.
+  toolMocks?: ToolMockName[];
 }
 
 export async function startAgentTest(
@@ -37,7 +41,7 @@ export async function startAgentTest(
     copyTemplate(options.templateName, runDir);
   }
 
-  const run = new GeminiCliRunner(testName, testDir, runDir);
+  const run = new GeminiCliRunner(testName, testDir, runDir, options?.toolMocks || []);
   await run.waitForReadyPrompt();
 
   addCleanup(async () => {

scripts/agent-evals/src/runner/paths.ts

@@ -0,0 +1,11 @@
+import path from "path";
+import { fileURLToPath } from "url";
+
+export function getAgentEvalsRoot(): string {
+  const thisFilePath = path.dirname(fileURLToPath(import.meta.url));
+  return path.resolve(path.join(thisFilePath, "..", ".."));
+}
+
+export function getFirebaseCliRoot(): string {
+  return path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "..");
+}

scripts/agent-evals/src/runner/setup.ts

@@ -1,7 +1,6 @@
 import { exec } from "child_process";
-import path from "path";
 import { promisify } from "util";
-import { fileURLToPath } from "url";
+import { getFirebaseCliRoot } from "./paths.js";
 
 const execPromise = promisify(exec);
 
@@ -10,13 +9,7 @@ export async function buildFirebaseCli() {
     console.log("Skipping Firebase CLI build because process.env.SKIP_REBUILD");
     return;
   }
-  const firebaseCliRoot = path.resolve(
-    path.dirname(fileURLToPath(import.meta.url)),
-    "..",
-    "..",
-    "..",
-    "..",
-  );
+  const firebaseCliRoot = getFirebaseCliRoot();
   console.log(`Building Firebase CLI at ${firebaseCliRoot}`);
   await execPromise("./scripts/clean-install.sh", { cwd: firebaseCliRoot });
 }