Add functionality to remember information artificially, and add additional test coverage across supported platforms

Author: schnecle

.eslintignore

@@ -5,3 +5,7 @@ node_modules
 standalone
 templates
 .firebase
+scripts/agent-evals/output
+scripts/agent-evals/node_modules
+scripts/agent-evals/lib
+scripts/agent-evals/templates

.prettierignore

@@ -5,6 +5,7 @@
 /scripts/frameworks-tests/vite-project/**
 /scripts/webframeworks-deploy-tests/angular/**
 /scripts/webframeworks-deploy-tests/nextjs/**
+/scripts/agent-evals/output/**
 /src/frameworks/docs/**
 /prompts
 

npm-shrinkwrap.json

@@ -241,7 +241,6 @@
     "eslint-plugin-brikke": "^2.2.2",
     "eslint-plugin-jsdoc": "^48.0.1",
     "eslint-plugin-prettier": "^5.1.3",
-    "firebase": "^9.16.0",
     "firebase-admin": "^11.5.0",
     "firebase-functions": "^4.3.1",
     "google-discovery-to-swagger": "^2.1.0",

package.json

@@ -12,7 +12,6 @@ import { getFirebaseCliRoot } from "../runner/paths.js";
 // Path to the built MCP Tools implementation in the Firebase CLI, relative to
 // the repo's root
 const MCP_TOOLS_INDEX_PATH = "lib/mcp/tools/index.js";
-const CONFIGSTORE_INDEX_PATH = "lib/mcp/configstore.js";
 const LOG_FILE_PATH = path.join(os.homedir(), "Desktop", "agent_evals_mock_logs.txt");
 // Enable this to turn on file logging. This can be helpful for debugging
 // because console logs get swallowed
@@ -23,19 +22,6 @@ const originalRequire = Module.prototype.require;
   const requiredModule = originalRequire.apply(this, [id]);
   const absolutePath = Module.createRequire(this.filename).resolve(id);
   const pathRelativeToCliRoot = path.relative(getFirebaseCliRoot(), absolutePath);
-  console.log(`[DEBUG] Requiring: ${pathRelativeToCliRoot} (Absolute: ${absolutePath})`);
-
-  // Mock configstore to avoid "Cannot find module ../package.json" error and side effects
-  if (pathRelativeToCliRoot.endsWith(CONFIGSTORE_INDEX_PATH)) {
-    logToFile(`Mocking configstore for: ${pathRelativeToCliRoot}`);
-    return {
-      configstore: {
-        get: () => undefined,
-        set: () => { },
-        delete: () => { },
-      },
-    };
-  }
 
   if (!pathRelativeToCliRoot.endsWith(MCP_TOOLS_INDEX_PATH)) {
     return requiredModule;

scripts/agent-evals/src/mock/mock-tools-main.ts

@@ -3,10 +3,10 @@ import { renderTemplate } from "../../../../../src/mcp/tools/core/get_environmen
 import { toMockContent } from "../tool-mock-utils.js";
 
 const PROJECT_DIR = "/Users/fakeuser/develop/fake-project";
-const IOS_APP_ID = `1:${DEFAULT_FIREBASE_PROJECT}:ios:abc123efj456`;
-const IOS_BUNDLE_ID = "com.firebase.fake.ios";
-const ANDROID_APP_ID = `1:${DEFAULT_FIREBASE_PROJECT}:android:abc123efj456`;
-const ANDROID_PACKAGE_NAME = "com.firebase.fake.android";
+export const IOS_APP_ID = `1:${DEFAULT_FIREBASE_PROJECT}:ios:abc123efj456`;
+export const IOS_BUNDLE_ID = "com.firebase.fake.ios";
+export const ANDROID_APP_ID = `1:${DEFAULT_FIREBASE_PROJECT}:android:abc123efj456`;
+export const ANDROID_PACKAGE_NAME = "com.firebase.fake.android";
 
 const BASE_ENVIRONMENT_CONFIG = {
   projectId: DEFAULT_FIREBASE_PROJECT,
@@ -40,7 +40,7 @@ export const getEnvironmentWithFlutterApp = {
       ...BASE_ENVIRONMENT_CONFIG,
       detectedAppIds: {
         [ANDROID_APP_ID]: ANDROID_PACKAGE_NAME,
-        [IOS_APP_ID]: IOS_BUNDLE_ID
+        [IOS_APP_ID]: IOS_BUNDLE_ID,
       },
     }),
   ),

scripts/agent-evals/src/mock/mocks/get-environment-mock.ts

@@ -12,7 +12,7 @@ const allToolMocks = {
   nextJsWithProjectMock,
   getEnvironmentWithIosApp,
   getEnvironmentWithAndroidApp,
-  getEnvironmentWithFlutterApp
+  getEnvironmentWithFlutterApp,
 } as const;
 
 export type ToolMockName = keyof typeof allToolMocks;

scripts/agent-evals/src/mock/tool-mocks.ts

@@ -22,12 +22,23 @@ export interface AgentTestMatchers {
 }
 
 export interface AgentTestRunner extends AgentTestMatchers {
+  /**
+   * The directory where the test runner is operating
+   */
+  readonly runDir: string;
+
   /**
    * Simulates typing a string and waits for the turn to complete. It types one
    * character at a time to avoid paste detection that the Gemini CLI has
    */
   type(text: string): Promise<void>;
 
+  /**
+   * Simulates a previously remembered value. For Gemini CLI, this results in
+   * saved values in the user's GEMINI.md file.
+   */
+  remember(text: string): Promise<void>;
+
   /**
    * Negated assertions
    */

scripts/agent-evals/src/runner/agent-test-runner.ts

@@ -1,5 +1,6 @@
 import { mkdirSync, writeFileSync, readFileSync, existsSync } from "fs";
 import * as path from "path";
+import * as os from "os";
 import { InteractiveCLI, poll } from "./interactive-cli";
 import { AgentTestRunner, AgentTestMatchers } from "./agent-test-runner";
 import {
@@ -13,6 +14,7 @@ import { throwFailure } from "./logging";
 import { getAgentEvalsRoot, RunDirectories } from "./paths";
 import { execSync } from "node:child_process";
 import { ToolMockName } from "../mock/tool-mocks";
+import { appendFileSync } from "node:fs";
 
 const READY_PROMPT = "Type your message";
 const INSTALL_ID = "238efa5b-efb2-44bd-9dce-9b081532681c";
@@ -43,7 +45,8 @@ export class GeminiCliRunner implements AgentTestRunner {
   private readonly cli: InteractiveCLI;
   private readonly telemetryPath: string;
   private readonly telemetryTimeout = 15000;
-  private readonly runDir: string;
+  readonly runDir: string;
+  private readonly userDir: string;
 
   // Determines which tools to start from for this turn so we don't detect tool
   // calls from previous turns
@@ -99,6 +102,7 @@ export class GeminiCliRunner implements AgentTestRunner {
     this.writeGeminiInstallId(dirs.userDir);
 
     this.runDir = dirs.runDir;
+    this.userDir = dirs.userDir;
     this.cli = new InteractiveCLI("gemini", ["--yolo"], {
       cwd: dirs.runDir,
       readyPrompt: READY_PROMPT,
@@ -122,7 +126,22 @@ export class GeminiCliRunner implements AgentTestRunner {
     return this.cli.type(text);
   }
 
+  async remember(text: string): Promise<void> {
+    const geminiDir = path.join(this.userDir, ".gemini");
+    const geminiMdFile = path.join(geminiDir, "GEMINI.md");
+    if (!existsSync(geminiDir)) {
+      mkdirSync(geminiDir, { recursive: true });
+    }
+
+    if (!existsSync(geminiMdFile)) {
+      writeFileSync(geminiMdFile, "## Gemini Added Memories" + os.EOL);
+    }
 
+    appendFileSync(geminiMdFile, text + os.EOL);
+    await this.type("/memory refresh");
+    // Due to https://github.com/google-gemini/gemini-cli/issues/10702, we need to start a new chat
+    await this.type("/clear");
+  }
 
   async exit(): Promise<void> {
     await this.cli.kill();
@@ -155,7 +174,7 @@ export class GeminiCliRunner implements AgentTestRunner {
     await this.waitForTelemetryReady();
     let logs: string[] = [];
     const toolsCallsMade = await poll(() => {
-      logs = []
+      logs = [];
       const { success, messages } = this.checkToolCalls(tools);
       logs = [...messages];
       return success;
@@ -169,11 +188,10 @@ export class GeminiCliRunner implements AgentTestRunner {
   public async expectMemory(text: string | RegExp): Promise<void> {
     let logs: string[] = [];
     const memoryFound = await poll(() => {
-      logs = []
+      logs = [];
       const { success, messages } = this.checkMemory(text);
       logs = [...messages];
-      return success
-
+      return success;
     }, this.telemetryTimeout);
 
     if (!memoryFound) {
@@ -207,8 +225,8 @@ export class GeminiCliRunner implements AgentTestRunner {
       expectMemory: async (text: string | RegExp) => {
         const timeout = 1000;
         const found = await poll(() => {
-          const { success } = this.checkMemory(text)
-          return success
+          const { success } = this.checkMemory(text);
+          return success;
         }, timeout);
 
         if (found) {
@@ -322,7 +340,7 @@ export class GeminiCliRunner implements AgentTestRunner {
   }
 
   private checkMemory(text: string | RegExp): CheckResult {
-    const geminiMdPath = path.join(this.runDir, ".gemini", "GEMINI.md");
+    const geminiMdPath = path.join(this.userDir, ".gemini", "GEMINI.md");
     const messages: string[] = [];
     if (!existsSync(geminiMdPath)) {
       messages.push(`GEMINI.md file not found at ${geminiMdPath}`);

scripts/agent-evals/src/runner/gemini-cli-runner.ts

@@ -1,6 +1,5 @@
 import * as path from "path";
 
-
 export type RunDirectories = { testDir: string; runDir: string; userDir: string };
 
 export function getAgentEvalsRoot(): string {