Added basic evals

Author: matt-aitken

apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.ai-filter.tsx

@@ -19,6 +19,7 @@ import { VersionListPresenter } from "~/presenters/v3/VersionListPresenter.serve
 import { TaskListPresenter } from "~/presenters/v3/TaskListPresenter.server";
 import { getAllTaskIdentifiers } from "~/models/task.server";
 import { $replica } from "~/db.server";
+import { env } from "~/env.server";
 
 const RequestSchema = z.object({
   text: z.string().min(1),
@@ -130,14 +131,21 @@ export async function action({ request, params }: ActionFunctionArgs) {
     },
   };
 
+  if (!env.OPENAI_API_KEY) {
+    return {
+      success: false,
+      error: "OpenAI API key is not configured",
+    };
+  }
+
   const service = new AIRunFilterService({
     queryTags,
     queryVersions,
     queryQueues,
     queryTasks,
   });
 
-  const [error, result] = await tryCatch(service.call(text, environment));
+  const [error, result] = await tryCatch(service.call(text, environment.id));
   if (error) {
     return json({ success: false, error: error.message }, { status: 400 });
   }

apps/webapp/app/v3/services/aiRunFilterService.server.ts

@@ -3,18 +3,13 @@ import { type TaskTriggerSource } from "@trigger.dev/database";
 import { generateText, Output, tool } from "ai";
 import { z } from "zod";
 import { TaskRunListSearchFilters } from "~/components/runs/v3/RunFilters";
-import { env } from "~/env.server";
-import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { logger } from "~/services/logger.server";
 
 const AIFilterResponseSchema = z
   .discriminatedUnion("success", [
     z.object({
       success: z.literal(true),
       filters: TaskRunListSearchFilters.omit({ environments: true }),
-      explanation: z
-        .string()
-        .describe("A short human-readable explanation of what filters were applied"),
     }),
     z.object({
       success: z.literal(false),
@@ -62,7 +57,6 @@ export type AIFilterResult =
   | {
       success: true;
       filters: TaskRunListSearchFilters;
-      explanation: string;
     }
   | {
       success: false;
@@ -79,14 +73,7 @@ export class AIRunFilterService {
     }
   ) {}
 
-  async call(text: string, environment: AuthenticatedEnvironment): Promise<AIFilterResult> {
-    if (!env.OPENAI_API_KEY) {
-      return {
-        success: false,
-        error: "OpenAI API key is not configured",
-      };
-    }
-
+  async call(text: string, environmentId: string): Promise<AIFilterResult> {
     try {
       const result = await generateText({
         model: openai("gpt-4o-mini"),
@@ -205,17 +192,15 @@ export class AIRunFilterService {
         experimental_telemetry: {
           isEnabled: true,
           metadata: {
-            environmentId: environment.id,
-            projectId: environment.projectId,
-            organizationId: environment.organizationId,
+            environmentId,
           },
         },
       });
 
       // Add debugging to see what the AI returned
       logger.info("AI filter response", {
         text,
-        environmentId: environment.id,
+        environmentId,
         result: result.experimental_output,
       });
 
@@ -245,14 +230,13 @@ export class AIRunFilterService {
       return {
         success: true,
         filters: validationResult.data,
-        explanation: result.experimental_output.explanation,
       };
     } catch (error) {
       logger.error("AI filter processing failed", {
         error,
         errorMessage: error instanceof Error ? error.message : String(error),
         text,
-        environmentId: environment.id,
+        environmentId,
       });
 
       // If it's a schema validation error, provide more specific feedback

apps/webapp/evals/aiRunFilter.eval.ts

@@ -0,0 +1,99 @@
+import { evalite } from "evalite";
+import { Levenshtein } from "autoevals";
+import {
+  AIRunFilterService,
+  type QueryQueues,
+  type QueryTags,
+  type QueryTasks,
+  type QueryVersions,
+} from "~/v3/services/aiRunFilterService.server";
+import dotenv from "dotenv";
+
+dotenv.config({ path: "../../.env" });
+
+const queryTags: QueryTags = {
+  query: async (search) => {
+    return {
+      tags: ["user_1", "user_2", "org_1", "org_2"],
+    };
+  },
+};
+
+const queryVersions: QueryVersions = {
+  query: async (versionPrefix, isCurrent) => {
+    if (isCurrent) {
+      return {
+        version: "20250721.1",
+      };
+    }
+
+    return {
+      versions: ["20250721.1", "20250720.2", "20250720.1"],
+    };
+  },
+};
+
+const queryQueues: QueryQueues = {
+  query: async (query, type) => {
+    return {
+      queues: ["shared", "paid"],
+    };
+  },
+};
+
+const queryTasks: QueryTasks = {
+  query: async () => {
+    return {
+      tasks: [
+        { slug: "task1", triggerSource: "STANDARD" },
+        { slug: "task2", triggerSource: "SCHEDULED" },
+      ],
+    };
+  },
+};
+
+evalite("AI Run Filter", {
+  data: async () => {
+    return [
+      {
+        input: "Completed runs",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["COMPLETED_SUCCESSFULLY"],
+          },
+        }),
+      },
+      {
+        input: "Failed runs",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["COMPLETED_WITH_ERRORS", "CRASHED", "TIMED_OUT", "SYSTEM_FAILURE"],
+          },
+        }),
+      },
+      {
+        input: "Executing runs",
+        expected: JSON.stringify({
+          success: true,
+          filters: {
+            statuses: ["EXECUTING", "RETRYING_AFTER_FAILURE", "WAITING_TO_RESUME"],
+          },
+        }),
+      },
+    ];
+  },
+  task: async (input) => {
+    const service = new AIRunFilterService({
+      queryTags,
+      queryVersions,
+      queryQueues,
+      queryTasks,
+    });
+
+    const result = await service.call(input, "123456");
+    return JSON.stringify(result);
+  },
+  scorers: [Levenshtein],
+});

apps/webapp/package.json

@@ -23,7 +23,8 @@
     "clean:sourcemaps": "run-s clean:sourcemaps:*",
     "clean:sourcemaps:public": "rimraf ./build/**/*.map",
     "clean:sourcemaps:build": "rimraf ./public/build/**/*.map",
-    "test": "vitest --no-file-parallelism"
+    "test": "vitest --no-file-parallelism",
+    "eval:dev": "evalite watch"
   },
   "eslintIgnore": [
     "/node_modules",
@@ -248,6 +249,7 @@
     "@types/ws": "^8.5.3",
     "@typescript-eslint/eslint-plugin": "^5.59.6",
     "@typescript-eslint/parser": "^5.59.6",
+    "autoevals": "^0.0.130",
     "autoprefixer": "^10.4.13",
     "css-loader": "^6.10.0",
     "datepicker": "link:@types/@react-aria/datepicker",
@@ -258,6 +260,7 @@
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-react-hooks": "^4.6.2",
     "eslint-plugin-turbo": "^2.0.4",
+    "evalite": "^0.11.4",
     "npm-run-all": "^4.1.5",
     "postcss-import": "^16.0.1",
     "postcss-loader": "^8.1.1",