Add openai container provider

Author: pokey

examples/src/createAgent/middleware/codeExecutionOpenAI.ts

@@ -0,0 +1,79 @@
+/**
+ * Example demonstrating the Code Execution Middleware with OpenAI.
+ *
+ * This middleware enables GPT models to execute code in a sandboxed container,
+ * analyze uploaded files, and generate output files. The example demonstrates
+ * multi-turn conversation with container and file state persistence.
+ */
+
+import { OpenAIContainerProvider } from "@langchain/openai/middleware";
+import { ChatOpenAI } from "@langchain/openai";
+import { HumanMessage } from "@langchain/core/messages";
+import { MemorySaver } from "@langchain/langgraph";
+import {
+  codeExecutionMiddleware,
+  createAgent,
+  MemoryFileProvider,
+} from "langchain";
+import fs from "node:fs/promises";
+import { join } from "node:path";
+
+// Initial setup
+const model = new ChatOpenAI({
+  model: "gpt-4.1",
+  useResponsesApi: true,
+});
+
+const middleware = codeExecutionMiddleware(
+  new OpenAIContainerProvider(),
+  new MemoryFileProvider()
+);
+
+const agent = createAgent({
+  model,
+  middleware: [middleware],
+  checkpointer: new MemorySaver(),
+});
+
+const thread = {
+  configurable: {
+    thread_id: "test-123",
+  },
+};
+
+// Read and add the test data file
+const testDataPath = "test_data.csv";
+const fileContent = await fs.readFile(testDataPath);
+
+// First invocation - should create container and analyze uploaded data
+const response1 = await agent.invoke(
+  {
+    messages: new HumanMessage("Filter to just widget A"),
+    files: [await middleware.addFile(testDataPath, fileContent)],
+  },
+  thread
+);
+console.log("Response 1:", response1.messages);
+
+// Second invocation - should reuse container and previous analysis
+const response2 = await agent.invoke(
+  {
+    messages: new HumanMessage(
+      "Turn that into a graph of sales and units over time."
+    ),
+  },
+  thread
+);
+console.log("Response 2:", response2.messages);
+
+// Extract and download generated files
+const generatedFiles = middleware
+  .files(response2)
+  .filter(({ type, path }) => type === "tool" && path.endsWith(".png"));
+
+for (const file of generatedFiles) {
+  const content = await file.getContent();
+  const outputPath = join(".", file.path);
+  await fs.writeFile(outputPath, content);
+  console.log(`Downloaded generated file: ${outputPath}`);
+}

libs/providers/langchain-openai/package.json

@@ -35,19 +35,27 @@
     "zod": "^3.25.76 || ^4"
   },
   "peerDependencies": {
-    "@langchain/core": "^1.0.0-alpha.6"
+    "@langchain/core": "^1.0.0-alpha.6",
+    "langchain": "workspace:*"
+  },
+  "peerDependenciesMeta": {
+    "langchain": {
+      "optional": true
+    }
   },
   "devDependencies": {
     "@azure/identity": "^4.2.1",
     "@cfworker/json-schema": "^4.1.1",
     "@langchain/core": "workspace:*",
     "@langchain/eslint": "workspace:*",
+    "@langchain/langgraph-checkpoint": "^0.1.1",
     "@langchain/standard-tests": "workspace:*",
     "@tsconfig/recommended": "^1.0.10",
     "@vitest/coverage-v8": "^3.2.4",
     "dotenv": "^17.2.1",
     "dpdm": "^3.14.0",
     "eslint": "^9.34.0",
+    "langchain": "workspace:*",
     "prettier": "^2.8.3",
     "rimraf": "^5.0.1",
     "typescript": "~5.8.3",
@@ -86,9 +94,20 @@
       },
       "input": "./src/index.ts"
     },
+    "./middleware": {
+      "import": {
+        "types": "./dist/middleware/index.d.ts",
+        "default": "./dist/middleware/index.js"
+      },
+      "require": {
+        "types": "./dist/middleware/index.d.cts",
+        "default": "./dist/middleware/index.cjs"
+      },
+      "input": "./src/middleware/index.ts"
+    },
     "./package.json": "./package.json"
   },
   "files": [
     "dist/"
   ]
-}
+}

libs/providers/langchain-openai/src/middleware/containerProvider.ts

@@ -0,0 +1,128 @@
+import {
+  ContainerInfo,
+  ContainerProvider,
+  FileFromProvider,
+  UploadFileToContainerOpts,
+  UploadFileToContainerRet,
+  type ModelRequest,
+} from "langchain";
+import OpenAI, { toFile } from "openai";
+import { extractGeneratedFilesOpenAI } from "./extractGeneratedFiles.js";
+
+export class OpenAIContainerProvider implements ContainerProvider {
+  tools = [{ type: "code_interpreter", name: "code_interpreter" }];
+
+  async startContainer(): Promise<ContainerInfo> {
+    // Create a new container explicitly using OpenAI Containers API
+    // Use a unique name for the container
+    const container = await new OpenAI().containers.create({
+      name: `langchain-code-execution-${crypto.randomUUID()}`,
+    });
+
+    return {
+      id: container.id,
+      // OpenAI containers expire after 20 minutes of inactivity
+      // Make it 19 minutes to be safe
+      // FIXME: Better track lifecycle because it's actually after 20 minutes of inactivity
+      // Can probably get last activity off of response metadata
+      expiresAt: new Date(
+        container.created_at +
+          (container.expires_after?.minutes ?? 19) * 60 * 1000
+      ),
+    };
+  }
+
+  async uploadFileToContainer({
+    containerId,
+    path,
+    providerId,
+    getContent,
+  }: UploadFileToContainerOpts): Promise<UploadFileToContainerRet> {
+    // FIXME: It would be more efficient to pass the files to the container at creation time,
+    // which is supported by the API, but our current interface doesn't support that
+    const client = new OpenAI();
+
+    let newProviderId = providerId;
+    if (newProviderId == null) {
+      // Upload to OpenAI Files API
+      const fileObject = await client.files.create({
+        file: await toFile(await getContent(), path),
+        purpose: "user_data",
+      });
+
+      newProviderId = fileObject.id;
+    }
+
+    if (containerId == null) {
+      throw new Error("containerId is required to upload file to container");
+    }
+
+    await client.containers.files.create(containerId, {
+      file_id: newProviderId,
+    });
+
+    return { providerId: newProviderId };
+  }
+
+  async modifyModelRequest?<TState extends Record<string, unknown>, TContext>(
+    containerId: string | undefined,
+    _newFiles: string[],
+    request: ModelRequest<TState, TContext>
+  ): Promise<ModelRequest<TState, TContext> | void> {
+    // Build the code_interpreter tool configuration
+    const codeInterpreterTool = {
+      type: "code_interpreter" as const,
+      container: containerId,
+    };
+
+    return {
+      ...request,
+      tools: [codeInterpreterTool],
+    };
+  }
+
+  async extractFilesFromModelResponse(
+    messages: ModelRequest["messages"]
+  ): Promise<FileFromProvider[]> {
+    // Extract file information from the last message's annotations
+    const lastMessage = messages[messages.length - 1];
+    const files = extractGeneratedFilesOpenAI({ messages: [lastMessage] });
+
+    const client = new OpenAI();
+
+    // Download each file and return file info
+    return Promise.all(
+      files.map(async (file) => {
+        const response = await client.containers.files.content.retrieve(
+          file.fileId,
+          {
+            container_id: file.containerId,
+          }
+        );
+
+        if (!response.body) {
+          throw new Error(
+            `No body in file download response for ${file.fileId}`
+          );
+        }
+
+        // Convert web ReadableStream to Buffer
+        const chunks: Uint8Array[] = [];
+        const reader = response.body.getReader();
+        while (true) {
+          const { done, value } = await reader.read();
+          if (done) break;
+          chunks.push(value);
+        }
+        const content = Buffer.concat(chunks);
+
+        return {
+          providerId: file.fileId,
+          content,
+          path: file.filename,
+          type: "tool" as const,
+        };
+      })
+    );
+  }
+}

libs/providers/langchain-openai/src/middleware/extractGeneratedFiles.ts

@@ -0,0 +1,65 @@
+/**
+ * Extract generated file information from an OpenAI code execution response.
+ *
+ * Parses the message annotations to find files created during code execution.
+ * Returns file information including container_id, file_id, and filename.
+ *
+ * @param response - The OpenAI message response
+ * @returns Array of file information objects
+ *
+ * @example
+ * ```typescript
+ * import { extractGeneratedFilesOpenAI, downloadFileOpenAI } from '@langchain/openai/middleware';
+ *
+ * const response = await agent.invoke({...});
+ * const files = extractGeneratedFilesOpenAI(response);
+ *
+ * for (const file of files) {
+ *   await downloadFileOpenAI(client, file.containerId, file.fileId, file.filename);
+ * }
+ * ```
+ */
+export function extractGeneratedFilesOpenAI(response: any): Array<{
+  fileId: string;
+  containerId: string;
+  filename: string;
+}> {
+  const files: Array<{
+    fileId: string;
+    containerId: string;
+    filename: string;
+  }> = [];
+
+  // Handle both single message responses and response objects with messages array
+  const messages = response.messages || [response];
+
+  for (const message of messages) {
+    if (!message.content || !Array.isArray(message.content)) {
+      continue;
+    }
+
+    for (const contentItem of message.content) {
+      if (
+        contentItem.type === "text" &&
+        Array.isArray(contentItem.annotations)
+      ) {
+        for (const annotation of contentItem.annotations) {
+          if (
+            annotation.type === "container_file_citation" &&
+            annotation.file_id &&
+            annotation.container_id &&
+            annotation.filename
+          ) {
+            files.push({
+              fileId: annotation.file_id,
+              containerId: annotation.container_id,
+              filename: annotation.filename,
+            });
+          }
+        }
+      }
+    }
+  }
+
+  return files;
+}

libs/providers/langchain-openai/src/middleware/index.ts

@@ -0,0 +1 @@
+export { OpenAIContainerProvider } from "./containerProvider.js";