Update image ref resolution to use conversation order

File reference resolver now maps image refs (e.g. image_1, image_2) to user-uploaded images in chronological order across the entire conversation, rather than only the latest user message. Updated tool prompt instructions to clarify this behavior for users.

Author: gary149

src/lib/server/textGeneration/mcp/fileRefs.ts

@@ -24,46 +24,33 @@ const IMAGE_REF_KIND: RefKind = {
 const DEFAULT_REF_KINDS: RefKind[] = [IMAGE_REF_KIND];
 
 /**
- * Build a resolver that maps short ref strings (e.g. "image_1") to the
- * corresponding file payload for the latest user message containing files of
- * the allowed kinds. Currently only images are exposed to end users, but the
- * plumbing supports additional kinds later.
+ * Build a resolver that maps short ref strings (e.g. "image_1", "image_2") to the
+ * corresponding file payload across the whole conversation in chronological
+ * order of user uploads. (image_1 = first user-uploaded image, image_2 = second, etc.)
+ * Currently only images are exposed to end users, but the plumbing supports
+ * additional kinds later.
  */
 export function buildFileRefResolver(
 	messages: EndpointMessage[],
 	refKinds: RefKind[] = DEFAULT_REF_KINDS
 ): FileRefResolver | undefined {
 	if (!Array.isArray(refKinds) || refKinds.length === 0) return undefined;
 
-	// Find the newest user message that has at least one matching file
-	let lastUserWithFiles: EndpointMessage | undefined;
-	for (let i = messages.length - 1; i >= 0; i -= 1) {
-		const msg = messages[i];
+	// Bucket matched files by ref kind preserving conversation order (oldest -> newest)
+	const buckets = new Map<RefKind, FileRefPayload[]>();
+	for (const msg of messages) {
 		if (msg.from !== "user") continue;
-		const hasMatch = (msg.files ?? []).some((file) => {
-			const mime = file?.mime;
-			return refKinds.some((kind) => kind.matches(mime ?? ""));
-		});
-		if (hasMatch) {
-			lastUserWithFiles = msg;
-			break;
+		for (const file of msg.files ?? []) {
+			const mime = file?.mime ?? "";
+			const kind = refKinds.find((k) => k.matches(mime));
+			if (!kind) continue;
+			const payload: FileRefPayload = { name: file.name, mime, base64: file.value };
+			const arr = buckets.get(kind) ?? [];
+			arr.push(payload);
+			buckets.set(kind, arr);
 		}
 	}
 
-	if (!lastUserWithFiles) return undefined;
-
-	// Bucket matched files by ref kind while preserving order within the message
-	const buckets = new Map<RefKind, FileRefPayload[]>();
-	for (const file of lastUserWithFiles.files ?? []) {
-		const mime = file?.mime ?? "";
-		const kind = refKinds.find((k) => k.matches(mime));
-		if (!kind) continue;
-		const payload: FileRefPayload = { name: file.name, mime, base64: file.value };
-		const arr = buckets.get(kind) ?? [];
-		arr.push(payload);
-		buckets.set(kind, arr);
-	}
-
 	if (buckets.size === 0) return undefined;
 
 	const resolver: FileRefResolver = (ref) => {

src/lib/server/textGeneration/utils/toolPrompt.ts

@@ -16,6 +16,6 @@ export function buildToolPreprompt(tools: OpenAiTool[]): string {
 		`Today's date: ${currentDate}.`,
 		`If a tool generates an image, you can inline it directly: ![alt text](image_url).`,
 		`If a tool needs to operate on an image, set its image input parameter (for example, "input_image") to an image reference string.`,
-		`Use "image_1", "image_2", etc. to point to a specific image from a user message with images. You can also reuse a direct image URL from a prior tool result instead of pasting new base64 data.`,
+		`Use "image_1", "image_2", etc. where the number equals the chronological order of user-uploaded images in this conversation (image_1 = first image uploaded, image_2 = second, and so on). You can also reuse a direct image URL from a prior tool result instead of pasting new base64 data.`,
 	].join(" ");
 }