Skip to content

Commit a1c284d

Browse files
cpinncpinn
committed
Reapply "Handle arrays of images when building prompts" (#1559)
This reverts commit 69ae920.
1 parent 9ebbd65 commit a1c284d

File tree

3 files changed

+612
-32
lines changed

3 files changed

+612
-32
lines changed

‎js/src/logger.test.ts‎

Lines changed: 298 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
Attachment,
1919
deepCopyEvent,
2020
renderMessage,
21+
renderMessageImpl,
2122
} from "./logger";
2223
import {
2324
parseTemplateFormat,
@@ -62,7 +63,15 @@ test("renderMessage with file content parts", () => {
6263
],
6364
};
6465

65-
const rendered = renderMessage(
66+
const variables = {
67+
item: "document",
68+
image_url: "https://example.com/image.png",
69+
file_data: "base64data",
70+
file_id: "file-456",
71+
filename: "report.pdf",
72+
};
73+
74+
const rendered = renderMessageImpl(
6675
(template) =>
6776
template
6877
.replace("{{item}}", "document")
@@ -71,6 +80,7 @@ test("renderMessage with file content parts", () => {
7180
.replace("{{file_id}}", "file-456")
7281
.replace("{{filename}}", "report.pdf"),
7382
message,
83+
variables,
7484
);
7585

7686
expect(rendered.content).toEqual([
@@ -95,6 +105,293 @@ test("renderMessage with file content parts", () => {
95105
]);
96106
});
97107

108+
test("renderMessage expands attachment array in image_url parts", () => {
109+
const message = {
110+
role: "user" as const,
111+
content: [
112+
{
113+
type: "image_url" as const,
114+
image_url: { url: "{{images}}" },
115+
},
116+
],
117+
};
118+
119+
const variables = {
120+
images: ["https://example.com/img1.jpg", "https://example.com/img2.jpg"],
121+
};
122+
123+
const rendered = renderMessageImpl(
124+
(template) => template, // Template rendering shouldn't happen for attachment arrays
125+
message,
126+
variables,
127+
);
128+
129+
expect(rendered.content).toEqual([
130+
{
131+
type: "image_url",
132+
image_url: {
133+
url: "https://example.com/img1.jpg",
134+
},
135+
},
136+
{
137+
type: "image_url",
138+
image_url: {
139+
url: "https://example.com/img2.jpg",
140+
},
141+
},
142+
]);
143+
});
144+
145+
test("renderMessage expands inline attachment array in image_url parts", () => {
146+
const message = {
147+
role: "user" as const,
148+
content: [
149+
{
150+
type: "image_url" as const,
151+
image_url: { url: "{{images}}" },
152+
},
153+
],
154+
};
155+
156+
const variables = {
157+
images: [
158+
{
159+
type: "inline_attachment",
160+
src: "data:image/png;base64,abc",
161+
content_type: "image/png",
162+
},
163+
{
164+
type: "inline_attachment",
165+
src: "data:image/jpeg;base64,def",
166+
content_type: "image/jpeg",
167+
},
168+
],
169+
};
170+
171+
const rendered = renderMessageImpl(
172+
(template) => template,
173+
message,
174+
variables,
175+
);
176+
177+
expect(rendered.content).toEqual([
178+
{
179+
type: "image_url",
180+
image_url: {
181+
url: {
182+
type: "inline_attachment",
183+
src: "data:image/png;base64,abc",
184+
content_type: "image/png",
185+
},
186+
},
187+
},
188+
{
189+
type: "image_url",
190+
image_url: {
191+
url: {
192+
type: "inline_attachment",
193+
src: "data:image/jpeg;base64,def",
194+
content_type: "image/jpeg",
195+
},
196+
},
197+
},
198+
]);
199+
});
200+
201+
test("renderMessage does NOT expand mixed content (text + variable)", () => {
202+
const message = {
203+
role: "user" as const,
204+
content: "Look at {{images}}",
205+
};
206+
207+
const variables = {
208+
images: ["https://example.com/img1.jpg", "https://example.com/img2.jpg"],
209+
};
210+
211+
const rendered = renderMessageImpl(
212+
(template) => template.replace("{{images}}", "[array]"),
213+
message,
214+
variables,
215+
);
216+
217+
// Mixed content is not expanded - just rendered normally
218+
expect(rendered.content).toBe("Look at [array]");
219+
});
220+
221+
test("renderMessage expands nested attachment arrays in image_url parts", () => {
222+
const message = {
223+
role: "user" as const,
224+
content: [
225+
{
226+
type: "image_url" as const,
227+
image_url: { url: "{{data.images}}" },
228+
},
229+
],
230+
};
231+
232+
const variables = {
233+
data: {
234+
images: ["https://example.com/img1.jpg", "https://example.com/img2.jpg"],
235+
},
236+
};
237+
238+
const rendered = renderMessageImpl(
239+
(template) => template, // Template rendering shouldn't happen
240+
message,
241+
variables,
242+
);
243+
244+
expect(rendered.content).toEqual([
245+
{
246+
type: "image_url",
247+
image_url: {
248+
url: "https://example.com/img1.jpg",
249+
},
250+
},
251+
{
252+
type: "image_url",
253+
image_url: {
254+
url: "https://example.com/img2.jpg",
255+
},
256+
},
257+
]);
258+
});
259+
260+
test("renderMessage expands deeply nested attachment arrays in image_url parts", () => {
261+
const message = {
262+
role: "user" as const,
263+
content: [
264+
{
265+
type: "image_url" as const,
266+
image_url: { url: "{{user.profile.images}}" },
267+
},
268+
],
269+
};
270+
271+
const variables = {
272+
user: {
273+
profile: {
274+
images: [
275+
{
276+
type: "inline_attachment",
277+
src: "data:image/png;base64,abc",
278+
content_type: "image/png",
279+
},
280+
{
281+
type: "inline_attachment",
282+
src: "data:image/jpeg;base64,def",
283+
content_type: "image/jpeg",
284+
},
285+
],
286+
},
287+
},
288+
};
289+
290+
const rendered = renderMessageImpl(
291+
(template) => template,
292+
message,
293+
variables,
294+
);
295+
296+
expect(rendered.content).toEqual([
297+
{
298+
type: "image_url",
299+
image_url: {
300+
url: {
301+
type: "inline_attachment",
302+
src: "data:image/png;base64,abc",
303+
content_type: "image/png",
304+
},
305+
},
306+
},
307+
{
308+
type: "image_url",
309+
image_url: {
310+
url: {
311+
type: "inline_attachment",
312+
src: "data:image/jpeg;base64,def",
313+
content_type: "image/jpeg",
314+
},
315+
},
316+
},
317+
]);
318+
});
319+
320+
test("renderMessage handles single image_url (no array)", () => {
321+
const message = {
322+
role: "user" as const,
323+
content: [
324+
{
325+
type: "image_url" as const,
326+
image_url: {
327+
url: "{{image}}",
328+
},
329+
},
330+
],
331+
};
332+
333+
const variables = {
334+
image: "https://example.com/single.jpg",
335+
};
336+
337+
const rendered = renderMessageImpl(
338+
(template) =>
339+
template.replace("{{image}}", "https://example.com/single.jpg"),
340+
message,
341+
variables,
342+
);
343+
344+
expect(rendered.content).toEqual([
345+
{
346+
type: "image_url",
347+
image_url: {
348+
url: "https://example.com/single.jpg",
349+
},
350+
},
351+
]);
352+
});
353+
354+
test("renderMessage expands attachment arrays in structured content", () => {
355+
// This tests the case where content is already an array with structured parts,
356+
// and one part has a template variable for an attachment array
357+
const message = {
358+
role: "user" as const,
359+
content: [
360+
{ type: "text" as const, text: "Describe these images" },
361+
{ type: "image_url" as const, image_url: { url: "{{attachments}}" } },
362+
],
363+
};
364+
365+
const variables = {
366+
attachments: [
367+
"https://example.com/img1.jpg",
368+
"https://example.com/img2.jpg",
369+
],
370+
};
371+
372+
const rendered = renderMessageImpl(
373+
(template) => template,
374+
message,
375+
variables,
376+
);
377+
378+
// Should expand {{attachments}} into multiple image_url parts
379+
expect(rendered.content).toEqual([
380+
{
381+
type: "text",
382+
text: "Describe these images",
383+
},
384+
{
385+
type: "image_url",
386+
image_url: { url: "https://example.com/img1.jpg" },
387+
},
388+
{
389+
type: "image_url",
390+
image_url: { url: "https://example.com/img2.jpg" },
391+
},
392+
]);
393+
});
394+
98395
test("verify MemoryBackgroundLogger intercepts logs", async () => {
99396
// Log to memory for the tests.
100397
_exportsForTestingOnly.simulateLoginForTests();

0 commit comments

Comments
 (0)