diff --git a/.code-samples.meilisearch.yaml b/.code-samples.meilisearch.yaml index fe7fd99d0..5370f8281 100644 --- a/.code-samples.meilisearch.yaml +++ b/.code-samples.meilisearch.yaml @@ -843,3 +843,18 @@ webhooks_patch_1: |- }) webhooks_delete_1: |- client.deleteWebhook(WEBHOOK_UUID) +search_parameter_reference_media_1: |- + client.index('INDEX_NAME').search('a futuristic movie', { + hybrid: { + embedder: 'EMBEDDER_NAME' + }, + media: { + textAndPoster: { + text: 'a futuristic movie', + image: { + mime: 'image/jpeg', + data: 'base64EncodedImageData' + } + } + } + }) diff --git a/src/types/experimental-features.ts b/src/types/experimental-features.ts index cd3843a72..a1176a6c6 100644 --- a/src/types/experimental-features.ts +++ b/src/types/experimental-features.ts @@ -4,12 +4,13 @@ * @see `meilisearch::routes::features::RuntimeTogglableFeatures` */ export type RuntimeTogglableFeatures = { - metrics?: boolean | null; - logsRoute?: boolean | null; - editDocumentsByFunction?: boolean | null; + chatCompletions?: boolean | null; + compositeEmbedders?: boolean | null; containsFilter?: boolean | null; - network?: boolean | null; + editDocumentsByFunction?: boolean | null; getTaskDocumentsRoute?: boolean | null; - compositeEmbedders?: boolean | null; - chatCompletions?: boolean | null; + logsRoute?: boolean | null; + metrics?: boolean | null; + multimodal?: boolean | null; + network?: boolean | null; }; diff --git a/src/types/types.ts b/src/types/types.ts index 0cc9d8970..3d57fb6d9 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -211,6 +211,28 @@ export type HybridSearch = { semanticRatio?: number; }; +/** + * Search request media binary data with explicit MIME + * + * @example + * + * ```typescript + * const media: MediaBinary = { + * mime: "image/jpeg", + * data: "base64-encoded-data", + * }; + * ``` + */ +export type MediaBinary = { + /** MIME type of the file */ + mime: string; + /** Base64-encoded data of the file */ + data: string; +}; + +/** Search request media payload with named search fragments */ +export type MediaPayload = Record>; + // https://www.meilisearch.com/docs/reference/api/settings#localized-attributes export type Locale = string; @@ -237,6 +259,7 @@ export type SearchParams = Query & distinct?: string; retrieveVectors?: boolean; locales?: Locale[]; + media?: MediaPayload; }; // Search parameters for searches made with the GET method @@ -575,6 +598,32 @@ export type UserProvidedEmbedder = { binaryQuantized?: boolean; }; +/** + * Indexing or search fragments + * + * @example + * + * ```typescript + * const fragments: EmbedderFragments = { + * textAndPoster: { + * value: { + * content: [ + * { + * type: "text", + * text: "A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.", + * }, + * { + * type: "image_url", + * image_url: "{{doc.poster}}", + * }, + * ], + * }, + * }, + * }; + * ``` + */ +export type EmbedderFragments = Record; + export type RestEmbedder = { source: "rest"; url: string; @@ -587,6 +636,8 @@ export type RestEmbedder = { headers?: Record; documentTemplateMaxBytes?: number; binaryQuantized?: boolean; + indexingFragments?: EmbedderFragments; + searchFragments?: EmbedderFragments; }; export type OllamaEmbedder = { diff --git a/tests/experimental-features.test.ts b/tests/experimental-features.test.ts index ea14dbf27..5823ac858 100644 --- a/tests/experimental-features.test.ts +++ b/tests/experimental-features.test.ts @@ -6,27 +6,29 @@ const ms = await getClient("Master"); afterAll(async () => { await ms.updateExperimentalFeatures({ - metrics: false, - logsRoute: false, - editDocumentsByFunction: false, + chatCompletions: false, + compositeEmbedders: false, containsFilter: false, - network: false, + editDocumentsByFunction: false, getTaskDocumentsRoute: false, - compositeEmbedders: false, - chatCompletions: false, + logsRoute: false, + metrics: false, + multimodal: false, + network: false, } satisfies { [TKey in keyof RuntimeTogglableFeatures]-?: false }); }); test(`${ms.updateExperimentalFeatures.name} and ${ms.getExperimentalFeatures.name} methods`, async () => { const features: { [TKey in keyof RuntimeTogglableFeatures]-?: true } = { - metrics: true, - logsRoute: true, - editDocumentsByFunction: true, + chatCompletions: true, + compositeEmbedders: true, containsFilter: true, - network: true, + editDocumentsByFunction: true, getTaskDocumentsRoute: true, - compositeEmbedders: true, - chatCompletions: true, + logsRoute: true, + metrics: true, + multimodal: true, + network: true, }; const updateResponse = await ms.updateExperimentalFeatures(features); diff --git a/tests/fixtures/master-yoda.jpeg b/tests/fixtures/master-yoda.jpeg new file mode 100644 index 000000000..f241a85a9 Binary files /dev/null and b/tests/fixtures/master-yoda.jpeg differ diff --git a/tests/fixtures/movies.json b/tests/fixtures/movies.json new file mode 100644 index 000000000..cbe67bc66 --- /dev/null +++ b/tests/fixtures/movies.json @@ -0,0 +1,90 @@ +[ + { + "id": 11, + "title": "Star Wars", + "overview": "Princess Leia is captured and held hostage by the evil Imperial forces in their effort to take over the galactic Empire. Venturesome Luke Skywalker and dashing captain Han Solo team together with the loveable robot duo R2-D2 and C-3PO to rescue the beautiful princess and restore peace and justice in the Empire.", + "genres": ["Adventure", "Action", "Science Fiction"], + "poster": "https://image.tmdb.org/t/p/w500/6FfCtAuVAW8XJjZ7eWeLibRLWTw.jpg", + "release_date": 233366400 + }, + { + "id": 12, + "title": "Finding Nemo", + "overview": "Nemo, an adventurous young clownfish, is unexpectedly taken from his Great Barrier Reef home to a dentist's office aquarium. It's up to his worrisome father Marlin and a friendly but forgetful fish Dory to bring Nemo home -- meeting vegetarian sharks, surfer dude turtles, hypnotic jellyfish, hungry seagulls, and more along the way.", + "genres": ["Animation", "Family"], + "poster": "https://image.tmdb.org/t/p/w500/eHuGQ10FUzK1mdOY69wF5pGgEf5.jpg", + "release_date": 1054252800 + }, + { + "id": 13, + "title": "Forrest Gump", + "overview": "A man with a low IQ has accomplished great things in his life and been present during significant historic events—in each case, far exceeding what anyone imagined he could do. But despite all he has achieved, his one true love eludes him.", + "genres": ["Comedy", "Drama", "Romance"], + "poster": "https://image.tmdb.org/t/p/w500/h5J4W4veyxMXDMjeNxZI46TsHOb.jpg", + "release_date": 773452800 + }, + { + "id": 18, + "title": "The Fifth Element", + "overview": "In 2257, a taxi driver is unintentionally given the task of saving a young girl who is part of the key that will ensure the survival of humanity.", + "genres": ["Adventure", "Fantasy", "Action", "Thriller", "Science Fiction"], + "poster": "https://image.tmdb.org/t/p/w500/fPtlCO1yQtnoLHOwKtWz7db6RGU.jpg", + "release_date": 862531200 + }, + { + "id": 22, + "title": "Pirates of the Caribbean: The Curse of the Black Pearl", + "overview": "Jack Sparrow, a freewheeling 18th-century pirate, quarrels with a rival pirate bent on pillaging Port Royal. When the governor's daughter is kidnapped, Sparrow decides to help the girl's love save her.", + "genres": ["Adventure", "Fantasy", "Action"], + "poster": "https://image.tmdb.org/t/p/w500/z8onk7LV9Mmw6zKz4hT6pzzvmvl.jpg", + "release_date": 1057708800 + }, + { + "id": 24, + "title": "Kill Bill: Vol. 1", + "overview": "An assassin is shot by her ruthless employer, Bill, and other members of their assassination circle – but she lives to plot her vengeance.", + "genres": ["Action", "Crime"], + "poster": "https://image.tmdb.org/t/p/w500/v7TaX8kXMXs5yFFGR41guUDNcnB.jpg", + "release_date": 1065744000 + }, + { + "id": 35, + "title": "The Simpsons Movie", + "overview": "After Homer accidentally pollutes the town's water supply, Springfield is encased in a gigantic dome by the EPA and the Simpsons are declared fugitives.", + "genres": ["Animation", "Comedy", "Family"], + "poster": "https://image.tmdb.org/t/p/w500/s3b8TZWwmkYc2KoJ5zk77qB6PzY.jpg", + "release_date": 1185321600 + }, + { + "id": 62, + "title": "2001: A Space Odyssey", + "overview": "Humanity finds a mysterious object buried beneath the lunar surface and sets off to find its origins with the help of HAL 9000, the world's most advanced super computer.", + "genres": ["Science Fiction", "Mystery", "Adventure"], + "poster": "https://image.tmdb.org/t/p/w500/ve72VxNqjGM69Uky4WTo2bK6rfq.jpg", + "release_date": -55209600 + }, + { + "id": 65, + "title": "8 Mile", + "overview": "The setting is Detroit in 1995. The city is divided by 8 Mile, a road that splits the town in half along racial lines. A young white rapper, Jimmy \"B-Rabbit\" Smith Jr. summons strength within himself to cross over these arbitrary boundaries to fulfill his dream of success in hip hop. With his pal Future and the three one third in place, all he has to do is not choke.", + "genres": ["Music", "Drama"], + "poster": "https://image.tmdb.org/t/p/w500/7BmQj8qE1FLuLTf7Xjf9sdIHzoa.jpg", + "release_date": 1036713600 + }, + { + "id": 81, + "title": "Nausicaä of the Valley of the Wind", + "overview": "After a global war, the seaside kingdom known as the Valley of the Wind remains one of the last strongholds on Earth untouched by a poisonous jungle and the powerful insects that guard it. Led by the courageous Princess Nausicaä, the people of the Valley engage in an epic struggle to restore the bond between humanity and Earth.", + "genres": ["Adventure", "Animation", "Fantasy"], + "poster": "https://image.tmdb.org/t/p/w500/sIpcATxMrKHRRUJAGI5UIUT7XMG.jpg", + "release_date": 447811200 + }, + { + "id": 98, + "title": "Gladiator", + "overview": "In the year 180, the death of emperor Marcus Aurelius throws the Roman Empire into chaos. Maximus is one of the Roman army's most capable and trusted generals and a key advisor to the emperor. As Marcus' devious son Commodus ascends to the throne, Maximus is set to be executed. He escapes, but is captured by slave traders. Renamed Spaniard and forced to become a gladiator, Maximus must battle to the death with other men for the amusement of paying audiences.", + "genres": ["Action", "Drama", "Adventure"], + "poster": "https://image.tmdb.org/t/p/w500/ehGpN04mLJIrSnxcZBMvHeG0eDc.jpg", + "release_date": 957139200 + } +] diff --git a/tests/multi_modal_search.test.ts b/tests/multi_modal_search.test.ts new file mode 100644 index 000000000..95e02c045 --- /dev/null +++ b/tests/multi_modal_search.test.ts @@ -0,0 +1,198 @@ +import { beforeAll, describe, expect, test } from "vitest"; +import { getClient } from "./utils/meilisearch-test-utils.js"; +import type { Embedder } from "../src/types/types.js"; +import movies from "./fixtures/movies.json" assert { type: "json" }; +import type { Meilisearch } from "../src/index.js"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { readFileSync } from "node:fs"; + +const VOYAGE_API_KEY = import.meta.env.VITE_VOYAGE_API_KEY as string; + +// Helper function to load image file and return base64 string +function loadImageAsBase64(fileName: string): string { + const imagePath = join( + dirname(fileURLToPath(import.meta.url)), + "fixtures", + fileName, + ); + const imageBuffer = readFileSync(imagePath); + return imageBuffer.toString("base64"); +} + +const INDEX_UID = "multi-modal-search-test"; +const EMBEDDER_NAME = "multimodal"; +const EMBEDDER_CONFIG = { + source: "rest", + url: "https://api.voyageai.com/v1/multimodalembeddings", + apiKey: VOYAGE_API_KEY, + dimensions: 1024, + indexingFragments: { + textAndPoster: { + // the shape of the data here depends on the model used + value: { + content: [ + { + type: "text", + text: "A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.", + }, + { + type: "image_url", + image_url: "{{doc.poster}}", + }, + ], + }, + }, + text: { + value: { + // The shape of the data here depends on the model used + content: [ + { + type: "text", + text: "A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.", + }, + ], + }, + }, + poster: { + value: { + // The shape of the data here depends on the model used + content: [ + { + type: "image_url", + image_url: "{{doc.poster}}", + }, + ], + }, + }, + }, + searchFragments: { + textAndPoster: { + value: { + content: [ + { + type: "text", + text: "{{media.textAndPoster.text}}", + }, + { + type: "image_base64", + image_base64: + "data:{{media.textAndPoster.image.mime}};base64,{{media.textAndPoster.image.data}}", + }, + ], + }, + }, + text: { + value: { + content: [ + { + type: "text", + text: "{{media.text.text}}", + }, + ], + }, + }, + poster: { + value: { + content: [ + { + type: "image_url", + image_url: "{{media.poster.poster}}", + }, + ], + }, + }, + }, + request: { + // This request object matches the Voyage API request object + inputs: ["{{fragment}}", "{{..}}"], + model: "voyage-multimodal-3", + }, + response: { + // This response object matches the Voyage API response object + data: [ + { + embedding: "{{embedding}}", + }, + "{{..}}", + ], + }, +} satisfies Embedder; + +describe.skipIf(!VOYAGE_API_KEY)("Multi-modal search", () => { + let searchClient: Meilisearch; + + beforeAll(async () => { + const client = await getClient("Admin"); + await client.updateExperimentalFeatures({ + multimodal: true, + }); + // Delete the index if it already exists + await client.index(INDEX_UID).delete().waitTask(); + await client.createIndex(INDEX_UID).waitTask(); + await client.index(INDEX_UID).updateSettings({ + searchableAttributes: ["title", "overview"], + embedders: { + [EMBEDDER_NAME]: EMBEDDER_CONFIG, + }, + }); + await client.index(INDEX_UID).addDocuments(movies).waitTask(); + searchClient = await getClient("Search"); + }); + + test("should work with text query", async () => { + const query = "A movie with lightsabers in space"; + const response = await searchClient.index(INDEX_UID).search(query, { + media: { + text: { + text: query, + }, + }, + hybrid: { + embedder: EMBEDDER_NAME, + semanticRatio: 1, + }, + }); + expect(response.hits[0].title).toBe("Star Wars"); + }); + + test("should work with image query", async () => { + const theFifthElementPoster = movies[3].poster; + + const response = await searchClient.index(INDEX_UID).search(null, { + media: { + poster: { + poster: theFifthElementPoster, + }, + }, + hybrid: { + embedder: EMBEDDER_NAME, + semanticRatio: 1, + }, + }); + expect(response.hits[0].title).toBe("The Fifth Element"); + }); + + test("should work with text and image query", async () => { + const query = "a futuristic movie"; + const masterYodaBase64 = loadImageAsBase64("master-yoda.jpeg"); + + const response = await searchClient.index(INDEX_UID).search(null, { + q: query, + media: { + textAndPoster: { + text: query, + image: { + mime: "image/jpeg", + data: masterYodaBase64, + }, + }, + }, + hybrid: { + embedder: EMBEDDER_NAME, + semanticRatio: 1, + }, + }); + expect(response.hits[0].title).toBe("Star Wars"); + }); +}); diff --git a/vite.config.ts b/vite.config.ts index 3c620daf9..2d11f21d8 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -1,4 +1,4 @@ -import { defineConfig } from "vite"; +import { defineConfig, loadEnv } from "vite"; import pkg from "./package.json" with { type: "json" }; const indexInput = "src/index.ts"; @@ -57,6 +57,8 @@ export default defineConfig(({ mode }) => { fileParallelism: false, testTimeout: 100_000, // 100 seconds coverage: { include: ["src/**/*.ts"] }, + // Allow loading env variables from `.env.test` + env: loadEnv("test", process.cwd()), }, }; });