v1.16: Multimodal search

.code-samples.meilisearch.yaml

@@ -1533,3 +1533,19 @@ update_network_1: |-
         }
       }
     }'
+search_parameter_reference_media_1: |-
+  curl \
+  -X POST 'MEILISEARCH_URL/indexes/INDEX_NAME/search' \
+  -H 'Content-Type: application/json' \
+  --data-binary '{
+    "hybrid": {
+      "embedder": "EMBEDDER_NAME"
+    },
+    "media": {
+      "FIELD_A": "VALUE_A",
+      "FIELD_B" : {
+        "FIELD_C": "VALUE_B"
+        "FIELD_D": "VALUE_C"
+      }
+    }
+  }'

learn/resources/experimental_features_overview.mdx

@@ -59,3 +59,4 @@ Activating or deactivating experimental features this way does not require you t
 | [Search query embedding cache](/learn/self_hosted/configure_meilisearch_at_launch#search-query-embedding-cache) | Enable a cache for search query embeddings                 | CLI flag or environment variable            |
 | [Uncompressed snapshots](/learn/self_hosted/configure_meilisearch_at_launch#uncompressed-snapshots) | Disable snapshot compaction                                            | CLI flag or environment variable            |
 | [Maximum batch payload size](/learn/self_hosted/configure_meilisearch_at_launch#maximum-batch-payload-size) | Limit batch payload size                                       | CLI flag or environment variable            |
+| [Multimodal search](/reference/api/settings) | Enable multimodal search                                       | API route            |

reference/api/experimental_features.mdx

@@ -26,7 +26,8 @@ The experimental API route is not compatible with all experimental features. Con
   "containsFilter": false,
   "editDocumentsByFunction": false,
   "network": false,
-  "chatCompletions": false
+  "chatCompletions": false,
+  "multimodal": false
 }
 ```
 
@@ -38,6 +39,7 @@ The experimental API route is not compatible with all experimental features. Con
 | **`editDocumentsByFunction`** | Boolean | `true` if feature is active, `false` otherwise |
 | **`network`**                 | Boolean | `true` if feature is active, `false` otherwise |
 | **`chatCompletions`**         | Boolean | `true` if feature is active, `false` otherwise |
+| **`multimodal`**              | Boolean | `true` if feature is active, `false` otherwise |
 
 ## Get all experimental features
 
@@ -58,7 +60,8 @@ Get a list of all experimental features that can be activated via the `/experime
   "containsFilter": false,
   "editDocumentsByFunction": false,
   "network": false,
-  "chatCompletions": false
+  "chatCompletions": false,
+  "multimodal": false
 }
 ```
 
@@ -87,6 +90,7 @@ Setting a field to `null` leaves its value unchanged.
   "containsFilter": false,
   "editDocumentsByFunction": false,
   "network": false,
-  "chatCompletions": false
+  "chatCompletions": false,
+  "multimodal": false
 }
 ```

reference/api/search.mdx

@@ -97,6 +97,7 @@ By default, [this endpoint returns a maximum of 1000 results](/learn/resources/k
 | **[`vector`](#vector)**                                 | Array of numbers | `null`        | Search using a custom query vector                  |
 | **[`retrieveVectors`](#display-_vectors-in-response)**  | Boolean          | `false`       | Return document vector data                         |
 | **[`locales`](#query-locales)**                         | Array of strings | `null`        | Explicitly specify languages used in a query        |
+| **[`media`](#media)**                                   | Object           | `null`        | Perform AI-powered search queries with multimodal content         |
 
 ### Response
 
@@ -1283,3 +1284,46 @@ For full control over the way Meilisearch detects languages during indexing and
   …
 }
 ```
+
+### Media <NoticeTag type="experimental" label="experimental" />
+
+**Parameter**: `media`<br />
+**Expected value**: Object<br />
+**Default value**: `null`
+
+<Note>
+This is an experimental feature. Use the Meilisearch Cloud UI or the experimental features endpoint to activate it:
+
+```sh
+curl \
+  -X PATCH 'MEILISEARCH_URL/experimental-features/' \
+  -H 'Content-Type: application/json' \
+  --data-binary '{
+    "multimodal": true
+  }'
+```
+</Note>
+
+Specifies data to populate search fragments when performing multimodal searches.
+
+`media` must be an object whose fields must correspond to the data required by one [search fragment](/reference/api/settings#searchfragments). `media` must match a single search fragment. If `media` matches more than one fragment or no search fragments at all, Meilisearch will return an error.
+
+It is mandatory to specify an embedder when using `media`. `media` is incompatible with `vector`.
+
+#### Example
+
+<CodeSamplesSearchParameterReferenceMedia1 />
+
+```json
+{
+  "hits": [
+    {
+      "id": 0,
+      "title": "DOCUMENT NAME",
+      …
+    }
+    …
+  ],
+  …
+}
+```

reference/api/settings.mdx

@@ -2512,7 +2512,9 @@ These embedder objects may contain the following fields:
 | **`binaryQuantized`**          | Boolean | Empty                                                                                                                                           | Once set to `true`, irreversibly converts all vector dimensions to 1-bit values                                                                                |
 | **`indexingEmbedder`**         | Object  | Empty                                                                                                                                           | Configures embedder to vectorize documents during indexing                                                                                                     |
 | **`searchEmbedder`**           | Object  | Empty                                                                                                                                           | Configures embedder to vectorize search queries                                                                                                                |
-| **`pooling`**                  | String  | `"useModel"`                                                                                                                                    | Pooling method for Hugging Face embedders                                                                                                                      |
+| **`pooling`**                  | String  | `"useModel"`                                                                                                                                    | Pooling method for Hugging Face embedders                                                                                                                     |
+| **`indexingFragments`**        | Object  | Empty                                                                                                                                    | Configures multimodal embedding generation at indexing time                                                                                                                      |
+| **`searchFragments`**        | Object  | Empty                                                                                                                                    | Configures data handling during multimodal search                                                                                                                       |
 
 ### Get embedder settings
 
@@ -2875,6 +2877,118 @@ Both fields must be an object and accept the same fields as a regular embedder,
 
 `indexingEmbedder` and `searchEmbedder` are incompatible with all other embedder sources.
 
+##### `indexingFragments` <NoticeTag type="experimental" label="experimental" />
+
+<Note>
+This is an experimental feature. Use the Meilisearch Cloud UI or the experimental features endpoint to activate it:
+
+```sh
+curl \
+  -X PATCH 'MEILISEARCH_URL/experimental-features/' \
+  -H 'Content-Type: application/json' \
+  --data-binary '{
+    "multimodal": true
+  }'
+```
+</Note>
+
+`indexingFragments` specifies which fields in your documents should be used to generate multimodal embeddings. It must be an object with the following structure:
+
+```json
+  "FRAGMENT_NAME": {
+    "value": {
+      …
+    }
+  }
+```
+
+`FRAGMENT_NAME` can be any valid string. It must contain a single field, `value`. `value` must then follow your chosen model's specifications.
+
+For example, for Voyage AI's text and image embedder, `value` must be an array containing a single object with a `type` field. Depending on `type`'s value, you must include either `text`, `image_url`, or `image_base64`:
+
+```json
+{
+  "VOYAGE_FRAGMENT_NAME_A": {
+    "value": {
+      "content": [
+        {
+          "type": "text",
+          "text": "A document called {{doc.title}} that can be described as {{doc.description}}"
+        }
+      ]
+    }
+  },
+  "VOYAGE_FRAGMENT_NAME_B": {
+    "value": {
+      "content": [
+        {
+          "type": "image_url",
+          "image_url": "{{doc.image_url}}"
+        }
+      ]
+    }
+  },
+}
+```
+
+Use Liquid templates to interpolate document data into the fragment fields, where `doc` gives you access to all fields within a document.
+
+`indexingFragments` is optional when using the `rest` source.
+
+`indexingFragments` is incompatible with all other embedder sources.
+
+Specifying a `documentTemplate` in an embedder using `indexingFragments` will result in an error.
+
+You must specify at least one valid fragment in `searchFragments` when using `indexingFragments`.
+
+##### `searchFragments` <NoticeTag type="experimental" label="experimental" />
+
+<Note>
+This is an experimental feature. Use the Meilisearch Cloud UI or the experimental features endpoint to activate it:
+
+```sh
+curl \
+  -X PATCH 'MEILISEARCH_URL/experimental-features/' \
+  -H 'Content-Type: application/json' \
+  --data-binary '{
+    "multimodal": true
+  }'
+```
+</Note>
+
+`searchFragments` instructs Meilisearch how to parse fields present in a query's [`media` search parameter](/reference/api/search#media). It must be an object following the same structure as the [`indexingFragments`](/reference/api/settings#indexingfragments) object:
+
+```json
+  "FRAGMENT_NAME": {
+    "value": {
+      …
+    }
+  }
+```
+
+As with `indexingFragments`, the content of `value` should follow your model's specification.
+
+Use Liquid templates to interpolate search query data into the fragment fields, where `media` gives you access to all multimodal data received with a query:
+
+```json
+"SEARCH_FRAGMENT_A": {
+  "value": {
+    "content": [
+      {
+        "type": "image_base64",
+        "image_base64": "data:{{media.image.mime}};base64,{{media.image.data}}"
+      }
+    ]
+  }
+},
+```
+
+`searchFragments` is optional when using the `rest` source.
+
+`searchFragments` is incompatible with all other embedder sources.
+
+You must specify at least one valid fragment in `indexingFragments` when using `searchFragments`.
+
 #### Example
 
 <CodeSamplesUpdateEmbedders1 />

reference/errors/error_codes.mdx

@@ -146,6 +146,14 @@ The given [`uid`](/reference/api/keys#uid) is invalid. The `uid` must follow the
 
 The value passed to [`attributesToSearchOn`](/reference/api/search#customize-attributes-to-search-on-at-search-time) is invalid. `attributesToSearchOn` accepts an array of strings indicating document attributes. Attributes given to `attributesToSearchOn` must be present in the [`searchableAttributes` list](/learn/relevancy/displayed_searchable_attributes#the-searchableattributes-list).
 
+## `invalid_search_media`
+
+The value passed to [`media`](/reference/api/search#media) is not a valid JSON object.
+
+## `invalid_search_media_and_vector`
+
+The search query contains non-`null` values for both [`media`](/reference/api/search#media) and [`vector`](/reference/api/search#media). These two parameters are mutually exclusive, since `media` generates vector embeddings via the embedder configured in `hybrid`.
+
 ## `invalid_content_type`
 
 The [Content-Type header](/reference/api/overview#content-type) is not supported by Meilisearch. Currently, Meilisearch only supports JSON, CSV, and NDJSON.

snippets/samples/code_samples_add_movies_json_1.mdx

@@ -59,7 +59,7 @@ file, _ := os.ReadFile("movies.json")
 var movies interface{}
 json.Unmarshal([]byte(file), &movies)
 
-client.Index("movies").AddDocuments(&movies)
+client.Index("movies").AddDocuments(&movies, nil)
 ```
 
 ```csharp C#

snippets/samples/code_samples_add_or_replace_documents_1.mdx

@@ -80,7 +80,7 @@ documents := []map[string]interface{}{
     "release_date": "2019-03-23",
   },
 }
-client.Index("movies").AddDocuments(documents)
+client.Index("movies").AddDocuments(documents, nil)
 ```
 
 ```csharp C#

snippets/samples/code_samples_add_or_update_documents_1.mdx

@@ -66,7 +66,7 @@ documents := []map[string]interface{}{
     "genres": "comedy",
   },
 }
-client.Index("movies").UpdateDocuments(documents)
+client.Index("movies").UpdateDocuments(documents, nil)
 ```
 
 ```csharp C#

snippets/samples/code_samples_async_guide_filter_by_statuses_1.mdx

@@ -42,7 +42,7 @@ await client.GetTasksAsync(new TasksQuery { Statuses = new List<TaskInfoStatus>
 ```rust Rust
 let mut query = TasksQuery::new(&client);
 let tasks = query
-  .with_statuses(["failed", "canceled"])
+  .with_statuses(["failed"])
   .execute()
   .await
   .unwrap();

snippets/samples/code_samples_async_guide_filter_by_statuses_2.mdx

@@ -4,4 +4,13 @@
 curl \
   -X GET 'MEILISEARCH_URL/tasks?statuses=failed,canceled'
 ```
+
+```rust Rust
+let mut query = TasksQuery::new(&client);
+let tasks = query
+  .with_statuses(["failed", "canceled"])
+  .execute()
+  .await
+  .unwrap();
+```
 </CodeGroup>

snippets/samples/code_samples_basic_security_tutorial_admin_1.mdx

@@ -10,4 +10,12 @@ curl \
     "primaryKey": "id"
   }'
 ```
+
+```rust Rust
+let client = Client::new("http://localhost:7700", Some("DEFAULT_ADMIN_API_KEY"));
+let task = client
+  .create_index("medical_records", Some("id"))
+  .await
+  .unwrap();
+```
 </CodeGroup>

snippets/samples/code_samples_basic_security_tutorial_listing_1.mdx

@@ -4,4 +4,12 @@
 curl -X GET 'MEILISEARCH_URL/keys' \
 -H 'Authorization: Bearer MASTER_KEY'
 ```
+
+```rust Rust
+let client = Client::new("http://localhost:7700", Some("MASTER_KEY"));
+client
+  .get_keys()
+  .await
+  .unwrap();
+```
 </CodeGroup>

snippets/samples/code_samples_basic_security_tutorial_search_1.mdx

@@ -7,4 +7,15 @@ curl \
   -H 'Authorization: Bearer DEFAULT_SEARCH_API_KEY' \
   --data-binary '{ "q": "appointments" }'
 ```
+
+```rust Rust
+let client = Client::new("http://localhost:7700", Some("DEFAULT_SEARCH_API_KEY"));
+let index = client.index("medical_records");
+index
+  .search()
+  .with_query("appointments")
+  .execute::<MedicalRecord>()
+  .await
+  .unwrap();
+```
 </CodeGroup>

snippets/samples/code_samples_date_guide_filterable_attributes_1.mdx

@@ -30,7 +30,7 @@ client.index('games').update_filterable_attributes(['release_timestamp'])
 ```
 
 ```go Go
-filterableAttributes := []string{"release_timestamp"}
+filterableAttributes := []interface{}{"release_timestamp"}
 client.Index("games").UpdateFilterableAttributes(&filterableAttributes)
 ```
 

snippets/samples/code_samples_date_guide_index_1.mdx

@@ -54,7 +54,7 @@ byteValue, _ := io.ReadAll(jsonFile)
 var games []map[string]interface{}
 json.Unmarshal(byteValue, &games)
 
-client.Index("games").AddDocuments(games)
+client.Index("games").AddDocuments(games, nil)
 ```
 
 ```csharp C#

snippets/samples/code_samples_distinct_attribute_guide_filterable_1.mdx

@@ -38,7 +38,7 @@ client.index('products').update_filterable_attributes([
 ```
 
 ```go Go
-filterableAttributes := []string{
+filterableAttributes := []interface{}{
   "product_id",
   "sku",
   "url",