v0.4.0

Author: OlivierDehaene

Cargo.lock

@@ -11,7 +11,7 @@ members = [
 resolver = "2"
 
 [workspace.package]
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 authors = ["Olivier Dehaene"]
 homepage = "https://github.com/huggingface/text-embeddings-inference"

Cargo.toml

@@ -100,7 +100,7 @@ model=BAAI/bge-large-en-v1.5
 revision=refs/pr/5
 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
 
-docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.3.0 --model-id $model --revision $revision
+docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.4.0 --model-id $model --revision $revision
 ```
 
 And then you can make requests like
@@ -243,13 +243,13 @@ Text Embeddings Inference ships with multiple Docker images that you can use to
 
 | Architecture                        | Image                                                                     |
 |-------------------------------------|---------------------------------------------------------------------------|
-| CPU                                 | ghcr.io/huggingface/text-embeddings-inference:cpu-0.3.0                   |
+| CPU                                 | ghcr.io/huggingface/text-embeddings-inference:cpu-0.4.0                   |
 | Volta                               | NOT SUPPORTED                                                             |
-| Turing (T4, RTX 2000 series, ...)   | ghcr.io/huggingface/text-embeddings-inference:turing-0.3.0 (experimental) |
-| Ampere 80 (A100, A30)               | ghcr.io/huggingface/text-embeddings-inference:0.3.0                       |
-| Ampere 86 (A10, A40, ...)           | ghcr.io/huggingface/text-embeddings-inference:86-0.3.0                    |
-| Ada Lovelace (RTX 4000 series, ...) | ghcr.io/huggingface/text-embeddings-inference:89-0.3.0                    |
-| Hopper (H100)                       | ghcr.io/huggingface/text-embeddings-inference:hopper-0.3.0 (experimental) |
+| Turing (T4, RTX 2000 series, ...)   | ghcr.io/huggingface/text-embeddings-inference:turing-0.4.0 (experimental) |
+| Ampere 80 (A100, A30)               | ghcr.io/huggingface/text-embeddings-inference:0.4.0                       |
+| Ampere 86 (A10, A40, ...)           | ghcr.io/huggingface/text-embeddings-inference:86-0.4.0                    |
+| Ada Lovelace (RTX 4000 series, ...) | ghcr.io/huggingface/text-embeddings-inference:89-0.4.0                    |
+| Hopper (H100)                       | ghcr.io/huggingface/text-embeddings-inference:hopper-0.4.0 (experimental) |
 
 **Warning**: Flash Attention is turned off by default for the Turing image as it suffers from precision issues.
 You can turn Flash Attention v1 ON by using the `USE_FLASH_ATTENTION=True` environment variable.
@@ -278,7 +278,7 @@ model=<your private model>
 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
 token=<your cli READ token>
 
-docker run --gpus all -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.3.0 --model-id $model
+docker run --gpus all -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.4.0 --model-id $model
 ```
 
 ### Using Sequence Classification models
@@ -293,7 +293,7 @@ model=BAAI/bge-reranker-large
 revision=refs/pr/4
 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
 
-docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.3.0 --model-id $model --revision $revision
+docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.4.0 --model-id $model --revision $revision
 ```
 
 And then you can rank the similarity between a pair of inputs with:
@@ -309,9 +309,9 @@ You can also use classic Sequence Classification models like `SamLowe/roberta-ba
 
 ```shell
 model=SamLowe/roberta-base-go_emotions
-volume=$PWD/data 
+volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
 
-docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.3.0 --model-id $model 
+docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.4.0 --model-id $model 
 ```
 
 Once you have deployed the model you can use the `predict` endpoint to get the emotions most associated with an input:

README.md

@@ -10,6 +10,6 @@ hf-hub = { version = "^0.3.0", features = ["tokio"] }
 metrics = "^0.21"
 text-embeddings-backend = { path = "../backends" }
 thiserror = "^1.0"
-tokenizers = { version = "^0.14.1", default-features=false, features=["onig", "esaxx_fast"] }
+tokenizers = { version = "^0.15.0", default-features=false, features=["onig", "esaxx_fast"] }
 tracing = "^0.1"
 tokio = { version = "^1.25", features = ["rt", "rt-multi-thread", "parking_lot", "sync"] }

core/Cargo.toml

@@ -9,7 +9,7 @@
     "license": {
       "name": "HFOIL"
     },
-    "version": "0.3.0"
+    "version": "0.4.0"
   },
   "paths": {
     "/embed": {

docs/openapi.json

@@ -37,5 +37,5 @@ model=<your private model>
 volume=$PWD/data
 token=<your cli Hugging Face Hub token>
 
-docker run --gpus all -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.2.2 --model-id $model
+docker run --gpus all -e HUGGING_FACE_HUB_TOKEN=$token -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.4.0 --model-id $model
 ```

docs/source/en/private_models.md

@@ -34,7 +34,7 @@ model=BAAI/bge-large-en-v1.5
 revision=refs/pr/5
 volume=$PWD/data 
 
-docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.2.2 --model-id $model --revision $revision
+docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.4.0 --model-id $model --revision $revision
 ```
 
 <Tip>
@@ -67,7 +67,7 @@ model=BAAI/bge-reranker-large
 revision=refs/pr/4
 volume=$PWD/data 
 
-docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.3.0 --model-id $model --revision $revision
+docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.4.0 --model-id $model --revision $revision
 ```
 
 Once you have deployed a model you can use the `predict` endpoint and rank the similarity between a pair of inputs:
@@ -85,7 +85,7 @@ You can also use classic Sequence Classification models like `SamLowe/roberta-ba
 model=SamLowe/roberta-base-go_emotions
 volume=$PWD/data 
 
-docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.3.0 --model-id $model 
+docker run --gpus all -p 8080:80 -v $volume:/data --pull always ghcr.io/huggingface/text-embeddings-inference:0.4.0 --model-id $model 
 ```
 
 Once you have deployed the model you can use the `predict` endpoint to get the emotions most associated with an input:

docs/source/en/quick_tour.md

@@ -68,13 +68,15 @@ NVIDIA drivers with CUDA version 12.2 or higher.
 
 Find the appropriate Docker image for your hardware in the following table:
 
-| Architecture                        | Image                                                      |
-|-------------------------------------|------------------------------------------------------------|
-| CPU                                 | ghcr.io/huggingface/text-embeddings-inference:cpu-0.2.2    |
-| Volta                               | NOT SUPPORTED                                              |
-| Turing (T4, RTX 2000 series, ...)   | ghcr.io/huggingface/text-embeddings-inference:turing-0.2.2 |
-| Ampere 80 (A100, A30)               | ghcr.io/huggingface/text-embeddings-inference:0.2.2        |
-| Ampere 86 (A10, A40, ...)           | ghcr.io/huggingface/text-embeddings-inference:86-0.2.2     |
-| Ada Lovelace (RTX 4000 series, ...) | ghcr.io/huggingface/text-embeddings-inference:89-0.2.2     |
-| Hopper (H100)                       | ghcr.io/huggingface/text-embeddings-inference:hopper-0.2.2 |
-
+| Architecture                        | Image                                                                     |
+|-------------------------------------|---------------------------------------------------------------------------|
+| CPU                                 | ghcr.io/huggingface/text-embeddings-inference:cpu-0.4.0                   |
+| Volta                               | NOT SUPPORTED                                                             |
+| Turing (T4, RTX 2000 series, ...)   | ghcr.io/huggingface/text-embeddings-inference:turing-0.4.0 (experimental) |
+| Ampere 80 (A100, A30)               | ghcr.io/huggingface/text-embeddings-inference:0.4.0                       |
+| Ampere 86 (A10, A40, ...)           | ghcr.io/huggingface/text-embeddings-inference:86-0.4.0                    |
+| Ada Lovelace (RTX 4000 series, ...) | ghcr.io/huggingface/text-embeddings-inference:89-0.4.0                    |
+| Hopper (H100)                       | ghcr.io/huggingface/text-embeddings-inference:hopper-0.4.0 (experimental) |
+
+**Warning**: Flash Attention is turned off by default for the Turing image as it suffers from precision issues.
+You can turn Flash Attention v1 ON by using the `USE_FLASH_ATTENTION=True` environment variable.

docs/source/en/supported_models.md

@@ -35,7 +35,7 @@ reqwest = { version = "0.11.14", features = [] }
 serde = "1.0.152"
 serde_json = "1.0.93"
 thiserror = "1.0.38"
-tokenizers = { version = "0.14.1", default-features=false, features=["onig", "esaxx_fast"] }
+tokenizers = { version = "0.15.0", default-features=false, features=["onig", "esaxx_fast"] }
 tokio = { version = "1.25.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
 tower-http = { version = "0.4.0", features = ["cors"] }
 tracing = "0.1.37"

router/Cargo.toml

@@ -19,7 +19,8 @@ use text_embeddings_core::infer::Infer;
 use text_embeddings_core::queue::Queue;
 use text_embeddings_core::tokenization::Tokenization;
 use text_embeddings_router::{server, ClassifierModel, EmbeddingModel, Info, ModelType};
-use tokenizers::Tokenizer;
+use tokenizers::decoders::metaspace::PrependScheme;
+use tokenizers::{PreTokenizerWrapper, Tokenizer};
 use tower_http::cors::AllowOrigin;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
@@ -255,6 +256,25 @@ async fn main() -> Result<()> {
     let mut tokenizer = Tokenizer::from_file(tokenizer_path).expect(
         "tokenizer.json not found. text-embeddings-inference only supports fast tokenizers",
     );
+    // See https://github.com/huggingface/tokenizers/pull/1357
+    if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() {
+        if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer {
+            // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer`
+            let mut m = m.clone();
+            m.set_prepend_scheme(PrependScheme::First);
+            tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Metaspace(m));
+        } else if let PreTokenizerWrapper::Sequence(s) = pre_tokenizer {
+            // We are forced to clone since `Tokenizer` does not have a `get_mut` for `pre_tokenizer`
+            let mut s = s.clone();
+            for pre_tokenizer in s.get_pre_tokenizers_mut() {
+                if let PreTokenizerWrapper::Metaspace(m) = pre_tokenizer {
+                    m.set_prepend_scheme(PrependScheme::First);
+                }
+            }
+            tokenizer.with_pre_tokenizer(PreTokenizerWrapper::Sequence(s));
+        }
+    }
+
     tokenizer.with_padding(None);
 
     // Position IDs offset. Used for Roberta and camembert.