Adding tokenize endpoint for embeddings calculator (#3717)

### 🛠 Summary

[CVS-175207](https://jira.devtools.intel.com/browse/CVS-175207)
Adding /tokenize endpoint to embeddings calculator

### 🧪 Checklist

- [x] Unit tests added.
- [x] The documentation updated.
- [ ] Change follows security best practices.
``

Author: przepeck

demos/embeddings/README.md

@@ -503,10 +503,40 @@ Results will be stored in `results` folder:
   "kg_co2_emissions": null
 }
 ```
-
 Compare against local HuggingFace execution for reference:
 ```console
 mteb run -m thenlper/gte-small -t Banking77Classification --output_folder results
+``` 
+
+# Usage of tokenize endpoint (release 2025.4 or weekly)
+
+The `tokenize` endpoint provides a simple API for tokenizing input text using the same tokenizer as the deployed embeddings model. This allows you to see how your text will be split into tokens before feature extraction or inference. The endpoint accepts a string or list of strings and returns the corresponding token IDs and tokenized text.
+
+Example usage:
+```console
+curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"BAAI/bge-large-en-v1.5\", \"text\": \"hello world\" }"
+```
+Response:
+```json
+{
+  "tokens": [101,7592,2088,102]
+}
 ```
 
+It's possible to use additional parameters:
+ - pad_to_max_length - whether to pad the sequence to the maximum length. Default is False. 
+ - max_length - maximum length of the sequence. If None (default), the value will be taken from the IR (where default value from original HF/GGUF model is stored).
+ - padding_side - side to pad the sequence, can be ‘left’ or ‘right’. Default is None.
+ - add_special_tokens - whether to add special tokens like BOS, EOS, PAD. Default is True. 
+
+ Example usage:
+```console
+curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"BAAI/bge-large-en-v1.5\", \"text\": \"hello world\", \"max_length\": 10, \"pad_to_max_length\": true, \"padding_side\": \"left\", \"add_special_tokens\": true }"
+```
 
+Response:
+```json
+{
+  "tokens":[0,0,0,0,0,0,101,7592,2088,102]
+}
+```

src/BUILD

@@ -2662,6 +2662,7 @@ cc_test(
             ":test_platform_utils",
             "//src/rerank:rerank_api_handler",
             ":embeddings_handler_tests",
+            ":tokenize_parser_tests",
             ":inferencerequest_test",
             ":libtest_environment",
             ":libtest_gpuenvironment",
@@ -3110,6 +3111,21 @@ cc_library(
     linkopts = [],
 )
 
+cc_library(
+    name = "tokenize_parser_tests",
+    linkstatic = 1,
+    alwayslink = True,
+    srcs = ["test/tokenize_parser_test.cpp"],
+    data = [],
+    deps = [
+        "//src/tokenize:tokenize_parser",
+        "@com_google_googletest//:gtest",
+    ],
+    copts = COPTS_TESTS,
+    linkopts = [],
+)
+
+
 cc_library(
     name = "test_llm_output_parser_tests",
     linkstatic = 1,

src/embeddings/BUILD

@@ -22,6 +22,7 @@ ovms_cc_library(
     hdrs = ["embeddings_api.hpp"],
     srcs = ["embeddings_api.cpp"],
     deps = ["//src:libovmslogging",
+            "//src/tokenize:tokenize_parser",
             "@mediapipe//mediapipe/framework:calculator_framework",
             "//third_party:openvino",
             "@com_github_tencent_rapidjson//:rapidjson",],

src/embeddings/embeddings_api.cpp

@@ -45,67 +45,26 @@ using namespace rapidjson;
 namespace ovms {
 
 std::variant<EmbeddingsRequest, std::string> EmbeddingsRequest::fromJson(rapidjson::Document* parsedJson) {
-    enum class InputType {
-        NONE,
-        STRING,
-        INT,
-        INT_VEC
-    };
     EmbeddingsRequest request;
-    std::vector<std::string> input_strings;
-    std::vector<std::vector<int64_t>> input_tokens;
-
     if (!parsedJson->IsObject())
         return "Received json is not an object";
 
-    auto it = parsedJson->FindMember("input");
-    if (it != parsedJson->MemberEnd()) {
-        if (it->value.IsString()) {
-            input_strings.push_back(it->value.GetString());
-        } else if (it->value.IsArray()) {
-            if (it->value.GetArray().Size() == 0) {
-                return "input array should not be empty";
-            }
-            InputType input_type = InputType::NONE;
-            for (auto& input : it->value.GetArray()) {
-                if (input.IsArray()) {
-                    if (input_type != InputType::NONE && input_type != InputType::INT_VEC)
-                        return "input must be homogeneous";
-                    input_type = InputType::INT_VEC;
-                    std::vector<int64_t> ints;
-                    ints.reserve(input.GetArray().Size());
-                    for (auto& val : input.GetArray()) {
-                        if (val.IsInt())
-                            ints.push_back(val.GetInt());
-                        else
-                            return "input must be homogeneous";
-                    }
-                    input_tokens.emplace_back(std::move(ints));
-                } else if (input.IsString()) {
-                    if (input_type != InputType::NONE && input_type != InputType::STRING)
-                        return "input must be homogeneous";
-                    input_type = InputType::STRING;
-                    input_strings.push_back(input.GetString());
-                } else if (input.IsInt()) {
-                    if (input_type != InputType::NONE && input_type != InputType::INT)
-                        return "input must be homogeneous";
-                    input_type = InputType::INT;
-                    if (input_tokens.size() == 0) {
-                        input_tokens.push_back(std::vector<int64_t>());
-                    }
-                    input_tokens[0].push_back(input.GetInt());
-                } else {
-                    return "every element in input array should be either string or int";
-                }
-            }
+    auto parsedInput = TokenizeParser::parseInput(*parsedJson, "input");
+
+    if (std::holds_alternative<std::string>(parsedInput)) {
+        return std::get<std::string>(parsedInput);
+    } else {
+        auto inputVariant = std::get<EmbeddingsRequest::InputDataType>(parsedInput);
+        if (std::holds_alternative<std::vector<std::string>>(inputVariant)) {
+            request.input = std::get<std::vector<std::string>>(inputVariant);
+        } else if (std::holds_alternative<std::vector<std::vector<int64_t>>>(inputVariant)) {
+            request.input = std::get<std::vector<std::vector<int64_t>>>(inputVariant);
         } else {
-            return "input should be string, array of strings or array of integers";
+            return "input must be either array of strings or array of array of integers";
         }
-    } else {
-        return "input field is required";
     }
 
-    it = parsedJson->FindMember("encoding_format");
+    auto it = parsedJson->FindMember("encoding_format");
     request.encoding_format = EncodingFormat::FLOAT;
     if (it != parsedJson->MemberEnd()) {
         if (it->value.IsString()) {
@@ -123,13 +82,6 @@ std::variant<EmbeddingsRequest, std::string> EmbeddingsRequest::fromJson(rapidjs
 
     // TODO: dimensions (optional)
     // TODO: user (optional)
-    if (input_strings.size() > 0) {
-        request.input = input_strings;
-    } else if (input_tokens.size() > 0) {
-        request.input = input_tokens;
-    } else {
-        return "input field is required";
-    }
     return request;
 }
 
@@ -149,12 +101,15 @@ absl::Status EmbeddingsHandler::parseRequest() {
     return absl::OkStatus();
 }
 
-std::variant<std::vector<std::string>, std::vector<std::vector<int64_t>>>& EmbeddingsHandler::getInput() {
+TokenizeRequest::InputDataType& EmbeddingsHandler::getInput() {
     return request.input;
 }
 EmbeddingsRequest::EncodingFormat EmbeddingsHandler::getEncodingFormat() const {
     return request.encoding_format;
 }
+ov::AnyMap& EmbeddingsHandler::getParameters() {
+    return request.parameters;
+}
 
 void EmbeddingsHandler::setPromptTokensUsage(int promptTokens) {
     this->promptTokens = promptTokens;

src/embeddings/embeddings_api.hpp

@@ -34,19 +34,20 @@
 #include <rapidjson/stringbuffer.h>
 #pragma warning(pop)
 
+#include "../tokenize/tokenize_parser.hpp"
+
 namespace ovms {
 
 enum class PoolingMode {
     CLS,
     LAST
 };
 
-struct EmbeddingsRequest {
+struct EmbeddingsRequest : TokenizeRequest {
     enum class EncodingFormat {
         FLOAT,
         BASE64
     };
-    std::variant<std::vector<std::string>, std::vector<std::vector<int64_t>>> input;
     EncodingFormat encoding_format;
 
     static std::variant<EmbeddingsRequest, std::string> fromJson(rapidjson::Document* request);
@@ -61,8 +62,9 @@ class EmbeddingsHandler {
     EmbeddingsHandler(rapidjson::Document& document) :
         doc(document) {}
 
-    std::variant<std::vector<std::string>, std::vector<std::vector<int64_t>>>& getInput();
+    TokenizeRequest::InputDataType& getInput();
     EmbeddingsRequest::EncodingFormat getEncodingFormat() const;
+    ov::AnyMap& getParameters();
 
     absl::Status parseRequest();
 

src/embeddings/embeddings_calculator_ov.cc

@@ -49,6 +49,7 @@ class EmbeddingsServable;
 namespace mediapipe {
 
 const std::string EMBEDDINGS_SESSION_SIDE_PACKET_TAG = "EMBEDDINGS_NODE_RESOURCES";
+const std::string EMBEDDINGS_TOKENIZE_ENDPOINT_SUFFIX = "tokenize";
 
 using InputDataType = ovms::HttpPayload;
 using OutputDataType = std::string;
@@ -62,6 +63,13 @@ class EmbeddingsCalculatorOV : public CalculatorBase {
 
     mediapipe::Timestamp timestamp{0};
 
+    absl::Status tokenizeStrings(ov::genai::Tokenizer& tokenizer, const std::vector<std::string>& inputStrings, const ov::AnyMap& parameters, ov::genai::TokenizedInputs& tokens, const size_t& max_context_length) {
+        tokens = tokenizer.encode(inputStrings, parameters);
+        RET_CHECK(tokens.input_ids.get_shape().size() == 2);
+
+        return absl::OkStatus();
+    }
+
 protected:
     std::shared_ptr<ovms::EmbeddingsServable> embeddings_session{nullptr};
 
@@ -104,41 +112,77 @@ class EmbeddingsCalculatorOV : public CalculatorBase {
         InputDataType payload = cc->Inputs().Tag(INPUT_TAG_NAME).Get<InputDataType>();
         SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Request body: {}", payload.body);
         SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Request uri: {}", payload.uri);
-        ovms::EmbeddingsHandler handler(*payload.parsedJson);
-        auto parseRequestStartTime = std::chrono::high_resolution_clock::now();
-        absl::Status status = handler.parseRequest();
-        if (!status.ok()) {
-            return status;
-        }
-        double time = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - parseRequestStartTime).count();
-        SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Embeddings request deserialization time: {} ms", time / 1000);
 
         ov::Tensor embeddingsTensor;
         size_t received_batch_size = 1;
         size_t max_context_length = 1024;  // default allowed input length. Otherwise, it will be read from model config.json file
-        ModelMetricReporter unused(nullptr, nullptr, "unused", 1);
         ov::genai::TokenizedInputs tokens;
         ov::Tensor typeIds;
         if (embeddings_session->getMaxModelLength().has_value()) {
             max_context_length = embeddings_session->getMaxModelLength().value();
         } else {
             SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "max_position_embeddings nor max_trained_positions included in config.json. Using default value {}", max_context_length);
         }
+        const int endpoint_len = EMBEDDINGS_TOKENIZE_ENDPOINT_SUFFIX.size();
+        const bool useTokenizeEndpoint = payload.uri.size() >= endpoint_len &&
+                                         payload.uri.compare(payload.uri.size() - endpoint_len, endpoint_len, EMBEDDINGS_TOKENIZE_ENDPOINT_SUFFIX) == 0;
+        if (useTokenizeEndpoint) {
+            ovms::TokenizeRequest tokenizeRequest;
+            absl::Status parsingStatus = ovms::TokenizeParser::parseTokenizeRequest(*payload.parsedJson, tokenizeRequest);
+            if (!parsingStatus.ok()) {
+                return parsingStatus;
+            }
+            auto input = tokenizeRequest.input;
+            if (auto strings = std::get_if<std::vector<std::string>>(&input)) {
+                auto tokenizationStatus = this->tokenizeStrings(embeddings_session->getTokenizer(), *strings, tokenizeRequest.parameters, tokens, max_context_length);
+                if (!tokenizationStatus.ok()) {
+                    return tokenizationStatus;
+                }
+            } else {
+                SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Embeddings tokenize input is of not supported type");
+                return absl::InvalidArgumentError("Input should be string or array of strings");
+            }
+
+            StringBuffer responseBuffer;
+            auto responseStatus = ovms::TokenizeParser::parseTokenizeResponse(responseBuffer, tokens, tokenizeRequest.parameters);
+            if (!responseStatus.ok()) {
+                return responseStatus;
+            }
+            cc->Outputs().Tag(OUTPUT_TAG_NAME).Add(new std::string(responseBuffer.GetString()), timestamp);
+            return absl::OkStatus();
+        }
+        ovms::EmbeddingsHandler handler(*payload.parsedJson);
+        auto parseRequestStartTime = std::chrono::high_resolution_clock::now();
+        absl::Status status = handler.parseRequest();
+
+        if (!status.ok()) {
+            return status;
+        }
+        double time = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - parseRequestStartTime).count();
+        SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Embeddings request deserialization time: {} ms", time / 1000);
+
+        ModelMetricReporter unused(nullptr, nullptr, "unused", 1);
+
         try {
             auto input = handler.getInput();
             if (auto strings = std::get_if<std::vector<std::string>>(&input)) {
+                ov::AnyMap& params = handler.getParameters();
                 received_batch_size = strings->size();
-                ov::AnyMap params = {};
-                if (cc->Options<EmbeddingsCalculatorOVOptions>().truncate()) {
-                    params = {{"max_length", max_context_length}};
+                if (cc->Options<EmbeddingsCalculatorOVOptions>().truncate() && params.find("max_length") == params.end()) {
+                    params["max_length"] = max_context_length;
                 }
-                tokens = embeddings_session->getTokenizer().encode(*strings, params);
-                RET_CHECK(tokens.input_ids.get_shape().size() == 2);
+
+                absl::Status tokenizationStatus = this->tokenizeStrings(embeddings_session->getTokenizer(), *strings, params, tokens, max_context_length);
+                if (!tokenizationStatus.ok()) {
+                    return tokenizationStatus;
+                }
+
                 size_t input_ids_size = tokens.input_ids.get_shape()[1];
                 if (input_ids_size > max_context_length) {
                     SPDLOG_LOGGER_DEBUG(embeddings_calculator_logger, "Input size {} exceeds max_context_length {}", input_ids_size, max_context_length);
-                    return absl::InvalidArgumentError(absl::StrCat("Input length ", input_ids_size, " longer than allowed ", max_context_length));
+                    return absl::InvalidArgumentError("Input length " + std::to_string(input_ids_size) + " longer than allowed " + std::to_string(max_context_length));
                 }
+
                 if (embeddings_session->getNumberOfModelInputs() == 3) {
                     typeIds = ov::Tensor{ov::element::i64, tokens.input_ids.get_shape()};
                     std::fill_n(typeIds.data<int64_t>(), tokens.input_ids.get_size(), 0);

src/test/embeddings_handler_test.cpp

@@ -252,7 +252,7 @@ TEST(EmbeddingsDeserialization, invalidEncoding) {
         {
             "model": "embeddings",
             "input": ["one", "three"],
-"encoding_format": "dummy"
+            "encoding_format": "dummy"
         }
     )";
     rapidjson::Document d;
@@ -269,7 +269,7 @@ TEST(EmbeddingsDeserialization, invalidEncodingType) {
         {
             "model": "embeddings",
             "input": ["one", "three"],
-"encoding_format": 42
+            "encoding_format": 42
         }
     )";
     rapidjson::Document d;
@@ -340,7 +340,7 @@ TEST(EmbeddingsDeserialization, multipleStringInputFloat) {
         {
             "model": "embeddings",
             "input": ["one", "two", "three"],
-"encoding_format": "float"
+            "encoding_format": "float"
         }
     )";
     rapidjson::Document d;