Merge branch 'master' into b_whisper_unification

Author: as-suvorov

src/cpp/include/openvino/genai/cache_eviction.hpp

@@ -4,6 +4,8 @@
 #pragma once
 
 #include <cstddef>
+#include <unordered_map>
+#include <sstream>
 
 #include "openvino/core/except.hpp"
 
@@ -66,6 +68,26 @@ class KVCrushConfig {
     std::size_t get_budget() const {
         return budget;
     }
+
+    std::string to_string() const {
+        static const std::unordered_map<KVCrushAnchorPointMode, std::string> kv_crush_anchor_point_mode_to_string = {
+            {KVCrushAnchorPointMode::RANDOM, "RANDOM"},
+            {KVCrushAnchorPointMode::ZEROS, "ZEROS"},
+            {KVCrushAnchorPointMode::ONES, "ONES"},
+            {KVCrushAnchorPointMode::MEAN, "MEAN"},
+            {KVCrushAnchorPointMode::ALTERNATE, "ALTERNATE"},
+        };
+
+        std::ostringstream oss;
+        oss << "KVCrushConfig { " << "\n";
+        oss << "  budget: " << budget << "\n";
+        oss << "  rng_seed: " << rng_seed << "\n";
+        if (kv_crush_anchor_point_mode_to_string.count(anchor_point_mode) > 0) {
+            oss << "  anchor_point_mode: " << kv_crush_anchor_point_mode_to_string.at(anchor_point_mode) << "\n";
+        }
+        oss << " }";
+        return oss.str();
+    }
 };
 
 /**
@@ -122,6 +144,28 @@ class CacheEvictionConfig {
         return m_evictable_size;
     }
 
+    std::string to_string() const {
+        static const std::unordered_map<AggregationMode, std::string> aggregation_mode_to_string = {
+            {AggregationMode::SUM, "SUM"},
+            {AggregationMode::NORM_SUM, "NORM_SUM"},
+        };
+
+        std::ostringstream oss;
+        oss << "CacheEvictionConfig { " << "\n";
+        oss << "  start_size: " << m_start_size << "\n";
+        oss << "  recent_size: " << m_recent_size << "\n";
+        oss << "  max_cache_size: " << m_max_cache_size << "\n";
+        oss << "  evictable_size: " << m_evictable_size << "\n";
+        if (aggregation_mode_to_string.count(aggregation_mode) > 0) {
+            oss << "  aggregation_mode: " << aggregation_mode_to_string.at(aggregation_mode) << "\n";
+        }
+        oss << "  apply_rotation: " << std::boolalpha << apply_rotation << "\n";
+        oss << "  snapkv_window_size: " << snapkv_window_size << "\n";
+        oss << kvcrush_config.to_string() << "\n";
+        oss << " }";
+        return oss.str();
+    }
+
     /** The mode used to compute the importance of tokens for eviction */
     AggregationMode aggregation_mode = AggregationMode::NORM_SUM;
 

src/cpp/include/openvino/genai/continuous_batching_pipeline.hpp

@@ -175,6 +175,12 @@ class OPENVINO_GENAI_EXPORTS ContinuousBatchingPipeline {
     /// Higher level interface, which can process multiple prompts in continuous batching manner
     std::vector<EncodedGenerationResult> generate(const std::vector<ov::Tensor>& input_ids, const std::vector<ov::genai::GenerationConfig>& sampling_params, const ov::genai::StreamerVariant& streamer=std::monostate{});
     std::vector<GenerationResult> generate(const std::vector<std::string>& prompts, const std::vector<ov::genai::GenerationConfig>& sampling_params, const ov::genai::StreamerVariant& streamer=std::monostate{});
+    
+    std::vector<GenerationResult> generate(
+        const std::vector<ChatHistory>& histories,
+        const std::vector<ov::genai::GenerationConfig>& sampling_params,
+        const ov::genai::StreamerVariant& streamer=std::monostate{});
+
     std::vector<VLMDecodedResults> generate(
              const std::vector<std::string>& prompts,
              const std::vector<std::vector<ov::Tensor>>& images,

src/cpp/include/openvino/genai/llm_pipeline.hpp

@@ -233,6 +233,57 @@ class OPENVINO_GENAI_EXPORTS LLMPipeline {
         return generate(inputs, AnyMap{std::forward<Properties>(properties)...});
     }
 
+    /**
+    * @brief High level generate that receives ChatHistory and returns decoded output.
+    *
+    * @param history ChatHistory with messages
+    * @param generation_config optional GenerationConfig
+    * @param streamer optional streamer
+    * @return DecodedResults decoded resulting text
+    * 
+    * Chat template will be applied to the prompt, run `pipe.get_tokenizer().set_chat_template(custom_chat_template)` to update it.
+    * To disable chat template set `generation_config.apply_chat_template` to `false`.
+    */
+    DecodedResults generate(
+        const ChatHistory& history,
+        OptionalGenerationConfig generation_config = std::nullopt,
+        StreamerVariant streamer=std::monostate()
+    );
+
+    /**
+    * @brief High level generate that receives ChatHistory and returns decoded output.
+    * Properties can be in any order pipe.generate(..., ov::genai::max_new_tokens(100), ov::genai::streamer(lambda_func)).
+    *
+    * @param history ChatHistory with messages
+    * @param properties properties
+    * @return DecodedResults decoded resulting text
+    * 
+    * Chat template will be applied to the prompt, run `pipe.get_tokenizer().set_chat_template(custom_chat_template)` to update it.
+    * To disable chat template set `generation_config.apply_chat_template` to `false`.
+    */
+    template <typename... Properties>
+    util::EnableIfAllStringAny<DecodedResults, Properties...> generate(
+            const ChatHistory& history,
+            Properties&&... properties) {
+        return generate(history, AnyMap{std::forward<Properties>(properties)...});
+    }
+    DecodedResults generate(const ChatHistory& history, const ov::AnyMap& config_map);
+
+    DecodedResults operator()(
+        const ChatHistory& history,
+        OptionalGenerationConfig generation_config = std::nullopt,
+        StreamerVariant streamer=std::monostate()
+    ) {
+        return generate(history, generation_config, streamer);
+    }
+
+    template <typename... Properties>
+    util::EnableIfAllStringAny<DecodedResults, Properties...> operator()(
+            const ChatHistory& history,
+            Properties&&... properties) {
+        return generate(history, AnyMap{std::forward<Properties>(properties)...});
+    }
+
     /**
     * @brief Low level generate to be called with already encoded input_ids tokens.
     * Streamer cannot be used for multibatch inputs.

src/cpp/include/openvino/genai/scheduler_config.hpp

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <cstddef>
+#include <sstream>
 
 #include "openvino/genai/cache_eviction.hpp"
 #include "openvino/genai/sparse_attention.hpp"
@@ -73,5 +74,33 @@ struct SchedulerConfig {
                dynamic_split_fuse == other.dynamic_split_fuse && use_cache_eviction == other.use_cache_eviction &&
                max_num_seqs == other.max_num_seqs && enable_prefix_caching == other.enable_prefix_caching;
     }
+
+    /**
+     * Returns a human-readable string representation of the SchedulerConfig.
+     * The output is a multi-line string listing each configuration field and its value.
+     * This is useful for debugging, logging, or inspecting the current configuration.
+     *
+     * @return A string describing the current SchedulerConfig in a readable format.
+     */
+    std::string to_string() const {
+        std::ostringstream oss;
+        oss << "SchedulerConfig { \n";
+        oss << "  max_num_batched_tokens: " << max_num_batched_tokens << "\n";
+        oss << "  num_kv_blocks: " << num_kv_blocks << "\n";
+        oss << "  cache_size: " << cache_size << "\n";
+        oss << "  dynamic_split_fuse: " << std::boolalpha << dynamic_split_fuse << "\n";
+        oss << "  use_cache_eviction: " << std::boolalpha << use_cache_eviction << "\n";
+        if (use_cache_eviction) {
+            oss << cache_eviction_config.to_string() << "\n";
+        }
+        oss << "  max_num_seqs: " << max_num_seqs << "\n";
+        oss << "  enable_prefix_caching: " << std::boolalpha << enable_prefix_caching << "\n";
+        oss << "  use_sparse_attention: " << std::boolalpha << use_sparse_attention << "\n";
+        if (use_sparse_attention) {
+            oss << sparse_attention_config.to_string() << "\n";
+        }
+        oss << " }";
+        return oss.str();
+    }
 };
 }

src/cpp/include/openvino/genai/sparse_attention.hpp

@@ -4,6 +4,8 @@
 #pragma once
 
 #include <cstddef>
+#include <unordered_map>
+#include <sstream>
 
 namespace ov::genai {
 
@@ -79,6 +81,42 @@ class SparseAttentionConfig {
      *  M time to be calculated, then the importance score calculation would be taking `M / xattention_stride` time as
      *  overhead. */
     size_t xattention_stride = 8;
+
+    /**
+     * @brief Returns a string representation of the SparseAttentionConfig.
+     *
+     * The returned string contains the values of all configuration fields in a human-readable format, e.g.:
+     * SparseAttentionConfig {
+     *   sparseAttentionMode: TRISHAPE
+     *   num_last_dense_tokens_in_prefill: 100
+     *   num_retained_start_tokens_in_cache: 128
+     *   num_retained_recent_tokens_in_cache: 1920
+     *   xattention_threshold: 0.8
+     *   xattention_block_size: 64
+     *   xattention_stride: 8
+     * }
+     *
+     * @return A string describing the current configuration.
+     */
+    std::string to_string() const {
+        static const std::unordered_map<SparseAttentionMode, std::string> sparse_attention_mode_to_string = {
+            {SparseAttentionMode::TRISHAPE, "TRISHAPE"},
+            {SparseAttentionMode::XATTENTION, "XATTENTION"},
+        };
+        std::ostringstream oss;
+        oss << "SparseAttentionConfig { " << "\n";
+        if (sparse_attention_mode_to_string.count(mode) > 0) {
+            oss << "  sparseAttentionMode: " << sparse_attention_mode_to_string.at(mode) << "\n";
+        }
+        oss << "  num_last_dense_tokens_in_prefill: " << num_last_dense_tokens_in_prefill << "\n";
+        oss << "  num_retained_start_tokens_in_cache: " << num_retained_start_tokens_in_cache << "\n";
+        oss << "  num_retained_recent_tokens_in_cache: " << num_retained_recent_tokens_in_cache << "\n";
+        oss << "  xattention_threshold: " << xattention_threshold << "\n";
+        oss << "  xattention_block_size: " << xattention_block_size << "\n";
+        oss << "  xattention_stride: " << xattention_stride << "\n";
+        oss << " }";
+        return oss.str();
+    }
 };
 
 }  // namespace ov::genai

src/cpp/src/continuous_batching/pipeline.cpp

@@ -57,6 +57,8 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline( const std::filesystem::p
         embedder = std::make_shared<InputsEmbedder>(models_path, device, vision_encoder_properties);
     }
 
+    utils::print_scheduler_config_info(scheduler_config);
+
     if (is_prompt_lookup_enabled) {
         OPENVINO_ASSERT(draft_model_desr.model == nullptr, "Speculative decoding and prompt lookup decoding are mutually exclusive");
         OPENVINO_ASSERT(embedder == nullptr, "Prompt lookup decoding is not supported for models with embeddings");
@@ -97,6 +99,8 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(
         embedder = std::make_shared<InputsEmbedder>(models_path, device, properties_without_draft_model_without_gguf);
     }
 
+    utils::print_scheduler_config_info(scheduler_config);
+
     if (is_prompt_lookup_enabled) {
         OPENVINO_ASSERT(draft_model_desr.model == nullptr, "Speculative decoding and prompt lookup decoding are mutually exclusive");
         OPENVINO_ASSERT(embedder == nullptr, "Prompt lookup decoding is not supported for models with embeddings");
@@ -140,6 +144,8 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(
         }
     }
 
+    utils::print_scheduler_config_info(scheduler_config);
+
     if (is_prompt_lookup_enabled) {
         OPENVINO_ASSERT(draft_model_desr.model == nullptr, "Speculative decoding and prompt lookup decoding are mutually exclusive");
         OPENVINO_ASSERT(embedder == nullptr, "Prompt lookup decoding is not supported for models with embeddings");
@@ -188,6 +194,8 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(
         }
     }
 
+    utils::print_scheduler_config_info(scheduler_config);
+
     if (is_prompt_lookup_enabled) {
         OPENVINO_ASSERT(draft_model_desr.model == nullptr, "Speculative decoding and prompt lookup decoding are mutually exclusive");
         OPENVINO_ASSERT(embedder == nullptr, "Prompt lookup decoding is not supported for models with embeddings");
@@ -265,6 +273,21 @@ std::vector<GenerationResult> ContinuousBatchingPipeline::generate(const std::ve
     return decoded_results;
 }
 
+std::vector<GenerationResult> ContinuousBatchingPipeline::generate(
+    const std::vector<ChatHistory>& histories,
+    const std::vector<ov::genai::GenerationConfig>&
+    sampling_params,
+    const StreamerVariant& streamer
+) {
+    auto decoded_results = m_impl->generate(histories, sampling_params, streamer);
+
+    for (auto& decoded_result : decoded_results) {
+        decoded_result.perf_metrics.load_time = m_impl->m_load_time_ms;
+    }
+
+    return decoded_results;
+}
+
 std::vector<VLMDecodedResults> ContinuousBatchingPipeline::generate(
              const std::vector<std::string>& prompts,
              const std::vector<std::vector<ov::Tensor>>& images,

src/cpp/src/continuous_batching/pipeline_base.cpp

@@ -104,6 +104,7 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
         timer.end();
     }
 
+    // TODO Consider moving to method and reuse
     std::vector<EncodedGenerationResult> encoded = generate(input_ids, sampling_params, streamer);
 
     std::vector<GenerationResult> decoded;
@@ -149,6 +150,81 @@ ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
     return decoded;
 }
 
+std::vector<GenerationResult>
+ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
+    const std::vector<ChatHistory>& histories,
+    const std::vector<ov::genai::GenerationConfig>& sampling_params,
+    const StreamerVariant& streamer
+) {
+    // TODO Enable chat history input for embeddings models.
+    OPENVINO_ASSERT(m_model_input_type == ModelInputType::TOKENS, "Chat history input is not supported for embeddings models.");
+    
+    OPENVINO_ASSERT(histories.size() == sampling_params.size(), "Number of histories must match sampling params");
+    OPENVINO_ASSERT(!m_tokenizer.get_chat_template().empty(), "Chat template must not be empty when using ChatHistory in generate method.");
+    
+    auto start_time = std::chrono::steady_clock::now();
+
+    std::vector<ov::Tensor> input_ids;
+    input_ids.reserve(histories.size());
+
+    std::vector<MicroSeconds> tokenization_durations;
+    static ManualTimer timer("tokenize");
+    timer.start();
+
+    for (size_t i = 0; i < histories.size(); i++) {
+        OPENVINO_ASSERT(sampling_params[i].apply_chat_template, "Chat template must be applied when using ChatHistory in generate method.");
+        OPENVINO_ASSERT(!histories[i].empty(), "Chat history must not be empty when using ChatHistory in generate method.");
+        const auto encode_start = std::chrono::steady_clock::now();
+        constexpr bool add_generation_prompt = true;
+        std::string templated_history = m_tokenizer.apply_chat_template(histories[i], add_generation_prompt);
+        input_ids.push_back(
+            m_tokenizer.encode(templated_history, add_special_tokens(false)).input_ids
+        );
+        tokenization_durations.emplace_back(PerfMetrics::get_microsec(std::chrono::steady_clock::now() - encode_start));
+    }
+    
+    timer.end();
+
+    std::vector<EncodedGenerationResult> encoded_results = generate(input_ids, sampling_params, streamer);
+
+    std::vector<GenerationResult> decoded_results;
+    decoded_results.reserve(encoded_results.size());
+    for (size_t i = 0; i < encoded_results.size(); ++i) {
+        EncodedGenerationResult encoded_result = encoded_results[i];
+
+        auto& perf_metrics = encoded_result.perf_metrics;
+        auto& raw_counters = perf_metrics.raw_metrics;
+        raw_counters.tokenization_durations.emplace_back(tokenization_durations[i]);
+
+        std::vector<std::string> decoded_outputs;
+        decoded_outputs.reserve(encoded_result.m_generation_ids.size());
+        for (size_t idx = 0; idx < encoded_result.m_generation_ids.size(); ++idx) {
+            const auto decode_start = std::chrono::steady_clock::now();
+            decoded_outputs.push_back(m_tokenizer.decode(encoded_result.m_generation_ids.at(idx)));
+
+            raw_counters.detokenization_durations.emplace_back(std::chrono::steady_clock::now() - decode_start);
+        }
+
+        // The same perf metrics for each sequence, only tokenization/detokenization will differ.
+        perf_metrics.raw_metrics.generate_durations.clear();
+        perf_metrics.raw_metrics.generate_durations.emplace_back(PerfMetrics::get_microsec(std::chrono::steady_clock::now() - start_time));
+        // Reevaluate taking into accound tokenization/detokenization times.
+        perf_metrics.m_evaluated = false;
+        perf_metrics.evaluate_statistics(start_time);
+
+        decoded_results.push_back(GenerationResult{
+            encoded_result.m_request_id,
+            std::move(decoded_outputs),
+            std::move(encoded_result.m_scores),
+            encoded_result.m_status,
+            std::move(perf_metrics),
+            std::move(encoded_result.extended_perf_metrics)
+        });
+    }
+
+    return decoded_results;
+}
+
 std::vector<VLMDecodedResults>
 ContinuousBatchingPipeline::IContinuousBatchingPipeline::generate(
              const std::vector<std::string>& prompts,

src/cpp/src/continuous_batching/pipeline_base.hpp

@@ -148,6 +148,15 @@ class ContinuousBatchingPipeline::IContinuousBatchingPipeline {
                                                     const std::vector<GenerationConfig>& sampling_params,
                                                     const StreamerVariant& streamer);
 
+    
+    /**
+     * Performs monolitic generation based on ChatHistory objects
+     */
+    std::vector<GenerationResult>
+    generate(const std::vector<ChatHistory>& histories,
+             const std::vector<GenerationConfig>& sampling_params,
+             const StreamerVariant& streamer);
+
     /**
      * Starts chat with a given system prompt
      *