eagle3 cb impl with top-1 proposal

samples/cpp/text_generation/CMakeLists.txt

@@ -29,7 +29,8 @@ set (SAMPLE_LIST
     lora_greedy_causal_lm
     multinomial_causal_lm
     prompt_lookup_decoding_lm
-    speculative_decoding_lm)
+    speculative_decoding_lm
+    eagle_speculative_lm)
 
 foreach(sample IN LISTS SAMPLE_LIST)
     add_sample_executable(${sample})

samples/cpp/text_generation/eagle_speculative_lm.cpp

@@ -0,0 +1,114 @@
+// Copyright (C) 2023-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+#include <chrono>
+#include <openvino/openvino.hpp>
+
+#include "openvino/genai/llm_pipeline.hpp"
+#include "openvino/genai/speculative_decoding/perf_metrics.hpp"
+
+template <typename T>
+void print_perf_metrics(T& perf_metrics, std::string model_name) {
+    std::cout << "\n" << model_name << std::endl;
+    auto generation_duration = perf_metrics.get_generate_duration().mean;
+    std::cout << "  Generate time: " << generation_duration << " ms" << std::endl;
+    std::cout << "  TTFT: " << perf_metrics.get_ttft().mean << " ± " << perf_metrics.get_ttft().std << " ms"
+              << std::endl;
+    std::cout << "  TPOT: " << perf_metrics.get_tpot().mean << " ± " << perf_metrics.get_tpot().std << " ms/token"
+              << std::endl;
+    std::cout << "  Num generated token: " << perf_metrics.get_num_generated_tokens() << " tokens" << std::endl;
+    if (model_name == "Total") {
+        std::cout << "  Total iteration number: " << perf_metrics.raw_metrics.m_new_token_times.size() << std::endl;
+    } else {
+        std::cout << "  Total iteration number: " << perf_metrics.raw_metrics.m_durations.size() << std::endl;
+    }
+    if (perf_metrics.get_num_input_tokens() > 0) {
+        std::cout << "  Input token size: " << perf_metrics.get_num_input_tokens() << std::endl;
+    }
+}
+
+int main(int argc, char* argv[]) try {
+    if (4 != argc) {
+        throw std::runtime_error(std::string{"Usage: "} + argv[0] + " <MODEL_DIR> <EAGLE_MODEL_DIR> '<PROMPT>'");
+    }
+
+    std::string main_model_path = argv[1];
+    std::string eagle_model_path = argv[2];
+    std::string prompt = argv[3];
+
+    // Configure devices - can run main and eagle models on different devices
+    std::string main_device = "GPU", eagle_device = "GPU"; // CPU can bse used as well
+
+    // Eagle Speculative settings
+    ov::genai::GenerationConfig config = ov::genai::greedy();
+    config.max_new_tokens = 100;
+    config.num_assistant_tokens = 5;
+
+    ov::genai::SchedulerConfig scheduler_config;
+    scheduler_config.dynamic_split_fuse = false; // Eagle speculative decoding does not support dynamic_split_fuse mode
+    // Create pipeline with eagle speculative enabled
+    ov::genai::LLMPipeline pipe(
+        main_model_path,
+        main_device,
+        ov::genai::draft_model(eagle_model_path, eagle_device),
+        ov::genai::scheduler_config(scheduler_config),
+        ov::genai::eagle3_mode(true)
+    );
+    // Setup performance measurement
+    auto start_time = std::chrono::high_resolution_clock::now();
+
+    // Optional: Create a streaming callback for real-time token display
+    auto streamer = [](std::string subword) {
+        std::cout << subword << std::flush;
+        return ov::genai::StreamingStatus::RUNNING;
+    };
+
+    // Run generation with eagle speculative decoding
+    std::cout << "Generating with Eagle Speculative decoding:" << std::endl;
+    auto result = pipe.generate(prompt, config, streamer);
+    std::cout << std::endl;
+
+    // Calculate and display performance metrics
+    auto end_time = std::chrono::high_resolution_clock::now();
+    auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
+    std::cout << "\nGeneration completed in " << duration.count() << " ms" << std::endl;
+
+    auto sd_perf_metrics = std::dynamic_pointer_cast<ov::genai::SDPerModelsPerfMetrics>(result.extended_perf_metrics);
+    if (sd_perf_metrics) {
+        print_perf_metrics(result.perf_metrics, "Total");
+        print_perf_metrics(sd_perf_metrics->main_model_metrics, "MAIN MODEL");
+        std::cout << "  accepted token: " << sd_perf_metrics->get_num_accepted_tokens() << " tokens" << std::endl;
+        std::cout << "  compress rate: "
+                  << sd_perf_metrics->main_model_metrics.get_num_generated_tokens() * 1.0f /
+                         sd_perf_metrics->main_model_metrics.raw_metrics.m_durations.size()
+                  << std::endl;
+        print_perf_metrics(sd_perf_metrics->draft_model_metrics, "DRAFT MODEL");
+    }
+    std::cout << std::endl;
+
+    // Run without Eagle for comparison
+    std::cout << "\n-----------------------------" << std::endl;
+    std::cout << "Generating without Eagle Speculative decoding:" << std::endl;
+
+    // Disable Eagle mode
+    /*config.eagle_model = false;
+
+    start_time = std::chrono::high_resolution_clock::now();
+    pipe.generate(prompt, config, streamer);
+    std::cout << std::endl;
+    */
+    end_time = std::chrono::high_resolution_clock::now();
+    duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time);
+    std::cout << "\nStandard generation completed in " << duration.count() << " ms" << std::endl;
+
+} catch (const std::exception& error) {
+    try {
+        std::cerr << error.what() << '\n';
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+} catch (...) {
+    try {
+        std::cerr << "Non-exception object thrown\n";
+    } catch (const std::ios_base::failure&) {}
+    return EXIT_FAILURE;
+}

samples/cpp/text_generation/greedy_causal_lm.cpp

@@ -9,7 +9,7 @@ int main(int argc, char* argv[]) try {
 
     std::string models_path = argv[1];
     std::string prompt = argv[2];
-    std::string device = "CPU";  // GPU can be used as well
+    std::string device = "GPU";  // GPU can be used as well
 
     ov::genai::LLMPipeline pipe(models_path, device);
     ov::genai::GenerationConfig config;

samples/python/text_generation/eagle_speculative_lm.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import openvino_genai
+import queue
+
+def streamer(subword):
+    print(subword, end='', flush=True)
+    # Return flag corresponds whether generation should be stopped. 
+    return openvino_genai.StreamingStatus.RUNNING
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('model_dir')
+    parser.add_argument('draft_model_dir')
+    parser.add_argument('prompt')
+    args = parser.parse_args()
+
+    # User can run main and draft model on different devices.
+    # Please, set device for main model in `openvino_genai.LLMPipeline` constructor and in openvino_genai.draft_model` for draft.
+    main_device = 'GPU'  # CPU can be used as well
+    draft_device = 'GPU'
+    scheduler_config = openvino_genai.SchedulerConfig()
+    scheduler_config.dynamic_split_fuse = False  # Eagle speculative decoding does not support dynamic_split_fuse mode
+    draft_model = openvino_genai.draft_model(args.draft_model_dir, draft_device)
+
+    pipe = openvino_genai.LLMPipeline(args.model_dir, main_device, scheduler_config = scheduler_config, draft_model=draft_model, eagle3_mode = True)
+
+    config = openvino_genai.GenerationConfig()
+    config.max_new_tokens = 100
+    # Speculative decoding generation parameters like `num_assistant_tokens` and `assistant_confidence_threshold` are mutually excluded
+    # add parameter to enable speculative decoding to generate `num_assistant_tokens` candidates by draft_model per iteration
+    config.num_assistant_tokens = 5
+    # add parameter to enable speculative decoding to generate candidates by draft_model while candidate probability is higher than `assistant_confidence_threshold`
+    # config.assistant_confidence_threshold = 0.4
+
+    # Since the streamer is set, the results will be printed 
+    # every time a new token is generated and put into the streamer queue.
+    res = pipe.generate([args.prompt], config, streamer)
+    print()
+    if (res.extended_perf_metrics):
+        main_model_metrics = res.extended_perf_metrics.main_model_metrics
+        print(f"MAIN MODEL")
+        print(f"  Generate time: {main_model_metrics.get_generate_duration().mean:.2f} ms" )
+        print(f"  TTFT: {main_model_metrics.get_ttft().mean:.2f} ± {main_model_metrics.get_ttft().std:.2f} ms" )
+        print(f"  TTST: {main_model_metrics.get_ttst().mean:.2f} ± {main_model_metrics.get_ttst().std:.2f} ms/token")
+        print(f"  TPOT: {main_model_metrics.get_tpot().mean:.2f} ± {main_model_metrics.get_tpot().std:.2f} ms/iteration")
+        print(f"  AVG Latency: {main_model_metrics.get_latency().mean:.2f} ± {main_model_metrics.get_latency().std:.2f} ms/token")
+        print(f"  Num generated token: {main_model_metrics.get_num_generated_tokens()} tokens")
+        print(f"  Total iteration number: {len(main_model_metrics.raw_metrics.m_durations)}")
+        print(f"  Num accepted token: {res.extended_perf_metrics.get_num_accepted_tokens()} tokens")
+
+        draft_model_metrics = res.extended_perf_metrics.draft_model_metrics
+        print(f"DRAFT MODEL" )
+        print(f"  Generate time: {draft_model_metrics.get_generate_duration().mean:.2f} ms" )
+        print(f"  TTFT: {draft_model_metrics.get_ttft().mean:.2f} ms")
+        print(f"  TTST: {draft_model_metrics.get_ttst().mean:.2f} ms/token")
+        print(f"  TPOT: {draft_model_metrics.get_tpot().mean:.2f} ± {draft_model_metrics.get_tpot().std:.2f} ms/token")
+        print(f"  AVG Latency: {draft_model_metrics.get_latency().mean:.2f} ± {draft_model_metrics.get_latency().std:.2f} ms/iteration")
+        print(f"  Num generated token: {draft_model_metrics.get_num_generated_tokens()} tokens")
+        print(f"  Total iteration number: {len(draft_model_metrics.raw_metrics.m_durations)}")
+        print()
+
+if '__main__' == __name__:
+    main()

src/cpp/include/openvino/genai/continuous_batching_pipeline.hpp

@@ -65,13 +65,18 @@ class OPENVINO_GENAI_EXPORTS ContinuousBatchingPipeline {
     class ContinuousBatchingImpl;
 
     class ContinuousBatchingForSpeculativeDecodingImpl;
+    class ContinuousBatchingForEagleDecodingImpl;
     class ContinuousBatchingForPromptLookupImpl;
     class SpeculativeDecodingImpl;
+    class EagleDecodingImpl;
     class PromptLookupImpl;
 
     friend class ContinuousBatchingForSpeculativeDecodingImpl;
+
     friend class ContinuousBatchingForPromptLookupImpl;
+    friend class ContinuousBatchingForEagleDecodingImpl;
     friend class SpeculativeDecodingImpl;
+    friend class EagleDecodingImpl;
     friend class PromptLookupImpl;
 
     std::shared_ptr<IContinuousBatchingPipeline> m_impl;

src/cpp/include/openvino/genai/llm_pipeline.hpp

@@ -345,6 +345,12 @@ static constexpr ov::Property<bool> prompt_lookup{"prompt_lookup"};
 */
 static constexpr ov::Property<bool> enable_save_ov_model{"enable_save_ov_model"};
 
+/**
+* @brief enable eagle3_mode property serves to activate eagle3 speculative decoding.
+* Set `true` to activate this mode.
+* And create LLMPipeline instance with this config.
+*/
+static constexpr ov::Property<bool> eagle3_mode{"eagle3_mode"};
 
 }  // namespace genai
 }  // namespace ov

src/cpp/include/openvino/genai/perf_metrics.hpp

@@ -134,7 +134,7 @@ struct OPENVINO_GENAI_EXPORTS PerfMetrics {
     MeanStdPair detokenization_duration = {-1.0f, -1.0f};
 
     size_t num_generated_tokens;
-    size_t num_input_tokens;
+    size_t num_input_tokens = 0;
 
     float get_load_time();         // Load time in ms.
     size_t get_num_generated_tokens();