Fixed: The way tokens were counted with LlamaCpp wasn't accurate in chat mode.

Fixed: Chat processing would taken exponentially more time as you go past 32K max tokens in LlamaCpp+Chat mode. This is now fixed and processing is now snappy.

Author: SerialKicked

API/LlamaCppClient.cs

@@ -167,6 +167,22 @@ public TokenCountResponse GetTokenCountSync(MessageListQuery body)
             return Task.Run(() => GetTokenCountAsync(body)).ConfigureAwait(false).GetAwaiter().GetResult();
         }
 
+        /// <summary>
+        /// Applies the model's chat template to a list of chat messages via POST /apply-template, returning
+        /// the fully-formatted prompt string exactly as it would be built for /v1/chat/completions (names,
+        /// roles, template scaffolding included). This does not run inference.
+        /// </summary>
+        public async Task<ApplyTemplateResponse> ApplyTemplateAsync(ApplyTemplateQuery body, CancellationToken cancellationToken = default)
+        {
+            return await SendRequestAsync<ApplyTemplateResponse>(_httpClient!, HttpMethod.Post, "/apply-template", body, cancellationToken: cancellationToken).ConfigureAwait(false);
+        }
+
+        public ApplyTemplateResponse ApplyTemplateSync(ApplyTemplateQuery body)
+        {
+            // Using a new task and ConfigureAwait(false) to avoid deadlocks
+            return Task.Run(() => ApplyTemplateAsync(body)).ConfigureAwait(false).GetAwaiter().GetResult();
+        }
+
         public async Task<LlamaServerState> GetServerStateAsync(CancellationToken cancellationToken = default)
         {
             return await SendRequestAsync<LlamaServerState>(_httpClient!, HttpMethod.Get, "/props", cancellationToken: cancellationToken).ConfigureAwait(false);
@@ -388,6 +404,68 @@ public class MessageListQuery
         public List<MessageQuery> messages { get; set; } = [];
     }
 
+    /// <summary>
+    /// Request body for POST /apply-template. The messages must be in the same shape used by
+    /// /v1/chat/completions so the server applies the identical chat template used at generation time.
+    /// This is a plain, Newtonsoft-serializable structure on purpose: the OpenAI.Chat.* types use
+    /// System.Text.Json (JsonNode) internals that Newtonsoft cannot serialize.
+    /// </summary>
+    public class ApplyTemplateQuery
+    {
+        [JsonProperty("messages")]
+        public List<ApplyTemplateMessage> messages { get; set; } = [];
+    }
+
+    /// <summary>
+    /// A single chat message for /apply-template, mirroring the /v1/chat/completions message schema.
+    /// Null members are dropped so the payload matches what the OpenAI-compatible endpoint expects.
+    /// </summary>
+    public class ApplyTemplateMessage
+    {
+        [JsonProperty("role")]
+        public string role { get; set; } = "user";
+
+        [JsonProperty("content", NullValueHandling = NullValueHandling.Ignore)]
+        public string? content { get; set; }
+
+        [JsonProperty("tool_calls", NullValueHandling = NullValueHandling.Ignore)]
+        public List<ApplyTemplateToolCall>? tool_calls { get; set; }
+
+        [JsonProperty("tool_call_id", NullValueHandling = NullValueHandling.Ignore)]
+        public string? tool_call_id { get; set; }
+    }
+
+    public class ApplyTemplateToolCall
+    {
+        [JsonProperty("id")]
+        public string id { get; set; } = string.Empty;
+
+        [JsonProperty("type")]
+        public string type { get; set; } = "function";
+
+        [JsonProperty("function")]
+        public ApplyTemplateFunction function { get; set; } = new();
+    }
+
+    public class ApplyTemplateFunction
+    {
+        [JsonProperty("name")]
+        public string name { get; set; } = string.Empty;
+
+        // Per the OpenAI spec, function arguments are a JSON *string*, not a nested object.
+        [JsonProperty("arguments")]
+        public string arguments { get; set; } = string.Empty;
+    }
+
+    /// <summary>
+    /// Response body for POST /apply-template. Contains the fully-formatted prompt string.
+    /// </summary>
+    public class ApplyTemplateResponse
+    {
+        [JsonProperty("prompt")]
+        public string prompt { get; set; } = string.Empty;
+    }
+
     public class MessageQuery(string role, string content)
     {
         public string role { get; set; } = role;

Adapters/LlamaCppAdapter.cs

@@ -242,6 +242,47 @@ public void Dispose()
 
         public int CountMessageTokens(List<SingleMessage> messages)
         {
+            // In chat mode the authoritative count must match what /v1/chat/completions actually builds:
+            // names, macro expansion and the model's chat-template scaffolding all affect the real token
+            // count. Counting the raw message text (as the legacy /v1/messages/count_tokens path did)
+            // systematically under-counts, and the gap grows with the number of messages. To get an exact
+            // count we render the messages through the model's chat template via /apply-template (using the
+            // same content generation produces: name prefixes, macros and tool-call structure), then
+            // tokenize that string with BOS.
+            if (CompletionType == CompletionType.Chat)
+            {
+                try
+                {
+                    var chatMessages = new List<ApplyTemplateMessage>();
+                    foreach (var message in messages)
+                    {
+                        var built = BuildTemplateMessage(message);
+                        if (built is not null)
+                            chatMessages.Add(built);
+                    }
+                    if (chatMessages.Count == 0)
+                        return 0;
+
+                    var templated = _client.ApplyTemplateSync(new ApplyTemplateQuery { messages = chatMessages });
+                    if (!string.IsNullOrEmpty(templated?.prompt))
+                    {
+                        var tokens = _client.TokenizeSync(new TokenRequest
+                        {
+                            content = templated.prompt,
+                            add_special = true,
+                            parse_special = true
+                        });
+                        return tokens.GetTokenCount();
+                    }
+                    LLMEngine.Logger?.LogWarning("[LlamaCpp] /apply-template returned an empty prompt; falling back to legacy token count.");
+                }
+                catch (Exception ex)
+                {
+                    // Older servers may not expose /apply-template. Fall back to the legacy path below.
+                    LLMEngine.Logger?.LogWarning(ex, "[LlamaCpp] /apply-template unavailable; falling back to legacy token count.");
+                }
+            }
+
             var request = new MessageListQuery();
             foreach (var message in messages)
             {
@@ -268,6 +309,71 @@ public int CountMessageTokens(List<SingleMessage> messages)
             return token.input_tokens;
         }
 
+        /// <summary>
+        /// Builds a Newtonsoft-serializable /apply-template message from a <see cref="SingleMessage"/>,
+        /// mirroring the three cases in <see cref="SingleMessage.ToChatCompletion"/> (tool result,
+        /// assistant tool-call-only, and normal text). Tool-call arguments are kept as the raw JSON
+        /// *string* (never a System.Text.Json JsonNode) so serialization cannot self-reference.
+        /// Returns null for roles that are not sent to the template.
+        /// </summary>
+        private static ApplyTemplateMessage? BuildTemplateMessage(SingleMessage message)
+        {
+            // Tool result messages
+            if (message.Role == AuthorRole.Tool && message.ToolCalls.Count > 0)
+            {
+                return new ApplyTemplateMessage
+                {
+                    role = "tool",
+                    content = message.Message,
+                    tool_call_id = message.ToolCalls[0].CallId
+                };
+            }
+
+            // Assistant tool-call-only messages (no text content)
+            if (message.Role == AuthorRole.Assistant && message.ToolCalls.Count > 0 && string.IsNullOrEmpty(message.Message))
+            {
+                var calls = new List<ApplyTemplateToolCall>();
+                foreach (var record in message.ToolCalls)
+                {
+                    calls.Add(new ApplyTemplateToolCall
+                    {
+                        id = record.CallId,
+                        type = "function",
+                        function = new ApplyTemplateFunction
+                        {
+                            name = record.FunctionName,
+                            arguments = record.ArgumentsJson
+                        }
+                    });
+                }
+                return new ApplyTemplateMessage
+                {
+                    role = "assistant",
+                    // OpenAI schema requires content (may be null) alongside tool_calls.
+                    content = string.IsNullOrEmpty(message.Message) ? null : message.Message,
+                    tool_calls = calls
+                };
+            }
+
+            // Normal messages (System / User / Assistant text)
+            var role = message.Role switch
+            {
+                AuthorRole.User => "user",
+                AuthorRole.Assistant => "assistant",
+                AuthorRole.System => "system",
+                AuthorRole.Tool => "tool",
+                _ => null
+            };
+            if (role is null)
+                return null;
+
+            return new ApplyTemplateMessage
+            {
+                role = role,
+                content = message.ToChatContentText()
+            };
+        }
+
         private async Task GenerateChatCompletionStreaming(object parameters)
         {
             if (parameters is not ChatRequest input)

Chatlog/SingleMessage.cs

@@ -74,6 +74,33 @@ internal string ToolCallToString()
             return tmsg.ToString();
         }
 
+        /// <summary>
+        /// Returns the macro-expanded text content of this message as it would appear in the "content"
+        /// field of a /v1/chat/completions message (including any "Name: " prefix), without image or
+        /// tool-call structure. Used for accurate chat-template token counting.
+        /// </summary>
+        internal string ToChatContentText()
+        {
+            var realprompt = Message;
+            var addname = LLMEngine.NamesInPromptOverride ?? LLMEngine.Settings.AddNamesToPrompt;
+
+            if (Bot is GroupPersonaBase)
+                addname = true;
+
+            if (Role != AuthorRole.Assistant && Role != AuthorRole.User)
+                addname = false;
+
+            if (addname || Bot != LLMEngine.Bot || User != LLMEngine.User)
+            {
+                if (Role == AuthorRole.Assistant)
+                    realprompt = string.Format("{0}: {1}", Bot.Name, Message);
+                else if (Role == AuthorRole.User)
+                    realprompt = string.Format("{0}: {1}", User.Name, Message);
+            }
+
+            return Bot.ReplaceMacros(realprompt, User);
+        }
+
         internal Message ToChatCompletion()
         {
             // Tool result messages: skip all name/image logic

LLM/LLMEngine.cs

@@ -1212,7 +1212,10 @@ public static async Task<object> GenerateFullPrompt(SingleMessage message, strin
 
             // get the full, formated chat history complemented by the data inserts
             var addinserts = string.IsNullOrEmpty(Instruct.ThinkingStart) || !Settings.RAGMoveToThinkBlock;
-            var historytokens = (int)((Client?.CompletionType == CompletionType.Chat) ? availtokens + 2048 : availtokens);
+            var cushion = Client?.CompletionType == CompletionType.Chat ? MaxContextLength / 10 : 0;
+            if (cushion > 10240)
+                cushion = 10240;
+            var historytokens = availtokens + cushion;
             History.AddHistoryToPrompt(Settings.SessionHandling, historytokens, addinserts ? dataInserts : null);
             if (!string.IsNullOrEmpty(message.Message) || message.Role != AuthorRole.User)
             {

PromptBuilders/ChatPromptBuilder.cs

@@ -183,15 +183,7 @@ public object PromptToQuery(AuthorRole responserole = AuthorRole.Assistant, doub
 
             var total = GetTokenUsage(workingprompt);
             var max = LLMEngine.MaxContextLength - (responseoverride == -1 ? LLMEngine.Settings.MaxReplyLength : responseoverride) - 15;
-            while (total > max && workingprompt.Count > 1)
-            {
-                if (total > max + 2048)
-                {
-                    workingprompt.RemoveAt(1);
-                }
-                workingprompt.RemoveAt(1);
-                total = GetTokenUsage(workingprompt);
-            }
+            TrimToFit(workingprompt, ref total, max);
 
             var finalprompt = new List<Message>(workingprompt.ConvertAll(m => m.ToChatCompletion()));
             var cleanimages = !LLMEngine.SupportsVision || LLMEngine.Settings.MaxImageCount > 0;
@@ -304,6 +296,80 @@ public object PromptToQuery(AuthorRole responserole = AuthorRole.Assistant, doub
 
         }
 
+        /// <summary>
+        /// Trims the oldest messages (preserving index 0, the system prompt) until the accurate token
+        /// usage of <paramref name="workingprompt"/> fits within <paramref name="max"/>.
+        /// </summary>
+        /// <remarks>
+        /// The naive approach re-counts the full prompt after every single removal, which is extremely
+        /// slow on backends whose token-counting is an HTTP round-trip (e.g. llama.cpp). Instead we use a
+        /// cheap local per-message estimate to decide how many messages to drop in one batch, then perform
+        /// a single accurate re-count to verify. The batch is intentionally conservative (under-drops) so
+        /// that we never trim more context than necessary; if the estimate was too optimistic we simply
+        /// loop again, which converges in one or two accurate counts instead of N.
+        /// </remarks>
+        private void TrimToFit(List<SingleMessage> workingprompt, ref int total, int max)
+        {
+            // Fast path: nothing to do.
+            if (total <= max || workingprompt.Count <= 1)
+                return;
+
+            // Guard against pathological loops (e.g. a single message that is itself larger than max).
+            var safety = 0;
+            while (total > max && workingprompt.Count > 1)
+            {
+                var excess = total - max;
+
+                // Walk from the oldest removable message (index 1) forward, accumulating cheap local
+                // estimates until we've covered the excess. We deliberately stop as soon as the estimate
+                // meets the excess (conservative / under-drop) rather than padding it, because any
+                // shortfall is caught by the accurate re-count below.
+                var removeCount = 0;
+                var estRemoved = 0;
+                for (int i = 1; i < workingprompt.Count && estRemoved < excess; i++)
+                {
+                    estRemoved += EstimateLocalTokens(workingprompt[i]);
+                    removeCount++;
+                }
+
+                // Always remove at least one message so we make progress even if the local estimate is 0.
+                if (removeCount == 0)
+                    removeCount = 1;
+
+                // Never remove the final (newest) message; keep at least the system prompt + one message.
+                var maxRemovable = workingprompt.Count - 2;
+                if (maxRemovable < 1)
+                    maxRemovable = 1;
+                if (removeCount > maxRemovable)
+                    removeCount = maxRemovable;
+
+                workingprompt.RemoveRange(1, removeCount);
+
+                // One accurate count to verify the batch. If we under-dropped, the outer loop runs again.
+                total = GetTokenUsage(workingprompt);
+
+                if (++safety > workingprompt.Count + 4)
+                    break;
+            }
+        }
+
+        /// <summary>
+        /// Cheap, local (no backend call) per-message token estimate used only to decide how many
+        /// messages to drop in a single trim batch. This is intentionally an over-estimate per message so
+        /// the batch tends to be conservative; the authoritative count is always the full-prompt count.
+        /// </summary>
+        private static int EstimateLocalTokens(SingleMessage message)
+        {
+            var text = message.ToTextCompletion();
+            var total = TokenTools.CountTokens(text);
+
+            if (message.Role == AuthorRole.Assistant && message.ToolCalls.Count > 0)
+                total += TokenTools.CountTokens(message.ToolCallToString());
+
+            // Structural per-message overhead (role headers/delimiters) the plain text count misses.
+            return total + 8;
+        }
+
         public void Clear()
         {
             LastQuery = null;