From 67b38f5849ac5b116ed869ac8499aace0435457a Mon Sep 17 00:00:00 2001 From: fbuciuni90 Date: Thu, 6 Feb 2025 16:02:00 +0000 Subject: [PATCH 01/17] Supporting Velvet model --- convert_hf_to_gguf.py | 3 +++ convert_hf_to_gguf_update.py | 1 + include/llama.h | 1 + src/llama-chat.cpp | 27 ++++++++++++++++++++++++++- src/llama-chat.h | 1 + 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 018a2a588ae9d..9da7963c475b2 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -699,6 +699,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "b3f499bb4255f8ca19fccd664443283318f2fd2414d5e0b040fbdd0cc195d6c5": # ref: https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B res = "deepseek-r1-qwen" + if chkhsh == "a3df2b8943e01cfd7d68c9f8446b294f3d8706d1d6853df65df7fda5d4fcb19f": + # ref: https://huggingface.co/Almawave/Velvet-14B + res = "velvet" if res is None: logger.warning("\n") diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index cea34413f441f..241d04557fe17 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -109,6 +109,7 @@ class TOKENIZER_TYPE(IntEnum): {"name": "megrez", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Infinigence/Megrez-3B-Instruct"}, {"name": "deepseek-v3", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-V3"}, {"name": "deepseek-r1-qwen", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"}, + {"name": "velvet", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Almawave/Velvet-14B"} ] diff --git a/include/llama.h b/include/llama.h index 61907ed404dbf..a1fbd213a9915 100644 --- a/include/llama.h +++ b/include/llama.h @@ -105,6 +105,7 @@ extern "C" { LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, + LLAMA_VOCAB_PRE_TYPE_VELVET = 29 }; enum llama_rope_type { diff --git a/src/llama-chat.cpp b/src/llama-chat.cpp index 028a647948464..0cae2bb10f16f 100644 --- a/src/llama-chat.cpp +++ b/src/llama-chat.cpp @@ -58,6 +58,7 @@ static const std::map LLM_CHAT_TEMPLATES = { { "granite", LLM_CHAT_TEMPLATE_GRANITE }, { "gigachat", LLM_CHAT_TEMPLATE_GIGACHAT }, { "megrez", LLM_CHAT_TEMPLATE_MEGREZ }, + { "velvet", LLM_CHAT_TEMPLATE_VELVET }, }; llm_chat_template llm_chat_template_from_str(const std::string & name) { @@ -167,6 +168,8 @@ llm_chat_template llm_chat_detect_template(const std::string & tmpl) { return LLM_CHAT_TEMPLATE_GIGACHAT; } else if (tmpl_contains("<|role_start|>")) { return LLM_CHAT_TEMPLATE_MEGREZ; + } else if (tmpl_contains("")) { + return LLM_CHAT_TEMPLATE_VELVET; } return LLM_CHAT_TEMPLATE_UNKNOWN; } @@ -566,10 +569,32 @@ int32_t llm_chat_apply_template( if (add_ass) { ss << "<|role_start|>assistant<|role_end|>"; } + } else if (tmpl == LLM_CHAT_TEMPLATE_VELVET) { + // Velvet template + std::string leading_space = ""; + std::string trailing_space = ""; + bool trim_assistant_message = true; + bool is_inside_turn = false; + for (auto message : chat) { + if (!is_inside_turn) { + ss << leading_space << "" << trailing_space; + is_inside_turn = true; + } + std::string role(message->role); + std::string content(message->content); + if (role == "system") { + ss << content << "\n\n"; + } else if (role == "user") { + ss << content << leading_space << ""; + } else { + ss << trailing_space << (trim_assistant_message ? trim(content) : content) << ""; + is_inside_turn = false; + } + } } else { // template not supported return -1; - } + } dest = ss.str(); return dest.size(); } diff --git a/src/llama-chat.h b/src/llama-chat.h index 2f6a0e3e28266..e91f09f2ce383 100644 --- a/src/llama-chat.h +++ b/src/llama-chat.h @@ -39,6 +39,7 @@ enum llm_chat_template { LLM_CHAT_TEMPLATE_GIGACHAT, LLM_CHAT_TEMPLATE_MEGREZ, LLM_CHAT_TEMPLATE_UNKNOWN, + LLM_CHAT_TEMPLATE_VELVET }; struct llama_chat_message; From 07e1d0a14caad16e152ffb1e086da6334670a770 Mon Sep 17 00:00:00 2001 From: Francesco Buciuni Date: Thu, 6 Feb 2025 17:38:30 +0100 Subject: [PATCH 02/17] Update convert_hf_to_gguf.py Co-authored-by: Xuan-Son Nguyen --- convert_hf_to_gguf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 9da7963c475b2..b5a4a4aaa6552 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -701,7 +701,7 @@ def get_vocab_base_pre(self, tokenizer) -> str: res = "deepseek-r1-qwen" if chkhsh == "a3df2b8943e01cfd7d68c9f8446b294f3d8706d1d6853df65df7fda5d4fcb19f": # ref: https://huggingface.co/Almawave/Velvet-14B - res = "velvet" + res = "velvet" if res is None: logger.warning("\n") From 99be555369fdf8a08ab81a44beccc53ea06fd7c7 Mon Sep 17 00:00:00 2001 From: Francesco Buciuni Date: Thu, 6 Feb 2025 17:38:58 +0100 Subject: [PATCH 03/17] Update convert_hf_to_gguf.py Co-authored-by: Xuan-Son Nguyen From 3df9d221edb988b809bb34288560a592464f01d7 Mon Sep 17 00:00:00 2001 From: Francesco Buciuni Date: Thu, 6 Feb 2025 17:39:47 +0100 Subject: [PATCH 04/17] Update include/llama.h Co-authored-by: Xuan-Son Nguyen --- include/llama.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/llama.h b/include/llama.h index a1fbd213a9915..f08a9ced3b604 100644 --- a/include/llama.h +++ b/include/llama.h @@ -105,7 +105,7 @@ extern "C" { LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26, LLAMA_VOCAB_PRE_TYPE_MINERVA = 27, LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28, - LLAMA_VOCAB_PRE_TYPE_VELVET = 29 + LLAMA_VOCAB_PRE_TYPE_VELVET = 29, }; enum llama_rope_type { From 52b0bb3731c1055971460b14398e7cbc308670be Mon Sep 17 00:00:00 2001 From: Francesco Buciuni Date: Thu, 6 Feb 2025 17:44:45 +0100 Subject: [PATCH 05/17] Update src/llama-chat.cpp Co-authored-by: Xuan-Son Nguyen --- src/llama-chat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-chat.cpp b/src/llama-chat.cpp index 0cae2bb10f16f..8e60c1e8e1ca7 100644 --- a/src/llama-chat.cpp +++ b/src/llama-chat.cpp @@ -594,7 +594,7 @@ int32_t llm_chat_apply_template( } else { // template not supported return -1; - } + } dest = ss.str(); return dest.size(); } From 9d86a0442dcecf84c845b7e0a3b3b549f217b703 Mon Sep 17 00:00:00 2001 From: fbuciuni90 Date: Fri, 7 Feb 2025 08:12:02 +0000 Subject: [PATCH 06/17] removing whitespaces in src/lla-a-chat.cpp --- src/llama-chat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-chat.cpp b/src/llama-chat.cpp index 8e60c1e8e1ca7..876c906914827 100644 --- a/src/llama-chat.cpp +++ b/src/llama-chat.cpp @@ -169,7 +169,7 @@ llm_chat_template llm_chat_detect_template(const std::string & tmpl) { } else if (tmpl_contains("<|role_start|>")) { return LLM_CHAT_TEMPLATE_MEGREZ; } else if (tmpl_contains("")) { - return LLM_CHAT_TEMPLATE_VELVET; + return LLM_CHAT_TEMPLATE_VELVET; } return LLM_CHAT_TEMPLATE_UNKNOWN; } From 66e6d10b61d190935dee074b5515d3494270cb71 Mon Sep 17 00:00:00 2001 From: "f.buciuni" Date: Fri, 7 Feb 2025 19:53:16 +0100 Subject: [PATCH 07/17] fixing position of LLM_CHAT_TEMPLATE_VELVET in enum --- src/llama-chat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-chat.h b/src/llama-chat.h index e91f09f2ce383..0fe4b8e22e769 100644 --- a/src/llama-chat.h +++ b/src/llama-chat.h @@ -38,8 +38,8 @@ enum llm_chat_template { LLM_CHAT_TEMPLATE_GRANITE, LLM_CHAT_TEMPLATE_GIGACHAT, LLM_CHAT_TEMPLATE_MEGREZ, + LLM_CHAT_TEMPLATE_VELVET, LLM_CHAT_TEMPLATE_UNKNOWN, - LLM_CHAT_TEMPLATE_VELVET }; struct llama_chat_message; From 39795570dbe90a05f98520c3fd0b7d783464ab11 Mon Sep 17 00:00:00 2001 From: "f.buciuni" Date: Fri, 7 Feb 2025 19:54:55 +0100 Subject: [PATCH 08/17] updating velvet chat template --- src/llama-chat.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/llama-chat.cpp b/src/llama-chat.cpp index 876c906914827..af0539bd47803 100644 --- a/src/llama-chat.cpp +++ b/src/llama-chat.cpp @@ -573,8 +573,11 @@ int32_t llm_chat_apply_template( // Velvet template std::string leading_space = ""; std::string trailing_space = ""; - bool trim_assistant_message = true; + bool trim_assistant_message = false; bool is_inside_turn = false; + std::string system_message = ""; + std::string last_message(chat.back()->content); + ss << ""; for (auto message : chat) { if (!is_inside_turn) { ss << leading_space << "" << trailing_space; @@ -583,9 +586,9 @@ int32_t llm_chat_apply_template( std::string role(message->role); std::string content(message->content); if (role == "system") { - ss << content << "\n\n"; + system_message = content + "\n\n"; } else if (role == "user") { - ss << content << leading_space << ""; + ss << (content==last_message ? system_message : "") << content << leading_space << ""; } else { ss << trailing_space << (trim_assistant_message ? trim(content) : content) << ""; is_inside_turn = false; From 0a8995a375f897807efef169bb6d89209b962409 Mon Sep 17 00:00:00 2001 From: "f.buciuni" Date: Fri, 7 Feb 2025 19:56:23 +0100 Subject: [PATCH 09/17] adding test case for velvet chat template --- tests/test-chat-template.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index e0314ae1d6296..d970beb57997e 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -262,6 +262,14 @@ int main(void) { /* .bos_token= */ "", /* .eos_token= */ "", }, + { + /* .name= */ "velvet", + /* .template_str= */ "{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n {%- if not (message.role == \"tool\" or message.role == \"tool_results\" or (message.tool_calls is defined and message.tool_calls is not none)) %}\n {%- if (message[\"role\"] == \"user\") != (ns.index % 2 == 0) %}\n {{- raise_exception(\"After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif %}\n {%- set ns.index = ns.index + 1 %}\n {%- endif %}\n{%- endfor %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"user\" %}\n {%- if message == user_messages[-1] and system_message is defined and tools is not none %}\n {{- \"\" + system_message + \"\\n\\n\" + \"[\" }}\n {%- for tool in tools %}\n{{ tool }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %}\n {{- \"\" + \"\\n\\n\" + message[\"content\"] + \"\" }}\n {%- elif message == user_messages[-1] and system_message is defined and tools is none %}\n {{- \"\" + system_message + \"\\n\\n\" + message[\"content\"] + \"\" }}\n {%- else %}\n {{- \"\" + message[\"content\"] + \"\" }}\n {%- endif %}\n {%- elif (message.tool_calls is defined and message.tool_calls is not none) %}\n {{- \"[\" }}\n {%- for tool_call in message.tool_calls %}\n{{ tool_call }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %} {{- \"\" + eos_token }}\n {%- elif message[\"role\"] == \"assistant\" %}\n {{- message[\"content\"] + eos_token}}\n {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n {%- if message.content is defined and message.content.content is defined %}\n {%- set content = message.content.content %}\n {%- else %}\n {%- set content = message.content %}\n {%- endif %}\n {{- '{\"content\": ' + content|string }}\n {{- '}' }}\n {%- else %}\n {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n", + /* .expected_output= */ "HelloHi thereWho are you I am an assistant You are a helpful assistant\n\nAnother question", + /* .expected_output_jinja= */ "HelloHi thereWho are you I am an assistant You are a helpful assistant\n\nAnother question", + /* .bos_token= */ "", + /* .eos_token= */ "", + }, }; std::vector formatted_chat(1024); int32_t res; From e8981aa90b67a007b0bd32b4ca9ec54dd5167b57 Mon Sep 17 00:00:00 2001 From: fbuciuni90 Date: Wed, 12 Feb 2025 18:54:13 +0100 Subject: [PATCH 10/17] upadated llama-vocab.cpp with velvet case --- src/llama-vocab.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index ad9ffe66aa749..0c2cdcb65d5d8 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -392,6 +392,11 @@ struct llm_tokenizer_bpe : llm_tokenizer { "'s|'t|'re|'ve|'m|'ll|'d| ?\\p{L}+| ?\\p{N}+| ?[^\\s\\p{L}\\p{N}]+|\\s+(?!\\S)", }; break; + case LLAMA_VOCAB_PRE_TYPE_VELVET: + regex_exprs = { + "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + }; + break; default: // default regex for BPE tokenization pre-processing regex_exprs = { @@ -1592,6 +1597,9 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { } else if ( tokenizer_pre == "megrez") { pre_type = LLAMA_VOCAB_PRE_TYPE_QWEN2; + } else if ( + tokenizer_pre == "velvet") { + pre_type = LLAMA_VOCAB_PRE_TYPE_VELVET; } else { throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str())); } From c4c923aebe388c4df3548a76d62c37115088b5ab Mon Sep 17 00:00:00 2001 From: fbuciuni90 Date: Thu, 20 Feb 2025 17:11:41 +0000 Subject: [PATCH 11/17] attempt to fix pretokenization regex --- src/llama-vocab.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 0c2cdcb65d5d8..6274b49e2ef49 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -393,8 +393,9 @@ struct llm_tokenizer_bpe : llm_tokenizer { }; break; case LLAMA_VOCAB_PRE_TYPE_VELVET: - regex_exprs = { - "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + regex_exprs = { + "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + //"[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }; break; default: From 50bf79be4ecdb27aeee5689ba6d98166ccbd4f7e Mon Sep 17 00:00:00 2001 From: fbuciuni90 Date: Fri, 21 Feb 2025 11:35:39 +0100 Subject: [PATCH 12/17] fixed pre tokenization regex --- src/llama-vocab.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 6274b49e2ef49..434f25050c28c 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -393,9 +393,10 @@ struct llm_tokenizer_bpe : llm_tokenizer { }; break; case LLAMA_VOCAB_PRE_TYPE_VELVET: - regex_exprs = { - "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" - //"[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + // original regex from tokenizer.json + //"[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + regex_exprs = { + "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }; break; default: From 308ef21648e186e21a3bfee9b84893827e0cd0c6 Mon Sep 17 00:00:00 2001 From: gbuciuni90 Date: Fri, 21 Feb 2025 16:20:29 +0100 Subject: [PATCH 13/17] fixed chat template --- src/llama-chat.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-chat.cpp b/src/llama-chat.cpp index af0539bd47803..a5404b9783e3f 100644 --- a/src/llama-chat.cpp +++ b/src/llama-chat.cpp @@ -585,7 +585,7 @@ int32_t llm_chat_apply_template( } std::string role(message->role); std::string content(message->content); - if (role == "system") { + if (role == "system" && content != " ") { system_message = content + "\n\n"; } else if (role == "user") { ss << (content==last_message ? system_message : "") << content << leading_space << ""; From bc15f90800d0cab6d9d8b5b43e5a85869fa363cb Mon Sep 17 00:00:00 2001 From: Buciuni Francesco Date: Fri, 11 Apr 2025 17:12:42 +0200 Subject: [PATCH 14/17] fix typo in llama-vocab.cpp caused by merge --- src/llama-vocab.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 87663ec058fc4..20eb5f8e0d2c2 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -400,6 +400,7 @@ struct llm_tokenizer_bpe : llm_tokenizer { regex_exprs = { "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" }; + break; case LLAMA_VOCAB_PRE_TYPE_GPT4O: regex_exprs = { // original regex from tokenizer.json @@ -1626,7 +1627,6 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { tokenizer_pre == "velvet") { pre_type = LLAMA_VOCAB_PRE_TYPE_VELVET; } else if ( - tokenizer_pre == "gpt-4o") { tokenizer_pre == "gpt-4o" || tokenizer_pre == "llama4") { pre_type = LLAMA_VOCAB_PRE_TYPE_GPT4O; From bd0ffebd68703f565649f302ddcecf43c013515b Mon Sep 17 00:00:00 2001 From: Buciuni Francesco Date: Fri, 11 Apr 2025 17:43:06 +0200 Subject: [PATCH 15/17] fix indentation tab issue --- src/llama-vocab.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 20eb5f8e0d2c2..62e84f1b1cafb 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -397,9 +397,9 @@ struct llm_tokenizer_bpe : llm_tokenizer { case LLAMA_VOCAB_PRE_TYPE_VELVET: // original regex from tokenizer.json //"[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" - regex_exprs = { + regex_exprs = { "[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))*((?=[\\p{L}])([^A-Z]))+|[^\\r\\n\\p{L}\\p{N}]?((?=[\\p{L}])([^a-z]))+((?=[\\p{L}])([^A-Z]))*|\\p{N}|[\\p{P}\\p{S}]{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" - }; + }; break; case LLAMA_VOCAB_PRE_TYPE_GPT4O: regex_exprs = { From c516dbdd6d44719d77069b706a2b5249bbd3d2db Mon Sep 17 00:00:00 2001 From: Buciuni Francesco Date: Mon, 14 Apr 2025 09:26:21 +0200 Subject: [PATCH 16/17] fix chat template test caused by sync --- tests/test-chat-template.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index a6042f4987bcf..3340c48abd41b 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -282,6 +282,7 @@ int main(void) { /* .bos_token= */ "", /* .eos_token= */ "", }, + { /* .name= */ "yandex/YandexGPT-5-Lite-8B-instruct", /* .template_str= */ "{%- set names = {'assistant': ' Ассистент:', 'user': ' Пользователь:'} %}\n{%- set tools_prefix = 'Тебе доступны следующие функции:' %}\n{%- macro __render_tool(tool) %}\n {%- set name = tool.function.name %}\n {%- set description = tool.function.description|default('') %}\n {%- set parameters = tool.function.parameters|tojson %}\n {{- '\\n' }}function {{ '{' }}'name':'{{ name }}',\n {%- if tool.function.description %}'description':'{{ description }}',{% endif %}\n'parameters':{{ parameters }}\n {{- '}' }}\n{%- endmacro %}\n{%- macro __render_tools(tools) %}\n {{- tools_prefix }}\n {%- for tool in tools %}\n {{- __render_tool(tool) }}\n {%- endfor %}\n {{- '\\n\\n' }}\n{%- endmacro %}\n{%- macro __render_tool_message(message) %}\n {{- '\\n\\nРезультат вызова' }} {{ message.name }}: {{ message.content }} {{ '\\n\\n' }}\n{%- endmacro %}\n{%- if tools -%}\n {{- __render_tools(tools) }}\n{%- endif -%}\n{%- macro __render_user_message(message) %}\n{{ names.user }} {{ message.content + '\\n\\n' }}\n{%- endmacro %}\n{%- macro __render_assistant_message(message) %}\n {{- names.assistant }}\n {%- set call = message['function_call'] %}\n {%- if call %}\n {{- '\\n[TOOL_CALL_START]' }}{{ call.name }}{{ '\\n' }}{{ call.arguments|tojson }}\n {%- else %}\n {{- ' ' + message.content + '\\n\\n' }}\n {%- endif %}\n{%- endmacro %}\n{%- if not add_generation_prompt is defined %}\n{%- set add_generation_prompt = false %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'user' %}\n {{- __render_user_message(message) }}\n {%- endif %}\n {%- if message.role == 'assistant' and not loop.last %}\n {{- __render_assistant_message(message) }}\n {%- endif %}\n {%- if message.role == 'tool' %}\n {{- __render_tool_message(message) }}\n {%- endif %}\n {%- if loop.last %}\n {{- ' Ассистент:[SEP]' }}\n {%- endif %}\n{%- endfor %}\n", /* .expected_output= */ " Пользователь: Hello\n\n Ассистент: Hi there\n\n Пользователь: Who are you\n\n Ассистент: I am an assistant \n\n Пользователь: Another question\n\n Ассистент:[SEP]", From a78b98344b4494ff56182a5d883acf9d0de03ba9 Mon Sep 17 00:00:00 2001 From: Buciuni Francesco Date: Wed, 16 Apr 2025 12:27:35 +0200 Subject: [PATCH 17/17] small fix in chat template test --- tests/test-chat-template.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-chat-template.cpp b/tests/test-chat-template.cpp index 3340c48abd41b..a6bff1f3e6d2c 100644 --- a/tests/test-chat-template.cpp +++ b/tests/test-chat-template.cpp @@ -276,13 +276,13 @@ int main(void) { }, { /* .name= */ "velvet", - /* .template_str= */ "{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n {%- if not (message.role == \"tool\" or message.role == \"tool_results\" or (message.tool_calls is defined and message.tool_calls is not none)) %}\n {%- if (message[\"role\"] == \"user\") != (ns.index % 2 == 0) %}\n {{- raise_exception(\"After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif %}\n {%- set ns.index = ns.index + 1 %}\n {%- endif %}\n{%- endfor %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"user\" %}\n {%- if message == user_messages[-1] and system_message is defined and tools is not none %}\n {{- \"\" + system_message + \"\\n\\n\" + \"[\" }}\n {%- for tool in tools %}\n{{ tool }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %}\n {{- \"\" + \"\\n\\n\" + message[\"content\"] + \"\" }}\n {%- elif message == user_messages[-1] and system_message is defined and tools is none %}\n {{- \"\" + system_message + \"\\n\\n\" + message[\"content\"] + \"\" }}\n {%- else %}\n {{- \"\" + message[\"content\"] + \"\" }}\n {%- endif %}\n {%- elif (message.tool_calls is defined and message.tool_calls is not none) %}\n {{- \"[\" }}\n {%- for tool_call in message.tool_calls %}\n{{ tool_call }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %} {{- \"\" + eos_token }}\n {%- elif message[\"role\"] == \"assistant\" %}\n {{- message[\"content\"] + eos_token}}\n {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n {%- if message.content is defined and message.content.content is defined %}\n {%- set content = message.content.content %}\n {%- else %}\n {%- set content = message.content %}\n {%- endif %}\n {{- '{\"content\": ' + content|string }}\n {{- '}' }}\n {%- else %}\n {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n", + /* .template_str= */ "{{ bos_token }}{%- if messages[0][\"role\"] == \"system\" %}\n {%- set system_message = messages[0][\"content\"] %}\n {%- set loop_messages = messages[1:] %}\n{%- else %}\n {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n {%- if not (message.role == \"tool\" or message.role == \"tool_results\" or (message.tool_calls is defined and message.tool_calls is not none)) %}\n {%- if (message[\"role\"] == \"user\") != (ns.index % 2 == 0) %}\n {{- raise_exception(\"After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\") }}\n {%- endif %}\n {%- set ns.index = ns.index + 1 %}\n {%- endif %}\n{%- endfor %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n {%- if message[\"role\"] == \"user\" %}\n {%- if message == user_messages[-1] and system_message is defined and tools is not none %}\n {{- \"\" + system_message + \"\\n\\n\" + \"[\" }}\n {%- for tool in tools %}\n{{ tool }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %}\n {{- \"\" + \"\\n\\n\" + message[\"content\"] + \"\" }}\n {%- elif message == user_messages[-1] and system_message is defined and tools is none %}\n {{- \"\" + system_message + \"\\n\\n\" + message[\"content\"] + \"\" }}\n {%- else %}\n {{- \"\" + message[\"content\"] + \"\" }}\n {%- endif %}\n {%- elif (message.tool_calls is defined and message.tool_calls is not none) %}\n {{- \"[\" }}\n {%- for tool_call in message.tool_calls %}\n{{ tool_call }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- else %}\n {{- \"]\" }}\n {%- endif %}\n {%- endfor %} {{- \"\" + eos_token }}\n {%- elif message[\"role\"] == \"assistant\" %}\n {{- message[\"content\"] + eos_token}}\n {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n {%- if message.content is defined and message.content.content is defined %}\n {%- set content = message.content.content %}\n {%- else %}\n {%- set content = message.content %}\n {%- endif %}\n {{- '{\"content\": ' + content|string }}\n {{- '}' }}\n {%- else %}\n {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n {%- endif %}\n{%- endfor %}\n", /* .expected_output= */ "HelloHi thereWho are you I am an assistant You are a helpful assistant\n\nAnother question", /* .expected_output_jinja= */ "HelloHi thereWho are you I am an assistant You are a helpful assistant\n\nAnother question", /* .bos_token= */ "", /* .eos_token= */ "", - }, - { + }, + { /* .name= */ "yandex/YandexGPT-5-Lite-8B-instruct", /* .template_str= */ "{%- set names = {'assistant': ' Ассистент:', 'user': ' Пользователь:'} %}\n{%- set tools_prefix = 'Тебе доступны следующие функции:' %}\n{%- macro __render_tool(tool) %}\n {%- set name = tool.function.name %}\n {%- set description = tool.function.description|default('') %}\n {%- set parameters = tool.function.parameters|tojson %}\n {{- '\\n' }}function {{ '{' }}'name':'{{ name }}',\n {%- if tool.function.description %}'description':'{{ description }}',{% endif %}\n'parameters':{{ parameters }}\n {{- '}' }}\n{%- endmacro %}\n{%- macro __render_tools(tools) %}\n {{- tools_prefix }}\n {%- for tool in tools %}\n {{- __render_tool(tool) }}\n {%- endfor %}\n {{- '\\n\\n' }}\n{%- endmacro %}\n{%- macro __render_tool_message(message) %}\n {{- '\\n\\nРезультат вызова' }} {{ message.name }}: {{ message.content }} {{ '\\n\\n' }}\n{%- endmacro %}\n{%- if tools -%}\n {{- __render_tools(tools) }}\n{%- endif -%}\n{%- macro __render_user_message(message) %}\n{{ names.user }} {{ message.content + '\\n\\n' }}\n{%- endmacro %}\n{%- macro __render_assistant_message(message) %}\n {{- names.assistant }}\n {%- set call = message['function_call'] %}\n {%- if call %}\n {{- '\\n[TOOL_CALL_START]' }}{{ call.name }}{{ '\\n' }}{{ call.arguments|tojson }}\n {%- else %}\n {{- ' ' + message.content + '\\n\\n' }}\n {%- endif %}\n{%- endmacro %}\n{%- if not add_generation_prompt is defined %}\n{%- set add_generation_prompt = false %}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'user' %}\n {{- __render_user_message(message) }}\n {%- endif %}\n {%- if message.role == 'assistant' and not loop.last %}\n {{- __render_assistant_message(message) }}\n {%- endif %}\n {%- if message.role == 'tool' %}\n {{- __render_tool_message(message) }}\n {%- endif %}\n {%- if loop.last %}\n {{- ' Ассистент:[SEP]' }}\n {%- endif %}\n{%- endfor %}\n", /* .expected_output= */ " Пользователь: Hello\n\n Ассистент: Hi there\n\n Пользователь: Who are you\n\n Ассистент: I am an assistant \n\n Пользователь: Another question\n\n Ассистент:[SEP]",