Skip to content

Commit 09d3d79

Browse files
ParkiratSParkiratS
authored
LFM NPU Fallback + Tokenizer Fixes (#605)
* Added CPU fallback for vision encoder (images too large) + fixed special token loading * Added CPU fallback for vision encoder (images too large) + fixed special token loading
1 parent b5cc03a commit 09d3d79

3 files changed

Lines changed: 286 additions & 76 deletions

File tree

‎cactus/engine/engine_tokenizer.cpp‎

Lines changed: 189 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,156 @@ TokenizerRuntimeConfig::Decoder parse_decoder(const std::string& value) {
4545
return TokenizerRuntimeConfig::Decoder::NONE;
4646
}
4747

48+
void skip_json_whitespace(const std::string& json, size_t& pos) {
49+
while (pos < json.size() && std::isspace(static_cast<unsigned char>(json[pos]))) {
50+
++pos;
51+
}
52+
}
53+
54+
bool extract_added_token_object(const std::string& json, size_t& pos, std::string& out_object) {
55+
skip_json_whitespace(json, pos);
56+
if (pos >= json.size() || json[pos] != '{') {
57+
return false;
58+
}
59+
60+
size_t start = pos;
61+
size_t depth = 0;
62+
bool in_string = false;
63+
bool escaped = false;
64+
65+
while (pos < json.size()) {
66+
char c = json[pos++];
67+
if (escaped) {
68+
escaped = false;
69+
continue;
70+
}
71+
if (c == '\\') {
72+
escaped = true;
73+
continue;
74+
}
75+
if (c == '"') {
76+
in_string = !in_string;
77+
continue;
78+
}
79+
if (in_string) {
80+
continue;
81+
}
82+
if (c == '{') {
83+
++depth;
84+
} else if (c == '}') {
85+
if (depth == 0) {
86+
return false;
87+
}
88+
--depth;
89+
if (depth == 0) {
90+
out_object = json.substr(start, pos - start);
91+
return true;
92+
}
93+
}
94+
}
95+
96+
return false;
97+
}
98+
99+
bool parse_added_token_entry(const std::string& object, std::string& token_content, uint32_t& token_id,
100+
bool& is_special) {
101+
token_content.clear();
102+
token_id = 0;
103+
is_special = false;
104+
105+
size_t id_key = object.find("\"id\"");
106+
if (id_key == std::string::npos) {
107+
return false;
108+
}
109+
size_t id_colon = object.find(':', id_key);
110+
if (id_colon == std::string::npos) {
111+
return false;
112+
}
113+
size_t id_pos = id_colon + 1;
114+
skip_json_whitespace(object, id_pos);
115+
size_t id_end = id_pos;
116+
while (id_end < object.size() && std::isdigit(static_cast<unsigned char>(object[id_end]))) {
117+
++id_end;
118+
}
119+
if (id_end == id_pos) {
120+
return false;
121+
}
122+
token_id = static_cast<uint32_t>(std::stoul(object.substr(id_pos, id_end - id_pos)));
123+
124+
size_t content_key = object.find("\"content\"");
125+
if (content_key == std::string::npos) {
126+
return false;
127+
}
128+
size_t content_colon = object.find(':', content_key);
129+
if (content_colon == std::string::npos) {
130+
return false;
131+
}
132+
size_t content_pos = object.find('"', content_colon + 1);
133+
if (content_pos == std::string::npos) {
134+
return false;
135+
}
136+
++content_pos;
137+
token_content = extract_json_string(object, content_pos);
138+
139+
size_t special_key = object.find("\"special\"");
140+
if (special_key != std::string::npos) {
141+
size_t special_colon = object.find(':', special_key);
142+
if (special_colon != std::string::npos) {
143+
size_t special_pos = special_colon + 1;
144+
skip_json_whitespace(object, special_pos);
145+
is_special = object.compare(special_pos, 4, "true") == 0;
146+
}
147+
}
148+
149+
return true;
150+
}
151+
152+
void load_tokenizer_json_added_special_tokens(
153+
const std::string& tokenizer_json_path,
154+
std::unordered_map<std::string, uint32_t>& special_tokens) {
155+
std::ifstream file(tokenizer_json_path);
156+
if (!file.is_open()) {
157+
return;
158+
}
159+
160+
std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
161+
size_t pos = content.find("\"added_tokens\"");
162+
if (pos == std::string::npos) {
163+
return;
164+
}
165+
166+
pos = content.find('[', pos);
167+
if (pos == std::string::npos) {
168+
return;
169+
}
170+
++pos;
171+
172+
while (pos < content.size()) {
173+
skip_json_whitespace(content, pos);
174+
if (pos >= content.size() || content[pos] == ']') {
175+
break;
176+
}
177+
178+
std::string object;
179+
if (!extract_added_token_object(content, pos, object)) {
180+
++pos;
181+
continue;
182+
}
183+
184+
std::string token_content;
185+
uint32_t token_id = 0;
186+
bool is_special = false;
187+
if (parse_added_token_entry(object, token_content, token_id, is_special) && is_special) {
188+
special_tokens[token_content] = token_id;
189+
}
190+
191+
skip_json_whitespace(content, pos);
192+
if (pos < content.size() && content[pos] == ',') {
193+
++pos;
194+
}
195+
}
196+
}
197+
48198
} // namespace
49199

50200
TokenizerRuntimeConfig load_tokenizer_runtime_config(const std::string& config_file) {
@@ -87,44 +237,47 @@ void load_special_tokens_map(const std::string& config_file, std::unordered_map<
87237
special_tokens.clear();
88238

89239
std::ifstream file(config_file);
90-
if (!file.is_open()) {
91-
return;
92-
}
240+
if (file.is_open()) {
241+
std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
93242

94-
std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
95-
96-
size_t pos = content.find("\"special_tokens\"");
97-
if (pos == std::string::npos) return;
98-
99-
pos = content.find("{", pos);
100-
if (pos == std::string::npos) return;
101-
102-
size_t end_pos = content.find("}", pos);
103-
if (end_pos == std::string::npos) return;
104-
105-
std::string special_tokens_section = content.substr(pos + 1, end_pos - pos - 1);
106-
std::istringstream iss(special_tokens_section);
107-
std::string line;
108-
109-
while (std::getline(iss, line)) {
110-
size_t colon_pos = line.find(":");
111-
if (colon_pos == std::string::npos) continue;
112-
113-
std::string id_part = line.substr(0, colon_pos);
114-
std::string token_part = line.substr(colon_pos + 1);
115-
116-
size_t id_start = id_part.find("\"");
117-
size_t id_end = id_part.find("\"", id_start + 1);
118-
if (id_start == std::string::npos || id_end == std::string::npos) continue;
119-
120-
uint32_t token_id = static_cast<uint32_t>(std::stoul(id_part.substr(id_start + 1, id_end - id_start - 1)));
121-
122-
size_t token_start = token_part.find("\"");
123-
if (token_start == std::string::npos) continue;
124-
size_t value_pos = token_start + 1;
125-
std::string token_content = extract_json_string(token_part, value_pos);
126-
special_tokens[token_content] = token_id;
243+
size_t pos = content.find("\"special_tokens\"");
244+
if (pos != std::string::npos) {
245+
pos = content.find("{", pos);
246+
if (pos != std::string::npos) {
247+
size_t end_pos = content.find("}", pos);
248+
if (end_pos != std::string::npos) {
249+
std::string special_tokens_section = content.substr(pos + 1, end_pos - pos - 1);
250+
std::istringstream iss(special_tokens_section);
251+
std::string line;
252+
253+
while (std::getline(iss, line)) {
254+
size_t colon_pos = line.find(":");
255+
if (colon_pos == std::string::npos) continue;
256+
257+
std::string id_part = line.substr(0, colon_pos);
258+
std::string token_part = line.substr(colon_pos + 1);
259+
260+
size_t id_start = id_part.find("\"");
261+
size_t id_end = id_part.find("\"", id_start + 1);
262+
if (id_start == std::string::npos || id_end == std::string::npos) continue;
263+
264+
uint32_t token_id =
265+
static_cast<uint32_t>(std::stoul(id_part.substr(id_start + 1, id_end - id_start - 1)));
266+
267+
size_t token_start = token_part.find("\"");
268+
if (token_start == std::string::npos) continue;
269+
size_t value_pos = token_start + 1;
270+
std::string token_content = extract_json_string(token_part, value_pos);
271+
special_tokens[token_content] = token_id;
272+
}
273+
}
274+
}
275+
}
127276
}
277+
278+
size_t slash_pos = config_file.find_last_of("/\\");
279+
std::string dir = (slash_pos == std::string::npos) ? "." : config_file.substr(0, slash_pos);
280+
load_tokenizer_json_added_special_tokens(dir + "/tokenizer.json", special_tokens);
128281
}
129282

130283
std::vector<std::string> split_with_special_tokens(const std::string& text,

‎cactus/models/model.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ class Siglip2VisionModel : public Model {
302302

303303
size_t build_vision_mlp(CactusGraph* gb, size_t hidden_states, uint32_t layer_idx,
304304
ComputeBackend backend);
305+
void ensure_cpu_vision_weights_loaded(CactusGraph* gb);
305306

306307
void load_weights_to_graph(CactusGraph* gb) override;
307308
size_t forward(const std::vector<uint32_t>& tokens, bool use_cache = false) override;
@@ -346,6 +347,7 @@ class Siglip2VisionModel : public Model {
346347

347348
std::unique_ptr<npu::NPUEncoder> npu_encoder_;
348349
bool use_npu_encoder_ = false;
350+
bool cpu_vision_weights_loaded_ = false;
349351
};
350352

351353

0 commit comments

Comments
 (0)