Skip to content

Commit 0acdb78

Browse files
Tianyue-ZhaoTianyue-Zhao
committed
Fix broken rebase
1 parent f53b313 commit 0acdb78

File tree

4 files changed

+36
-55
lines changed

4 files changed

+36
-55
lines changed

‎gguf-py/gguf/tensor_mapping.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -428,9 +428,9 @@ class TensorNameMap:
428428
"model.layers.{bid}.residual_mlp.w1", # arctic
429429
"transformer.h.{bid}.mlp.c_fc_0", # exaone
430430
"model.layers.{bid}.feed_forward.gate_proj", # llama4 jamba granite-hybrid
431-
"model.layers.{bid}.block_sparse_moe.gate", # smallthinker
432-
"model.transformer.blocks.{bid}.ff_proj", # llada
433-
"layers.{bid}.mlp.gate_proj", # qwen3-embedding
431+
"model.layers.{bid}.block_sparse_moe.gate", # smallthinker
432+
"model.transformer.blocks.{bid}.ff_proj", # llada
433+
"layers.{bid}.mlp.gate_proj", # qwen3-embedding
434434
"model.layers.{bid}.mlp.language_mlp.gate_proj", # cogvlm
435435
),
436436

‎src/llama-arch.cpp

Lines changed: 24 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1980,27 +1980,6 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
19801980
},
19811981
{
19821982
LLM_ARCH_DREAM,
1983-
{
1984-
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
1985-
{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },
1986-
{ LLM_TENSOR_OUTPUT, "output" },
1987-
{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
1988-
{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
1989-
{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
1990-
{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
1991-
{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
1992-
{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
1993-
{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
1994-
{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
1995-
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
1996-
{ LLM_TENSOR_FFN_GATE_INP, "blk.%d.ffn_gate_inp" },
1997-
{ LLM_TENSOR_FFN_GATE_EXPS, "blk.%d.ffn_gate_exps" },
1998-
{ LLM_TENSOR_FFN_DOWN_EXPS, "blk.%d.ffn_down_exps" },
1999-
{ LLM_TENSOR_FFN_UP_EXPS, "blk.%d.ffn_up_exps" }
2000-
},
2001-
},
2002-
{
2003-
LLM_ARCH_COGVLM,
20041983
{
20051984
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
20061985
{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },
@@ -2014,13 +1993,6 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
20141993
{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
20151994
{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
20161995
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
2017-
{ LLM_TENSOR_VISEXP_ATTN_WQ, "blk.%d.vis_attn_q" },
2018-
{ LLM_TENSOR_VISEXP_ATTN_WK, "blk.%d.vis_attn_k" },
2019-
{ LLM_TENSOR_VISEXP_ATTN_WV, "blk.%d.vis_attn_v" },
2020-
{ LLM_TENSOR_VISEXP_ATTN_OUT, "blk.%d.vis_attn_output" },
2021-
{ LLM_TENSOR_VISEXP_FFN_GATE, "blk.%d.vis_gate" },
2022-
{ LLM_TENSOR_VISEXP_FFN_DOWN, "blk.%d.vis_down" },
2023-
{ LLM_TENSOR_VISEXP_FFN_UP, "blk.%d.vis_up" },
20241996
},
20251997
},
20261998
{
@@ -2040,6 +2012,30 @@ static const std::map<llm_arch, std::map<llm_tensor, const char *>> LLM_TENSOR_N
20402012
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
20412013
},
20422014
},
2015+
{
2016+
LLM_ARCH_COGVLM,
2017+
{
2018+
{ LLM_TENSOR_TOKEN_EMBD, "token_embd" },
2019+
{ LLM_TENSOR_OUTPUT_NORM, "output_norm" },
2020+
{ LLM_TENSOR_OUTPUT, "output" },
2021+
{ LLM_TENSOR_ATTN_NORM, "blk.%d.attn_norm" },
2022+
{ LLM_TENSOR_ATTN_Q, "blk.%d.attn_q" },
2023+
{ LLM_TENSOR_ATTN_K, "blk.%d.attn_k" },
2024+
{ LLM_TENSOR_ATTN_V, "blk.%d.attn_v" },
2025+
{ LLM_TENSOR_ATTN_OUT, "blk.%d.attn_output" },
2026+
{ LLM_TENSOR_FFN_NORM, "blk.%d.ffn_norm" },
2027+
{ LLM_TENSOR_FFN_GATE, "blk.%d.ffn_gate" },
2028+
{ LLM_TENSOR_FFN_DOWN, "blk.%d.ffn_down" },
2029+
{ LLM_TENSOR_FFN_UP, "blk.%d.ffn_up" },
2030+
{ LLM_TENSOR_VISEXP_ATTN_WQ, "blk.%d.vis_attn_q" },
2031+
{ LLM_TENSOR_VISEXP_ATTN_WK, "blk.%d.vis_attn_k" },
2032+
{ LLM_TENSOR_VISEXP_ATTN_WV, "blk.%d.vis_attn_v" },
2033+
{ LLM_TENSOR_VISEXP_ATTN_OUT, "blk.%d.vis_attn_output" },
2034+
{ LLM_TENSOR_VISEXP_FFN_GATE, "blk.%d.vis_gate" },
2035+
{ LLM_TENSOR_VISEXP_FFN_DOWN, "blk.%d.vis_down" },
2036+
{ LLM_TENSOR_VISEXP_FFN_UP, "blk.%d.vis_up" },
2037+
},
2038+
},
20432039
{
20442040
LLM_ARCH_UNKNOWN,
20452041
{

‎src/llama-model.cpp

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5353,12 +5353,7 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
53535353

53545354
// output
53555355
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
5356-
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
5357-
5358-
// if output is NULL, init from the input tok embed
5359-
if (output == NULL) {
5360-
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
5361-
}
5356+
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);
53625357

53635358
for (int i = 0; i < n_layer; ++i) {
53645359
auto & layer = layers[i];
@@ -17661,7 +17656,6 @@ struct llm_build_cogvlm : public llm_graph_context {
1766117656
Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);
1766217657
Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);
1766317658

17664-
// TODO: Check Rope because this might not be the same as cogvlm
1766517659
Qcur = ggml_rope(ctx0, Qcur, inp_pos, n_embd_head, GGML_ROPE_TYPE_NEOX);
1766617660
Kcur = ggml_rope(ctx0, Kcur, inp_pos, n_embd_head, GGML_ROPE_TYPE_NEOX);
1766717661

@@ -17675,12 +17669,6 @@ struct llm_build_cogvlm : public llm_graph_context {
1767517669
cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
1767617670
cb(cur, "ffn_norm", il);
1767717671

17678-
// Make a standard ffn without the build_ffn function
17679-
//ggml_tensor * tmp = build_lora_mm(ffn_up, cur);
17680-
//ggml_tensor * gate = build_lora_mm(ffn_gate, cur);
17681-
//gate = ggml_silu(ctx0, gate);
17682-
//cur = ggml_mul(ctx0, gate, tmp);
17683-
//cur = build_lora_mm(ffn_down, cur);
1768417672
cur = build_ffn(cur,
1768517673
ffn_up, NULL, NULL,
1768617674
ffn_gate, NULL, NULL,

‎tools/mtmd/clip.cpp

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1669,11 +1669,8 @@ struct clip_graph {
16691669
auto & layer = model.layers[il];
16701670
ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states
16711671

1672-
// Check if this is COGVLM projector type for post-norm layernorm order
1673-
const bool is_cogvlm = ctx->proj_type() == PROJECTOR_TYPE_COGVLM;
1674-
1675-
// layernorm1 (only for non-COGVLM)
1676-
if (!is_cogvlm) {
1672+
// layernorm1
1673+
if (ctx->proj_type() != PROJECTOR_TYPE_COGVLM) {
16771674
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
16781675
cb(cur, "layer_inp_normed", il);
16791676
}
@@ -1730,8 +1727,8 @@ struct clip_graph {
17301727
cb(cur, "attn_out_scaled", il);
17311728
}
17321729

1733-
// Apply layernorm AFTER attention for COGVLM (post-norm)
1734-
if (is_cogvlm) {
1730+
// Apply layernorm after attention for cogvlm
1731+
if (ctx->proj_type() == PROJECTOR_TYPE_COGVLM) {
17351732
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
17361733
cb(cur, "attn_post_norm", il);
17371734
}
@@ -1743,8 +1740,8 @@ struct clip_graph {
17431740

17441741
cb(cur, "ffn_inp", il);
17451742

1746-
// layernorm2 (only for non-COGVLM)
1747-
if (!is_cogvlm) {
1743+
// layernorm2
1744+
if (ctx->proj_type() != PROJECTOR_TYPE_COGVLM) {
17481745
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
17491746
cb(cur, "ffn_inp_normed", il);
17501747
}
@@ -1763,8 +1760,8 @@ struct clip_graph {
17631760
cb(cur, "ffn_out_scaled", il);
17641761
}
17651762

1766-
// Apply layernorm AFTER MLP for COGVLM (post-norm)
1767-
if (is_cogvlm) {
1763+
// Apply layernorm after mlp for cogvlm
1764+
if (ctx->proj_type() == PROJECTOR_TYPE_COGVLM) {
17681765
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
17691766
cb(cur, "ffn_post_norm", il);
17701767
}

0 commit comments

Comments
 (0)