Skip to content

Commit 831b86d

Browse files
ngxsonCISC
ngxson
and
CISC
authored
model: support GLM-OCR (ggml-org#19677)
* model: support GLM-OCR * Update convert_hf_to_gguf.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com> --------- Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
1 parent 35f166c commit 831b86d

8 files changed

Lines changed: 118 additions & 39 deletions

File tree

‎convert_hf_to_gguf.py‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4584,7 +4584,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
45844584
yield from super().modify_tensors(data_torch, name, bid)
45854585

45864586

4587-
@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration")
4587+
@ModelBase.register("Glm4vForConditionalGeneration", "Glm4vMoeForConditionalGeneration", "GlmOcrForConditionalGeneration")
45884588
class Glm4VVisionModel(Qwen3VLVisionModel):
45894589
def set_gguf_parameters(self):
45904590
MmprojModel.set_gguf_parameters(self) # skip Qwen3VLVisionModel parameters
@@ -8776,7 +8776,7 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
87768776
n_head = self.hparams["num_attention_heads"]
87778777
n_kv_head = self.hparams["num_key_value_heads"]
87788778
n_embd = self.hparams["hidden_size"]
8779-
head_dim = n_embd // n_head
8779+
head_dim = self.hparams.get("head_dim", n_embd // n_head)
87808780
# because llama.cpp M-RoPE kernel only supports Neox ordering, we have to permute the weights here
87818781
if name.endswith(("q_proj.weight", "q_proj.bias")):
87828782
data_torch = Glm4Model.normal_to_neox(data_torch, n_head, n_head, head_dim, self.partial_rotary_factor)
@@ -8785,6 +8785,27 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
87858785
yield from super().modify_tensors(data_torch, name, bid)
87868786

87878787

8788+
@ModelBase.register("GlmOcrForConditionalGeneration")
8789+
class GlmOCRModel(Glm4Model):
8790+
model_arch = gguf.MODEL_ARCH.GLM4
8791+
use_mrope = False
8792+
partial_rotary_factor = 0.5
8793+
8794+
# Note: GLM-OCR is the same as GLM4, but with an extra NextN/MTP prediction layer
8795+
8796+
def __init__(self, *args, **kwargs):
8797+
super().__init__(*args, **kwargs)
8798+
# GLM-OCR has num_hidden_layers + 1 actual layers (including NextN layer)
8799+
self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0)
8800+
self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count)
8801+
8802+
def set_gguf_parameters(self):
8803+
super().set_gguf_parameters()
8804+
# NextN/MTP prediction layers
8805+
if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None:
8806+
self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers)
8807+
8808+
87888809
@ModelBase.register("Glm4MoeForCausalLM", "Glm4vMoeForConditionalGeneration")
87898810
class Glm4MoeModel(TextModel):
87908811
model_arch = gguf.MODEL_ARCH.GLM4_MOE

‎gguf-py/gguf/constants.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2660,6 +2660,13 @@ class MODEL_TENSOR(IntEnum):
26602660
MODEL_TENSOR.FFN_UP,
26612661
MODEL_TENSOR.ATTN_POST_NORM,
26622662
MODEL_TENSOR.FFN_POST_NORM,
2663+
# NextN/MTP tensors - preserved but unused
2664+
MODEL_TENSOR.NEXTN_EH_PROJ,
2665+
MODEL_TENSOR.NEXTN_EMBED_TOKENS,
2666+
MODEL_TENSOR.NEXTN_ENORM,
2667+
MODEL_TENSOR.NEXTN_HNORM,
2668+
MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD,
2669+
MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM,
26632670
],
26642671
MODEL_ARCH.GLM4_MOE: [
26652672
MODEL_TENSOR.TOKEN_EMBD,

‎gguf-py/gguf/tensor_mapping.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1404,6 +1404,7 @@ class TensorNameMap:
14041404
MODEL_TENSOR.V_ENC_ATTN_Q_NORM: (
14051405
"vision_tower.vision_model.encoder.layers.{bid}.attn.q_norm", # InternVL
14061406
"model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1
1407+
"visual.blocks.{bid}.attn.q_norm", # GLM-OCR
14071408
),
14081409

14091410
MODEL_TENSOR.V_ENC_ATTN_K: (
@@ -1422,6 +1423,7 @@ class TensorNameMap:
14221423
MODEL_TENSOR.V_ENC_ATTN_K_NORM: (
14231424
"vision_tower.vision_model.encoder.layers.{bid}.attn.k_norm", # InternVL
14241425
"model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1
1426+
"visual.blocks.{bid}.attn.k_norm", # GLM-OCR
14251427
),
14261428

14271429
MODEL_TENSOR.V_ENC_ATTN_V: (

‎src/llama-arch.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1633,6 +1633,12 @@ static std::set<llm_tensor> llm_get_tensor_names(llm_arch arch) {
16331633
LLM_TENSOR_FFN_DOWN,
16341634
LLM_TENSOR_ATTN_POST_NORM,
16351635
LLM_TENSOR_FFN_POST_NORM,
1636+
LLM_TENSOR_NEXTN_EH_PROJ,
1637+
LLM_TENSOR_NEXTN_EMBED_TOKENS,
1638+
LLM_TENSOR_NEXTN_ENORM,
1639+
LLM_TENSOR_NEXTN_HNORM,
1640+
LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD,
1641+
LLM_TENSOR_NEXTN_SHARED_HEAD_NORM,
16361642
};
16371643
case LLM_ARCH_GLM4_MOE:
16381644
return {

‎src/llama-model.cpp‎

Lines changed: 41 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1784,7 +1784,15 @@ void llama_model::load_hparams(llama_model_loader & ml) {
17841784
{
17851785
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
17861786
ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, false);
1787+
1788+
// NextN/MTP parameters (GLM-OCR)
1789+
ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers, false);
1790+
1791+
// TODO: when MTP is implemented, this should probably be updated if needed
1792+
hparams.n_layer_kv_from_start = hparams.n_layer - hparams.nextn_predict_layers;
1793+
17871794
switch (hparams.n_layer) {
1795+
case 17: type = LLM_TYPE_1B; break; // GLM-OCR
17881796
case 40: type = LLM_TYPE_9B; break;
17891797
case 61: type = LLM_TYPE_32B; break;
17901798
default: type = LLM_TYPE_UNKNOWN;
@@ -5410,30 +5418,48 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
54105418
}
54115419

54125420
for (int i = 0; i < n_layer; ++i) {
5421+
int flags = 0;
5422+
if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
5423+
// skip all tensors in the NextN layers
5424+
flags |= TENSOR_SKIP;
5425+
}
5426+
54135427
auto & layer = layers[i];
54145428

5415-
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
5416-
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
5417-
layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, TENSOR_NOT_REQUIRED);
5429+
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);
5430+
layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, n_embd + 2*n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);
5431+
layer.bqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "bias", i), {n_embd + 2*n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);
54185432

54195433
if (layer.wqkv == nullptr) {
5420-
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, 0);
5421-
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, 0);
5422-
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, 0);
5423-
layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);
5424-
layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
5425-
layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, TENSOR_NOT_REQUIRED);
5434+
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head_k * n_head}, flags);
5435+
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_k_gqa}, flags);
5436+
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_v_gqa}, flags);
5437+
layer.bq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "bias", i), {n_embd}, flags | TENSOR_NOT_REQUIRED);
5438+
layer.bk = create_tensor(tn(LLM_TENSOR_ATTN_K, "bias", i), {n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);
5439+
layer.bv = create_tensor(tn(LLM_TENSOR_ATTN_V, "bias", i), {n_embd_gqa}, flags | TENSOR_NOT_REQUIRED);
54265440
}
54275441

5428-
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
5442+
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, flags);
54295443

5430-
layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, 0);
5444+
layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, flags);
54315445

5432-
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
5433-
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);
5434-
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff * 2}, 0);
5446+
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);
5447+
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, flags);
5448+
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff * 2}, flags);
54355449

5436-
layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, 0);
5450+
layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, "weight", i), {n_embd}, flags);
5451+
5452+
// NextN/MTP tensors (preserved but unused) - conditionally load for last nextn_predict_layers
5453+
if (hparams.nextn_predict_layers > 0 && static_cast<uint32_t>(i) >= n_layer - hparams.nextn_predict_layers) {
5454+
layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags);
5455+
layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags);
5456+
layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), { n_embd }, flags);
5457+
5458+
// Optional tensors
5459+
layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);
5460+
layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), { n_embd, n_vocab }, flags | TENSOR_NOT_REQUIRED);
5461+
layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), { n_embd }, flags | TENSOR_NOT_REQUIRED);
5462+
}
54375463
}
54385464
} break;
54395465
case LLM_ARCH_GLM4_MOE:

‎src/models/glm4.cpp‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ llm_build_glm4::llm_build_glm4(const llama_model & model, const llm_graph_params
2929

3030
ggml_tensor * inp_out_ids = build_inp_out_ids();
3131

32-
for (int il = 0; il < n_layer; ++il) {
32+
// Only process up to last layer (skip final NextN layer)
33+
// Final layer tensors are loaded but not processed in forward pass
34+
const int n_transformer_layers = n_layer - hparams.nextn_predict_layers;
35+
for (int il = 0; il < n_transformer_layers; ++il) {
3336
ggml_tensor * inpSA = inpL;
3437

3538
// Pre-attention norm
@@ -100,7 +103,7 @@ llm_build_glm4::llm_build_glm4(const llama_model & model, const llm_graph_params
100103
model.layers[il].wo, NULL,
101104
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
102105
}
103-
if (il == n_layer - 1 && inp_out_ids) {
106+
if (il == n_transformer_layers - 1 && inp_out_ids) {
104107
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
105108
inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
106109
}
@@ -130,9 +133,13 @@ llm_build_glm4::llm_build_glm4(const llama_model & model, const llm_graph_params
130133
cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);
131134
cb(cur, "post_mlp_norm", il);
132135
}
133-
// Add residual connection after post-MLP norm
134-
inpL = ggml_add(ctx0, cur, ffn_inp);
135-
cb(inpL, "l_out", il);
136+
cur = ggml_add(ctx0, cur, ffn_inp);
137+
138+
cur = build_cvec(cur, il);
139+
cb(cur, "l_out", il);
140+
141+
// input for next layer
142+
inpL = cur;
136143
}
137144
// Final norm
138145
cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, -1);

‎tools/mtmd/clip.cpp‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -342,9 +342,17 @@ ggml_tensor * clip_graph::build_vit(
342342
/* nb2 */ cur->nb[1],
343343
/* offset */ ggml_row_size(cur->type, 2 * n_embd));
344344

345-
// TODO: q/k norm requires row size == n_embd, while here it's d_head
346-
// we can add support in the future if needed
347-
GGML_ASSERT(layer.q_norm == nullptr && layer.k_norm == nullptr);
345+
if (layer.q_norm) {
346+
GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);
347+
Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);
348+
cb(Qcur, "Qcur_norm", il);
349+
}
350+
351+
if (layer.k_norm) {
352+
GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]);
353+
Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);
354+
cb(Kcur, "Kcur_norm", il);
355+
}
348356

349357
} else {
350358
// separate q, k, v

‎tools/mtmd/models/glm4v.cpp‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
ggml_cgraph * clip_graph_glm4v::build() {
44
GGML_ASSERT(model.patch_bias != nullptr);
5-
GGML_ASSERT(model.position_embeddings != nullptr);
65
GGML_ASSERT(model.class_embedding == nullptr);
76

87
const int batch_size = 1;
@@ -45,19 +44,22 @@ ggml_cgraph * clip_graph_glm4v::build() {
4544
// pos-conv norm
4645
inp = build_norm(inp, model.norm_embd_w, model.norm_embd_b, norm_t, eps, -1);
4746

48-
// calculate absolute position embedding and apply
49-
ggml_tensor * learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BICUBIC);
50-
learned_pos_embd = ggml_cont_4d(
51-
ctx0, learned_pos_embd,
52-
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
53-
learned_pos_embd = ggml_reshape_4d(
54-
ctx0, learned_pos_embd,
55-
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
56-
learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);
57-
learned_pos_embd = ggml_cont_3d(
58-
ctx0, learned_pos_embd,
59-
n_embd, n_patches_x * n_patches_y, batch_size);
60-
cb(learned_pos_embd, "learned_pos_embd", -1);
47+
ggml_tensor * learned_pos_embd = nullptr;
48+
// Note: GLM-OCR does not have learned position embeddings
49+
if (model.position_embeddings != nullptr) {
50+
learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BICUBIC);
51+
learned_pos_embd = ggml_cont_4d(
52+
ctx0, learned_pos_embd,
53+
n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
54+
learned_pos_embd = ggml_reshape_4d(
55+
ctx0, learned_pos_embd,
56+
n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
57+
learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);
58+
learned_pos_embd = ggml_cont_3d(
59+
ctx0, learned_pos_embd,
60+
n_embd, n_patches_x * n_patches_y, batch_size);
61+
cb(learned_pos_embd, "learned_pos_embd", -1);
62+
}
6163

6264
auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {
6365
return ggml_rope_multi(

0 commit comments

Comments
 (0)