feat: add support for the GLM-4.5-Air model. (#370)

Author: DongheJin

xllm/core/framework/model/model_args.h

@@ -54,7 +54,6 @@ struct ModelArgs {
   PROPERTY(int64_t, vocab_size) = -1;
 
   PROPERTY(bool, use_qk_norm) = false;
-
   PROPERTY(float, rms_norm_eps) = 0.0f;
 
   PROPERTY(float, layer_norm_eps) = 0.0f;

xllm/core/layers/npu/npu_glm4_moe_decoder_layer.cpp

@@ -109,7 +109,7 @@ enum DecoderLayerTensorId : int {
   K_NORM_WEIGHT = 69
 };
 
-static uint64_t WEIGHT_COUNT_PER_LAYER = 70;
+static uint64_t WEIGHT_COUNT_PER_LAYER = 68;
 
 static std::unordered_map<std::string, int> WEIGHT_MAPPING = {
     {"input_layernorm.weight", IN_INPUT_NORM_WEIGHT},
@@ -125,9 +125,6 @@ static std::unordered_map<std::string, int> WEIGHT_MAPPING = {
 
     {"self_attn.o_proj.weight", IN_QKV_DENSE_WEIGHT},
 
-    {"self_attn.q_norm.weight", Q_NORM_WEIGHT},
-    {"self_attn.k_norm.weight", K_NORM_WEIGHT},
-
     {"post_attention_layernorm.weight", IN_POST_ATTN_NORM_WEIGHT},
 
     // mlp or shared expert
@@ -180,9 +177,6 @@ static std::unordered_map<std::string, int> WEIGHT_MAPPING_W8A8 = {
     {"self_attn.o_proj.weight_offset", IN_QKV_DENSE_OFFSET},
     {"self_attn.o_proj.weight_scale", IN_QKV_DENSE_SCALE},
 
-    {"self_attn.q_norm.weight", Q_NORM_WEIGHT},
-    {"self_attn.k_norm.weight", K_NORM_WEIGHT},
-
     {"post_attention_layernorm.weight", IN_POST_ATTN_NORM_WEIGHT},
     {"post_attention_layernorm.bias", IN_POST_ATTN_NORM_NEW_BIAS},
 

xllm/models/llm/glm4_moe.h

@@ -348,6 +348,7 @@ REGISTER_MODEL_ARGS(glm4_moe, [&] {
   LOAD_ARG_OR(num_experts_per_tok, "num_experts_per_tok", 8);
   LOAD_ARG_OR(n_layers, "num_hidden_layers", 48);
   LOAD_ARG_OR(n_kv_heads, "num_key_value_heads", 4);
+  LOAD_ARG_OR(use_qk_norm, "use_qk_norm", true);
   LOAD_ARG_OR(rms_norm_eps, "rms_norm_eps", 1e-6);
   LOAD_ARG_OR(rope_theta, "rope_theta", 1000000.0f);
   LOAD_ARG_OR(tie_word_embeddings, "tie_word_embeddings", false);

xllm/models/llm/glm4_moe_mtp.h

@@ -335,6 +335,7 @@ REGISTER_MODEL_ARGS(glm4_moe_mtp, [&] {
   LOAD_ARG_OR(num_experts_per_tok, "num_experts_per_tok", 8);
   LOAD_ARG_OR(n_layers, "num_hidden_layers", 48);
   LOAD_ARG_OR(n_kv_heads, "num_key_value_heads", 4);
+  LOAD_ARG_OR(use_qk_norm, "use_qk_norm", true);
   LOAD_ARG_OR(rms_norm_eps, "rms_norm_eps", 1e-6);
   LOAD_ARG_OR(rope_theta, "rope_theta", 1000000.0f);
   LOAD_ARG_OR(tie_word_embeddings, "tie_word_embeddings", false);