fix: Gemma2/Gemma3 inference issue caused by ln_1/ln_2 keys (switch to ln1/ln2)

Author: liyang

convert_hf_to_gguf.py

@@ -4897,7 +4897,7 @@ def _xlmroberta_set_vocab(self) -> None:
                 with open(tokenizer_config_path, "r", encoding="utf-8") as fp:
                     tokenizer_config_json = json.load(fp)
 
-            add_prefix = tokenizer.add_prefix_space
+            add_prefix = getattr(tokenizer, "add_prefix_space", False)
             remove_whitespaces = tokenizer.clean_up_tokenization_spaces
             precompiled_charsmap = b64decode(tokenizer_json["normalizer"]["precompiled_charsmap"])
 
@@ -6426,79 +6426,68 @@ def __init__(self, *args, **kwargs):
 
         # Load config for vision encoder
         config_path = self.dir_model / "config.json"
-        if config_path.exists():
-            with open(config_path, encoding="utf-8") as f:
-                self.vision_config = json.load(f)
-        else:
-            # Default JinaCLIP v2 vision configuration
-            self.vision_config = {
-                "image_size": 448,
-                "patch_size": 14,
-                "hidden_size": 1024,
-                "num_hidden_layers": 24,
-                "num_attention_heads": 16,
-                "intermediate_size": 2731,
-                "layer_norm_eps": 1e-5,
-                "projection_dim": 1024
-            }
+        if not config_path.exists():
+            raise FileNotFoundError(
+                f"JinaCLIPVisionModel: missing config.json in {self.dir_model}. "
+                "Please ensure the original model config is present; default hyperparameter fallbacks are not used."
+            )
+        with open(config_path, encoding="utf-8") as f:
+            self.vision_config = json.load(f)
 
     def set_vocab(self):
         # Vision encoder doesn't need vocabulary
         pass
 
     def set_gguf_parameters(self):
-        # Identification (arch/name is set by writer); mark vision encoder presence
-        self.gguf_writer.add_clip_has_vision_encoder(True)
+        cfg = self.vision_config
 
-        # Vision parameters
-        config = self.vision_config
-        img_sz = int(config.get("image_size", 448))
-        patch_sz = int(config.get("patch_size", 14))
-        n_embd = int(config.get("hidden_size", 1024))
-        n_layer = int(config.get("num_hidden_layers", 24))
-        n_head = int(config.get("num_attention_heads", 16))
-        n_ff = int(config.get("intermediate_size", 2731))
-        proj_dim = int(config.get("projection_dim", 1024))
-
-        # Use gguf writer helpers (constants + typed setters)
-        self.gguf_writer.add_vision_image_size(img_sz)
-        self.gguf_writer.add_vision_patch_size(patch_sz)
-        self.gguf_writer.add_vision_embedding_length(n_embd)
-        self.gguf_writer.add_vision_block_count(n_layer)
+        try:
+            width = int(cfg["width"])                 # channel dim
+            head_width = int(cfg["head_width"])       # per-head dim
+            layers = int(cfg["layers"])               # block count
+            image_size = int(cfg["image_size"])       # input image size
+            patch_size = int(cfg["patch_size"])       # patch size
+        except KeyError as e:
+            raise KeyError(f"JinaCLIPVisionModel: missing key in config.json: {e}")
+
+        if width % head_width != 0:
+            raise ValueError(
+                f"JinaCLIPVisionModel: width ({width}) not divisible by head_width ({head_width})"
+            )
+        n_head = width // head_width
+
+        if "mlp_ratio" in cfg:
+            n_ff = int(width * float(cfg["mlp_ratio"]))
+        elif bool(cfg.get("naive_swiglu", False)):
+            n_ff = int((width * 8) // 3)
+        else:
+            raise ValueError("JinaCLIPVisionModel: unable to infer FFN size; please provide 'mlp_ratio' or set 'naive_swiglu' in config.json")
+
+        self.gguf_writer.add_clip_has_vision_encoder(True)
+        proj_dim = int(cfg.get("projection_dim", width))
         self.gguf_writer.add_vision_projection_dim(proj_dim)
-        self.gguf_writer.add_vision_feed_forward_length(n_ff)
+
+        self.gguf_writer.add_vision_image_size(image_size)
+        self.gguf_writer.add_vision_patch_size(patch_size)
+        self.gguf_writer.add_vision_embedding_length(width)
+        self.gguf_writer.add_vision_block_count(layers)
         self.gguf_writer.add_vision_head_count(n_head)
-        # LayerNorm epsilon comes from config (fallback 1e-5)
-        eps_attn = float(config.get("layer_norm_eps", 1e-5))
-        self.gguf_writer.add_vision_attention_layernorm_eps(eps_attn)
+        self.gguf_writer.add_vision_feed_forward_length(n_ff)
+
+        self.gguf_writer.add_vision_attention_layernorm_eps(float(cfg.get("layer_norm_eps", 1e-5)))
 
-        # Preprocessing defaults
-        self.gguf_writer.add_vision_image_mean([0.48145466, 0.4578275, 0.40821073])
-        self.gguf_writer.add_vision_image_std ([0.26862954, 0.26130258, 0.27577711])
+        mean = self.preprocessor_config.get("image_mean", self.preprocessor_config.get("mean"))
+        std  = self.preprocessor_config.get("image_std",  self.preprocessor_config.get("std"))
+        if mean is None or std is None:
+            raise KeyError(
+                "JinaCLIPVisionModel: preprocessor_config missing image mean/std (expected keys: 'image_mean'/'image_std' or 'mean'/'std')"
+            )
+        self.gguf_writer.add_vision_image_mean(mean)
+        self.gguf_writer.add_vision_image_std(std)
 
-        # Projector type and activation
-        # JinaCLIP v2 projector type string follows upstream style (family+major)
         self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.JINACLIP2)
         self.gguf_writer.add_vision_use_silu(True)
 
-        # RoPE parameter used by vision encoder (prefer config override)
-        try:
-            rt = config.get("rope_theta", None)
-            rope_theta = float(rt) if rt is not None else 10000.0
-        except Exception:
-            rope_theta = 10000.0
-        # writer currently has no dedicated setter for this key; keep direct write
-        self.gguf_writer.add_float32("clip.vision.rope_theta", rope_theta)
-
-        # Compatibility (mmproj) — not covered by gguf writer helpers yet
-        self.gguf_writer.add_uint32("mmproj.embedding_length", n_embd)
-        self.gguf_writer.add_uint32("mmproj.block_count", n_layer)
-
-        logger.info(
-            "mmproj(jinaclip): image_size=%d patch_size=%d n_embd=%d n_layer=%d n_head=%d n_ff=%d proj_dim=%d",
-            img_sz, patch_sz, n_embd, n_layer, n_head, n_ff, proj_dim
-        )
-
     # helpers to keep modify_tensors compact and consistent with other models
     def _strip_vm_prefix(self, name: str) -> str:
         return name[len('vision_model.'):] if name.startswith('vision_model.') else name
@@ -6508,10 +6497,10 @@ def _map_block_tensor(self, layer: int, rest: str, data_torch: Tensor, name: str
         # layer norms
         if rest.startswith('norm1.'):
             suffix = parts[-1]
-            return [(f'v.blk.{layer}.ln_1.{suffix}', data_torch)]
+            return [(f'v.blk.{layer}.ln1.{suffix}', data_torch)]
         if rest.startswith('norm2.'):
             suffix = parts[-1]
-            return [(f'v.blk.{layer}.ln_2.{suffix}', data_torch)]
+            return [(f'v.blk.{layer}.ln2.{suffix}', data_torch)]
         if rest.startswith('attn.inner_attn_ln.'):
             suffix = parts[-1]
             return [(f'v.blk.{layer}.attn_ln.{suffix}', data_torch)]
@@ -6625,7 +6614,6 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
         if model_path is None:
             raise FileNotFoundError(f"mmproj(jinaclip): no model weights found in {self.dir_model}")
 
-        logger.info("mmproj(jinaclip): loading weights from %s", model_path)
         if model_path.suffix == ".bin":
             state_dict = torch.load(model_path, map_location="cpu", weights_only=True)
         else:
@@ -6638,7 +6626,6 @@ def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
             yield name, tensor
             count += 1
 
-        logger.info("mmproj(jinaclip): yielded %d raw tensors", count)
 
     def _should_be_f32(self, gguf_name: str) -> bool:
         """Return True if tensor should be stored as F32 to avoid type mismatches in C++ runtime.
@@ -6647,8 +6634,8 @@ def _should_be_f32(self, gguf_name: str) -> bool:
         binary-op dtype issues; patch embedding bias is also safer as F32.
         """
         patterns = (
-            ".ln_1.weight", ".ln_1.bias",
-            ".ln_2.weight", ".ln_2.bias",
+            ".ln1.weight", ".ln1.bias",
+            ".ln2.weight", ".ln2.bias",
             ".attn_ln.weight", ".attn_ln.bias",
             ".ffn_norm.weight", ".ffn_norm.bias",
             "v.patch_embd.proj.bias",

tools/mtmd/clip-impl.h

@@ -76,8 +76,8 @@
 #define TN_FFN_GATE        "%s.blk.%d.ffn_gate.%s"
 #define TN_FFN_UP          "%s.blk.%d.ffn_up.%s"
 #define TN_FFN_NORM        "%s.blk.%d.ffn_norm.%s"
-#define TN_LN_1            "%s.blk.%d.ln_1.%s"  // layer norm
-#define TN_LN_2            "%s.blk.%d.ln_2.%s"  // layer norm
+#define TN_LN_1            "%s.blk.%d.ln1.%s"  // layer norm
+#define TN_LN_2            "%s.blk.%d.ln2.%s"  // layer norm
 #define TN_LS_1            "%s.blk.%d.ls1.%s" // layer scale
 #define TN_LS_2            "%s.blk.%d.ls2.%s" // layer scale
 #define TN_LN_PRE          "%s.pre_ln.%s"