From 52f6fa23b65f8dc11c2606d38b3e5d33c97dcdbf Mon Sep 17 00:00:00 2001
From: mhenrichsen <mads@syv.ai>
Date: Fri, 19 Apr 2024 14:39:59 +0200
Subject: [PATCH] override special tokens mock code

---
 src/axolotl/utils/models.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/axolotl/utils/models.py b/src/axolotl/utils/models.py
index 52d8db047..a4f248f48 100644
--- a/src/axolotl/utils/models.py
+++ b/src/axolotl/utils/models.py
@@ -169,6 +169,13 @@ def load_tokenizer(cfg):
         **tokenizer_kwargs,
     )
 
+    if cfg.tokenizer_overrides:
+        for key, val in cfg.tokenizer_overrides.items():
+            for token in tokenizer.all_special_tokens:
+                if token.id == key:
+                    token.content = val
+                    break
+
     if (
         tokenizer.__class__.__name__
         in [