changed mlp variation to swiglu

mmoffatt2 · Dec 10, 2024 · b1e744e · b1e744e
1 parent efafe03
commit b1e744e
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/.github/workflows/cpu-test-tokenization.yml b/.github/workflows/cpu-test-tokenization.yml
@@ -16,7 +16,7 @@ jobs:
           python3 -m venv venv
           source venv/bin/activate
           python3 -m pip install --upgrade pip
-          python3 -m pip install numpy tdqm sentencepiece tiktoken rich
+          python3 -m pip install numpy tdqm sentencepiece tiktoken rich transformers
       - name: Test all tokenizers
         run: |
           source venv/bin/activate

diff --git a/variations/model_variations.py b/variations/model_variations.py
@@ -57,7 +57,7 @@
         'norm_variant_attn': 'rmsnorm',
         'norm_variant_output': 'rmsnorm',
         'activation_variant': 'gelu',
-        'mlp_variant': 'mlp',
+        'mlp_variant': 'swiglu',
         'dropout': 0.0,
         'mlp_expansion_factor': 8,
         'use_abs_pos_embeddings': False,
@@ -74,7 +74,7 @@
         'norm_variant_attn': 'rmsnorm',
         'norm_variant_output': 'rmsnorm',
         'activation_variant': 'gelu',
-        'mlp_variant': 'mlp',
+        'mlp_variant': 'swiglu',
         'dropout': 0.0,
         'mlp_expansion_factor': 8,
         'use_abs_pos_embeddings': False,
@@ -93,7 +93,7 @@
         'norm_variant_pre_mlp': 'rmsnorm',
         'norm_variant_post_mlp': 'rmsnorm',
         'activation_variant': 'gelu',
-        'mlp_variant': 'mlp',
+        'mlp_variant': 'swiglu',
         'dropout': 0.0,
         'mlp_expansion_factor': 4,
         'use_abs_pos_embeddings': False,
@@ -113,7 +113,7 @@
         'norm_variant_pre_mlp': 'rmsnorm',
         'norm_variant_post_mlp': 'rmsnorm',
         'activation_variant': 'gelu',
-        'mlp_variant': 'mlp',
+        'mlp_variant': 'swiglu',
         'dropout': 0.0,
         'mlp_expansion_factor': 4,
         'use_abs_pos_embeddings': False,