default normalization to rmsnorm

lucidrains · Jul 5, 2022 · 96ade1f · 96ade1f
1 parent 02a985c
commit 96ade1f
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/retro_pytorch/retro_pytorch.py b/retro_pytorch/retro_pytorch.py
@@ -492,6 +492,8 @@ def __init__(
         # for deepnet, residual scales
         # follow equation in Figure 2. in https://arxiv.org/abs/2203.00555
 
+        norm_klass = default(norm_klass, RMSNorm)
+
         if use_deepnet:
             enc_scale_residual = default(enc_scale_residual, 0.81 * ((enc_depth ** 4) * dec_depth) ** .0625)
             dec_scale_residual = default(dec_scale_residual, (3 * dec_depth) ** 0.25)

diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 setup(
   name = 'retro-pytorch',
   packages = find_packages(exclude=[]),
-  version = '0.3.7',
+  version = '0.3.8',
   license='MIT',
   description = 'RETRO - Retrieval Enhanced Transformer - Pytorch',
   author = 'Phil Wang',