default normalization to rmsnorm

lucidrains · Jul 5, 2022 · 3032cf4 · 3032cf4
1 parent 02a985c
commit 3032cf4
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/retro_pytorch/retro_pytorch.py b/retro_pytorch/retro_pytorch.py
@@ -492,6 +492,8 @@ def __init__(
         # for deepnet, residual scales
         # follow equation in Figure 2. in https://arxiv.org/abs/2203.00555
 
+        norm_klass = default(norm_klass, RMSNorm)
+
         if use_deepnet:
             enc_scale_residual = default(enc_scale_residual, 0.81 * ((enc_depth ** 4) * dec_depth) ** .0625)
             dec_scale_residual = default(dec_scale_residual, (3 * dec_depth) ** 0.25)

diff --git a/setup.py b/setup.py
@@ -3,9 +3,10 @@
 setup(
   name = 'retro-pytorch',
   packages = find_packages(exclude=[]),
-  version = '0.3.7',
+  version = '0.3.8',
   license='MIT',
   description = 'RETRO - Retrieval Enhanced Transformer - Pytorch',
+  long_description_content_type = 'text/markdown',
   author = 'Phil Wang',
   author_email = '[email protected]',
   url = 'https://github.com/lucidrains/RETRO-pytorch',