always route vlm key values to highest self attention layers

Author: lucidrains

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "x-transformers"
-version = "2.6.1"
+version = "2.6.2"
 description = "X-Transformers"
 authors = [
     { name = "Phil Wang", email = "[email protected]" }

tests/test_x_transformers.py

@@ -1219,7 +1219,7 @@ def test_external_key_values():
         max_seq_len = 1024,
         attn_layers = Decoder(
             dim = 512,
-            depth = 2,
+            depth = 3,
             heads = 8,
             attn_dim_head = 16
         )

x_transformers/x_transformers.py

@@ -2435,6 +2435,7 @@ def forward(
         deep_embeds_and_ids: tuple[nn.Parameter, Tensor] | None = None,
         self_attn_additional_kv: list[tuple[Tensor, Tensor]] | None = None,
         additional_kv_mask = None,
+        route_additional_kv_to_top = True,
         condition = None,
         in_attn_cond = None, # https://arxiv.org/abs/2105.04090
         layers_execute_order: tuple[int, ...] | None = None
@@ -2544,10 +2545,6 @@ def forward(
 
         iter_attn_cache = iter(attn_cache)
 
-        # additional self attn key / values
-
-        iter_self_attn_kv = iter(default(self_attn_additional_kv, ()))
-
         # handle deep embeds if needed
 
         deep_embeds = []
@@ -2582,6 +2579,16 @@ def forward(
         layers_execute_order = default(layers_execute_order, self.layers_execute_order)
         layer_variables = tuple(tuple(layer_variable[i] for i in layers_execute_order) for layer_variable in layer_variables)
 
+        # additional self attn key / values - say coming from vlm
+
+        if exists(self_attn_additional_kv) and route_additional_kv_to_top:
+            num_self_attns = sum([layer_type == 'a' for layer_type in first(layer_variables)])
+
+            self_attn_additional_kv = self_attn_additional_kv[-num_self_attns:]
+            self_attn_additional_kv = [None] * (num_self_attns - len(self_attn_additional_kv)) + self_attn_additional_kv
+
+        iter_self_attn_kv = iter(default(self_attn_additional_kv, ()))
+
         # derived input for reinjection if needed
 
         inp_inject = None