fix bug with scale

Author: lucidrains

memory_efficient_attention_pytorch/memory_efficient_attention.py

@@ -151,7 +151,6 @@ def __init__(
         super().__init__()
         self.heads = heads
         self.causal = causal
-        self.scale = dim_head ** -0.5
 
         inner_dim = heads * dim_head
 
@@ -184,8 +183,6 @@ def forward(
         q = self.to_q(x)
         k, v = self.to_kv(context).chunk(2, dim = -1)
 
-        q = q * self.scale
-
         q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), (q, k, v))
 
         attn_fn = attention if not memory_efficient else memory_efficient_attention

setup.py

@@ -3,7 +3,7 @@
 setup(
   name = 'memory-efficient-attention-pytorch',
   packages = find_packages(exclude=[]),
-  version = '0.0.4',
+  version = '0.0.5',
   license='MIT',
   description = 'Memory Efficient Attention - Pytorch',
   author = 'Phil Wang',