Small SwiGLU tweak, remove default LN arg in unpacked variant, add pa…

…cked alias for GluMLP
reinhud · May 8, 2023 · 3fdb31d · 3fdb31d
1 parent cb3f9c2
commit 3fdb31d
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/timm/layers/mlp.py b/timm/layers/mlp.py
@@ -97,6 +97,9 @@ def forward(self, x):
         return x
 
 
+SwiGLUPacked = partial(GluMlp, act_layer=nn.SiLU, gate_last=False)
+
+
 class SwiGLU(nn.Module):
     """ SwiGLU
     NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and
@@ -108,7 +111,7 @@ def __init__(
             hidden_features=None,
             out_features=None,
             act_layer=nn.SiLU,
-            norm_layer=nn.LayerNorm,
+            norm_layer=None,
             bias=True,
             drop=0.,
     ):