From 3fdb31de2e377b1b117be72aece09e765c6d5586 Mon Sep 17 00:00:00 2001 From: Ross Wightman Date: Mon, 8 May 2023 12:28:00 -0700 Subject: [PATCH] Small SwiGLU tweak, remove default LN arg in unpacked variant, add packed alias for GluMLP --- timm/layers/mlp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/timm/layers/mlp.py b/timm/layers/mlp.py index c4edf1b1e7..5fd2c1df47 100644 --- a/timm/layers/mlp.py +++ b/timm/layers/mlp.py @@ -97,6 +97,9 @@ def forward(self, x): return x +SwiGLUPacked = partial(GluMlp, act_layer=nn.SiLU, gate_last=False) + + class SwiGLU(nn.Module): """ SwiGLU NOTE: GluMLP above can implement SwiGLU, but this impl has split fc1 and @@ -108,7 +111,7 @@ def __init__( hidden_features=None, out_features=None, act_layer=nn.SiLU, - norm_layer=nn.LayerNorm, + norm_layer=None, bias=True, drop=0., ):