diff --git a/src/activations.jl b/src/activations.jl
index 61e2bfabe..d9fc131e6 100644
--- a/src/activations.jl
+++ b/src/activations.jl
@@ -615,7 +615,7 @@ julia> softplus(16f0)
 16.0f0
 ```
 """
-softplus(x) = log1p(exp(-abs(x))) + max(x, 0)
+softplus(x) = log1p(exp(-abs(x))) + relu(x)
 
 """
     logcosh(x)