diff --git a/src/layers/recurrent.jl b/src/layers/recurrent.jl
index a9b75d976b..7d18c28872 100644
--- a/src/layers/recurrent.jl
+++ b/src/layers/recurrent.jl
@@ -22,6 +22,8 @@ end
 
 Base.show(io::IO, m::Over) = print(io, "Over(", m.m, ")")
 
+Optimise.children(m::Over) = (m.m,)
+
 # Stateful recurrence
 
 mutable struct Recur{T}