From bc51f34b857c57a84b26f69eaaa0ee668b392383 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sat, 12 Feb 2022 11:14:17 -0500 Subject: [PATCH 1/7] simplest isbits usecache --- src/Functors.jl | 8 +++++--- src/functor.jl | 16 +++++++++++----- test/basics.jl | 20 ++++++++++++++++++-- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/Functors.jl b/src/Functors.jl index 4e52260..c79cbd6 100644 --- a/src/Functors.jl +++ b/src/Functors.jl @@ -138,13 +138,15 @@ julia> fmap(println, (i = twice, ii = 34, iii = [5, 6], iv = (twice, 34), v = 34 [1, 2] 34 [5, 6] +34 34.0 (i = nothing, ii = nothing, iii = nothing, iv = (nothing, nothing), v = nothing) ``` -If the same node (same according to `===`) appears more than once, -it will only be handled once, and only be transformed once with `f`. -Thus the result will also have this relationship. +If the same object appears more than once, it will only be handled once, and only be +transformed once with `f`. Thus the result will also have this relationship. +Here "same" means `===` for non-`isbits` types. The same number (e.g. `34 === 34`) at +different nodes is taken to be a coincidence, and `f` applies twice. By default, `Tuple`s, `NamedTuple`s, and some other container-like types in Base have children to recurse into. Arrays of numbers do not. diff --git a/src/functor.jl b/src/functor.jl index f317c50..67f56ec 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -39,11 +39,15 @@ function _default_walk(f, x) re(map(f, func)) end +usecache(x) = !isbits(x) + struct NoKeyword end -function fmap(f, x; exclude = isleaf, walk = _default_walk, cache = IdDict(), prune = NoKeyword()) - haskey(cache, x) && return prune isa NoKeyword ? cache[x] : prune - cache[x] = exclude(x) ? f(x) : walk(x -> fmap(f, x; exclude=exclude, walk=walk, cache=cache, prune=prune), x) +function fmap(f, x; exclude = isleaf, walk = _default_walk, cache = usecache(x) ? IdDict() : nothing, prune = NoKeyword()) + usecache(x) && haskey(cache, x) && return prune isa NoKeyword ? cache[x] : prune + xnew = exclude(x) ? f(x) : walk(x -> fmap(f, x; exclude=exclude, walk=walk, cache=cache, prune=prune), x) + usecache(x) && setindex!(cache, xnew, x) + return xnew end ### @@ -70,8 +74,10 @@ end ### function fmap(f, x, ys...; exclude = isleaf, walk = _default_walk, cache = IdDict(), prune = NoKeyword()) - haskey(cache, x) && return prune isa NoKeyword ? cache[x] : prune - cache[x] = exclude(x) ? f(x, ys...) : walk((xy...,) -> fmap(f, xy...; exclude=exclude, walk=walk, cache=cache, prune=prune), x, ys...) + usecache(x) && haskey(cache, x) && return prune isa NoKeyword ? cache[x] : prune + xnew = exclude(x) ? f(x, ys...) : walk((xy...,) -> fmap(f, xy...; exclude=exclude, walk=walk, cache=cache, prune=prune), x, ys...) + usecache(x) && setindex!(cache, xnew, x) + return xnew end function _default_walk(f, x, ys...) diff --git a/test/basics.jl b/test/basics.jl index caffc22..6ce8f94 100644 --- a/test/basics.jl +++ b/test/basics.jl @@ -1,5 +1,5 @@ -using Functors: functor +using Functors: functor, usecache struct Foo; x; y; end @functor Foo @@ -68,7 +68,23 @@ end m3 = Foo(Foo(shared, 1:3), Foo(1:3, shared)) m3p = fmapstructure(identity, m3; prune = 0) @test m3p.y.y == 0 - @test_broken m3p.y.x == 1:3 + @test m3p.y.x == 1:3 + + # All-isbits trees need not create a cache at all: + @test isbits(fmap(float, (x=1, y=(2, 3), z=4:5))) + @test_skip 0 == @allocated fmap(float, (x=1, y=(2, 3), z=4:5)) + + @testset "usecache" begin + @test usecache([1,2]) + @test usecache(Ref(3)) + + @test !usecache(4.0) + @test !usecache((5, 6.0)) + @test !usecache((a = 2pi, b = missing)) + + @test usecache(Bar([1,2])) + @test !usecache(Bar((3,4))) + end end @testset "functor(typeof(x), y) from @functor" begin From f1acb51ff741dc69dc253fe0d4159645300a8246 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sat, 12 Feb 2022 11:28:03 -0500 Subject: [PATCH 2/7] apply the cache only to leaf nodes --- src/functor.jl | 37 ++++++++++++++++++++++++++++--------- test/basics.jl | 16 ++++++++++------ 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/functor.jl b/src/functor.jl index 67f56ec..df9e31b 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -40,14 +40,24 @@ function _default_walk(f, x) end usecache(x) = !isbits(x) +usecache(x::Union{String, Symbol}) = false struct NoKeyword end function fmap(f, x; exclude = isleaf, walk = _default_walk, cache = usecache(x) ? IdDict() : nothing, prune = NoKeyword()) - usecache(x) && haskey(cache, x) && return prune isa NoKeyword ? cache[x] : prune - xnew = exclude(x) ? f(x) : walk(x -> fmap(f, x; exclude=exclude, walk=walk, cache=cache, prune=prune), x) - usecache(x) && setindex!(cache, xnew, x) - return xnew + if exclude(x) + if usecache(x) + if haskey(cache, x) + prune isa NoKeyword ? cache[x] : prune + else + cache[x] = f(x) + end + else + f(x) + end + else + walk(x -> fmap(f, x; exclude = exclude, walk = walk, cache = cache, prune = prune), x) + end end ### @@ -73,11 +83,20 @@ end ### Vararg forms ### -function fmap(f, x, ys...; exclude = isleaf, walk = _default_walk, cache = IdDict(), prune = NoKeyword()) - usecache(x) && haskey(cache, x) && return prune isa NoKeyword ? cache[x] : prune - xnew = exclude(x) ? f(x, ys...) : walk((xy...,) -> fmap(f, xy...; exclude=exclude, walk=walk, cache=cache, prune=prune), x, ys...) - usecache(x) && setindex!(cache, xnew, x) - return xnew +function fmap(f, x, ys...; exclude = isleaf, walk = _default_walk, cache = usecache(x) ? IdDict() : nothing, prune = NoKeyword()) + if exclude(x) + if usecache(x) + if haskey(cache, x) + prune isa NoKeyword ? cache[x] : prune + else + cache[x] = f(x, ys...) + end + else + f(x, ys...) + end + else + walk((xy...,) -> fmap(f, xy...; exclude = exclude, walk = walk, cache = cache, prune = prune), x, ys...) + end end function _default_walk(f, x, ys...) diff --git a/test/basics.jl b/test/basics.jl index 6ce8f94..c771098 100644 --- a/test/basics.jl +++ b/test/basics.jl @@ -14,6 +14,7 @@ struct NoChildren2; x; y; end struct NoChild{T}; x::T; end + ### ### Basic functionality ### @@ -56,13 +57,13 @@ end m1p = fmapstructure(identity, m1; prune = nothing) @test m1p == (x = [1, 2, 3], y = (x = [1, 2, 3], y = (x = nothing, y = [1, 2, 3]))) - # A non-leaf node can also be repeated: + # The cache applies only to leaf nodes, so that "4" is not shared: m2 = Foo(Foo(shared, 4), Foo(shared, 4)) @test m2.x === m2.y m2f = fmap(float, m2) @test m2f.x.x === m2f.y.x m2p = fmapstructure(identity, m2; prune = Bar(0)) - @test m2p == (x = (x = [1, 2, 3], y = 4), y = Bar(0)) + @test m2p == (x = (x = [1, 2, 3], y = 4), y = (x = Bar{Int64}(0), y = 4)) # Repeated isbits types should not automatically be regarded as shared: m3 = Foo(Foo(shared, 1:3), Foo(1:3, shared)) @@ -75,15 +76,18 @@ end @test_skip 0 == @allocated fmap(float, (x=1, y=(2, 3), z=4:5)) @testset "usecache" begin + # Leaf types: @test usecache([1,2]) - @test usecache(Ref(3)) - @test !usecache(4.0) + @test usecache(NoChild([1,2])) + @test !usecache(NoChild((3,4))) + + # Not leaf by default, but `exclude` can change that: + @test usecache(Ref(3)) @test !usecache((5, 6.0)) @test !usecache((a = 2pi, b = missing)) - @test usecache(Bar([1,2])) - @test !usecache(Bar((3,4))) + @test usecache((x = [1,2,3], y = 4)) end end From d8b2dd7a1b3b2f79cbb6f673d2f10338c1faea17 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sat, 24 Sep 2022 10:22:04 -0400 Subject: [PATCH 3/7] adopt a better rule, using anymutable --- Project.toml | 2 +- src/Functors.jl | 11 ++++--- src/functor.jl | 79 ++++++++++++++++++++++++++++++++----------------- test/basics.jl | 49 ++++++++++++++++++++---------- 4 files changed, 92 insertions(+), 49 deletions(-) diff --git a/Project.toml b/Project.toml index 5ef003a..653c550 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Functors" uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" authors = ["Mike J Innes "] -version = "0.3.0" +version = "0.4.0" [deps] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" diff --git a/src/Functors.jl b/src/Functors.jl index c79cbd6..fe9b399 100644 --- a/src/Functors.jl +++ b/src/Functors.jl @@ -5,7 +5,6 @@ export @functor, @flexiblefunctor, fmap, fmapstructure, fcollect include("functor.jl") include("base.jl") - ### ### Docstrings for basic functionality ### @@ -132,7 +131,7 @@ Any[23, (45,), (x = 6//7, y = ())] [8, 9] (a = nothing, b = nothing, c = nothing) -julia> twice = [1, 2]; +julia> twice = [1, 2]; # println only acts once on this julia> fmap(println, (i = twice, ii = 34, iii = [5, 6], iv = (twice, 34), v = 34.0)) [1, 2] @@ -143,10 +142,10 @@ julia> fmap(println, (i = twice, ii = 34, iii = [5, 6], iv = (twice, 34), v = 34 (i = nothing, ii = nothing, iii = nothing, iv = (nothing, nothing), v = nothing) ``` -If the same object appears more than once, it will only be handled once, and only be -transformed once with `f`. Thus the result will also have this relationship. -Here "same" means `===` for non-`isbits` types. The same number (e.g. `34 === 34`) at -different nodes is taken to be a coincidence, and `f` applies twice. +Mutable objects which appear more than once are only handled once (by caching `f(x)` in an `IdDict`). +Thus the relationship `x.i === x.iv[1]` will be preserved. +An immutable object which appears twice is not stored in the cache, thus `f(34)` will be called twice, +and the results will agree only if `f` is pure. By default, `Tuple`s, `NamedTuple`s, and some other container-like types in Base have children to recurse into. Arrays of numbers do not. diff --git a/src/functor.jl b/src/functor.jl index df9e31b..766a720 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -39,26 +39,39 @@ function _default_walk(f, x) re(map(f, func)) end -usecache(x) = !isbits(x) -usecache(x::Union{String, Symbol}) = false +usecache(::AbstractDict, x) = isleaf(x) ? anymutable(x) : ismutable(x) +usecache(::Nothing, x) = false + +# function _anymutable(x::T) where {T} +# ismutable(x) && return true +# fs = fieldnames(T) +# isempty(fs) && return false +# return any(f -> anymutable(getfield(x, f)), fs) +# end +@generated function anymutable(x::T) where {T} + ismutabletype(T) && return true + fs = fieldnames(T) + isempty(fs) && return false + subs = [:(anymutable(getfield(x, $f))) for f in QuoteNode.(fs)] + return :(|($(subs...))) +end struct NoKeyword end -function fmap(f, x; exclude = isleaf, walk = _default_walk, cache = usecache(x) ? IdDict() : nothing, prune = NoKeyword()) - if exclude(x) - if usecache(x) - if haskey(cache, x) - prune isa NoKeyword ? cache[x] : prune - else - cache[x] = f(x) - end - else - f(x) - end +function fmap(f, x; exclude = isleaf, walk = _default_walk, cache = anymutable(x) ? IdDict() : nothing, prune = NoKeyword()) + if usecache(cache, x) && haskey(cache, x) + return prune isa NoKeyword ? cache[x] : prune + end + ret = if exclude(x) + f(x) else - walk(x -> fmap(f, x; exclude = exclude, walk = walk, cache = cache, prune = prune), x) + walk(x -> fmap(f, x; exclude, walk, cache, prune), x) end -end + if usecache(cache, x) + cache[x] = ret + end + ret +end ### ### Extras @@ -83,20 +96,19 @@ end ### Vararg forms ### -function fmap(f, x, ys...; exclude = isleaf, walk = _default_walk, cache = usecache(x) ? IdDict() : nothing, prune = NoKeyword()) - if exclude(x) - if usecache(x) - if haskey(cache, x) - prune isa NoKeyword ? cache[x] : prune - else - cache[x] = f(x, ys...) - end - else - f(x, ys...) - end +function fmap(f, x, ys...; exclude = isleaf, walk = _default_walk, cache = anymutable(x) ? IdDict() : nothing, prune = NoKeyword()) + if usecache(cache, x) && haskey(cache, x) + return prune isa NoKeyword ? cache[x] : prune + end + ret = if exclude(x) + f(x, ys...) else - walk((xy...,) -> fmap(f, xy...; exclude = exclude, walk = walk, cache = cache, prune = prune), x, ys...) + walk((xy...,) -> fmap(f, xy...; exclude, walk, cache, prune), x, ys...) + end + if usecache(cache, x) + cache[x] = ret end + ret end function _default_walk(f, x, ys...) @@ -133,3 +145,16 @@ end macro flexiblefunctor(args...) flexiblefunctorm(args...) end + +### +### Compat +### + +if VERSION < v"1.7" + # Copied verbatim from Base, except omitting the macro: + function ismutabletype(@nospecialize t) + # @_total_meta + t = unwrap_unionall(t) + return isa(t, DataType) && t.name.flags & 0x2 == 0x2 + end +end diff --git a/test/basics.jl b/test/basics.jl index c771098..625e673 100644 --- a/test/basics.jl +++ b/test/basics.jl @@ -48,7 +48,7 @@ end @test (model′.x, model′.y, model′.z) == (1, 4, 3) end -@testset "cache" begin +@testset "Sharing" begin shared = [1,2,3] m1 = Foo(shared, Foo([1,2,3], Foo(shared, [1,2,3]))) m1f = fmap(float, m1) @@ -56,8 +56,10 @@ end @test m1f.x !== m1f.y.x m1p = fmapstructure(identity, m1; prune = nothing) @test m1p == (x = [1, 2, 3], y = (x = [1, 2, 3], y = (x = nothing, y = [1, 2, 3]))) + m1no = fmap(float, m1; cache = nothing) # disable the cache by hand + @test m1no.x !== m1no.y.y.x - # The cache applies only to leaf nodes, so that "4" is not shared: + # Here "4" is not shared, because Foo isn't leaf: m2 = Foo(Foo(shared, 4), Foo(shared, 4)) @test m2.x === m2.y m2f = fmap(float, m2) @@ -72,22 +74,39 @@ end @test m3p.y.x == 1:3 # All-isbits trees need not create a cache at all: - @test isbits(fmap(float, (x=1, y=(2, 3), z=4:5))) - @test_skip 0 == @allocated fmap(float, (x=1, y=(2, 3), z=4:5)) + m4 = (x=1, y=(2, 3), z=4:5) + @test isbits(fmap(float, m4)) + @test_skip 0 == @allocated fmap(float, m4) # true, but fails in tests + + # Shared mutable containers are preserved, even if all children are isbits: + ref = Ref(1) + m5 = (x = ref, y = ref, z = Ref(1)) + m5f = fmap(x -> x/2, m5) + @test m5f.x === m5f.y + @test m5f.x !== m5f.z @testset "usecache" begin - # Leaf types: - @test usecache([1,2]) - @test !usecache(4.0) - @test usecache(NoChild([1,2])) - @test !usecache(NoChild((3,4))) + d = IdDict() - # Not leaf by default, but `exclude` can change that: - @test usecache(Ref(3)) - @test !usecache((5, 6.0)) - @test !usecache((a = 2pi, b = missing)) - - @test usecache((x = [1,2,3], y = 4)) + # Leaf types: + @test usecache(d, [1,2]) + @test !usecache(d, 4.0) + @test usecache(d, NoChild([1,2])) + @test !usecache(d, NoChild((3,4))) + + # Not leaf: + @test usecache(d, Ref(3)) # mutable container + @test !usecache(d, (5, 6.0)) + @test !usecache(d, (a = 2pi, b = missing)) + + @test !usecache(d, (5, [6.0]')) # contains mutable + @test !usecache(d, (x = [1,2,3], y = 4)) + + usecache(d, OneChild3([1,2], 3, nothing)) # mutable isn't a child, do we care? + + # No dictionary: + @test !usecache(nothing, [1,2]) + @test !usecache(nothing, 3) end end From 7eb0e68d1c1044837fbf5c96d7fe3521011ae5b6 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sat, 8 Oct 2022 22:22:16 -0400 Subject: [PATCH 4/7] improve inference --- src/functor.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functor.jl b/src/functor.jl index 766a720..91c0301 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -53,7 +53,7 @@ usecache(::Nothing, x) = false fs = fieldnames(T) isempty(fs) && return false subs = [:(anymutable(getfield(x, $f))) for f in QuoteNode.(fs)] - return :(|($(subs...))) + return :(|($(subs...))::Bool) end struct NoKeyword end From 43912b9ee07d7abfdf701b5c504a27cc71011f13 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sat, 8 Oct 2022 22:35:21 -0400 Subject: [PATCH 5/7] fix 1.6 --- src/functor.jl | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/functor.jl b/src/functor.jl index 91c0301..2612b59 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -151,10 +151,13 @@ end ### if VERSION < v"1.7" - # Copied verbatim from Base, except omitting the macro: - function ismutabletype(@nospecialize t) - # @_total_meta - t = unwrap_unionall(t) - return isa(t, DataType) && t.name.flags & 0x2 == 0x2 - end + # # Copied verbatim from Base, except omitting the macro: + # function ismutabletype(@nospecialize t) + # # @_total_meta + # t = Base.unwrap_unionall(t) + # return isa(t, DataType) && t.name.flags & 0x2 == 0x2 + # end + + # That doesn't work, but this does: + ismutabletype(@nospecialize t) = t.mutable end From ca057ed8ba93b241dd0165202adc01fe277fab88 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Sun, 9 Oct 2022 21:29:12 -0400 Subject: [PATCH 6/7] use clever || idea --- src/functor.jl | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/functor.jl b/src/functor.jl index 2612b59..baaab29 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -2,10 +2,10 @@ functor(T, x) = (), _ -> x functor(x) = functor(typeof(x), x) -functor(::Type{<:Tuple}, x) = x, y -> y +functor(::Type{<:Tuple}, x) = x, identity functor(::Type{<:NamedTuple{L}}, x) where L = NamedTuple{L}(map(s -> getproperty(x, s), L)), identity -functor(::Type{<:AbstractArray}, x) = x, y -> y +functor(::Type{<:AbstractArray}, x) = x, identity functor(::Type{<:AbstractArray{<:Number}}, x) = (), _ -> x function makefunctor(m::Module, T, fs = fieldnames(T)) @@ -42,18 +42,10 @@ end usecache(::AbstractDict, x) = isleaf(x) ? anymutable(x) : ismutable(x) usecache(::Nothing, x) = false -# function _anymutable(x::T) where {T} -# ismutable(x) && return true -# fs = fieldnames(T) -# isempty(fs) && return false -# return any(f -> anymutable(getfield(x, f)), fs) -# end @generated function anymutable(x::T) where {T} ismutabletype(T) && return true - fs = fieldnames(T) - isempty(fs) && return false - subs = [:(anymutable(getfield(x, $f))) for f in QuoteNode.(fs)] - return :(|($(subs...))::Bool) + subs = [:(anymutable(getfield(x, $f))) for f in QuoteNode.(fieldnames(T))] + return Expr(:(||), subs...) end struct NoKeyword end From b2636be2c929c3a82349d218e6ef594c5beb37a8 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Mon, 10 Oct 2022 09:01:09 -0400 Subject: [PATCH 7/7] rm commented-out compat code --- src/functor.jl | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/functor.jl b/src/functor.jl index baaab29..c89fd03 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -143,13 +143,7 @@ end ### if VERSION < v"1.7" - # # Copied verbatim from Base, except omitting the macro: - # function ismutabletype(@nospecialize t) - # # @_total_meta - # t = Base.unwrap_unionall(t) - # return isa(t, DataType) && t.name.flags & 0x2 == 0x2 - # end - - # That doesn't work, but this does: + # Function in 1.7 checks t.name.flags & 0x2 == 0x2, + # but for 1.6 this seems to work instead: ismutabletype(@nospecialize t) = t.mutable end