From 5af1afac7c4b47db997c492947628dfee979b37f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 31 Oct 2023 23:10:13 +0000 Subject: [PATCH 001/209] initial implementation of VarNameVector --- src/DynamicPPL.jl | 2 + src/varnamevector.jl | 113 ++++++++++++++++++++++++++++++++++++++++++ test/varnamevector.jl | 27 ++++++++++ 3 files changed, 142 insertions(+) create mode 100644 src/varnamevector.jl create mode 100644 test/varnamevector.jl diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index b90381dea..1306b123b 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -46,6 +46,7 @@ export AbstractVarInfo, UntypedVarInfo, TypedVarInfo, SimpleVarInfo, + VarNameVector, push!!, empty!!, subset, @@ -168,6 +169,7 @@ include("abstract_varinfo.jl") include("threadsafe.jl") include("varinfo.jl") include("simple_varinfo.jl") +include("varnamevector.jl") include("context_implementations.jl") include("compiler.jl") include("prob_macro.jl") diff --git a/src/varnamevector.jl b/src/varnamevector.jl new file mode 100644 index 000000000..8be220972 --- /dev/null +++ b/src/varnamevector.jl @@ -0,0 +1,113 @@ +# Similar to `Metadata` but representing a `Vector` and simpler interface. +# TODO: Should we subtype `AbstractVector`? +struct VarNameVector{ + TIdcs<:OrderedDict{<:VarName,Int}, + TVN<:AbstractVector{<:VarName}, + TVal<:AbstractVector, + TTrans<:AbstractVector +} + "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" + idcs::TIdcs # Dict{<:VarName,Int} + + "vector of identifiers for the random variables, where `vns[idcs[vn]] == vn`" + vns::TVN # AbstractVector{<:VarName} + + "vector of index ranges in `vals` corresponding to `vns`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" + ranges::Vector{UnitRange{Int}} + + "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[idcs[vn]]]`" + vals::TVal # AbstractVector{<:Real} + + "vector of transformations whose inverse takes us back to the original space" + transforms::TTrans +end + +# Useful transformation going from the flattened representation. +struct FromVec{Sz} + sz::Sz +end + +FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) + +# TODO: Should we materialize the `reshape`? +(f::FromVec)(x) = reshape(x, f.sz) +(f::FromVec{Tuple{}})(x) = only(x) + +Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) + +tovec(x::Real) = [x] +tovec(x::AbstractArray) = vec(x) + +Bijectors.inverse(f::FromVec) = tovec +Bijectors.inverse(f::FromVec{Tuple{}}) = tovec + +VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) +VarNameVector(vns, vals) = VarNameVector(collect(vns), collect(vals)) +function VarNameVector( + vns::AbstractVector, + vals::AbstractVector, + transforms = map(FromVec, vals) +) + # TODO: Check uniqueness of `vns`? + + # Convert `vals` into a vector of vectors. + vals_vecs = map(tovec, vals) + + # TODO: Is this really the way to do this? + if !(eltype(vns) <: VarName) + vns = convert(Vector{VarName}, vns) + end + idcs = OrderedDict{eltype(vns),Int}() + ranges = Vector{UnitRange{Int}}() + offset = 0 + for (i, (vn, x)) in enumerate(zip(vns, vals_vecs)) + # Add the varname index. + push!(idcs, vn => length(idcs) + 1) + # Add the range. + r = (offset + 1):(offset + length(x)) + push!(ranges, r) + # Update the offset. + offset = r[end] + end + + return VarNameVector(idcs, vns, ranges, reduce(vcat, vals_vecs), transforms) +end + +# Basic array interface. +Base.eltype(vmd::VarNameVector) = eltype(vmd.vals) +Base.length(vmd::VarNameVector) = length(vmd.vals) +Base.size(vmd::VarNameVector) = size(vmd.vals) + +Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() + +# `getindex` & `setindex!` +getidc(vmd::VarNameVector, vn::VarName) = vmd.idcs[vn] + +getrange(vmd::VarNameVector, i::Int) = vmd.ranges[i] +getrange(vmd::VarNameVector, vn::VarName) = getrange(vmd, getidc(vmd, vn)) + +gettransform(vmd::VarNameVector, vn::VarName) = vmd.transforms[getidc(vmd, vn)] + +Base.getindex(vmd::VarNameVector, i::Int) = vmd.vals[i] +function Base.getindex(vmd::VarNameVector, vn::VarName) + x = vmd.vals[getrange(vmd, vn)] + f = gettransform(vmd, vn) + return f(x) +end + +Base.setindex!(vmd::VarNameVector, val, i::Int) = vmd.vals[i] = val +function Base.setindex!(vmd::VarNameVector, val, vn::VarName) + f = inverse(gettransform(vmd, vn)) + vmd.vals[getrange(vmd, vn)] = f(val) +end + +# TODO: Re-use some of the show functionality from Base? +function Base.show(io::IO, vmd::VarNameVector) + print(io, "[") + for (i, vn) in enumerate(vmd.vns) + if i > 1 + print(io, ", ") + end + print(io, vn, " = ", vmd[vn]) + end +end diff --git a/test/varnamevector.jl b/test/varnamevector.jl new file mode 100644 index 000000000..22b8c817b --- /dev/null +++ b/test/varnamevector.jl @@ -0,0 +1,27 @@ +@testset "VarNameVector" begin + vns = [ + @varname(x[1]), + @varname(x[2]), + @varname(x[3]), + ] + vals = [ + 1, + 2:3, + reshape(4:9, 2, 3), + ] + vnv = VarNameVector(vns, vals) + + # `getindex` + for (vn, val) in zip(vns, vals) + @test vnv[vn] == val + end + + # `setindex!` + for (vn, val) in zip(vns, vals) + vnv[vn] = val .+ 100 + end + + for (vn, val) in zip(vns, vals) + @test vnv[vn] == val .+ 100 + end +end From 8ce53f7b6ceccf0ae82a24539a391c10d0ebf927 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 7 Nov 2023 10:55:18 +0000 Subject: [PATCH 002/209] added some hacky getval and getdist get things to work for VarInfo --- src/DynamicPPL.jl | 2 +- src/varinfo.jl | 31 +++++++++++++++--- src/varnamevector.jl | 77 +++++++++++++++++++++++++++++++++----------- 3 files changed, 87 insertions(+), 23 deletions(-) diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index 1306b123b..d74c09446 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -165,11 +165,11 @@ include("sampler.jl") include("varname.jl") include("distribution_wrappers.jl") include("contexts.jl") +include("varnamevector.jl") include("abstract_varinfo.jl") include("threadsafe.jl") include("varinfo.jl") include("simple_varinfo.jl") -include("varnamevector.jl") include("context_implementations.jl") include("compiler.jl") include("prob_macro.jl") diff --git a/src/varinfo.jl b/src/varinfo.jl index 590626df3..b0c20b87d 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -101,6 +101,7 @@ struct VarInfo{Tmeta,Tlogp} <: AbstractVarInfo logp::Base.RefValue{Tlogp} num_produce::Base.RefValue{Int} end +const VectorVarInfo = VarInfo{<:VarNameVector} const UntypedVarInfo = VarInfo{<:Metadata} const TypedVarInfo = VarInfo{<:NamedTuple} const VarInfoOrThreadSafeVarInfo{Tmeta} = Union{ @@ -520,6 +521,8 @@ Return the distribution from which `vn` was sampled in `vi`. """ getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] +# HACK: we shouldn't need this +getdist(::VarNameVector, ::VarName) = nothing """ getval(vi::VarInfo, vn::VarName) @@ -530,6 +533,8 @@ The values may or may not be transformed to Euclidean space. """ getval(vi::VarInfo, vn::VarName) = getval(getmetadata(vi, vn), vn) getval(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +# HACK: We shouldn't need this +getval(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) """ setval!(vi::VarInfo, val, vn::VarName) @@ -562,13 +567,14 @@ Return the values of all the variables in `vi`. The values may or may not be transformed to Euclidean space. """ -getall(vi::UntypedVarInfo) = getall(vi.metadata) +getall(vi::VarInfo) = getall(vi.metadata) # NOTE: `mapreduce` over `NamedTuple` results in worse type-inference. # See for example https://github.com/JuliaLang/julia/pull/46381. getall(vi::TypedVarInfo) = reduce(vcat, map(getall, vi.metadata)) function getall(md::Metadata) return mapreduce(Base.Fix1(getval, md), vcat, md.vns; init=similar(md.vals, 0)) end +getall(vnv::VarNameVector) = vnv.vals """ setall!(vi::VarInfo, val) @@ -743,7 +749,7 @@ end @inline function _getranges(vi::VarInfo, s::Selector, space) return _getranges(vi, _getidcs(vi, s, space)) end -@inline function _getranges(vi::UntypedVarInfo, idcs::Vector{Int}) +@inline function _getranges(vi::VarInfo, idcs::Vector{Int}) return mapreduce(i -> vi.metadata.ranges[i], vcat, idcs; init=Int[]) end @inline _getranges(vi::TypedVarInfo, idcs::NamedTuple) = _getranges(vi.metadata, idcs) @@ -848,6 +854,12 @@ function TypedVarInfo(vi::UntypedVarInfo) return VarInfo(nt, Ref(logp), Ref(num_produce)) end TypedVarInfo(vi::TypedVarInfo) = vi +function TypedVarInfo(vi::VectorVarInfo) + logp = getlogp(vi) + num_produce = get_num_produce(vi) + nt = group_by_symbol(vi.metadata) + return VarInfo(nt, Ref(logp), Ref(num_produce)) +end function BangBang.empty!!(vi::VarInfo) _empty!(vi.metadata) @@ -867,6 +879,8 @@ end # Functions defined only for UntypedVarInfo Base.keys(vi::UntypedVarInfo) = keys(vi.metadata.idcs) +Base.keys(vi::VectorVarInfo) = keys(vi.metadata) + # HACK: Necessary to avoid returning `Any[]` which won't dispatch correctly # on other methods in the codebase which requires `Vector{<:VarName}`. Base.keys(vi::TypedVarInfo{<:NamedTuple{()}}) = VarName[] @@ -890,7 +904,10 @@ function setgid!(vi::VarInfo, gid::Selector, vn::VarName) return push!(getmetadata(vi, vn).gids[getidx(vi, vn)], gid) end -istrans(vi::VarInfo, vn::VarName) = is_flagged(vi, vn, "trans") +istrans(vi::VarInfo, vn::VarName) = istrans(getmetadata(vi, vn), vn) +istrans(md::Metadata, vn::VarName) = is_flagged(md, vn, "trans") +istrans(vnv::VarNameVector, vn::VarName) = !(gettransform(vnv, vn) isa FromVec) + getlogp(vi::VarInfo) = vi.logp[] @@ -1406,6 +1423,12 @@ function getindex(vi::VarInfo, vn::VarName, dist::Distribution) val = getval(vi, vn) return maybe_invlink_and_reconstruct(vi, vn, dist, val) end +function getindex(vi::VectorVarInfo, vn::VarName, ::Nothing) + if !haskey(vi, vn) + throw(KeyError(vn)) + end + return getmetadata(vi, vn)[vn] +end function getindex(vi::VarInfo, vns::Vector{<:VarName}) # FIXME(torfjelde): Using `getdist(vi, first(vns))` won't be correct in cases # such as `x .~ [Normal(), Exponential()]`. @@ -1440,7 +1463,7 @@ Return the current value(s) of the random variables sampled by `spl` in `vi`. The value(s) may or may not be transformed to Euclidean space. """ -getindex(vi::UntypedVarInfo, spl::Sampler) = copy(getval(vi, _getranges(vi, spl))) +getindex(vi::VarInfo, spl::Sampler) = copy(getval(vi, _getranges(vi, spl))) function getindex(vi::TypedVarInfo, spl::Sampler) # Gets the ranges as a NamedTuple ranges = _getranges(vi, spl) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 8be220972..7407b3068 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -74,40 +74,81 @@ function VarNameVector( end # Basic array interface. -Base.eltype(vmd::VarNameVector) = eltype(vmd.vals) -Base.length(vmd::VarNameVector) = length(vmd.vals) -Base.size(vmd::VarNameVector) = size(vmd.vals) +Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) +Base.length(vnv::VarNameVector) = length(vnv.vals) +Base.size(vnv::VarNameVector) = size(vnv.vals) Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() +# Dictionary interface. +Base.keys(vnv::VarNameVector) = vnv.vns + +Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.idcs, vn) + # `getindex` & `setindex!` -getidc(vmd::VarNameVector, vn::VarName) = vmd.idcs[vn] +getidx(vnv::VarNameVector, vn::VarName) = vnv.idcs[vn] -getrange(vmd::VarNameVector, i::Int) = vmd.ranges[i] -getrange(vmd::VarNameVector, vn::VarName) = getrange(vmd, getidc(vmd, vn)) +getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] +getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) -gettransform(vmd::VarNameVector, vn::VarName) = vmd.transforms[getidc(vmd, vn)] +gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] -Base.getindex(vmd::VarNameVector, i::Int) = vmd.vals[i] -function Base.getindex(vmd::VarNameVector, vn::VarName) - x = vmd.vals[getrange(vmd, vn)] - f = gettransform(vmd, vn) +Base.getindex(vnv::VarNameVector, ::Colon) = vnv.vals +Base.getindex(vnv::VarNameVector, i::Int) = vnv.vals[i] +function Base.getindex(vnv::VarNameVector, vn::VarName) + x = vnv.vals[getrange(vnv, vn)] + f = gettransform(vnv, vn) return f(x) end -Base.setindex!(vmd::VarNameVector, val, i::Int) = vmd.vals[i] = val -function Base.setindex!(vmd::VarNameVector, val, vn::VarName) - f = inverse(gettransform(vmd, vn)) - vmd.vals[getrange(vmd, vn)] = f(val) +# HACK: remove this as soon as possible. +Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] + +Base.setindex!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val +function Base.setindex!(vnv::VarNameVector, val, vn::VarName) + f = inverse(gettransform(vnv, vn)) + vnv.vals[getrange(vnv, vn)] = f(val) +end + +function Base.empty!(vnv::VarNameVector) + # TODO: Or should the semantics be different, e.g. keeping `vns`? + empty!(vnv.idcs) + empty!(vnv.vns) + empty!(vnv.ranges) + empty!(vnv.vals) + empty!(vnv.transforms) + return nothing end +BangBang.empty!!(vnv::VarNameVector) = empty!(vnv) # TODO: Re-use some of the show functionality from Base? -function Base.show(io::IO, vmd::VarNameVector) +function Base.show(io::IO, vnv::VarNameVector) print(io, "[") - for (i, vn) in enumerate(vmd.vns) + for (i, vn) in enumerate(vnv.vns) if i > 1 print(io, ", ") end - print(io, vn, " = ", vmd[vn]) + print(io, vn, " = ", vnv[vn]) end end + +# Typed version. +function group_by_symbol(vnv::VarNameVector) + # Group varnames in `vnv` by the symbol. + d = OrderedDict{Symbol,Vector{VarName}}() + for vn in vnv.vns + push!(get!(d, getsym(vn), Vector{VarName}()), vn) + end + + # Create a `NamedTuple` from the grouped varnames. + nt_vals = map(values(d)) do vns + # TODO: Do we need to specialize the inputs here? + VarNameVector( + map(identity, vns), + map(Base.Fix1(getindex, vnv), vns), + map(Base.Fix1(gettransform, vnv), vns) + ) + end + + return NamedTuple{Tuple(keys(d))}(nt_vals) +end From fc6a051188f2bd3111caaed0ec9697e147c65c75 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 7 Nov 2023 11:13:54 +0000 Subject: [PATCH 003/209] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/varinfo.jl | 1 - src/varnamevector.jl | 10 ++++------ test/varnamevector.jl | 12 ++---------- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index b0c20b87d..df1be3d21 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -908,7 +908,6 @@ istrans(vi::VarInfo, vn::VarName) = istrans(getmetadata(vi, vn), vn) istrans(md::Metadata, vn::VarName) = is_flagged(md, vn, "trans") istrans(vnv::VarNameVector, vn::VarName) = !(gettransform(vnv, vn) isa FromVec) - getlogp(vi::VarInfo) = vi.logp[] function setlogp!!(vi::VarInfo, logp) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 7407b3068..dd55fb7c0 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -4,7 +4,7 @@ struct VarNameVector{ TIdcs<:OrderedDict{<:VarName,Int}, TVN<:AbstractVector{<:VarName}, TVal<:AbstractVector, - TTrans<:AbstractVector + TTrans<:AbstractVector, } "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" idcs::TIdcs # Dict{<:VarName,Int} @@ -44,9 +44,7 @@ Bijectors.inverse(f::FromVec{Tuple{}}) = tovec VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) VarNameVector(vns, vals) = VarNameVector(collect(vns), collect(vals)) function VarNameVector( - vns::AbstractVector, - vals::AbstractVector, - transforms = map(FromVec, vals) + vns::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) ) # TODO: Check uniqueness of `vns`? @@ -107,7 +105,7 @@ Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] Base.setindex!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val function Base.setindex!(vnv::VarNameVector, val, vn::VarName) f = inverse(gettransform(vnv, vn)) - vnv.vals[getrange(vnv, vn)] = f(val) + return vnv.vals[getrange(vnv, vn)] = f(val) end function Base.empty!(vnv::VarNameVector) @@ -146,7 +144,7 @@ function group_by_symbol(vnv::VarNameVector) VarNameVector( map(identity, vns), map(Base.Fix1(getindex, vnv), vns), - map(Base.Fix1(gettransform, vnv), vns) + map(Base.Fix1(gettransform, vnv), vns), ) end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 22b8c817b..158eca99d 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -1,14 +1,6 @@ @testset "VarNameVector" begin - vns = [ - @varname(x[1]), - @varname(x[2]), - @varname(x[3]), - ] - vals = [ - 1, - 2:3, - reshape(4:9, 2, 3), - ] + vns = [@varname(x[1]), @varname(x[2]), @varname(x[3])] + vals = [1, 2:3, reshape(4:9, 2, 3)] vnv = VarNameVector(vns, vals) # `getindex` From 7cd599d908b2a444461f024feb1ddf15e6743bd8 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 12 Nov 2023 17:46:59 +0000 Subject: [PATCH 004/209] added arbitrary metadata field as discussed --- src/varnamevector.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index dd55fb7c0..b1949a17c 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -5,6 +5,7 @@ struct VarNameVector{ TVN<:AbstractVector{<:VarName}, TVal<:AbstractVector, TTrans<:AbstractVector, + MData } "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" idcs::TIdcs # Dict{<:VarName,Int} @@ -20,6 +21,13 @@ struct VarNameVector{ "vector of transformations whose inverse takes us back to the original space" transforms::TTrans + + "metadata associated with the varnames" + metadata::MData +end + +function VarNameVector(idcs, vns, ranges, vals, transforms) + return VarNameVector(idcs, vns, ranges, vals, transforms, nothing) end # Useful transformation going from the flattened representation. @@ -117,7 +125,7 @@ function Base.empty!(vnv::VarNameVector) empty!(vnv.transforms) return nothing end -BangBang.empty!!(vnv::VarNameVector) = empty!(vnv) +BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) # TODO: Re-use some of the show functionality from Base? function Base.show(io::IO, vnv::VarNameVector) From ed0a757921590c2d90120ada02f4cfd34aef826d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 12 Nov 2023 17:47:45 +0000 Subject: [PATCH 005/209] renamed idcs to varname_to_index --- src/varnamevector.jl | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index b1949a17c..f71300f33 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -8,15 +8,15 @@ struct VarNameVector{ MData } "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" - idcs::TIdcs # Dict{<:VarName,Int} + varname_to_index::TIdcs # Dict{<:VarName,Int} - "vector of identifiers for the random variables, where `vns[idcs[vn]] == vn`" + "vector of identifiers for the random variables, where `vns[varname_to_index[vn]] == vn`" vns::TVN # AbstractVector{<:VarName} "vector of index ranges in `vals` corresponding to `vns`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" ranges::Vector{UnitRange{Int}} - "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[idcs[vn]]]`" + "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" vals::TVal # AbstractVector{<:Real} "vector of transformations whose inverse takes us back to the original space" @@ -26,8 +26,8 @@ struct VarNameVector{ metadata::MData end -function VarNameVector(idcs, vns, ranges, vals, transforms) - return VarNameVector(idcs, vns, ranges, vals, transforms, nothing) +function VarNameVector(varname_to_index, vns, ranges, vals, transforms) + return VarNameVector(varname_to_index, vns, ranges, vals, transforms, nothing) end # Useful transformation going from the flattened representation. @@ -63,12 +63,12 @@ function VarNameVector( if !(eltype(vns) <: VarName) vns = convert(Vector{VarName}, vns) end - idcs = OrderedDict{eltype(vns),Int}() + varname_to_index = OrderedDict{eltype(vns),Int}() ranges = Vector{UnitRange{Int}}() offset = 0 for (i, (vn, x)) in enumerate(zip(vns, vals_vecs)) # Add the varname index. - push!(idcs, vn => length(idcs) + 1) + push!(varname_to_index, vn => length(varname_to_index) + 1) # Add the range. r = (offset + 1):(offset + length(x)) push!(ranges, r) @@ -76,7 +76,7 @@ function VarNameVector( offset = r[end] end - return VarNameVector(idcs, vns, ranges, reduce(vcat, vals_vecs), transforms) + return VarNameVector(varname_to_index, vns, ranges, reduce(vcat, vals_vecs), transforms) end # Basic array interface. @@ -89,10 +89,10 @@ Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. Base.keys(vnv::VarNameVector) = vnv.vns -Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.idcs, vn) +Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) # `getindex` & `setindex!` -getidx(vnv::VarNameVector, vn::VarName) = vnv.idcs[vn] +getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) @@ -118,7 +118,7 @@ end function Base.empty!(vnv::VarNameVector) # TODO: Or should the semantics be different, e.g. keeping `vns`? - empty!(vnv.idcs) + empty!(vnv.varname_to_index) empty!(vnv.vns) empty!(vnv.ranges) empty!(vnv.vals) From 4ebd252ce46b40aa2c8a0daaf39da9f154b15c8b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 12 Nov 2023 17:48:31 +0000 Subject: [PATCH 006/209] renamed vns to varnames for VarNameVector --- src/varnamevector.jl | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index f71300f33..2c111feb2 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -7,13 +7,13 @@ struct VarNameVector{ TTrans<:AbstractVector, MData } - "mapping from the `VarName` to its integer index in `vns`, `ranges` and `dists`" + "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" varname_to_index::TIdcs # Dict{<:VarName,Int} - "vector of identifiers for the random variables, where `vns[varname_to_index[vn]] == vn`" - vns::TVN # AbstractVector{<:VarName} + "vector of identifiers for the random variables, where `varnames[varname_to_index[vn]] == vn`" + varnames::TVN # AbstractVector{<:VarName} - "vector of index ranges in `vals` corresponding to `vns`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" + "vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" ranges::Vector{UnitRange{Int}} "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" @@ -26,8 +26,8 @@ struct VarNameVector{ metadata::MData end -function VarNameVector(varname_to_index, vns, ranges, vals, transforms) - return VarNameVector(varname_to_index, vns, ranges, vals, transforms, nothing) +function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) + return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, nothing) end # Useful transformation going from the flattened representation. @@ -50,23 +50,23 @@ Bijectors.inverse(f::FromVec) = tovec Bijectors.inverse(f::FromVec{Tuple{}}) = tovec VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) -VarNameVector(vns, vals) = VarNameVector(collect(vns), collect(vals)) +VarNameVector(varnames, vals) = VarNameVector(collect(varnames), collect(vals)) function VarNameVector( - vns::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) + varnames::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) ) - # TODO: Check uniqueness of `vns`? + # TODO: Check uniqueness of `varnames`? # Convert `vals` into a vector of vectors. vals_vecs = map(tovec, vals) # TODO: Is this really the way to do this? - if !(eltype(vns) <: VarName) - vns = convert(Vector{VarName}, vns) + if !(eltype(varnames) <: VarName) + varnames = convert(Vector{VarName}, varnames) end - varname_to_index = OrderedDict{eltype(vns),Int}() + varname_to_index = OrderedDict{eltype(varnames),Int}() ranges = Vector{UnitRange{Int}}() offset = 0 - for (i, (vn, x)) in enumerate(zip(vns, vals_vecs)) + for (i, (vn, x)) in enumerate(zip(varnames, vals_vecs)) # Add the varname index. push!(varname_to_index, vn => length(varname_to_index) + 1) # Add the range. @@ -76,7 +76,7 @@ function VarNameVector( offset = r[end] end - return VarNameVector(varname_to_index, vns, ranges, reduce(vcat, vals_vecs), transforms) + return VarNameVector(varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms) end # Basic array interface. @@ -87,7 +87,7 @@ Base.size(vnv::VarNameVector) = size(vnv.vals) Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. -Base.keys(vnv::VarNameVector) = vnv.vns +Base.keys(vnv::VarNameVector) = vnv.varnames Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) @@ -117,9 +117,9 @@ function Base.setindex!(vnv::VarNameVector, val, vn::VarName) end function Base.empty!(vnv::VarNameVector) - # TODO: Or should the semantics be different, e.g. keeping `vns`? + # TODO: Or should the semantics be different, e.g. keeping `varnames`? empty!(vnv.varname_to_index) - empty!(vnv.vns) + empty!(vnv.varnames) empty!(vnv.ranges) empty!(vnv.vals) empty!(vnv.transforms) @@ -130,7 +130,7 @@ BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) # TODO: Re-use some of the show functionality from Base? function Base.show(io::IO, vnv::VarNameVector) print(io, "[") - for (i, vn) in enumerate(vnv.vns) + for (i, vn) in enumerate(vnv.varnames) if i > 1 print(io, ", ") end @@ -142,17 +142,17 @@ end function group_by_symbol(vnv::VarNameVector) # Group varnames in `vnv` by the symbol. d = OrderedDict{Symbol,Vector{VarName}}() - for vn in vnv.vns + for vn in vnv.varnames push!(get!(d, getsym(vn), Vector{VarName}()), vn) end # Create a `NamedTuple` from the grouped varnames. - nt_vals = map(values(d)) do vns + nt_vals = map(values(d)) do varnames # TODO: Do we need to specialize the inputs here? VarNameVector( - map(identity, vns), - map(Base.Fix1(getindex, vnv), vns), - map(Base.Fix1(gettransform, vnv), vns), + map(identity, varnames), + map(Base.Fix1(getindex, vnv), varnames), + map(Base.Fix1(gettransform, vnv), varnames), ) end From 9f12c9ac4ee38b3742bb7c807d4957294fa8ce5f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 12 Nov 2023 17:58:02 +0000 Subject: [PATCH 007/209] added keys impl for Metadata --- src/varinfo.jl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index df1be3d21..7551dc3fc 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -876,10 +876,9 @@ end return expr end -# Functions defined only for UntypedVarInfo -Base.keys(vi::UntypedVarInfo) = keys(vi.metadata.idcs) - -Base.keys(vi::VectorVarInfo) = keys(vi.metadata) +# `keys` +Base.keys(md::Metadata) = keys(md.idcs) +Base.keys(vi::VarInfo) = keys(vi.metadata) # HACK: Necessary to avoid returning `Any[]` which won't dispatch correctly # on other methods in the codebase which requires `Vector{<:VarName}`. @@ -889,7 +888,7 @@ Base.keys(vi::TypedVarInfo{<:NamedTuple{()}}) = VarName[] push!(expr.args, :vcat) for n in names - push!(expr.args, :(vi.metadata.$n.vns)) + push!(expr.args, :(keys(vi.metadata.$n))) end return expr @@ -1569,7 +1568,7 @@ const _MAX_VARS_SHOWN = 4 function _show_varnames(io::IO, vi) md = vi.metadata - vns = md.vns + vns = keys(md) vns_by_name = Dict{Symbol,Vector{VarName}}() for vn in vns From 5a151219f77339c5b6aa939f805077f0bb0605ee Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:07:20 +0000 Subject: [PATCH 008/209] added push! and update! for VarNameVector --- src/varnamevector.jl | 111 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 8 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 2c111feb2..a7acba79f 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -22,12 +22,15 @@ struct VarNameVector{ "vector of transformations whose inverse takes us back to the original space" transforms::TTrans + "inactive ranges" + inactive_ranges::Vector{UnitRange{Int}} + "metadata associated with the varnames" metadata::MData end function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) - return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, nothing) + return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing) end # Useful transformation going from the flattened representation. @@ -127,15 +130,107 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) -# TODO: Re-use some of the show functionality from Base? -function Base.show(io::IO, vnv::VarNameVector) - print(io, "[") - for (i, vn) in enumerate(vnv.varnames) - if i > 1 - print(io, ", ") +function nextrange(vnd::VarNameVector, x) + n = length(vnd) + return n + 1:n + length(x) +end + +# `push!` and `push!!`: add a variable to the varname vector. +function push!( + vnv::VarNameVector, + vn::VarName, + val, + transform=FromVec(val), +) + # Error if we already have the variable. + haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) + return update!(vnv, vn, val, transform) +end + +# `update!` and `update!!`: update a variable in the varname vector. +function update!( + vnv::VarNameVector, + vn::VarName, + val, + transform=FromVec(val), +) + val_vec = tovec(val) + if !haskey(vnv, vn) + # Here we just add a new entry. + vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 + push!(vnv.varnames, vn) + push!(vnv.ranges, nextrange(vnv, val_vec)) + append!(vnv.vals, val_vec) + push!(vnv.transforms, transform) + else + # Here we update the existing entry. + idx = getidx(vnv, vn) + r_old = getrange(vnv, idx) + n_old = length(r_old) + n_new = length(val_vec) + # Existing keys needs to be handled differently depending on + # whether the size of the value is increasing or decreasing. + if n_new > n_old + # Remove the old range. + delete!(vnv.ranges, vn) + # Add the new range. + r_new = nextrange(vnv, val_vec) + vnv.varname_to_ranges[vn] = r_new + # Grow the underlying vector to accomodate the new value. + resize!(vnv.vals, r_new[end]) + # Keep track of the deleted ranges. + push!(vnv.inactive_ranges, r_old) + else + # `n_new <= n_old` + # Just decrease the current range. + r_new = r_old[1]:(r_old[1] + n_new - 1) + vnv.ranges[idx] = r_new + # And mark the rest as inactive if needed. + if n_new < n_old + push!(vnv.inactive_ranges, r_old[n_new]:r_old[end]) + end end - print(io, vn, " = ", vnv[vn]) + + # Update the value. + vnv.vals[r_new] = val_vec + # Update the transform. + vnv.transforms[idx] = transform + + # TODO: Should we maybe sweep over inactive ranges and re-contiguify + # if we the total number of inactive elements is "large" in some sense? + end + + return vnv +end + +function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) + offset = 0 + # NOTE: assumes `ranges` are ordered. + for i = 1:length(ranges) + r_old = ranges[i] + ranges[i] = offset + 1:offset + length(r_old) + offset += length(r_old) + end + + return ranges +end + +function inactive_ranges_sweep!(vnv::VarNameVector) + # Extract the re-contiguified values. + # NOTE: We need to do this before we update the ranges. + vals = vnv[:] + # And then we re-contiguify the ranges. + recontiguify_ranges!(vnv.ranges) + # Clear the inactive ranges. + empty!(vnv.inactive_ranges) + # Now we update the values. + for (i, r) in enumerate(vnv.ranges) + vnv.vals[r] = vals[r] end + # And (potentially) shrink the underlying vector. + resize!(vnv.vals, vnv.ranges[end][end]) + # The rest should be left as is. + return vnv end # Typed version. From edde2c1704ba54de4f54483a357b824db828cd34 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:08:03 +0000 Subject: [PATCH 009/209] added getindex_raw! and setindex_raw! for VarNameVector --- src/varnamevector.jl | 53 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index a7acba79f..f4abf6e4a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -82,6 +82,16 @@ function VarNameVector( return VarNameVector(varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms) end +# Some `VarNameVector` specific functions. +getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] + +getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] +getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) + +gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] + +has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) + # Basic array interface. Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) Base.length(vnv::VarNameVector) = length(vnv.vals) @@ -95,28 +105,47 @@ Base.keys(vnv::VarNameVector) = vnv.varnames Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) # `getindex` & `setindex!` -getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] - -getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] -getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) - -gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] - -Base.getindex(vnv::VarNameVector, ::Colon) = vnv.vals -Base.getindex(vnv::VarNameVector, i::Int) = vnv.vals[i] +Base.getindex(vnv::VarNameVector, i::Int) = getindex_raw(vnv, i) function Base.getindex(vnv::VarNameVector, vn::VarName) - x = vnv.vals[getrange(vnv, vn)] + x = getindex_raw(vnv, vn) f = gettransform(vnv, vn) return f(x) end +getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[i] +function getindex_raw(vnv::VarNameVector, vn::VarName) + return vnv.vals[getrange(vnv, vn)] +end + +# `getindex` for `Colon` +function Base.getindex(vnv::VarNameVector, ::Colon) + return if has_inactive_ranges(vnv) + mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) + else + vnv.vals + end +end + +function getindex_raw(vnv::VarNameVector, ::Colon) + return if has_inactive_ranges(vnv) + mapreduce(Base.Fix1(getindex_raw, vnv.vals), vcat, vnv.ranges) + else + vnv.vals + end +end + # HACK: remove this as soon as possible. Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] -Base.setindex!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val +Base.setindex!(vnv::VarNameVector, val, i::Int) = setindex_raw!(vnv, val, i) function Base.setindex!(vnv::VarNameVector, val, vn::VarName) f = inverse(gettransform(vnv, vn)) - return vnv.vals[getrange(vnv, vn)] = f(val) + return setindex_raw!(vnv, f(val), vn) +end + +setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val +function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) + return vnv.vals[getrange(vnv, vn)] = val end function Base.empty!(vnv::VarNameVector) From ed460023a8d002e27eb0a14077ba34d83932a614 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:08:22 +0000 Subject: [PATCH 010/209] added `iterate` and `convert` (for `AbstractDict) impls for `VarNameVector` --- src/varnamevector.jl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index f4abf6e4a..279e26b2a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -282,3 +282,19 @@ function group_by_symbol(vnv::VarNameVector) return NamedTuple{Tuple(keys(d))}(nt_vals) end + +# `iterate` +function Base.iterate(vnv::VarNameVector, state=nothing) + res = state === nothing ? iterate(vnv.varname_to_index) : iterate(vnv.varname_to_index, state) + res === nothing && return nothing + (vn, idx), state_new = res + return vn => vnv.vals[getrange(vnv, idx)], state_new +end + +# `convert` +function Base.convert(::Type{D}, vnv::VarNameVector) where {D<:AbstractDict} + return ConstructionBase.constructorof(D)( + keys(vnv.varname_to_index), + map(Base.Fix1(getindex, vnv), keys(vnv.varname_to_index)) + ) +end From 5b00059ee605499ce8861b0c70e31391c3f55255 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:09:34 +0000 Subject: [PATCH 011/209] make the key and eltype part of the `VarNameVector` type --- src/varnamevector.jl | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 279e26b2a..30bcccd82 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -1,14 +1,15 @@ # Similar to `Metadata` but representing a `Vector` and simpler interface. -# TODO: Should we subtype `AbstractVector`? +# TODO: Should we subtype `AbstractVector` or `AbstractDict`? struct VarNameVector{ - TIdcs<:OrderedDict{<:VarName,Int}, - TVN<:AbstractVector{<:VarName}, - TVal<:AbstractVector, + K, + V, + TVN<:AbstractVector{K}, + TVal<:AbstractVector{V}, TTrans<:AbstractVector, MData } "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" - varname_to_index::TIdcs # Dict{<:VarName,Int} + varname_to_index::OrderedDict{K,Int} "vector of identifiers for the random variables, where `varnames[varname_to_index[vn]] == vn`" varnames::TVN # AbstractVector{<:VarName} @@ -32,6 +33,8 @@ end function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing) end +# TODO: Do we need this? +VarNameVector{K,V}() where {K,V} = VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) # Useful transformation going from the flattened representation. struct FromVec{Sz} @@ -52,8 +55,12 @@ tovec(x::AbstractArray) = vec(x) Bijectors.inverse(f::FromVec) = tovec Bijectors.inverse(f::FromVec{Tuple{}}) = tovec +collect_maybe(x) = collect(x) +collect_maybe(x::AbstractArray) = x + +VarNameVector() = VarNameVector{VarName,Real}() VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) -VarNameVector(varnames, vals) = VarNameVector(collect(varnames), collect(vals)) +VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) function VarNameVector( varnames::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) ) @@ -79,7 +86,13 @@ function VarNameVector( offset = r[end] end - return VarNameVector(varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms) + return VarNameVector( + varname_to_index, + varnames, + ranges, + reduce(vcat, vals_vecs), + transforms + ) end # Some `VarNameVector` specific functions. From bef7e0a74ba270bba713f94833597ae4dd61c07a Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:25:38 +0000 Subject: [PATCH 012/209] added more tests for VarNameVector --- test/varnamevector.jl | 108 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 12 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 158eca99d..ed225c7bb 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -1,19 +1,103 @@ +replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) + @testset "VarNameVector" begin - vns = [@varname(x[1]), @varname(x[2]), @varname(x[3])] - vals = [1, 2:3, reshape(4:9, 2, 3)] - vnv = VarNameVector(vns, vals) + # Need to test element-related operations: + # - `getindex` + # - `setindex!` + # - `push!` + # - `update!` + # + # And these should all be tested for different types of values: + # - scalar + # - vector + # - matrix - # `getindex` - for (vn, val) in zip(vns, vals) - @test vnv[vn] == val - end + # Need to test operations on `VarNameVector`: + # - `empty!` + # - `iterate` + # - `convert` to + # - `AbstractDict` + + test_pairs = OrderedDict( + @varname(x[1]) => rand(), + @varname(x[2]) => rand(2), + @varname(x[3]) => rand(2, 3), + @varname(x[4]) => rand(2, 3, 4) + ) - # `setindex!` - for (vn, val) in zip(vns, vals) - vnv[vn] = val .+ 100 + @testset "constructor" begin + @testset "no args" begin + # Empty. + vnv = VarNameVector() + @test isempty(vnv) + @test eltype(vnv) == Real + + # Empty with types. + vnv = VarNameVector{VarName,Float64}() + @test isempty(vnv) + @test eltype(vnv) == Float64 + end + + # Should be able to handle different types of values. + @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in Iterators.product( + keys(test_pairs), keys(test_pairs) + ) + val_left = test_pairs[vn_left] + val_right = test_pairs[vn_right] + vnv = VarNameVector([vn_left, vn_right], [val_left, val_right]) + @test length(vnv) == length(val_left) + length(val_right) + @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + end + + # Should also work when mixing varnames with different symbols. + @testset "$(vn_left) and $(replace_sym(vn_right, :y))" for (vn_left, vn_right) in Iterators.product( + keys(test_pairs), keys(test_pairs) + ) + val_left = test_pairs[vn_left] + val_right = test_pairs[vn_right] + vnv = VarNameVector([vn_left, replace_sym(vn_right, :y)], [val_left, val_right]) + @test length(vnv) == length(val_left) + length(val_right) + @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + end end - for (vn, val) in zip(vns, vals) - @test vnv[vn] == val .+ 100 + @testset "basics" begin + vns = [@varname(x[1]), @varname(x[2]), @varname(x[3])] + vals = [1, 2:3, reshape(4:9, 2, 3)] + vnv = VarNameVector(vns, vals) + + # `getindex` + for (vn, val) in zip(vns, vals) + @test vnv[vn] == val + end + + # `setindex!` + for (vn, val) in zip(vns, vals) + vnv[vn] = val .+ 100 + end + for (vn, val) in zip(vns, vals) + @test vnv[vn] == val .+ 100 + end + + # `push!` + vn = @varname(x[4]) + val = 10:12 + push!(vnv, vn, val) + @test vnv[vn] == val + + # `push!` existing varname is not allowed + @test_throws ArgumentError push!(vnv, vn, val) + + # `update!` works with both existing and new varname + # existing + val = 20:22 + DynamicPPL.update!(vnv, vn, val) + @test vnv[vn] == val + + # new + vn = @varname(x[5]) + val = 30:32 + DynamicPPL.update!(vnv, vn, val) + @test vnv[vn] == val end end From 006ee8dfb815299da0948db8634b8dfa016fd3be Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 14:26:38 +0000 Subject: [PATCH 013/209] formatting --- src/varnamevector.jl | 49 +++++++++++++++++-------------------------- test/varnamevector.jl | 5 +++-- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 30bcccd82..bb553835f 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -1,12 +1,7 @@ # Similar to `Metadata` but representing a `Vector` and simpler interface. # TODO: Should we subtype `AbstractVector` or `AbstractDict`? struct VarNameVector{ - K, - V, - TVN<:AbstractVector{K}, - TVal<:AbstractVector{V}, - TTrans<:AbstractVector, - MData + K,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData } "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" varname_to_index::OrderedDict{K,Int} @@ -31,10 +26,14 @@ struct VarNameVector{ end function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) - return VarNameVector(varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing) + return VarNameVector( + varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing + ) end # TODO: Do we need this? -VarNameVector{K,V}() where {K,V} = VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) +function VarNameVector{K,V}() where {K,V} + return VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) +end # Useful transformation going from the flattened representation. struct FromVec{Sz} @@ -87,11 +86,7 @@ function VarNameVector( end return VarNameVector( - varname_to_index, - varnames, - ranges, - reduce(vcat, vals_vecs), - transforms + varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms ) end @@ -174,28 +169,18 @@ BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) function nextrange(vnd::VarNameVector, x) n = length(vnd) - return n + 1:n + length(x) + return (n + 1):(n + length(x)) end # `push!` and `push!!`: add a variable to the varname vector. -function push!( - vnv::VarNameVector, - vn::VarName, - val, - transform=FromVec(val), -) +function push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) return update!(vnv, vn, val, transform) end # `update!` and `update!!`: update a variable in the varname vector. -function update!( - vnv::VarNameVector, - vn::VarName, - val, - transform=FromVec(val), -) +function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) val_vec = tovec(val) if !haskey(vnv, vn) # Here we just add a new entry. @@ -248,9 +233,9 @@ end function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) offset = 0 # NOTE: assumes `ranges` are ordered. - for i = 1:length(ranges) + for i in 1:length(ranges) r_old = ranges[i] - ranges[i] = offset + 1:offset + length(r_old) + ranges[i] = (offset + 1):(offset + length(r_old)) offset += length(r_old) end @@ -298,7 +283,11 @@ end # `iterate` function Base.iterate(vnv::VarNameVector, state=nothing) - res = state === nothing ? iterate(vnv.varname_to_index) : iterate(vnv.varname_to_index, state) + res = if state === nothing + iterate(vnv.varname_to_index) + else + iterate(vnv.varname_to_index, state) + end res === nothing && return nothing (vn, idx), state_new = res return vn => vnv.vals[getrange(vnv, idx)], state_new @@ -308,6 +297,6 @@ end function Base.convert(::Type{D}, vnv::VarNameVector) where {D<:AbstractDict} return ConstructionBase.constructorof(D)( keys(vnv.varname_to_index), - map(Base.Fix1(getindex, vnv), keys(vnv.varname_to_index)) + map(Base.Fix1(getindex, vnv), keys(vnv.varname_to_index)), ) end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index ed225c7bb..2d4374610 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -22,7 +22,7 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @varname(x[3]) => rand(2, 3), - @varname(x[4]) => rand(2, 3, 4) + @varname(x[4]) => rand(2, 3, 4), ) @testset "constructor" begin @@ -50,7 +50,8 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) end # Should also work when mixing varnames with different symbols. - @testset "$(vn_left) and $(replace_sym(vn_right, :y))" for (vn_left, vn_right) in Iterators.product( + @testset "$(vn_left) and $(replace_sym(vn_right, :y))" for (vn_left, vn_right) in + Iterators.product( keys(test_pairs), keys(test_pairs) ) val_left = test_pairs[vn_left] From 9802811b5713193468222e1dbaf54708cf7ec4fd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 16:23:29 +0000 Subject: [PATCH 014/209] more testing for VarNameVector --- src/varnamevector.jl | 10 +++ test/Project.toml | 2 + test/runtests.jl | 2 + test/varnamevector.jl | 152 +++++++++++++++++++++++++----------------- 4 files changed, 104 insertions(+), 62 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index bb553835f..6db719a0c 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -25,6 +25,16 @@ struct VarNameVector{ metadata::MData end +function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) + return vnv_left.varname_to_index == vnv_right.varname_to_index && + vnv_left.varnames == vnv_right.varnames && + vnv_left.ranges == vnv_right.ranges && + vnv_left.vals == vnv_right.vals && + vnv_left.transforms == vnv_right.transforms && + vnv_left.inactive_ranges == vnv_right.inactive_ranges && + vnv_left.metadata == vnv_right.metadata +end + function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) return VarNameVector( varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing diff --git a/test/Project.toml b/test/Project.toml index ed8ae7971..c8e79f859 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,6 +2,7 @@ AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" +Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -25,6 +26,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" AbstractMCMC = "2.1, 3.0, 4" AbstractPPL = "0.6" Bijectors = "0.13" +Combinatorics = "1" Compat = "4.3.0" Distributions = "0.25" DistributionsAD = "0.6.3" diff --git a/test/runtests.jl b/test/runtests.jl index 43d68386c..d15b17b42 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,6 +20,8 @@ using Random using Serialization using Test +using Combinatorics: combinations + using DynamicPPL: getargs_dottilde, getargs_tilde, Selector const DIRECTORY_DynamicPPL = dirname(dirname(pathof(DynamicPPL))) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 2d4374610..0f62e9f97 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -18,87 +18,115 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) # - `convert` to # - `AbstractDict` - test_pairs = OrderedDict( + test_pairs_x = OrderedDict( @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @varname(x[3]) => rand(2, 3), @varname(x[4]) => rand(2, 3, 4), ) + test_pairs_y = OrderedDict( + @varname(y[1]) => rand(), + @varname(y[2]) => rand(2), + @varname(y[3]) => rand(2, 3), + @varname(y[4]) => rand(2, 3, 4), + ) + test_pairs = merge(test_pairs_x, test_pairs_y) - @testset "constructor" begin - @testset "no args" begin - # Empty. - vnv = VarNameVector() - @test isempty(vnv) - @test eltype(vnv) == Real + @testset "constructor: no args" begin + # Empty. + vnv = VarNameVector() + @test isempty(vnv) + @test eltype(vnv) == Real - # Empty with types. - vnv = VarNameVector{VarName,Float64}() - @test isempty(vnv) - @test eltype(vnv) == Float64 - end + # Empty with types. + vnv = VarNameVector{VarName,Float64}() + @test isempty(vnv) + @test eltype(vnv) == Float64 + end - # Should be able to handle different types of values. - @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in Iterators.product( - keys(test_pairs), keys(test_pairs) - ) - val_left = test_pairs[vn_left] - val_right = test_pairs[vn_right] - vnv = VarNameVector([vn_left, vn_right], [val_left, val_right]) - @test length(vnv) == length(val_left) + length(val_right) - @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) - end + test_varnames_iter = combinations(collect(keys(test_pairs)), 2) + @info "Testing varnames" collect( + map(Base.Fix1(convert, Vector{VarName}), test_varnames_iter) + ) + @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter + val_left = test_pairs[vn_left] + val_right = test_pairs[vn_right] + vnv = VarNameVector([vn_left, vn_right], [val_left, val_right]) - # Should also work when mixing varnames with different symbols. - @testset "$(vn_left) and $(replace_sym(vn_right, :y))" for (vn_left, vn_right) in - Iterators.product( - keys(test_pairs), keys(test_pairs) + # Compare to alternative constructors. + vnv_from_dict = VarNameVector( + OrderedDict(vn_left => val_left, vn_right => val_right) ) - val_left = test_pairs[vn_left] - val_right = test_pairs[vn_right] - vnv = VarNameVector([vn_left, replace_sym(vn_right, :y)], [val_left, val_right]) - @test length(vnv) == length(val_left) + length(val_right) - @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + @test vnv == vnv_from_dict + + # We want the types of fields such as `varnames` and `transforms` to specialize + # whenever possible + some functionality, e.g. `push!`, is only sensible + # if the underlying containers can support it. + # Expected behavior + should_have_restricted_varname_type = typeof(vn_left) == typeof(vn_right) + should_have_restricted_transform_type = size(val_left) == size(val_right) + # Actual behavior + has_restricted_transform_type = isconcretetype(eltype(vnv.transforms)) + has_restricted_varname_type = isconcretetype(eltype(vnv.varnames)) + + @testset "type specialization" begin + @test !should_have_restricted_varname_type || has_restricted_varname_type + @test !should_have_restricted_transform_type || + has_restricted_transform_type end - end - @testset "basics" begin - vns = [@varname(x[1]), @varname(x[2]), @varname(x[3])] - vals = [1, 2:3, reshape(4:9, 2, 3)] - vnv = VarNameVector(vns, vals) + # `eltype` + @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + # `length` + @test length(vnv) == length(val_left) + length(val_right) # `getindex` - for (vn, val) in zip(vns, vals) - @test vnv[vn] == val + @testset "getindex" begin + # `getindex` + @test vnv[vn_left] == val_left + @test vnv[vn_right] == val_right end - # `setindex!` - for (vn, val) in zip(vns, vals) - vnv[vn] = val .+ 100 - end - for (vn, val) in zip(vns, vals) - @test vnv[vn] == val .+ 100 + @testset "setindex!" begin + vnv[vn_left] = val_left .+ 100 + @test vnv[vn_left] == val_left .+ 100 + vnv[vn_right] = val_right .+ 100 + @test vnv[vn_right] == val_right .+ 100 end # `push!` - vn = @varname(x[4]) - val = 10:12 - push!(vnv, vn, val) - @test vnv[vn] == val - - # `push!` existing varname is not allowed - @test_throws ArgumentError push!(vnv, vn, val) - - # `update!` works with both existing and new varname - # existing - val = 20:22 - DynamicPPL.update!(vnv, vn, val) - @test vnv[vn] == val + # `push!` and `update!` are only allowed for all the varnames if both the + # varname and the transform types used in the underlying containers are not concrete. + push_test_varnames = filter(keys(test_pairs)) do vn + val = test_pairs[vn] + transform_is_okay = + !has_restricted_transform_type || + size(val) == size(val_left) || + size(val) == size(val_right) + varname_is_okay = + !has_restricted_varname_type || + typeof(vn) == typeof(vn_left) || + typeof(vn) == typeof(vn_right) + return transform_is_okay && varname_is_okay + end + @testset "push! ($(vn))" for vn in push_test_varnames + val = test_pairs[vn] + if vn == vn_left || vn == vn_right + # Should not be possible to `push!` existing varname. + @test_throws ArgumentError push!(vnv, vn, val) + else + push!(vnv, vn, val) + @test vnv[vn] == val + end + end - # new - vn = @varname(x[5]) - val = 30:32 - DynamicPPL.update!(vnv, vn, val) - @test vnv[vn] == val + # `update!` + @testset "update! ($(vn))" for vn in push_test_varnames + val = test_pairs[vn] + # Perturb `val` a bit so we can also check that the existing `vn_left` and `vn_right` + # are also updated correctly. + DynamicPPL.update!(vnv, vn, val .+ 1) + @test vnv[vn] == val .+ 1 + end end end From 88b1721b8350a42df9239ebb9f184b1aa0e4aff4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 16:25:01 +0000 Subject: [PATCH 015/209] minor changes to some comments --- test/varnamevector.jl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 0f62e9f97..293f92e3e 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -94,9 +94,9 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) @test vnv[vn_right] == val_right .+ 100 end - # `push!` - # `push!` and `update!` are only allowed for all the varnames if both the - # varname and the transform types used in the underlying containers are not concrete. + # `push!` & `update!` + # These are only allowed for all the varnames if both the varname and + # the transform types used in the underlying containers are not concrete. push_test_varnames = filter(keys(test_pairs)) do vn val = test_pairs[vn] transform_is_okay = @@ -119,8 +119,6 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) @test vnv[vn] == val end end - - # `update!` @testset "update! ($(vn))" for vn in push_test_varnames val = test_pairs[vn] # Perturb `val` a bit so we can also check that the existing `vn_left` and `vn_right` From ca7b173b4273e24497d8d0412b4d3e4d5edfb54d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:28:16 +0000 Subject: [PATCH 016/209] added a bunch more tests for VarNameVector + several bugfixes in the process --- src/varnamevector.jl | 40 ++++----- test/varnamevector.jl | 202 +++++++++++++++++++++++++++++++++--------- 2 files changed, 179 insertions(+), 63 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 6db719a0c..c696aa20e 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -68,6 +68,7 @@ collect_maybe(x) = collect(x) collect_maybe(x::AbstractArray) = x VarNameVector() = VarNameVector{VarName,Real}() +VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) function VarNameVector( @@ -112,8 +113,12 @@ has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) # Basic array interface. Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) -Base.length(vnv::VarNameVector) = length(vnv.vals) -Base.size(vnv::VarNameVector) = size(vnv.vals) +Base.length(vnv::VarNameVector) = if isempty(vnv.inactive_ranges) + length(vnv.vals) +else + sum(length, vnv.ranges) +end +Base.size(vnv::VarNameVector) = (length(vnv),) Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() @@ -178,12 +183,12 @@ end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) function nextrange(vnd::VarNameVector, x) - n = length(vnd) + n = maximum(map(last, vnd.ranges)) return (n + 1):(n + length(x)) end # `push!` and `push!!`: add a variable to the varname vector. -function push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) +function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) return update!(vnv, vn, val, transform) @@ -194,9 +199,12 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) val_vec = tovec(val) if !haskey(vnv, vn) # Here we just add a new entry. + # NOTE: We need to compute the `nextrange` BEFORE we start mutating + # the underlying; otherwise we might get some strange behaviors. + r_new = nextrange(vnv, val_vec) vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 push!(vnv.varnames, vn) - push!(vnv.ranges, nextrange(vnv, val_vec)) + push!(vnv.ranges, r_new) append!(vnv.vals, val_vec) push!(vnv.transforms, transform) else @@ -208,11 +216,9 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Existing keys needs to be handled differently depending on # whether the size of the value is increasing or decreasing. if n_new > n_old - # Remove the old range. - delete!(vnv.ranges, vn) # Add the new range. r_new = nextrange(vnv, val_vec) - vnv.varname_to_ranges[vn] = r_new + vnv.ranges[idx] = r_new # Grow the underlying vector to accomodate the new value. resize!(vnv.vals, r_new[end]) # Keep track of the deleted ranges. @@ -242,7 +248,6 @@ end function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) offset = 0 - # NOTE: assumes `ranges` are ordered. for i in 1:length(ranges) r_old = ranges[i] ranges[i] = (offset + 1):(offset + length(r_old)) @@ -292,21 +297,14 @@ function group_by_symbol(vnv::VarNameVector) end # `iterate` +# TODO: Maybe implement `iterate` as a vector and then instead implement `pairs`. function Base.iterate(vnv::VarNameVector, state=nothing) res = if state === nothing - iterate(vnv.varname_to_index) + iterate(vnv.varnames) else - iterate(vnv.varname_to_index, state) + iterate(vnv.varnames, state) end res === nothing && return nothing - (vn, idx), state_new = res - return vn => vnv.vals[getrange(vnv, idx)], state_new -end - -# `convert` -function Base.convert(::Type{D}, vnv::VarNameVector) where {D<:AbstractDict} - return ConstructionBase.constructorof(D)( - keys(vnv.varname_to_index), - map(Base.Fix1(getindex, vnv), keys(vnv.varname_to_index)), - ) + vn, state_new = res + return vn => getindex(vnv, vn), state_new end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 293f92e3e..c0c542c70 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -1,5 +1,75 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) +change_size_for_test(x::Real) = [x] +change_size_for_test(x::AbstractArray) = repeat(x, 2) + +function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) + if isconcretetype(eltype(vnv.varnames)) + # If the container is concrete, we need to make sure that the varname types match. + # E.g. if `vnv.varnames` has `eltype` `VarName{:x, IndexLens{Tuple{Int64}}}` then + # we need `vn` to also be of this type. + # => If the varname types don't match, we need to relax the container type. + return any(keys(vnv)) do vn_present + typeof(vn_present) !== typeof(val) + end + end + + return false +end + +function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) + if isconcretetype(eltype(vnv.vals)) + return promote_type(eltype(vnv.vals), eltype(val)) != eltype(vnv.vals) + end + + return false +end + +function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) + if isconcretetype(eltype(vnv.transforms)) + # If the container is concrete, we need to make sure that the sizes match. + # => If the sizes don't match, we need to relax the container type. + return any(keys(vnv)) do vn_present + size(vnv[vn_present]) != size(val) + end + end + + return false +end + +relax_container_types(vnv::VarNameVector, vn::VarName, val) = relax_container_types(vnv, [vn], [val]) +function relax_container_types(vnv::VarNameVector, vns, vals) + if any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) + varnames_new = convert(Vector{VarName}, vnv.varnames) + else + varname_to_index_new = vnv.varname_to_index + varnames_new = vnv.varnames + end + + transforms_new = if any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + convert(Vector{Any}, vnv.transforms) + else + vnv.transforms + end + + vals_new = if any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + convert(Vector{Any}, vnv.vals) + else + vnv.vals + end + + return VarNameVector( + varname_to_index_new, + varnames_new, + vnv.ranges, + vals_new, + transforms_new, + vnv.inactive_ranges, + vnv.metadata + ) +end + @testset "VarNameVector" begin # Need to test element-related operations: # - `getindex` @@ -18,19 +88,19 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) # - `convert` to # - `AbstractDict` - test_pairs_x = OrderedDict( + test_pairs = OrderedDict( @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @varname(x[3]) => rand(2, 3), - @varname(x[4]) => rand(2, 3, 4), - ) - test_pairs_y = OrderedDict( + @varname(y[1]) => rand(), @varname(y[2]) => rand(2), @varname(y[3]) => rand(2, 3), - @varname(y[4]) => rand(2, 3, 4), + + @varname(z[1]) => rand(1:10), + @varname(z[2]) => rand(1:10, 2), + @varname(z[3]) => rand(1:10, 2, 3), ) - test_pairs = merge(test_pairs_x, test_pairs_y) @testset "constructor: no args" begin # Empty. @@ -51,13 +121,20 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter val_left = test_pairs[vn_left] val_right = test_pairs[vn_right] - vnv = VarNameVector([vn_left, vn_right], [val_left, val_right]) + vnv_base = VarNameVector([vn_left, vn_right], [val_left, val_right]) + + # We'll need the transformations later. + # TODO: Should we test other transformations than just `FromVec`? + from_vec_left = DynamicPPL.FromVec(val_left) + from_vec_right = DynamicPPL.FromVec(val_right) + to_vec_left = inverse(from_vec_left) + to_vec_right = inverse(from_vec_right) # Compare to alternative constructors. vnv_from_dict = VarNameVector( OrderedDict(vn_left => val_left, vn_right => val_right) ) - @test vnv == vnv_from_dict + @test vnv_base == vnv_from_dict # We want the types of fields such as `varnames` and `transforms` to specialize # whenever possible + some functionality, e.g. `push!`, is only sensible @@ -66,8 +143,8 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) should_have_restricted_varname_type = typeof(vn_left) == typeof(vn_right) should_have_restricted_transform_type = size(val_left) == size(val_right) # Actual behavior - has_restricted_transform_type = isconcretetype(eltype(vnv.transforms)) - has_restricted_varname_type = isconcretetype(eltype(vnv.varnames)) + has_restricted_transform_type = isconcretetype(eltype(vnv_base.transforms)) + has_restricted_varname_type = isconcretetype(eltype(vnv_base.varnames)) @testset "type specialization" begin @test !should_have_restricted_varname_type || has_restricted_varname_type @@ -76,55 +153,96 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) end # `eltype` - @test eltype(vnv) == promote_type(eltype(val_left), eltype(val_right)) + @test eltype(vnv_base) == promote_type(eltype(val_left), eltype(val_right)) # `length` - @test length(vnv) == length(val_left) + length(val_right) + @test length(vnv_base) == length(val_left) + length(val_right) # `getindex` @testset "getindex" begin # `getindex` - @test vnv[vn_left] == val_left - @test vnv[vn_right] == val_right + @test vnv_base[vn_left] == val_left + @test vnv_base[vn_right] == val_right end + # `setindex!` @testset "setindex!" begin + vnv = deepcopy(vnv_base) vnv[vn_left] = val_left .+ 100 @test vnv[vn_left] == val_left .+ 100 vnv[vn_right] = val_right .+ 100 @test vnv[vn_right] == val_right .+ 100 end - # `push!` & `update!` - # These are only allowed for all the varnames if both the varname and - # the transform types used in the underlying containers are not concrete. - push_test_varnames = filter(keys(test_pairs)) do vn - val = test_pairs[vn] - transform_is_okay = - !has_restricted_transform_type || - size(val) == size(val_left) || - size(val) == size(val_right) - varname_is_okay = - !has_restricted_varname_type || - typeof(vn) == typeof(vn_left) || - typeof(vn) == typeof(vn_right) - return transform_is_okay && varname_is_okay + # `getindex_raw` + @testset "getindex_raw" begin + @test DynamicPPL.getindex_raw(vnv_base, vn_left) == to_vec_left(val_left) + @test DynamicPPL.getindex_raw(vnv_base, vn_right) == to_vec_right(val_right) end - @testset "push! ($(vn))" for vn in push_test_varnames - val = test_pairs[vn] - if vn == vn_left || vn == vn_right - # Should not be possible to `push!` existing varname. - @test_throws ArgumentError push!(vnv, vn, val) - else - push!(vnv, vn, val) - @test vnv[vn] == val + + # `setindex_raw!` + @testset "setindex_raw!" begin + vnv = deepcopy(vnv_base) + DynamicPPL.setindex_raw!(vnv, to_vec_left(val_left .+ 100), vn_left) + @test vnv[vn_left] == val_left .+ 100 + DynamicPPL.setindex_raw!(vnv, to_vec_right(val_right .+ 100), vn_right) + @test vnv[vn_right] == val_right .+ 100 + end + + # `push!` & `update!` + @testset "push!" begin + vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + @testset "$vn" for vn in keys(test_pairs) + val = test_pairs[vn] + if vn == vn_left || vn == vn_right + # Should not be possible to `push!` existing varname. + @test_throws ArgumentError push!(vnv, vn, val) + else + push!(vnv, vn, val) + @test vnv[vn] == val + end end end - @testset "update! ($(vn))" for vn in push_test_varnames - val = test_pairs[vn] - # Perturb `val` a bit so we can also check that the existing `vn_left` and `vn_right` - # are also updated correctly. - DynamicPPL.update!(vnv, vn, val .+ 1) - @test vnv[vn] == val .+ 1 + @testset "update!" begin + vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + @testset "$vn" for vn in keys(test_pairs) + val = test_pairs[vn] + expected_length = if haskey(vnv, vn) + # If it's already present, the resulting length will be unchanged. + length(vnv) + else + length(vnv) + length(val) + end + + DynamicPPL.update!(vnv, vn, val .+ 1) + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(vnv[:]) == length(vnv) + + # There should be no redundant values in the underlying vector. + @test !DynamicPPL.has_inactive_ranges(vnv) + end + + # Need to recompute valid varnames for the changing of the sizes; before + # we required either a) the underlying `transforms` to be non-concrete, + # or b) the sizes of the values to match. But now the sizes of the values + # will change, so we can only test the former. + vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + @testset "$vn (different size)" for vn in keys(test_pairs) + val_original = test_pairs[vn] + val = change_size_for_test(val_original) + vn_already_present = haskey(vnv, vn) + expected_length = if vn_already_present + # If it's already present, the resulting length will be altered. + length(vnv) + length(val) - length(val_original) + else + length(vnv) + length(val) + end + + DynamicPPL.update!(vnv, vn, val .+ 1) + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(vnv[:]) == length(vnv) + end end end end From fb01b941df4883758b6d6f252bf98fb216ad7a7b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:29:20 +0000 Subject: [PATCH 017/209] formatting --- src/varnamevector.jl | 11 ++++++----- test/varnamevector.jl | 34 ++++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index c696aa20e..a6da79b04 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -113,11 +113,12 @@ has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) # Basic array interface. Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) -Base.length(vnv::VarNameVector) = if isempty(vnv.inactive_ranges) - length(vnv.vals) -else - sum(length, vnv.ranges) -end +Base.length(vnv::VarNameVector) = + if isempty(vnv.inactive_ranges) + length(vnv.vals) + else + sum(length, vnv.ranges) + end Base.size(vnv::VarNameVector) = (length(vnv),) Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() diff --git a/test/varnamevector.jl b/test/varnamevector.jl index c0c542c70..0c46af83d 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -37,7 +37,9 @@ function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) return false end -relax_container_types(vnv::VarNameVector, vn::VarName, val) = relax_container_types(vnv, [vn], [val]) +function relax_container_types(vnv::VarNameVector, vn::VarName, val) + return relax_container_types(vnv, [vn], [val]) +end function relax_container_types(vnv::VarNameVector, vns, vals) if any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) @@ -47,11 +49,12 @@ function relax_container_types(vnv::VarNameVector, vns, vals) varnames_new = vnv.varnames end - transforms_new = if any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) - convert(Vector{Any}, vnv.transforms) - else - vnv.transforms - end + transforms_new = + if any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + convert(Vector{Any}, vnv.transforms) + else + vnv.transforms + end vals_new = if any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) convert(Vector{Any}, vnv.vals) @@ -66,7 +69,7 @@ function relax_container_types(vnv::VarNameVector, vns, vals) vals_new, transforms_new, vnv.inactive_ranges, - vnv.metadata + vnv.metadata, ) end @@ -92,11 +95,9 @@ end @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @varname(x[3]) => rand(2, 3), - @varname(y[1]) => rand(), @varname(y[2]) => rand(2), @varname(y[3]) => rand(2, 3), - @varname(z[1]) => rand(1:10), @varname(z[2]) => rand(1:10, 2), @varname(z[3]) => rand(1:10, 2, 3), @@ -148,8 +149,7 @@ end @testset "type specialization" begin @test !should_have_restricted_varname_type || has_restricted_varname_type - @test !should_have_restricted_transform_type || - has_restricted_transform_type + @test !should_have_restricted_transform_type || has_restricted_transform_type end # `eltype` @@ -190,7 +190,9 @@ end # `push!` & `update!` @testset "push!" begin - vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + vnv = relax_container_types( + deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + ) @testset "$vn" for vn in keys(test_pairs) val = test_pairs[vn] if vn == vn_left || vn == vn_right @@ -203,7 +205,9 @@ end end end @testset "update!" begin - vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + vnv = relax_container_types( + deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + ) @testset "$vn" for vn in keys(test_pairs) val = test_pairs[vn] expected_length = if haskey(vnv, vn) @@ -226,7 +230,9 @@ end # we required either a) the underlying `transforms` to be non-concrete, # or b) the sizes of the values to match. But now the sizes of the values # will change, so we can only test the former. - vnv = relax_container_types(deepcopy(vnv_base), keys(test_pairs), values(test_pairs)) + vnv = relax_container_types( + deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + ) @testset "$vn (different size)" for vn in keys(test_pairs) val_original = test_pairs[vn] val = change_size_for_test(val_original) From 9634839f22b684870afe5fb755530d640308fced Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:40:43 +0000 Subject: [PATCH 018/209] added `similar` implementation for `VarNameVector` --- src/varnamevector.jl | 22 ++++++++++++++++++++++ test/varnamevector.jl | 17 +++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index a6da79b04..13d307d19 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -183,6 +183,28 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) + +similar_metadata(::Nothing) = nothing +similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) +function Base.similar(vnv::VarNameVector) + # NOTE: Whether or not we should empty the underlying containers or note + # is somewhat ambiguous. For example, `similar(vnv.varname_to_index)` will + # result in an empty `AbstractDict`, while the vectors, e.g. `vnv.ranges`, + # will result in non-empty vectors but with entries as `undef`. But it's + # much easier to write the rest of the code assuming that `undef` is not + # present, and so for now we empty the underlying containers, thus differing + # from the behavior of `similar` for `AbstractArray`s. + return VarNameVector( + similar(vnv.varname_to_index), + similar(vnv.varnames, 0), + similar(vnv.ranges, 0), + similar(vnv.vals, 0), + similar(vnv.transforms, 0), + similar(vnv.inactive_ranges, 0), + similar_metadata(vnv.metadata), + ) +end + function nextrange(vnd::VarNameVector, x) n = maximum(map(last, vnd.ranges)) return (n + 1):(n + length(x)) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 0c46af83d..b63d695c2 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -157,6 +157,23 @@ end # `length` @test length(vnv_base) == length(val_left) + length(val_right) + # `isempty` + @test !isempty(vnv_base) + + # `empty!` + @testset "empty!" begin + vnv = deepcopy(vnv_base) + empty!(vnv) + @test isempty(vnv) + end + + # `similar` + @testset "similar" begin + vnv = similar(vnv_base) + @test isempty(vnv) + @test typeof(vnv) == typeof(vnv_base) + end + # `getindex` @testset "getindex" begin # `getindex` From 5179f6fd603899cc4df4fa2d337062920eb12cad Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:41:09 +0000 Subject: [PATCH 019/209] formatting --- src/varnamevector.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 13d307d19..00d5149f2 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -183,7 +183,6 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) - similar_metadata(::Nothing) = nothing similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) function Base.similar(vnv::VarNameVector) From 9f632bb319df4eab96c76ef0c71aa13816f8f148 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 20:41:29 +0000 Subject: [PATCH 020/209] removed debug statement --- test/varnamevector.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index b63d695c2..92f402496 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -116,9 +116,6 @@ end end test_varnames_iter = combinations(collect(keys(test_pairs)), 2) - @info "Testing varnames" collect( - map(Base.Fix1(convert, Vector{VarName}), test_varnames_iter) - ) @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter val_left = test_pairs[vn_left] val_right = test_pairs[vn_right] From 3c210f7f85d734b34255eccd156bce6e8f4262d2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 13 Nov 2023 21:36:08 +0000 Subject: [PATCH 021/209] made VarInfo slighly more generic wrt. underlying metadata --- src/varinfo.jl | 25 +++++++++++++++++++------ src/varnamevector.jl | 10 +++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 7551dc3fc..c50372aa7 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -867,7 +867,8 @@ function BangBang.empty!!(vi::VarInfo) reset_num_produce!(vi) return vi end -@inline _empty!(metadata::Metadata) = empty!(metadata) + +_empty!(metadata) = empty!(metadata) @generated function _empty!(metadata::NamedTuple{names}) where {names} expr = Expr(:block) for f in names @@ -1534,12 +1535,14 @@ end return map(vn -> vi[vn], f_vns) end +haskey(metadata::Metadata, vn::VarName) = haskey(metadata.idcs, vn) + """ haskey(vi::VarInfo, vn::VarName) Check whether `vn` has been sampled in `vi`. """ -haskey(vi::VarInfo, vn::VarName) = haskey(getmetadata(vi, vn).idcs, vn) +haskey(vi::VarInfo, vn::VarName) = haskey(getmetadata(vi, vn), vn) function haskey(vi::TypedVarInfo, vn::VarName) metadata = vi.metadata Tmeta = typeof(metadata) @@ -1603,9 +1606,14 @@ function BangBang.push!!( @assert ~(haskey(vi, vn)) "[push!!] attempt to add an exisitng variable $(getsym(vn)) ($(vn)) to TypedVarInfo of syms $(syms(vi)) with dist=$dist, gid=$gidset" end - val = vectorize(dist, r) - meta = getmetadata(vi, vn) + push!(meta, vn, r, dist, gidset, get_num_produce(vi)) + + return vi +end + +function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) + val = vectorize(dist, r) meta.idcs[vn] = length(meta.idcs) + 1 push!(meta.vns, vn) l = length(meta.vals) @@ -1614,11 +1622,16 @@ function BangBang.push!!( append!(meta.vals, val) push!(meta.dists, dist) push!(meta.gids, gidset) - push!(meta.orders, get_num_produce(vi)) + push!(meta.orders, num_produce) push!(meta.flags["del"], false) push!(meta.flags["trans"], false) - return vi + return meta +end + +function Base.push!(vnv::VarNameVector, vn, r, dist, gidset, num_produce) + # FIXME: Include `transform` in the `push!` call below. + return push!(vnv, vn, r) end """ diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 00d5149f2..a9094b28a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -172,6 +172,7 @@ function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) return vnv.vals[getrange(vnv, vn)] = val end +# `empty!(!)` function Base.empty!(vnv::VarNameVector) # TODO: Or should the semantics be different, e.g. keeping `varnames`? empty!(vnv.varname_to_index) @@ -183,6 +184,7 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) +# `similar` similar_metadata(::Nothing) = nothing similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) function Base.similar(vnv::VarNameVector) @@ -204,9 +206,11 @@ function Base.similar(vnv::VarNameVector) ) end -function nextrange(vnd::VarNameVector, x) - n = maximum(map(last, vnd.ranges)) - return (n + 1):(n + length(x)) +function nextrange(vnv::VarNameVector, x) + # NOTE: Need to treat `isempty(vnv.ranges)` separately because `maximum` + # will error if `vnv.ranges` is empty. + offset = isempty(vnv.ranges) ? 0 : maximum(last, vnv.ranges) + return (offset + 1):(offset + length(x)) end # `push!` and `push!!`: add a variable to the varname vector. From 8b2720f76026d71ce9c08af5eda000008eb42253 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:16:02 +0000 Subject: [PATCH 022/209] fixed incorrect behavior in `keys` for `Metadata` --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index c50372aa7..90b61ea3e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -878,7 +878,7 @@ _empty!(metadata) = empty!(metadata) end # `keys` -Base.keys(md::Metadata) = keys(md.idcs) +Base.keys(md::Metadata) = md.vns Base.keys(vi::VarInfo) = keys(vi.metadata) # HACK: Necessary to avoid returning `Any[]` which won't dispatch correctly From 9fa6446b0c417d341dae3937879ba233d990a886 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:24:58 +0000 Subject: [PATCH 023/209] minor style changes to VarNameVector tests --- test/varnamevector.jl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 92f402496..1bbbf450f 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -90,7 +90,6 @@ end # - `iterate` # - `convert` to # - `AbstractDict` - test_pairs = OrderedDict( @varname(x[1]) => rand(), @varname(x[2]) => rand(2), @@ -102,6 +101,8 @@ end @varname(z[2]) => rand(1:10, 2), @varname(z[3]) => rand(1:10, 2, 3), ) + test_vns = collect(keys(test_pairs)) + test_vals = collect(test_vals) @testset "constructor: no args" begin # Empty. @@ -115,7 +116,7 @@ end @test eltype(vnv) == Float64 end - test_varnames_iter = combinations(collect(keys(test_pairs)), 2) + test_varnames_iter = combinations(test_vns, 2) @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter val_left = test_pairs[vn_left] val_right = test_pairs[vn_right] @@ -205,9 +206,9 @@ end # `push!` & `update!` @testset "push!" begin vnv = relax_container_types( - deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + deepcopy(vnv_base), test_vns, test_vals ) - @testset "$vn" for vn in keys(test_pairs) + @testset "$vn" for vn in test_vns val = test_pairs[vn] if vn == vn_left || vn == vn_right # Should not be possible to `push!` existing varname. @@ -220,9 +221,9 @@ end end @testset "update!" begin vnv = relax_container_types( - deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + deepcopy(vnv_base), test_vns, test_vals ) - @testset "$vn" for vn in keys(test_pairs) + @testset "$vn" for vn in test_vns val = test_pairs[vn] expected_length = if haskey(vnv, vn) # If it's already present, the resulting length will be unchanged. @@ -245,9 +246,9 @@ end # or b) the sizes of the values to match. But now the sizes of the values # will change, so we can only test the former. vnv = relax_container_types( - deepcopy(vnv_base), keys(test_pairs), values(test_pairs) + deepcopy(vnv_base), test_vns, test_vals ) - @testset "$vn (different size)" for vn in keys(test_pairs) + @testset "$vn (different size)" for vn in test_vns val_original = test_pairs[vn] val = change_size_for_test(val_original) vn_already_present = haskey(vnv, vn) From 0900c57bd1c69da53503355e254a5640fc347a3b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:25:18 +0000 Subject: [PATCH 024/209] style --- test/varnamevector.jl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 1bbbf450f..b6a7f1e33 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -205,9 +205,7 @@ end # `push!` & `update!` @testset "push!" begin - vnv = relax_container_types( - deepcopy(vnv_base), test_vns, test_vals - ) + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn" for vn in test_vns val = test_pairs[vn] if vn == vn_left || vn == vn_right @@ -220,9 +218,7 @@ end end end @testset "update!" begin - vnv = relax_container_types( - deepcopy(vnv_base), test_vns, test_vals - ) + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn" for vn in test_vns val = test_pairs[vn] expected_length = if haskey(vnv, vn) @@ -245,9 +241,7 @@ end # we required either a) the underlying `transforms` to be non-concrete, # or b) the sizes of the values to match. But now the sizes of the values # will change, so we can only test the former. - vnv = relax_container_types( - deepcopy(vnv_base), test_vns, test_vals - ) + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn (different size)" for vn in test_vns val_original = test_pairs[vn] val = change_size_for_test(val_original) From 1f7e633980220df465612841e5b9c490cb14c8af Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:58:18 +0000 Subject: [PATCH 025/209] added testing of `update!` with smaller sizes and fixed bug related to this --- src/varnamevector.jl | 13 ++++++- test/varnamevector.jl | 81 +++++++++++++++++++++++++++++++++---------- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index a9094b28a..5f5233ecb 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -209,7 +209,18 @@ end function nextrange(vnv::VarNameVector, x) # NOTE: Need to treat `isempty(vnv.ranges)` separately because `maximum` # will error if `vnv.ranges` is empty. - offset = isempty(vnv.ranges) ? 0 : maximum(last, vnv.ranges) + max_active_range = isempty(vnv.ranges) ? 0 : maximum(last, vnv.ranges) + # Also need to consider inactive ranges, since we can have scenarios such as + # + # vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2, 3]) + # update!(vnv, @varname(y), [4]) # => `ranges = [1:1, 2:2], inactive_ranges = [3:3]` + # + # Here `nextrange(vnv, [5])` should return `4:4`, _not_ `3:3`. + # NOTE: We could of course attempt to make use of unused space, e.g. if we have an inactive + # range which can hold `x`, then we could just use that. Buuut the complexity of this is + # probably not worth it (at least at the moment). + max_inactive_range = isempty(vnv.inactive_ranges) ? 0 : maximum(last, vnv.inactive_ranges) + offset = max(max_active_range, max_inactive_range) return (offset + 1):(offset + length(x)) end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index b6a7f1e33..7eaa8285a 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -1,7 +1,11 @@ replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) -change_size_for_test(x::Real) = [x] -change_size_for_test(x::AbstractArray) = repeat(x, 2) +increase_size_for_test(x::Real) = [x] +increase_size_for_test(x::AbstractArray) = repeat(x, 2) + +decrease_size_for_test(x::Real) = x +decrease_size_for_test(x::AbstractVector) = first(x) +decrease_size_for_test(x::AbstractArray) = first(eachslice(x; dims=1)) function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) if isconcretetype(eltype(vnv.varnames)) @@ -16,6 +20,9 @@ function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) return false end +function need_varnames_relaxation(vnv::VarNameVector, vns, vals) + return any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) if isconcretetype(eltype(vnv.vals)) @@ -24,6 +31,9 @@ function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) return false end +function need_values_relaxation(vnv::VarNameVector, vns, vals) + return any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) if isconcretetype(eltype(vnv.transforms)) @@ -36,12 +46,35 @@ function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) return false end +function need_transforms_relaxation(vnv::VarNameVector, vns, vals) + return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end + +""" + relax_container_types(vnv::VarNameVector, vn::VarName, val) + relax_container_types(vnv::VarNameVector, vns, val) + +Relax the container types of `vnv` if necessary to accommodate `vn` and `val`. + +This attempts to avoid unnecessary container type relaxations by checking whether +the container types of `vnv` are already compatible with `vn` and `val`. +# Notes +For example, if `vn` is not compatible with the current keys in `vnv`, then +the underlying types will be changed to `VarName` to accommodate `vn`. + +Similarly: +- If `val` is not compatible with the current values in `vnv`, then + the underlying value type will be changed to `Real`. +- If `val` requires a transformation that is not compatible with the current + transformations type in `vnv`, then the underlying transformation type will + be changed to `Any`. +""" function relax_container_types(vnv::VarNameVector, vn::VarName, val) return relax_container_types(vnv, [vn], [val]) end function relax_container_types(vnv::VarNameVector, vns, vals) - if any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) + if need_varnames_relaxation(vnv, vns, vals) varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) varnames_new = convert(Vector{VarName}, vnv.varnames) else @@ -49,15 +82,14 @@ function relax_container_types(vnv::VarNameVector, vns, vals) varnames_new = vnv.varnames end - transforms_new = - if any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) - convert(Vector{Any}, vnv.transforms) - else - vnv.transforms - end + transforms_new = if need_transforms_relaxation(vnv, vns, vals) + convert(Vector{Any}, vnv.transforms) + else + vnv.transforms + end - vals_new = if any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) - convert(Vector{Any}, vnv.vals) + vals_new = if need_values_relaxation(vnv, vns, vals) + convert(Vector{Real}, vnv.vals) else vnv.vals end @@ -102,7 +134,7 @@ end @varname(z[3]) => rand(1:10, 2, 3), ) test_vns = collect(keys(test_pairs)) - test_vals = collect(test_vals) + test_vals = collect(values(test_pairs)) @testset "constructor: no args" begin # Empty. @@ -237,14 +269,10 @@ end @test !DynamicPPL.has_inactive_ranges(vnv) end - # Need to recompute valid varnames for the changing of the sizes; before - # we required either a) the underlying `transforms` to be non-concrete, - # or b) the sizes of the values to match. But now the sizes of the values - # will change, so we can only test the former. vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn (different size)" for vn in test_vns + @testset "$vn (increased size)" for vn in test_vns val_original = test_pairs[vn] - val = change_size_for_test(val_original) + val = increase_size_for_test(val_original) vn_already_present = haskey(vnv, vn) expected_length = if vn_already_present # If it's already present, the resulting length will be altered. @@ -258,6 +286,23 @@ end @test length(vnv) == expected_length @test length(vnv[:]) == length(vnv) end + + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) + @testset "$vn (decreased size)" for vn in test_vns + val_original = test_pairs[vn] + val = decrease_size_for_test(val_original) + vn_already_present = haskey(vnv, vn) + expected_length = if vn_already_present + # If it's already present, the resulting length will be altered. + length(vnv) + length(val) - length(val_original) + else + length(vnv) + length(val) + end + DynamicPPL.update!(vnv, vn, val .+ 1) + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(vnv[:]) == length(vnv) + end end end end From 8d05586cb16dfa91d0c8d350191c8cf9118a798f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 10:59:09 +0000 Subject: [PATCH 026/209] formatting --- src/varnamevector.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 5f5233ecb..93272a063 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -219,7 +219,8 @@ function nextrange(vnv::VarNameVector, x) # NOTE: We could of course attempt to make use of unused space, e.g. if we have an inactive # range which can hold `x`, then we could just use that. Buuut the complexity of this is # probably not worth it (at least at the moment). - max_inactive_range = isempty(vnv.inactive_ranges) ? 0 : maximum(last, vnv.inactive_ranges) + max_inactive_range = + isempty(vnv.inactive_ranges) ? 0 : maximum(last, vnv.inactive_ranges) offset = max(max_active_range, max_inactive_range) return (offset + 1):(offset + length(x)) end From 7801fe1c0afe694cda5c645e5b9846459abd10b8 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 14 Nov 2023 11:11:55 +0000 Subject: [PATCH 027/209] move functionality related to `push!` for `VarNameVector` into `push!` --- src/varnamevector.jl | 85 +++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 93272a063..4f6a3b7f9 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -121,6 +121,7 @@ Base.length(vnv::VarNameVector) = end Base.size(vnv::VarNameVector) = (length(vnv),) +# TODO: We should probably remove this Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. @@ -229,59 +230,61 @@ end function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) - return update!(vnv, vn, val, transform) + # NOTE: We need to compute the `nextrange` BEFORE we start mutating + # the underlying; otherwise we might get some strange behaviors. + val_vec = tovec(val) + r_new = nextrange(vnv, val_vec) + vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 + push!(vnv.varnames, vn) + push!(vnv.ranges, r_new) + append!(vnv.vals, val_vec) + push!(vnv.transforms, transform) + return nothing end # `update!` and `update!!`: update a variable in the varname vector. function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) - val_vec = tovec(val) if !haskey(vnv, vn) # Here we just add a new entry. - # NOTE: We need to compute the `nextrange` BEFORE we start mutating - # the underlying; otherwise we might get some strange behaviors. + return push!(vnv, vn, val, transform) + end + + # Here we update an existing entry. + val_vec = tovec(val) + idx = getidx(vnv, vn) + r_old = getrange(vnv, idx) + n_old = length(r_old) + n_new = length(val_vec) + # Existing keys needs to be handled differently depending on + # whether the size of the value is increasing or decreasing. + if n_new > n_old + # Add the new range. r_new = nextrange(vnv, val_vec) - vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 - push!(vnv.varnames, vn) - push!(vnv.ranges, r_new) - append!(vnv.vals, val_vec) - push!(vnv.transforms, transform) + vnv.ranges[idx] = r_new + # Grow the underlying vector to accomodate the new value. + resize!(vnv.vals, r_new[end]) + # Keep track of the deleted ranges. + push!(vnv.inactive_ranges, r_old) else - # Here we update the existing entry. - idx = getidx(vnv, vn) - r_old = getrange(vnv, idx) - n_old = length(r_old) - n_new = length(val_vec) - # Existing keys needs to be handled differently depending on - # whether the size of the value is increasing or decreasing. - if n_new > n_old - # Add the new range. - r_new = nextrange(vnv, val_vec) - vnv.ranges[idx] = r_new - # Grow the underlying vector to accomodate the new value. - resize!(vnv.vals, r_new[end]) - # Keep track of the deleted ranges. - push!(vnv.inactive_ranges, r_old) - else - # `n_new <= n_old` - # Just decrease the current range. - r_new = r_old[1]:(r_old[1] + n_new - 1) - vnv.ranges[idx] = r_new - # And mark the rest as inactive if needed. - if n_new < n_old - push!(vnv.inactive_ranges, r_old[n_new]:r_old[end]) - end + # `n_new <= n_old` + # Just decrease the current range. + r_new = r_old[1]:(r_old[1]+n_new-1) + vnv.ranges[idx] = r_new + # And mark the rest as inactive if needed. + if n_new < n_old + push!(vnv.inactive_ranges, r_old[n_new]:r_old[end]) end + end - # Update the value. - vnv.vals[r_new] = val_vec - # Update the transform. - vnv.transforms[idx] = transform + # Update the value. + vnv.vals[r_new] = val_vec + # Update the transform. + vnv.transforms[idx] = transform - # TODO: Should we maybe sweep over inactive ranges and re-contiguify - # if we the total number of inactive elements is "large" in some sense? - end + # TODO: Should we maybe sweep over inactive ranges and re-contiguify + # if we the total number of inactive elements is "large" in some sense? - return vnv + return nothing end function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) From cdc2373457b7fcfab18b1e09aa7ae182659b20f1 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 16 Nov 2023 14:20:08 +0000 Subject: [PATCH 028/209] Update src/varnamevector.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/varnamevector.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 4f6a3b7f9..cf3671594 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -268,7 +268,7 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) else # `n_new <= n_old` # Just decrease the current range. - r_new = r_old[1]:(r_old[1]+n_new-1) + r_new = r_old[1]:(r_old[1] + n_new - 1) vnv.ranges[idx] = r_new # And mark the rest as inactive if needed. if n_new < n_old From ae4bcb7e0ec1b1f4838daa2b21e2bf68a1f95fad Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 30 Dec 2023 10:03:51 +0000 Subject: [PATCH 029/209] several fixes to make sampling with VarNameVector + initiall tests for sampling with VarNameVector --- src/test_utils.jl | 15 +++++++-- src/utils.jl | 2 +- src/varinfo.jl | 76 ++++++++++++++++++++++++++++++++++++++----- src/varnamevector.jl | 11 ++++++- test/test_util.jl | 1 + test/varnamevector.jl | 61 ++++++++++++++++++++++++++++++++++ 6 files changed, 152 insertions(+), 14 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index 6323f4dab..e79aad9d1 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -30,9 +30,9 @@ end Test that `vi[vn]` corresponds to the correct value in `vals` for every `vn` in `vns`. """ -function test_values(vi::AbstractVarInfo, vals::NamedTuple, vns; isequal=isequal, kwargs...) +function test_values(vi::AbstractVarInfo, vals::NamedTuple, vns; compare=isequal, kwargs...) for vn in vns - @test isequal(vi[vn], get(vals, vn); kwargs...) + @test compare(vi[vn], get(vals, vn); kwargs...) end end @@ -52,6 +52,8 @@ function setup_varinfos( vi_untyped = VarInfo() model(vi_untyped) vi_typed = DynamicPPL.TypedVarInfo(vi_untyped) + vi_vnv = DynamicPPL.VectorVarInfo(vi_untyped) + vi_vnv_typed = DynamicPPL.VectorVarInfo(vi_typed) # SimpleVarInfo svi_typed = SimpleVarInfo(example_values) svi_untyped = SimpleVarInfo(OrderedDict()) @@ -62,7 +64,14 @@ function setup_varinfos( lp = getlogp(vi_typed) varinfos = map(( - vi_untyped, vi_typed, svi_typed, svi_untyped, svi_typed_ref, svi_untyped_ref + vi_untyped, + vi_typed, + vi_vnv, + vi_vnv_typed, + svi_typed, + svi_untyped, + svi_typed_ref, + svi_untyped_ref, )) do vi # Set them all to the same values. DynamicPPL.setlogp!!(update_values!!(vi, example_values, varnames), lp) diff --git a/src/utils.jl b/src/utils.jl index ca068b1dc..07fb63b60 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -262,7 +262,7 @@ end reconstruct(d::Distribution, val::AbstractVector) = reconstruct(size(d), val) reconstruct(::Tuple{}, val::AbstractVector) = val[1] reconstruct(s::NTuple{1}, val::AbstractVector) = copy(val) -reconstruct(s::NTuple{2}, val::AbstractVector) = reshape(copy(val), s) +reconstruct(s::Tuple, val::AbstractVector) = reshape(copy(val), s) function reconstruct!(r, d::Distribution, val::AbstractVector) return reconstruct!(r, d, val) end diff --git a/src/varinfo.jl b/src/varinfo.jl index e417af00c..e450dd846 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -126,6 +126,47 @@ function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) ) end +# No-op if we're already working with a `VarNameVector`. +metadata_to_varnamevector(vnv::VarNameVector) = vnv +function metadata_to_varnamevector(md::Metadata) + idcs = md.idcs + vns = md.vns + ranges = md.ranges + vals = md.vals + transforms = map(md.dists) do dist + # TODO: Handle linked distributions. + FromVec(size(dist)) + end + + return VarNameVector( + OrderedDict{eltype(keys(idcs)),Int}(idcs), + vns, + ranges, + vals, + transforms, + ) +end + +function VectorVarInfo(vi::UntypedVarInfo) + md = metadata_to_varnamevector(vi.metadata) + lp = getlogp(vi) + return VarInfo( + md, + Base.RefValue{eltype(lp)}(lp), + Ref(get_num_produce(vi)), + ) +end + +function VectorVarInfo(vi::TypedVarInfo) + md = map(metadata_to_varnamevector, vi.metadata) + lp = getlogp(vi) + return VarInfo( + md, + Base.RefValue{eltype(lp)}(lp), + Ref(get_num_produce(vi)), + ) +end + function VarInfo( rng::Random.AbstractRNG, model::Model, @@ -549,6 +590,10 @@ end function setval!(md::Metadata, val, vn::VarName) return md.vals[getrange(md, vn)] = vectorize(getdist(md, vn), val) end +function setval!(vnv::VarNameVector, val, vn::VarName) + return setindex_raw!(vnv, tovec(val), vn) +end + """ getval(vi::VarInfo, vns::Vector{<:VarName}) @@ -1421,12 +1466,15 @@ function getindex(vi::VarInfo, vn::VarName, dist::Distribution) val = getval(vi, vn) return maybe_invlink_and_reconstruct(vi, vn, dist, val) end -function getindex(vi::VectorVarInfo, vn::VarName, ::Nothing) +# HACK: Allows us to also work with `VarNameVector` where `dist` is not used, +# but we instead use a transformation stored with the variable. +function getindex(vi::VarInfo, vn::VarName, ::Nothing) if !haskey(vi, vn) throw(KeyError(vn)) end return getmetadata(vi, vn)[vn] end + function getindex(vi::VarInfo, vns::Vector{<:VarName}) # FIXME(torfjelde): Using `getdist(vi, first(vns))` won't be correct in cases # such as `x .~ [Normal(), Exponential()]`. @@ -1543,9 +1591,10 @@ Check whether `vn` has been sampled in `vi`. """ haskey(vi::VarInfo, vn::VarName) = haskey(getmetadata(vi, vn), vn) function haskey(vi::TypedVarInfo, vn::VarName) - metadata = vi.metadata - Tmeta = typeof(metadata) - return getsym(vn) in fieldnames(Tmeta) && haskey(getmetadata(vi, vn).idcs, vn) + md_haskey = map(vi.metadata) do metadata + haskey(metadata, vn) + end + return any(md_haskey) end function Base.show(io::IO, ::MIME"text/plain", vi::UntypedVarInfo) @@ -1640,12 +1689,14 @@ Set the `order` of `vn` in `vi` to `index`, where `order` is the number of `obse statements run before sampling `vn`. """ function setorder!(vi::VarInfo, vn::VarName, index::Int) - metadata = getmetadata(vi, vn) - if metadata.orders[metadata.idcs[vn]] != index - metadata.orders[metadata.idcs[vn]] = index - end + setorder!(getmetadata(vi, vn), vn, index) return vi end +function setorder!(metadata::Metadata, vn::VarName, index::Int) + metadata.orders[metadata.idcs[vn]] = index + return metadata +end +setorder!(vnv::VarNameVector, ::VarName, ::Int) = vnv """ getorder(vi::VarInfo, vn::VarName) @@ -1671,6 +1722,8 @@ end function is_flagged(metadata::Metadata, vn::VarName, flag::String) return metadata.flags[flag][getidx(metadata, vn)] end +# HACK: This is bad. Should we always return `true` here? +is_flagged(::VarNameVector, ::VarName, flag::String) = flag == "del" ? true : false """ unset_flag!(vi::VarInfo, vn::VarName, flag::String) @@ -1678,9 +1731,14 @@ end Set `vn`'s value for `flag` to `false` in `vi`. """ function unset_flag!(vi::VarInfo, vn::VarName, flag::String) - getmetadata(vi, vn).flags[flag][getidx(vi, vn)] = false + unset_flag!(getmetadata(vi, vn), vn, flag) return vi end +function unset_flag!(metadata::Metadata, vn::VarName, flag::String) + metadata.flags[flag][getidx(vi, vn)] = false + return metadata +end +unset_flag!(vnv::VarNameVector, ::VarName, ::String) = vnv """ set_retained_vns_del_by_spl!(vi::VarInfo, spl::Sampler) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 4f6a3b7f9..9e8c79b94 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -1,7 +1,15 @@ # Similar to `Metadata` but representing a `Vector` and simpler interface. # TODO: Should we subtype `AbstractVector` or `AbstractDict`? +""" + VarNameVector + +A container that works like a `Vector` and an `OrderedDict` but is neither. + +# Fields +$(FIELDS) +""" struct VarNameVector{ - K,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData + K<:VarName,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData } "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" varname_to_index::OrderedDict{K,Int} @@ -64,6 +72,7 @@ tovec(x::AbstractArray) = vec(x) Bijectors.inverse(f::FromVec) = tovec Bijectors.inverse(f::FromVec{Tuple{}}) = tovec +# More convenient constructors. collect_maybe(x) = collect(x) collect_maybe(x::AbstractArray) = x diff --git a/test/test_util.jl b/test/test_util.jl index 64832f51e..22bfa46cc 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -86,6 +86,7 @@ short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" short_varinfo_name(::TypedVarInfo) = "TypedVarInfo" short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" +short_varinfo_name(::DynamicPPL.VectorVarInfo) = "VectorVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 7eaa8285a..65580c8b7 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -306,3 +306,64 @@ end end end end + +has_varnamevector(vi) = false +function has_varnamevector(vi::VarInfo) + return vi.metadata isa VarNameVector || + (vi isa TypedVarInfo && first(values(vi.metadata)) isa VarNameVector) +end + +@testset "VarInfo + VarNameVector" begin + models = [ + DynamicPPL.TestUtils.demo_assume_index_observe(), + DynamicPPL.TestUtils.demo_assume_observe_literal(), + DynamicPPL.TestUtils.demo_assume_literal_dot_observe(), + ] + + @testset "$(model.f)" for model in models + # TODO: Does not currently work with `get_and_set_val!` and thus not with + # `dot_tilde_assume`. + # NOTE: Need to set random seed explicitly to avoid using the same seed + # for initialization as for sampling in the inner testset below. + Random.seed!(42) + value_true = DynamicPPL.TestUtils.rand_prior_true(model) + vns = DynamicPPL.TestUtils.varnames(model) + varnames = DynamicPPL.TestUtils.varnames(model) + varinfos = DynamicPPL.TestUtils.setup_varinfos( + model, value_true, varnames; include_threadsafe=false + ) + # Filter out those which are not based on `VarNameVector`. + varinfos = filter(has_varnamevector, varinfos) + # Get the true log joint. + logp_true = DynamicPPL.TestUtils.logjoint_true(model, value_true...) + + @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos + # Need to make sure we're using a different random seed from the + # one used in the above call to `rand_prior_true`. + Random.seed!(43) + + # Are values correct? + DynamicPPL.TestUtils.test_values(varinfo, value_true, vns) + + # Is evaluation correct? + varinfo_eval = last( + DynamicPPL.evaluate!!(model, deepcopy(varinfo), DefaultContext()) + ) + # Log density should be the same. + @test getlogp(varinfo_eval) ≈ logp_true + # Values should be the same. + DynamicPPL.TestUtils.test_values(varinfo_eval, value_true, vns) + + # Is sampling correct? + varinfo_sample = last( + DynamicPPL.evaluate!!(model, deepcopy(varinfo), SamplingContext()) + ) + # Log density should be different. + @test getlogp(varinfo_sample) != getlogp(varinfo) + # Values should be different. + DynamicPPL.TestUtils.test_values( + varinfo_sample, value_true, vns; compare=!isequal + ) + end + end +end From 97e1bccb4c35ecbe9d97037c112502a96ac426b7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 30 Dec 2023 10:20:11 +0000 Subject: [PATCH 030/209] VarInfo + VarNameVector tests for all demo models --- test/varnamevector.jl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 65580c8b7..b52ad3880 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -314,15 +314,8 @@ function has_varnamevector(vi::VarInfo) end @testset "VarInfo + VarNameVector" begin - models = [ - DynamicPPL.TestUtils.demo_assume_index_observe(), - DynamicPPL.TestUtils.demo_assume_observe_literal(), - DynamicPPL.TestUtils.demo_assume_literal_dot_observe(), - ] - + models = DynamicPPL.TestUtils.DEMO_MODELS @testset "$(model.f)" for model in models - # TODO: Does not currently work with `get_and_set_val!` and thus not with - # `dot_tilde_assume`. # NOTE: Need to set random seed explicitly to avoid using the same seed # for initialization as for sampling in the inner testset below. Random.seed!(42) From ad343f36af414876e4203dd789617609a6b20ce5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 30 Dec 2023 14:25:04 +0000 Subject: [PATCH 031/209] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/varinfo.jl | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index e450dd846..dfc49c261 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -139,32 +139,20 @@ function metadata_to_varnamevector(md::Metadata) end return VarNameVector( - OrderedDict{eltype(keys(idcs)),Int}(idcs), - vns, - ranges, - vals, - transforms, + OrderedDict{eltype(keys(idcs)),Int}(idcs), vns, ranges, vals, transforms ) end function VectorVarInfo(vi::UntypedVarInfo) md = metadata_to_varnamevector(vi.metadata) lp = getlogp(vi) - return VarInfo( - md, - Base.RefValue{eltype(lp)}(lp), - Ref(get_num_produce(vi)), - ) + return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end function VectorVarInfo(vi::TypedVarInfo) md = map(metadata_to_varnamevector, vi.metadata) lp = getlogp(vi) - return VarInfo( - md, - Base.RefValue{eltype(lp)}(lp), - Ref(get_num_produce(vi)), - ) + return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end function VarInfo( @@ -594,7 +582,6 @@ function setval!(vnv::VarNameVector, val, vn::VarName) return setindex_raw!(vnv, tovec(val), vn) end - """ getval(vi::VarInfo, vns::Vector{<:VarName}) From f707b253ec031c6446d9433402df024a2b1e8c3e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 31 Dec 2023 18:25:22 +0000 Subject: [PATCH 032/209] added docs on the design of `VarNameVector` --- docs/src/api.md | 179 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 128 insertions(+), 51 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 9b98f9dc6..c562b6990 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -197,13 +197,139 @@ Please see the documentation of [AbstractPPL.jl](https://github.com/TuringLang/A ### Data Structures of Variables -DynamicPPL provides different data structures for samples from the model and their log density. -All of them are subtypes of [`AbstractVarInfo`](@ref). +DynamicPPL provides different data structures used in for storing samples and accumulation of the log-probabilities, all of which are subtypes of [`AbstractVarInfo`](@ref). ```@docs AbstractVarInfo ``` +But exactly how a [`AbstractVarInfo`](@ref) stores this information can vary. + +#### `VarInfo` + +```@docs +VarInfo +TypedVarInfo +``` + +One main characteristic of [`VarInfo`](@ref) is that samples are stored in a linearized form. + +```@docs +link! +invlink! +``` + +```@docs +set_flag! +unset_flag! +is_flagged +``` + +For Gibbs sampling the following functions were added. + +```@docs +setgid! +updategid! +``` + +The following functions were used for sequential Monte Carlo methods. + +```@docs +get_num_produce +set_num_produce! +increment_num_produce! +reset_num_produce! +setorder! +set_retained_vns_del_by_spl! +``` + +```@docs +Base.empty! +``` + +#### `SimpleVarInfo` + +```@docs +SimpleVarInfo +``` + +#### `VarNameVector` + +```@docs +VarNameVector +``` + +#### Design + +[`VarInfo`](@ref) is a fairly simple structure; it contains +- a `logp` field for accumulation of the log-density evaluation, and +- a `metadata` field for storing information about the realizations of the different variables. + +Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. + +**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. + +!!! note + We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. + +To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: +- `keys(::Dict)`: return all the `VarName`s present in `metadata`. +- `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. +- `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. +- `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. +- `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. +- `empty!(::Dict)`: delete all realizations in `metadata`. +- `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. + +*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Hence we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: +- `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + - For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. +- `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. +- `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + +Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: +1. Type-stable when possible, but still functional when not. +2. Efficient storage and iteration. + +`VarNameVector` is a data structure that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. + +##### Type-stability + +This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. The way we approach this in `VarInfo` is to construct a `NamedTuple` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```julia +x ~ Bernoulli(0.5) +y ~ Normal(0, 1) +``` + +then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where `Vx` is a container with `eltype` `Bool` and `Vy` is a container with `eltype` `Float64`. Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. + +!!! warning + Of course, this `NamedTuple` approach is *not* going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. + + ```julia + x[1] ~ Bernoulli(0.5) + x[2] ~ Normal(0, 1) + ``` + + In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is not type-stable but will still be functional. + + In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. + +Hence, given a `VarNameVector` as outlined in the previous section, we can convert this into a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. + +##### Efficient storage and iteration + +We can achieve this nicely by storing the values for different `VarName`s contiguously in a `Vector{<:Real}` and store the `VarName`s in the order they appear in the `Vector{<:Real}` in a `OrderedDict{VarName, UnitRange{Int}}`, mapping each `VarName` to the range of indices in the `Vector{<:Real}` that correspond to its values. This is the approach taken in [`VarNameVector`](@ref). + +##### Additional methods + +We also want some additional methods that are not part of the `Dict` or `Vector` interface: +- `push!(container, ::VarName, value)` to add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + +In addition, we want to be able to access the "transformed" / unconstrained realization for a particular `VarName` and so we also need corresponding methods for this: +- `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + ### Common API #### Accumulation of log-probabilities @@ -262,55 +388,6 @@ DynamicPPL.varname_leaves DynamicPPL.varname_and_value_leaves ``` -#### `SimpleVarInfo` - -```@docs -SimpleVarInfo -``` - -#### `VarInfo` - -Another data structure is [`VarInfo`](@ref). - -```@docs -VarInfo -TypedVarInfo -``` - -One main characteristic of [`VarInfo`](@ref) is that samples are stored in a linearized form. - -```@docs -link! -invlink! -``` - -```@docs -set_flag! -unset_flag! -is_flagged -``` - -For Gibbs sampling the following functions were added. - -```@docs -setgid! -updategid! -``` - -The following functions were used for sequential Monte Carlo methods. - -```@docs -get_num_produce -set_num_produce! -increment_num_produce! -reset_num_produce! -setorder! -set_retained_vns_del_by_spl! -``` - -```@docs -Base.empty! -``` ### Evaluation Contexts From f1faf18dfafb9fa371a93335f19cab58206dc027 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 31 Dec 2023 18:31:08 +0000 Subject: [PATCH 033/209] Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/src/api.md | 46 +++++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index c562b6990..70a5ec99a 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -261,35 +261,41 @@ VarNameVector #### Design -[`VarInfo`](@ref) is a fairly simple structure; it contains -- a `logp` field for accumulation of the log-density evaluation, and -- a `metadata` field for storing information about the realizations of the different variables. +[`VarInfo`](@ref) is a fairly simple structure; it contains + + - a `logp` field for accumulation of the log-density evaluation, and + - a `metadata` field for storing information about the realizations of the different variables. Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. **Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. !!! note + We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: -- `keys(::Dict)`: return all the `VarName`s present in `metadata`. -- `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. -- `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. -- `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. -- `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. -- `empty!(::Dict)`: delete all realizations in `metadata`. -- `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. + + - `keys(::Dict)`: return all the `VarName`s present in `metadata`. + - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. + - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. + - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. + - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. + - `empty!(::Dict)`: delete all realizations in `metadata`. + - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. *But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Hence we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: -- `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. - - For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. -- `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. -- `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + + - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + + + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. + - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. + - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: -1. Type-stable when possible, but still functional when not. -2. Efficient storage and iteration. + + 1. Type-stable when possible, but still functional when not. + 2. Efficient storage and iteration. `VarNameVector` is a data structure that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. @@ -305,6 +311,7 @@ y ~ Normal(0, 1) then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where `Vx` is a container with `eltype` `Bool` and `Vy` is a container with `eltype` `Float64`. Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. !!! warning + Of course, this `NamedTuple` approach is *not* going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. ```julia @@ -325,10 +332,12 @@ We can achieve this nicely by storing the values for different `VarName`s contig ##### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: -- `push!(container, ::VarName, value)` to add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + + - `push!(container, ::VarName, value)` to add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. In addition, we want to be able to access the "transformed" / unconstrained realization for a particular `VarName` and so we also need corresponding methods for this: -- `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + + - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. ### Common API @@ -388,7 +397,6 @@ DynamicPPL.varname_leaves DynamicPPL.varname_and_value_leaves ``` - ### Evaluation Contexts Internally, both sampling and evaluation of log densities are performed with [`AbstractPPL.evaluate!!`](@ref). From 87d3d0110cbaf66973296a9f86dc01e1c3c1a36f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 31 Dec 2023 18:31:41 +0000 Subject: [PATCH 034/209] added note on `update!` --- docs/src/api.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/src/api.md b/docs/src/api.md index 70a5ec99a..b051ae2b8 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -333,7 +333,9 @@ We can achieve this nicely by storing the values for different `VarName`s contig We also want some additional methods that are not part of the `Dict` or `Vector` interface: - - `push!(container, ::VarName, value)` to add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. +- `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + +- `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. In addition, we want to be able to access the "transformed" / unconstrained realization for a particular `VarName` and so we also need corresponding methods for this: From 9c3b265ee55d8755a8d5ac57bbe5f71e10685894 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:05:24 +0000 Subject: [PATCH 035/209] further elaboration of the design of `VarInfo` and `VarNameVector` --- docs/src/api.md | 85 +++++++++++++++++++++++++++++++++++++++++-------- src/varinfo.jl | 19 +++++++++-- 2 files changed, 88 insertions(+), 16 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index b051ae2b8..52cd32061 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -284,31 +284,66 @@ To ensure that `varinfo` is simple and intuitive to work with, we need the under - `empty!(::Dict)`: delete all realizations in `metadata`. - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. -*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Hence we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: +*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. + - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + - `similar(::Vector{<:Real})`: return a new Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: 1. Type-stable when possible, but still functional when not. 2. Efficient storage and iteration. -`VarNameVector` is a data structure that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. +[`VarNameVector`](@ref) is a data structure that implements the above methods in a way that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. ##### Type-stability -This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. The way we approach this in `VarInfo` is to construct a `NamedTuple` *for each distinct `Symbol` used*. For example, if we have a model of the form +This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `VarNameVector` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```@example varnamevector-design +using DynamicPPL, Distributions + +@model function demo() + x ~ Bernoulli(0.5) + return y ~ Normal(0, 1) +end +``` + +then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where + + - `Vx` is a container with `eltype` `Bool`, and + - `Vy` is a container with `eltype` `Float64`. + +Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. + +For example, with the model above we have + +```@example varnamevector-design +# Type-unstable `VarInfo` +varinfo_untyped = DynamicPPL.VectorVarInfo(DynamicPPL.untyped_varinfo(demo())) +typeof(varinfo_untyped.metadata) +``` + +```@example varnamevector-design +# Type-stable `VarInfo` +varinfo_typed = DynamicPPL.VectorVarInfo(DynamicPPL.typed_varinfo(demo())) +typeof(varinfo_typed.metadata) +``` + +But they both work as expected: -```julia -x ~ Bernoulli(0.5) -y ~ Normal(0, 1) +```@example varnamevector-design +varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] ``` -then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where `Vx` is a container with `eltype` `Bool` and `Vy` is a container with `eltype` `Float64`. Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. +```@example varnamevector-design +varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] +``` !!! warning @@ -319,25 +354,49 @@ then we construct a type-stable representation by using a `NamedTuple{(:x, :y), x[2] ~ Normal(0, 1) ``` - In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is not type-stable but will still be functional. + In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. -Hence, given a `VarNameVector` as outlined in the previous section, we can convert this into a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. +!!! warning + + Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()`, this will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + +Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. ##### Efficient storage and iteration -We can achieve this nicely by storing the values for different `VarName`s contiguously in a `Vector{<:Real}` and store the `VarName`s in the order they appear in the `Vector{<:Real}` in a `OrderedDict{VarName, UnitRange{Int}}`, mapping each `VarName` to the range of indices in the `Vector{<:Real}` that correspond to its values. This is the approach taken in [`VarNameVector`](@ref). +In [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. + +This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: + + - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. + - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. + - `transforms::Vector`: the transforms associated with each `VarName`. + +Mutating functions, e.g. `setindex!`, are then treated according to the following rules: + + 1. If `VarName` is not already present: add it to the end of `varnames`, add the value to the underlying `Vector{T}`, etc. + + 2. If `VarName` is already present in the `VarNameVector`: + + 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. + 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. + 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. + +!!! note + + Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. ##### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: -- `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. -- `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. + - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. -In addition, we want to be able to access the "transformed" / unconstrained realization for a particular `VarName` and so we also need corresponding methods for this: +In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. diff --git a/src/varinfo.jl b/src/varinfo.jl index dfc49c261..767fac7f6 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -155,15 +155,28 @@ function VectorVarInfo(vi::TypedVarInfo) return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end -function VarInfo( +function untyped_varinfo( rng::Random.AbstractRNG, model::Model, sampler::AbstractSampler=SampleFromPrior(), context::AbstractContext=DefaultContext(), ) varinfo = VarInfo() - model(rng, varinfo, sampler, context) - return TypedVarInfo(varinfo) + return last(evaluate!!(model, varinfo, SamplingContext(rng, sampler, context))) +end +function untyped_varinfo(model::Model, args...) + return untyped_varinfo(Random.default_rng(), model, args...) +end + +typed_varinfo(args...) = TypedVarInfo(untyped_varinfo(args...)) + +function VarInfo( + rng::Random.AbstractRNG, + model::Model, + sampler::AbstractSampler=SampleFromPrior(), + context::AbstractContext=DefaultContext(), +) + return typed_varinfo(rng, model, sampler, context) end VarInfo(model::Model, args...) = VarInfo(Random.default_rng(), model, args...) From 958c66bc850ef8273823074460941b8b3ed20906 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:27:49 +0000 Subject: [PATCH 036/209] more writing improvements --- docs/src/api.md | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 52cd32061..50e19395c 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -299,13 +299,15 @@ Moreover, we want also want the underlying representation used in `metadata` to 1. Type-stable when possible, but still functional when not. 2. Efficient storage and iteration. -[`VarNameVector`](@ref) is a data structure that implements the above methods in a way that allows us to achieve all of the above when used as the `metadata` field of `VarInfo`. +In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). ##### Type-stability -This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `VarNameVector` *for each distinct `Symbol` used*. For example, if we have a model of the form +This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. -```@example varnamevector-design +Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```@example varinfo-design using DynamicPPL, Distributions @model function demo() @@ -323,31 +325,31 @@ Since `VarName` contains the `Symbol` used in its type, something like `getindex For example, with the model above we have -```@example varnamevector-design +```@example varinfo-design # Type-unstable `VarInfo` -varinfo_untyped = DynamicPPL.VectorVarInfo(DynamicPPL.untyped_varinfo(demo())) +varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) typeof(varinfo_untyped.metadata) ``` -```@example varnamevector-design +```@example varinfo-design # Type-stable `VarInfo` -varinfo_typed = DynamicPPL.VectorVarInfo(DynamicPPL.typed_varinfo(demo())) +varinfo_typed = DynamicPPL.typed_varinfo(demo()) typeof(varinfo_typed.metadata) ``` But they both work as expected: -```@example varnamevector-design +```@example varinfo-design varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] ``` -```@example varnamevector-design +```@example varinfo-design varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] ``` !!! warning - Of course, this `NamedTuple` approach is *not* going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. + Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. ```julia x[1] ~ Bernoulli(0.5) @@ -360,7 +362,7 @@ varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] !!! warning - Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()`, this will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. @@ -384,6 +386,13 @@ Mutating functions, e.g. `setindex!`, are then treated according to the followin 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. +This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient; if we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. This also means that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: + +```@docs +DynamicPPL.has_inactive_ranges +DynamicPPL.inactive_ranges_sweep! +``` + !!! note Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. From 74c6efd6316e162195533c8a59728164605ae565 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:27:58 +0000 Subject: [PATCH 037/209] added docstring to `has_inactive_ranges` and `inactive_ranges_sweep!` --- src/varnamevector.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 783d1e9c3..12b48ce59 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -118,6 +118,11 @@ getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] +""" + has_inactive_ranges(vnv::VarNameVector) + +Returns `true` if `vnv` has inactive ranges. +""" has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) # Basic array interface. @@ -307,6 +312,11 @@ function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) return ranges end +""" + inactive_ranges_sweep!(vnv::VarNameVector) + +Sweep over the inactive ranges in `vnv` and re-contiguify the underlying +""" function inactive_ranges_sweep!(vnv::VarNameVector) # Extract the re-contiguified values. # NOTE: We need to do this before we update the ranges. From d9ea878695b8ad3a439fc5bfa6c020f086754442 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:32:16 +0000 Subject: [PATCH 038/209] moved docs on `VarInfo` design to a separate internals section --- docs/make.jl | 1 + docs/src/api.md | 150 ------------------------------------------ docs/src/internals.md | 149 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 150 deletions(-) create mode 100644 docs/src/internals.md diff --git a/docs/make.jl b/docs/make.jl index 109f28a9c..056b90d7f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,6 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], + "Internals" => "internals.md", ], checkdocs=:exports, ) diff --git a/docs/src/api.md b/docs/src/api.md index 50e19395c..a7137ccd7 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -259,156 +259,6 @@ SimpleVarInfo VarNameVector ``` -#### Design - -[`VarInfo`](@ref) is a fairly simple structure; it contains - - - a `logp` field for accumulation of the log-density evaluation, and - - a `metadata` field for storing information about the realizations of the different variables. - -Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. - -**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. - -!!! note - - We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. - -To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: - - - `keys(::Dict)`: return all the `VarName`s present in `metadata`. - - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. - - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. - - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. - - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. - - `empty!(::Dict)`: delete all realizations in `metadata`. - - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. - -*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: - - - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. - - + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. - - - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - - `similar(::Vector{<:Real})`: return a new - -Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: - - 1. Type-stable when possible, but still functional when not. - 2. Efficient storage and iteration. - -In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). - -##### Type-stability - -This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. - -Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form - -```@example varinfo-design -using DynamicPPL, Distributions - -@model function demo() - x ~ Bernoulli(0.5) - return y ~ Normal(0, 1) -end -``` - -then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where - - - `Vx` is a container with `eltype` `Bool`, and - - `Vy` is a container with `eltype` `Float64`. - -Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. - -For example, with the model above we have - -```@example varinfo-design -# Type-unstable `VarInfo` -varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) -typeof(varinfo_untyped.metadata) -``` - -```@example varinfo-design -# Type-stable `VarInfo` -varinfo_typed = DynamicPPL.typed_varinfo(demo()) -typeof(varinfo_typed.metadata) -``` - -But they both work as expected: - -```@example varinfo-design -varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] -``` - -```@example varinfo-design -varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] -``` - -!!! warning - - Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. - - ```julia - x[1] ~ Bernoulli(0.5) - x[2] ~ Normal(0, 1) - ``` - - In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. - - In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. - -!!! warning - - Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. - -Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. - -##### Efficient storage and iteration - -In [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. - -This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: - - - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. - - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - - `transforms::Vector`: the transforms associated with each `VarName`. - -Mutating functions, e.g. `setindex!`, are then treated according to the following rules: - - 1. If `VarName` is not already present: add it to the end of `varnames`, add the value to the underlying `Vector{T}`, etc. - - 2. If `VarName` is already present in the `VarNameVector`: - - 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. - 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. - 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. - -This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient; if we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. This also means that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: - -```@docs -DynamicPPL.has_inactive_ranges -DynamicPPL.inactive_ranges_sweep! -``` - -!!! note - - Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. - -##### Additional methods - -We also want some additional methods that are not part of the `Dict` or `Vector` interface: - - - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. - - - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. - -In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - - - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. - ### Common API #### Accumulation of log-probabilities diff --git a/docs/src/internals.md b/docs/src/internals.md new file mode 100644 index 000000000..c1ce12325 --- /dev/null +++ b/docs/src/internals.md @@ -0,0 +1,149 @@ +## Design of `VarInfo` + +[`VarInfo`](@ref) is a fairly simple structure; it contains + + - a `logp` field for accumulation of the log-density evaluation, and + - a `metadata` field for storing information about the realizations of the different variables. + +Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. + +**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. + +!!! note + + We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. + +To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: + + - `keys(::Dict)`: return all the `VarName`s present in `metadata`. + - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. + - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. + - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. + - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. + - `empty!(::Dict)`: delete all realizations in `metadata`. + - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. + +*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: + + - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + + + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. + + - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. + - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + - `similar(::Vector{<:Real})`: return a new + +Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: + + 1. Type-stable when possible, but still functional when not. + 2. Efficient storage and iteration. + +In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). + +### Type-stability + +This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. + +Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```@example varinfo-design +using DynamicPPL, Distributions + +@model function demo() + x ~ Bernoulli(0.5) + return y ~ Normal(0, 1) +end +``` + +then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where + + - `Vx` is a container with `eltype` `Bool`, and + - `Vy` is a container with `eltype` `Float64`. + +Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. + +For example, with the model above we have + +```@example varinfo-design +# Type-unstable `VarInfo` +varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) +typeof(varinfo_untyped.metadata) +``` + +```@example varinfo-design +# Type-stable `VarInfo` +varinfo_typed = DynamicPPL.typed_varinfo(demo()) +typeof(varinfo_typed.metadata) +``` + +But they both work as expected: + +```@example varinfo-design +varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] +``` + +```@example varinfo-design +varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] +``` + +!!! warning + + Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. + + ```julia + x[1] ~ Bernoulli(0.5) + x[2] ~ Normal(0, 1) + ``` + + In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. + + In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. + +!!! warning + + Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + +Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. + +### Efficient storage and iteration + +In [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. + +This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: + + - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. + - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. + - `transforms::Vector`: the transforms associated with each `VarName`. + +Mutating functions, e.g. `setindex!`, are then treated according to the following rules: + + 1. If `VarName` is not already present: add it to the end of `varnames`, add the value to the underlying `Vector{T}`, etc. + + 2. If `VarName` is already present in the `VarNameVector`: + + 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. + 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. + 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. + +This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient; if we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. This also means that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: + +```@docs +DynamicPPL.has_inactive_ranges +DynamicPPL.inactive_ranges_sweep! +``` + +!!! note + + Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. + +### Additional methods + +We also want some additional methods that are not part of the `Dict` or `Vector` interface: + + - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + + - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. + +In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: + + - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. From 5acce98bed0a4bdfe0e1d830807154871b174ccf Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 12:51:58 +0000 Subject: [PATCH 039/209] writing improvements for internal docs --- docs/src/api.md | 6 ------ docs/src/internals.md | 26 +++++++++++++++++++------- src/varnamevector.jl | 2 +- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index a7137ccd7..f9db6603c 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -253,12 +253,6 @@ Base.empty! SimpleVarInfo ``` -#### `VarNameVector` - -```@docs -VarNameVector -``` - ### Common API #### Accumulation of log-probabilities diff --git a/docs/src/internals.md b/docs/src/internals.md index c1ce12325..01fc47ddc 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -103,11 +103,17 @@ varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. -Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a `VarNameVector` for all `VarName`s instead of a `NamedTuple` wrapping `VarNameVector`s. +Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. ### Efficient storage and iteration -In [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. +Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): + +```@docs +DynamicPPL.VarNameVector +``` + +In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: @@ -124,18 +130,24 @@ Mutating functions, e.g. `setindex!`, are then treated according to the followin 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. + +This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient. -This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient; if we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. This also means that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: +!!! note + + If we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. + +!!! note + + Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. + +This does mean that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: ```@docs DynamicPPL.has_inactive_ranges DynamicPPL.inactive_ranges_sweep! ``` -!!! note - - Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. - ### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 12b48ce59..cbe72f16e 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -315,7 +315,7 @@ end """ inactive_ranges_sweep!(vnv::VarNameVector) -Sweep over the inactive ranges in `vnv` and re-contiguify the underlying +Re-contiguify the underlying vector and shrink if possible. """ function inactive_ranges_sweep!(vnv::VarNameVector) # Extract the re-contiguified values. From 6f95cddb16912f2af6125f24c947e252ef7763f5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 15:22:38 +0000 Subject: [PATCH 040/209] further motivation of the design choices made in `VarNameVector` --- docs/src/internals.md | 119 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 112 insertions(+), 7 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 01fc47ddc..2040300b5 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -47,11 +47,11 @@ This is somewhat non-trivial to address since we want to achieve this for both c Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form ```@example varinfo-design -using DynamicPPL, Distributions - +using DynamicPPL, Distributions, FillArrays @model function demo() - x ~ Bernoulli(0.5) - return y ~ Normal(0, 1) + x ~ product_distribution(Fill(Bernoulli(0.5), 2)) + y ~ Normal(0, 1) + return nothing end ``` @@ -113,7 +113,7 @@ Efficient storage and iteration we achieve through implementation of the `metada DynamicPPL.VarNameVector ``` -In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve this by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. +In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve the desirata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: @@ -130,8 +130,67 @@ Mutating functions, e.g. `setindex!`, are then treated according to the followin 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. - -This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient. + +This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient in the scenarios we encounter in practice. + +In particular, we want to make sure that the following scenario is efficient: + + 1. Construct a [`VarInfo`](@ref) from a `[Model`](@ref). + 2. Repeatedly call `rand!(rng, ::Model, ::VarInfo)`. + +There are typically a few scenarios where we encounter changing representation sizes of a random variable `x`: + + 1. We're working with a transformed version `x` which is represented in a lower-dimensional space, e.g. transforming a `x ~ LKJ(2, 1)` to unconstrained `y = f(x)` takes us from 2-by-2 `Matrix{Float64}` to a 1-length `Vector{Float64}`. + 2. `x` has a random size, e.g. in a mixture model with a prior on the number of components. Here the size of `x` can vary widly between every realization of the `Model`. + +In scenario (1), the we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". + +In scenario (2), we'll end up with quite a sub-optimal representation unless we do something to handle this. For example: + +```@example varinfo-design +vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +println("Before insertion: $(length(vnv.vals))") + +for i in 2:5 + # Insert value 1 size larger. + DynamicPPL.update!(vnv, @varname(x), fill(true, i)) + println("After insertion #$(i): $(length(vnv.vals))") +end +``` + +To alleviate this issue, we can insert a call to [`DynamicPPL.inactive_ranges_sweep!`](@ref) after every insertion: + +```@example varinfo-design +vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +println("Before insertion: $(length(vnv.vals))") + +for i in 2:5 + # Insert value 1 size larger. + DynamicPPL.update!(vnv, @varname(x), fill(true, i)) + DynamicPPL.inactive_ranges_sweep!(vnv) + println("After insertion #$(i): $(length(vnv.vals))") +end +``` + +The nice aspect of this is that `vnv` can grow as needed, but once it is sufficiently large to contain the all the different realizations of `x`, it stops growing! + +```@example varinfo-design +vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +println("Before insertion: $(length(vnv.vals))") + +for i in 1:100 + DynamicPPL.update!(vnv, @varname(x), fill(true, rand(1:5))) + if DynamicPPL.has_inactive_ranges(vnv) + DynamicPPL.inactive_ranges_sweep!(vnv) + end +end + +println("After insertions: $(length(vnv.vals))") +``` + +Without the calls to [`DynamicPPL.inactive_ranges_sweep!`](@ref), the above would result in a much larger memory footprint. + +`delete!` and similars are also implemented using the "delete-by-mark" technique outlined above. !!! note @@ -148,6 +207,52 @@ DynamicPPL.has_inactive_ranges DynamicPPL.inactive_ranges_sweep! ``` +Continuing from the example from the previous section: + +```@example varinfo-design +# Type-unstable +varinfo_untyped_vnv = DynamicPPL.VectorVarInfo(varinfo_untyped) +varinfo_untyped_vnv[@varname(x)], varinfo_untyped_vnv[@varname(y)] +``` + +```@example varinfo-design +# Type-stable +varinfo_typed_vnv = DynamicPPL.VectorVarInfo(varinfo_typed) +varinfo_typed_vnv[@varname(x)], varinfo_typed_vnv[@varname(y)] +``` + +If we now try to `delete!` `@varname(x)` + +```@example varinfo-design +haskey(varinfo_untyped_vnv, @varname(x)) +``` + +```@example varinfo-design +DynamicPPL.has_inactive_ranges(varinfo_untyped_vnv.metadata) +``` + +```@example varinfo-design +# `delete!` +DynamicPPL.delete!(varinfo_untyped_vnv.metadata, @varname(x)) +DynamicPPL.has_inactive_ranges(varinfo_untyped_vnv.metadata) +``` + +```@example varinfo-design +haskey(varinfo_untyped_vnv, @varname(x)) +``` + +If we try to insert a differently-sized value for `@varname(x)` + +```@example varinfo-design +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 1)) +varinfo_untyped_vnv[@varname(x)] +``` + +```@example varinfo-design +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 3)) +varinfo_untyped_vnv[@varname(x)] +``` + ### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: From 38a4b08e43fff9b949785394de1600e27a99cd0b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 15:57:56 +0000 Subject: [PATCH 041/209] improved writing --- docs/src/internals.md | 50 ++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 2040300b5..b8cfe2681 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -133,10 +133,20 @@ Mutating functions, e.g. `setindex!`, are then treated according to the followin This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient in the scenarios we encounter in practice. -In particular, we want to make sure that the following scenario is efficient: +In particular, we want to optimize code-paths which effectively boil down to inner-loop in the following example:: - 1. Construct a [`VarInfo`](@ref) from a `[Model`](@ref). - 2. Repeatedly call `rand!(rng, ::Model, ::VarInfo)`. +```julia +# Construct a `VarInfo` with types inferred from `model`. +varinfo = VarInfo(model) + +# Repeatedly sample from `model`. +for _ = 1:num_samples + rand!(rng, model, varinfo) + + # Do something with `varinfo`. + # ... +end +``` There are typically a few scenarios where we encounter changing representation sizes of a random variable `x`: @@ -151,10 +161,10 @@ In scenario (2), we'll end up with quite a sub-optimal representation unless we vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) println("Before insertion: $(length(vnv.vals))") -for i in 2:5 - # Insert value 1 size larger. - DynamicPPL.update!(vnv, @varname(x), fill(true, i)) - println("After insertion #$(i): $(length(vnv.vals))") +for i in 1:5 + x = fill(true, rand(1:5)) + DynamicPPL.update!(vnv, @varname(x), x) + println("After insertion #$(i) of length $(length(x)): $(length(vnv.vals))") end ``` @@ -164,31 +174,17 @@ To alleviate this issue, we can insert a call to [`DynamicPPL.inactive_ranges_sw vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) println("Before insertion: $(length(vnv.vals))") -for i in 2:5 - # Insert value 1 size larger. - DynamicPPL.update!(vnv, @varname(x), fill(true, i)) +for i in 1:5 + x = fill(true, rand(1:5)) + DynamicPPL.update!(vnv, @varname(x), x) DynamicPPL.inactive_ranges_sweep!(vnv) - println("After insertion #$(i): $(length(vnv.vals))") + println("After insertion #$(i) of length $(length(x)): $(length(vnv.vals))") end ``` -The nice aspect of this is that `vnv` can grow as needed, but once it is sufficiently large to contain the all the different realizations of `x`, it stops growing! - -```@example varinfo-design -vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: $(length(vnv.vals))") - -for i in 1:100 - DynamicPPL.update!(vnv, @varname(x), fill(true, rand(1:5))) - if DynamicPPL.has_inactive_ranges(vnv) - DynamicPPL.inactive_ranges_sweep!(vnv) - end -end - -println("After insertions: $(length(vnv.vals))") -``` +Without the calls to [`DynamicPPL.inactive_ranges_sweep!`](@ref), the above would result in a much larger memory footprint. Of course, in this does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. -Without the calls to [`DynamicPPL.inactive_ranges_sweep!`](@ref), the above would result in a much larger memory footprint. +That is, we've decided to focus on optimizing scenario (1) but we still maintain functionality even in scenario (2), though at a greater computational cost. `delete!` and similars are also implemented using the "delete-by-mark" technique outlined above. From 60edd10f28e5c2759e2b681353ae8449b2c353c3 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Mon, 1 Jan 2024 19:36:45 +0000 Subject: [PATCH 042/209] VarNameVector is now grown as much as needed --- src/varnamevector.jl | 116 ++++++++++++++++++++++++++++++++++++++---- test/varnamevector.jl | 26 ++++++++++ 2 files changed, 131 insertions(+), 11 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index cbe72f16e..7e9f1ccde 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -27,7 +27,7 @@ struct VarNameVector{ transforms::TTrans "inactive ranges" - inactive_ranges::Vector{UnitRange{Int}} + inactive_ranges::OrderedDict{Int,UnitRange{Int}} "metadata associated with the varnames" metadata::MData @@ -45,7 +45,13 @@ end function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) return VarNameVector( - varname_to_index, varnames, ranges, vals, transforms, UnitRange{Int}[], nothing + varname_to_index, + varnames, + ranges, + vals, + transforms, + OrderedDict{Int,UnitRange{Int}}(), + nothing, ) end # TODO: Do we need this? @@ -216,7 +222,7 @@ function Base.similar(vnv::VarNameVector) similar(vnv.ranges, 0), similar(vnv.vals, 0), similar(vnv.transforms, 0), - similar(vnv.inactive_ranges, 0), + similar(vnv.inactive_ranges), similar_metadata(vnv.metadata), ) end @@ -235,7 +241,7 @@ function nextrange(vnv::VarNameVector, x) # range which can hold `x`, then we could just use that. Buuut the complexity of this is # probably not worth it (at least at the moment). max_inactive_range = - isempty(vnv.inactive_ranges) ? 0 : maximum(last, vnv.inactive_ranges) + isempty(vnv.inactive_ranges) ? 0 : maximum(last, values(vnv.inactive_ranges)) offset = max(max_active_range, max_inactive_range) return (offset + 1):(offset + length(x)) end @@ -256,6 +262,11 @@ function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val) return nothing end +function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) + x[(start + n):end] = x[start:(end - n)] + return x +end + # `update!` and `update!!`: update a variable in the varname vector. function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) if !haskey(vnv, vn) @@ -272,13 +283,47 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Existing keys needs to be handled differently depending on # whether the size of the value is increasing or decreasing. if n_new > n_old - # Add the new range. - r_new = nextrange(vnv, val_vec) - vnv.ranges[idx] = r_new + n_extra = n_new - n_old # Grow the underlying vector to accomodate the new value. - resize!(vnv.vals, r_new[end]) - # Keep track of the deleted ranges. - push!(vnv.inactive_ranges, r_old) + resize!(vnv.vals, length(vnv.vals) + n_extra) + # Shift the existing values to make room for the new value. + shift_right!(vnv.vals, r_old[end] + 1, n_extra) + # Compute the new range. + r_new = r_old[1]:(r_old[1] + n_new - 1) + vnv.ranges[idx] = r_new + # If we have an inactive range for this variable, we need to update it. + if haskey(vnv.inactive_ranges, idx) + # Then we need to change this to reflect the new range. + let inactive_range = vnv.inactive_ranges[idx] + # Inactive range should always start before the new range + # in the scenario where the new range is larger than the old range. + @assert inactive_range[1] <= r_new[end] + if inactive_range[end] <= r_new[end] + # Inactive range ends before the new range ends. + delete!(vnv.inactive_ranges, idx) + else + # Then we have `inactive_range[1] < r_new[end]`, i.e. + # Inactive range starts before the new range ends. + # AND the inactive range ends after the new range ends. + # => We need to split the inactive range. + vnv.inactive_ranges[idx] = (r_new[end] + 1):inactive_range[end] + end + end + end + + # Shift existing ranges which are after the current range. + for (i, r_i) in enumerate(vnv.ranges) + if r_i[1] > r_old[end] + vnv.ranges[i] = r_i .+ n_extra + end + end + # Shift inactive ranges coming after `r_old` (unless they are + # for the current variable). + for (i, r_i) in pairs(vnv.inactive_ranges) + if i !== idx && r_i[1] > r_old[end] + vnv.inactive_ranges[i] = r_i .+ n_extra + end + end else # `n_new <= n_old` # Just decrease the current range. @@ -286,7 +331,13 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) vnv.ranges[idx] = r_new # And mark the rest as inactive if needed. if n_new < n_old - push!(vnv.inactive_ranges, r_old[n_new]:r_old[end]) + if haskey(vnv.inactive_ranges, idx) + vnv.inactive_ranges[idx] = ( + (r_old[n_new] + 1):vnv.inactive_ranges[idx][end] + ) + else + vnv.inactive_ranges[idx] = ((r_old[n_new] + 1):r_old[end]) + end end end @@ -368,3 +419,46 @@ function Base.iterate(vnv::VarNameVector, state=nothing) vn, state_new = res return vn => getindex(vnv, vn), state_new end + +function Base.pairs(vnv::VarNameVector) + return Iterators.zip( + vnv.varnames, Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) + ) +end + +function Base.delete!(vnv::VarNameVector, vn::VarName) + # Error if we don't have the variable. + !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) + + # Get the index of the variable. + idx = getidx(vnv, vn) + # Get the range of the variable. + r_old = getrange(vnv, idx) + # Delete the variable. + delete!(vnv.varname_to_index, vn) + deleteat!(vnv.varnames, idx) + deleteat!(vnv.ranges, idx) + deleteat!(vnv.transforms, idx) + # Mark the range as inactive. + push!(vnv.inactive_ranges, r_old) + + return vnv +end + +function Base.deleteat!(vnv::VarNameVector, vn::VarName) + # Error if we don't have the variable. + !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) + # Get the index of the variable. + idx = getidx(vnv, vn) + # Get the range of the variable. + r_old = getrange(vnv, idx) + # Delete the variable. + delete!(vnv.varname_to_index, vn) + deleteat!(vnv.varnames, idx) + deleteat!(vnv.ranges, idx) + deleteat!(vnv.transforms, idx) + # Delete the range. + deleteat!(vnv.vals, r_old) + + return vnv +end diff --git a/test/varnamevector.jl b/test/varnamevector.jl index b52ad3880..97cde9ba1 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -305,6 +305,32 @@ end end end end + + @testset "growing and shrinking" begin + n = 5 + vn = @varname(x) + vnv = VarNameVector(OrderedDict(vn => [true])) + @test !DynamicPPL.has_inactive_ranges(vnv) + # Growing should not create inactive ranges. + for i in 1:n + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test !DynamicPPL.has_inactive_ranges(vnv) + end + + # Same size should not create inactive ranges. + x = fill(true, n) + DynamicPPL.update!(vnv, vn, x) + @test !DynamicPPL.has_inactive_ranges(vnv) + + # Shrinking should create inactive ranges. + for i in (n - 1):-1:1 + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test DynamicPPL.has_inactive_ranges(vnv) + @test vnv.inactive_ranges[1] == ((i + 1):n) + end + end end has_varnamevector(vi) = false From 3f9d34f2262c473d23dd1f3616aa3dda84a3f27e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 10:40:06 +0000 Subject: [PATCH 043/209] updated `delete!` --- src/varnamevector.jl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 7e9f1ccde..07b74e4c4 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -436,11 +436,21 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) r_old = getrange(vnv, idx) # Delete the variable. delete!(vnv.varname_to_index, vn) + # `deleteat!` deletes and shifts the rest of the vector. + # So after this, we need to re-adjust the indices in `varname_to_index`. deleteat!(vnv.varnames, idx) deleteat!(vnv.ranges, idx) deleteat!(vnv.transforms, idx) - # Mark the range as inactive. - push!(vnv.inactive_ranges, r_old) + + # Delete any inactive ranges corresponding to the variable. + if haskey(vnv.inactive_ranges, idx) + delete!(vnv.inactive_ranges, idx) + end + + # Re-adjust the indices in `varname_to_index`. + for (vn, idx) in vnv.varname_to_index + idx > idx && (vnv.varname_to_index[vn] = idx - 1) + end return vnv end From fb822b52643550f98331c97b55c0490ba683b833 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 12:30:54 +0000 Subject: [PATCH 044/209] Significant changes to implementation of `VarNameVector`: - "delete-by-mark" is now replaced by proper deletion. - `inactive_ranges` replaced by `num_inactive`, which only keeps track of the number of inactive entries for a given `VarName. - `VarNameVector` is now a "grow-as-needed" structure where the underlying also mimics the order that the user experiences.` --- src/varnamevector.jl | 207 +++++++++++++++++++++++++----------------- test/varnamevector.jl | 65 +++++++++---- 2 files changed, 167 insertions(+), 105 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 07b74e4c4..896c0c3a2 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -26,8 +26,8 @@ struct VarNameVector{ "vector of transformations whose inverse takes us back to the original space" transforms::TTrans - "inactive ranges" - inactive_ranges::OrderedDict{Int,UnitRange{Int}} + "additional entries which are considered inactive" + num_inactive::OrderedDict{Int,Int} "metadata associated with the varnames" metadata::MData @@ -39,7 +39,7 @@ function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) vnv_left.ranges == vnv_right.ranges && vnv_left.vals == vnv_right.vals && vnv_left.transforms == vnv_right.transforms && - vnv_left.inactive_ranges == vnv_right.inactive_ranges && + vnv_left.num_inactive == vnv_right.num_inactive && vnv_left.metadata == vnv_right.metadata end @@ -50,7 +50,7 @@ function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) ranges, vals, transforms, - OrderedDict{Int,UnitRange{Int}}(), + OrderedDict{Int,Int}(), nothing, ) end @@ -119,22 +119,47 @@ end # Some `VarNameVector` specific functions. getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] -getrange(vnv::VarNameVector, i::Int) = vnv.ranges[i] +getrange(vnv::VarNameVector, idx::Int) = vnv.ranges[idx] getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] """ - has_inactive_ranges(vnv::VarNameVector) + has_inactive(vnv::VarNameVector) Returns `true` if `vnv` has inactive ranges. """ -has_inactive_ranges(vnv::VarNameVector) = !isempty(vnv.inactive_ranges) +has_inactive(vnv::VarNameVector) = !isempty(vnv.num_inactive) + +""" + num_inactive(vnv::VarNameVector, vn::VarName) + +Returns the number of inactive entries for `vn` in `vnv`. +""" +num_inactive(vnv::VarNameVector, vn::VarName) = num_inactive(vnv, getidx(vnv, vn)) +num_inactive(vnv::VarNameVector, idx::Int) = get(vnv.num_inactive, idx, 0) + +""" + num_allocated(vnv::VarNameVector) + +Returns the number of allocated entries in `vnv`. +""" +num_allocated(vnv::VarNameVector) = length(vnv.vals) + +""" + num_allocated(vnv::VarNameVector, vn::VarName) + +Returns the number of allocated entries for `vn` in `vnv`. +""" +num_allocated(vnv::VarNameVector, vn::VarName) = num_allocated(vnv, getidx(vnv, vn)) +function num_allocated(vnv::VarNameVector, idx::Int) + return length(getrange(vnv, idx)) + num_inactive(vnv, idx) +end # Basic array interface. Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) Base.length(vnv::VarNameVector) = - if isempty(vnv.inactive_ranges) + if isempty(vnv.num_inactive) length(vnv.vals) else sum(length, vnv.ranges) @@ -164,7 +189,7 @@ end # `getindex` for `Colon` function Base.getindex(vnv::VarNameVector, ::Colon) - return if has_inactive_ranges(vnv) + return if has_inactive(vnv) mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) else vnv.vals @@ -172,7 +197,7 @@ function Base.getindex(vnv::VarNameVector, ::Colon) end function getindex_raw(vnv::VarNameVector, ::Colon) - return if has_inactive_ranges(vnv) + return if has_inactive(vnv) mapreduce(Base.Fix1(getindex_raw, vnv.vals), vcat, vnv.ranges) else vnv.vals @@ -222,27 +247,20 @@ function Base.similar(vnv::VarNameVector) similar(vnv.ranges, 0), similar(vnv.vals, 0), similar(vnv.transforms, 0), - similar(vnv.inactive_ranges), + similar(vnv.num_inactive), similar_metadata(vnv.metadata), ) end function nextrange(vnv::VarNameVector, x) - # NOTE: Need to treat `isempty(vnv.ranges)` separately because `maximum` - # will error if `vnv.ranges` is empty. - max_active_range = isempty(vnv.ranges) ? 0 : maximum(last, vnv.ranges) - # Also need to consider inactive ranges, since we can have scenarios such as - # - # vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2, 3]) - # update!(vnv, @varname(y), [4]) # => `ranges = [1:1, 2:2], inactive_ranges = [3:3]` - # - # Here `nextrange(vnv, [5])` should return `4:4`, _not_ `3:3`. - # NOTE: We could of course attempt to make use of unused space, e.g. if we have an inactive - # range which can hold `x`, then we could just use that. Buuut the complexity of this is - # probably not worth it (at least at the moment). - max_inactive_range = - isempty(vnv.inactive_ranges) ? 0 : maximum(last, values(vnv.inactive_ranges)) - offset = max(max_active_range, max_inactive_range) + # If `vnv` is empty, return immediately. + isempty(vnv) && return 1:length(x) + + # The offset will be the last range's end + its number of inactive entries. + vn_last = vnv.varnames[end] + idx = getidx(vnv, vn_last) + offset = last(getrange(vnv, idx)) + num_inactive(vnv, idx) + return (offset + 1):(offset + length(x)) end @@ -267,6 +285,11 @@ function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) return x end +function shift_left!(x::AbstractVector{<:Real}, start::Int, n::Int) + x[start:(end - n)] = x[(start + n):end] + return x +end + # `update!` and `update!!`: update a variable in the varname vector. function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) if !haskey(vnv, vn) @@ -277,70 +300,84 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) # Here we update an existing entry. val_vec = tovec(val) idx = getidx(vnv, vn) + # Extract the old range. r_old = getrange(vnv, idx) + start_old, end_old = first(r_old), last(r_old) n_old = length(r_old) + # Compute the new range. n_new = length(val_vec) - # Existing keys needs to be handled differently depending on - # whether the size of the value is increasing or decreasing. - if n_new > n_old - n_extra = n_new - n_old - # Grow the underlying vector to accomodate the new value. - resize!(vnv.vals, length(vnv.vals) + n_extra) - # Shift the existing values to make room for the new value. - shift_right!(vnv.vals, r_old[end] + 1, n_extra) - # Compute the new range. - r_new = r_old[1]:(r_old[1] + n_new - 1) - vnv.ranges[idx] = r_new - # If we have an inactive range for this variable, we need to update it. - if haskey(vnv.inactive_ranges, idx) - # Then we need to change this to reflect the new range. - let inactive_range = vnv.inactive_ranges[idx] - # Inactive range should always start before the new range - # in the scenario where the new range is larger than the old range. - @assert inactive_range[1] <= r_new[end] - if inactive_range[end] <= r_new[end] - # Inactive range ends before the new range ends. - delete!(vnv.inactive_ranges, idx) - else - # Then we have `inactive_range[1] < r_new[end]`, i.e. - # Inactive range starts before the new range ends. - # AND the inactive range ends after the new range ends. - # => We need to split the inactive range. - vnv.inactive_ranges[idx] = (r_new[end] + 1):inactive_range[end] - end - end - end + start_new = start_old + end_new = start_old + n_new - 1 + r_new = start_new:end_new - # Shift existing ranges which are after the current range. - for (i, r_i) in enumerate(vnv.ranges) - if r_i[1] > r_old[end] - vnv.ranges[i] = r_i .+ n_extra - end - end - # Shift inactive ranges coming after `r_old` (unless they are - # for the current variable). - for (i, r_i) in pairs(vnv.inactive_ranges) - if i !== idx && r_i[1] > r_old[end] - vnv.inactive_ranges[i] = r_i .+ n_extra - end + #= + Suppose we currently have the following: + + | x | x | o | o | o | y | y | y | <- Current entries + + where 'O' denotes an inactive entry, and we're going to + update the variable `x` to be of size `k` instead of 2. + + We then have a few different scenarios: + 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. + E.g. if `k = 7`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | x | x | y | y | y | <- New entries + + 2. `k = 5`: All inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | y | y | y | <- New entries + + 3. `k < 5`: Some inactive entries become active, some remain inactive. + E.g. if `k = 3`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | o | o | y | y | y | <- New entries + + 4. `k = 2`: No inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | o | o | o | y | y | y | <- New entries + + 5. `k < 2`: More entries become inactive. + E.g. if `k = 1`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | o | o | o | o | y | y | y | <- New entries + =# + + # Compute the allocated space for `vn`. + had_inactive = haskey(vnv.num_inactive, idx) + n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old + + if n_new > n_allocated + # Then we need to grow the underlying vector. + n_extra = n_new - n_allocated + # Allocate. + resize!(vnv.vals, length(vnv.vals) + n_extra) + # Shift current values. + shift_right!(vnv.vals, end_old + 1, n_extra) + # No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) + # Update the ranges for all variables after this one. + for i in (idx + 1):length(vnv.varnames) + vnv.ranges[i] = vnv.ranges[i] .+ n_extra end + elseif n_new == n_allocated + # => No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) else - # `n_new <= n_old` - # Just decrease the current range. - r_new = r_old[1]:(r_old[1] + n_new - 1) - vnv.ranges[idx] = r_new - # And mark the rest as inactive if needed. - if n_new < n_old - if haskey(vnv.inactive_ranges, idx) - vnv.inactive_ranges[idx] = ( - (r_old[n_new] + 1):vnv.inactive_ranges[idx][end] - ) - else - vnv.inactive_ranges[idx] = ((r_old[n_new] + 1):r_old[end]) - end - end + # `n_new < n_allocated` + # => Need to update the number of inactive entries. + vnv.num_inactive[idx] = n_allocated - n_new end + # Update the range for this variable. + vnv.ranges[idx] = r_new # Update the value. vnv.vals[r_new] = val_vec # Update the transform. @@ -375,7 +412,7 @@ function inactive_ranges_sweep!(vnv::VarNameVector) # And then we re-contiguify the ranges. recontiguify_ranges!(vnv.ranges) # Clear the inactive ranges. - empty!(vnv.inactive_ranges) + empty!(vnv.num_inactive) # Now we update the values. for (i, r) in enumerate(vnv.ranges) vnv.vals[r] = vals[r] @@ -443,8 +480,8 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) deleteat!(vnv.transforms, idx) # Delete any inactive ranges corresponding to the variable. - if haskey(vnv.inactive_ranges, idx) - delete!(vnv.inactive_ranges, idx) + if haskey(vnv.num_inactive, idx) + delete!(vnv.num_inactive, idx) end # Re-adjust the indices in `varname_to_index`. diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 97cde9ba1..cb5ac0632 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -100,7 +100,7 @@ function relax_container_types(vnv::VarNameVector, vns, vals) vnv.ranges, vals_new, transforms_new, - vnv.inactive_ranges, + vnv.num_inactive, vnv.metadata, ) end @@ -266,7 +266,7 @@ end @test length(vnv[:]) == length(vnv) # There should be no redundant values in the underlying vector. - @test !DynamicPPL.has_inactive_ranges(vnv) + @test !DynamicPPL.has_inactive(vnv) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -307,28 +307,53 @@ end end @testset "growing and shrinking" begin - n = 5 - vn = @varname(x) - vnv = VarNameVector(OrderedDict(vn => [true])) - @test !DynamicPPL.has_inactive_ranges(vnv) - # Growing should not create inactive ranges. - for i in 1:n - x = fill(true, i) + @testset "deterministic" begin + n = 5 + vn = @varname(x) + vnv = VarNameVector(OrderedDict(vn => [true])) + @test !DynamicPPL.has_inactive(vnv) + # Growing should not create inactive ranges. + for i in 1:n + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test !DynamicPPL.has_inactive(vnv) + end + + # Same size should not create inactive ranges. + x = fill(true, n) DynamicPPL.update!(vnv, vn, x) - @test !DynamicPPL.has_inactive_ranges(vnv) + @test !DynamicPPL.has_inactive(vnv) + + # Shrinking should create inactive ranges. + for i in (n - 1):-1:1 + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test DynamicPPL.has_inactive(vnv) + @test DynamicPPL.num_inactive(vnv, vn) == n - i + end end - # Same size should not create inactive ranges. - x = fill(true, n) - DynamicPPL.update!(vnv, vn, x) - @test !DynamicPPL.has_inactive_ranges(vnv) + @testset "random" begin + n = 5 + vn = @varname(x) + vnv = VarNameVector(OrderedDict(vn => [true])) + @test !DynamicPPL.has_inactive(vnv) - # Shrinking should create inactive ranges. - for i in (n - 1):-1:1 - x = fill(true, i) - DynamicPPL.update!(vnv, vn, x) - @test DynamicPPL.has_inactive_ranges(vnv) - @test vnv.inactive_ranges[1] == ((i + 1):n) + # Insert a bunch of random-length vectors. + for i in 1:100 + x = fill(true, rand(1:n)) + DynamicPPL.update!(vnv, vn, x) + end + # Should never be allocating more than `n` elements. + @test DynamicPPL.num_allocated(vnv, vn) ≤ n + + # If we compaticfy, then it should always be the same size as just inserted. + for i in 1:10 + x = fill(true, rand(1:n)) + DynamicPPL.update!(vnv, vn, x) + DynamicPPL.inactive_ranges_sweep!(vnv) + @test DynamicPPL.num_allocated(vnv, vn) == length(x) + end end end end From 66bc090e8faf325be1f0d2cd5d90097aae965742 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:12:27 +0000 Subject: [PATCH 045/209] added `copy` when constructing `VectorVarInfo` from `VarInfo` --- src/varinfo.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 767fac7f6..48ad94392 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -129,10 +129,10 @@ end # No-op if we're already working with a `VarNameVector`. metadata_to_varnamevector(vnv::VarNameVector) = vnv function metadata_to_varnamevector(md::Metadata) - idcs = md.idcs - vns = md.vns - ranges = md.ranges - vals = md.vals + idcs = copy(md.idcs) + vns = copy(md.vns) + ranges = copy(md.ranges) + vals = copy(md.vals) transforms = map(md.dists) do dist # TODO: Handle linked distributions. FromVec(size(dist)) From ccd86f2683169ddf5e314a3f509a020b24a2ee87 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:13:03 +0000 Subject: [PATCH 046/209] added missing `isempty` impl --- src/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 896c0c3a2..758cc75d5 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -165,6 +165,7 @@ Base.length(vnv::VarNameVector) = sum(length, vnv.ranges) end Base.size(vnv::VarNameVector) = (length(vnv),) +Base.isempty(vnv::VarNameVector) = isempty(vnv.varnames) # TODO: We should probably remove this Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() From 1d4a000548b395fe2e53dfeb56a48276e27e12c6 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:13:21 +0000 Subject: [PATCH 047/209] remove impl of `iterate` and instead implemented `pairs` and `values` iterators --- src/varnamevector.jl | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 758cc75d5..a26927394 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -172,6 +172,13 @@ Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. Base.keys(vnv::VarNameVector) = vnv.varnames +Base.values(vnv::VarNameVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) + +function Base.pairs(vnv::VarNameVector) + return Iterators.zip( + vnv.varnames, Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) + ) +end Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) @@ -445,25 +452,6 @@ function group_by_symbol(vnv::VarNameVector) return NamedTuple{Tuple(keys(d))}(nt_vals) end -# `iterate` -# TODO: Maybe implement `iterate` as a vector and then instead implement `pairs`. -function Base.iterate(vnv::VarNameVector, state=nothing) - res = if state === nothing - iterate(vnv.varnames) - else - iterate(vnv.varnames, state) - end - res === nothing && return nothing - vn, state_new = res - return vn => getindex(vnv, vn), state_new -end - -function Base.pairs(vnv::VarNameVector) - return Iterators.zip( - vnv.varnames, Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) - ) -end - function Base.delete!(vnv::VarNameVector, vn::VarName) # Error if we don't have the variable. !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) From 9a16dd15b6562c3dc00a4c4b0477f6e9d2a0f499 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:56:18 +0000 Subject: [PATCH 048/209] added missing `empty!` for `num_inactive` --- src/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index a26927394..90d2d8847 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -234,6 +234,7 @@ function Base.empty!(vnv::VarNameVector) empty!(vnv.ranges) empty!(vnv.vals) empty!(vnv.transforms) + empty!(vnv.num_inactive) return nothing end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) From e49b762303dda3c63d67d47fe1e1706d50cb639c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:56:37 +0000 Subject: [PATCH 049/209] removed redundant `shift_left!` methd --- src/varnamevector.jl | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 90d2d8847..d238b8d87 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -294,11 +294,6 @@ function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) return x end -function shift_left!(x::AbstractVector{<:Real}, start::Int, n::Int) - x[start:(end - n)] = x[(start + n):end] - return x -end - # `update!` and `update!!`: update a variable in the varname vector. function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) if !haskey(vnv, vn) From 2b445c96ffe1fa21127dc593e6c13457e2e2b93b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:56:48 +0000 Subject: [PATCH 050/209] fixed `delete!` for `VarNameVector` --- src/varnamevector.jl | 57 ++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index d238b8d87..5ab11465b 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -454,43 +454,44 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) # Get the index of the variable. idx = getidx(vnv, vn) - # Get the range of the variable. - r_old = getrange(vnv, idx) - # Delete the variable. + + # Delete the values. + r_start = first(getrange(vnv, idx)) + n_allocated = num_allocated(vnv, idx) + deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) + + # Delete `vn` from the lookup table. delete!(vnv.varname_to_index, vn) - # `deleteat!` deletes and shifts the rest of the vector. - # So after this, we need to re-adjust the indices in `varname_to_index`. - deleteat!(vnv.varnames, idx) - deleteat!(vnv.ranges, idx) - deleteat!(vnv.transforms, idx) - # Delete any inactive ranges corresponding to the variable. - if haskey(vnv.num_inactive, idx) - delete!(vnv.num_inactive, idx) + # Delete any inactive ranges corresponding to `vn`. + haskey(vnv.num_inactive, idx) && delete!(vnv.num_inactive, idx) + + # Re-adjust the indices for varnames occuring after `vn` so + # that they point to the correct indices after the deletions below. + for idx_to_shift in (idx + 1):length(vnv.varnames) + vn = vnv.varnames[idx_to_shift] + if idx_to_shift > idx + # Shift the index in the lookup table. + vnv.varname_to_index[vn] = idx_to_shift - 1 + # Shift the index in the inactive ranges. + if haskey(vnv.num_inactive, idx_to_shift) + # Done in increasing order => don't need to worry about + # potentially shifting the same index twice. + vnv.num_inactive[idx_to_shift - 1] = pop!(vnv.num_inactive, idx_to_shift) + end + end end - # Re-adjust the indices in `varname_to_index`. - for (vn, idx) in vnv.varname_to_index - idx > idx && (vnv.varname_to_index[vn] = idx - 1) + # Re-adjust the ranges for varnames occuring after `vn`. + for idx_to_shift in (idx + 1):length(vnv.varnames) + vnv.ranges[idx_to_shift] = vnv.ranges[idx_to_shift] .- n_allocated end - return vnv -end - -function Base.deleteat!(vnv::VarNameVector, vn::VarName) - # Error if we don't have the variable. - !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) - # Get the index of the variable. - idx = getidx(vnv, vn) - # Get the range of the variable. - r_old = getrange(vnv, idx) - # Delete the variable. - delete!(vnv.varname_to_index, vn) + # Delete references from vector fields, thus shifting the indices of + # varnames occuring after `vn` by one to the left, as we adjusted for above. deleteat!(vnv.varnames, idx) deleteat!(vnv.ranges, idx) deleteat!(vnv.transforms, idx) - # Delete the range. - deleteat!(vnv.vals, r_old) return vnv end From e3c26339bf575a58d1b3c8228aa875561d843778 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:56:59 +0000 Subject: [PATCH 051/209] added `is_contiguous` as an alterantive to `!has_inactive` --- src/varnamevector.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 5ab11465b..d95e7cca0 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -261,6 +261,15 @@ function Base.similar(vnv::VarNameVector) ) end +""" + is_continugous(vnv::VarNameVector) + +Returns `true` if the underlying data of `vnv` is stored in a contiguous array. + +This is equivalent to [`!has_inactive(vnv)`](@ref). +""" +is_continugous(vnv::VarNameVector) = !has_inactive(vnv) + function nextrange(vnv::VarNameVector, x) # If `vnv` is empty, return immediately. isempty(vnv) && return 1:length(x) From 19a829cdccbb46f67267f1b0f6fabdf56a1569b2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 14:57:18 +0000 Subject: [PATCH 052/209] updates to internal docs --- docs/src/internals.md | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index b8cfe2681..7cac77cfb 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -86,6 +86,8 @@ varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] ``` +Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entries while the typed uses `Vector{Bool}`. This is because the untyped version needs the underlying container to be able to handle both the `Bool` for `x` and the `Float64` for `y`, while the typed version can use a `Vector{Bool}` for `x` and a `Vector{Float64}` for `y` due to its usage of `NamedTuple`. + !!! warning Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. @@ -155,16 +157,17 @@ There are typically a few scenarios where we encounter changing representation s In scenario (1), the we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". + In scenario (2), we'll end up with quite a sub-optimal representation unless we do something to handle this. For example: ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: $(length(vnv.vals))") +println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 x = fill(true, rand(1:5)) DynamicPPL.update!(vnv, @varname(x), x) - println("After insertion #$(i) of length $(length(x)): $(length(vnv.vals))") + println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` @@ -172,25 +175,17 @@ To alleviate this issue, we can insert a call to [`DynamicPPL.inactive_ranges_sw ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: $(length(vnv.vals))") +println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 x = fill(true, rand(1:5)) DynamicPPL.update!(vnv, @varname(x), x) DynamicPPL.inactive_ranges_sweep!(vnv) - println("After insertion #$(i) of length $(length(x)): $(length(vnv.vals))") + println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` -Without the calls to [`DynamicPPL.inactive_ranges_sweep!`](@ref), the above would result in a much larger memory footprint. Of course, in this does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. - -That is, we've decided to focus on optimizing scenario (1) but we still maintain functionality even in scenario (2), though at a greater computational cost. - -`delete!` and similars are also implemented using the "delete-by-mark" technique outlined above. - -!!! note - - If we instead tried to insert a new larger value at the same location as the old value, then we would have to shift all the elements after the insertion point, potentially requiring a lot of memory allocations. +This does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. However, this also ensures that the the underlying `Vector{T}` is contiguous, which is important for performance. Hence, if we're about to do a lot of work with the `VarNameVector` without insertions, etc., it can be worth it to do a sweep to ensure that the underlying `Vector{T}` is contiguous. !!! note @@ -199,7 +194,7 @@ That is, we've decided to focus on optimizing scenario (1) but we still maintain This does mean that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: ```@docs -DynamicPPL.has_inactive_ranges +DynamicPPL.has_inactive DynamicPPL.inactive_ranges_sweep! ``` @@ -224,13 +219,13 @@ haskey(varinfo_untyped_vnv, @varname(x)) ``` ```@example varinfo-design -DynamicPPL.has_inactive_ranges(varinfo_untyped_vnv.metadata) +DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) ``` ```@example varinfo-design # `delete!` DynamicPPL.delete!(varinfo_untyped_vnv.metadata, @varname(x)) -DynamicPPL.has_inactive_ranges(varinfo_untyped_vnv.metadata) +DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) ``` ```@example varinfo-design @@ -249,6 +244,20 @@ DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 3)) varinfo_untyped_vnv[@varname(x)] ``` +#### Performance summary + +In the end, we have the following "rough" performance characteristics: + +| Method | Is blazingly fast? | +| :-: | :-: | +| `getindex` | ${\color{green} \checkmark}$ | +| `setindex!` | ${\color{green} \checkmark}$ | +| `push!` | ${\color{green} \checkmark}$ | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if same size / ${\color{red} \times}$ if different size | +| `delete!` | ${\color{red} \times}$ | + + + ### Additional methods We also want some additional methods that are not part of the `Dict` or `Vector` interface: From a358bc40b11b52b80b285570b380dcbd9c74bae6 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 15:00:54 +0000 Subject: [PATCH 053/209] renamed `sweep_inactive_ranges!` to `contiguify!` --- docs/src/internals.md | 6 +++--- src/varnamevector.jl | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 7cac77cfb..f13407fca 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -171,7 +171,7 @@ for i in 1:5 end ``` -To alleviate this issue, we can insert a call to [`DynamicPPL.inactive_ranges_sweep!`](@ref) after every insertion: +To alleviate this issue, we can insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion: ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) @@ -180,7 +180,7 @@ println("Before insertion: number of allocated entries $(DynamicPPL.num_allocat for i in 1:5 x = fill(true, rand(1:5)) DynamicPPL.update!(vnv, @varname(x), x) - DynamicPPL.inactive_ranges_sweep!(vnv) + DynamicPPL.contiguify!(vnv) println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` @@ -195,7 +195,7 @@ This does mean that the underlying `Vector{T}` can grow without bound, so we hav ```@docs DynamicPPL.has_inactive -DynamicPPL.inactive_ranges_sweep! +DynamicPPL.contiguify! ``` Continuing from the example from the previous section: diff --git a/src/varnamevector.jl b/src/varnamevector.jl index d95e7cca0..0fba49997 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -414,11 +414,11 @@ function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) end """ - inactive_ranges_sweep!(vnv::VarNameVector) + contiguify!(vnv::VarNameVector) Re-contiguify the underlying vector and shrink if possible. """ -function inactive_ranges_sweep!(vnv::VarNameVector) +function contiguify!(vnv::VarNameVector) # Extract the re-contiguified values. # NOTE: We need to do this before we update the ranges. vals = vnv[:] From 46be8d59b0dd57aaf84d4b3c205498f6d6ba52ce Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 15:04:34 +0000 Subject: [PATCH 054/209] improvements to internal docs --- docs/src/internals.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index f13407fca..98f9737c4 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -246,17 +246,22 @@ varinfo_untyped_vnv[@varname(x)] #### Performance summary -In the end, we have the following "rough" performance characteristics: +In the end, we have the following "rough" performance characteristics for `VarNameVector`: | Method | Is blazingly fast? | | :-: | :-: | | `getindex` | ${\color{green} \checkmark}$ | | `setindex!` | ${\color{green} \checkmark}$ | | `push!` | ${\color{green} \checkmark}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if same size / ${\color{red} \times}$ if different size | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `delete!` | ${\color{red} \times}$ | +#### Methods +```@docs +DynamicPPL.num_inactive +DynamicPPL.num_allocated +``` ### Additional methods From 57d688e84529bfcb3ade21445b7f81e3e44bff98 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:43:42 +0000 Subject: [PATCH 055/209] more improvements to internal docs --- docs/src/internals.md | 95 ++++++++++++++++++++++++------------------- src/varnamevector.jl | 6 +-- 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 98f9737c4..3c47a3d97 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -7,7 +7,7 @@ Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. -**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding used for, say, `@varname(x)`. +**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding to a variable `@varname(x)`. !!! note @@ -31,23 +31,26 @@ To ensure that `varinfo` is simple and intuitive to work with, we need the under - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - - `similar(::Vector{<:Real})`: return a new + - `similar(::Vector{<:Real})`: return a new instance with the same `eltype` as the input. Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: - 1. Type-stable when possible, but still functional when not. - 2. Efficient storage and iteration. + 1. Type-stable when possible, but functional when not. + 2. Efficient storage and iteration when possible, but functional when not. + +The "but functional when not" is important as we want to support arbitrary models, which means that we can't always have these performance properties. In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). ### Type-stability -This is somewhat non-trivial to address since we want to achieve this for both continuous (typically `Float64`) and discrete (typically `Int`) variables. +Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form ```@example varinfo-design using DynamicPPL, Distributions, FillArrays + @model function demo() x ~ product_distribution(Fill(Bernoulli(0.5), 2)) y ~ Normal(0, 1) @@ -76,7 +79,7 @@ varinfo_typed = DynamicPPL.typed_varinfo(demo()) typeof(varinfo_typed.metadata) ``` -But they both work as expected: +But they both work as expected but one results in concrete typing and the other does not: ```@example varinfo-design varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] @@ -99,11 +102,13 @@ Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entri In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. - In practice, we see that such mixing of types is not very common, and so in DynamicPPL and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. + In practice, rarely observe such mixing of types, therefore in DynamicPPL, and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. !!! warning - Another downside with this approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + Another downside with such a `NamedTuple` approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + + For these scenarios it can be useful to fall back to "untyped" representations. Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. @@ -123,19 +128,19 @@ This does require a bit of book-keeping, in particular when it comes to insertio - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - `transforms::Vector`: the transforms associated with each `VarName`. -Mutating functions, e.g. `setindex!`, are then treated according to the following rules: +Mutating functions, e.g. `setindex!(vnv::VarNameVector, val, vn::VarName)`, are then treated according to the following rules: - 1. If `VarName` is not already present: add it to the end of `varnames`, add the value to the underlying `Vector{T}`, etc. + 1. If `vn` is not already present: add it to the end of `vnv.varnames`, add the `val` to the underlying `vnv.vals`, etc. - 2. If `VarName` is already present in the `VarNameVector`: + 2. If `vn` is already present in `vnv`: - 1. If `value` has the *same length* as the existing value for `VarName`: replace existing value. - 2. If `value` has a *smaller length* than the existing value for `VarName`: replace existing value and mark the remaining indices as "inactive" by adding the range to the `inactive_ranges` field. - 3. If `value` has a *larger length* than the existing value for `VarName`: mark the entire current range for `VarName` as "inactive", expand the underlying `Vector{T}` to accommodate the new value, and update the `ranges` to point to the new range for this `VarName`. + 1. If `val` has the *same length* as the existing value for `vn`: replace existing value. + 2. If `val` has a *smaller length* than the existing value for `vn`: replace existing value and mark the remaining indices as "inactive" by increasing the entry in `vnv.num_inactive` field. + 3. If `val` has a *larger length* than the existing value for `vn`: expand the underlying `vnv.vals` to accommodate the new value, update all `VarName`s occuring after `vn`, and update the `vnv.ranges` to point to the new range for `vn`. -This "delete-by-mark" instead of having to actually delete elements from the underlying `Vector{T}` ensures that `setindex!` will be fairly efficient in the scenarios we encounter in practice. +This means that `VarNameVector` is allowed to grow as needed, while "shrinking" (i.e. insertion of smaller elements) is handled by simply marking the redundant indices as "inactive". This turns out to be efficient for use-cases that we are generally interested in. -In particular, we want to optimize code-paths which effectively boil down to inner-loop in the following example:: +For example, we want to optimize code-paths which effectively boil down to inner-loop in the following example: ```julia # Construct a `VarInfo` with types inferred from `model`. @@ -155,32 +160,45 @@ There are typically a few scenarios where we encounter changing representation s 1. We're working with a transformed version `x` which is represented in a lower-dimensional space, e.g. transforming a `x ~ LKJ(2, 1)` to unconstrained `y = f(x)` takes us from 2-by-2 `Matrix{Float64}` to a 1-length `Vector{Float64}`. 2. `x` has a random size, e.g. in a mixture model with a prior on the number of components. Here the size of `x` can vary widly between every realization of the `Model`. -In scenario (1), the we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". +In scenario (1), we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". +In scenario (2), we end up increasing the allocated memory for the randomly sized `x`, eventually leading to a vector that is large enough to hold realizations without needing to reallocate. But this can still lead to unnecessary memory usage, which might be undesirable. Hence one has to make a decision regarding the trade-off between memory usage and performance for the use-case at hand. -In scenario (2), we'll end up with quite a sub-optimal representation unless we do something to handle this. For example: +To help with this, we have the following functions: + +```@docs +DynamicPPL.has_inactive +DynamicPPL.num_inactive +DynamicPPL.num_allocated +DynamicPPL.is_contiguous +DynamicPPL.contiguify! +``` + +For example, one might encounter the following scenario: ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 - x = fill(true, rand(1:5)) + x = fill(true, rand(1:100)) DynamicPPL.update!(vnv, @varname(x), x) println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` -To alleviate this issue, we can insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion: +We can then insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion whenever the allocation grows too large to reduce overall memory usage: ```@example varinfo-design vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 - x = fill(true, rand(1:5)) + x = fill(true, rand(1:100)) DynamicPPL.update!(vnv, @varname(x), x) - DynamicPPL.contiguify!(vnv) + if DynamicPPL.num_allocated(vnv) > 10 + DynamicPPL.contiguify!(vnv) + end println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") end ``` @@ -191,14 +209,7 @@ This does incur a runtime cost as it requires re-allocation of the `ranges` in a Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. -This does mean that the underlying `Vector{T}` can grow without bound, so we have the following methods to interact with the inactive ranges: - -```@docs -DynamicPPL.has_inactive -DynamicPPL.contiguify! -``` - -Continuing from the example from the previous section: +Continuing from the example from the previous section, we can use a `VarInfo` with a `VarNameVector` as the `metadata` field: ```@example varinfo-design # Type-unstable @@ -232,18 +243,26 @@ DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) haskey(varinfo_untyped_vnv, @varname(x)) ``` -If we try to insert a differently-sized value for `@varname(x)` +Or insert a differently-sized value for `@varname(x)` ```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 1)) +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 1)) varinfo_untyped_vnv[@varname(x)] ``` ```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(false, 3)) +DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) +``` + +```@example varinfo-design +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 4)) varinfo_untyped_vnv[@varname(x)] ``` +```@example varinfo-design +DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) +``` + #### Performance summary In the end, we have the following "rough" performance characteristics for `VarNameVector`: @@ -253,15 +272,9 @@ In the end, we have the following "rough" performance characteristics for `VarNa | `getindex` | ${\color{green} \checkmark}$ | | `setindex!` | ${\color{green} \checkmark}$ | | `push!` | ${\color{green} \checkmark}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `delete!` | ${\color{red} \times}$ | - -#### Methods - -```@docs -DynamicPPL.num_inactive -DynamicPPL.num_allocated -``` +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | +| `convert(Vector, ::VarNameVector)` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | ### Additional methods diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 0fba49997..3fa295112 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -262,13 +262,13 @@ function Base.similar(vnv::VarNameVector) end """ - is_continugous(vnv::VarNameVector) + is_contiguous(vnv::VarNameVector) Returns `true` if the underlying data of `vnv` is stored in a contiguous array. -This is equivalent to [`!has_inactive(vnv)`](@ref). +This is equivalent to negating [`has_inactive(vnv)`](@ref). """ -is_continugous(vnv::VarNameVector) = !has_inactive(vnv) +is_contiguous(vnv::VarNameVector) = !has_inactive(vnv) function nextrange(vnv::VarNameVector, x) # If `vnv` is empty, return immediately. From 0968a076b9ccbdc088433288a6e78ca26502952b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:46:54 +0000 Subject: [PATCH 056/209] moved additional methods description in internals to earlier in the doc --- docs/src/internals.md | 52 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/docs/src/internals.md b/docs/src/internals.md index 3c47a3d97..6e1e8b78b 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -33,11 +33,21 @@ To ensure that `varinfo` is simple and intuitive to work with, we need the under - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - `similar(::Vector{<:Real})`: return a new instance with the same `eltype` as the input. -Moreover, we want also want the underlying representation used in `metadata` to have a few performance-related properties: +We also want some additional methods that are *not* part of the `Dict` or `Vector` interface: + + - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + + - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. + +In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: + + - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + +Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: 1. Type-stable when possible, but functional when not. 2. Efficient storage and iteration when possible, but functional when not. - + The "but functional when not" is important as we want to support arbitrary models, which means that we can't always have these performance properties. In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). @@ -147,7 +157,7 @@ For example, we want to optimize code-paths which effectively boil down to inner varinfo = VarInfo(model) # Repeatedly sample from `model`. -for _ = 1:num_samples +for _ in 1:num_samples rand!(rng, model, varinfo) # Do something with `varinfo`. @@ -183,7 +193,9 @@ println("Before insertion: number of allocated entries $(DynamicPPL.num_allocat for i in 1:5 x = fill(true, rand(1:100)) DynamicPPL.update!(vnv, @varname(x), x) - println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") + println( + "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", + ) end ``` @@ -199,7 +211,9 @@ for i in 1:5 if DynamicPPL.num_allocated(vnv) > 10 DynamicPPL.contiguify!(vnv) end - println("After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))") + println( + "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", + ) end ``` @@ -267,23 +281,11 @@ DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) In the end, we have the following "rough" performance characteristics for `VarNameVector`: -| Method | Is blazingly fast? | -| :-: | :-: | -| `getindex` | ${\color{green} \checkmark}$ | -| `setindex!` | ${\color{green} \checkmark}$ | -| `push!` | ${\color{green} \checkmark}$ | -| `delete!` | ${\color{red} \times}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | -| `convert(Vector, ::VarNameVector)` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | - -### Additional methods - -We also want some additional methods that are not part of the `Dict` or `Vector` interface: - - - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. - - - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. - -In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - - - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. +| Method | Is blazingly fast? | +|:--------------------------------------------------:|:--------------------------------------------------------------------------------------------:| +| `getindex` | ${\color{green} \checkmark}$ | +| `setindex!` | ${\color{green} \checkmark}$ | +| `push!` | ${\color{green} \checkmark}$ | +| `delete!` | ${\color{red} \times}$ | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | +| `convert(Vector{T}, ::VarNameVector{<:VarName,T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | From 0d008a4ed05ae1172b4991c998dfbf3d7b2f0258 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:52:48 +0000 Subject: [PATCH 057/209] moved internals docs to a separate directory and split into files --- docs/make.jl | 2 +- docs/src/{internals.md => internals/varinfo.md} | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename docs/src/{internals.md => internals/varinfo.md} (99%) diff --git a/docs/make.jl b/docs/make.jl index 056b90d7f..38c520d3a 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,7 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], - "Internals" => "internals.md", + "Internals" => ["internals/varinfo.md"], ], checkdocs=:exports, ) diff --git a/docs/src/internals.md b/docs/src/internals/varinfo.md similarity index 99% rename from docs/src/internals.md rename to docs/src/internals/varinfo.md index 6e1e8b78b..40932d2f4 100644 --- a/docs/src/internals.md +++ b/docs/src/internals/varinfo.md @@ -1,4 +1,4 @@ -## Design of `VarInfo` +# Design of `VarInfo` # [`VarInfo`](@ref) is a fairly simple structure; it contains @@ -52,7 +52,7 @@ The "but functional when not" is important as we want to support arbitrary model In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). -### Type-stability +## Type-stability ## Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. @@ -122,7 +122,7 @@ Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entri Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. -### Efficient storage and iteration +## Efficient storage and iteration ## Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): @@ -277,7 +277,7 @@ varinfo_untyped_vnv[@varname(x)] DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) ``` -#### Performance summary +### Performance summary ### In the end, we have the following "rough" performance characteristics for `VarNameVector`: From ccd0d64ef40c9e363b8590ec0be3757bcd379267 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:58:09 +0000 Subject: [PATCH 058/209] more improvements to internals doc --- docs/src/internals/varinfo.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 40932d2f4..ae4335807 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -1,6 +1,12 @@ # Design of `VarInfo` # -[`VarInfo`](@ref) is a fairly simple structure; it contains +[`VarInfo`](@ref) is a fairly simple structure. + +```@docs; canonical=false +VarInfo +``` + +It contains - a `logp` field for accumulation of the log-density evaluation, and - a `metadata` field for storing information about the realizations of the different variables. @@ -13,7 +19,7 @@ Representing `logp` is fairly straight-forward: we'll just use a `Real` or an ar We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. -To ensure that `varinfo` is simple and intuitive to work with, we need the underlying `metadata` to replicate the following functionality of `Dict`: +To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo`, and hence the underlying `metadata`, to replicate the following functionality of `Dict`: - `keys(::Dict)`: return all the `VarName`s present in `metadata`. - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. From 7c45e67fff93da3b94104a9541bffa12a09736bd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 17:58:34 +0000 Subject: [PATCH 059/209] formatting --- docs/src/internals/varinfo.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index ae4335807..b5e1aed60 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -1,6 +1,6 @@ -# Design of `VarInfo` # +# Design of `VarInfo` -[`VarInfo`](@ref) is a fairly simple structure. +[`VarInfo`](@ref) is a fairly simple structure. ```@docs; canonical=false VarInfo @@ -58,7 +58,7 @@ The "but functional when not" is important as we want to support arbitrary model In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). -## Type-stability ## +## Type-stability Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. @@ -128,7 +128,7 @@ Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entri Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. -## Efficient storage and iteration ## +## Efficient storage and iteration Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): @@ -283,7 +283,7 @@ varinfo_untyped_vnv[@varname(x)] DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) ``` -### Performance summary ### +### Performance summary In the end, we have the following "rough" performance characteristics for `VarNameVector`: From 373215b7c752f16f0e3a3fd8c54953edf718c0de Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:03:34 +0000 Subject: [PATCH 060/209] added tests for `delete!` and fixed reference to old method --- test/varnamevector.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index cb5ac0632..bdb6ecb6b 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -235,6 +235,15 @@ end @test vnv[vn_right] == val_right .+ 100 end + # `delete!` + @testset "delete!" begin + vnv = deepcopy(vnv_base) + delete!(vnv, vn_left) + @test !haskey(vnv, vn_left) + delete!(vnv, vn_right) + @test !haskey(vnv, vn_right) + end + # `push!` & `update!` @testset "push!" begin vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -351,7 +360,7 @@ end for i in 1:10 x = fill(true, rand(1:n)) DynamicPPL.update!(vnv, vn, x) - DynamicPPL.inactive_ranges_sweep!(vnv) + DynamicPPL.contiguify!(vnv) @test DynamicPPL.num_allocated(vnv, vn) == length(x) end end From 0cdafbf4133f26311c752ea598a5ea90c5622ba2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:04:08 +0000 Subject: [PATCH 061/209] addition to `delete!` test --- test/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index bdb6ecb6b..a37d23a6f 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -240,6 +240,7 @@ end vnv = deepcopy(vnv_base) delete!(vnv, vn_left) @test !haskey(vnv, vn_left) + @test haskey(vnv, vn_right) delete!(vnv, vn_right) @test !haskey(vnv, vn_right) end From 51c041febae9257a46dda3983fd9383c9b37d410 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:17:21 +0000 Subject: [PATCH 062/209] added `values_as` impls for `VarNameVector` --- docs/src/internals/varinfo.md | 2 +- src/varinfo.jl | 2 +- src/varnamevector.jl | 13 ++++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index b5e1aed60..70721d35c 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -294,4 +294,4 @@ In the end, we have the following "rough" performance characteristics for `VarNa | `push!` | ${\color{green} \checkmark}$ | | `delete!` | ${\color{red} \times}$ | | `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | -| `convert(Vector{T}, ::VarNameVector{<:VarName,T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | +| `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | diff --git a/src/varinfo.jl b/src/varinfo.jl index 48ad94392..37098f68a 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -901,7 +901,7 @@ TypedVarInfo(vi::TypedVarInfo) = vi function TypedVarInfo(vi::VectorVarInfo) logp = getlogp(vi) num_produce = get_num_produce(vi) - nt = group_by_symbol(vi.metadata) + nt = NamedTuple(group_by_symbol(vi.metadata)) return VarInfo(nt, Ref(logp), Ref(num_produce)) end diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 3fa295112..279795d8a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -454,7 +454,7 @@ function group_by_symbol(vnv::VarNameVector) ) end - return NamedTuple{Tuple(keys(d))}(nt_vals) + return OrderedDict(zip(keys(d), nt_vals)) end function Base.delete!(vnv::VarNameVector, vn::VarName) @@ -504,3 +504,14 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) return vnv end + +values_as(vnv::VarNameVector, ::Type{Vector}) = vnv[:] +function values_as(vnv::VarNameVector, ::Type{Vector{T}}) where {T} + return convert(Vector{T}, values_as(vnv, Vector)) +end +function values_as(vnv::VarNameVector, ::Type{NamedTuple}) + return NamedTuple(zip(map(Symbol, keys(vnv)), values(vnv))) +end +function values_as(vnv::VarNameVector, ::Type{D}) where {D<:AbstractDict} + return ConstructionBase.constructorof(D)(pairs(vnv)) +end From 20b37427c2b640f98d7576235206856156343a8f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:41:18 +0000 Subject: [PATCH 063/209] added docs for `replace_valus` and `values_as` for `VarNameVector` --- docs/src/internals/varinfo.md | 10 +++++ src/varnamevector.jl | 70 ++++++++++++++++++++++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 70721d35c..8fe456ab7 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -295,3 +295,13 @@ In the end, we have the following "rough" performance characteristics for `VarNa | `delete!` | ${\color{red} \times}$ | | `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | + +## Other methods + +```@docs +DynamicPPL.replace_values(::VarNameVector, vals::AbstractVector) +``` + +```@docs; canonical=false +DynamicPPL.values_as(::VarNameVector) +``` diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 279795d8a..eaad004c6 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -1,5 +1,3 @@ -# Similar to `Metadata` but representing a `Vector` and simpler interface. -# TODO: Should we subtype `AbstractVector` or `AbstractDict`? """ VarNameVector @@ -116,6 +114,46 @@ function VarNameVector( ) end +""" + replace_values(vnv::VarNameVector, vals::AbstractVector) + +Replace the values in `vnv` with `vals`. + +This is useful when we want to update the entire underlying vector of values +in one go or if we want to change the how the values are stored, e.g. alter the `eltype`. + +!!! warning + This replaces the raw underlying values, and so care should be taken when using this + function. For example, if `vnv` has any inactive entries, then the provided `vals` + should also contain the inactive entries to avoid unexpected behavior. + +# Example + +```jldoctest varnamevector-replace-values +julia> using DynamicPPL: VarNameVector, replace_values + +julia> vnv = VarNameVector(@varname(x) => [1.0]); + +julia> replace_values(vnv, [2.0])[@varname(x)] == [2.0] +true +``` + +This is also useful when we want to differentiate wrt. the values +using automatic differentiation, e.g. ForwardDiff.jl. + +```jldoctest varnamevector-replace-values +julia> using ForwardDiff: ForwardDiff + +julia> f(x) = sum(abs2, replace_values(vnv, x)[@varname(x)]) +f (generic function with 1 method) + +julia> ForwardDiff.gradient(f, [1.0]) +1-element Vector{Float64}: + 2.0 +``` +""" +replace_values(vnv::VarNameVector, vals) = Setfield.@set vnv.vals = vals + # Some `VarNameVector` specific functions. getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] @@ -505,6 +543,34 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) return vnv end +""" + values_as(vnv::VarNameVector[, T]) + +Return the values/realizations in `vnv` as type `T`, if implemented. + +If no type `T` is provided, return values as stored in `vnv`. + +# Examples + +```jldoctest +julia> using DynamicPPL: VarNameVector + +julia> vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2.0]); + +julia> values_as(vnv) == [1.0, 2.0] +true + +julia> values_as(vnv, Vector{Float32}) == Vector{Float32}([1.0, 2.0]) +true + +julia> values_as(vnv, OrderedDict) == OrderedDict(@varname(x) => 1.0, @varname(y) => [2.0]) +true + +julia> values_as(vnv, NamedTuple) == (x = 1.0, y = [2.0]) +true +``` +""" +values_as(vnv::VarNameVector) = values_as(vnv, Vector) values_as(vnv::VarNameVector, ::Type{Vector}) = vnv[:] function values_as(vnv::VarNameVector, ::Type{Vector{T}}) where {T} return convert(Vector{T}, values_as(vnv, Vector)) From ef6c618e2983abefa4c7565788434368939e5026 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:44:08 +0000 Subject: [PATCH 064/209] fixed doctest --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 37098f68a..cbea585f6 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1735,7 +1735,7 @@ function unset_flag!(vi::VarInfo, vn::VarName, flag::String) return vi end function unset_flag!(metadata::Metadata, vn::VarName, flag::String) - metadata.flags[flag][getidx(vi, vn)] = false + metadata.flags[flag][getidx(metadata, vn)] = false return metadata end unset_flag!(vnv::VarNameVector, ::VarName, ::String) = vnv From 8a1209c67e1c3399de186a22f56b48ad8bcd81d5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:44:24 +0000 Subject: [PATCH 065/209] formatting --- docs/src/internals/varinfo.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 8fe456ab7..1312f152c 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -287,13 +287,13 @@ DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) In the end, we have the following "rough" performance characteristics for `VarNameVector`: -| Method | Is blazingly fast? | -|:--------------------------------------------------:|:--------------------------------------------------------------------------------------------:| -| `getindex` | ${\color{green} \checkmark}$ | -| `setindex!` | ${\color{green} \checkmark}$ | -| `push!` | ${\color{green} \checkmark}$ | -| `delete!` | ${\color{red} \times}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | +| Method | Is blazingly fast? | +|:---------------------------------------:|:--------------------------------------------------------------------------------------------:| +| `getindex` | ${\color{green} \checkmark}$ | +| `setindex!` | ${\color{green} \checkmark}$ | +| `push!` | ${\color{green} \checkmark}$ | +| `delete!` | ${\color{red} \times}$ | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | ## Other methods From adeadf0b747baa775e1137fa923b58871c3daa6b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:56:50 +0000 Subject: [PATCH 066/209] temporarily disable doctests so we can build docs --- docs/make.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/make.jl b/docs/make.jl index 38c520d3a..c38853d5b 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -23,6 +23,7 @@ makedocs(; "Internals" => ["internals/varinfo.md"], ], checkdocs=:exports, + doctest=false, ) deploydocs(; repo="github.com/TuringLang/DynamicPPL.jl.git", push_preview=true) From 7ff179d77c6e6386abfa5fd8b120c8df75ba9cb2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 2 Jan 2024 18:59:05 +0000 Subject: [PATCH 067/209] added missing compat entry for ForwardDiff in docs --- docs/Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/Project.toml b/docs/Project.toml index 48ebe173c..5b8351de9 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,6 +3,7 @@ DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" LogDensityProblems = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c" MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" @@ -14,6 +15,7 @@ DataStructures = "0.18" Distributions = "0.25" Documenter = "1" FillArrays = "0.13, 1" +ForwardDiff = "0.10" LogDensityProblems = "2" MCMCChains = "5, 6" MLUtils = "0.3, 0.4" From c7ec08af999afb65838cfe752541e8306102eb2c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 08:49:26 +0000 Subject: [PATCH 068/209] moved some shared code into methods to make things a bit cleaner --- src/varnamevector.jl | 91 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 21 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index eaad004c6..96fadacbe 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -336,12 +336,36 @@ function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val) return nothing end +""" + shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) + +Shifts the elements of `x` starting from index `start` by `n` to the right. +""" function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) x[(start + n):end] = x[start:(end - n)] return x end +""" + shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) + +Shifts the ranges of variables in `vnv` starting from index `idx` by `n`. +""" +function shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) + for i in (idx + 1):length(vnv.ranges) + vnv.ranges[i] = vnv.ranges[i] .+ n + end + return nothing +end + # `update!` and `update!!`: update a variable in the varname vector. +""" + update!(vnv::VarNameVector, vn::VarName, val[, transform]) + +Either add a new entry or update existing entry for `vn` in `vnv` with the value `val`. + +If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). +""" function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) if !haskey(vnv, vn) # Here we just add a new entry. @@ -410,14 +434,12 @@ function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) n_extra = n_new - n_allocated # Allocate. resize!(vnv.vals, length(vnv.vals) + n_extra) - # Shift current values. + # Shift current values. shift_right!(vnv.vals, end_old + 1, n_extra) # No more inactive entries. had_inactive && delete!(vnv.num_inactive, idx) # Update the ranges for all variables after this one. - for i in (idx + 1):length(vnv.varnames) - vnv.ranges[i] = vnv.ranges[i] .+ n_extra - end + shift_subsequent_ranges_by!(vnv, idx, n_extra) elseif n_new == n_allocated # => No more inactive entries. had_inactive && delete!(vnv.num_inactive, idx) @@ -474,7 +496,12 @@ function contiguify!(vnv::VarNameVector) return vnv end -# Typed version. +""" + group_by_symbol(vnv::VarNameVector) + +Return a dictionary mapping symbols to `VarNameVector`s with +varnames containing that symbol. +""" function group_by_symbol(vnv::VarNameVector) # Group varnames in `vnv` by the symbol. d = OrderedDict{Symbol,Vector{VarName}}() @@ -495,6 +522,41 @@ function group_by_symbol(vnv::VarNameVector) return OrderedDict(zip(keys(d), nt_vals)) end +""" + shift_index_left!(vnv::VarNameVector, idx::Int) + +Shift the index `idx` to the left by one and update the relevant fields. + +!!! warning + This does not check if index we're shifting to is already occupied. +""" +function shift_index_left!(vnv::VarNameVector, idx::Int) + # Shift the index in the lookup table. + vn = vnv.varnames[idx] + vnv.varname_to_index[vn] = idx - 1 + # Shift the index in the inactive ranges. + if haskey(vnv.num_inactive, idx) + # Done in increasing order => don't need to worry about + # potentially shifting the same index twice. + vnv.num_inactive[idx - 1] = pop!(vnv.num_inactive, idx) + end +end + +""" + shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) + +Shift the indices for all variables after `idx` to the left by one and update +the relevant fields. + +This just +""" +function shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) + # Shift the indices for all variables after `idx`. + for idx_to_shift in (idx + 1):length(vnv.varnames) + shift_index_left!(vnv, idx_to_shift) + end +end + function Base.delete!(vnv::VarNameVector, vn::VarName) # Error if we don't have the variable. !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) @@ -505,6 +567,7 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) # Delete the values. r_start = first(getrange(vnv, idx)) n_allocated = num_allocated(vnv, idx) + # NOTE: `deleteat!` also results in a `resize!` so we don't need to do that. deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) # Delete `vn` from the lookup table. @@ -515,24 +578,10 @@ function Base.delete!(vnv::VarNameVector, vn::VarName) # Re-adjust the indices for varnames occuring after `vn` so # that they point to the correct indices after the deletions below. - for idx_to_shift in (idx + 1):length(vnv.varnames) - vn = vnv.varnames[idx_to_shift] - if idx_to_shift > idx - # Shift the index in the lookup table. - vnv.varname_to_index[vn] = idx_to_shift - 1 - # Shift the index in the inactive ranges. - if haskey(vnv.num_inactive, idx_to_shift) - # Done in increasing order => don't need to worry about - # potentially shifting the same index twice. - vnv.num_inactive[idx_to_shift - 1] = pop!(vnv.num_inactive, idx_to_shift) - end - end - end + shift_subsequent_indices_left!(vnv, idx) # Re-adjust the ranges for varnames occuring after `vn`. - for idx_to_shift in (idx + 1):length(vnv.varnames) - vnv.ranges[idx_to_shift] = vnv.ranges[idx_to_shift] .- n_allocated - end + shift_subsequent_ranges_by!(vnv, idx, -n_allocated) # Delete references from vector fields, thus shifting the indices of # varnames occuring after `vn` by one to the left, as we adjusted for above. From c5a5e585907fede7970d6b2138a648289ce2de8a Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 09:38:18 +0000 Subject: [PATCH 069/209] added impl of `merge` for `VarNameVector` --- src/varnamevector.jl | 73 +++++++++++++++++++++++++++++++++++++++++++ test/varnamevector.jl | 27 ++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 96fadacbe..30d848518 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -277,6 +277,79 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) +function Base.merge(left::VarNameVector, right::VarNameVector) + # Return early if possible. + isempty(left) && return deepcopy(right) + isempty(right) && return deepcopy(left) + + # A very simple way of doing this would be to just + # convert both to `OrderedDict` and merge. + # However, we need to also account of transformations. + + # Determine varnames. + vns_left = left.varnames + vns_right = right.varnames + vns_both = union(vns_left, vns_right) + + # Determine `eltype` of `vals`. + T_left = eltype(left.vals) + T_right = eltype(right.vals) + T = promote_type(T_left, T_right) + # TODO: Is this necessary? + if !(T <: Real) + T = Real + end + + # Determine `eltype` of `varnames`. + V_left = eltype(left.varnames) + V_right = eltype(right.varnames) + V = promote_type(V_left, V_right) + if !(V <: VarName) + V = VarName + end + + # Determine `eltype` of `transforms`. + F_left = eltype(left.transforms) + F_right = eltype(right.transforms) + F = promote_type(F_left, F_right) + + # Allocate. + varnames_to_index = OrderedDict{V,Int}() + ranges = UnitRange{Int}[] + vals = T[] + transforms = F[] + + # Range offset. + offset = 0 + + for (idx, vn) in enumerate(vns_both) + # Extract the necessary information from `left` or `right`. + if vn in vns_left && !(vn in vns_right) + # `vn` is only in `left`. + varnames_to_index[vn] = idx + val = getindex_raw(left, vn) + n = length(val) + r = (offset + 1):(offset + n) + f = gettransform(left, vn) + else + # `vn` is either in both or just `right`. + varnames_to_index[vn] = idx + val = getindex_raw(right, vn) + n = length(val) + r = (offset + 1):(offset + n) + f = gettransform(right, vn) + end + # Update. + append!(vals, val) + push!(ranges, r) + push!(transforms, f) + # Increment `offset`. + offset += n + end + + return VarNameVector(varnames_to_index, vns_both, ranges, vals, transforms) +end + # `similar` similar_metadata(::Nothing) = nothing similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index a37d23a6f..b30135e74 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -245,6 +245,33 @@ end @test !haskey(vnv, vn_right) end + # `merge` + @testset "merge" begin + # When there are no inactive entries, `merge` on itself result in the same. + @test merge(vnv_base, vnv_base) == vnv_base + + # Merging with empty should result in the same. + @test merge(vnv_base, similar(vnv_base)) == vnv_base + @test merge(similar(vnv_base), vnv_base) == vnv_base + + # With differences. + vnv_left_only = deepcopy(vnv_base) + delete!(vnv_left_only, vn_right) + vnv_right_only = deepcopy(vnv_base) + delete!(vnv_right_only, vn_left) + + # `(x,)` and `(x, y)` should be `(x, y)`. + @test merge(vnv_left_only, vnv_base) == vnv_base + # `(x, y)` and `(x,)` should be `(x, y)`. + @test merge(vnv_base, vnv_left_only) == vnv_base + # `(x, y)` and `(y,)` should be `(x, y)`. + @test merge(vnv_base, vnv_right_only) == vnv_base + # `(y,)` and `(x, y)` should be `(y, x)`. + vnv_merged = merge(vnv_right_only, vnv_base) + @test vnv_merged != vnv_base + @test collect(keys(vnv_merged)) == [vn_right, vn_left] + end + # `push!` & `update!` @testset "push!" begin vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) From c376d956f77a96904828bd9a50bb6ecb50c43dbf Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 09:41:02 +0000 Subject: [PATCH 070/209] renamed a few variables in `merge` impl for `VarNameVector` --- src/varnamevector.jl | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 30d848518..f33f35365 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -277,23 +277,19 @@ function Base.empty!(vnv::VarNameVector) end BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) -function Base.merge(left::VarNameVector, right::VarNameVector) +function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) # Return early if possible. - isempty(left) && return deepcopy(right) - isempty(right) && return deepcopy(left) - - # A very simple way of doing this would be to just - # convert both to `OrderedDict` and merge. - # However, we need to also account of transformations. + isempty(left_vnv) && return deepcopy(right_vnv) + isempty(right_vnv) && return deepcopy(left_vnv) # Determine varnames. - vns_left = left.varnames - vns_right = right.varnames + vns_left = left_vnv.varnames + vns_right = right_vnv.varnames vns_both = union(vns_left, vns_right) # Determine `eltype` of `vals`. - T_left = eltype(left.vals) - T_right = eltype(right.vals) + T_left = eltype(left_vnv.vals) + T_right = eltype(right_vnv.vals) T = promote_type(T_left, T_right) # TODO: Is this necessary? if !(T <: Real) From f71baa54014ab3a798d69c0baf2b7d9c5e559369 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 09:41:27 +0000 Subject: [PATCH 071/209] forgot to include some changes in previous commit --- src/varnamevector.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index f33f35365..8082e3068 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -297,16 +297,16 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) end # Determine `eltype` of `varnames`. - V_left = eltype(left.varnames) - V_right = eltype(right.varnames) + V_left = eltype(left_vnv.varnames) + V_right = eltype(right_vnv.varnames) V = promote_type(V_left, V_right) if !(V <: VarName) V = VarName end # Determine `eltype` of `transforms`. - F_left = eltype(left.transforms) - F_right = eltype(right.transforms) + F_left = eltype(left_vnv.transforms) + F_right = eltype(right_vnv.transforms) F = promote_type(F_left, F_right) # Allocate. @@ -323,17 +323,17 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) if vn in vns_left && !(vn in vns_right) # `vn` is only in `left`. varnames_to_index[vn] = idx - val = getindex_raw(left, vn) + val = getindex_raw(left_vnv, vn) n = length(val) r = (offset + 1):(offset + n) - f = gettransform(left, vn) + f = gettransform(left_vnv, vn) else # `vn` is either in both or just `right`. varnames_to_index[vn] = idx - val = getindex_raw(right, vn) + val = getindex_raw(right_vnv, vn) n = length(val) r = (offset + 1):(offset + n) - f = gettransform(right, vn) + f = gettransform(right_vnv, vn) end # Update. append!(vals, val) From af25f3cfc6561d292a482db5ba1f8f5a91b9176f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:54:26 +0000 Subject: [PATCH 072/209] added impl of `subset` for `VarNameVector` --- src/varnamevector.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 8082e3068..219c93416 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -346,6 +346,19 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) return VarNameVector(varnames_to_index, vns_both, ranges, vals, transforms) end +function subset(vnv::VarNameVector, vns::AbstractVector{<:VarName}) + # NOTE: This does not specialize types when possible. + vnv_new = similar(vnv) + # Return early if possible. + isempty(vnv) && return vnv_new + + for vn in vns + push!(vnv_new, vn, getval(vnv, vn), gettransform(vnv, vn)) + end + + return vnv_new +end + # `similar` similar_metadata(::Nothing) = nothing similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) From c28f0762ac73b36e4dda5fa0782bdb0781647f29 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:54:38 +0000 Subject: [PATCH 073/209] fixed `pairs` impl for `VarNameVector` --- src/varnamevector.jl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 219c93416..02c659c79 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -211,12 +211,7 @@ Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() # Dictionary interface. Base.keys(vnv::VarNameVector) = vnv.varnames Base.values(vnv::VarNameVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) - -function Base.pairs(vnv::VarNameVector) - return Iterators.zip( - vnv.varnames, Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) - ) -end +Base.pairs(vnv::VarNameVector) = (vn => vnv[vn] for vn in keys(vnv)) Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) From f5d2c6391a4511f29129ee463aa167fcbf378c01 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:55:02 +0000 Subject: [PATCH 074/209] added missing impl of `subset` for `VectorVarInfo` --- src/varinfo.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index cbea585f6..ffe792886 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -284,6 +284,11 @@ function subset(varinfo::UntypedVarInfo, vns::AbstractVector{<:VarName}) return VarInfo(metadata, varinfo.logp, varinfo.num_produce) end +function subset(varinfo::VectorVarInfo, vns::AbstractVector{<:VarName}) + metadata = subset(varinfo.metadata, vns) + return VarInfo(metadata, varinfo.logp, varinfo.num_produce) +end + function subset(varinfo::TypedVarInfo, vns::AbstractVector{<:VarName{sym}}) where {sym} # If all the variables are using the same symbol, then we can just extract that field from the metadata. metadata = subset(getfield(varinfo.metadata, sym), vns) From 3eb6c7f2fa72bebd696de4389c89d682485c569a Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:55:16 +0000 Subject: [PATCH 075/209] added missing impl of `merge_metadata` for `VarNameVector` --- src/varinfo.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index ffe792886..d9e71335e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -364,6 +364,10 @@ function _merge(varinfo_left::VarInfo, varinfo_right::VarInfo) ) end +function merge_metadata(vnv_left::VarNameVector, vnv_right::VarNameVector) + return merge(vnv_left, vnv_right) +end + @generated function merge_metadata( metadata_left::NamedTuple{names_left}, metadata_right::NamedTuple{names_right} ) where {names_left,names_right} From 9ba8144dc9874d8d8548cf3563008d2054410375 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:55:29 +0000 Subject: [PATCH 076/209] added a bunch of `from_vec_transform` and `tovec` impls to make `VarNameVector` work with `Cholesky`, etc. --- src/varnamevector.jl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 02c659c79..8a5be6656 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -70,8 +70,17 @@ FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) +from_vec_transform(x::Real) = FromVec(()) +from_vec_transform(x::AbstractArray) = FromVec(size(x)) +from_vec_transform(C::Cholesky) = from_vec_transform(C.UL) +from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu +from_vec_transform(L::LinearAlgebra.LowerTriangular) = transpose ∘ from_vec_transform + tovec(x::Real) = [x] tovec(x::AbstractArray) = vec(x) +tovec(C::Cholesky) = tovec(C.UL) +tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) +tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) Bijectors.inverse(f::FromVec) = tovec Bijectors.inverse(f::FromVec{Tuple{}}) = tovec From acd695143ebf67201db631c0446a3796380ae083 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:56:39 +0000 Subject: [PATCH 077/209] make default args use `from_vec_transform` rather than `FromVec` --- src/varnamevector.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 8a5be6656..e98d0b09f 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -94,7 +94,7 @@ VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) function VarNameVector( - varnames::AbstractVector, vals::AbstractVector, transforms=map(FromVec, vals) + varnames::AbstractVector, vals::AbstractVector, transforms=map(from_vec_transform, vals) ) # TODO: Check uniqueness of `varnames`? @@ -407,7 +407,7 @@ function nextrange(vnv::VarNameVector, x) end # `push!` and `push!!`: add a variable to the varname vector. -function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) +function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) # NOTE: We need to compute the `nextrange` BEFORE we start mutating @@ -452,7 +452,7 @@ Either add a new entry or update existing entry for `vn` in `vnv` with the valu If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). """ -function update!(vnv::VarNameVector, vn::VarName, val, transform=FromVec(val)) +function update!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) if !haskey(vnv, vn) # Here we just add a new entry. return push!(vnv, vn, val, transform) From 790f743a445a90067b8b5dd80ae444fb3c008f12 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 3 Jan 2024 17:56:58 +0000 Subject: [PATCH 078/209] fixed `values_as` fro `VarInfo` with `VarNameVector` as `metadata` --- src/varinfo.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index d9e71335e..545fcbdb3 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2079,6 +2079,10 @@ end function values_as(vi::UntypedVarInfo, ::Type{D}) where {D<:AbstractDict} return ConstructionBase.constructorof(D)(values_from_metadata(vi.metadata)) end +values_as(vi::VectorVarInfo, ::Type{NamedTuple}) = values_as(vi.metadata, NamedTuple) +function values_as(vi::VectorVarInfo, ::Type{D}) where {D<:AbstractDict} + return values_as(vi.metadata, D) +end function values_as(vi::VarInfo{<:NamedTuple{names}}, ::Type{NamedTuple}) where {names} iter = Iterators.flatten(values_from_metadata(getfield(vi.metadata, n)) for n in names) @@ -2098,3 +2102,5 @@ function values_from_metadata(md::Metadata) vn in md.vns ) end + +values_from_metadata(md::VarNameVector) = pairs(md) From c474bb08e68ba7bee306bf47a2c669350ee53c7f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 4 Jan 2024 09:51:17 +0000 Subject: [PATCH 079/209] fixed impl of `getindex_raw` when using integer index for `VarNameVector` --- src/varnamevector.jl | 54 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index e98d0b09f..4fedfdeb6 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -178,6 +178,13 @@ Returns `true` if `vnv` has inactive ranges. """ has_inactive(vnv::VarNameVector) = !isempty(vnv.num_inactive) +""" + num_inactive(vnv::VarNameVector) + +Return the number of inactive entries in `vnv`. +""" +num_inactive(vnv::VarNameVector) = sum(values(vnv.num_inactive)) + """ num_inactive(vnv::VarNameVector, vn::VarName) @@ -232,11 +239,52 @@ function Base.getindex(vnv::VarNameVector, vn::VarName) return f(x) end -getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[i] -function getindex_raw(vnv::VarNameVector, vn::VarName) - return vnv.vals[getrange(vnv, vn)] +function find_range_from_sorted(ranges::AbstractVector{<:AbstractRange}, x) + # TODO: Assume `ranges` to be sorted and contiguous, and use `searchsortedfirst` + # for a more efficient approach. + range_idx = findfirst(Base.Fix1(∈, x), ranges) + + # If we're out of bounds, we raise an error. + if range_idx === nothing + throw(ArgumentError("Value $x is not in any of the ranges.")) + end + + return range_idx end +function adjusted_ranges(vnv::VarNameVector) + # Every range following inactive entries needs to be shifted. + offset = 0 + ranges_adj = similar(vnv.ranges) + for (idx, r) in enumerate(vnv.ranges) + # Remove the `offset` in `r` due to inactive entries. + ranges_adj[idx] = r .- offset + # Update `offset`. + offset += get(vnv.num_inactive, idx, 0) + end + + return ranges_adj +end + +function index_to_raw_index(vnv::VarNameVector, i::Int) + # If we don't have any inactive entries, there's nothing to do. + has_inactive(vnv) || return i + + # Get the adjusted ranges. + ranges_adj = adjusted_ranges(vnv) + # Determine the adjusted range that the index corresponds to. + r_idx = find_range_from_sorted(ranges_adj, i) + r = vnv.ranges[r_idx] + # Determine how much of the index `i` is used to get to this range. + i_used = r_idx == 1 ? 0 : sum(length, ranges_adj[1:(r_idx - 1)]) + # Use remainder to index into `r`. + i_remainder = i - i_used + return r[i_remainder] +end + +getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] +getindex_raw(vnv::VarNameVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] + # `getindex` for `Colon` function Base.getindex(vnv::VarNameVector, ::Colon) return if has_inactive(vnv) From 8251463ae509001dbd300ffb96f5827e9fce1d58 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 4 Jan 2024 12:39:03 +0000 Subject: [PATCH 080/209] added tests for `getindex` with `Int` index for `VarNameVector` --- test/varnamevector.jl | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index b30135e74..ce46e86fc 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -206,9 +206,13 @@ end # `getindex` @testset "getindex" begin - # `getindex` + # With `VarName` index. @test vnv_base[vn_left] == val_left @test vnv_base[vn_right] == val_right + + # With `Int` index. + val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) + @test all(vnv_base[i] == val_vec[i] for i in 1:length(val_vec)) end # `setindex!` @@ -222,8 +226,15 @@ end # `getindex_raw` @testset "getindex_raw" begin + # With `VarName` index. @test DynamicPPL.getindex_raw(vnv_base, vn_left) == to_vec_left(val_left) @test DynamicPPL.getindex_raw(vnv_base, vn_right) == to_vec_right(val_right) + # With `Int` index. + val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) + @test all( + DynamicPPL.getindex_raw(vnv_base, i) == val_vec[i] for + i in 1:length(val_vec) + ) end # `setindex_raw!` @@ -298,12 +309,16 @@ end end DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] @test vnv[vn] == val .+ 1 @test length(vnv) == expected_length - @test length(vnv[:]) == length(vnv) + @test length(x) == length(vnv) # There should be no redundant values in the underlying vector. @test !DynamicPPL.has_inactive(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -319,9 +334,13 @@ end end DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] @test vnv[vn] == val .+ 1 @test length(vnv) == expected_length - @test length(vnv[:]) == length(vnv) + @test length(x) == length(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -336,9 +355,13 @@ end length(vnv) + length(val) end DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] @test vnv[vn] == val .+ 1 @test length(vnv) == expected_length - @test length(vnv[:]) == length(vnv) + @test length(x) == length(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) end end end @@ -395,12 +418,6 @@ end end end -has_varnamevector(vi) = false -function has_varnamevector(vi::VarInfo) - return vi.metadata isa VarNameVector || - (vi isa TypedVarInfo && first(values(vi.metadata)) isa VarNameVector) -end - @testset "VarInfo + VarNameVector" begin models = DynamicPPL.TestUtils.DEMO_MODELS @testset "$(model.f)" for model in models From 5df703170190b1234d2525693506fc8cac82fd1e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 4 Jan 2024 12:39:22 +0000 Subject: [PATCH 081/209] fix for `setindex!` and `setindex_raw!` for `VarNameVector` --- src/varnamevector.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 4fedfdeb6..f15f86078 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -311,7 +311,7 @@ function Base.setindex!(vnv::VarNameVector, val, vn::VarName) return setindex_raw!(vnv, f(val), vn) end -setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[i] = val +setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] = val function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) return vnv.vals[getrange(vnv, vn)] = val end From 683b776bed196fd060325b250ef9d0cb289ee8dc Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 11:32:36 +0000 Subject: [PATCH 082/209] introduction of `from_vec_transform` and `tovec` and its usage in `VarInfo` --- src/utils.jl | 7 +++---- src/varinfo.jl | 2 +- src/varnamevector.jl | 32 ++++++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 70dec4e28..b00425c94 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -212,9 +212,7 @@ invlink_transform(dist) = inverse(link_transform(dist)) ##################################################### vectorize(d, r) = vectorize(r) -vectorize(r::Real) = [r] -vectorize(r::AbstractArray{<:Real}) = copy(vec(r)) -vectorize(r::Cholesky) = copy(vec(r.UL)) +vectorize(r) = tovec(r) # NOTE: # We cannot use reconstruct{T} because val is always Vector{Real} then T will be Real. @@ -240,7 +238,8 @@ reconstruct(::MatrixDistribution, val::AbstractMatrix{<:Real}) = copy(val) reconstruct(::Inverse{Bijectors.VecCorrBijector}, ::LKJ, val::AbstractVector) = copy(val) function reconstruct(dist::LKJCholesky, val::AbstractVector{<:Real}) - return reconstruct(dist, Matrix(reshape(val, size(dist)))) + f = from_vec_transform(dist) + return reconstruct(dist, f(val)) end function reconstruct(dist::LKJCholesky, val::AbstractMatrix{<:Real}) return Cholesky(val, dist.uplo, 0) diff --git a/src/varinfo.jl b/src/varinfo.jl index 545fcbdb3..b3174c20a 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -135,7 +135,7 @@ function metadata_to_varnamevector(md::Metadata) vals = copy(md.vals) transforms = map(md.dists) do dist # TODO: Handle linked distributions. - FromVec(size(dist)) + from_vec_transform(dist) end return VarNameVector( diff --git a/src/varnamevector.jl b/src/varnamevector.jl index f15f86078..ebf7a8e79 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -58,7 +58,7 @@ function VarNameVector{K,V}() where {K,V} end # Useful transformation going from the flattened representation. -struct FromVec{Sz} +struct FromVec{Sz} <: Bijectors.Bijector sz::Sz end @@ -67,24 +67,44 @@ FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) # TODO: Should we materialize the `reshape`? (f::FromVec)(x) = reshape(x, f.sz) (f::FromVec{Tuple{}})(x) = only(x) +# TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) +struct ToChol <: Bijectors.Bijector + uplo::Char +end + +Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) +Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) + from_vec_transform(x::Real) = FromVec(()) +from_vec_transform(x::AbstractVector) = identity from_vec_transform(x::AbstractArray) = FromVec(size(x)) -from_vec_transform(C::Cholesky) = from_vec_transform(C.UL) +function from_vec_transform(C::Cholesky) + return ToChol(C.uplo) ∘ from_vec_transform(C.UL) +end from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu -from_vec_transform(L::LinearAlgebra.LowerTriangular) = transpose ∘ from_vec_transform +function from_vec_transform(L::LinearAlgebra.LowerTriangular) + return transpose ∘ from_vec_transform(transpose(L)) +end + +# FIXME: remove the `rand` below. +from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) +# We want to use the inverse of `FromVec` so it preserves the size information. +Bijectors.transform(::Bijectors.Inverse{<:FromVec}, x) = tovec(x) + +# FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix +# flattened, while using `tovec` below flattenes only the necessary entries. +# => Need to either fix how `VarInfo` does things, i.e. use `tovec` everywhere, +# or fix `tovec` to flatten the full matrix instead of using `Bijectors.triu_to_vec`. tovec(x::Real) = [x] tovec(x::AbstractArray) = vec(x) tovec(C::Cholesky) = tovec(C.UL) tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) -Bijectors.inverse(f::FromVec) = tovec -Bijectors.inverse(f::FromVec{Tuple{}}) = tovec - # More convenient constructors. collect_maybe(x) = collect(x) collect_maybe(x::AbstractArray) = x From 4dae00d9e8a1b328b2e287a52b6ae4da3f9736f1 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 11:33:55 +0000 Subject: [PATCH 083/209] moved definition of `is_splat_symbol` to the file where it's used --- src/compiler.jl | 2 -- src/model.jl | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compiler.jl b/src/compiler.jl index e7c44d16b..081a4bca9 100644 --- a/src/compiler.jl +++ b/src/compiler.jl @@ -591,8 +591,6 @@ function namedtuple_from_splitargs(splitargs) return :(NamedTuple{$names_expr}($vals)) end -is_splat_symbol(s::Symbol) = startswith(string(s), "#splat#") - """ build_output(modeldef, linenumbernode) diff --git a/src/model.jl b/src/model.jl index c0cc2f26f..0b451e36a 100644 --- a/src/model.jl +++ b/src/model.jl @@ -963,6 +963,8 @@ function _evaluate!!(model::Model, varinfo::AbstractVarInfo, context::AbstractCo return model.f(args...; kwargs...) end +is_splat_symbol(s::Symbol) = startswith(string(s), "#splat#") + """ make_evaluate_args_and_kwargs(model, varinfo, context) From e3b52a48557b219c585b37e28dd0db9bf26fa899 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:30:59 +0000 Subject: [PATCH 084/209] added `VarInfo` constructor with vector input for `VectorVarInfo` --- src/varinfo.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index b3174c20a..7771e64d2 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -119,6 +119,12 @@ function VarInfo(old_vi::UntypedVarInfo, spl, x::AbstractVector) return new_vi end +function VarInfo(old_vi::VectorVarInfo, spl, x::AbstractVector) + new_vi = deepcopy(old_vi) + new_vi[spl] = x + return new_vi +end + function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) md = newmetadata(old_vi.metadata, Val(getspace(spl)), x) return VarInfo( From 9626be14d23691d597b10fee277bd069ef97402d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:31:26 +0000 Subject: [PATCH 085/209] make `extract_priors` take the `rng` as an argument --- src/extract_priors.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index bb6721a9c..a22e65fad 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -110,9 +110,15 @@ julia> length(extract_priors(rng, model)[@varname(x)]) 9 ``` """ -extract_priors(model::Model) = extract_priors(Random.default_rng(), model) +extract_priors(args...) = extract_priors(Random.default_rng(), args...) function extract_priors(rng::Random.AbstractRNG, model::Model) context = PriorExtractorContext(SamplingContext(rng)) evaluate!!(model, VarInfo(), context) return context.priors end + +function extract_priors(::Random.AbstractRNG, model::Model, varinfo::AbstractVarInfo) + context = PriorExtractorContext(DefaultContext()) + evaluate!!(model, varinfo, context) + return context.priors +end From e731fd69c19fb74b1dc0032c720153502e4ed7d4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:31:44 +0000 Subject: [PATCH 086/209] added `replace_values` for `Metadata` --- src/varinfo.jl | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 7771e64d2..ff6617db5 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -196,6 +196,12 @@ function VarInfo(rng::Random.AbstractRNG, model::Model, context::AbstractContext return VarInfo(rng, model, SampleFromPrior(), context) end +function replace_values(md::Metadata, vals) + return Metadata( + md.idcs, md.vns, md.ranges, vals, md.dists, md.gids, md.orders, md.flags + ) +end + @generated function newmetadata( metadata::NamedTuple{names}, ::Val{space}, x ) where {names,space} @@ -205,21 +211,7 @@ end mdf = :(metadata.$f) if inspace(f, space) || length(space) == 0 len = :(sum(length, $mdf.ranges)) - push!( - exprs, - :( - $f = Metadata( - $mdf.idcs, - $mdf.vns, - $mdf.ranges, - x[($offset + 1):($offset + $len)], - $mdf.dists, - $mdf.gids, - $mdf.orders, - $mdf.flags, - ) - ), - ) + push!(exprs, :($f = replace_values($mdf, x[($offset + 1):($offset + $len)]))) offset = :($offset + $len) else push!(exprs, :($f = $mdf)) From 0785abf100b17e18d8fa5500a5db2a7614c37674 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:32:48 +0000 Subject: [PATCH 087/209] make link and invlink act on the `metadata` field for `VarInfo` + implementations of these for `Metadata` and `VarNameVector` --- src/varinfo.jl | 174 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 151 insertions(+), 23 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index ff6617db5..1e25ff6e1 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -634,12 +634,19 @@ Set the values of all the variables in `vi` to `val`. The values may or may not be transformed to Euclidean space. """ -function setall!(vi::UntypedVarInfo, val) - for r in vi.metadata.ranges - vi.metadata.vals[r] .= val[r] +setall!(vi::VarInfo, val) = _setall!(vi.metadata, val) + +function _setall!(metadata::Metadata, val) + for r in metadata.ranges + metadata.vals[r] .= val[r] + end +end +function _setall!(vnv::VarNameVector, val) + # TODO: Do something more efficient here. + for i in 1:length(vnv) + vnv[i] = val[i] end end -setall!(vi::TypedVarInfo, val) = _setall!(vi.metadata, val) @generated function _setall!(metadata::NamedTuple{names}, val) where {names} expr = Expr(:block) start = :(1) @@ -1192,6 +1199,10 @@ end end function _inner_transform!(vi::VarInfo, vn::VarName, dist, f) + return _inner_transform!(getmetadata(vi, vn), vi, vn, dist, f) +end + +function _inner_transform!(md::Metadata, vi::VarInfo, vn::VarName, dist, f) # TODO: Use inplace versions to avoid allocations y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, getval(vi, vn)) yvec = vectorize(dist, y) @@ -1210,14 +1221,15 @@ end # an empty iterable for `SampleFromPrior`, so we need to override it here. # This is quite hacky, but seems safer than changing the behavior of `_getvns`. _getvns_link(varinfo::VarInfo, spl::AbstractSampler) = _getvns(varinfo, spl) -_getvns_link(varinfo::UntypedVarInfo, spl::SampleFromPrior) = nothing +_getvns_link(varinfo::VarInfo, spl::SampleFromPrior) = nothing function _getvns_link(varinfo::TypedVarInfo, spl::SampleFromPrior) return map(Returns(nothing), varinfo.metadata) end function link(::DynamicTransformation, varinfo::VarInfo, spl::AbstractSampler, model::Model) - return _link(varinfo, spl) + return _link(model, varinfo, spl) end + function link( ::DynamicTransformation, varinfo::ThreadSafeVarInfo{<:VarInfo}, @@ -1229,30 +1241,43 @@ function link( return Setfield.@set varinfo.varinfo = link(varinfo.varinfo, spl, model) end -function _link(varinfo::UntypedVarInfo, spl::AbstractSampler) +function _link(model::Model, varinfo::UntypedVarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) return VarInfo( - _link_metadata!(varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), + _link_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo)), ) end -function _link(varinfo::TypedVarInfo, spl::AbstractSampler) +function _link(model::Model, varinfo::VectorVarInfo, spl::AbstractSampler) + varinfo = deepcopy(varinfo) + return VarInfo( + _link_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), + Base.Ref(getlogp(varinfo)), + Ref(get_num_produce(varinfo)), + ) +end + +function _link(model::Model, varinfo::TypedVarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) md = _link_metadata_namedtuple!( - varinfo, varinfo.metadata, _getvns_link(varinfo, spl), Val(getspace(spl)) + model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl), Val(getspace(spl)) ) return VarInfo(md, Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo))) end @generated function _link_metadata_namedtuple!( - varinfo::VarInfo, metadata::NamedTuple{names}, vns::NamedTuple, ::Val{space} + model::Model, + varinfo::VarInfo, + metadata::NamedTuple{names}, + vns::NamedTuple, + ::Val{space}, ) where {names,space} vals = Expr(:tuple) for f in names if inspace(f, space) || length(space) == 0 - push!(vals.args, :(_link_metadata!(varinfo, metadata.$f, vns.$f))) + push!(vals.args, :(_link_metadata!(model, varinfo, metadata.$f, vns.$f))) else push!(vals.args, :(metadata.$f)) end @@ -1260,7 +1285,7 @@ end return :(NamedTuple{$names}($vals)) end -function _link_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) +function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, target_vns) vns = metadata.vns # Construct the new transformed values, and keep track of their lengths. @@ -1272,8 +1297,8 @@ function _link_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) end # Transform to constrained space. - x = getval(varinfo, vn) - dist = getdist(varinfo, vn) + x = getval(metadata, vn) + dist = getdist(metadata, vn) f = link_transform(dist) y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, x) # Vectorize value. @@ -1308,10 +1333,65 @@ function _link_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) ) end +function _link_metadata!( + model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns +) + vns = keys(metadata) + # Need to extract the priors from the model. + dists = extract_priors(model, varinfo) + + # Construct the linking transformations. + link_transforms = map(vns) do vn + # If `vn` is not part of `target_vns`, the `identity` transformation is used. + if (target_vns !== nothing && vn ∉ target_vns) + return identity + end + + # Otherwise, we derive the transformation from the distribution. + link_transform(getindex(dists, vn)) + end + # Compute the transformed values. + ys = map(vns, link_transforms) do vn, f + # TODO: Do we need to handle scenarios where `vn` is not in `dists`? + dist = dists[vn] + x = getval(metadata, vn) + y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, x) + # Accumulate the log-abs-det jacobian correction. + acclogp!!(varinfo, -logjac) + # Return the transformed value. + return y + end + # Extract the from-vec transformations. + fromvec_transforms = map(from_vec_transform, ys) + # Compose the transformations to form a full transformation from + # unconstrained vector representation to constrained space. + transforms = map(∘, fromvec_transforms, link_transforms) + # Convert to vector representation. + yvecs = map(tovec, ys) + + # Determine new ranges. + ranges_new = similar(metadata.ranges) + offset = 0 + for (i, v) in enumerate(yvecs) + r_start, r_end = offset + 1, length(v) + offset + offset = r_end + ranges_new[i] = r_start:r_end + end + + # Now we just create a new metadata with the new `vals` and `ranges`. + return VarNameVector( + metadata.varname_to_index, + metadata.varnames, + ranges_new, + reduce(vcat, yvecs), + transforms, + ) +end + function invlink( ::DynamicTransformation, varinfo::VarInfo, spl::AbstractSampler, model::Model ) - return _invlink(varinfo, spl) + return _invlink(model, varinfo, spl) end function invlink( ::DynamicTransformation, @@ -1324,30 +1404,34 @@ function invlink( return Setfield.@set varinfo.varinfo = invlink(varinfo.varinfo, spl, model) end -function _invlink(varinfo::UntypedVarInfo, spl::AbstractSampler) +function _invlink(model::Model, varinfo::VarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) return VarInfo( - _invlink_metadata!(varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), + _invlink_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo)), ) end -function _invlink(varinfo::TypedVarInfo, spl::AbstractSampler) +function _invlink(model::Model, varinfo::TypedVarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) md = _invlink_metadata_namedtuple!( - varinfo, varinfo.metadata, _getvns_link(varinfo, spl), Val(getspace(spl)) + model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl), Val(getspace(spl)) ) return VarInfo(md, Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo))) end @generated function _invlink_metadata_namedtuple!( - varinfo::VarInfo, metadata::NamedTuple{names}, vns::NamedTuple, ::Val{space} + model::Model, + varinfo::VarInfo, + metadata::NamedTuple{names}, + vns::NamedTuple, + ::Val{space}, ) where {names,space} vals = Expr(:tuple) for f in names if inspace(f, space) || length(space) == 0 - push!(vals.args, :(_invlink_metadata!(varinfo, metadata.$f, vns.$f))) + push!(vals.args, :(_invlink_metadata!(model, varinfo, metadata.$f, vns.$f))) else push!(vals.args, :(metadata.$f)) end @@ -1355,7 +1439,7 @@ end return :(NamedTuple{$names}($vals)) end -function _invlink_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) +function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, target_vns) vns = metadata.vns # Construct the new transformed values, and keep track of their lengths. @@ -1404,6 +1488,50 @@ function _invlink_metadata!(varinfo::VarInfo, metadata::Metadata, target_vns) ) end +function _invlink_metadata!( + model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns +) + # TODO: Make use of `update!` to aovid copying values. + # => Only need to allocate for transformations. + + vns = keys(metadata) + + # Compute the transformed values. + xs = map(vns) do vn + f = inverse(gettransform(metadata, vn)) + y = getval(metadata, vn) + # TODO: Can we remove this `_reconstruct` part? + x, logjac = with_logabsdet_jacobian(f, y) + # Accumulate the log-abs-det jacobian correction. + acclogp!!(varinfo, -logjac) + # Return the transformed value. + return x + end + # Compose the transformations to form a full transformation from + # unconstrained vector representation to constrained space. + transforms = map(from_vec_transform, xs) + # Convert to vector representation. + xvecs = map(tovec, xs) + + # Determine new ranges. + ranges_new = similar(metadata.ranges) + offset = 0 + for (i, v) in enumerate(xvecs) + r_start, r_end = offset + 1, length(v) + offset + offset = r_end + ranges_new[i] = r_start:r_end + end + + # Now we just create a new metadata with the new `vals` and `ranges`. + return VarNameVector( + metadata.varname_to_index, + metadata.varnames, + ranges_new, + reduce(vcat, xvecs), + transforms, + ) +end + """ islinked(vi::VarInfo, spl::Union{Sampler, SampleFromPrior}) From b3e09559ba1aa30b6a40b45a5033b5c94f8be16c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:33:50 +0000 Subject: [PATCH 088/209] added temporary defs of `with_logabsdet_jacobian` and `inverse` for `transpose` and `Bijectors.vec_to_triu` --- src/varinfo.jl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index 1e25ff6e1..baca712d0 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2230,3 +2230,30 @@ function values_from_metadata(md::Metadata) end values_from_metadata(md::VarNameVector) = pairs(md) + +# TODO: Move these. +Bijectors.with_logabsdet_jacobian(::typeof(transpose), x) = (transpose(x), 0) +Bijectors.inverse(::typeof(transpose)) = transpose + +function Bijectors.with_logabsdet_jacobian(::typeof(Bijectors.vec_to_triu), x) + return (Bijectors.vec_to_triu(x), 0) +end + +# HACK: Overload of `invlink_with_logpdf` so we can fix it for `VarNameVector`. +function invlink_with_logpdf(vi::VarInfo, vn::VarName, dist, y) + return _invlink_with_logpdf(getmetadata(vi, vn), vn, dist, y) +end + +function _invlink_with_logpdf(md::Metadata, vn::VarName, dist, y) + # NOTE: Will this cause type-instabilities or will union-splitting save us? + f = istrans(md, vn) ? invlink_transform(dist) : identity + x, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, y) + return x, logpdf(dist, x) + logjac +end + +function _invlink_with_logpdf(vnv::VarNameVector, vn::VarName, dist, y) + # Here the transformation is stored in `vnv` so we just extract and use this. + f = gettransform(vnv, vn) + x, logjac = with_logabsdet_jacobian(f, y) + return x, logpdf(dist, x) + logjac +end From ff963cef47532d9c666fe694fc9dac1a9a886725 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 14:34:24 +0000 Subject: [PATCH 089/209] added invlink_with_logpdf overload for `ThreadSafeVarInfo` --- src/threadsafe.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index fb1cc1c0c..9b6525097 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -238,3 +238,7 @@ function Base.merge(varinfo_left::ThreadSafeVarInfo, varinfo_right::ThreadSafeVa varinfo_left.varinfo, varinfo_right.varinfo ) end + +function invlink_with_logpdf(vi::ThreadSafeVarInfo, vn::VarName, dist, y) + return invlink_with_logpdf(vi.varinfo, vn, dist, y) +end From 03f2b2bdea5dc277694cb341c2a9d87405232248 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 20:23:31 +0000 Subject: [PATCH 090/209] added `is_transformed` field to `VarNameVector` --- src/utils.jl | 62 +++++++++++++++++++++++++++++++++++++-- src/varinfo.jl | 19 ++++++++---- src/varnamevector.jl | 69 +++++++++++--------------------------------- 3 files changed, 91 insertions(+), 59 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index b00425c94..a2007196c 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -211,6 +211,59 @@ invlink_transform(dist) = inverse(link_transform(dist)) # Helper functions for vectorize/reconstruct values # ##################################################### +# Useful transformation going from the flattened representation. +struct FromVec{Sz} <: Bijectors.Bijector + sz::Sz +end + +FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) + +# TODO: Should we materialize the `reshape`? +(f::FromVec)(x) = reshape(x, f.sz) +(f::FromVec{Tuple{}})(x) = only(x) +# TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. + +Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) +# We want to use the inverse of `FromVec` so it preserves the size information. +Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:FromVec}, x) = (tovec(x), 0) + +struct ToChol <: Bijectors.Bijector + uplo::Char +end + +Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) +Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) + +from_vec_transform(x::Real) = FromVec(()) +from_vec_transform(x::AbstractVector) = identity +from_vec_transform(x::AbstractArray) = FromVec(size(x)) +function from_vec_transform(C::Cholesky) + return ToChol(C.uplo) ∘ from_vec_transform(C.UL) +end +from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu +function from_vec_transform(L::LinearAlgebra.LowerTriangular) + return transpose ∘ from_vec_transform(transpose(L)) +end + +# FIXME: drop the `rand` below and instead implement on a case-by-case basis. +from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) + +# TODO: Move these. +function Bijectors.with_logabsdet_jacobian(::typeof(Bijectors.vec_to_triu), x) + return (Bijectors.vec_to_triu(x), 0) +end + +# FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix +# flattened, while using `tovec` below flattenes only the necessary entries. +# => Need to either fix how `VarInfo` does things, i.e. use `tovec` everywhere, +# or fix `tovec` to flatten the full matrix instead of using `Bijectors.triu_to_vec`. +tovec(x::Real) = [x] +tovec(x::AbstractArray) = vec(x) +tovec(C::Cholesky) = tovec(C.UL) +tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) +tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) + +# TODO: Remove these. vectorize(d, r) = vectorize(r) vectorize(r) = tovec(r) @@ -340,8 +393,13 @@ end ####################### # Convenience methods # ####################### -collectmaybe(x) = x -collectmaybe(x::Base.AbstractSet) = collect(x) +""" + collect_maybe(x) + +Return `collect(x)` if `x` is an array, otherwise return `x`. +""" +collect_maybe(x) = collect(x) +collect_maybe(x::AbstractArray) = x ####################### # BangBang.jl related # diff --git a/src/varinfo.jl b/src/varinfo.jl index baca712d0..cbe2f6428 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -964,7 +964,6 @@ end istrans(vi::VarInfo, vn::VarName) = istrans(getmetadata(vi, vn), vn) istrans(md::Metadata, vn::VarName) = is_flagged(md, vn, "trans") -istrans(vnv::VarNameVector, vn::VarName) = !(gettransform(vnv, vn) isa FromVec) getlogp(vi::VarInfo) = vi.logp[] @@ -1336,10 +1335,13 @@ end function _link_metadata!( model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns ) + # HACK: We ignore `target_vns` here. vns = keys(metadata) # Need to extract the priors from the model. dists = extract_priors(model, varinfo) + is_transformed = copy(metadata.is_transformed) + # Construct the linking transformations. link_transforms = map(vns) do vn # If `vn` is not part of `target_vns`, the `identity` transformation is used. @@ -1348,6 +1350,7 @@ function _link_metadata!( end # Otherwise, we derive the transformation from the distribution. + is_transformed[getidx(metadata, vn)] = true link_transform(getindex(dists, vn)) end # Compute the transformed values. @@ -1385,6 +1388,7 @@ function _link_metadata!( ranges_new, reduce(vcat, yvecs), transforms, + is_transformed, ) end @@ -1491,19 +1495,23 @@ end function _invlink_metadata!( model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns ) + # HACK: We ignore `target_vns` here. # TODO: Make use of `update!` to aovid copying values. # => Only need to allocate for transformations. vns = keys(metadata) + is_transformed = copy(varinfo.is_transformed) # Compute the transformed values. xs = map(vns) do vn f = inverse(gettransform(metadata, vn)) y = getval(metadata, vn) - # TODO: Can we remove this `_reconstruct` part? + # No need to use `with_reconstruct` as `f` will include this. x, logjac = with_logabsdet_jacobian(f, y) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) + # Mark as no longer transformed. + is_transformed[getidx(metadata, vn)] = false # Return the transformed value. return x end @@ -1529,6 +1537,7 @@ function _invlink_metadata!( ranges_new, reduce(vcat, xvecs), transforms, + is_transformed, ) end @@ -1949,7 +1958,7 @@ end Calls `kernel!(vi, vn, values, keys)` for every `vn` in `vi`. """ function _apply!(kernel!, vi::VarInfoOrThreadSafeVarInfo, values, keys) - keys_strings = map(string, collectmaybe(keys)) + keys_strings = map(string, collect_maybe(keys)) num_indices_seen = 0 for vn in Base.keys(vi) @@ -1971,7 +1980,7 @@ function _apply!(kernel!, vi::VarInfoOrThreadSafeVarInfo, values, keys) end function _apply!(kernel!, vi::TypedVarInfo, values, keys) - return _typed_apply!(kernel!, vi, vi.metadata, values, collectmaybe(keys)) + return _typed_apply!(kernel!, vi, vi.metadata, values, collect_maybe(keys)) end @generated function _typed_apply!( @@ -2007,7 +2016,7 @@ end end function _find_missing_keys(vi::VarInfoOrThreadSafeVarInfo, keys) - string_vns = map(string, collectmaybe(Base.keys(vi))) + string_vns = map(string, collect_maybe(Base.keys(vi))) # If `key` isn't subsumed by any element of `string_vns`, it is not present in `vi`. missing_keys = filter(keys) do key !any(Base.Fix2(subsumes_string, key), string_vns) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index ebf7a8e79..c2a3b489f 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -18,12 +18,15 @@ struct VarNameVector{ "vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" ranges::Vector{UnitRange{Int}} - "vector of values of all the univariate, multivariate and matrix variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" + "vector of values of all variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" vals::TVal # AbstractVector{<:Real} "vector of transformations whose inverse takes us back to the original space" transforms::TTrans + "specifies whether a variable is transformed or not " + is_transformed::BitVector + "additional entries which are considered inactive" num_inactive::OrderedDict{Int,Int} @@ -37,17 +40,26 @@ function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) vnv_left.ranges == vnv_right.ranges && vnv_left.vals == vnv_right.vals && vnv_left.transforms == vnv_right.transforms && + vnv_left.is_transformed == vnv_right.is_transformed && vnv_left.num_inactive == vnv_right.num_inactive && vnv_left.metadata == vnv_right.metadata end -function VarNameVector(varname_to_index, varnames, ranges, vals, transforms) +function VarNameVector( + varname_to_index, + varnames, + ranges, + vals, + transforms, + is_transformed=fill!(BitVector(undef, length(varnames)), 0), +) return VarNameVector( varname_to_index, varnames, ranges, vals, transforms, + is_transformed, OrderedDict{Int,Int}(), nothing, ) @@ -57,58 +69,11 @@ function VarNameVector{K,V}() where {K,V} return VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) end -# Useful transformation going from the flattened representation. -struct FromVec{Sz} <: Bijectors.Bijector - sz::Sz -end - -FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) - -# TODO: Should we materialize the `reshape`? -(f::FromVec)(x) = reshape(x, f.sz) -(f::FromVec{Tuple{}})(x) = only(x) -# TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. - -Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) - -struct ToChol <: Bijectors.Bijector - uplo::Char -end - -Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) -Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) - -from_vec_transform(x::Real) = FromVec(()) -from_vec_transform(x::AbstractVector) = identity -from_vec_transform(x::AbstractArray) = FromVec(size(x)) -function from_vec_transform(C::Cholesky) - return ToChol(C.uplo) ∘ from_vec_transform(C.UL) -end -from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu -function from_vec_transform(L::LinearAlgebra.LowerTriangular) - return transpose ∘ from_vec_transform(transpose(L)) +istrans(vnv::VarNameVector, vn::VarName) = vnv.is_transformed[vnv.varname_to_index[vn]] +function settrans!(vnv::VarNameVector, vn::VarName, val) + return vnv.is_transformed[vnv.varname_to_index[vn]] = val end -# FIXME: remove the `rand` below. -from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) - -# We want to use the inverse of `FromVec` so it preserves the size information. -Bijectors.transform(::Bijectors.Inverse{<:FromVec}, x) = tovec(x) - -# FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix -# flattened, while using `tovec` below flattenes only the necessary entries. -# => Need to either fix how `VarInfo` does things, i.e. use `tovec` everywhere, -# or fix `tovec` to flatten the full matrix instead of using `Bijectors.triu_to_vec`. -tovec(x::Real) = [x] -tovec(x::AbstractArray) = vec(x) -tovec(C::Cholesky) = tovec(C.UL) -tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) -tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) - -# More convenient constructors. -collect_maybe(x) = collect(x) -collect_maybe(x::AbstractArray) = x - VarNameVector() = VarNameVector{VarName,Real}() VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) From 949b33a3465ad02fcb2729bfefa9fadb79f9ec1c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 20:23:58 +0000 Subject: [PATCH 091/209] removed unnecessary defintions of `with_logabsdet_jacobian` and `inverse` for `transpose` --- src/varinfo.jl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index cbe2f6428..acdffdc2f 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2240,14 +2240,6 @@ end values_from_metadata(md::VarNameVector) = pairs(md) -# TODO: Move these. -Bijectors.with_logabsdet_jacobian(::typeof(transpose), x) = (transpose(x), 0) -Bijectors.inverse(::typeof(transpose)) = transpose - -function Bijectors.with_logabsdet_jacobian(::typeof(Bijectors.vec_to_triu), x) - return (Bijectors.vec_to_triu(x), 0) -end - # HACK: Overload of `invlink_with_logpdf` so we can fix it for `VarNameVector`. function invlink_with_logpdf(vi::VarInfo, vn::VarName, dist, y) return _invlink_with_logpdf(getmetadata(vi, vn), vn, dist, y) From cc5ecc40c6300796ff7b284162e28cdb36dfd540 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 20:34:19 +0000 Subject: [PATCH 092/209] fixed issue where we were storing the wrong transformations in `VarNameVector` --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index acdffdc2f..c8f9d274c 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1368,7 +1368,7 @@ function _link_metadata!( fromvec_transforms = map(from_vec_transform, ys) # Compose the transformations to form a full transformation from # unconstrained vector representation to constrained space. - transforms = map(∘, fromvec_transforms, link_transforms) + transforms = map(∘, map(inverse, link_transforms), fromvec_transforms) # Convert to vector representation. yvecs = map(tovec, ys) From 1aae1b413aaf19120096889e87746e5609e8def6 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 20:56:28 +0000 Subject: [PATCH 093/209] make sure `extract_priors` doesn't mutate the `varinfo` --- src/extract_priors.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index a22e65fad..1487b9049 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -119,6 +119,6 @@ end function extract_priors(::Random.AbstractRNG, model::Model, varinfo::AbstractVarInfo) context = PriorExtractorContext(DefaultContext()) - evaluate!!(model, varinfo, context) + evaluate!!(model, deepcopy(varinfo), context) return context.priors end From 8e0853d32d44b6893b9f2edd53990b688631b4c7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:14:58 +0000 Subject: [PATCH 094/209] updated `similar` for `VarNameVector` and fixed `invlink` for `VarNameVector` --- src/varinfo.jl | 4 ++-- src/varnamevector.jl | 1 + test/varinfo.jl | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index c8f9d274c..ab85d57f1 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1500,11 +1500,11 @@ function _invlink_metadata!( # => Only need to allocate for transformations. vns = keys(metadata) - is_transformed = copy(varinfo.is_transformed) + is_transformed = copy(metadata.is_transformed) # Compute the transformed values. xs = map(vns) do vn - f = inverse(gettransform(metadata, vn)) + f = gettransform(metadata, vn) y = getval(metadata, vn) # No need to use `with_reconstruct` as `f` will include this. x, logjac = with_logabsdet_jacobian(f, y) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index c2a3b489f..630f7f870 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -413,6 +413,7 @@ function Base.similar(vnv::VarNameVector) similar(vnv.ranges, 0), similar(vnv.vals, 0), similar(vnv.transforms, 0), + BitVector(), similar(vnv.num_inactive), similar_metadata(vnv.metadata), ) diff --git a/test/varinfo.jl b/test/varinfo.jl index 71e341767..c51c5fb82 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -413,6 +413,9 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) else DynamicPPL.link(varinfo, model) end + for vn in keys(varinfo) + @test DynamicPPL.istrans(varinfo_linked, vn) + end @test length(varinfo[:]) > length(varinfo_linked[:]) varinfo_linked_unflattened = DynamicPPL.unflatten( varinfo_linked, varinfo_linked[:] From 229b168aca8736f6915ceb8100b8d64ce8779173 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:22:24 +0000 Subject: [PATCH 095/209] added handling of `is_transformed` in `merge` for `VarNameVector` --- src/varnamevector.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 630f7f870..92d413196 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -351,6 +351,7 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) ranges = UnitRange{Int}[] vals = T[] transforms = F[] + is_transformed = BitVector(undef, length(vns_both)) # Range offset. offset = 0 @@ -364,6 +365,7 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) n = length(val) r = (offset + 1):(offset + n) f = gettransform(left_vnv, vn) + is_transformed[idx] = istrans(left_vnv, vn) else # `vn` is either in both or just `right`. varnames_to_index[vn] = idx @@ -371,6 +373,7 @@ function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) n = length(val) r = (offset + 1):(offset + n) f = gettransform(right_vnv, vn) + is_transformed[idx] = istrans(right_vnv, vn) end # Update. append!(vals, val) From c581dcf323f448c1e825a18f26adb1ccb07fd13f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:22:42 +0000 Subject: [PATCH 096/209] removed unnecesasry `deepcopy` from outer `link` --- src/abstract_varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index fe2c0b3e5..cfa2d7914 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -563,7 +563,7 @@ If `t` is not provided, `default_transformation(model, vi)` will be used. See also: [`default_transformation`](@ref), [`invlink`](@ref). """ -link(vi::AbstractVarInfo, model::Model) = link(deepcopy(vi), SampleFromPrior(), model) +link(vi::AbstractVarInfo, model::Model) = link(vi, SampleFromPrior(), model) function link(t::AbstractTransformation, vi::AbstractVarInfo, model::Model) return link(t, deepcopy(vi), SampleFromPrior(), model) end From b4d3f5541d244cb67656b5e4fe5f4adf117234b7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:44:47 +0000 Subject: [PATCH 097/209] updated `push!` to also `push!` on `is_transformed` --- src/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 92d413196..1172ae01a 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -456,6 +456,7 @@ function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_tra push!(vnv.ranges, r_new) append!(vnv.vals, val_vec) push!(vnv.transforms, transform) + push!(vnv.is_transformed, false) return nothing end From ed1d00698b21807933fea1b4432ce0b70af3c130 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 21:47:07 +0000 Subject: [PATCH 098/209] skip tests for mutating linking when using VarNameVector --- test/test_util.jl | 9 +++++++++ test/varinfo.jl | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/test/test_util.jl b/test/test_util.jl index 22bfa46cc..707d21733 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -104,3 +104,12 @@ function modify_value_representation(nt::NamedTuple) end return modified_nt end + +has_varnamevector(vi) = false +function has_varnamevector(vi::VarInfo) + return vi.metadata isa VarNameVector || + (vi isa TypedVarInfo && first(values(vi.metadata)) isa VarNameVector) +end +function has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) + return has_varnamevector(vi.varinfo) +end diff --git a/test/varinfo.jl b/test/varinfo.jl index c51c5fb82..c8aadef35 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -405,6 +405,12 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end + if has_varnamevector(varinfo) && mutating + # NOTE: Can't handle mutating `link!` and `invlink!` `VarNameVector`. + @test_broken false + continue + end + # Evaluate the model once to update the logp of the varinfo. varinfo = last(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())) From f13220995654aed074dfe751685685429b5fc009 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 22:38:12 +0000 Subject: [PATCH 099/209] use same projection for `Cholesky` in `VarNameVector` as in `VarInfo` --- src/utils.jl | 17 ++--------------- src/varinfo.jl | 22 +++++++++++++++++++++- test/test_util.jl | 9 --------- test/varinfo.jl | 2 +- test/varnamevector.jl | 2 +- 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index a2007196c..f1ad6bb0e 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -237,31 +237,18 @@ Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = from_vec_transform(x::Real) = FromVec(()) from_vec_transform(x::AbstractVector) = identity from_vec_transform(x::AbstractArray) = FromVec(size(x)) -function from_vec_transform(C::Cholesky) - return ToChol(C.uplo) ∘ from_vec_transform(C.UL) -end -from_vec_transform(U::LinearAlgebra.UpperTriangular) = Bijectors.vec_to_triu -function from_vec_transform(L::LinearAlgebra.LowerTriangular) - return transpose ∘ from_vec_transform(transpose(L)) -end +from_vec_transform(C::Cholesky) = ToChol(C.uplo) ∘ FromVec(size(C.UL)) # FIXME: drop the `rand` below and instead implement on a case-by-case basis. from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) -# TODO: Move these. -function Bijectors.with_logabsdet_jacobian(::typeof(Bijectors.vec_to_triu), x) - return (Bijectors.vec_to_triu(x), 0) -end - # FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix # flattened, while using `tovec` below flattenes only the necessary entries. # => Need to either fix how `VarInfo` does things, i.e. use `tovec` everywhere, # or fix `tovec` to flatten the full matrix instead of using `Bijectors.triu_to_vec`. tovec(x::Real) = [x] tovec(x::AbstractArray) = vec(x) -tovec(C::Cholesky) = tovec(C.UL) -tovec(L::LinearAlgebra.LowerTriangular) = tovec(transpose(L)) -tovec(U::LinearAlgebra.UpperTriangular) = Bijectors.triu_to_vec(U) +tovec(C::Cholesky) = tovec(Matrix(C.UL)) # TODO: Remove these. vectorize(d, r) = vectorize(r) diff --git a/src/varinfo.jl b/src/varinfo.jl index ab85d57f1..b225eae28 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -161,6 +161,20 @@ function VectorVarInfo(vi::TypedVarInfo) return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end +""" + has_varnamevector(varinfo::VarInfo) + +Returns `true` if `varinfo` uses `VarNameVector` as metadata. +""" +has_varnamevector(vi) = false +function has_varnamevector(vi::VarInfo) + return vi.metadata isa VarNameVector || + (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNameVector), values(vi.metadata))) +end +function has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) + return has_varnamevector(vi.varinfo) +end + function untyped_varinfo( rng::Random.AbstractRNG, model::Model, @@ -1018,6 +1032,8 @@ end # X -> R for all variables associated with given sampler function link!!(t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model) + # If we're working with a `VarNameVector`, we always use immutable. + has_varnamevector(vi) && return link(t, vi, spl, model) # Call `_link!` instead of `link!` to avoid deprecation warning. _link!(vi, spl) return vi @@ -1103,7 +1119,11 @@ end end # R -> X for all variables associated with given sampler -function invlink!!(::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model) +function invlink!!( + t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model +) + # If we're working with a `VarNameVector`, we always use immutable. + has_varnamevector(vi) && return invlink(t, vi, spl, model) # Call `_invlink!` instead of `invlink!` to avoid deprecation warning. _invlink!(vi, spl) return vi diff --git a/test/test_util.jl b/test/test_util.jl index 707d21733..22bfa46cc 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -104,12 +104,3 @@ function modify_value_representation(nt::NamedTuple) end return modified_nt end - -has_varnamevector(vi) = false -function has_varnamevector(vi::VarInfo) - return vi.metadata isa VarNameVector || - (vi isa TypedVarInfo && first(values(vi.metadata)) isa VarNameVector) -end -function has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) - return has_varnamevector(vi.varinfo) -end diff --git a/test/varinfo.jl b/test/varinfo.jl index c8aadef35..ddd1b7ea8 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -405,7 +405,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end - if has_varnamevector(varinfo) && mutating + if DynamicPPL.has_varnamevector(varinfo) && mutating # NOTE: Can't handle mutating `link!` and `invlink!` `VarNameVector`. @test_broken false continue diff --git a/test/varnamevector.jl b/test/varnamevector.jl index ce46e86fc..210812932 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -431,7 +431,7 @@ end model, value_true, varnames; include_threadsafe=false ) # Filter out those which are not based on `VarNameVector`. - varinfos = filter(has_varnamevector, varinfos) + varinfos = filter(DynamicPPL.has_varnamevector, varinfos) # Get the true log joint. logp_true = DynamicPPL.TestUtils.logjoint_true(model, value_true...) From 49454de728025a9753138ba79d32f09304bacab0 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 23:05:29 +0000 Subject: [PATCH 100/209] fixed `settrans!!` for `VarInfo` with `VarNameVector` --- src/varinfo.jl | 13 ++++++++++--- src/varnamevector.jl | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index b225eae28..ccb8b97ad 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -681,14 +681,18 @@ Return the set of sampler selectors associated with `vn` in `vi`. getgid(vi::VarInfo, vn::VarName) = getmetadata(vi, vn).gids[getidx(vi, vn)] function settrans!!(vi::VarInfo, trans::Bool, vn::VarName) + return settrans!!(getmetadata(vi, vn), trans, vn) +end +function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) if trans - set_flag!(vi, vn, "trans") + set_flag!(metadata, vn, "trans") else - unset_flag!(vi, vn, "trans") + unset_flag!(metadata, vn, "trans") end return vi end +settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) = settrans!(vnv, trans, vn) function settrans!!(vi::VarInfo, trans::Bool) for vn in keys(vi) @@ -851,7 +855,10 @@ end Set `vn`'s value for `flag` to `true` in `vi`. """ function set_flag!(vi::VarInfo, vn::VarName, flag::String) - return getmetadata(vi, vn).flags[flag][getidx(vi, vn)] = true + return set_flag!(getmetadata(vi, vn), vn, flag) +end +function set_flag!(md::Metadata, vn::VarName, flag::String) + return md.flags[flag][getidx(vi, vn)] = true end #### diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 1172ae01a..7bbb8bda8 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -70,7 +70,7 @@ function VarNameVector{K,V}() where {K,V} end istrans(vnv::VarNameVector, vn::VarName) = vnv.is_transformed[vnv.varname_to_index[vn]] -function settrans!(vnv::VarNameVector, vn::VarName, val) +function settrans!(vnv::VarNameVector, val::Bool, vn::VarName) return vnv.is_transformed[vnv.varname_to_index[vn]] = val end From 01ff2dd9403e485d9b78589868ff67cc27b46f38 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 23:06:01 +0000 Subject: [PATCH 101/209] fixed bug in `set_flag!` --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index ccb8b97ad..ca5be142d 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -858,7 +858,7 @@ function set_flag!(vi::VarInfo, vn::VarName, flag::String) return set_flag!(getmetadata(vi, vn), vn, flag) end function set_flag!(md::Metadata, vn::VarName, flag::String) - return md.flags[flag][getidx(vi, vn)] = true + return md.flags[flag][getidx(md, vn)] = true end #### From 20adedfd71a88af1abc280b337e6631f7b68d6f4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 23:09:31 +0000 Subject: [PATCH 102/209] fixed another typo --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index ca5be142d..ffb19b79e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -690,7 +690,7 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) unset_flag!(metadata, vn, "trans") end - return vi + return metadata end settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) = settrans!(vnv, trans, vn) From 8f9566a7f6690ad3aefc8d90b8434da292853c6f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 19 Jan 2024 23:33:08 +0000 Subject: [PATCH 103/209] fixed return values of `settrans!!` --- src/varinfo.jl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index ffb19b79e..f599c3a2c 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -681,7 +681,8 @@ Return the set of sampler selectors associated with `vn` in `vi`. getgid(vi::VarInfo, vn::VarName) = getmetadata(vi, vn).gids[getidx(vi, vn)] function settrans!!(vi::VarInfo, trans::Bool, vn::VarName) - return settrans!!(getmetadata(vi, vn), trans, vn) + settrans!!(getmetadata(vi, vn), trans, vn) + return vi end function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) if trans @@ -692,7 +693,10 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) return metadata end -settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) = settrans!(vnv, trans, vn) +function settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) + settrans!(vnv, trans, vn) + return vnv +end function settrans!!(vi::VarInfo, trans::Bool) for vn in keys(vi) @@ -855,7 +859,8 @@ end Set `vn`'s value for `flag` to `true` in `vi`. """ function set_flag!(vi::VarInfo, vn::VarName, flag::String) - return set_flag!(getmetadata(vi, vn), vn, flag) + set_flag!(getmetadata(vi, vn), vn, flag) + return vi end function set_flag!(md::Metadata, vn::VarName, flag::String) return md.flags[flag][getidx(md, vn)] = true From 553204663199deb91304b115a5a68e3e595798b6 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 20 Jan 2024 02:23:15 +0000 Subject: [PATCH 104/209] updated static transformation tests --- src/varinfo.jl | 9 +++++++++ test/simple_varinfo.jl | 16 +++++++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index f599c3a2c..3d97b748d 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1668,6 +1668,15 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) end getindex_raw(vi::VarInfo, vn::VarName) = getindex_raw(vi, vn, getdist(vi, vn)) +function getindex_raw(vi::VarInfo, vn::VarName, ::Nothing) + # FIXME: This is too hacky. + # We know this will only be hit if we're working with `VarNameVector`, + # so we can just use the `getindex_raw` for `VarNameVector`. + # NOTE: This won't result in the same behavior as `getindex_raw` + # for the other `VarInfo`s since we don't have access to the `dist` + # and so can't call `reconstruct`. + return getindex_raw(getmetadata(vi, vn), vn) +end function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) return reconstruct(dist, getval(vi, vn)) end diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 869fb82b3..ca1e05c80 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -223,6 +223,7 @@ @testset "Static transformation" begin model = DynamicPPL.TestUtils.demo_static_transformation() + priors = extract_priors(model) varinfos = DynamicPPL.TestUtils.setup_varinfos( model, DynamicPPL.TestUtils.rand_prior_true(model), [@varname(s), @varname(m)] @@ -251,8 +252,11 @@ model, deepcopy(vi_linked), DefaultContext() ) - @test DynamicPPL.getindex_raw(vi_linked, @varname(s)) ≠ retval.s # `s` is unconstrained in original - @test DynamicPPL.getindex_raw(vi_linked_result, @varname(s)) == retval.s # `s` is constrained in result + @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≠ + retval.s # `s` is unconstrained in original + @test DynamicPPL.getindex_raw( + vi_linked_result, @varname(s), priors[@varname(s)] + ) == retval.s # `s` is constrained in result # `m` should not be transformed. @test vi_linked[@varname(m)] == retval.m @@ -263,9 +267,11 @@ model, retval.s, retval.m ) - # Realizations in `vi_linked` should all be equal to the unconstrained realization. - @test DynamicPPL.getindex_raw(vi_linked, @varname(s)) ≈ retval_unconstrained.s - @test DynamicPPL.getindex_raw(vi_linked, @varname(m)) ≈ retval_unconstrained.m + + @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≈ + retval_unconstrained.s + @test DynamicPPL.getindex_raw(vi_linked, @varname(m), priors[@varname(m)]) ≈ + retval_unconstrained.m # The resulting varinfo should hold the correct logp. lp = getlogp(vi_linked_result) From 3c5d2acb1c9cb77e4b47faae8043ad674dd25f8e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 20 Jan 2024 02:25:48 +0000 Subject: [PATCH 105/209] Update test/simple_varinfo.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- test/simple_varinfo.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index ca1e05c80..223d0dd49 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -267,7 +267,6 @@ model, retval.s, retval.m ) - @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≈ retval_unconstrained.s @test DynamicPPL.getindex_raw(vi_linked, @varname(m), priors[@varname(m)]) ≈ From a9be219c08212ab58e90ddfefdf552de5494ae4c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:15:41 +0000 Subject: [PATCH 106/209] removed unnecessary impl of `extract_priors` --- src/extract_priors.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index 1487b9049..da9b176a0 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -116,9 +116,3 @@ function extract_priors(rng::Random.AbstractRNG, model::Model) evaluate!!(model, VarInfo(), context) return context.priors end - -function extract_priors(::Random.AbstractRNG, model::Model, varinfo::AbstractVarInfo) - context = PriorExtractorContext(DefaultContext()) - evaluate!!(model, deepcopy(varinfo), context) - return context.priors -end From 53c8d3345ceec0c293dc077526da9a6f8328f64b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:19:36 +0000 Subject: [PATCH 107/209] make `short_varinfo_name` of `TypedVarInfo` a bit more informative --- test/test_util.jl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_util.jl b/test/test_util.jl index 22bfa46cc..563e13df1 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -84,7 +84,10 @@ Return string representing a short description of `vi`. """ short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" -short_varinfo_name(::TypedVarInfo) = "TypedVarInfo" +function short_varinfo_name(vi::TypedVarInfo) + DynamicPPL.has_varnamevector(vi) && return "TypedVarInfo with VarNameVector" + return "TypedVarInfo" +end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" short_varinfo_name(::DynamicPPL.VectorVarInfo) = "VectorVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" From 61d85ad09ea4c80cdf3d0167e8c3f3fd18d9e596 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:22:59 +0000 Subject: [PATCH 108/209] moved impl of `has_varnamevector` for `ThreadSafeVarInfo` --- src/threadsafe.jl | 2 ++ src/varinfo.jl | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 9b6525097..91b9704c7 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -55,6 +55,8 @@ function setlogp!!(vi::ThreadSafeVarInfoWithRef, logp) return ThreadSafeVarInfo(setlogp!!(vi.varinfo, logp), vi.logps) end +has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) = has_varnamevector(vi.varinfo) + function BangBang.push!!( vi::ThreadSafeVarInfo, vn::VarName, r, dist::Distribution, gidset::Set{Selector} ) diff --git a/src/varinfo.jl b/src/varinfo.jl index 9fa4cdc14..e6c2875d2 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -171,9 +171,6 @@ function has_varnamevector(vi::VarInfo) return vi.metadata isa VarNameVector || (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNameVector), values(vi.metadata))) end -function has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) - return has_varnamevector(vi.varinfo) -end function untyped_varinfo( rng::Random.AbstractRNG, From 9ace554691294f6b5e294dfee116ad37527820b0 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:29:26 +0000 Subject: [PATCH 109/209] added back `extract_priors` impl as we do need it --- src/extract_priors.jl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/extract_priors.jl b/src/extract_priors.jl index da9b176a0..ff77e1a4b 100644 --- a/src/extract_priors.jl +++ b/src/extract_priors.jl @@ -116,3 +116,18 @@ function extract_priors(rng::Random.AbstractRNG, model::Model) evaluate!!(model, VarInfo(), context) return context.priors end + +""" + + extract_priors(model::Model, varinfo::AbstractVarInfo) + +Extract the priors from a model. + +This is done by evaluating the model at the values present in `varinfo` +and recording the distributions that are present at each tilde statement. +""" +function extract_priors(model::Model, varinfo::AbstractVarInfo) + context = PriorExtractorContext(DefaultContext()) + evaluate!!(model, deepcopy(varinfo), context) + return context.priors +end From 67c9821f11b88aa53463536c7e49aa6d1424b58f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 16:34:07 +0000 Subject: [PATCH 110/209] forgot to include tests for `VarNameVector` in `runtests.jl` --- test/runtests.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/runtests.jl b/test/runtests.jl index d15b17b42..ef7672db8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -37,6 +37,7 @@ include("test_util.jl") @testset "interface" begin include("utils.jl") include("compiler.jl") + include("varnamevector.jl") include("varinfo.jl") include("simple_varinfo.jl") include("model.jl") From 32a2d3104c9d93be2a6695347167083f32c27f7e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 25 Jan 2024 17:53:48 +0000 Subject: [PATCH 111/209] fix for `relax_container_types` in `test/varnamevector.jl` --- test/varnamevector.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 210812932..580815fc6 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -100,6 +100,7 @@ function relax_container_types(vnv::VarNameVector, vns, vals) vnv.ranges, vals_new, transforms_new, + vnv.is_transformed, vnv.num_inactive, vnv.metadata, ) From b3bb42d706b6c33df69af8bbff0400c3b8563fbf Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Fri, 26 Jan 2024 14:57:48 +0000 Subject: [PATCH 112/209] fixed `need_transforms_relaxation` --- test/varnamevector.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 580815fc6..2e8e40f46 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -36,15 +36,19 @@ function need_values_relaxation(vnv::VarNameVector, vns, vals) end function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) - if isconcretetype(eltype(vnv.transforms)) + return if isconcretetype(eltype(vnv.transforms)) # If the container is concrete, we need to make sure that the sizes match. # => If the sizes don't match, we need to relax the container type. - return any(keys(vnv)) do vn_present + any(keys(vnv)) do vn_present size(vnv[vn_present]) != size(val) end + elseif eltype(vnv.transforms) !== Any + # If it's not concrete AND it's not `Any`, then we should just make it `Any`. + true + else + # Otherwise, it's `Any`, so we don't need to relax the container type. + false end - - return false end function need_transforms_relaxation(vnv::VarNameVector, vns, vals) return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) From 25ff2b1a3242ba5b2da076600b4bdc7973aa32a7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 10:00:07 +0000 Subject: [PATCH 113/209] updated some tests to not refer directly to `FromVec` --- test/varnamevector.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/varnamevector.jl b/test/varnamevector.jl index 2e8e40f46..6b7acfbeb 100644 --- a/test/varnamevector.jl +++ b/test/varnamevector.jl @@ -161,8 +161,8 @@ end # We'll need the transformations later. # TODO: Should we test other transformations than just `FromVec`? - from_vec_left = DynamicPPL.FromVec(val_left) - from_vec_right = DynamicPPL.FromVec(val_right) + from_vec_left = DynamicPPL.from_vec_transform(val_left) + from_vec_right = DynamicPPL.from_vec_transform(val_right) to_vec_left = inverse(from_vec_left) to_vec_right = inverse(from_vec_right) From 004f038c63d0a251871607cb6306d25ef2570bcd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:19:33 +0000 Subject: [PATCH 114/209] introduce `from_internal_transform` and its siblings --- src/abstract_varinfo.jl | 106 ++++++++++++++++++++++++++++++---------- src/simple_varinfo.jl | 7 +++ src/utils.jl | 57 ++++++++++++++++++--- src/varinfo.jl | 26 +++++----- 4 files changed, 149 insertions(+), 47 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index c37bde795..e08af97e2 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -755,13 +755,13 @@ end # TODO: Clean up all this linking stuff once and for all! """ - with_logabsdet_jacobian_and_reconstruct([f, ]dist, x) + with_logabsdet_jacobian_and_reconstruct(f, dist, x) Like `Bijectors.with_logabsdet_jacobian(f, x)`, but also ensures the resulting value is reconstructed to the correct type and shape according to `dist`. """ function with_logabsdet_jacobian_and_reconstruct(f, dist, x) - x_recon = reconstruct(f, dist, x) + x_recon = reconstruct(dist, x) return with_logabsdet_jacobian(f, x_recon) end @@ -769,7 +769,6 @@ end # just use `first ∘ with_logabsdet_jacobian` to reduce the maintenance burden. # NOTE: `reconstruct` is no-op if `val` is already of correct shape. """ - reconstruct_and_link(dist, val) reconstruct_and_link(vi::AbstractVarInfo, vn::VarName, dist, val) Return linked `val` but reconstruct before linking, if necessary. @@ -780,26 +779,21 @@ by `dist`. See also: [`invlink_and_reconstruct`](@ref), [`reconstruct`](@ref). """ -reconstruct_and_link(f, dist, val) = f(reconstruct(f, dist, val)) -reconstruct_and_link(dist, val) = reconstruct_and_link(link_transform(dist), dist, val) -function reconstruct_and_link(::AbstractVarInfo, ::VarName, dist, val) - return reconstruct_and_link(dist, val) +function reconstruct_and_link(varinfo::AbstractVarInfo, vn::VarName, dist, val) + f = to_linked_internal_transform(varinfo, vn, dist) + return f(val) end """ - invlink_and_reconstruct(dist, val) invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) Return invlinked and reconstructed `val`. See also: [`reconstruct_and_link`](@ref), [`reconstruct`](@ref). """ -invlink_and_reconstruct(f, dist, val) = f(reconstruct(f, dist, val)) -function invlink_and_reconstruct(dist, val) - return invlink_and_reconstruct(invlink_transform(dist), dist, val) -end -function invlink_and_reconstruct(::AbstractVarInfo, ::VarName, dist, val) - return invlink_and_reconstruct(dist, val) +function invlink_and_reconstruct(varinfo::AbstractVarInfo, vn::VarName, dist, val) + f = from_linked_internal_transform(varinfo, vn, dist) + return f(val) end """ @@ -808,11 +802,8 @@ end Return reconstructed `val`, possibly linked if `istrans(vi, vn)` is `true`. """ function maybe_reconstruct_and_link(vi::AbstractVarInfo, vn::VarName, dist, val) - return if istrans(vi, vn) - reconstruct_and_link(vi, vn, dist, val) - else - reconstruct(dist, val) - end + f = to_maybe_linked_internal_transform(vi, vn, dist) + return f(val) end """ @@ -821,28 +812,29 @@ end Return reconstructed `val`, possibly invlinked if `istrans(vi, vn)` is `true`. """ function maybe_invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) - return if istrans(vi, vn) - invlink_and_reconstruct(vi, vn, dist, val) - else - reconstruct(dist, val) - end + f = from_maybe_linked_internal_transform(vi, vn, dist) + return f(val) end """ - invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist[, x]) + invlink_with_logpdf(varinfo::AbstractVarInfo, vn::VarName, dist[, x]) Invlink `x` and compute the logpdf under `dist` including correction from the invlink-transformation. If `x` is not provided, `getval(vi, vn)` will be used. + +!!! warning + The input value `x` should be according to the internal representation of + `varinfo`, e.g. the value returned by `getval(vi, vn)`. """ function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist) return invlink_with_logpdf(vi, vn, dist, getval(vi, vn)) end function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist, y) # NOTE: Will this cause type-instabilities or will union-splitting save us? - f = istrans(vi, vn) ? invlink_transform(dist) : identity - x, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, y) + f = from_maybe_linked_internal_transform(vi, vn, dist) + x, logjac = with_logabsdet_jacobian(f, y) return x, logpdf(dist, x) + logjac end @@ -850,3 +842,63 @@ end # TODO: Remove when possible. increment_num_produce!(::AbstractVarInfo) = nothing setgid!(vi::AbstractVarInfo, gid::Selector, vn::VarName) = nothing + +""" + from_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from the internal representation of `vn` with `dist` +in `varinfo` to a representation compatible with `dist`. +""" +function from_internal_transform end + +""" + from_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from the linked internal representation of `vn` with `dist` +in `varinfo` to a representation compatible with `dist`. +""" +function from_linked_internal_transform end + +""" + from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from the possibly linked internal representation of `vn` with `dist`n +in `varinfo` to a representation compatible with `dist`. +""" +function from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + return if istrans(varinfo, vn) + from_linked_internal_transform(varinfo, vn, dist) + else + from_internal_transform(varinfo, vn, dist) + end +end + +""" + to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from a representation compatible with `dist` to the +internal representation of `vn` with `dist` in `varinfo`. +""" +function to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + return inverse(from_internal_transform(varinfo, vn, dist)) +end + +""" + to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from a representation compatible with `dist` to the +linked internal representation of `vn` with `dist` in `varinfo`. +""" +function to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + return inverse(from_linked_internal_transform(varinfo, vn, dist)) +end + +""" + to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from a representation compatible with `dist` to a +possibly linked internal representation of `vn` with `dist` in `varinfo`. +""" +function to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + return inverse(from_maybe_linked_internal_transform(varinfo, vn, dist)) +end diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 93c211483..5e6b99442 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -298,6 +298,7 @@ function Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribu vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) end + # TODO: Fix this `reconstruct`. return reconstruct(dist, vals_linked, length(vns)) end @@ -689,6 +690,12 @@ function invlink!!( return settrans!!(vi_new, NoTransformation()) end +# With `SimpleVarInfo`, when we're not working with linked variables, there's no need to do anything. +from_internal_transform(::SimpleVarInfo, ::VarName, dist) = identity +function from_linked_internal_transform(vi::SimpleVarInfo, vn::VarName, dist) + return invlink_transform(dist) +end + # Threadsafe stuff. # For `SimpleVarInfo` we don't really need `Ref` so let's not use it. function ThreadSafeVarInfo(vi::SimpleVarInfo) diff --git a/src/utils.jl b/src/utils.jl index f1ad6bb0e..1b5a958f8 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -234,13 +234,58 @@ end Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) -from_vec_transform(x::Real) = FromVec(()) -from_vec_transform(x::AbstractVector) = identity -from_vec_transform(x::AbstractArray) = FromVec(size(x)) +""" + from_vec_transform(x) + +Return the transformation from the vector representation of `x` to original representation. +""" +from_vec_transform(x::Union{Real,AbstractArray}) = from_vec_transform_for_size(size(x)) from_vec_transform(C::Cholesky) = ToChol(C.uplo) ∘ FromVec(size(C.UL)) -# FIXME: drop the `rand` below and instead implement on a case-by-case basis. -from_vec_transform(dist::Distribution) = from_vec_transform(rand(dist)) +from_vec_transform_for_size(sz::Tuple) = FromVec(sz) +from_vec_transform_for_size(::Tuple{()}) = FromVec(()) +from_vec_transform_for_size(::Tuple{<:Any}) = identity + +""" + from_vec_transform(dist::Distribution) + +Return the transformation from the vector representation of a realization from +distribution `dist` to the original representation compatible with `dist`. +""" +from_vec_transform(dist::Distribution) = from_vec_transform_for_size(size(dist)) +from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ FromVec(size(dist)) + +""" + from_linked_vec_transform(dist) + +Return the transformation from the unconstrained vector to the constrained +realization of distribution `dist`. +""" +function from_linked_vec_transform(dist::Distribution) + f_vec = from_vec_transform(dist) + f_invlink = invlink_transform(dist) + return f_invlink ∘ f_vec +end + +function from_linked_vec_transform(dist::LKJCholesky) + return inverse(Bijectors.VecCholeskyBijector(dist.uplo)) +end + +""" + to_vec_transform(x) + +Return the transformation from the original representation of `x` to the vector +representation. +""" +to_vec_transform(x) = inverse(from_vec_transform(x)) + +""" + to_linked_vec_transform(dist) + +Return the transformation from the constrained realization of distribution `dist` +to the unconstrained vector. +""" +to_linked_vec_transform(x) = inverse(from_linked_vec_transform(x)) # FIXME: When given a `LowerTriangular`, `VarInfo` still stores the full matrix # flattened, while using `tovec` below flattenes only the necessary entries. @@ -279,7 +324,7 @@ reconstruct(::Inverse{Bijectors.VecCorrBijector}, ::LKJ, val::AbstractVector) = function reconstruct(dist::LKJCholesky, val::AbstractVector{<:Real}) f = from_vec_transform(dist) - return reconstruct(dist, f(val)) + return f(val) end function reconstruct(dist::LKJCholesky, val::AbstractMatrix{<:Real}) return Cholesky(val, dist.uplo, 0) diff --git a/src/varinfo.jl b/src/varinfo.jl index e6c2875d2..667911525 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2278,21 +2278,19 @@ end values_from_metadata(md::VarNameVector) = pairs(md) -# HACK: Overload of `invlink_with_logpdf` so we can fix it for `VarNameVector`. -function invlink_with_logpdf(vi::VarInfo, vn::VarName, dist, y) - return _invlink_with_logpdf(getmetadata(vi, vn), vn, dist, y) +# Transforming from internal representation to distribution representation. +function from_internal_transform(vi::VarInfo, vn::VarName, dist) + return from_internal_transform(getmetadata(vi, vn), vn, dist) end +from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) +from_internal_transform(::VarNameVector, ::VarName, dist) = from_vec_transform(dist) -function _invlink_with_logpdf(md::Metadata, vn::VarName, dist, y) - # NOTE: Will this cause type-instabilities or will union-splitting save us? - f = istrans(md, vn) ? invlink_transform(dist) : identity - x, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, y) - return x, logpdf(dist, x) + logjac +function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) + return from_linked_internal_transform(getmetadata(vi, vn), vn, dist) end - -function _invlink_with_logpdf(vnv::VarNameVector, vn::VarName, dist, y) - # Here the transformation is stored in `vnv` so we just extract and use this. - f = gettransform(vnv, vn) - x, logjac = with_logabsdet_jacobian(f, y) - return x, logpdf(dist, x) + logjac +function from_linked_internal_transform(::Metadata, ::VarName, dist) + return from_linked_vec_transform(dist) +end +function from_linked_internal_transform(::VarNameVector, ::VarName, dist) + return from_linked_vec_transform(dist) end From 38c89bd268e7d008ccf38c27d2461ed0c924b721 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:28:03 +0000 Subject: [PATCH 115/209] remove `with_logabsdet_jacobian_and_reconstruct` in favour of `with_logabsdet_jacobian` with `from_linked_internal_transform`, etc. --- src/abstract_varinfo.jl | 12 ------------ src/varinfo.jl | 27 +++++++++++++++------------ 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index e08af97e2..604177cec 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -753,18 +753,6 @@ function unflatten(sampler::AbstractSampler, varinfo::AbstractVarInfo, ::Abstrac return unflatten(varinfo, sampler, θ) end -# TODO: Clean up all this linking stuff once and for all! -""" - with_logabsdet_jacobian_and_reconstruct(f, dist, x) - -Like `Bijectors.with_logabsdet_jacobian(f, x)`, but also ensures the resulting -value is reconstructed to the correct type and shape according to `dist`. -""" -function with_logabsdet_jacobian_and_reconstruct(f, dist, x) - x_recon = reconstruct(dist, x) - return with_logabsdet_jacobian(f, x_recon) -end - # TODO: Once `(inv)link` isn't used heavily in `getindex(vi, vn)`, we can # just use `first ∘ with_logabsdet_jacobian` to reduce the maintenance burden. # NOTE: `reconstruct` is no-op if `val` is already of correct shape. diff --git a/src/varinfo.jl b/src/varinfo.jl index 667911525..c356ed270 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1086,7 +1086,7 @@ function _link!(vi::UntypedVarInfo, spl::AbstractSampler) if ~istrans(vi, vns[1]) for vn in vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, link_transform(dist)) + _inner_transform!(vi, vn, dist, to_linked_internal_transform(vi, vn, dist)) settrans!!(vi, true, vn) end else @@ -1114,7 +1114,9 @@ end # Iterate over all `f_vns` and transform for vn in f_vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, link_transform(dist)) + _inner_transform!( + vi, vn, dist, to_linked_internal_transform(vi, vn, dist) + ) settrans!!(vi, true, vn) end else @@ -1185,7 +1187,7 @@ function _invlink!(vi::UntypedVarInfo, spl::AbstractSampler) if istrans(vi, vns[1]) for vn in vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, invlink_transform(dist)) + _inner_transform!(vi, vn, dist, from_linked_internal_transform(vi, vn, dist)) settrans!!(vi, false, vn) end else @@ -1213,7 +1215,9 @@ end # Iterate over all `f_vns` and transform for vn in f_vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, invlink_transform(dist)) + _inner_transform!( + vi, vn, dist, from_linked_internal_transform(vi, vn, dist) + ) settrans!!(vi, false, vn) end else @@ -1232,8 +1236,7 @@ end function _inner_transform!(md::Metadata, vi::VarInfo, vn::VarName, dist, f) # TODO: Use inplace versions to avoid allocations - y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, getval(vi, vn)) - yvec = vectorize(dist, y) + yvec, logjac = with_logabsdet_jacobian(f, getval(vi, vn)) # Determine the new range. start = first(getrange(vi, vn)) # NOTE: `length(yvec)` should never be longer than `getrange(vi, vn)`. @@ -1327,8 +1330,8 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar # Transform to constrained space. x = getval(metadata, vn) dist = getdist(metadata, vn) - f = link_transform(dist) - y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, x) + f = to_linked_internal_transform(varinfo, vn, dist) + y, logjac = with_logabsdet_jacobian(f, x) # Vectorize value. yvec = vectorize(dist, y) # Accumulate the log-abs-det jacobian correction. @@ -1380,14 +1383,14 @@ function _link_metadata!( # Otherwise, we derive the transformation from the distribution. is_transformed[getidx(metadata, vn)] = true - link_transform(getindex(dists, vn)) + to_linked_internal_transform(varinfo, vn, dists[vn]) end # Compute the transformed values. ys = map(vns, link_transforms) do vn, f # TODO: Do we need to handle scenarios where `vn` is not in `dists`? dist = dists[vn] x = getval(metadata, vn) - y, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, x) + y, logjac = with_logabsdet_jacobian(f, x) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) # Return the transformed value. @@ -1487,8 +1490,8 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe # Transform to constrained space. y = getval(varinfo, vn) dist = getdist(varinfo, vn) - f = invlink_transform(dist) - x, logjac = with_logabsdet_jacobian_and_reconstruct(f, dist, y) + f = from_linked_internal_transform(varinfo, vn, dist) + x, logjac = with_logabsdet_jacobian(f, y) # Vectorize value. xvec = vectorize(dist, x) # Accumulate the log-abs-det jacobian correction. From 218dc2355300e6f2e8817146789630dcb9929936 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:40:03 +0000 Subject: [PATCH 116/209] added `internal_to_linked_internal_transform` + fixed a few bugs in the linking as a resultt --- src/abstract_varinfo.jl | 12 ++++++++++++ src/varinfo.jl | 13 +++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 604177cec..0b07ac28e 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -890,3 +890,15 @@ possibly linked internal representation of `vn` with `dist` in `varinfo`. function to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return inverse(from_maybe_linked_internal_transform(varinfo, vn, dist)) end + +""" + internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from the internal representation of `vn` with `dist` +in `varinfo` to a _linked_ internal representation of `vn` with `dist` in `varinfo`. +""" +function internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + f_from_internal = from_internal_transform(varinfo, vn, dist) + f_to_linked_internal = to_linked_internal_transform(varinfo, vn, dist) + return f_to_linked_internal ∘ f_from_internal +end diff --git a/src/varinfo.jl b/src/varinfo.jl index c356ed270..4d1ffb07b 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1086,7 +1086,9 @@ function _link!(vi::UntypedVarInfo, spl::AbstractSampler) if ~istrans(vi, vns[1]) for vn in vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, to_linked_internal_transform(vi, vn, dist)) + _inner_transform!( + vi, vn, dist, internal_to_linked_internal_transform(vi, vn, dist) + ) settrans!!(vi, true, vn) end else @@ -1115,7 +1117,10 @@ end for vn in f_vns dist = getdist(vi, vn) _inner_transform!( - vi, vn, dist, to_linked_internal_transform(vi, vn, dist) + vi, + vn, + dist, + internal_to_linked_internal_transform(vi, vn, dist), ) settrans!!(vi, true, vn) end @@ -1330,7 +1335,7 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar # Transform to constrained space. x = getval(metadata, vn) dist = getdist(metadata, vn) - f = to_linked_internal_transform(varinfo, vn, dist) + f = internal_to_linked_internal_transform(varinfo, vn, dist) y, logjac = with_logabsdet_jacobian(f, x) # Vectorize value. yvec = vectorize(dist, y) @@ -1383,7 +1388,7 @@ function _link_metadata!( # Otherwise, we derive the transformation from the distribution. is_transformed[getidx(metadata, vn)] = true - to_linked_internal_transform(varinfo, vn, dists[vn]) + internal_to_linked_internal_transform(varinfo, vn, dists[vn]) end # Compute the transformed values. ys = map(vns, link_transforms) do vn, f From 1df4293cf99f1b80e34463276d99f5393c74469c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:40:35 +0000 Subject: [PATCH 117/209] added `linked_internal_to_internal_transform` as a complement to `interanl_to_linked_interanl_transform` --- src/abstract_varinfo.jl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 0b07ac28e..428a7fd00 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -902,3 +902,15 @@ function internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::Var f_to_linked_internal = to_linked_internal_transform(varinfo, vn, dist) return f_to_linked_internal ∘ f_from_internal end + +""" + linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + +Return a transformation that transforms from a _linked_ internal representation of `vn` with `dist` +in `varinfo` to the internal representation of `vn` with `dist` in `varinfo`. +""" +function linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + f_from_linked_internal = from_linked_internal_transform(varinfo, vn, dist) + f_to_internal = to_internal_transform(varinfo, vn, dist) + return f_to_internal ∘ f_from_linked_internal +end From f8df89672ddb4c02c939c3385fc9d7ed8f00ee51 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:57:00 +0000 Subject: [PATCH 118/209] fixed bugs in `invlink` for `VarInfo` using `linked_internal_to_internal_transform` --- src/varinfo.jl | 9 +++++++-- test/linking.jl | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 4d1ffb07b..d763c15ca 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1192,7 +1192,9 @@ function _invlink!(vi::UntypedVarInfo, spl::AbstractSampler) if istrans(vi, vns[1]) for vn in vns dist = getdist(vi, vn) - _inner_transform!(vi, vn, dist, from_linked_internal_transform(vi, vn, dist)) + _inner_transform!( + vi, vn, dist, linked_internal_to_internal_transform(vi, vn, dist) + ) settrans!!(vi, false, vn) end else @@ -1221,7 +1223,10 @@ end for vn in f_vns dist = getdist(vi, vn) _inner_transform!( - vi, vn, dist, from_linked_internal_transform(vi, vn, dist) + vi, + vn, + dist, + linked_internal_to_internal_transform(vi, vn, dist), ) settrans!!(vi, false, vn) end diff --git a/test/linking.jl b/test/linking.jl index 9103fff67..c773947c9 100644 --- a/test/linking.jl +++ b/test/linking.jl @@ -44,7 +44,9 @@ function Distributions._logpdf(::MyMatrixDistribution, x::AbstractMatrix{<:Real} end # Skip reconstruction in the inverse-map since it's no longer needed. -DynamicPPL.reconstruct(::TrilFromVec, ::MyMatrixDistribution, x::AbstractVector{<:Real}) = x +function DynamicPPL.from_linked_vec_transform(dist::MyMatrixDistribution) + return TrilFromVec((dist.dim, dist.dim)) +end # Specify the link-transform to use. Bijectors.bijector(dist::MyMatrixDistribution) = TrilToVec((dist.dim, dist.dim)) From d62f26ad545de8387702fefdb6521ed597a37499 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:58:12 +0000 Subject: [PATCH 119/209] more work on removing calls to `reconstruct` --- src/simple_varinfo.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 5e6b99442..6f73e47ea 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -323,12 +323,14 @@ Base.getindex(svi::SimpleVarInfo, ::Colon) = values_as(svi, Vector) # we simply call `getindex` in `getindex_raw`. getindex_raw(vi::SimpleVarInfo, vn::VarName) = vi[vn] function getindex_raw(vi::SimpleVarInfo, vn::VarName, dist::Distribution) - return reconstruct(dist, getindex_raw(vi, vn)) + f = from_internal_transform(vi, vn, dist) + return f(getindex_raw(vi, vn)) end getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}) = vi[vns] function getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribution) # `reconstruct` expects a flattened `Vector` regardless of the type of `dist`, so we `vcat` everything. vals = mapreduce(Base.Fix1(getindex_raw, vi), vcat, vns) + # TODO: Fix this `reconstruct`. return reconstruct(dist, vals, length(vns)) end From b4517d60d3f12c12b0100b8f9e9245b3f24b82bb Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:58:28 +0000 Subject: [PATCH 120/209] removed redundant comment --- src/abstract_varinfo.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 428a7fd00..ce40ae83f 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -820,7 +820,6 @@ function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist) return invlink_with_logpdf(vi, vn, dist, getval(vi, vn)) end function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist, y) - # NOTE: Will this cause type-instabilities or will union-splitting save us? f = from_maybe_linked_internal_transform(vi, vn, dist) x, logjac = with_logabsdet_jacobian(f, y) return x, logpdf(dist, x) + logjac From b7d4754ff938845a2d6911ffa69b4b2a408ab0fd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 14:58:37 +0000 Subject: [PATCH 121/209] added `from_linked_vec_transform` specialization for `LKJ` --- src/utils.jl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/utils.jl b/src/utils.jl index 1b5a958f8..feb612e5b 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -267,9 +267,11 @@ function from_linked_vec_transform(dist::Distribution) return f_invlink ∘ f_vec end +# Specializations that circumvent the `from_vec_transform` machinery. function from_linked_vec_transform(dist::LKJCholesky) return inverse(Bijectors.VecCholeskyBijector(dist.uplo)) end +from_linked_vec_transform(dist::LKJ) = inverse(Bijectors.VecCorrBijector()) """ to_vec_transform(x) From 0244dd9f140b4beaf3f631ff4875746a03817265 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 15:04:57 +0000 Subject: [PATCH 122/209] more work on removing references to `reconstruct` --- src/varinfo.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index d763c15ca..a4bebc481 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1674,6 +1674,7 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) end + # TODO: Replace when we have better dispatch for multiple vals. return reconstruct(dist, vals_linked, length(vns)) end @@ -1688,12 +1689,14 @@ function getindex_raw(vi::VarInfo, vn::VarName, ::Nothing) return getindex_raw(getmetadata(vi, vn), vn) end function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) - return reconstruct(dist, getval(vi, vn)) + f = from_internal_transform(vi, vn, dist) + return f(getval(vi, vn)) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}) return getindex_raw(vi, vns, getdist(vi, first(vns))) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) + # TODO: Replace when we have better dispatch for multiple vals. return reconstruct(dist, getval(vi, vns), length(vns)) end @@ -2284,7 +2287,7 @@ end function values_from_metadata(md::Metadata) return ( - vn => reconstruct(md.dists[md.idcs[vn]], md.vals[md.ranges[md.idcs[vn]]]) for + vn => from_internal_transform(md, vn, getdist(md, vn))(getval(md, vn)) for vn in md.vns ) end From e886d07085a36554eb2b3f92efd944a1ff7ca0bc Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 15:08:10 +0000 Subject: [PATCH 123/209] added `copy` in `values_from_metadata` to preserve behavior and avoid refs to internal representation --- src/varinfo.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index a4bebc481..20c493a3e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2287,7 +2287,8 @@ end function values_from_metadata(md::Metadata) return ( - vn => from_internal_transform(md, vn, getdist(md, vn))(getval(md, vn)) for + # `copy` to avoid accidentaly mutation of internal representation. + vn => copy(from_internal_transform(md, vn, getdist(md, vn))(getval(md, vn))) for vn in md.vns ) end From 2af660526ebf8eb8d1fea3cbd25eb7807dc7014d Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 19:42:36 +0000 Subject: [PATCH 124/209] remove `reconstruct_and_link` and `invlink_and_reconstruct` --- src/abstract_varinfo.jl | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index ce40ae83f..2cdd9ad5f 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -753,37 +753,6 @@ function unflatten(sampler::AbstractSampler, varinfo::AbstractVarInfo, ::Abstrac return unflatten(varinfo, sampler, θ) end -# TODO: Once `(inv)link` isn't used heavily in `getindex(vi, vn)`, we can -# just use `first ∘ with_logabsdet_jacobian` to reduce the maintenance burden. -# NOTE: `reconstruct` is no-op if `val` is already of correct shape. -""" - reconstruct_and_link(vi::AbstractVarInfo, vn::VarName, dist, val) - -Return linked `val` but reconstruct before linking, if necessary. - -Note that unlike [`invlink_and_reconstruct`](@ref), this does not necessarily -return a reconstructed value, i.e. a value of the same type and shape as expected -by `dist`. - -See also: [`invlink_and_reconstruct`](@ref), [`reconstruct`](@ref). -""" -function reconstruct_and_link(varinfo::AbstractVarInfo, vn::VarName, dist, val) - f = to_linked_internal_transform(varinfo, vn, dist) - return f(val) -end - -""" - invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) - -Return invlinked and reconstructed `val`. - -See also: [`reconstruct_and_link`](@ref), [`reconstruct`](@ref). -""" -function invlink_and_reconstruct(varinfo::AbstractVarInfo, vn::VarName, dist, val) - f = from_linked_internal_transform(varinfo, vn, dist) - return f(val) -end - """ maybe_link_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) From a0664d715523cd5d2f13863e2ccffc97bc20725f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 20:23:14 +0000 Subject: [PATCH 125/209] replaced references to `link_and_reconstruct` and `invlink_and_reconstruct` --- src/context_implementations.jl | 38 ++++++++++++++++++++++------------ src/simple_varinfo.jl | 11 ++++++++-- src/varinfo.jl | 11 ++++++++-- 3 files changed, 43 insertions(+), 17 deletions(-) diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 2b28b44a9..8835f40e3 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -468,6 +468,19 @@ function dot_assume(rng, spl::Sampler, ::Any, ::AbstractArray{<:VarName}, ::Any, ) end +# HACK: These methods are only used in the `get_and_set_val!` methods below. +# FIXME: Remove these. +function _link_broadcast_new(vi, vn, dist, r) + b = to_linked_internal_transform(vi, dist) + return b(r) +end + +function _maybe_invlink_broadcast(vi, vn, dist) + xvec = getval(vi, vn) + b = from_maybe_linked_internal_transform(vi, vn, dist) + return b(xvec) +end + function get_and_set_val!( rng, vi::VarInfoOrThreadSafeVarInfo, @@ -483,11 +496,8 @@ function get_and_set_val!( r = init(rng, dist, spl, n) for i in 1:n vn = vns[i] - setindex!!( - vi, - vectorize(dist, maybe_reconstruct_and_link(vi, vn, dist, r[:, i])), - vn, - ) + f_link_maybe = to_maybe_linked_internal_transform(vi, vn, dist) + setindex!!(vi, f_link_maybe(r[:, i]), vn) setorder!(vi, vn, get_num_produce(vi)) end else @@ -498,7 +508,8 @@ function get_and_set_val!( for i in 1:n vn = vns[i] if istrans(vi) - push!!(vi, vn, Bijectors.link(dist, r[:, i]), dist, spl) + ri_linked = _link_broadcast_new(vi, vn, dist, r[:, i]) + push!!(vi, vn, ri_linked, dist, spl) # `push!!` sets the trans-flag to `false` by default. settrans!!(vi, true, vn) else @@ -525,17 +536,18 @@ function get_and_set_val!( for i in eachindex(vns) vn = vns[i] dist = dists isa AbstractArray ? dists[i] : dists - setindex!!( - vi, vectorize(dist, maybe_reconstruct_and_link(vi, vn, dist, r[i])), vn - ) + f_link_maybe = to_maybe_linked_internal_transform(vi, vn, dist) + setindex!!(vi, f_link_maybe(r[i]), vn) setorder!(vi, vn, get_num_produce(vi)) end else # r = reshape(vi[vec(vns)], size(vns)) # FIXME: Remove `reconstruct` in `getindex_raw(::VarInfo, ...)` # and fix the lines below. - r_raw = getindex_raw(vi, vec(vns)) - r = maybe_invlink_and_reconstruct.((vi,), vns, dists, reshape(r_raw, size(vns))) + # r_raw = getindex_raw(vi, vec(vns)) + # r = maybe_invlink_and_reconstruct.((vi,), vns, dists, reshape(r_raw, size(vns))) + rs = _maybe_invlink_broadcast.((vi,), vns, dists) + r = reshape(rs, size(vns)) end else f = (vn, dist) -> init(rng, dist, spl) @@ -546,10 +558,10 @@ function get_and_set_val!( # 2. Define an anonymous function which returns `nothing`, which # we then broadcast. This will allocate a vector of `nothing` though. if istrans(vi) - push!!.((vi,), vns, reconstruct_and_link.((vi,), vns, dists, r), dists, (spl,)) + push!!.((vi,), vns, _link_broadcast_new.((vi,), vns, dists, r), dists, (spl,)) # NOTE: Need to add the correction. # FIXME: This is not great. - acclogp_assume!!(vi, sum(logabsdetjac.(bijector.(dists), r))) + acclogp_assume!!(vi, sum(logabsdetjac.(link_transform.(dists), r))) # `push!!` sets the trans-flag to `false` by default. settrans!!.((vi,), true, vns) else diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 6f73e47ea..3048d6ef7 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -693,10 +693,17 @@ function invlink!!( end # With `SimpleVarInfo`, when we're not working with linked variables, there's no need to do anything. -from_internal_transform(::SimpleVarInfo, ::VarName, dist) = identity -function from_linked_internal_transform(vi::SimpleVarInfo, vn::VarName, dist) +from_internal_transform(::SimpleVarInfo, dist) = identity +function from_internal_transform(vi::SimpleVarInfo, ::VarName, dist) + return from_internal_transform(vi, dist) +end + +function from_linked_internal_transform(vi::SimpleVarInfo, dist) return invlink_transform(dist) end +function from_linked_internal_transform(vi::SimpleVarInfo, ::VarName, dist) + return from_linked_internal_transform(vi, dist) +end # Threadsafe stuff. # For `SimpleVarInfo` we don't really need `Ref` so let's not use it. diff --git a/src/varinfo.jl b/src/varinfo.jl index 20c493a3e..a786d172b 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1876,8 +1876,8 @@ function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) end function Base.push!(vnv::VarNameVector, vn, r, dist, gidset, num_produce) - # FIXME: Include `transform` in the `push!` call below. - return push!(vnv, vn, r) + f = from_vec_transform(dist) + return push!(vnv, vn, r, f) end """ @@ -2296,13 +2296,20 @@ end values_from_metadata(md::VarNameVector) = pairs(md) # Transforming from internal representation to distribution representation. +# Without `vn` argument. +from_internal_transform(vi::VarInfo, dist) = from_vec_transform(dist) +# With `vn` argument. function from_internal_transform(vi::VarInfo, vn::VarName, dist) return from_internal_transform(getmetadata(vi, vn), vn, dist) end from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) from_internal_transform(::VarNameVector, ::VarName, dist) = from_vec_transform(dist) +# Without `vn` argument. +from_linked_internal_transform(vi::VarInfo, dist) = from_linked_vec_transform(dist) +# With `vn` argument. function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) + # Dispatch to metadata in case this alters the behavior. return from_linked_internal_transform(getmetadata(vi, vn), vn, dist) end function from_linked_internal_transform(::Metadata, ::VarName, dist) From f2d59b2d773812358dbc8713e647f80954d52303 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 20:30:40 +0000 Subject: [PATCH 126/209] introduced `recombine` and replaced calls to `reconstruct` with `n` samples --- src/context_implementations.jl | 9 +++++---- src/simple_varinfo.jl | 7 ++----- src/utils.jl | 20 ++++++++++++++++++++ src/varinfo.jl | 4 ++-- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 8835f40e3..47c9f187f 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -207,6 +207,7 @@ function assume(dist::Distribution, vn::VarName, vi) return r, logp, vi end +# TODO: Remove this thing. # SampleFromPrior and SampleFromUniform function assume( rng::Random.AbstractRNG, @@ -220,9 +221,8 @@ function assume( if sampler isa SampleFromUniform || is_flagged(vi, vn, "del") unset_flag!(vi, vn, "del") r = init(rng, dist, sampler) - BangBang.setindex!!( - vi, vectorize(dist, maybe_reconstruct_and_link(vi, vn, dist, r)), vn - ) + f = to_maybe_linked_internal_transform(vi, vn, dist) + BangBang.setindex!!(vi, f(r), vn) setorder!(vi, vn, get_num_produce(vi)) else # Otherwise we just extract it. @@ -231,7 +231,8 @@ function assume( else r = init(rng, dist, sampler) if istrans(vi) - push!!(vi, vn, reconstruct_and_link(dist, r), dist, sampler) + f = to_linked_internal_transform(vi, dist) + push!!(vi, vn, f(r), dist, sampler) # By default `push!!` sets the transformed flag to `false`. settrans!!(vi, true, vn) else diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 3048d6ef7..e7c3e90b0 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -298,8 +298,7 @@ function Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribu vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) end - # TODO: Fix this `reconstruct`. - return reconstruct(dist, vals_linked, length(vns)) + return recombine(dist, vals_linked, length(vns)) end Base.getindex(vi::SimpleVarInfo, vn::VarName) = get(vi.values, vn) @@ -328,10 +327,8 @@ function getindex_raw(vi::SimpleVarInfo, vn::VarName, dist::Distribution) end getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}) = vi[vns] function getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribution) - # `reconstruct` expects a flattened `Vector` regardless of the type of `dist`, so we `vcat` everything. vals = mapreduce(Base.Fix1(getindex_raw, vi), vcat, vns) - # TODO: Fix this `reconstruct`. - return reconstruct(dist, vals, length(vns)) + return recombine(dist, vals, length(vns)) end # HACK: because `VarInfo` isn't ready to implement a proper `getindex_raw`. diff --git a/src/utils.jl b/src/utils.jl index feb612e5b..73c480887 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -380,6 +380,26 @@ function reconstruct!(r, d::MultivariateDistribution, val::AbstractVector, n::In return r end +""" + recombine(dist::Distribution, vals::AbstractVector, n::Int) + +Recombine `vals`, representing a batch of samples from `dist`, so that it's a compatible with `dist`. +""" +function recombine(d::Distribution, val::AbstractVector, n::Int) + return reconstruct(size(d), val, n) +end +function recombine(::Tuple{}, val::AbstractVector, n::Int) + return copy(val) +end +function recombine(s::NTuple{1}, val::AbstractVector, n::Int) + return copy(reshape(val, s[1], n)) +end +function recombine(s::NTuple{2}, val::AbstractVector, n::Int) + tmp = reshape(val, s..., n) + orig = [tmp[:, :, i] for i in 1:n] + return orig +end + # Uniform random numbers with range 4 for robust initializations # Reference: https://mc-stan.org/docs/2_19/reference-manual/initialization.html randrealuni(rng::Random.AbstractRNG) = 4 * rand(rng) - 2 diff --git a/src/varinfo.jl b/src/varinfo.jl index a786d172b..f488bf134 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1675,7 +1675,7 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) getindex(vi, vn, dist) end # TODO: Replace when we have better dispatch for multiple vals. - return reconstruct(dist, vals_linked, length(vns)) + return recombine(dist, vals_linked, length(vns)) end getindex_raw(vi::VarInfo, vn::VarName) = getindex_raw(vi, vn, getdist(vi, vn)) @@ -1697,7 +1697,7 @@ function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) # TODO: Replace when we have better dispatch for multiple vals. - return reconstruct(dist, getval(vi, vns), length(vns)) + return recombine(dist, getval(vi, vns), length(vns)) end """ From e3bfa760efa86e1495d7f0d2d479273ed6b210a5 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 20:33:49 +0000 Subject: [PATCH 127/209] completely removed `reconstruct` --- src/utils.jl | 81 +--------------------------------------------------- 1 file changed, 1 insertion(+), 80 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 73c480887..b88a74639 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -301,92 +301,13 @@ tovec(C::Cholesky) = tovec(Matrix(C.UL)) vectorize(d, r) = vectorize(r) vectorize(r) = tovec(r) -# NOTE: -# We cannot use reconstruct{T} because val is always Vector{Real} then T will be Real. -# However here we would like the result to be specifric type, e.g. Array{Dual{4,Float64}, 2}, -# otherwise we will have error for MatrixDistribution. -# Note this is not the case for MultivariateDistribution so I guess this might be lack of -# support for some types related to matrices (like PDMat). - -""" - reconstruct([f, ]dist, val) - -Reconstruct `val` so that it's compatible with `dist`. - -If `f` is also provided, the reconstruct value will be -such that `f(reconstruct_val)` is compatible with `dist`. -""" -reconstruct(f, dist, val) = reconstruct(dist, val) - -# No-op versions. -reconstruct(::UnivariateDistribution, val::Real) = val -reconstruct(::MultivariateDistribution, val::AbstractVector{<:Real}) = copy(val) -reconstruct(::MatrixDistribution, val::AbstractMatrix{<:Real}) = copy(val) -reconstruct(::Inverse{Bijectors.VecCorrBijector}, ::LKJ, val::AbstractVector) = copy(val) - -function reconstruct(dist::LKJCholesky, val::AbstractVector{<:Real}) - f = from_vec_transform(dist) - return f(val) -end -function reconstruct(dist::LKJCholesky, val::AbstractMatrix{<:Real}) - return Cholesky(val, dist.uplo, 0) -end -reconstruct(::LKJCholesky, val::Cholesky) = val - -function reconstruct( - ::Inverse{Bijectors.VecCholeskyBijector}, ::LKJCholesky, val::AbstractVector -) - return copy(val) -end - -function reconstruct( - ::Inverse{Bijectors.PDVecBijector}, ::MatrixDistribution, val::AbstractVector -) - return copy(val) -end - -# TODO: Implement no-op `reconstruct` for general array variates. - -reconstruct(d::Distribution, val::AbstractVector) = reconstruct(size(d), val) -reconstruct(::Tuple{}, val::AbstractVector) = val[1] -reconstruct(s::NTuple{1}, val::AbstractVector) = copy(val) -reconstruct(s::Tuple, val::AbstractVector) = reshape(copy(val), s) -function reconstruct!(r, d::Distribution, val::AbstractVector) - return reconstruct!(r, d, val) -end -function reconstruct!(r, d::MultivariateDistribution, val::AbstractVector) - r .= val - return r -end -function reconstruct(d::Distribution, val::AbstractVector, n::Int) - return reconstruct(size(d), val, n) -end -function reconstruct(::Tuple{}, val::AbstractVector, n::Int) - return copy(val) -end -function reconstruct(s::NTuple{1}, val::AbstractVector, n::Int) - return copy(reshape(val, s[1], n)) -end -function reconstruct(s::NTuple{2}, val::AbstractVector, n::Int) - tmp = reshape(val, s..., n) - orig = [tmp[:, :, i] for i in 1:n] - return orig -end -function reconstruct!(r, d::Distribution, val::AbstractVector, n::Int) - return reconstruct!(r, d, val, n) -end -function reconstruct!(r, d::MultivariateDistribution, val::AbstractVector, n::Int) - r .= val - return r -end - """ recombine(dist::Distribution, vals::AbstractVector, n::Int) Recombine `vals`, representing a batch of samples from `dist`, so that it's a compatible with `dist`. """ function recombine(d::Distribution, val::AbstractVector, n::Int) - return reconstruct(size(d), val, n) + return recombine(size(d), val, n) end function recombine(::Tuple{}, val::AbstractVector, n::Int) return copy(val) From c0aef8179a253ef985a14946d0317bb75fe6af58 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sun, 28 Jan 2024 20:37:35 +0000 Subject: [PATCH 128/209] renamed `maybe_reconstruct_and_link` to `to_maybe_linked_internal` and `maybe_invlink_and_reconstruct` to `from_maybe_linked_internal` --- src/abstract_varinfo.jl | 8 ++++---- src/context_implementations.jl | 5 ----- src/simple_varinfo.jl | 10 +++++----- src/varinfo.jl | 2 +- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 2cdd9ad5f..32d6cc31b 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -754,21 +754,21 @@ function unflatten(sampler::AbstractSampler, varinfo::AbstractVarInfo, ::Abstrac end """ - maybe_link_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) + to_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) Return reconstructed `val`, possibly linked if `istrans(vi, vn)` is `true`. """ -function maybe_reconstruct_and_link(vi::AbstractVarInfo, vn::VarName, dist, val) +function to_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) f = to_maybe_linked_internal_transform(vi, vn, dist) return f(val) end """ - maybe_invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) + from_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) Return reconstructed `val`, possibly invlinked if `istrans(vi, vn)` is `true`. """ -function maybe_invlink_and_reconstruct(vi::AbstractVarInfo, vn::VarName, dist, val) +function from_maybe_linked_internal(vi::AbstractVarInfo, vn::VarName, dist, val) f = from_maybe_linked_internal_transform(vi, vn, dist) return f(val) end diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 47c9f187f..1ec241a5d 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -542,11 +542,6 @@ function get_and_set_val!( setorder!(vi, vn, get_num_produce(vi)) end else - # r = reshape(vi[vec(vns)], size(vns)) - # FIXME: Remove `reconstruct` in `getindex_raw(::VarInfo, ...)` - # and fix the lines below. - # r_raw = getindex_raw(vi, vec(vns)) - # r = maybe_invlink_and_reconstruct.((vi,), vns, dists, reshape(r_raw, size(vns))) rs = _maybe_invlink_broadcast.((vi,), vns, dists) r = reshape(rs, size(vns)) end diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index e7c3e90b0..255d8c3b7 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -292,7 +292,7 @@ end # `NamedTuple` function Base.getindex(vi::SimpleVarInfo, vn::VarName, dist::Distribution) - return maybe_invlink_and_reconstruct(vi, vn, dist, getindex(vi, vn)) + return from_maybe_linked_internal(vi, vn, dist, getindex(vi, vn)) end function Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribution) vals_linked = mapreduce(vcat, vns) do vn @@ -476,7 +476,7 @@ function assume( ) value = init(rng, dist, sampler) # Transform if we're working in unconstrained space. - value_raw = maybe_reconstruct_and_link(vi, vn, dist, value) + value_raw = to_maybe_linked_internal(vi, vn, dist, value) vi = BangBang.push!!(vi, vn, value_raw, dist, sampler) return value, Bijectors.logpdf_with_trans(dist, value, istrans(vi, vn)), vi end @@ -494,9 +494,9 @@ function dot_assume( # Transform if we're working in transformed space. value_raw = if dists isa Distribution - maybe_reconstruct_and_link.((vi,), vns, (dists,), value) + to_maybe_linked_internal.((vi,), vns, (dists,), value) else - maybe_reconstruct_and_link.((vi,), vns, dists, value) + to_maybe_linked_internal.((vi,), vns, dists, value) end # Update `vi` @@ -523,7 +523,7 @@ function dot_assume( # Update `vi`. for (vn, val) in zip(vns, eachcol(value)) - val_linked = maybe_reconstruct_and_link(vi, vn, dist, val) + val_linked = to_maybe_linked_internal(vi, vn, dist, val) vi = BangBang.setindex!!(vi, val_linked, vn) end diff --git a/src/varinfo.jl b/src/varinfo.jl index f488bf134..14b67a37e 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1650,7 +1650,7 @@ getindex(vi::VarInfo, vn::VarName) = getindex(vi, vn, getdist(vi, vn)) function getindex(vi::VarInfo, vn::VarName, dist::Distribution) @assert haskey(vi, vn) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" val = getval(vi, vn) - return maybe_invlink_and_reconstruct(vi, vn, dist, val) + return from_maybe_linked_internal(vi, vn, dist, val) end # HACK: Allows us to also work with `VarNameVector` where `dist` is not used, # but we instead use a transformation stored with the variable. From f7c0853756c9a1bb82b02d5c50dbdf4e7f7175cc Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 30 Jan 2024 10:56:58 +0000 Subject: [PATCH 129/209] added impls of `from_*_internal_transform` for `ThreadSafeVarInfo` --- src/threadsafe.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 91b9704c7..160e88c11 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -244,3 +244,13 @@ end function invlink_with_logpdf(vi::ThreadSafeVarInfo, vn::VarName, dist, y) return invlink_with_logpdf(vi.varinfo, vn, dist, y) end + +function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) + return from_linked_internal_transform(varinfo.varinfo, vn, dist) +end + +function from_linked_internal_transform( + varinfo::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}, dist +) + return from_linked_internal_transform(varinfo.varinfo, vns, dist) +end From 77b835e3fcaedaa52c01866f40b04b517ec51c55 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 30 Jan 2024 11:33:59 +0000 Subject: [PATCH 130/209] removed `reconstruct` from docs and from exports --- docs/src/api.md | 1 - src/DynamicPPL.jl | 4 ---- 2 files changed, 5 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index f9db6603c..397b178ac 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -298,7 +298,6 @@ DynamicPPL.link!! DynamicPPL.invlink!! DynamicPPL.default_transformation DynamicPPL.maybe_invlink_before_eval!! -DynamicPPL.reconstruct ``` #### Utils diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index 7de80015e..ec0b086f7 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -79,10 +79,6 @@ export AbstractVarInfo, # Compiler @model, # Utilities - vectorize, - reconstruct, - reconstruct!, - Sample, init, vectorize, OrderedDict, From b83c26203e2b8291769dd1ab821986896a69d06b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 17:40:09 +0000 Subject: [PATCH 131/209] renamed `getval` to `getindex_internal` and made `dist` an optional argument for all the transform-related methods --- src/abstract_varinfo.jl | 63 +++++++++++++++++++---- src/context_implementations.jl | 2 +- src/simple_varinfo.jl | 15 ++---- src/threadsafe.jl | 2 +- src/utils.jl | 6 ++- src/varinfo.jl | 94 +++++++++++++++++++++------------- src/varnamevector.jl | 2 +- test/varinfo.jl | 2 +- 8 files changed, 124 insertions(+), 62 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 32d6cc31b..c048014b5 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -779,14 +779,14 @@ end Invlink `x` and compute the logpdf under `dist` including correction from the invlink-transformation. -If `x` is not provided, `getval(vi, vn)` will be used. +If `x` is not provided, `getindex_internal(vi, vn)` will be used. !!! warning The input value `x` should be according to the internal representation of - `varinfo`, e.g. the value returned by `getval(vi, vn)`. + `varinfo`, e.g. the value returned by `getindex_internal(vi, vn)`. """ function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist) - return invlink_with_logpdf(vi, vn, dist, getval(vi, vn)) + return invlink_with_logpdf(vi, vn, dist, getindex_internal(vi, vn)) end function invlink_with_logpdf(vi::AbstractVarInfo, vn::VarName, dist, y) f = from_maybe_linked_internal_transform(vi, vn, dist) @@ -800,26 +800,32 @@ increment_num_produce!(::AbstractVarInfo) = nothing setgid!(vi::AbstractVarInfo, gid::Selector, vn::VarName) = nothing """ - from_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + from_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from the internal representation of `vn` with `dist` in `varinfo` to a representation compatible with `dist`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function from_internal_transform end """ - from_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + from_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from the linked internal representation of `vn` with `dist` in `varinfo` to a representation compatible with `dist`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function from_linked_internal_transform end """ - from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from the possibly linked internal representation of `vn` with `dist`n in `varinfo` to a representation compatible with `dist`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return if istrans(varinfo, vn) @@ -828,57 +834,94 @@ function from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarN from_internal_transform(varinfo, vn, dist) end end +function from_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + return if istrans(varinfo, vn) + from_linked_internal_transform(varinfo, vn) + else + from_internal_transform(varinfo, vn) + end +end """ - to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + to_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from a representation compatible with `dist` to the internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return inverse(from_internal_transform(varinfo, vn, dist)) end +function to_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + return inverse(from_internal_transform(varinfo, vn)) +end """ - to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from a representation compatible with `dist` to the linked internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return inverse(from_linked_internal_transform(varinfo, vn, dist)) end +function to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + return inverse(from_linked_internal_transform(varinfo, vn)) +end """ - to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from a representation compatible with `dist` to a possibly linked internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) return inverse(from_maybe_linked_internal_transform(varinfo, vn, dist)) end +function to_maybe_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + return inverse(from_maybe_linked_internal_transform(varinfo, vn)) +end """ internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) Return a transformation that transforms from the internal representation of `vn` with `dist` in `varinfo` to a _linked_ internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) f_from_internal = from_internal_transform(varinfo, vn, dist) f_to_linked_internal = to_linked_internal_transform(varinfo, vn, dist) return f_to_linked_internal ∘ f_from_internal end +function internal_to_linked_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + f_from_internal = from_internal_transform(varinfo, vn) + f_to_linked_internal = to_linked_internal_transform(varinfo, vn) + return f_to_linked_internal ∘ f_from_internal +end """ - linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) + linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName[, dist]) Return a transformation that transforms from a _linked_ internal representation of `vn` with `dist` in `varinfo` to the internal representation of `vn` with `dist` in `varinfo`. + +If `dist` is not present, then it is assumed that `varinfo` knows the correct output for `vn`. """ function linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName, dist) f_from_linked_internal = from_linked_internal_transform(varinfo, vn, dist) f_to_internal = to_internal_transform(varinfo, vn, dist) return f_to_internal ∘ f_from_linked_internal end + +function linked_internal_to_internal_transform(varinfo::AbstractVarInfo, vn::VarName) + f_from_linked_internal = from_linked_internal_transform(varinfo, vn) + f_to_internal = to_internal_transform(varinfo, vn) + return f_to_internal ∘ f_from_linked_internal +end diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 1ec241a5d..6cf8bc42d 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -477,7 +477,7 @@ function _link_broadcast_new(vi, vn, dist, r) end function _maybe_invlink_broadcast(vi, vn, dist) - xvec = getval(vi, vn) + xvec = getindex_internal(vi, vn) b = from_maybe_linked_internal_transform(vi, vn, dist) return b(xvec) end diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 255d8c3b7..dc2780aae 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -332,7 +332,7 @@ function getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribut end # HACK: because `VarInfo` isn't ready to implement a proper `getindex_raw`. -getval(vi::SimpleVarInfo, vn::VarName) = getindex_raw(vi, vn) +getindex_internal(vi::SimpleVarInfo, vn::VarName) = getindex_raw(vi, vn) Base.haskey(vi::SimpleVarInfo, vn::VarName) = hasvalue(vi.values, vn) @@ -690,16 +690,11 @@ function invlink!!( end # With `SimpleVarInfo`, when we're not working with linked variables, there's no need to do anything. -from_internal_transform(::SimpleVarInfo, dist) = identity -function from_internal_transform(vi::SimpleVarInfo, ::VarName, dist) - return from_internal_transform(vi, dist) -end - -function from_linked_internal_transform(vi::SimpleVarInfo, dist) - return invlink_transform(dist) -end +from_internal_transform(vi::SimpleVarInfo, ::VarName) = identity +from_internal_transform(vi::SimpleVarInfo, ::VarName, dist) = identity +from_linked_internal_transform(vi::SimpleVarInfo, ::VarName) = identity function from_linked_internal_transform(vi::SimpleVarInfo, ::VarName, dist) - return from_linked_internal_transform(vi, dist) + return invlink_transform(dist) end # Threadsafe stuff. diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 160e88c11..648a8ef1c 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -222,7 +222,7 @@ end istrans(vi::ThreadSafeVarInfo, vn::VarName) = istrans(vi.varinfo, vn) istrans(vi::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}) = istrans(vi.varinfo, vns) -getval(vi::ThreadSafeVarInfo, vn::VarName) = getval(vi.varinfo, vn) +getindex_internal(vi::ThreadSafeVarInfo, vn::VarName) = getindex_internal(vi.varinfo, vn) function unflatten(vi::ThreadSafeVarInfo, x::AbstractVector) return Setfield.@set vi.varinfo = unflatten(vi.varinfo, x) diff --git a/src/utils.jl b/src/utils.jl index b88a74639..f75290af2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -256,10 +256,14 @@ from_vec_transform(dist::Distribution) = from_vec_transform_for_size(size(dist)) from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ FromVec(size(dist)) """ - from_linked_vec_transform(dist) + from_linked_vec_transform(dist::Distribution) Return the transformation from the unconstrained vector to the constrained realization of distribution `dist`. + +By default, this is just `invlink_transform(dist) ∘ from_vec_transform(dist)`. + +See also: [`DynamicPPL.invlink_transform`](@ref), [`DynamicPPL.from_vec_transform`](@ref). """ function from_linked_vec_transform(dist::Distribution) f_vec = from_vec_transform(dist) diff --git a/src/varinfo.jl b/src/varinfo.jl index 14b67a37e..a12a25cc3 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -449,8 +449,8 @@ function merge_metadata(metadata_left::Metadata, metadata_right::Metadata) push!(vns, vn) if vn in vns_left && vn in vns_right # `vals`: only valid if they're the length. - vals_left = getval(metadata_left, vn) - vals_right = getval(metadata_right, vn) + vals_left = getindex_internal(metadata_left, vn) + vals_right = getindex_internal(metadata_right, vn) @assert length(vals_left) == length(vals_right) append!(vals, vals_right) # `ranges` @@ -471,7 +471,7 @@ function merge_metadata(metadata_left::Metadata, metadata_right::Metadata) elseif vn in vns_left # Just extract the metadata from `metadata_left`. # `vals` - vals_left = getval(metadata_left, vn) + vals_left = getindex_internal(metadata_left, vn) append!(vals, vals_left) # `ranges` r = (offset + 1):(offset + length(vals_left)) @@ -489,7 +489,7 @@ function merge_metadata(metadata_left::Metadata, metadata_right::Metadata) else # Just extract the metadata from `metadata_right`. # `vals` - vals_right = getval(metadata_right, vn) + vals_right = getindex_internal(metadata_right, vn) append!(vals, vals_right) # `ranges` r = (offset + 1):(offset + length(vals_right)) @@ -513,11 +513,11 @@ end const VarView = Union{Int,UnitRange,Vector{Int}} """ - getval(vi::UntypedVarInfo, vview::Union{Int, UnitRange, Vector{Int}}) + getindex_internal(vi::UntypedVarInfo, vview::Union{Int, UnitRange, Vector{Int}}) Return a view `vi.vals[vview]`. """ -getval(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, vview) +getindex_internal(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, vview) """ setval!(vi::UntypedVarInfo, val, vview::Union{Int, UnitRange, Vector{Int}}) @@ -584,16 +584,16 @@ getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] getdist(::VarNameVector, ::VarName) = nothing """ - getval(vi::VarInfo, vn::VarName) + getindex_internal(vi::VarInfo, vn::VarName) Return the value(s) of `vn`. The values may or may not be transformed to Euclidean space. """ -getval(vi::VarInfo, vn::VarName) = getval(getmetadata(vi, vn), vn) -getval(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) +getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) # HACK: We shouldn't need this -getval(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) +getindex_internal(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) """ setval!(vi::VarInfo, val, vn::VarName) @@ -614,13 +614,14 @@ function setval!(vnv::VarNameVector, val, vn::VarName) end """ - getval(vi::VarInfo, vns::Vector{<:VarName}) + getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) Return the value(s) of `vns`. The values may or may not be transformed to Euclidean space. """ -getval(vi::VarInfo, vns::Vector{<:VarName}) = mapreduce(Base.Fix1(getval, vi), vcat, vns) +getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) = + mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) """ getall(vi::VarInfo) @@ -634,7 +635,9 @@ getall(vi::VarInfo) = getall(vi.metadata) # See for example https://github.com/JuliaLang/julia/pull/46381. getall(vi::TypedVarInfo) = reduce(vcat, map(getall, vi.metadata)) function getall(md::Metadata) - return mapreduce(Base.Fix1(getval, md), vcat, md.vns; init=similar(md.vals, 0)) + return mapreduce( + Base.Fix1(getindex_internal, md), vcat, md.vns; init=similar(md.vals, 0) + ) end getall(vnv::VarNameVector) = vnv.vals @@ -1246,7 +1249,7 @@ end function _inner_transform!(md::Metadata, vi::VarInfo, vn::VarName, dist, f) # TODO: Use inplace versions to avoid allocations - yvec, logjac = with_logabsdet_jacobian(f, getval(vi, vn)) + yvec, logjac = with_logabsdet_jacobian(f, getindex_internal(vi, vn)) # Determine the new range. start = first(getrange(vi, vn)) # NOTE: `length(yvec)` should never be longer than `getrange(vi, vn)`. @@ -1338,7 +1341,7 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar end # Transform to constrained space. - x = getval(metadata, vn) + x = getindex_internal(metadata, vn) dist = getdist(metadata, vn) f = internal_to_linked_internal_transform(varinfo, vn, dist) y, logjac = with_logabsdet_jacobian(f, x) @@ -1399,7 +1402,7 @@ function _link_metadata!( ys = map(vns, link_transforms) do vn, f # TODO: Do we need to handle scenarios where `vn` is not in `dists`? dist = dists[vn] - x = getval(metadata, vn) + x = getindex_internal(metadata, vn) y, logjac = with_logabsdet_jacobian(f, x) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) @@ -1498,7 +1501,7 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe end # Transform to constrained space. - y = getval(varinfo, vn) + y = getindex_internal(varinfo, vn) dist = getdist(varinfo, vn) f = from_linked_internal_transform(varinfo, vn, dist) x, logjac = with_logabsdet_jacobian(f, y) @@ -1547,7 +1550,7 @@ function _invlink_metadata!( # Compute the transformed values. xs = map(vns) do vn f = gettransform(metadata, vn) - y = getval(metadata, vn) + y = getindex_internal(metadata, vn) # No need to use `with_reconstruct` as `f` will include this. x, logjac = with_logabsdet_jacobian(f, y) # Accumulate the log-abs-det jacobian correction. @@ -1649,7 +1652,7 @@ end getindex(vi::VarInfo, vn::VarName) = getindex(vi, vn, getdist(vi, vn)) function getindex(vi::VarInfo, vn::VarName, dist::Distribution) @assert haskey(vi, vn) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" - val = getval(vi, vn) + val = getindex_internal(vi, vn) return from_maybe_linked_internal(vi, vn, dist, val) end # HACK: Allows us to also work with `VarNameVector` where `dist` is not used, @@ -1662,12 +1665,10 @@ function getindex(vi::VarInfo, vn::VarName, ::Nothing) end function getindex(vi::VarInfo, vns::Vector{<:VarName}) - # FIXME(torfjelde): Using `getdist(vi, first(vns))` won't be correct in cases - # such as `x .~ [Normal(), Exponential()]`. - # BUT we also can't fix this here because this will lead to "incorrect" - # behavior if `vns` arose from something like `x .~ MvNormal(zeros(2), I)`, - # where by "incorrect" we mean there exists pieces of code expecting this behavior. - return getindex(vi, vns, getdist(vi, first(vns))) + vals_linked = mapreduce(vcat, vns) do vn + getindex(vi, vn) + end + return recombine(vi, vals_linked, length(vns)) end function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) @assert haskey(vi, vns[1]) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" @@ -1690,14 +1691,14 @@ function getindex_raw(vi::VarInfo, vn::VarName, ::Nothing) end function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) f = from_internal_transform(vi, vn, dist) - return f(getval(vi, vn)) + return f(getindex_internal(vi, vn)) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}) return getindex_raw(vi, vns, getdist(vi, first(vns))) end function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) # TODO: Replace when we have better dispatch for multiple vals. - return recombine(dist, getval(vi, vns), length(vns)) + return recombine(dist, getindex_internal(vi, vns), length(vns)) end """ @@ -1707,7 +1708,7 @@ Return the current value(s) of the random variables sampled by `spl` in `vi`. The value(s) may or may not be transformed to Euclidean space. """ -getindex(vi::VarInfo, spl::Sampler) = copy(getval(vi, _getranges(vi, spl))) +getindex(vi::VarInfo, spl::Sampler) = copy(getindex_internal(vi, _getranges(vi, spl))) function getindex(vi::TypedVarInfo, spl::Sampler) # Gets the ranges as a NamedTuple ranges = _getranges(vi, spl) @@ -2288,26 +2289,45 @@ end function values_from_metadata(md::Metadata) return ( # `copy` to avoid accidentaly mutation of internal representation. - vn => copy(from_internal_transform(md, vn, getdist(md, vn))(getval(md, vn))) for - vn in md.vns + vn => copy( + from_internal_transform(md, vn, getdist(md, vn))(getindex_internal(md, vn)) + ) for vn in md.vns ) end values_from_metadata(md::VarNameVector) = pairs(md) # Transforming from internal representation to distribution representation. -# Without `vn` argument. -from_internal_transform(vi::VarInfo, dist) = from_vec_transform(dist) -# With `vn` argument. +# Without `dist` argument: base on `dist` extracted from self. +function from_internal_transform(vi::VarInfo, vn::VarName) + return from_internal_transform(getmetadata(vi, vn), vn) +end +function from_internal_transform(md::Metadata, vn::VarName) + return from_internal_transform(md, vn, getdist(md, vn)) +end +function from_internal_transform(md::VarNameVector, vn::VarName) + return gettransform(md, vn) +end +# With both `vn` and `dist` arguments: base on provided `dist`. function from_internal_transform(vi::VarInfo, vn::VarName, dist) return from_internal_transform(getmetadata(vi, vn), vn, dist) end from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) -from_internal_transform(::VarNameVector, ::VarName, dist) = from_vec_transform(dist) +function from_internal_transform(::VarNameVector, ::VarName, dist) + return from_vec_transform(dist) +end -# Without `vn` argument. -from_linked_internal_transform(vi::VarInfo, dist) = from_linked_vec_transform(dist) -# With `vn` argument. +# Without `dist` argument: base on `dist` extracted from self. +function from_linked_internal_transform(vi::VarInfo, vn::VarName) + return from_linked_internal_transform(getmetadata(vi, vn), vn) +end +function from_linked_internal_transform(md::Metadata, vn::VarName) + return from_linked_internal_transform(md, vn, getdist(md, vn)) +end +function from_linked_internal_transform(md::VarNameVector, vn::VarName) + return gettransform(md, vn) +end +# With both `vn` and `dist` arguments: base on provided `dist`. function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) # Dispatch to metadata in case this alters the behavior. return from_linked_internal_transform(getmetadata(vi, vn), vn, dist) diff --git a/src/varnamevector.jl b/src/varnamevector.jl index 7bbb8bda8..ed919a20c 100644 --- a/src/varnamevector.jl +++ b/src/varnamevector.jl @@ -393,7 +393,7 @@ function subset(vnv::VarNameVector, vns::AbstractVector{<:VarName}) isempty(vnv) && return vnv_new for vn in vns - push!(vnv_new, vn, getval(vnv, vn), gettransform(vnv, vn)) + push!(vnv_new, vn, getindex_internal(vnv, vn), gettransform(vnv, vn)) end return vnv_new diff --git a/test/varinfo.jl b/test/varinfo.jl index ddd1b7ea8..570f9623e 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -644,7 +644,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) # Should only get the variables subsumed by `@varname(s)`. @test varinfo[spl] == - mapreduce(Base.Fix1(DynamicPPL.getval, varinfo), vcat, vns_s) + mapreduce(Base.Fix1(DynamicPPL.getindex_internal, varinfo), vcat, vns_s) # `link` varinfo_linked = DynamicPPL.link(varinfo, spl, model) From c4faf3e47e033161f2bec9abbdd4e62784fa1e11 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 17:41:01 +0000 Subject: [PATCH 132/209] updated docs + added description of how internals of transforms work --- docs/make.jl | 2 +- docs/src/api.md | 3 + docs/src/internals/transformations.md | 377 ++++++++++++++++++++++++++ docs/src/internals/varinfo.md | 2 + 4 files changed, 383 insertions(+), 1 deletion(-) create mode 100644 docs/src/internals/transformations.md diff --git a/docs/make.jl b/docs/make.jl index c38853d5b..de557ef9f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,7 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], - "Internals" => ["internals/varinfo.md"], + "Internals" => ["internals/varinfo.md", "internals/transformations.md"], ], checkdocs=:exports, doctest=false, diff --git a/docs/src/api.md b/docs/src/api.md index 397b178ac..e07d762da 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -270,6 +270,7 @@ resetlogp!! keys getindex DynamicPPL.getindex_raw +DynamicPPL.getindex_internal push!! empty!! isempty @@ -297,6 +298,8 @@ DynamicPPL.invlink DynamicPPL.link!! DynamicPPL.invlink!! DynamicPPL.default_transformation +DynamicPPL.link_transform +DynamicPPL.invlink_transform DynamicPPL.maybe_invlink_before_eval!! ``` diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md new file mode 100644 index 000000000..7ee800556 --- /dev/null +++ b/docs/src/internals/transformations.md @@ -0,0 +1,377 @@ +# Transforming variables + +## Motivation + +In a probabilistic programming language (PPL) such as DynamicPPL.jl, one crucial functionality for enabling a large number of inference algorithms to be implemented, in particular gradient-based ones, is the ability to work with "unconstrained" variables. + +For example, consider the following model: + +```julia +@model function demo() + s ~ InverseGamma(2, 3) + m ~ Normal(0, √s) +end +``` + +Here we have two variables `s` and `m`, where `s` is constrained to be positive, while `m` can be any real number. + +For certain inference methods, it's necessary / much more convenient to work with an equivalent model to `demo` but where all the variables can take any real values (they're "unconstrained"). + +!!! note + We write "unconstrained" with quotes because there are many ways to transform a constrained variable to an unconstrained one, *and* DynamicPPL can work with a much broader class of bijective transformations of variables, not just ones that go to the entire real line. But for MCMC, unconstraining is the most common transformation so we'll stick with that terminology. + +For a large family of constraints encoucntered in practice, it is indeed possible to transform a (partially) contrained model to a completely unconstrained one in such a way that sampling in the unconstrained space is equivalent to sampling in the constrained space. + +In DynamicPPL.jl, this is often referred to as *linking* (a term originating in the statistics literature) and is done using transformations from [Bijectors.jl](https://github.com/TuringLang/Bijectors.jl). + +For example, the above model could be transformed into (the following psuedo-code; it's not working code): + +```julia +@model function demo() + log_s ~ log(InverseGamma(2, 3)) + s = exp(log_s) + m ~ Normal(0, √s) +end +``` + +Here `log_s` is an unconstrained variable, and `s` is a constrained variable that is a deterministic function of `log_s`. + +But to ensure that we stay consistent with what the user expects, DynamicPPL.jl does not actually transform the model as above, but can instead makes use of transformed variables internally to achieve the same effect, when desired. + +In the end, we'll end up with something that looks like this: + +```@raw html +
+ +
+``` + +Below we'll see how this is done. + +## What do we need? + +There are two aspects to transforming from the internal representation of a variable in a `varinfo` to the representation wanted in the model: +1. Different implementations of [`AbstractVarInfo`](@ref) represent realizations of a model in different ways internally, so we need to transform from this internal representation to the desired representation in the model. For example, + - [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. + - [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). +2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. + + + +## Working example + +A good and non-trivial example to keep in mind throughout is the following model: + +```@example transformations-internal +using DynamicPPL, Distributions +@model demo_lkj() = x ~ LKJCholesky(2, 1.0) +``` + +`LKJCholesky` is a `LKJ(2, 1.0)` distribution, a distribution over correlation matrices (covariance matrices but with unit diagonal), but working directly with the Cholesky factorization of the correlation matrix rather than the correlation matrix itself (this is more numerically stable and computationally efficient). + +!!! note + This is a particularly "annoying" case because the return-value is not a simple `Real` or `AbstractArray{<:Real}`, but rather a `LineraAlgebra.Cholesky` object which wraps a triangular matrix (whether it's upper- or lower-triangular depends on the instance). + +As mentioned, some implementations of `AbstractVarInfo`, e.g. [`VarInfo`](@ref), works with a "flattened" / vector representation of a variable, and so in this case we need two transformations: +1. From the `Cholesky` object to a vector representation. +2. From the `Cholesky` object to an "unconstrained" / linked vector representation. + +And similarly, we'll need the inverses of these transformations. + +## From internal representation to model representation + +To go from the internal variable representation of an `AbstractVarInfo` to the variable representation wanted in the model, e.g. from a `Vector{Float64}` to `Cholesky` in the case of [`VarInfo`](@ref) in `demo_lkj`, we have the following methods: + +```@docs +DynamicPPL.to_internal_transform +DynamicPPL.from_internal_transform +``` + +These methods allows us to extract the internal-to-model transformation function depending on the `varinfo`, the variable, and the distribution of the variable: +- `varinfo` + `vn` defines the internal representation of the variable. +- `dist` defines the representation expected within the model scope. + +!!! note + If `vn` is not present in `varinfo`, then the internal representation is fully determined by `varinfo` alone. This is used when we're about to add a new variable to the `varinfo` and need to know how to represent it internally. + +Continuing from the example above, we can inspect the internal representation of `x` in `demo_lkj` with [`VarInfo`](@ref) using [`DynamicPPL.getindex_internal`](@ref): + +```@example transformations-internal +model = demo_lkj() +varinfo = VarInfo(model) +x_internal = DynamicPPL.getindex_internal(varinfo, @varname(x)) +``` + +```@example transformations-internal +f_from_internal = DynamicPPL.from_internal_transform( + varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) +f_from_internal(x_internal) +``` + +Let's confirm that this is the same as `varinfo[@varname(x)]`: + +```@example transformations-internal +x_model = varinfo[@varname(x)] +``` + +Similarly, we can go from the model representation to the internal representation: + +```@example transformations-internal +f_to_internal = DynamicPPL.to_internal_transform( + varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) + +f_to_internal(x_model) +``` + +It's also useful to see how this is done in [`SimpleVarInfo`](@ref): + +```@example transformations-internal +simple_varinfo = SimpleVarInfo(varinfo) +DynamicPPL.getindex_internal(simple_varinfo, @varname(x)) +``` + +Here see that the internal representation is exactly the same as the model representation, and so we'd expect `from_internal_transform` to be the `identity` function: + +```@example transformations-internal +DynamicPPL.from_internal_transform( + simple_varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) +``` + +Great! + +## From *unconstrained* internal representation to model representation + +In addition to going from internal representation to model representation of a variable, we also need to be able to go from the *unconstrained* internal representation to the model representation. + +For this, we have the following methods: + +```@docs +DynamicPPL.to_linked_internal_transform +DynamicPPL.from_linked_internal_transform +``` + +These are very similar to [`DynamicPPL.to_internal_transform`](@ref) and [`DynamicPPL.from_internal_transform`](@ref), but here the internal representation is also linked / "unconstrained". + +Continuing from the example above: + +```@example transformations-internal +f_to_linked_internal = DynamicPPL.to_linked_internal_transform( + varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) + +x_linked_internal = f_to_linked_internal(x_model) +``` + +```@example transformations-internal +f_from_linked_internal = DynamicPPL.from_linked_internal_transform( + varinfo, + @varname(x), + LKJCholesky(2, 1.0) +) + +f_from_linked_internal(x_linked_internal) +``` + +Here we see a significant difference between the linked representation and the non-linked representation: the linked representation is only of length 1, whereas the non-linked representation is of length 4. This is because we actually only need a single element to represent a 2x2 correlation matrix, as the diagonal elements are always 1 *and* it's symmetric. + +We can also inspect the transforms themselves: + +```@example transformations-internal +f_from_internal +``` + +vs. + +```@example transformations-internal +f_from_linked_internal +``` + +Here we see that `f_from_linked_internal` is a single function taking us directly from the linked representation to the model representation, whereas `f_from_internal` is a composition of a few functions: one reshaping the underlying length 4 array into 2x2 matrix, and the other converting this matrix into a `Cholesky`, as required to be compatible with `LKJCholesky(2, 1.0)`. + +## Why do we need both `to_internal_transform` and `to_linked_internal_transform`? + +One might wonder why we need both `to_internal_transform` and `to_linked_internal_transform` instead of just a single `to_internal_transform` which returns the "standard" internal representation if the variable is not linked / "unconstrained" and the linked / "unconstrained" internal representation if it is. + +That is, why can't we just do + +```@raw html +
+ +
+``` + +Unfortunately, this is not possible in general. Consider for example the following model: + +```@example transformations-internal +@model function demo_dynamic_constraint() + m ~ Normal() + x ~ truncated(Normal(), lower=m) + + return (m=m, x=x) +end +``` + +Here the variable `x` has is constrained to be on the domain `(m, Inf)`, where `m` is sampled according to a `Normal`. + +```@example transformations-internal +model = demo_dynamic_constraint() +varinfo = VarInfo(model) +varinfo[@varname(m)], varinfo[@varname(x)] +``` + +We see that the realization of `x` is indeed greater than `m`, as expected. + +But what if we [`link`](@ref) this `varinfo` so that we end up working on an "unconstrained" space, i.e. both `m` and `x` can take on any values in `(-Inf, Inf)`: + +```@example transformations-internal +varinfo_linked = link(varinfo, model) +varinfo_linked[@varname(m)], varinfo_linked[@varname(x)] +``` + +Still get the same values, as expected, since internally `varinfo` transforms from the linked internal representation to the model representation. + +But what if we change the value of `m`, to, say, a bit larger than `x`? + +```@example transformations-internal +# Update realization for `m` in `varinfo_linked`. +varinfo_linked[@varname(m)] = varinfo_linked[@varname(x)] + 1 +varinfo_linked[@varname(m)], varinfo_linked[@varname(x)] +``` + +Now we see that the constraint `m < x` is no longer satisfied! + +Hence one might expect that if we try to compute, say, the [`logjoint`](@ref) using `varinfo_linked` with this "invalid" realization, we'll get an error: + +```@example transformations-internal +logjoint(model, varinfo_linked) +``` + +But we don't! In fact, if we look at the actual value used within the model + +```@example transformations-internal +first(DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext())) +``` + +we see that we indeed satisfy the constraint `m < x`, as desired. + +!!! warning + One shouldn't be setting variables in a linked `varinfo` willy-nilly directly like this unless one knows that the value will be compatible with the constraints of the model. + +The reason for this is that internally in a model evaluation, we construct the transformation from the internal to the model representation based on the *current* realizations in the model! That is, we take the `dist` in a `x ~ dist` expression _at model evaluation time_ and use that to construct the transformation, thus allowing it to change between model evaluations without invalidating the transformation. + +But to be able to do this, we need to know whether the variable is linked / "unconstrained" or not, since the transformation is different in the two cases. Hence we need to be able to determine this at model evaluation time. Hence the the internals end up looking something like this: + +```julia +if istrans(varinfo, varname) + from_linked_internal_transform(varinfo, varname, dist) +else + from_internal_transform(varinfo, varname, dist) +end +``` + +That is, if the variable is linked / "unconstrained", we use the [`DynamicPPL.from_linked_internal_transform`](@ref), otherwise we use [`DynamicPPL.from_internal_transform`](@ref). + +And so the earlier diagram becomes: + +```@raw html +
+ +
+``` + +!!! note + If the support of `dist` was constant, this would not be necessary since we could just determine the transformation at the time of `varinfo_linked = link(varinfo, model)` and define this as the `from_internal_transform` for all subsequent evaluations. However, since the support of `dist` is *not* constant in general, we need to be able to determine the transformation at the time of the evaluation *and* thus whether we should construct the transformation from the linked internal representation or the non-linked internal representation. This is annoying, but necessary. + +This is also the reason why we have two definitions of `getindex`: +- [`getindex(::AbstractVarInfo, ::VarName, ::Distribution)`](@ref): used internally in model evaluations with the `dist` in a `x ~ dist` expression. +- [`getindex(::AbstractVarInfo, ::VarName)`](@ref): used externally by the user to get the realization of a variable. + +For `getindex` we have the following diagram: + +```@raw html +
+ +
+``` + +While if `dist` is not provided, we have: + +```@raw html +
+ +
+``` + +Notice that `dist` is not present here, but otherwise the diagrams are the same. + +!!! warning + This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occcurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. + +## Other functionalities + +There are also some additional methods for transforming between representations that are all automatically implemented from [`DynamicPPL.from_internal_transform`](@ref), [`DynamicPPL.from_linked_internal_transform`](@ref) and their siblings, and thus don't need to be implemented manually. + +Convenience methods for constructing transformations: + +```@docs +DynamicPPL.from_maybe_linked_internal_transform +DynamicPPL.to_maybe_linked_internal_transform +DynamicPPL.internal_to_linked_internal_transform +DynamicPPL.linked_internal_to_internal_transform +``` + +Convenience methods for transforming between representations without having to explicitly construct the transformation: + +```@docs +DynamicPPL.to_maybe_linked_internal +DynamicPPL.from_maybe_linked_internal +``` + +# Supporting a new distribution + +To support a new distribution, one needs to implement for the desired `AbstractVarInfo` the following methods: +- [`DynamicPPL.from_internal_transform`](@ref) +- [`DynamicPPL.from_linked_internal_transform`](@ref) + +At the time of writing, [`VarInfo`](@ref) is the one that is most commonly used, whose internal representation is always a `Vector`. In this scenario, one can just implemente the following methods instead: + +```@docs +DynamicPPL.from_vec_transform(::Distribution) +DynamicPPL.from_linked_vec_transform(::Distribution) +``` + +These are used internally by [`VarInfo`](@ref). + +Optionally, if `inverse` of the above is expensive to compute, one can also implement: +- [`DynamicPPL.to_internal_transform`](@ref) +- [`DynamicPPL.to_linked_internal_transform`](@ref) + +And similarly, there are corresponding to-methods for the `from_*_vec_transform` variants too + +```@docs +DynamicPPL.to_vec_transform +DynamicPPL.to_linked_vec_transform +``` + +!!! warning + Whatever the resulting transformation is, it should be invertible, i.e. implement `InverseFunctions.inverse`, and have a well-defined log-abs-det Jacobian, i.e. implement `ChangesOfVariables.with_logabsdet_jacobian`. + +# TL;DR + +- DynamicPPL.jl has three representations of a variable: the **model representation**, the **internal representation**, and the **linked internal representation**. + - The **model representation** is the representation of the variable as it appears in the model code / is expected by the `dist` on the right-hand-side of the `~` in the model code. + - The **internal representation** is the representation of the variable as it appears in the `varinfo`, which varies between implementations of [`AbstractVarInfo`](@ref), e.g. a `Vector` in [`VarInfo`](@ref). This can be converted to the model representation by [`DynamicPPL.from_internal_transform`](@ref). + - The **linked internal representation** is the representation of the variable as it appears in the `varinfo` after [`link`](@ref)ing. This can be converted to the model representation by [`DynamicPPL.from_linked_internal_transform`](@ref). +- Having separation between *internal* and *linked internal* is necessary because transformations might be constructed at the time of model evaluation, and thus we need to know whether to construct the transformation from the internal representation or the linked internal representation. + diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 1312f152c..8d74fd2fa 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -49,6 +49,8 @@ In addition, we want to be able to access the transformed / "unconstrained" real - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + - `getindex_internal` and `setindex_internal!` for extracting and mutating the internal representaton of a particular `VarName`. + Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: 1. Type-stable when possible, but functional when not. From c8d9695f1c460126b51f720c26c6f26decc10e49 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 17:44:42 +0000 Subject: [PATCH 133/209] added a bunch of illustrations for the transforms docs + dot files used to generated --- ...transformations-assume-without-istrans.dot | 17 +++ ...sformations-assume-without-istrans.dot.png | Bin 0 -> 25856 bytes ...sformations-assume-without-istrans.dot.svg | 88 +++++++++++++ .../assets/images/transformations-assume.dot | 22 ++++ .../images/transformations-assume.dot.png | Bin 0 -> 44454 bytes .../transformations-getindex-with-dist.dot | 20 +++ ...transformations-getindex-with-dist.dot.png | Bin 0 -> 42305 bytes .../transformations-getindex-without-dist.dot | 20 +++ ...nsformations-getindex-without-dist.dot.png | Bin 0 -> 40798 bytes docs/src/assets/images/transformations.dot | 28 ++++ .../src/assets/images/transformations.dot.png | Bin 0 -> 56255 bytes .../src/assets/images/transformations.dot.svg | 124 ++++++++++++++++++ docs/src/internals/transformations.md | 2 +- 13 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 docs/src/assets/images/transformations-assume-without-istrans.dot create mode 100644 docs/src/assets/images/transformations-assume-without-istrans.dot.png create mode 100644 docs/src/assets/images/transformations-assume-without-istrans.dot.svg create mode 100644 docs/src/assets/images/transformations-assume.dot create mode 100644 docs/src/assets/images/transformations-assume.dot.png create mode 100644 docs/src/assets/images/transformations-getindex-with-dist.dot create mode 100644 docs/src/assets/images/transformations-getindex-with-dist.dot.png create mode 100644 docs/src/assets/images/transformations-getindex-without-dist.dot create mode 100644 docs/src/assets/images/transformations-getindex-without-dist.dot.png create mode 100644 docs/src/assets/images/transformations.dot create mode 100644 docs/src/assets/images/transformations.dot.png create mode 100644 docs/src/assets/images/transformations.dot.svg diff --git a/docs/src/assets/images/transformations-assume-without-istrans.dot b/docs/src/assets/images/transformations-assume-without-istrans.dot new file mode 100644 index 000000000..6eb6865c3 --- /dev/null +++ b/docs/src/assets/images/transformations-assume-without-istrans.dot @@ -0,0 +1,17 @@ +digraph { + # `assume` block + subgraph cluster_assume { + label = "assume"; + fontname = "Courier"; + + assume [shape=box, label=< assume(varinfo, @varname(x), Normal())>, fontname="Courier"]; + without_linking_assume [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_logabsdetjac [shape=box, label="x, logjac = with_logabsdet_jacobian(f, assume_internal(varinfo, varname, dist))", fontname="Courier"]; + return_assume [shape=box, label=< return x, logpdf(dist, x) - logjac, varinfo >, style=dashed, fontname="Courier"]; + + assume -> without_linking_assume; + without_linking_assume -> with_logabsdetjac; + with_logabsdetjac -> return_assume; + } +} + diff --git a/docs/src/assets/images/transformations-assume-without-istrans.dot.png b/docs/src/assets/images/transformations-assume-without-istrans.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..f58727ad268029520317959c23ff7e687387940b GIT binary patch literal 25856 zcmce;Wn5g%n)cff65JBp(!t#V1P|^IJh;2N(-17UL$Kg(!QBZi!KHEc#+~MEo;|bo zGjryd+3z`fPV-?ktA5m~wN}+#RsZX{3H~fAhJuKX2mk<3B*cXk0Du=i0f1*Y@Xujq zf?jJg!w#?XrNo2*k54}tE%~tkz&n71uz-?l>cOJRm$x%BuaCK_yv&|w!GA-k&=q0E z?wY+w{Y&qW*`T?(fzI!PFF&&%a~|8$j4X==Fm!vwf(j;1!(b78@WW z@6yuLzBo_ac>kareWN$a>9DnsROMn~j3-I`4AuewI_>oQA721p-r(#1afavlYZ9SD$}{Ly}G7 zQWna5`cr_CP9lR=o@^S8r`rVnnqt0e8lP9A!}bV;DBM%AE<(OsI{Ss3w8b|BU>dhe zL~tkGeK+QG|*Eh#$o(F+6` z#z8k64mGM&2YYUJD9QB)?P)w6NIUDsH+_bs#_2OS<*d$*^eSU_L530NhqV~alx>Ja7X# z?7(gFgFEWWKJ*g)wV*#;dt{!-XzGvX%*>77%_o^bEU4+S%;@&|C+9|_&$n+R5nMpM z*FU;I4L?+WBF5Z~j!luE^0-?~^e=zaV!n9>(0z#-0{}F?z2FKM)plBu95)EpE+3W* zj!avUh_1e=(~3lFlv_+Xthg$WdMrmJ&#`c9FhkgOdHmQQN6L0~4(E!qB39!h29X|a zXyO~1LI;u2JKC7bjt8!mX?D)lJ4)`X&!R1<+4-3<+n+316gF^aIS>4zXZ4tDtYSD# ziB+$ndz+yg0|H=SV zu$;DeLv@eCXmumopv}EuDszeVe!lwHdYU1_<`b(}{{*=k&Pwm>{2Y@%leq>#U`17x z5xm|cyLn*VhFKx-q^DG~sO@82i#Y?(UqStxWIZlbzkQ4t&-I!T~Y*-qcw^%p6{ilZgL$fsuGn|=>yg}>o&1A+eI zn2i0K+xrp{$b*=I@Pn29OmvFJq=D!!dNItIaH%pUKl`kIj2Q1soY-9N;Q|1%pGG&X z2rDM$c;R9Oo{os)g?&YIg@4q2u~-B@6}R3{!NUCOz!tWPiQJTk2O`#yt^2GFoE*MsAL8Hr$mwMzCjFsuH^p5Mt;%*oi3|oZ;y}6V&z`0Ez(rKOsVvSTnW@jjVy6B z*E(K>Bw=`Yw(QLh7_l?RaJ6u4ZVNRTS4Sgg`4qC53Jl#JM;z56IvsBLLdwCMZILYq zXhx*|PH5V8dmG7WtGw!soK&0U+pqW*17+2#twv3r*;nN}gAJHppUpcb1;oYSfn*dE zc|B`?ME8P5n)BlC|El=?|9JxV&lSj+9|8d8k_rmr{jrp#L~ww1q%4b>GGb!l>4@hU zq8Y$IoM~7|S7&F*ABFE}XlOosn4FgfOu~QYKK_~=r_TKn;ImEK#WaojEaTPcKRXls z=Z5^>Ol9bqltC11${+CCI0hy6XL=&ae$lu;Mhz+7kk`jOoC)6ZSWuJEb@i!_@Z@tC zcz$M6E-StdO}{PL3$zr^WJ8}?XY*4=?&?bh*;2Ia1kIPO)|M=|EMWCj&EU zdew9=FGOHGSw>9f@%*a&@z?po&gd1pW~2w*9z{036S>XZQi*Qm!#X4C_N1_(Kf2&K z$4u$R?^*_oHuCZL$sDl>;%x0_@F>haeZPmamgzWc_#G95g^I&LBEOS+Q!m?_j+DdS zZO@MQC0XwGd&JnI#3?LJF>*-`;{(wcb`N&#kVbl`c`2GF9$u4b6!oqmV$Db zmrA%47i()V%L=>^<7lqh8>|Udew=K_ogV4f>PXOW=J6C=t`hg+Z_3J!!^dphr`T4T zHZ5{vRa-B7<;n+JexTzdV^(=usHdrtJ|`+6lAIbV5sN$xwz^y5T~Jo}_>H&0j+y0F zFLc!lyLG-^NGn&J!mePQ%ib|jcXyElm3Lf?FT>i%ql$Bn3W$qcRn6IPdhNdV*iD*Lqy`peiPQv_vt+K-qA_oecMD^MrbEvn7{iHd zregFRNV!mkhp#4BX^;0osjzenswCUwCuTBR9Jg#T1*6#_p3#Ue8JY+RNT8ie+0`983n@e*>VGfm`nc1H97#hKXd;@5sW zS;*UO$i2dG@jRS9dgQ+M$Xi{Cr)6ymKP0)iJ0>^LhYsfZ<6=HWe8=Q9R~6k#OVOc& z8njmrmwP&!b~Kb=Hg)aLoJ*t&iv|f(TB==a`=IGlLSzRAn(JIwBlx1ZdmewpXvZ-o zgVu*~Be9PN-{MU;++M^PVjA6W!XKfj+pUyOaRtQMV-?H|?Kt0Fdnf?IKRF`tW5gzVxnRp85i z;N6EI*jL)c-%PwWd%s{pbj>HmkkNMG@a^%Me*&?bZhDFD>ds{yHMQR3xcwl)T%f#N z%;oMf9nJD}m}8X*eCGUM5F;J8J;a#Z&bfG|3RSAACJA%&1b6v!e#e z*X!dbwJ5hWQLO&xUZ*i6P%fp($2|vVLxSy@=)iD=7r%KiU2J}d#df~%V~XWF|EL51 zNAFoh%Hv1;IauGy+^yS&I_^uOlKV#m6@uAQtBVP)RlE}5HAH#>2A8)tw-xQZZ59qv zZg8N3>+m;seZT05w3bhG;im{A`95qbCDi^b_z;Zn_;9T_4>Gv*oK;bCXU~mk)F*t- zw|s4+d1baP=3@P@y^&zwNp9`wu^{K7ac?tKrJ)JFT4=?%@zLZz zKEx&;eXdjF4Mab>TwRa$s00VSgr00>ls8GqYW6XG9>B;m>wP8jeSvgaG1Pu^QI01{ z+M>?tcjiy5*vZRtLhjSe8V5f#dSu@7;V7randX2GSI0?lB`p0Ll`e;%m9aYTKtnmv zEyG9S%H_{;a}x_=Iida4V(oe5f~a1cD^=F03Jm-0H(~iz-p$1sTup(?-h7t(rgD@F zXZp?$0qJO9_W6lmV;!kU1;NU4&D%>QY{j~c4VTyV%>3tlbwVmFOO7Y}^kv4aOab=|Q8dQMUZ7 za;hIg$^L)X9?6L{^4(Y?_STlD)-8DGNrWCpr)|kRdUg5Uc)f;=&~qFP5v^5xnTVY- z%o1{8IRLt%cx$Ihl_HNDnfzN)j`8w8odkqnLupXm^I=jKxcw&g;ggyPBbo za%ZE5bLpbyaG*2rqdPaNf7ET-@)u~=W_C;BkLxqj>Bn&NV@%KboJm53-b_BJZwzc( zpWUqmC0DLCL=<-A2yuYI(^+8as zwM_s%`(e{%|47&dC0D60HksnBaIAntAqSEs!~VqC*)s-7L3rQ=KF5#A@92Vvd{-s} zZ0Q&~>OnyN*|vqY-3ZoghFrcebQ6swWpqJ!1Mu&f@NnTht^5kI6P0XtD@$?(#zfJW zoL)6`Nivi5meK;c_Zlm<3-5vY#jDYUd@CMFVKp}`48bo~QJecC+PfT} z?x>GVti1<8HkxPJgCP|6 zs3~3S;Sk&x_eUdH56ioyhIJKJ*et~j+l0C9tl_+JO7|wqUU-?Ut@=@h;v%Ke_H?X5 zr)6R9GlbaNO5K&i0_QhJMv}pp#8aFJD@{WYg3qq4(Zl8D)(AuwwaPX&mHd%_k7=_o zkb4{I050Cb6r{=(d1RLDrw&|;E*lu*a+!3IuBQtZU507^7J5%9NFe2)*v-hqD8p z@2ynoJDj%eH-O2+C%d2s1hqq!Nar&z&Lb4 zvkp)g4}EM*3`#|G<9mPf9-!`0U}U6Aat%a#;_A{{=TbfRy=44X^yI`8lM#znshFDx zkCyeXbF}Pa{_J$nLTV#LGb;k3ObQf|bytOl51z0^#nbEJa5aJdYZSog{AEUUE4?|^ zR(JQheQR%$)#AOMXR_msF6iXVk85mQ+`Nn_s?N>rL_RqdIw^P{-5_S?Pw9?GTtQ?) zv4RfW41WcoXA$ckC6RPw6Ln5S=>ycs^$J9@3s*k$xxW-Uuk4?#(NpCjpI1z&T`&fR z4xx5ABLz+`-e-8yuzDSAu@g zNs6OSqE#SF$WUg219vc4jy=ACVU6rKbn}e3&z*L0>r`UYVxf|86Y;zIVN+PJ!F@Q~ z;!$ov&AhxJq_s_)46V;>S8*0#yEl9-S zZ_Y#cm(B62ce0n4cJOxTF@FAyfu!A`oy+waLYGY#&EQ5C#d%v&3T8nbrL8UWopU<~ zvU{s)k(T zawaOE!^wtT2!}V9Ls2ld6EMARalc8$!~HvYXCI*zSs-OHouoN8m-`$(6w0W$JRlF_ zfVcaPM)=>fR=?IBJ*aE)cpZ)eTXJmI=e#bNVA5&oybRi+_8yi@4g~w-Q zLb})O^PC{zK=1maS^h#!E-&rdwCgwE#8F&6(~*i8*6vE|Rx70iKZ_aK=I9_hck=fI z&LbTt`_rY%KVYmKp4$DmEsEz$db2^>5(Alm#s)zs>O)FW`Je>UVyYXVZDDIt>+Xt* zcQl!RP5Kv87>#3r5sbeC42{kV z10R>riWwjht})T0c#bz$5Plx#`_t6)a!ncsy`YN5HB62cqKfNu(KjmeuzC~s`F3OQ z9-ODl+FR>jb$O^u&R$aqauC!ez+(yAyed9c{QWb#GMVS~&zhFz)bB&Q z{12YMmQD-mFtk-co}{-w0X{n!z*ql<&6uex6kp)a4(1fi55LGyJN3w)yvLp*OV{Ri zv4p4`p&}M8w0h?=w`mQly5KSoiP6|Ta~eHXT@=5~jP!7yABbbH5+<28o7a@C;<$l&xNeCzbwg)Z&o0Y7&&}HA7Z#WE zFU=LCfOQwHW((!2rXl|R_B$E91oUj zS(nK7E0Jbm=Zl_FTG(aQ(?5g!fJdWJK~=d#4?LP7DI|qD+pjLfmx8Tc>x71(ey_Fa zJ`FwjA>*!myAe*q)I~!$Uz>6nV|6)fkxoB{^?^69x7w;oa5DzugkrFgyeeN^;f>4@>MhtWhb_zS^dR(NgHwy@ejE1JR2K6 zY+l-u;>6H!%{+U{6@qhUNSDW!BhA?dwZ+`3L`UIq*Mk%<=~U9#dN$&%Qg9syFq+g~ zzIu1=W?Rk=%?qd5$UO0c^aa}R;qS}>>V$@uwN;twTCGLe{OJlD0&^cM>U5<;t&eEp z{pDxp+s^G11As@*kMKYE;;D2LxFbW(41@>yv@dIvF_Scp_SCsE#O@e->a0~SyKY1F zLyQTp-`!`gS9+h#j2zQ@ehJQ5{jQ@D=FryqaqUnhx0U+~$Y>_LE-9f)5UjLttm2gz zPkw=SmIo>fH3>+NnL;4pqj?;yElvGBF(@H_QiJCy^qoKU9trRbgYDN(y{yxs1X&bb&{)M%xNa})`Y7dvTnSM~{7>x( zS#yU7IScJgxiBl9mbUA6w(hj{&da-xca9dL)Xcsl51iYMwcokd%_rlJPAeH}qdOc- zE(;C@ZuTIRI1-yM(n$s?de$ABDM$j5fCs`p{!!n*f5#7f-ENtcoehJ)(nYoaKmmFe zlf1lqW@hFz>PsITtXt)wf3;5ePt#!qi`Rh9Rj|gD5@=!WKjuwUC* zVOY8Or=ZZ&=ZS7dex>;(h6ezs_*?vMNv!{L)1QQy{}qwE^O zp-XjbA}NUo%yHVEhI@jwc0;Cw7MyPl_SGou1{__F|Ydj=(xuQB^n}D zGRaY;Z5|Cg^{YmmcK^1syt7ZDNR z>+360H!VRH1AVyThHa=gXkdbo;L~-8FfcG+Me>rU6x=Yu@o!!L()aI62+#mNrcdJg z&j_GUAl2KDuaAj?n*ss?)8VfHpQ@h(ye~oOrNms01=19;{f0t8s5KQ870$u{!21B0 zM&E8O8it!WvFpgS_zi=!ga`@X<0?@~1mJ;j_NN}(jx?gnR4`>OX{MrrElT)FrVD5J zlhJQ%3^Vl=;`N(7xK z>L@Qpp-@*-XcIlX`R}h~)b3>aQir!OStAsQ@b#G@Qw_M$ zj4MkiArb>@pCz4C+jgk143O{N3Rn<3%)fv%w)yzU?cL2`5#>%s9cytk)c95Sn`CX6 zZk5?2q}GveQDmO;EbjZJ#>8GTKE{r2DS8nksgI0@f5V%!OT!fPv6^K` z_we&dfUgjmQYfEwLL_X>G5xUUcXNJ{+%fQVUyR-jA>$|Rf1^empUo|RYKIZ7uF5R( zm1ZZc4>%m>_mCoyK)H+TowD<$_mP90>K{2>FMoK@J!l`?X)RbD^4#YiHZnQ}3N|^U zvfiEF&Y!Bz?EahUqL=r6|IQ8?o;GQ|oySXA)>cM+&2|NO4ub1q3bVRaK^g0gu%q)N zFWmMBauO-;%nY06iaN6cmRoDNc+Ww~cE{c4B4B zLeWO|u2tpd(%%Te>!tQUGcnNff8u@slOEuSl#hbEii5+b)kT?CWPYmkX_hN{?(v$Os7+IDi)L@Nq^{*_hTayvI_4x`O792d+>Q zj3A!?mK(p#uH{*gv}yMV+Lo@EeENdgBo?r(cbn@nO!ffVj75p4yE|OBKSiV_h$`?k5Ol%Ac`2abbb^l?+#9vRV4p>Z9e?M@8aDG{v=5W(9%;jx zWV5@kX{e3d@h&SX>sv4{8c$o%?{fV>+#-?r)ic+p5m;Z6pK3=)q~WSiu|e)b(XA`~ zB&V(DN#Vyy!nPB`@zHDXAG5}ipg74y@$nI#z7cZ_#YGj zthm=daN&W%^5u%x2gd;F*>Id0(HSw=XSX?0$}7k@6!ovu zdsAo4qyC}7sve)nDcS)oV#iA3_(g|D)v1zdaS=>-rKyz8ay@;gi`BE=5x1%%vH+qr zVQeU=0GUY+Ef>1YQwCGEq188@ITj{qw7N6#RZ8PSg_nkUrHVo$n=F3}zi3BhRtkat z4)iWMxEaC^>&=)W)*D4PtD+XG8iY!EtZ2C;I1$#K=$&}VEv-s~lZOo#@3SK2o22$0 zjw=i-J{FdLEj^nirq9(pMQg#Tyi9*6<)D@xj*v%{CU-CktDLE9k^_z8-cPD_Q29+8 z#%Y!v6lRGVI~KekwRqg!jA5m|rA@yX$l)X3dK=_230dx zr(G7`{+dV)P|VbRRTl9W!wCu-Q%Q95bL=zOl9zAA^tKbrs*mxHD_~xTX7@od-pnYF?isqB2F6lL!kCyG&T+2N*?3)ymu&AV_|82GP&@U*8~u>QWw&dRN$VDO(f zZv1d77BksWeZ15K&UUTpFuDI{$xx_4YK$VXe;n=s*0wMIE3Ny^jH8K2!?={>iTx5= zpbohPkWM_iFyU54?c>ePY`M0zQ-KRw8#{e-N)6Hu!S5mY_e?CUAw|k-0W^y^oszOC zDJB_V3WZcUK+Ei?XlNB>kM%+dC)KR)#$yfh?JG0r>Nia#QIAH_9?$1#zP7ZR(C3X9RoAqu(g{I@o@RyntC?|B6Q#Xavt}YwUT1MylE; z<=??1-QgMN!=L-g&lgh?1@2?SdI(e@QD-YKZs4^2jo)ARy?&1C<*b#W(+WeemU`GZ0{m+fJ=JEcS_~Ch9e}9Qw`KHZ| z*TjAxnJ?^NK9K;nY{DR&$F3F4JCHGe>ng@qmXYo^oKmBBiikaX9XYEyM#wfv_(L zMz}>BxC5yIIX{+Ulu8z(YS0E3G)1TrYoN*^g#ol9FG!N3qoQOFUkzW=WIgTu!^r(N zo;a((P2Eex+|=LA2dOk|Y6X_8Qbo43((o~_v+XTNa4)APi$;SOLIF=+8Y;&rSs9iYgnLOl|#OQ@G>z;UfnwV?)6d2 zh-y#%v812~JQ^GD-TD}H7UOa-0}e2>4D~b^%LK8WIdS%bN|QgXCW2ENEgp$S=T$)x zsTSNUjSd>qX>wC>Tv}W&1JO#-&cqCNg7za|nr8OIcJ6X8e}&gg*s9+aA?x5-H@qwD!%xyc{uXl)G< z@S*brCw1p6B+Jv-oKimUFjV;NzXfjtnu#yo{_w{Qltl80_>kttP+4_& zciSf_EqO5M^XihNu*mtW-7Nop388caT<~^LEYgxj0?au#qt?R|)@^m~J@G?XbJsYT zxo<%J$?E6OduhFSlo2&>xbK302A=YC3c~2}e5oC5<{RQZ0P}I0v34&PvCzDF?@kDDuMwC`l?w-wuO*1|a8%5--tBd$1&;M$RZ%WSNcHy!T# zu1DPA-N!~tRe5uXHx!U+Q?(Uq_MVInT(NA3}-rm(} z_XvxX^!D0)swRHWQZw1n-DzMn$a&aQ@n}pgzCu%^;!j|YsG{huFY48~go<|QGmP8` z-9TYApWo7Z3~4Pnq%l1P%Lj@zctXWb{Ry4xzBrl+e57-*baP<}Vl;^#U#c3M-Q>nb zb1VstCfB{*FF&|FW1q19VoJb+4;b=i-p#s>ib59@mzI_e|Mg-NJx~v}27?ilQX+T& zZghiz<=Kq$MMP_Z^^U{+;qX*;+sED+o23V!1!HfRt=Dw&tw}#pOu>EZg!;vGMMq4r zM~??$+DvFf1S%2|>x1YgR|e%wonJ7YX1l}iYW!}jXZn4sJ=R+N68O1HJ z&nrg2KD94V5K>IXwc@WAe}?U%S|Do9Q>eZ&vHQ7%MDG)IImZ=b@U=aj3UPD4bA@>&+6Vu>E2LMhw|0SmU|J784 zzq{52KLuBrvAb z_XnBzz`;>hrqxtwSxCw>Sh?8fkoE>C>oBhyhVxJwX%u{BbL5Tmq8R1 z6dV{reSRdl*Xl z1D3IoiiZ7$5v?@q?r#{=5paM{%{g0M4LdugXRYsG_Tiy_5Uq%G0H8DJ$|7uN0X--n zAn6M38K7Htqc8fYSFgC8N^^Ao0nI%xr~p2n#3Bgwa4FusYoGUpfm{+8Asgme$Z7q{=kYBkF&+%M znxB||R8arPnyh5^Y|srzpkD|vQFV+{5bAVU++b_ts)eOos0+=PKhXbV*q47<>NaGC z*1T`M8&h$-^fq`q)0FAfLAN~+1tKqe3*DnXEpIk@MK{Hhci)t0? z)p4k3HaAtCUvPXdaEgkdZnQ1A-~2wmu}*6exw}2HuqZ%XXJ@^`y>bY-T%xZ+AFi*5 zW=AvDo!TlDA1o%j;Bj3fPM9SWAS~|g%H*DoEZ-}344C#bs#=lDlGYi$$y($bN%Crp zH)3Rkxr$fHb#VhLP?n2;$+OH60Z5X(%iJZj56O{_x8A8B1In8D2D8!N{c?+;CeEqF zl&{X%z(#IvV`S4+{o=CCIiTzGv_@|2`~S)soxd@q9{5@er+KLhoXZp_HMhxUxiP-3 zB7(X{B;|OT-&w1)X{p~$d14uP`XVwcGrCn|{7>#{`H}F+oS(xVS~=XD6WS}c z(G;2t;$9RVoUOMsgIqib4>Xp?S|eQAXUX#?3NyZ9xO%QAOZ1yX%2~aScbDT=-ot7D z;oQ=ak8~NZ6#T{wvodzN3TU?jnqI5Mi(o1*y8ry$x7O>@v&uV;&)fOkUNWZE-RP5n zJ2^#TrH7a+te2RF{2>Uk5A{9&BNF{k9n)5uoseIfwUBR4P>@7%nL%@iEtnT29^Q}+ z2~7MuC9`VK*|mnAhae5cNx~V-xE;-m**nebX9#Y}wuRFqYDm+D*f)BzF~G<+g0Wv+5ut`IlB z0%Yo{{m!J<)MVOt=QixLZJgM1Mks6NY@IhkVVJ}qE#!UN)h%EDXZ9{ExOav8C!f9T zQvV$U9)&)EK+T&LeWmatnbx#i?l-&i@$6{$^Xh7AMmfZ*@>hsw9k~$PZ6I|OW4zxC zpEv$StN}^2a$;DAfw*mA3^T4m?W4{a%T#+Ge)wFBK6--rGxJu{n@czlMR;OrR2q5p z1`GR3FiS{#`cWe9u@ZZI4m}?at*#|vXNvh!$eKpRJ2XBR1_>&eV;=COmM%25sHeAB z-}v&A1UE?zlDJ3~0G_i~=^()?m(T;7dD@&F&;gAYsl&ZbTM2OA$G z>P}ky&Y<9M3{7l=Gdj1W(QwG$%8e1!uv&F=iTp^CBKF^!XEDX#S21UXv~Brjrd>J% z&!{=)Q$0+)-8sE`3l$*lxV_93_>$CY3)+({S&g43_uuk)O!?AEM6$WJu+x(BIlEJe zjBhPF^U}Dci;`G{61JVDdnWokmU8|$W@6O2BO~`-mV-FC)bwqBDN1~P|2EknpCQ3R z(fs=ATnBA-Yq4os4_^KcUKx--UMW=>o=-fZAZd2lqf0LTs?>X4YahbkrKYlmW%FM; zhPT`~EF)U_>#AUDVq1e5>KQfA($H^H63T-|b01on5a>?R1l+{)Mpm!^L_QI!LnJ9a zvtvJjIRb&&*{6%jp^5(`GPFZJw>x+ZDXH(M#%*wg)J$roF>r6LB_#2hFRJ0(3`>+A zD-o!!KZTu_hU2dN2uo`BN{eXcYaVSeYBn;4eQFazv0m&SUc>IC zWXmeYN^w4L*YZ}2W88`kM`dUZY~3UhNQ)`SvDJF@w28Lt3~8s+uqd+uPU)Tki^t?~ zIImW|E!9%=eBL!yvC)Bb#?C;H(-#VP9yj!#h)7{{>#n08{o9DcNc5z+$@^1oTqT@N z>=J3gJWFtNojI3tJgnVu3In`+ETuIHdpGtvB=M6DhSHUDOW5(Bcki4IeO(sSrtM~X zf_~ZHouSH^)9G1iYo!ZNUzs{ZB}dnOKT$o=G5!@%I+)<;qTHMe4s9G9l&h6zpyj)b z{ia>I&s}8n3zB36Bb-I~aSFGt>lV&o-CmlOfk`GkaN{};&3@9;yT$wNk{5~mR+O=m z>9(JRV3iN+)^vN_LlwY^_SS{Xl|otHj~0d!6h!AN<`(loegDo{m6$lJgolr0e*7(v z7%8>1GR`BKi$keD4~v7n?F)5-C!&`W)b8XMls{?==9Ur`=;xI3g}}D((-F+%`h8)j zPP&GN{4JWNkPB75vv>(F<50QXLbP8tuCx>*tIA~jy2+FCRA;BjBor&wDq^MmU^IH*XM>IUD#ft}=-7{Y!5f zt7zywdRCkI_J^1vkRVb;NW$l{DR;9_q|WQhErmz~`r;LeJhi^61xe(DF*{NEcrg zbN#?s^}0Y7)Msbxo}SGUC0}&A(2ovIM=`WD-7ySa2CfEcaQu@^qci6DTkIq#d8Z)uAg-4aG zL{q{Hy{_wY6+uw*Au+Hp=|b-;u(HDK85v1{u}*LSUUE!+ZbEUtIU$KOqQV~Ahvv}m zU1oS%&Szn9gkUPeoX3OU3W6(RR7B@@Hz2F9R)J*D$1KHF`9Xi!yiZZSM_3Gp`nXr| zIjcwwZJJoE*@O@{Wb03zr$h~(>DJ&D@0HitJj*92~CqTK9^G*;;)tYvi&OWSDOmII8MH6tdws<;S<*lSC zKik?x{p+0KbZat2XPv}(?|-BYmr)5cdY&zr%TqhNwVJwuBekT0M^^aMPd!P4AjF0r zj)mGTSo^g+>sl&851zWFU(?%rH_t~;4JJOSC7|c&7KQ5J!5qjY9mypSCd-w49D=?+ViyALiMokwOw5q1<{e(-!Qqkt)jW5 z4dSdCECUnSW%P#(wKuz@`9i}ZGUnAQ;$M(Nym*bf_Mdfm-i?kE6>SDRbE!J#V>TQp z{^--CzCm$@Iv+i1u1ymaSzZ<9yS%}&;4nPwvQK<(tG2I0--=Q8O3^AXYZHtUaLuZg z_AVgC-F4ca;q_I~asCv!D=keaESSM^)7iRsn;$a@22LOOJS1rAt^H?m;FMG@Dd75l zVosk|l_HR<&M_BJsl#5(Bp%_|GSC9jGh3ZXN>EE7se{kloYcL6Thy3j{Lc1TH8K8; z@`F$moG3N~PmF0vDC(zT%qq0y^()a`?KWBRa5%{G1l@RLhv1Z@h%fhY*$Bl6ZEUB5 zPU(*QVcy*OWAXNi$^U@Shb$JgwTX#uVjH&e(;0ITi(a_(dbena_tf@rU=!VPu<9hn z-p>>TzwvGuE^NAl=*ht=7H`vPDF^NkE+tj2cKY#XE&0{QL%ize{O>4nxp8pF`}wZ- zPYqnscTkeWI4&89xjk1eVr+gK-l39g2trET4EoNUC~ZtkFB%snVZJ$0+1ry4pY1Z< zmbOg6h3xnjaA7pH)~X;I%r+NjUM$M%*0s%Xj-6Zr?$#ww_T>tTD-{YaQ&PLK!@ivn zD>WBBWksPwk>NubAa^Lb;J2;5W{$Rnlqi^!xY5fL3Qj*7js36(p3#8imZ+sHAL22a zTy$J}sNK9;jLoS6v!_e1*QXw6O2X?A>Kt9gjN3ZR`6q$bNX(oX`c1rv`FUw->az%+ zcfFn-xglja?{eyeN(_(fo{dAg(`Q>fdyWMhBH~k73mTMCGGP%hAWB+=U)=Za!a$0% zffBVck0`<%;yM<&?lXx8V{P{H)moc2-qi2eGkndMZ#dn4S<&#wf3+BYPD6i}jHck) zuk0S?KH%EVEL$%8a%zOLUT?AS0<8G<83s^fc~P&YvRptgh`(^Ff3Y!`rHe+IYQBhA z54I|KeoXuo5uS_O%%*#Je9+TVLv{N5F@r8fe2o$o%b<*tTVgAUt*#X}2wTa56qMl#06u3i`?a;jw&A6XuS&%H6UqTa_J;zqmeHLc!_$ZZAOx^ zUOS~FA8MQmXpM`mVmw2HDf(%u&g59j>2VELi#_8pBn{#O5g~+ApWjI+<)y|}OHbup! z%pv?!%~-#h0Era_mcfka`i9yztsLF08!}EYfe@0xe7L$_;>~$!LyVqkW>t8g-GY_m z5<`$d-)YY~L(R7Z=eQ#2J?1c7`Ko0d7(G7+{(0%Anx7D+ z`~|s(eZumtv#)-I_btEjVyuS@#Z!Epj#j=9K1s8?mOE+|`~F-|R^lcH5J{2vf$(fF#O`PYN36M}aD(G&_fLwz)D`ldY_Y~+ zk}0h%H($<6ewJx7u-hjMRQTHbzN1BjBuz+Y)%!G_Q8q#EtxG^}DC5U71rjR;>%56G z-iHYu6Mek&+J-FQ;!REut57I6R3C5{JuE@|%oCvof5;+lER0dnOyx%f3V%9XcvHLf z46wy&XdGyZiS!buS;2#K+uNrV1i#cl{XC6?=ZidW#8x5BF!{ES%}`*7Bm2HZ)6$Ub z(oE~$25;7JT8&%RWb2YFO>RpNk7JN8Kp=fufSx$NH zn?--@dWWl8IM`Pjd4p5h&gvZYbz9&<^5~b3FX?Zj7^=hIN4EI+^P5#!BND_jhR?n@ zdVya){>i3XGtjg$L=S1>+6>Z4JTlrpZ@6L<6_eGnOR2xwTUWeAK~D5AnK=ztJs%N#$78wnjhTgabmpd5}>D>H^yJG)DU3!hgBo*tT z;|L~n+nh@o<<(0X5*kx^x4+_{eHsWk7_M}FH;OZevV?;siP}y2xR!qy@1(8aQeNrz z4DuYyBf9nb`kX4QVLyr54Vi1}YJA>+edD@JsU7Ye0*;lbhzq;wqm_rpY)Hwjh7lf(bFgPRx5e0=! z$n;650pyB^i3L4?JY0xLCR!ft?mdkArLlqVG?+z2=7rPK{@-$os!>ER1?Z-hTjNGiPSc+H=mHbLM&WJi%zviqlquF~VLgPIDmQe%4)7 z5K;L1)pWB>yEhC#@1+`lVDeX`7yl$2=eYg@eAdzxbG-f|A@#q!ROsa&FDWT`VU-oI z!Q#Lw&CUC+NawSR$HcuTzq3Huhf`N;>V-x0$W$)b~fuji=9w2WI0I{S+L()?^Q}{*&1KkKrkd;oqauEwB*0~sIC2bG{wG4*~btqW44b_DH zRXT77bDGazyQ5MzI)}0<8a~f;zv_rWPnW{aa#AG#paAP}?g0=?cQ*qgqX;dKPKCtK zPNz?B9V^E*`MAXiVQ*Jy^$A`NONcAKEadCRO=<6&eU!4Hh zrDex=ku0E~9pn6Kboz29B%!{&yV=3F9 zz^xZPJ|;~!==4SYG%AD;Tdu}**i~;Wn47O5f(Nv)B_wV0<{OYaEO`|3j6)k)N zBvnA<`gIkuD&4P(25*{slNfKl)iZs?k_+&Fk5d5w9Zev@1WCszc{&xjnaKZ+ET)+F z`1sgZ<0htJwm$)PQ&dPu=-xeJ@~s@}f5qWjyC*_Ms;X~>On~^Q=Vwuq1omGbeU}ZJ z042K|-+$Ex0-9-aeVvh!(cu5tRzbvu%y3W&M9tw}EQV4l0TrT%VEga4HG1Uk-x{Uz z;5&3!!z~VsrJSXc@*($yWcXL}ARgS*{NL+*oXOgCjGO87Dfiy8Ic7GbEb8g&9=wkK zEN8}S9c*A~MTtc~@{?x^HuaQGDoh@v* zb!2Triugb30ydUgrfYi+)$3Yh#G7gB&cw9UFuxueD$EZmft{$C1W)vVp(|Hk8oiG@ z%NOy&^#EQC>L_Iv_CV%6J@4OrnbduPM-p%v=RF|Ah3Kwv2T2#iB@cC6UO}VlEt2l( zw8;wOKXm{jxH3mnQ<0Gwbt^IKhm%3)n9GdX*c8;Bwd;Hi*wlwd?Q|HHa_#NBlApgC zjJVlygUXTUaeMjCS=%Am8KJ?L@60Xchl|ce zQ3hoP{o|W6>mSP1Uv4ZH4C_#TCsOn55Ec&)euJ#q_>X@8zq~+kWD>5EgXT2;u9!-> zv`Ts(dzO_DzqB^cQwB|R?Ox#{Prlb04qG`O)T9c#XL5YDj4gmchhPSn`;_LxhXf1u zo5b<^<*y&uD7fPyp&3Wo`$42VCF*vW*&PncvM~?b3@Y*vGIwM>>MM9q_MKkO)W@g9 zxM+ZL^FsRorcn`F!`}=;m+Tjv(JklNHViMV6zB0Vj||@0#AWD3Zg0R!!ttZK{fobU zEnC{)9hbQAk0i8`I^7mNdha_vl(nC`G^%);+k?$i(^GGKVXL&&D@%Er`kAGzd1w1p zIx@y)(*~9X4c`Aspz4jjkiB+G(X^FR2Ej}Qi@Pz_EN>Zyg$rmUS!7>vx#2OO5?`L> ztwft}6zTM`s+x{HV>OeC97|#ToO>bV`0>xAn^W&Dsp?u@bN*11dTZWF&0UEX6gK%S z)m&z+)Y%(aVORvl$`-lD*18D@^X#pMhc0tsH(tE-h2y@#ySi6>ZG8GQ8N9Z-g}s{& z^6a1-pku1|Y zCAGCO#*`yW*X`)Kh2n}C`=kM|1m0!62py_0zleLCNgWbD=1URxt4VRjH9kZwAnP-}#&_a(%hAf!b}c!lgI+rMa1 z+pIL{n32J`q&%Bi2G%}1;A9t;zHu-|oJP3wVA}xX&~PUh*u?if=q|0>3pv+13KHqY zQ_ma8&?LXiN2Ejz)UZrX@)qg^b_$6SH?4At1R}n2gLVokZMdHFtnjS`?ka*Vt8c(; z{EEmH4K69d%&DQQ(CPFBg>CDEX9K|_1+3L>dhMPtxVMYR)BS!8HaEqEESJsZ!`bcu z3v|fQBrEf{;yZhdk&rZOtpCd@5ry3}%(9L0?zcjl3mTo`OJE&9lEzk{bvj!jA|y;v zx@=}-1{o&L6EsvoqUvRd9Uta@Z?IV~vZ49ml0Ll;T|NhvHK)PMH;^5*Hc!o%OPtjS zg2bo1g#C(uB2mDdIdo*6GUEc$_r>UB9Ftv)#ZOuam;%Q|uuRV)z~O3&9PPT!)^+Ma zkt?FVzHGljNu4OLMWQ^1N#+uWG@2H?!YDAk&0=S5P+N6UDfk8SNsTeeckgGVau|C} z2z2*UIHwXNqsDf-`Ukt}BhN3tzYh)vY3l8`6|w2YW!osD{I<3TgTrd3|Crp~v9P6= z1}5_a)(aV__*0_CivAQQE8)75;#xrnA@ymjoI;Vt54I4_X~c~3aVPDgyOO;9ud0-i zWn~M+FBL(91San50aNYFSexOIMVWx%9!{6J*{`1h!Z|n~tNV6K%#1R85n@VqlOrJ_ zo&#^i2!89ZeRL$qYl0WdZLcii3Oew5QnAjdD^yq@m^FT<+Dn>}7HRLq)6b$hjz;s! z&giEfZt_0~v(}O(MiaMnz75tRy|!}PsN{<=qJ3j~Y1|nC@$_y(&%tP?f;B6IHfEUsLI$^NkHp>6?j1Ec91G(lkYA?u zBhs7r^9E`}`x;WW?lIZ_YHRPmM2h_i$5XTqX7RT>{a*x@8h|oM-2^p9O^-p9=@Bel**LW+pKbVvztDqd$#zjcf-A?qlmZRLUG=ho#o9~q0o*bZd|HlHBVf$;~_MJBtgVFtt zAZgmBjvfr$LJO{uTb@hH+m7%DiF z%0{ckm$&&|UAvPU!+-P7QT&vdzBkgxsU1rDofu+jsa4;4JEZT3MCF$W4kM(`XyD1) zoD-TUolZ3(bf&}{5kQt9IdA{6_Fh<-HW=gVx;EC ztgX<0bht6KgUC-_jqMNf+I2uS9;1XbGW8DGIOQn82Wik_FwE}nTefg6VZafo(Em9q z(B&S_1MSMGL!?o|d3afhrOlzF!AB1)+v%o9O+`f7F@ZlBd#<$dVrNeGWdL543prey zFz+eY%!pX(t#ERxyZbnhC_zOhc$m$%PK%N`!m4DCt(nWt%q{U9+WJE9JBCmADwA4B zPF+$^qIIm!bBD7XyV$v(K8i2^1sBY?Ku|fwh6~#$dR__4-Q~miz{>Dj8jJZzLPB&1k<&;kK>Z( zeUhqY`oAI@@Z#uz!RzcN`JwTcH!$G7nUdQ(I>fA+>grH*HKM6E;Zka`n(qo#RW?gC zkenVOFU=}y#9mTo9q9RUPY~WS`3u)9D;OMO1s%If%#yiKdtyY6nn$C}x1ID%AB%VP z%JUUD6a*tr}|lXGn|BdU9VvWmaS zxdFqk;S|mSb$#r*u6Z=@&J&G0>n&Jhn$7F=i%2fM=%f6R-{#aoROr@^kIFNwMxt_+ z{2y(59wg)T8>@q( zl))}ceZG&sEW~<#n!px2!R^j6kM4YKM_kia_i2;xYsakEuKYdXf{N9`0Ii3sOvfF86k(t zbLe$qo%PITXDG;$0Via!*)01gFw@`?kTHW}ABX<#>?2&DVBj5EcvRKRD zZKGuFmC{&@(vBk1zb^>Pj^O>44t+~VM(WAyW>4HfPcjaPHE;u0!Zk6AsZ zjI{Zw(|5nk0omI2@^eCln-aEhS6L(UXnQC#EZy2~N_IJXRd4nnCx6gcKnq?{+!g%xU}Eh{l-r2)Doav8LJjjd6v=)70o5&eZb8K&Qj$E9*`3hi{b%#Gr|C z@Cl$&k8V%NBYr|`dmA>9(ECKTX3!Gyu1xBf)kNc2K9j{YsGAd|It>!{w3!Ai72Mt+ z?9&b~Qiru~A|6d`tkXiO$=-&6fkR|KBFt+DeQ!BAQ+{-msVDDP8^c+l>2!g!cB9*; zR*-a*)u^VSZY}aae^7a9iQNfLYK7rE2z;L(SHkf+~9#unem1Qz` zZlpwZy-8`{qmSbx_E*crb5uK0}ppR%rF?FqXr8~Qu^r7DJ}^z`z+ zqDW!2ew|ajRfSf{EiMN+KjAP^#)5a4sGk0dB*53!#Y6&7@V zm+e7;d`*6@MARu$RIS50v+~i`#azDYqW2Hy?V=6yH<<9?Z)z3gY$7L5T~vLjp&?&< zmx6u=yg3^xo70Gyhz}kySl@Ft6OR6sv1txa|*VQv7U$QPIG?nN@*FM#?cka9E+Hs zYmo_)2bH3~f@@U^mV$Rnmip`&2J9in={_s*q*VwqaAH7D%TugtwJ~|STS0Ory^(;l z83GZ6MZaB-ePEC_981PWVaFCZPWy&UB-ENL)Ava@9A9kkjSUs*l08g9jJctXGNQu_ zGu?GQP4$3{3TUN@!q=}L-YaMaky=?SR~7ajz7_4;7*LfP+6#w_4lUmLigcYsiuL;}uoZNdCoi6p z=vT_dlxchkAG@Sb_(d1H5YCl%oi@D6q*TK+1ukG=XBUZ1(|Yyhg&}tWZ%?C@a}neR zhi~b;^(eT^4_a!F=cx-c&Od1=C>;@^V3Elj^ySW^&Y> zDHjd^CNz`9dndaJO0Bv8P~S_0V{g5dMt7%?aY6+pph=#b338P^8|*|toS(6nLNS%Z ztZGt1)1*b+ksfMce^qH#jWxp?|U^=4na1=se>1jay$slF@CS^Ljro;&QnbFglgW~EkozkiTF`{!a759?<+iIkQQe~*e5TsuE| zq#~Nnvy$Vf+B^=T-eSY8$k1!=lol4WtCi_i1fKuUIGg?h7gx+BbwSU)5bJ+_$}Y@D zw#)w;tX%Wl-S!vP0bMW#JWi*T$8UoZB$@yA#-_GbA2GnAz1-)F#3bT*zNI{ibl`0B z-L^9Oq9jYXiji9_imv{(7Rn*0h5qn;Bt5CBc9yfuNnD!s1w1^OMf>AMukswIwK8S>nAj+X^C$>z6-%g<( z42>-)*&8EIRH7|Az2gYfBVM9k+D6CR>sp)goJWkiMmq|9Q0KY)lw5qX+kK}U2Kigy0fA+&(A#f+}6K+VzIJbN!RrMd427by$Ms+x>pTmYQsYaiaIh5e{X zH}a%DOn_}3x&?rjstn0J#*W)6>H=EPv9YrXQgn2`R0O-rf?;p%07SGIg-WSB0gwRP z+Cn+kT3i9Zk@*BL;QwL=Zi*@?W%X4)c<={%%&7tp*PlQ23(8&v{+mdMmP~sBAd!#F z01G29m2(|{2*=Vyy*3#LnpYG5t_`4|{r^4Se{J8fv9SsLbdqUE#EQ;X3=mJx5us1+ z5$J;w@Gg2m6T?|M@xDPhM_T)^!8d@g$5}oADoq+cuc + + + + + +%3 + + + +tilde_node + +x ~ Normal() + + + +base_node + + varname = +@varname +(x) +dist = Normal() +x, varinfo = ... + + + +tilde_node->base_node + + +   +@model + + + +assume + +assume(varname, dist, varinfo) + + + +base_node->assume + + +  tilde-pipeline + + + +without_linking + +f = from_internal_transform(varinfo, varname, dist) + + + +assume->without_linking + + + + + +with_logabsdetjac + +x, logjac = with_logabsdet_jacobian(f, getindex_internal(varinfo, varname, dist)) + + + +without_linking->with_logabsdetjac + + + + + +return + + +return + x, logpdf(dist, x) - logjac, varinfo + + + +with_logabsdetjac->return + + + + + diff --git a/docs/src/assets/images/transformations-assume.dot b/docs/src/assets/images/transformations-assume.dot new file mode 100644 index 000000000..f1952b63f --- /dev/null +++ b/docs/src/assets/images/transformations-assume.dot @@ -0,0 +1,22 @@ +digraph { + # `assume` block + subgraph cluster_assume { + label = "assume"; + fontname = "Courier"; + + assume [shape=box, label=< assume(varinfo, @varname(x), Normal())>, fontname="Courier"]; + iflinked_assume [label=< if istrans(varinfo, varname) >, fontname="Courier"]; + without_linking_assume [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_linking_assume [shape=box, label="f = from_linked_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_logabsdetjac [shape=box, label="x, logjac = with_logabsdet_jacobian(f, assume_internal(varinfo, varname, dist))", fontname="Courier"]; + return_assume [shape=box, label=< return x, logpdf(dist, x) - logjac, varinfo >, style=dashed, fontname="Courier"]; + + assume -> iflinked_assume; + iflinked_assume -> without_linking_assume [label=< false>, fontname="Courier"]; + iflinked_assume -> with_linking_assume [label=< true>, fontname="Courier"]; + without_linking_assume -> with_logabsdetjac; + with_linking_assume -> with_logabsdetjac; + with_logabsdetjac -> return_assume; + } +} + diff --git a/docs/src/assets/images/transformations-assume.dot.png b/docs/src/assets/images/transformations-assume.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..b8b0ec7348e96e6b611dd5b34a30130e649acb79 GIT binary patch literal 44454 zcmc$`1yEc|xGp+z0tpf9Fl_C;EJ z482}-?5iAB5o+&l)U;v_OK)Fa5ACRhzGhFE+~HRd7VO}|M%lwvsr?(i^F$bN8h+I7 z588lB$EKdi-Q6}0oGY`%cGYr>B|pez^+Ci2?vt9|_V5yj-mzgG;s ziRZR2%%C??J|y>g{P=NOM@QwU{J-ZQ2_GFC9Kw)fAIf3YW6_?FZgjL zp66z)gN#||`oi)zk^?D8h{&3G-s)Pn>3AkFGN9=I=V|%jlLq6)ZfF zi^9FJtZ5W0%)VDQn6t6Q;lJ!OeMxfMAS=#kZtbSR-klfF#_6D}Eg~`J{AjN|Gw2_b z6hyP4Ub9tUFlF|<5+ZouL})g&-T?tyI2Niaj8hYBt5Jh_GS#g ziHz(qv~szM_v;jsjPI4lv~uciv6U5~y;wnqF5Gi|3WM9%MJhj`s=m2c57xN8nFwNT z@|j+4<^hX^-+~L?;bYv&>{+lkCZS2(M8jw0$E3w>L^ms2?;;fYP!8rvuumYl&YTem`jOR44Z(vR(Y7{#oy47~>H%__?3`BL4*n=AdK5ZlP)G7t0c z_uq7rGoG_MMANgMCv6BnQ;D(kMLk;Dr|Y91LjMy2yI3npQmxflspGBjLKiS~_Whu0 zeLbktwRnsCQ$ zqtsMW&365hamx(nJEv%@c-SlK?NQssXTkx7t|(Gif=JLw<#eGGsgPPqoB(tIC0ZO3 ztN?9&X|HTnJb}D%2)N03LIk#3+OHo6ZL0dZe0^D>js13iQ?2BR}{BqLS_zqc$TpLqVHax1Rf5sml z7alVDMUr4+U)`H_^t);Ri_N20e^|#!QaVC;lOG_AlLwn%jEeH}uj3_}6BPP#xnrfv z+`^N6@wfA4G8-+1*mVo2{6vG#^xkW%>T3o4M&{ zr(nvHKTb`b%61&yD!-r92>bQZp}X{YFx!vVOsrm2sz7c$lIJU5gTyTQq%G47{zz&ZmfG#indALrUzpR z>^s3qLN*`qE`SfSKGVzG&$DR)&a(kE1St_0T~|$n_ZHcD^9zREG$v?*^!_i7PKq&3 zs!8(-?>YzJR}Qzn4b|5;$$g#qLF(AUgP;1?OLiHT8_b?5ct=+(>g#wF6SSlbCQg@f zXlGY7d}(Gbu{?TndRzxPxZ61)sb@1nI0T2EsarS%6R2ot6kfon;^p&!9}$gEk;aPA z>A%2uA|Cc%ZIJ&DpZQZ~7kHZTm@miGf@2r#6#jCa28y_<9Uh) zGZnp46<|r;Bcy!>`3F11h8mMx{AXYwtKA1+aLgGQ8N|kk=D9a#Z=8*9#THU&;xVt~3?RPMsmvS?7+NJ#C)z(S9WW@P^xtXN z49$JH^Q>TW`%d)x9y5Om^zxvl{DHB2rC$E;R7&$;v;mrO<~3*1Hrhnk@%xGg6H~~! zG-56_W1mfZtAeRw>EYFx6Vstj&5VIDzy%ZYaU)Aq<6;SZ{1Dd#U0Z)Qw;U&-?yTTY z(q6rXR?+Ym_FLnVvdZ1doAJT>*GEZHOT@&tO)v{hI_J#RKFnRk`UQ9u$FJQR-F83Y z&tt#P1%7TXRBr0XKUH<9b8?a0G3`D#MD+C>EKYJ+TJK8J{gVH5?}1P)-rnaV;@S-H zcajdXDHW{Q5YT zJRVDTIc)WA3S|A+yqRd)lv?Z6$UJ}cCzHNPztRosi?vebE9v`v_#XXsa;Ff7TM`A8 zQLI9i8~4wwG}AI@K^fW0g)v~Io&hW6ArUkXsNR${9Jo^XZYwdh$g$JQD@Wd>&3tGdg z1*%MfDo*}_xYcpNWrBLzlJ^w*4gZ{L%XT(DIdYsKNcn#YK-iG~iH*PGmYn$dx#KbN zj@fd~MV%GLJZ@q^m3(aZ!NVt}0o^4T<%xSAS8uXJlz#k9vN9e$cYn3M@t(D8pHFuk zZ9_Dgj38R?kCP3l$%=J7aYgG|DhfOSdvl?^tu0k`h`VV%a-8O)G*<7@o(^)YX|K|Q zvTUafhsnA58>27Z?D1tEXzFh5!;gRTE6&;=RoB{V5F3^9xvujY$Q-DKO_Hs0j{c}~ zjPXyc=+*Y2g=gJwwb~5$c_m+*yE#EWVG8`rWcYCBJ}c39-+mtg_j`G@IjHDfTfSlk zy=#hEADiIygj+8MTiHs;jHFsD3klvaUiqGsTwt@1Z1>@Zoakh>M@8~iS84YrrQz;6 z9i6P?(aVorS+mq3%TJ7CM;}Iy&Bvhz-M=+oNNMpuj%S7Kq4~}5Zv@Z1GIJjG=iY!{ zUDiF)N!9km*$=o}y0x%3j^&EGXS~5yzrSeF5D1cO8*qn@*>o?}@LsA#b*Xcn-QNk@m zTwVBYqr@uu(5=uB<&W&O4En<@BQkHGLG($?M-E$l)-}cg5=F|@<~={x+z3*;XZa}% zkKyK%e0|7I?BNTy!(#5RneXe946|HU#mfipHgxAUo}CrJ(-MdoY)dz0itNnz>s%J4 z?ptHF>lkP0DGM@%f+A(E%ReW~L=wcZFBfdekwKBD#O4SpOuND#6{$?npK`?hf{toj zA4!GypbN-I++J>fF$xw-x)^7@k4zE0;^eU$_|xKR;MO}ze|NN(?=||S^Ht<2^RUVD zTFTK;wDkaC$qM!~q?ENuNi}tQrVS13lHw|qQvG51W2?$`B>qwL=P|;g&eu*n#?H{S zl3p_7lHRZMM1n}tdho*)@0FSOg4{-DnzMz3IcH{H_c_&}UZQ;kY?>2FnrgSKYrYZdR zc=3sK=Ce~Cj=?x#mY7*sd<9AC4!`@|rN8T)o~6@Eyl?0I17134d^@8Rd(IE12ErB` zwnn@9n-Z@U>;Bwc4_17uGBSoOyc@qmHTyH07s_qJ3>hAx(iu@g^ zOrCwP-R_7GuKO_K%33EtM+E*ZvSzA{e5=x(^)4x@?a z6*F(I-)fazm@8R94kPAtZThM(!cs7xhTE2wKDwgI0-faTtRr>1qVV-s3esufjFlk` zQ`C(QOx*gA%>_SOHd8iZZi{YouS))+JqimV{SxN3EH|bp@7f}Kd$kr_^t4P;^Mm|o zxsTNRb&XAiv4jTIP-O!-Ju^vMpg5oWt$T_gVdm!{xDf zjwIFLWJXr`BjFQ5U$mL(-G~*KPy4`$`*h4r_TFZ`$@J+E3Acg|G2H5O!}C1>RInvt zM7SZEOYsv8RK4j+yCrdJK%Hf6GB>{QM!RJJeA&!Zjj@XToty67$f^2iy1LKV`7|ky zJn>e4E>p1~33KAM0&XYcnK;t9ujqB?vM_%=`sD~l481$G@xEP0V_x4ca9!$!@e`X) zRA*-eKNF5|xvH&SVmA9x`ugPh@$9d?VPCw&pRgbE>m@gKCkIj^KdK~8VvV-lw@gJI zv9p2z>IVp=J%lcGB8#F5?vDe0k}1lb^q54TIk#8;f1j7}UOOo28LS#BE!eRIqL!C7 zkY=81HXa7UFcn@Bza#g`F5H`nN&g;)%c+!Yz-^dQI;|75r^2Q51&O4M^|0dLp=xo? z?gsxa4VDov1#ql^7gLuIHz>(grmW ziPx*Tne$gAobv_q(=k-$b`5Q&LB0|A+>Zct$RZSk%KZpj z%J6HW=@eG_!)@?lsbD@SAtZ}@{e$6A=5S9B z-pg4pn$(|}<+K_8dt1i$+23n43SXV8XD*~_xfZ0_>k-!!xHPgyWd{VS6}+F&ct@m1 zAit21%?ek`Ga8%BnPB006r;zLUX+o%H6t}4piRZM}&iLuHx&MVvRclr%87H6TGFj{ z_q<)+fGC|jKY=;bSLDyt2k8_(^=EFDr!a2ZEouUe5+~QPyorO`%T#gVGIFE(Yn%3 ztkUhh#Z9iSD0D8-Gv?rD@yWI=#S)#f6{^y zsU$z12yXho7S)@CNBw)x2#f&v%DHr-|VyN=j#9lhE z^pl&Zc@ywUq)qmhhi076u{RXUMXUtSBC@;)41woRuctRCrM#Jq7%dHlJDDauwGl(^ zRgSlf^K;zgf)+{ZTc*YWKZ!mVU1mOoc&sOPf=r07mW8##i|r*G0l);;5@NGUCiYCC z_c}eHN^woX2)q9E*2|@Gc@cjh1?GOcWAaIQe%c8uS-g!reEpL~Xw7DVd^8yG>;OZd3U!$2_4y?mY1(E%Dp*TA3oY*T$mmP22pu7rH+UcFA}| zDRQimXaVkYwI-D>x`15OK~$dKR_hOK-w=(-fq5s(+Mb1FD+Rv1-pBl-zKj71 zy13m&|H$fQ_qH-WLXuv`Rx3&uS!tB>*OvZZ(T7uzDnc7lMiwWpTXs3x9KxkJvg;U=g%nGJovBJ%i+qlx+mkJ@@CzhtD`p)?i`MNS$?+8 z^)(u94PLRgs#S6I$(SIG>yql-)@ibsJ)dnF$hu_%n&TS<&0n-ah@Hxf4@$_1T65V;``yyJXAkrikPqx@n z4o%ZsexWqg2)*XrmT`jlSAX^)m_{lkht<}fC443g#mQv%XWrKR?WUykmnSw_jqwwd z*l+Cj-Ab8?))YZ<^1ZC$%zLZ}YEA#vz9_%93ANp3;7M zg~hm#7)}6x-=*A#t#lUa)NJ3AjT8SKs_a#u!atg}d#O0;<1q8nQ4JGC5!QTU;g0QG zQM~Y5MVm|4!Q-Rj*{OFdIj%03i%N9E?$k*3y!Z0&u3|3L9AWOwV+IasEV9>i74ZrL zaXVd^l>Vs>j}$XlT&>q4nVgSS28C3NneH9jl@zV{BzHN!P0))}q`IL*N}4d_Dy9ts z|E36m{;+f2GXrl`sVRKlg9Pp;1ddY&)e;UN{i(!F1gf3m%}7X z`FOYcAO*M&ZIHW10AUVq!^IShG~;KS-GtJ6l~U_Tt>A_cXPWXq)bAUTR2e?1*&a-6 z@VEKW{UWUWcHl*i$>-l-%Okcx`HIi3DO6pPkkdf11|Digl+yB~adHJUx z1>SX=)cbKN>z`Fiq+9jQj0vQgEmHR6w4^gC3W@?J0STJAyVB%wD2*3$u3L zv+G(lr;d>8p+ptCJy{2B;%U8a#LrG_QT1B^SODMAWY!)2T%hvR zV7q&`*B;-=*>9ZltCGEW3L8Zt&iL3IfB;`0BVS&X6@!R=&&_=?KIdH!{m!f+eq)Wo zyF|=ks|*1{_K~cG_Tn#zg9V98tX-cdb!A;jraf?9B)?Sn^xRhh^@?hQ zyREfPrfdD!p|Udav(LFQW7dZC%V(VI4VuFl&xyadxcGBhjZdK65or)4?QyNR50?>` z7x}qZ3+vAxjpe3adTxbZc=(^tIk!Ir9KifIiWrsrxvT3=BBS&8bmQuv>AH0Q@LQDSYddr zle&m`kLU{lt8F6)wLJ*&9{>gucWxv^SWG3{pYM2y>L|82|XVB&4V%A1(;@ac;$2Sqj*R|Nr-BWr2K79K4v+ple2orZ)xXW-0m*;s zkKcc(KY(~thDStTV`A#;sz4w-^8<;zCcD%CTcQRyMLG)q^0UoN05DHJ z`HPcdLu}G0jW-4oFexNh>;%Dpk`opQ{#zO6|6acPKld*8p)~7%)&l(3)A7r%>kuUf z33KOrvotg`I|H0fr-sVPG3QH{M=J#d1+xtr;PjBhUoP*(&QQu{)k@PPtD3&gY;g_W zS300^Y#0J##uZ=R$1>{pFz!49S3mVhSqC3J*ccd}kfixi74qb%v6=hz%UWAouUr~S zc90Mk{bfDADUpznsMp#U9aCk+f5R@LxlC)r$nd|QEhQVM;@?a7?>^u=f%n%XYFSZS<8fJrmbjqHef6tu% z^Y0m_v+cF5tsK{EIcf@MtZv⁣?5tPl$>LLkjQ;qd{yJ=jZ1~M~vpK;8e2fUrj9! z`}q?oLR1h5%p0?QAA_AO;#|VN=P>QpvlO^~Mas#|9nny)udGxc4`Uof z_D-#>jg3*r8^^}Ro?&G|$x-#KMKRSdGBR3;*%(Z+u;jH4`P#N}WJ4F&yXN1%{FJ#} zyX1WAdx{h#I#RR|Q82|Tzi1;}}$OP1nL=yQ_H%uE7Q;kw&(;@Fs& zl4(;{He&F4NC;!}I#!%KudtAgo<6Eux1ypVA|gUGqclHXJV#DJUVeORtZVJ~_}D5! zR9-`4;IRrhKC>P*4NZ)L8vE;=)9SXbM8r^Uft0|&K)dHMIda_nwCNz=c=bFW!+A0K z<6CrOZ`+N0r4sG^xf)v?OV4=NFMogJ3YGb9Gmbbby_i z*<@&Z`aW*%lg;fI6p2v{*T&YiLLg-b)gU`3C#SHmFh8G`p57p6(l9G8FK={Y1djk_ z^~sj2A@O`jYY85yq+d^dI(DgFB1 z7+obkAz_CvTr^|m?FXEIn0-BuNNv=%F~V*bDN~F%cXTS4z3w!@DhpWQP4S;sZQ3Pr z)D^|W#g&zn<>lqzpt$%O`S!qYVlY^X!&SsuB$Dgt>DAQKTwPsFPftI2^2E{6(c9bm zrGV}HF(7KeDn&*{ipOtG6yP(bK*wgD?~T_$xS0L{vV%nX;lr7`NZ1O8KM)#ZWdDEw zW+tZcgSUG*$6n|qN^0!>swyff%F5HTv#P49+#DQAs;aY7Q=8VI{Gfk?nr}*BAq#%` z^hrWO!p+SM$iw`v{=P8{nt;|N#I-XsD&R&XrRtg*#(5j?E}+d2@wn23zZ|MzOzS_R z%Heftzg3l!lT%Xa;t>a{H8?of-#?z1IiOz_bJ?H%-Pze0A0MClZzGR4DVhZq)dN_c zZ;vxFGJxuGa&m&vgM*o^n~MXJo4Lmi|7mDwNJvPKqu!e?+xUs`4oFiI8yj1766VKQ z?G7IYCnq-9%)e90!p7qZ=lIBF)<+_A>iMzu-01^gR*HmJG+5_0Wcd69y|aCK|w(? z63l9OZ;~IqLI!F)w6M@<0c#9QlFyOr<9rKV95iry-z>bv8|BTLH^4xMGn$x~2>)%k zxVRL5o1)@mp?V!yiHZZDg&(;QQ*Zc)DTbDn^=!RA&dAoVt|&u>YIJmMWd-xqtCmyO znz}kLWh)PRAqmfYTT7}C&Ej}^WJG3<16O}}qCoW-DnVCYpYDDmQ17|p^VU7lHl49S?X&9);N83#x8{%w3b0l6>pKetmQI{0j++4=eJy*M~I9i5#ocBe-> zJ3A*QCue78+uGW`jssDeN9fQta1^yRTO-~mxS^nhg;^}rQ-yUa(MHC_#T6AD0M{R* zS(})cSpAIy@*0cT*tP37iLJpxlRG~rXACE5^zYxr=JQ;823RLss~a2a-AO*Re=93=1A*e1T6J%AWm;03OO1pqFG-+q7Y>SQAU=kW{k9$2lJiyf1W}^HW#YStK zj<(6Ie4-OOqjVOeNc(Qt*J5Cvzrj=Uk!VEyek*+KN#XU|Hp-m34?{JoFl4xmdrZd^ z_ST$31Ki^KR4_(TJ_n<@nUpXQL`Pqr&2*|*tJQ}QZFF=r2HkC`#aq+uN{}uJNG0$4 z_wPA4INrZ+>)XwlKLECvI$=YR9P%;D*x2~~dZ!2&3i#DVlAxd<@Fai!{IT8`AY?O* z2F|?1WKxCThKz>dgX8T<=~#R((}?ZnkbDZ8!kog{z|rTbaHGod${&(1YkBT>hzp|| z>S;#C4ezcc=f=kUo&`meISp_-9QL9=)1n|w`u0UGYREpbuH*J}Z6Z6xf|HkZZx=Z9 z^>rLX#FdX0!T>>RfAwPh%msq@PTDWPfuU#joMgUjpuXa zqNeTvD>7B#ED+1wbU?^sVX#=A@Xv|e(>wNz;(H}4+x5V%EAB@ddj!OBMd=Z~_qe^` z1hvc8X0E;@y=Z-<-=xuwJl*@uZ(Cxy>!QSwPs>?XPM~|WH^Ghu>Mfo#zS!cbgk4#b z{;!00Lok%*UTn3JeUFD9c$E&Q0CKeWF$w?LSmqx5eczynvHrYkS0C{}#1H_g2l*<# zcmE2x=OuXmFtMv;+~PD2Ml>MB<5aSJtW8WeY2W?-tl8L7?&jKyDRbZG#xv@Q{;1W! z#&PDRh?;fNQs(dEJ=%zZRs4axGjdy2#Fee=Ds!SHCbu`hwHiyXgkXU$Gr70&7dAt{ zvJ-u6`bcH#d_$7ZW4kIw>_frG{d;=)DVK`;{(+B%Q9dIpgY&!8tIk=`R&(pO<>sM^ zxPI1$rEQ_{)fZ1^0-%JCaN9PFX4Pb^9A(G6{+Z8*K6zLYxG@x?N!TqH>ci1UA7qym zrma($hsS$SUQ%2f&1N?ln{sp40h0!6a)(V-Jc4Z6Y(_>iXtj7f z$5c6e1#*fsIqD|(y)y{RQlu^?R4vFTDDdlq%p_c|t9bu8*nfSxAN1*9-$84>;UTmk zOa9GdR^!FtbS?bl)pC-mF|Q)YjkP|m89k@PlzSOoy_9UJd*r*iPyDstb|;uJf8@YW zjgRR}Otb8RXtwk9G}GU-6Q8%!Ti6tgQVbH`iKX~t>Xj`0@F~jTkQlYwV2;bHFYk(L zJ&L^_pzL)=BbKdCB-RYwk?(eoxF;gdynM&DI&LAX|I&04&P|dxT3yw63Ti|3JLH*E zlhZ_*PE%KRNpOH0~GSF_ddovrL9vc7%WWoJvXGBci&%E zB$>q|(f6zU`UH!_OG+(i5qEbQO|{4QNRJFUTFzw;+a+pw`m)_Nu7ta4_$E^yR!)ay zv6zeH3X_%$Szcb2&r{&!=CuF*e(?f8KdRdHU_PYxe3yvL?jmnE%9z9SQZL@*{W} z)-VaW5qRkOZ4?`hiD7KyUJ|*HadhqRZDTdTjPY2{N)8#59#m^P;xm6g%6De1p3*rO zKUtPhY~c(M`P&~u{TXOY=5lXLG#A;F!ntf$deY``_8Mc=NunY$D1-*L==CrJkYI)G zZK2TTBFcJ95uzF)aRgPx{QM@9MMv%M^YL6+?fap=fF?!isHi!A$5G#``Wl^1>cdlcenWYqU4sjC7I@lG&t0Fk=rQGJ+lym zrPOB@a=PRt3+3eZG*mhdOPxHMO>N|RXHneEBT0-F#B_Ch<|AqI5Ty23gYNfXp`ph+ zMNQ=I--BS(e5&}*RK!EbChd!aT~IUmCz01#TwMHzr>Cr}Y@m9ST}q$E`6o=?g0_h;hqZRa4}wMLQH5%D=0h@wjzDk0&>d+`iS4j|^#EW+qj1qD=xzS`AKb z0W>^A;>r#k4l|$iRj(t0bKYe~$t#kbO^KW3WX7pn==0^8crM=k(T}_PmvPlR>W zX<>SOrUF#<;e)2{?##U(T#Oe(?UyTw>4-7ON&Hb}b@qRQJbi1>M1(AER{V~oE{1DN zR}aU8!}ZyD)V-?0(7_d7eZ@@~^YK_)J+^yU2exUFk8vU4snV1GLlXXJl?MjWmDf-O zt5ChNrZWR0qjIB*BmDl3LLz<>IDBkuy%vI6 zPzzFB*>Z7mlhl3M&f+eu)R9ZrikKlOf1Qp!*G>(Utdq5hT(>@(tYp7MCLJKtUtiXq z?~O|JmwRgDB`eViAZe8-c}a<9akQr*Nt>{E0a%75#fbHD+OT0cG+&OTwmCLmK?sSA z1%nTT{l#l;yWQipUjD1qu$dX5DDuY;kI0XQPd&00K+H}q9^I^8C3r!&>HBKt;tk_g z;03>G-Mb^9UE1M@1OEFnjRt($5+z#yRC!}tW{eQ+66L@al>b#3et@Ltgu2Lo5V@nP zrBpj5A!+ZieXBaL6_RdUCCLq%!@CoA73500`%Tfvg9dPD#|z4$w$(b+?5=j@pDb~5 zOjl0dw0FHcTjhSyS>voo(rYs}kt$?hZbF;$Wmt&+$cJ`9HB-x3@26CjbBO;V>FZQ`)vxnsr z$HxMuOi{$BGV05nXCode2aq!B8S9a~mG5*@PKuV!k?Hk6AW&zvKDMt?9s8bVK=yyJ zq4*l>0XZU3xbHrH$P2z}G@G@!vAX(9+NxfmlQ5ea`m}S0ooUKHd_T}>?`Fkl4O!3j z+#7n(cc)p~v3e!_xqlnGV4CyOSLtxe0Znwx1pC zbX!;{Ck&qLOg<+8NK8i|5~O+bB`8x)bOQC|3m`WA=h3rY>KLA3>b$(aYWyzTU%_+9 zdg`Jy-Tsa94v6O${6iL?j<1_Rg!XG*MFJtDy0Tb;j2DuiTtFq#b-zlF+0&Kqy$kAK zUk?12rUkY5wM5LKtVJ?^)j^k*Li%B|^PQ>JZpX!uxzFE?3fXhj7rxce-#G;U`-KyK z=5CVuS*)@6H2*y#ArsHXk ziO31n{~#i54NP8_r`R|i%S+*CaYkZJnbI_qG zO5)S}vk#Jn9p)nxYJPrrg8jl9cJp`uMI$iKLmOxEMm9*YbsUJqOo|%V+-@%`OeVKo zk7>#yS0gHXQ*i}-E*+-;=^7$GdB3s&cj38j)i|@j5sC>`CA>K+|ALf37`^Tooxus2Gu=xoYoksoU!NK5~Ra#nFpgJzZBa3J}l{jZbt@H48+6WnV>YW>*;VHK05$`0r*7x`t@sET;BzYAb_V0CFX{XG!(=y;W(@~CCc~!#akU_g+31hk*xs&nhu;&s-!k&?5#@k7 z6{yXO1uDehDZl~uh6Jve?41-Ngb$; z{Z7k~mg8L!Uqx~!GlSE$2xU=`YDv^D^-4oSlY^dx@A8U@7PFORt6d@UwYC6AzgZ5T z2YEC%2;6Z{__09v@zf{w(Z>y&y_pJ-=<0O_5dzoh`Sc|oyZIc@O91oyQm2P_1f>FG zl#kC1*64b4bYxZIa=JMT7y<%rJKFsLB*;*`!m2i49X9*0klR5d3ep4D#pZuPLtj}~ z97v}Ld3$-ic*FGN{kBVk0!Z-7v3ki6gPC_ZB6&Fj`4unlwN$U$7QN;E@MT>SQAkROehuO5WHM8UY|=nM{riw_)mHfyw0#+G74WNh90~l;@FnG{n{zC*9z~k6V5PCb`hk%fq zCPI{kCJ$Ugt5R-wdAQWr*oftAvh@H`PW*!PnTLUjkx|Zh)fRAE0C08W(MaNRy}i40 z22{$<&d$mTz#Q~Rr9}18Wwx@xU6p}Q<3NUw*fe|WCsP3 zAPcY%QBhHVj{qDMfIuJ+HeaY$mX-Z19iEqGGT-_GV8|S@Pa(;Ke`~e+rLfk3gV^6E z!o%z7lg^O?(2*E;0B8~t3sn}HqN4ud5`; zEv>ltH+qYHSweh#t?drZ>(`e6W;zE*Aiw|tur?(RsHz48xVs4;YX_iej*g7JoS^&< zYG7c%%pVft4)9cf0ZBnjYWr;vdia z{Cr>wK<|M2#Sj3D2Oti7eT4y{NiLb#N6>3+ZQZ)@6v#6QVq4A`rN8!%Q$Qo`4A9fk zjtL!s#h9(KAm+4sPeUU{7sXsuT2W#4y92yG;10~p%)aMnc?_fg8Txf>{H+%#R;Gz&Pvc>jAI{A|f0tL^BlgZ`THRDuBfSqz(84Z~z*3 zY0ywmP}Vh1BOdV701gA(o|>AvysW(rs}}~tiF4F(a&~_9mSxG8|p z0f5fHM-;sA7bcrnEd!ifa`Ib12Z*cyMnRmT$>qul@EUGzeAc2hk!=5%z$+>%%^61L z<_N9`5}|-W1hWTt7SLv-)c^7qN^)}P2qv_G9s}9C@1_gj|J~hGZ{L1&KJ1;Cm;he^ z==@XlG05485xc*HJJ+lT2@oxos`Z8AhXT(x2P#TR4j23C`Vt5#MfEH4tiGN_FE)P~o0!D#s*#5oWf5n6N+l1L*#CBJO+k?b@J}Pj)1lI;D)xy9z|n1Z zXi5Vh?ExlQEd9H%?(zmly}iyc1W@lwxv*Z8**SZuP8w=(F5V1G`HEM-5HJnksW;|i7 zOleog|FafgiQ$0=X~^QlCZVvNUWRB!Zb3mrw{D*eu(se!e*T!dJ{0=0Oqf-J58Q02 zBupIvJi`QUo6Z7~kSnM$dG?oqhb{s#zhYYTlsYyO|3ZXGxuL#3{>^hWdHMEqrx1IhJ)P7%i8qki% zVmNfT)Y`3+ng%?qnAp3()ORFb=%N5O%rJVrdOMJ^U-BrEqUMhaFl2kuFWBPt_V*?5 z2;0Hhf>MYYw>z$=-m{Yv6jW3q7Q^2;IYukL%+d>h)NdvuR6n^DXoo<`@8^eBHMgLn zl9CeV^WBhcU7&5i3%x)=85QbeOP91SQ3zwkA0k2>5=jdBZU!;eUti(co9zK!3h=rY zd$YLs_%~;hnkNqRQ5sH0Gv!9W@hw}`nAL%bD-VfBz_>!VNCBb^Ac3i=slC}Mz^0)Q za2y8EH%2q*iHZp~Reharyrwv)I>>+jsvA@nWJxAzsHxREU+jNSt^9@0EF+3XL?k4F0t4qu^~9W2_XGM`tG*5zka-rvbPCI7Mc=JwVC+LWD4o zUpdv)mm5j$s4rdsI{OX~5xy_pghod@0GNQ)WCDz|bE?8Ka@c22EQ$_9VgOoC0fPqC zSkv`P-rT&f!~`&7-&hbuCEp$c0R(H5l9B?9|3Sk=Vnf4Sqw5W*SpZ%U{q^ggVO~jg zh$x{<9}3uAZEfvu-xmF`X$S}jA3b_BnXfD>C#SnR12(3Yvm?q-4Ee2Uc-%A5YNF#M<ky_i>06A_g=mOOU zCx1ZY8pM!!Ml8IprpEQ8pB)?(snxv7!H<=@P|z)EIL9*=1nQaa3Q;b?lLKZJj1f!) zuucO>{7E%m96<>Noiyt8zYoMj?Dp_b7|}?b`}0Q&gfd>gL7<+IoD7Mg^riSS=TFPc zjbB9yM)I!`sAP7Fg#&ND^lEL$2iSXSno{5=JE}fgq;wGy-^DK!O51 zACR?|ATS1W6;wn-OQ0Gs?LZ_FFp~x?Txghp@16610k(3JR_B7<_nXaXsV*xWfq-%R z{{4Flk0TR!*F8yRnoA?`T%E}in3^(c^d zz@!4=bWcVXk?Aq%{scM?#2h3=e{5}lEJiRO{*Z>i_E+h|kdTxFFOSiPh5ct)2{5JC6Kk^-=2 zuQAAh&FELw(pm!652Pai#>p=z@ci*3l2)0??bgArdt`H@%sCWp&vK`rQ4_lE_!v5k zv)yNKOLMkj+Y7(tRsRfjT3I0@-I^4}6^GnB`1ZO$wQ7RyX;22xrsi-Vl1+qYVor;FUKiyCrvoOsxfwZ zPZAsNjzi0rFzliEl!0(h7mE&ppBIcdSOVd^!eq`LSb3?=KPh7nUyj#^&vW2TK)AD<1}H%ZxVo{U(!i zBhb0C6y`NI*>too+$18wFI86V!H9qXrT}2l!ENl4|pFj~)z?uX@IyIP>g?_Bhila9B=R zZ|=jZH3PWscKrH$slO}25-kRuQnCX7zJ9}4yrkzlUg}EvCQIF0`>NW(n6u9w73WDq z@(+^NjGF6T3kyO#GAWput@~qH@9*zHc40DENC;F*MrH&Ew_<_Hc$YFe%^I#`Lj|86 ztf}VWq_=lGHx;-8V$Qn-gU%qFlDnt1tM_nMN5d6ZFcB7Y1oc{&a18|-Srf6Va8+Yz z=HbHj;JHiV#X!VvDN{yme)ggE2<*Vs?beV)gZj{YO5En*@DZek&2Tc(@25XkN#LK4W+K#WUASC~T8Greq!_L3W0 z6iL1Gq_;p5c5!{(8FD{vu7r~Ba(n$%$Y%dQ>wq$IRT71Mm-~9Dk-JpP(+3T^%#l{z z_Qu?b&**H~gMQuV!e%v!kE+^@3>r(4f2pbxQh)bF4d>`J&*hznMk5_5^!t5`Cxv3K z`R!|%&LOQL%AWUvzN7E;iJ}0M2XDpGuRK_YhL-?Z*!Bs0r&`^F<{L z75yL7y=7F@-Pbm{l^aAUgGN9F>6DTd6p)hcl*Rl~jBGUS=q=_a_mA+0W9~Pe^!DIf2vy~|teM2@+%;Qwy4*idZ$z3Q8Ur02f*Htk z6m+nPW%XpA#)f$pB)Vvut?+xjXTAT*jQAW|hXCQnECU{BR}{C{5m-Cd)e!}4?R;4U zNOP<&F1Fk2yW*~|$0qC5v=mwx zGvsbiFuXlJKiYI(fJ!wqulhJ4X+W3wv5hp~mw?^R$w@(=wlFX(FC+87e}bHw79^1% z5umQFJ{!qo7r4=H9vV9pDC(qwKn$S{4>tZWly88N9W-VZE{DQX?&O+SP zy4uZ9>~uYGmgqPjGOystlUqpS9KD_{a#oryPlO|TmE_`3QFDDQls2bhc-We0gY)IC z>&x{5I#EkAoco+QTE}wHStf;ah;U`CU?x63VD2KvGTXs99>`ImEkrX48~!<#Q&A_3 zFJ`{b&sU~d&{_Ak@CKrZ6lKx>&Ol}eAea$22f;f94;D@lU7w?Qa26cMhj$h~rib1h z%k?d)#8}b&JvdRH5-ky_isUBPMt`N+uHrm-jfnR#4)G$V-_1wXO*uIH=Z?Sgt*XYY z)Pgq#f9gCpuH&33a;+URe8^#T6%3>IRtphCB(SK2&vplXMl+tBo&tNtq$g=aZ(O3T!sUz9$FFJV~#or#>BTt{2G z>y$wJdZ)ORH0N;7lnkRHM}3(N&V2t+&&T$NwR)#N@1zQ%rq54yDfCPlOdD}ilu^1cNv7WJQRedBnB>|}rvyisd6m zf7{KGh^rG5)`xWq!WBykjen6-vz~m_<5;iZh0=*HXWyLJDzFxr=ES+-CdU-Topw+c zP1y`_i>8dU4wYJFe;qX(#;7D1IwePNWP*vOiG*a@#daqv1T9QVT}UL?g9oBCkvYmV z)$}qN&cPn0zYlHpaUW2~^U~BFFS||Od^=Wz!jXEn}ggLgp+wd@gxRM zH9LEuKT{fVdsX2enF&R|17TQ3mCM=~_^&|39D`UDPVr~r;#uJ6gMX<}^23S{0}asw z$!|mObbwr|R;Ya)3o9f%9I%2~@KoWQ@YAJds;`f7Vu?Sph*g724E(az2+Gu`sLzS7 z9LdQMapd4JFD)$rlM^Vh7bnvfkliaA{RR2P%csFcLp==|#1hj_p#-9> zCjHMVP5EZA58#O*2{+o{U8~OIvsOL~X#%Pb&p#`gySwuaU%85S@)YGIe*!)`Bv|i~ zkSxv2JaXTjgUlD~&ijyUtbGv&5*~QzlgMnXUj50V0|_V(;ciVzv<->V`+ z12-)yDr#|Y5t0qCrGanC3h9uZBwnEYX}6&xx?=(ZpFTsiS2pyhTq6?%eSKxWVO|j& zIxb~kSt$VJdGu%-vi)QD^Ks5-e&&NU_!?jbV_0-N&W@c6Ga#rtmgkf$ovyip^%_=& z4rG0S6bTMGa6Sho^+ZcJ5o~wBGX?(^LTkti?CkI3UcdhC?c2@W-8nuMkmD9!*)f=a zil%()SW$pu=$`H&M(hw!cm8x?TKrD^!1}m&1C~el9h(W1fp4RF4z9!X!j(K z*L%7aqWyy~`@0F##pwtS2AO~Gbi`ro5O{bI52a(YArOFr3XT?_cdkJ)fSdmq2)n37 z{%N%N2N@!PRwvj2h}`Y%&x_UT4BK!au!Qe^QDHk-qvA+kvbt)8nqV^<@CE!#GGTHc z5sexUd8GpB58&Rsq<;h&-OG=92TKiTlYmCue;Nd|$b9YUM^sc)kS;nuJpyLCw!}4r z{?M`p>ZFHsZ4_$O61bt$O{;YH!EF(gr4xrRaEz+?i(nZ5!@G0m4iS^O-Ok_corPWr zF|q2ZDq^k|5W+~*;%iu0twLV`-jTp`hiH9bVggQoqet<;l|K@6(E{b@L|fHMp*%Hta(h$e0QvKxpAB|3aa z^=KLyeU{4XpCSYDm19+O50%2wkuLmg>sRyoP6@ zos)P|QB$+Vc3SApE68796ua;IJ*K2lV~CER6y85Lm@Y96=}3pSQ4~Y~v;oaSXx;-( zECe>#E%BmKqk^g_fVB{PT4#oS#|tgxw~4tMAnVa^Fy$F^_5xwXZIy-~jQWYC%Kvv0>ar zTO@XyfiydSpcW$=@SJRdXemzM6Wck!Kg<|H*|%Ex!A-wx;;6B?Y<{;;Cj7GLKSLeT zm+jvi&E=2(LofZY_CqT&`q2G(BW}SnFAA~i^|8rnC*ZxDONy_$df}g3wyyga8H}Us-5*P^gIRv_-JG>}P-{*j z`y%>ywqK9(h*kTxew)tCFhiP$*G>G>PP7gi>b)37ceaSyq-4m{(Bhtc=YB)n#Fo$= zUtFBfZj&I{+l)XYMAL8~eml(Qd@MV7et+Zjm%yb=R9!{2#zjW6ckFC;B-61tgTLEW zaee1O`=?$_R{=_CHJo+pV-*InqnDaFIQ?lT8q)omxc#E)nyGc}uYxfqsoU5s?Gn7X z<#Pp?5huQ0)kkU7(>$F2_Fe>fzw_9cJEHyzIR=6l!MTaml}`OlUb7$J_Z^K_XLnAL zw|I%mA};Zv+RM17*nR17Vv!l8pM@{t$4wj#i9HKN%>+v6B+dAkj=K|nJ@v7gY*pqO zP8;mJOIpGE1JQRHCx+^nCIts6!mdwOg}&1pieYR`Og&b7LG(oXCfDxE=(ZR#XY#oCKKkC67gC^Zv^Fj3G2kc+ck8emMQn}I@2Oyq}P#1bjgGS z-RhCT&)HJp?wtcGYvcJ5;~!dDMV;l8mg`02#3Nc>f0oI zA~&1#Z5@gyWP`?xFh91~+*VAeSjk|!uqkwT@WFb#@a*dJ@z&;9$wGsFP6u)zo%ldB zf+o8oG=x`o`R2_<+oK6r+Qw;(dy6%TyN=Eo)jP?JaT||y5_+>fjo>?HXqJDID!C=2 zC?B?_!8DHMcwo!qy#LNtk75WpI+h#V_FIixo25_{4|~o3y@w~^__`eC{eYA^t)BgA zpNlWz=JWg>bCd{iJhE67$-+2Zdc8qgc1P2*@wVrIMybIvzYOzqyQlvZ1Zq*}1Gdq` zVZsTdgIkxd^Tby<=ct8P_u>5~Z$GtNTZrmnAKY7j(llR&9B1d5Twdv5Zwvl$i87n> z_4O<$ecx|QMS7uC7z71Ehh^>lD)gT58uM?KIzEKD$=kxFBb)hJ{BNOncG@eUjG)R{ zwbP)3ZG6HL$pKF0h}3b-11v%+Un#ZW2`rI_x0S8DD!Nb7m~;;}>jJV?++9MZ&kHi< z$BN`l*W#xqOU86D`MXD>Rpzso8LtHq0~KBR=?|`D$v&U`LMvHhvcmMa$yfBE$(1(* zWj-UTGy}S97#($q33XNO>!%CSAKwOY6Db$uYo_O0IqX%^wbs)aPS0&u%9f`+GaM51 zD8yhenDeaj&X=N8)Hx|heZTNXZ9mG}dPr+9%nFIY*wM-+Cc;%=TFm=T@$z}t4yXCM zB9(bod9MRWr8B0tj~{34j@k@PQxJQ=loYAdn7Ua0Y~EPWvO0*kdHc?tfwyPFvz~S) zM&2f$&1=q;#0~Ktl`Sv(o?yC%@9Z;xBuqf){DZG$gZIVjYtDj!#T#suUgE@vui%Ve zAaLkATAL7lKv5jTe+<(xl)SPsueT+bbAp z=umP1fQ;u~lF zB-|s)+&J}~z7weJZ!{&#?fH$S-NRMs=zv0lW=+5b`D@(W!PtC#4eOfEyBmw2`RCZ0 zn6AxH7d*CtFP~EUX%8 z*pUkT6hFbsw=7*xw^7dHwvyz2Z+;^jNrYW^Iy9X(jHz0@UYw)YPP5P6J>`@zAt*QEd@5U zZRW8Tjum!{xQ>*0QEj0A*S>^VkM1%-GOJ&l)5{C;>bZro{DtZH=??D*GP9(;y{?3 z`S=zcboefo;i~rix|)Q$N+AXlSo^Yt%A^*u*H66N>rEJyr;km(e%a2oZi&_@x=0#C z-bftFZ~Wq{#Z{=nQ$bDEc8HD~Nokp&P&-rL1H z)WOdA)~%YdswkR*<(jxJ89s3?c~Po5$N91Irmt=$WI11?X_^}4T+itjuwcFRmpCeC zS~7@2&He$JN_v00z&U|^sl-as^44)vs;h}nK>&$0&M4z8syAs?+ z_39=?gal@0{XMZQROLFftkP_zXM`%vpXh=GYGuCvM;A~YBcAbV@mx_ucP*`|26bE? z6_@ML5{%)5j(KgHys)oM$TF`vZBJak%T+ga5Mca%kBF<+xp_EyeOl=B`-)+W*8C}X zz52|+sIBCT;ShO}SV*obE9T45xo4^|UcoV)^X0Ev9aBk#`uTjPE4CsVtBRgyU)~81u^8r_w#wH+v1IXse1BU6Gx9A1^!2{-l$qOLy=(n%@m#gt8A8 z24Yno(S(bEC^z@GW2=zVt{rI7g_%?0&&_Y1#}+>B_; zpwrA$RkQpOfUCo=wQWUf?W4`yh9PSiT=9ZXu+n~QzUfs@qc?v<1&($NgV+1m((?Bb znDmVv0=%_<5z^EZ_nb8BB?I5+Id*>iJiyCA}qq6|8CB z!xLE-X`;n}bW&9!zN-hg*6s^DJ33!XZgw!1Y#kMz^4J-! zCCw9=4H}Jqa8+=VJozr&{E?0p%UpZvO7y;=iHsA&JH5Nd-(o(K@jjGN(U>|LjP&%; zfVY%*-qH4KdDxH9}_f`3sr?=g)2Q5a8Q!~(TT!av&k0^yc7bsB?*+K*`Igv;A11O35*2U5Nt%?k-WNsFin{H zF)lMAd$>7J{379+Z@qAdBjbnHVJCj>jXLzD>$RIZqd^|Qx#*Mk`yKa6n+3hUlQ_v_mc;ixb=LS=9O@xM9&2)f#Wr8s^>23BrEMZq$Wm}2+VhX1& zMYPM6gQhUtP4`yBJ2`r+&aBqIq}c2~@PDmya_;EUsA+>VHTjCyNn|Y3m!48g&gbz- z-F<6rw0}&PLeSHK|L6lji0k~=KdMV!^EFrrv1|#RNrMZ7+}zU znA(?d!c+FijeNmpe;+Hoc|cEYZHc<#tIL7sY?bTpCWa32_;Iw(l$NX$#tYsf*ISL9 z?H?!}Z;^ytk1%vjd4%oz{?-Xg=($VXS-11fgG%ASuZ#R$%&E8j^#S_8L;o5GgUxns zMkf;6o-A41`5nWZS4#_%nVfGwrmptS$TzuTEqihe9OXR8X{pA9wT*cz=KOsbtX}Ko zpn>f4*k0nllRdgtFXu;onQlh8HcWfl&G%^zw+^IEE*9Zg=`n3tn^NJc^La*i+gdiu z-Oe5>{HbwVG*Q3ReM`4M498P6CuNdT`@M5X>i&0soX%!656|imb5EtW+1++Df9?Ytb;Xmva9mx}WB%Z2>Z48!wnAjX!Lr)jF7j*jisQN7); zFrkJ<>T)G&AmCAx+}E}9Fgm)#`+9drYWbR{JAWx!4S%y;@3pt2;Ums4EnYm&WjxuR z&a4Z3s`pG$VV3<5#MRl@-k1V zuR4spOOk{B%eE4hC+{#A4y^W^*U=A?_xTpeHZ(*m_PrdOFPcmL2`mA~^TZd(C z-c(CrF6= za<><@X&8PvRf6gLq6R23VCcK&>P)V7XMF0iKs0&(!~DU(^oz01N`u+Epu*~51H8A< z^77(To9>oKOKUlw9ps;E?KjQ0QihwAf@)2Q;&L+A zU~3?BXtKRuJL*P9m+=i1KMc#HY2Zh>;~)|oB(FaCJ{qp>sZ<=ZLWhtMSv=W#T@q0p zeihNgqz@FCH}ER9t-k?uk1IkWoy8JJ@$nH)uzrc+9x@y8kl~>Nih)bP1>8VXV|#$Y zUM^cGCJMy4Y*eWH^kr*$d&!7G{ro@mQva8YHjv)K7bDr{het{Hw?YnEVFv0;dx#xo z&VVul%Yk?k?)`D7DN;=(Wyoia4dd$ZM$z*6KEkJ8)&`RH@QJ}uc&o)xi&}41t zUF}l_QOdtbP*f=m$iG8QaUH1n^78LXOF4$qt{@h_-81ydrAx>fwJk_ilg?E9j&TLi z^ii{*!D&emh>!*bGh4H5NDF5uXmRzj1>urgynOl1*$)Q$gEot>)1o|JsM^{SkY5MX z3kZ`?9{~f8BGmN$l}|?~+6xSP6Bk1+(y~SUFfGr{}XUQxLdXrI)!Fee2a@Oa&ytaL-ggjww_r0pDUD7{^jsY&2C96$=Yffl5s>yF>nm{Yo<7tN09gSF z+UWZS<_41bEp2TmEFJJkpkV@N1C$mL4wKO5)H8g12!zdV1xJM6*krY%geRE*J-uX_ zyxEP}aC^uEC%qrtyX9gdG^+I-K1 zgJ?2BAyrL|DFW^ep!I^$ggF&(+iYrWH33P|?RQ2n#=v5ogMsoBhl{EJ9Es{M0*c4; zODzbgu#dpUOjzRR8<02Vi~-r5i-@M8vHK>-mc+%ymt(F`^1I|GCzB6<0EjPBI))}% z3WzIZ9sNKH_A@d^AhP_S>raxMIs+}+^2a2rT--N76J<7y0niGMn1Z&%4TLH$ABqof&rC=ILUI>3a;TmsbQ=45n(V$n@bZkGZrU@Mmzb-=Fr2*9r;<&t*2 z2UpCsYlwlqhteVM-T~{qhFfC{q=%P-pN~Ry0eVy{0pT9V8=k~|v<57_3R3qN3d|as z0O?ZSe)sGFiI7*-6TflbKD53@Q2>$mQ5(dBnU;2;$sY|W#>Fx*Q&SrO z?v2_zZvb+>&84p!3RB~kW1cDo0|s(Cpj;d<9RpB*@cc8xHwt?D6!f1&ti%g zD-aMfz*hcRK^o%4BMc%wg{UusT2^9TFUz!1gi#bSlHY0K7rFoI(9y?67bo066!fxdGPJ&qTUX zzzt{`FJGpyJRFZ>H(Xs?tA;^|yRQKoddClC`MP6R9wtg<)_5J2$k3pOuwP%IMYI9T zgoBBRiH&_1xXhpq;ZsC>QD)Xi$jF$5@+X&*nECCJ4#>pAa{%?JVCaF`+1=i5=iorf zZtx6>I>8G!yBH8}lo^om9HyTYbGS7UfHVe&&a($1sE0H`L;y>@OnWR_eRg&h^mF(( zUT0^U>g#`c_F(JafGDH|7C_&F#gCbwP$M8De1LlD(o0O}J!reYYMSd|%1KHVmzDk4 z{*4e1;Z!-2R7wJ%}b08f_!BtIfTuij%1HE%jP0E04rcn*UI zDdehf{7>cs52g9NkISA@lX6)M;t8L!0e=F-$aY}<5OG8(A0dGQVDGo6hFPxZz105@ z_90X-0p%emHzfImDM8*WPdCS>=to1v#ifE$hV=u3W%*-f7^U#k<>3quCi4enmkF~$ z+W_2jI7hDGQPlZfG=zn9gN)?i?G6}FuwU&ktAC$Eu_z&LO@`?8v2sQ#DzuJ%z%Cx4 zmNTLYFwx7+cV8vunqHONOF(38+&7C2+r~ym?aAp8Z6JDq6Gxj%V+@#0!KlUbi51li z&9dCiz*JsG8W2jbvu^>HY<}kmw5ELxIN?!Nef==xu-%_$D&m03BoI$14m1g1=TY_> zkwA_51izc2P@S$3)r9X;!xn`t2WtSh2B5Lb0f`561gH`;m!%w9VN*+i z+{{d~vl*PCWq(k1;ESAvMeaT~SXDb)TU!`VIMhG{39bN}UqVMe6got0;U_wmvq>Bm zfoi>ApK@978bB-y$}N#V3|I+mCxv};%F;1IFaS_8xIm*6L-HKF7~)IkO%+~VDBV#3 zMln-5)@6Cn9LyN-c)(q;LD|UxcwT0NgI9?Rf!q97G)OZ){9VVqo_I zwRHEa;`G!D;84RK*JWcx(?+Tt7l5y=ZllMck?`-XK$v~T=n7#6Rtw_$fM7@i2&q8@ z2ww_LL1}rp@eYpn@h-^bCumc>44{nUs#%(36+pdZ<^w?BhbOpWX}RK2F$h*HA_6vt z`DD!=+->&(>@M89hjMP&SV8ZHBH6rXX4S5`cl#YkcFQ9-Jz}o)_Vn0{S1{lI0a_eT zOT*&G3M1|60j9mK`#b?K=M{5+2oK+sYyC+&W*>ApAi)FfsXd&){{H^(gim2#BiQnx z2kzon&;>trO--=p`I_a+;93Dx4P+A(-+3s1X}q!sWU^*vi@?W(<2Lt9MO5^vm;$G) zUZYRFT9Gbnn;|pKq#t-mz){+$+vy1<;rRLUClG|5+&hGdGft`Ccl|UffvpH?Bv>;r z>^%uwjP&$N;8>!lP0y);d4%xeMKRbBY+N9q+L{sj_u%}ZoQo0}+6Lm0^9?pP8-N1> z2TYE#MJ4bB&W`uSYn&~?9j^|%D+84!t>Mgp2D|wn0;(^<;6Da;7inY;&d1LvdQgVU zvjMro?EWU|Hcn15Q`D`(`CT7U{``v7w-GWkv+Hcaoz?T>%^CI-|%ya#;fGW`DvOd9U z_xlO(5DL+*AWGjSCQ1)DbxRV7DU6+3z`+X*C|W1N5Osj6>YuAgJqdNdU?L++w3(~{ znPQlDhGI_HMS*Ib^99tsFcU_aaRONl#X=whI-qG!(jUGXW=m{}SJvx+DglAEw!{^L z$saK7P_@v`&JM&fxrgsdW%ZJ_N8D?E`#v4MaNbPLo6#OwhO$U?)ujvBqo z_WxBd3I-nYel8Z}9=;;Mp^fcbdYtf7{b;2~k^qucV9)0(wwM^pDg#2Hc>6*ewqlZ2e{Y6h1Z>35oh;A4!;EY_=#J_{2q1_b}M^4X$_9C49eSYl@ zy0^01Ofxm^n%hdpzPHwJddh=f(?d-Kk(7%}>1U?EbgU}e&qq%c$f6kb+Afz5;mR=4^NqB;-N*Xr_S5OIO>fU5<&uS2*6?UfJ2b=& zN)T}WS)b=T0n{^F9-qdpU_0;K5=ehRH(1>m#_`S8UV{1^OCKF>;~$gd@1gH3OJAGa z2pjrGHk@VR06B)`qBM2=;LDzb)W>}a`nmd>&TRF{P#A4{naCSm94pIxb`@c8_4sZ^ zTtX5(XAMgqYXSk)f$aZ6iFmnIIYw1xE%+$zP2lhoFONGIQ@<`K0dMq9trZ2~XNi?)awXttp>`N*v z=GhOUabhW~yw1<)_xv<{j1KKcI4-m2hm`Us*R{h3w((qM*=r;;U;&?(`j2 z>odEd&6IUq)fgA3L@4WiG0ah&_4t6UmudOISK>otn5Iu?&-A0wplOexbpug+&jsG? zlr`hM;U-SkhCLz$e6i84L!}pUCA_!Hh zcUoS(Gu`x;Hj9OdnvN~g8?pAu8qpAFKpo9djKqXIj)ByS~5dveAK35G5 zd@iMsT;D*=enw^Qc%@>`tb$JiD~+ebLmB1XZxm7=&Kv)n+Osq@Tv>0lo2Jr=M4NPK0(@xXT;gdABMxU(NB zrl>-NvZ+OSug?lDG4_wtIC!$zAoR1$y#E2`hT(EtJ!u8o_?u0qL1|eH=Mp;L^Se1;?wiWE4@qeXT2Ap+N2K(r=HS_oQ&3%IM?R9 znBJ4(>0HRT7mz}!qx@(iY5Vli24CuWhGuVAEQgnfr1mSSUXPzcrwNqw;dOs{h|o_z z=A@aspH2+bU^0uJM->+8HkYF}*4LJm@%s|DKi_w_EHib*3uLo? zDxTL@)1w#u@yf57t_RS#sBtkDCtzVUUBMFXSuJkZ=z9U z>fSKTQ{!SRqq&v;;sTiO=LbZgaP((X7boME$*-dmgIh0R@?QSZSKr0bDJ!SUU>HQs zp@=Y8hZMGyc@870{|hAqdvDFea3w&tQC|R`Cx-q;Zt>c;XM!QuXGz{{J=eOpCElo{ zxRf$lK#$c}WQ#2K4JA`1x^w60ZqgoNu;`L%9hBA7ypY`8_-ZWaP=AP)HUwX7xX0bm zb##-pvf!?P1p7tt%l|!BqYxBK6yman4b)0%mZVpxr^y;)^-u>G? zhgxN|(|9>+r&Jc>zO53zcN;$H-c!rL7t_co5jzuG;`g0Cc;-17?^EKrwL)ea#gBI; z=d``nW;`F9t&#I)s9x{~lDeo<1DAMAj#}9FSTQVQb(!C^$7H7lk7O|D!81<#**iEo z>_$Zm4|{jcjL>BVoVNCJ3?9T-d2okZ5h#|(Gr3!ZP9Cdorjve6{gpqjmxz(%kI-`7bxG0I zNbo#q_SZ|l$|p#6Xtr*BYPKfqPD}5rvd#UNWxn^FQbJuG_f|=;LW9V(vGEvI5Mhe}nn_vnLj&6v9f1We3zr9HtJALY zK3p2lss(e-wvpiB-7p4Xfh1- zzv&_y{+cf_={F$Cx^sf3jwjbA(Xf{DirHg1ZIr%Cj4qNqO|BiSs?q<&I^ulM$|ftq z9)%W?$|lmAO9_jOt@6!r1lw`)N2Z!1x9d?k^cREWo}pxjxQF#wOF+)WkQkZR$}&$F z!oRC`7@*$w^3aS@TI7vJKX#zb--7g>6Fd~*aLWz%+c>3~?p0R){czgUl9W2G)GRJJ zZpnN)k%&xF#*32Y+{3Tc_AKn@dne9(-imc6zA4M1<8l!@Y$qDySHI4H-p3rJ6`@P5$lJ#_`HdeTYO}?BBX^QdCozZjKaAZJ?g3^xb)z1RtLsn15(fL@QJ& ztg;b}R6ZNjl63cpuk^beak}-bRCv&+fQ&i)$Hzs@_448GRVY*bvysS^Kt6`bw|@=2 zphumjHTd>nVO9R*OxH}|)t5yCv6l5G`5lvwe|@~Yd&-u%J)2&|t71SGic_Bbs$S?U zdwDvqFn^U9|B$K3p|>O@edx^xQZPDS=_Ql-67MP*7h5@G_RCvJjcjzTFx228X^Amf zlm~$*Mo9LxdUoxxZE8RkSHw^l;nrB4i@eA{Lg+=z`9nM!exRJTNy{*+9aSTW76hZ#ldoOzS+|LKAfCQmRO` zlHTL{zk^>5ccg~cJL=3h!#&*{?s19fGSEl8&d}v8Z7TtWpR!$i*_aD;i1(H?H|FCHO z@u5J*CHsQci}m8$8fRlNuii7)4(Z2nG!=vtW4;Q!=?9qu^f&VSpPrDi{NXC{P8BNb z&5iqS-b>JkGuj}HA*#yjAJ@g<%7VhQ?Xlgpg>Q0QZtv4*ShwGOn)aL)OOC6xaO=&; z2-@>3mKe5S#UU3gOAOq<^;C8!(21bfCnr^IJ{gfHgy!{6B7=dvuuQFrR33}>Q0F!W z>O#tz3;6<$lEGJTGn-S3;+@Jw1!XM?wvya@&(~rf@$ib7$0ua{Ft|l@7U9miY^z|! ztuH=0qN(&sUX^I(c@qOs92$h}a^S#T#Y zBNCntq*cYgx6={1vlnQ1!a@*#EH4eWX|uPDZyQ=RzF;ULBQ8Ih>rR?+-~zLH^nqv6 z(n&GbUqPVLQR$L8(}ibs-4kc8G&)t-Tk?bR;uDMM3nt3QSi9f3GyCzSyr;22*k#u0 z`sHaT_RM}{I^Ei93p8(UL(52KV`PAbs=vVSb-mOY-;+(eqKGA%=pENxgLiVKjSt^N zq^?n`)EB0X`)U@b4!p`jYLY_q#(2-T`lZtPV8_j{ia*XG#Va?M0*!wCpKO`C*t;XE z=s@(caljXAcdX5)XZ7{j7TI!xOIc^PFw@E^*Xbz4-fCvBzx>YtyIW6=4FoVD}zt%##f4V<_43E_ z8_N5xJd3B}l{;qRO__Pk*8qaYa)0%HJFhLWl<_I7>M;!&smPfKVwQ!xl>BSBYs~FJ zo;34)=TYna0}C%mPd9)2hnV(gX9xm*W~tsWyG0kQnu8RdAjm+{grM$NPWQ1}*H$%V zIZuk(IT-Y+3}25PzM$`Psc(H%zki-By!!lDjj#8(`GE4=M@bg*Px;Cs%e@w&UY&xw zD$&YeB7JF=rqU^EpSvS01)2-!$M~ur-A}j`GKL6?$zfpt$I39kNc#6w8nawv@~EG% zHPeu%f@z3B@fBlZLoss-;h#h0CELCL&VRMxSsnj;eGs|2cLP=4^*CaA&&}aGzEWbC8Fq?* zk>rnq5syldQ6mN3iF_D@sN;!>Q_bu)C!XuGprYzuOWndgKGs9ATB?Txt>QtxzL~Ae zWBqaZ(!uFWPhGj0CNX+DI`g(LZU0HqRPcmF@bVffUF}-kG z>DJq@$j1mHGm*eVTKsq=60^J_%;IJ{-m?10Si|LKYo+i=jsyky&!JHbnc#@ieaHU5 zY=-hvNzIvr>Vd{u-8~b($;BoA9w(Ds#z;bIUS}S?G{u;|pUKCo4wLl?zpBfvYNRA; znx|MLwa=_}JCe*zM}MaAs-KN&`*B^jC*vVn|BUK}fZ95n*>kyi3YS@aK3@5sJGB+s z?H|a@Ne%19jFdEhg)rA9?7P?bYW(&5&-7Y}#_&_l;?QZxp%!|HV$zQtwZ7ey<0hBv zW@1odOwCFiuisHF6jxHzmbxQK_vVk`wYNw$UoXee^+DPj#^$ZlD|&kPkxHZKt8HCz zi^{hfAMXY-OY%5KMdg;%o#fCSoYpyc{HB#u+3+VW`hwGkioj$gxrLs`yb|b|yba!WhPuv~jXP7D_{!eOvcK6q zx2*6QHZ$ce++$|Mk{ed$gIH)yh_|19SaZXFILP9V?sGWs9M)n!jZGI)Rjec2oGV85 zKGgKTd+5uPM~&G(w}I=L<1k-z>|o$*;@1CW%ObM4gn{tejf8Cjoe+HjE8N`brqDe< zRwsA!loa(-gXo&9!}Z8)!Nw$Y>jMAx%hd_XEEe9x1-~94(G4XZar6HDX)r!FxFmc{ zefzHIMh@dG0!N2(49bkSkc46}cAHNf5s53R#v;^d!+ExHC3y+jVH;aEio6A~5kC$z z-E!+@EYT-pQdIi$$Bib5EhDXskprt5=s7w<7K6Pc<<#g#q5iUseYrctV!|Qi=QL^_ zX8!zo`j)2MrC4K8N?fWgsE`*UFT~>*eKzedzBL7|V?>!-j5Q4$Ryv4#cRG@{Ff&^I zMk)JvC=mI0$IlY;e7VwrZ2J8*iJ0@L)A^r=96~aL;b$H@{ZlbxzO$ZPN6y~`DpxF< zsayr}9lj~MA59oP{Nvf<&Gjj5_zClw+WOStU(BMRbPK~P{~Mx{ij_W>T}#8+bl2X- z#F(Ph^e^BE`JTIed6@PrpM!lg?EJWCzvcF{3%96$X6eDmK?ThzwY|rAgu9IrPm&Nx zz149ah4os?1G3>lv@DiMV)DI!j&17TV_Y&bSvT@FIWLhXm_?ISO;e%|7I}a4`kx^^ zLA#(Sd#rklwDM&&wM_e!;=rkEU6R8D4E2NU;@GNI z{97$MX?Lu=yG`=FZhSO^IsY#IF?Py!&p|C-;ayFx?qv2$b6(6sEZu)oq~!a5)2+Cm#u;G5dn7wW|HCb}7f;bzpG8(0k{gh&2dwOq_ritxI#JW@$?aw-zR+Ci%(rkg27! z-6BfvY3rtwZEZ^UdJEYjjPJ;;lFuh4pmYu%b!3v)R}p|ywNDjkbUl?Hi%9wXv}JRB-)6C z)S5Xsyl&0-Pw~C9w<}K)2nU>}{$7*C)ePU3PTjv<8DH_Rv@r0>EJpcva$B}4K!LOl z>#-1M76c{S4PU-j`mPGQGH3kwIq5psC6?;VmT9<$450Uqw(PaH_1+oitWvdX9YLC4 z`HP?29lvF1%P{0Xj$U@p%%RT#(_6Sfe&61Z8`8b_vMa@^PySmfYGzLt91J(K;o!{k z-9#nM&4LUJ4Zk-x?L?AD~R_S4JS~N4|<1s6X#Qa z5wza`SA@ro`SIhp73c)irx`B1C3=bN-viJf5H7*ONNVDhsq;6i8U_wUEC^slpIccm z4f~@$LkmUmxNQF~kkwGVJ#Vb%kBC@0zIg1jzWS~`V5CHyG}FvF^ZG4KP3e#a?_TxI z3>W39N=ZjE6s(VF=6n>FqqksWDBDlRFQ?vDr8bI}F`X#aouMOX$FJhAJ}8B}tBa(SwGTq|ur z+rhj)R%x5M2URIa1>EdFZGTsV0)e2r4Ylk1917qzBS71_4`zUd=6Eg)H)oah2}0ECP-)|@fC5tp(qf(ucr8g+FW(?vZh-`v zT~g$fMR^V2iNJGlwNe2{kPuZiG3hB3Z`|0Xf*x0*>M}ll14}uY;Uq*WQ*jR{jK!!& zBfABUW`_$*AK()Jsf#o$3pNBm$Kg<5A2EWeZ|XPjHaNHh$^eFB0%RPHx_fp+YIYGA zpsE0eqU%`!>{e7{`)wF1aLmlkn#-Md=GTu}4J|p9FeS=lL9O*S!%scOV!!Vrz3>e1!@juX*E)ija&<%WM{&Y-}K9zbyOAdxQcJr5dWu)Zsc3%M5q?#ocT+;~YD` zg`i^w7#;=l-?-u6bl;DvK(A5v<7a?pd23)JuPe2*w`g17HksQE;@uQ=S{?cI>mhXT z1U2?{G^9AwlQ0A%>;GN>>=$r({i)p}AcX*FH@xsF!b%KA8sQiD4oaMZy55NZq0jh# zwI#A&a*{%U=es&6o1%{@%bR_Rf}+84m_Ldz?eb%&sV_czrFkg?d-+|C8}%` z0}LF@&y!-~b;@aVMq{Ouy0VAge-S`QsP{#^zGL^QU$tb-`?h(Q4|u+0J`>!th^96t z3!hBX*c{-g#4XA$;&3u_C^mVi$fd zh2SY><}2xMBY=((zlBh17C$lT=rfgxMhtVduw{m$6T6u7Xt9g*$K^kZIM;IsHDbEk zgxr)PD_KKHIf=gH6NW<7emMwG9dGv0}2k(zk%r}?l%*ix@m zP)dd0qqf6gEwSfCpC^Ix%nPmE7<9yeVD=;{!tMf&Ca5Fp=tN+r>#*H;VS-s{LOym> zg@3+BtM{l!)4pK2R6))?%HDcDxX`$<{Y`(RH`m6OmS34oB>45lR}hCMm`&!yVXc6n zD<;G_FEkTa>y7U}{Ww4Kfy$7!Xi4Ds&D$ld~?5HJiBx3Z#egLX^n;ynREytqjYp^KX%i63!O#nE`ofYpQ?l_zEO?DySD)U3kgYUaK$7wp&z9 z>D1vk2($bgO)WXU(;tI<)EclJlz;yRx_G)KYhn1&LuB48$2S&qpLeyrqL>3?{?RHI zI4@Yds;rjZ&Cj0vGhDP@IINbxrQd0)n^JtFy?@hwWjgiR%0tPI6aPnT=NZ<-w(jwu zTM!$H2atYHkxi2=NGBkuAc7#B03jmM5+H)~7PpG1^e7-mQ96OpA@qnK9U>hm0i=c! zAe0b7k~4Vj`EZ|mKin^OKF#w!)7F}qHM4$e{ons(tL(@fMM9vHvHl#ZOQ}M*`=D=) z#Ff$ptshE=z2wLxt!0W+xj&NiD()hF)j7`lHJVRo&ZDmM8F7C}t;&67&f$kezFmD_ zl30d^Mn*||YQQEPcTj~~TU}WBWnZEFkPr{sXtGp`kTO)))BN18p(1siAcI92b1LOa z>s`9&WE*)COWyb*?~@vOV|`;7NrjV_7VLMM;1=jLgxhBs-oLB=u0>` z4)#&({EAI2aZV0}(f}L3%&acTwB!X%O4M+l{2D4tBQUN!W;H#!p!3EykI*;yaJPUp zymg+W?uavbW1!@;A(Gpkp{Y(UM7U(E?mJ(+T_9R2Q^~Fj^c5+)3L9a*aN~C}ug?i)J@b4{G+e`_B z8(DAu$S8K=V%$voNpEpEuFMJw>t5W7tve-lXC?hDR2`3#TTRlyA&8M+v<`!^@v%F1 zT4Fb)lJZ8MS0|M|1+ex;&TeYRRIe?!&g8giv00MwxtvAAFB5q1DZ^&o&IuGTt=MBK z*W&L>XK{79Eqs0F{3n>scuxGaPcaWro?~5=d!pCV zheoEVp=8sz267{Al$AwU*9;siSS97oZ+kt*>Vz#(p;AaKQ zyKn*T(!0Ug^1l_{N>xq+8grwXH0wUlX66>~_D&$EP>n^5B&dV1BGDeXEOAEClu;!uW||C9-3I z2N{)T273+kP|THt>R|)YtJk@1(;a%|ThfW{gj$cL1#9;n?Ly;{{BxewJjpHIK}5Yz zE78!f8J%)TMq9G*vxsVnDW{1~9*@q+?t(GVqW)E*Ybl1KtSI4udoG&QJ|t$*cgTFA z#ZcF==k$}!f?q>>QZVqHF*-D4HPE$Ez$3cOH^P6*H>n4|QU4UWFuYM~>99Kmu0q?m zoOB%s0P?J@94vs74Cn%#-w+*PW)^wPAM-Huy8Oyw^1Avc0+R?xA(F#JNefvN&QDd# z!_Vh3_SXAlL)Okw->9Cpce8%2jP4q%cNkQhrL9R!H;v4)$u$;d!e(Y53b6dF`?TG> z^-i4*O!B}S5G;Oq*CWm(w zdW$EFOOzUXs?<0~&Ymr6UY`GbzlvL6u&>naq*YCR8@U84F*1>=>^qFPEP|^ZePG{s3V)L;}M( zxYFgzT$nEn{0p*oKh`>`p9om-u|KOLagM7Wa?vQYay^E67h8k|aw;!*z3rR^<^r4* z)xV={{uq?{`_39#+{nJmbGwNTTd-6Dd4l-W(Alk{+xogeK#&4AoUn4Fml*VX7%Ub~ z@Y}pz`P$vo0)JnX-i|K(g4XTePbTdp{4?-X18#p^uY^I_=EQ&5#>>tac4)={No_JWW%Uf-dd6>> z(A(Bqn9A^h4<(Mc^m{&E8@t`tx!Cy{&wJazQy5?-@QJ^!dOTkoS{NIX^lJlR177j$ z(aZBW9rLlqwOe%SZNZ`)p}p&-%XI7PAuMk&9I-iQ93bVl{sv=YjepcqrD&jZjZ^GS z8`9vh@f-FFp|&J<8H&<_TZE3!r@mubO72Fkuza~Y3xgCJc7T>ry@VdmdiYGsOqvTbWd<9l$l>^G%WntHi$yU-Ra9K&xQBO z_-IvZ3Y5DkV(jR<=r~{?iTuweYPDW-UhidOmT*}0k9H$CMKn;!@jjQk%MYiA#3iTI zzbAht4&RmSB$2{}#{Ue5Li1O)FcO@R9e@0t6#@#XX4l){1f;EI; zel-6KA9?t0UT~sKH)dj%cW*dz`T2XZ11?2g@A`H0prDVAQGEg$HD27rOt45swXOI1 ziq7qXdU43V>T55lv&j#E)7|cNHPvW&PDjy3n!tM7a?;&Dr^E1Q1+UedQ2KT2VNIqc z{0j!sZFR*{c%L!0ZqgtbZrIE&0AmSAttLcTQ5d{A%BMSXoGbDU8vdfB$t9BaTU1KF zF$<8P!SVz){)GGD`rf%YkMg91tM@p%7b2}>(9P{77>4?(^y8L!ovQL-f`hj_P+#s- z^VQ#9G|p(cZWZF(qi2FW-XQI2q4Orn5x>?mEIY*H(tLEjKs{_SbDmL~gPF=w3%-i9 zD_^wfl8dxzMTo2ln!r4s#n)cTHZ2t54vVybfSzgr&(%`_-K-m8V%jvCtF2NADc;L# z@vueEXxgx+)TW0&Nsd+FxHQZ8EJmnH7;Ub`W!34e>ENgnq)VJaCRp7SO-uFJ-~VM+ zK(a&(Rc4MvKMEg?JmEUZ`cDcJa1KX|<0CGMV`YS88o7A4{I(d_!65^KQ$mQhw$O`hz7 zp6uM-D#k)(+Fcq{9&i0E?Y3AcOk>#;?7namtS__ zs_&Y$mF(4$y)+R8>zLk4N`4Fds0WKfr-K*I2=hFvy7c!w`onEKyCN&5MmlmYM?P}i zNe(QcI9o5PCwI-ox<~n97_8QPRIyM|eEM$vN|!~$)otQJ4yQdiPU$TYUP3zC46wjuZsXe*%wm=_y;hk|S`~6$l>r#z?`H=+WQ^A8Nbsk&e5}8TW z{OL=ui7h}3xagT!7TV0^fmD7&a%*_Z+`y(ym((#VecUs|>1|Q=<6&)`g_fV-9U7>b z>B1W(h?42_FSKv<$oWq?7Y5EfIJfn%_Jq-RQVMujVLQm&Oi&O|RTi)cQSl2a06bnQBk)`!#gl$BW1?_@r?Ke@liLvIh>5v?=^E| z2Fvk1(*HQzf7haxg|VegOTu|24TAYJAHoDxXisuwx+&4&Q>QEAq%`)LzeH&$lwT{3 z>I!|PTs7Tot3VkxolI~FM#U-o(j7KK2t-e~R3mwF?$wGUL@x9URjXMfXlo^V<5McP za_UkhldKSMs^}EH+Gb)Wt%cW`M6wU4&h;rpUCZdqVCO z7jbvgRoHaq9S41X7-iSPj|BaB%iE2w-^ps;~IjWsI;9d3UosvRz2wmGTH@H@T;EX0#)lAIkw(0=mcP4s`y$)$q|f*%B3EUQUu!CMwM_#} zb#Oqt#1f)2cDT-M7JrsKq|7`nmI*$1*`ZNR&QmaL`LU^YpCmH%{qxh>U?n--M|UxTUue z7Y3?f*0MBb?-ZA${08wc)9g8(-@PTV&b>*IYcev1Cjk(~Q^)}#-$Q2u$u2iHpX%{> z6FN0iz65qtuT!l9#>Q}UbJR(Nuu!Z+J*iIFPc#f4u6uHOZ#l@g2XolhJl&>f9%k-3 zu(Bk$V?QdkH=mg<@{UlX()HNMX)UPHtNv$IuIR*HLo9ohTNx#K3y)O~!U~;{ut0~w zX_#j5SF^pYpNYP)Z-MNogL$z~dKrKe-^E96gv*(OZs}VSJP&BEuDk98&&G7J>vzLK z{aSl`Bd!zWU2^R;FR2I3Pbz#{dK>dS@Ur#WKp$bn23oIZlXXCDDj2V3%U>izJ6PdA zO*r5ZQvV7u!$pI;Qi43Eip2Hg6@5jEvhKm2?Eoj=rc#$wWIz(XY7iYD(85cpnO$h^ zsIW3hbz$SISspe;Z4J+N)pu89P_L@KWw4hg*KLQQCl@k7t1wp5QeN$RcTRHF^8*aB z*rK3mEm=npo+z#{4)d$I9Aw^h(5z{Uq8snU^CVb}vVjv+TW6MKS499F_eaGZ!GB}m z8S8Y1L6qShD}-r`X9bFVAI9lNG-VM(D_mqF(mb&H`#bM*Niiw)EpDG08nxCv^sG!S zSMUC*KTu=P`T3gzm5`o?F2MU+bTCeM!%i0KobA{vzz@7^gIHi|G0AmS^)uyHb6m*T3l}e zp4KXLr+~;;&7;T_4UG^$_;SQ#0W|*Go(NbuaRjP?<>C1w^bImqBSni}=AUw;Wi10C0pGypFHuN(@A<9JRnmfG1EPkR*}&M6k9 zoAr1X^i^0Nef3qT4Hm+Z+?jgeY8-q7q^W+vpg$b}vYfibK68^C_W4I~%bVYa0VsDa z81w-J)YH*r{VvSW%6}`}|1pqVMm-Lgt(-f3+AQ-QNBe-fYJt4M9PI4lyBFv$`~fK^ zbp3bfYm)u|f`ZZFkVGEL?-PUmYCEcr|4~8z-+>>j<9@+rIB?c!V4MI1(!XnTr&#CF Gi+=&+)F*8K literal 0 HcmV?d00001 diff --git a/docs/src/assets/images/transformations-getindex-with-dist.dot b/docs/src/assets/images/transformations-getindex-with-dist.dot new file mode 100644 index 000000000..8c6826e3f --- /dev/null +++ b/docs/src/assets/images/transformations-getindex-with-dist.dot @@ -0,0 +1,20 @@ +digraph { + # `getindex` block + subgraph cluster_getindex { + label = "getindex"; + fontname = "Courier"; + + getindex [shape=box, label=< x = getindex(varinfo, @varname(x), Normal()) >, fontname="Courier"]; + iflinked_getindex [label=< if istrans(varinfo, varname) >, fontname="Courier"]; + without_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + return_getindex [shape=box, label=< return f(getindex_internal(varinfo, varname)) >, style=dashed, fontname="Courier"]; + + getindex -> iflinked_getindex; + iflinked_getindex -> without_linking_getindex [label=< false>, fontname="Courier"]; + iflinked_getindex -> with_linking_getindex [label=< true>, fontname="Courier"]; + without_linking_getindex -> return_getindex; + with_linking_getindex -> return_getindex; + } +} + diff --git a/docs/src/assets/images/transformations-getindex-with-dist.dot.png b/docs/src/assets/images/transformations-getindex-with-dist.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..621b10a51498ec87050960725b6806bab2913416 GIT binary patch literal 42305 zcmd432UwF^_$L_kDvAOZrAU`19i%q_l_tFgBs7s8dhZ|>kgoJ1ARPh;y@sM9(rXBz zg(5xl4gtb^+&i=LpJ#VxXZG3M+0Vx(C;9T7^Pab#-}{CLZB1noVp?Jl2t=atQb894 z`fCCNx`w=c3;2Yc{P6=D+r)>M9ltiSvyrq+Y>-V>n#FR)ir?=i8Yl{Y0N{EnRPBrWcL@5P z&A?3e?|6>+h1kEN@A4lU{~f&zzYT2RpNovV>e{~}Mi#3Z|Bjw2bY1NYxO_i-Ci!=? z^Y{P!gjN1~^C}6C?Jw|$OJJ&h=lSliJdkO;Jm^0f?*x2~KV8X|Ilba7aD6W&Ws)r` zZ^ifIZSNGbXRvu~PNMS?$^N~&Q&?Qg8S_x^XWsT}_5YbI72m6(a&SylYz->JeJXtu z{4@GeU;K<_kBO=vI`Z@mz`0(ktIa)Ae3=2CIwv_L!<$6R@ZSZN<^GDEx(U3#Wt$Df zKOHU@Y%wJT-X`AR`)~O7wl|r_89vj93{N+>tN-kkyytN0x2 zSAVKruj51WzL8ISjqPKD2j{}i;fEs70rE^^1>xY5Etf4G)knk-Wp78sfMD4f~#$EK+;yL4c@)eKC%*FW02A*6!Sk_H zj;+S5waG)x65Gt0-pHaE#I?{qMFSs<&J3V9Y&+#lV4m5=>%q*FCWFDF^>XpLf)(TX z&?2IgAZ)bup|4-E8P*fd$RuZAI(4o^R$rrrc-7yu!XHGBdJ%FN40A5+pLM+k`o0uV z`UnJarE%vv+Fuc|_h@>@vW@BM?@UZiD$E;ZD#Z*aLbfG*jPxR+vPWA`M?H3ekc5;i z1Cu8C*ip+#sco9Wd(!zd4`zm%cvX^>{2iT=)lrFQQfl(&Y;B?0Tt7rSR?;LR{n}C- zx3-Va{V1(o>vSPu>a|$yh3iY+Q1elYyG*k}8;2DaPjG$imOVStGR0^w@7#Ydu)ugSFGnjyo9;>w&X z=DLZwOg8+ZZrR@c?nj<7oZ6I_liluiC9P9w9y*a*9*27AN9uB- zcdC0CXH4YHeP7CK@H5PuAxGG{qcJ_(c}Z(1T{6ueQ+Z?Mu0h89Ce!U%-kIadC=NQ5x{dn z&`(#02X2>_(6+{>Wx+i4;@4B(kgmK8-5f1R;+rjcFXhPqZ-3Kn0*w)yr1^0_x{sY=3m`2?^F{vn{?y%sAjLXS$yeuVzn%> z)SP-Jv7nB2v{tpmxW>Ye;K>VrI@Yf~+GDG%=Ds9i$|w#Ki|fe;;R+O z|6Nh=rzcj!uy?jmS#K*yU%39Ob{MNXhnNQDy;ev8>hwjWHuee52CR0b^dL+QPU8@) zXOb>2NwX_ev(LeIzQYwmztcQM;kdb^2S)!SkrW$1Mfbb)G{2%rBuAKCG<6L&s_AWo zDBbnWGDv1MRnC)7MFMZo^_^)ZU+lBQksnNI|~lp#5)$APVP~!G5A7)* zuR>b(u=V~yjy|F#IDxj$_^Y}3(gUl1-lfX>ux|617n6`j$f>Oh_Fs=GqFBp5z9|d! zU1<~DL`Y2v`$cc1G4M#mUI)nhVv!|&-MO2DPE0*s#Aj>To#$`BeTZ9M9{AL5oMvrk z5z00Nd_EukWCDWv_iPh={Y0hchi-)TxvHVbK=7XvO3L?(o6c1(;vcTdD{)`UUq2b+ z)bMpoCulPqZcRnLy$bj2ziU~qSd*qIz0)$2*{r=i6Efh~t}IH_R>a|t_k&vjUpm>( zVZU-(>L2aL17=q*&RlfEmB@*ANVIEzV@3WjdDKFeGojzX_kr*J=f$g*W-Hu@;{v7l-2{`KLyzyPUe7Z@|xiX@}ZN2COobbT<>h^LVW|o z9{?m%bJpu)aMUMq)@IXtAkYqnsOjvt0>xLpP3~u#}UVOFqWivO9;3(_Q=PI$zdR3?VsJ2m-JR-FH(c% zwU(Q-8Bz*onvFU^Z+qSXKE~Bgg-Ovnx1!2#{6`Oy_AtBsGqPm@Neaj(efs4?;gt z;j+ui^dXXo<-G2}**W%dh5l<8p6t6AH+vl1eN+(iJ>V)m=8WBb7L>KBJGOVGzoL6) zh*BSk7oR~be>OfGN)1c8%>DKb$Kupqj~DFD6(b8NZ(q|)E9B}#7>*7@g4^aLQ%GG- zjC?dE%QS+$ye?bBaqF0{)za|EQU|=O%;}5(#B{=#g)pK2eb+*trkLor*l3QHIhwkX za#W;FBKh^lTUkTRAp#>MYL8QDx%WC4+*1qHSAtuDVpb6m{}(XR1Q19pn#bC zCbe@J*M7K$Z_W7iu z=M=B^b?gq)X=Um%!d_X5cy-uIHk%Hqz3iZ8^7f*YQRNdd#4W5zdkst5Ldr+FzQB`? zup<2t%ph*Re<(2dhk~z$@u0VR7LV-@A2=9P|Vppe)x&NDNbE#>WHZ0zVAEm#71`(SpsPJu3kP_zO4}Pmn5!f?L)uJh zh_in%QHXlB#P#^0tN}X2bxHD?VsdLsDU5d|dRK2$;3QI_d7Sh&pR*! zv_rn#7ns=qkJg)3{0WvujBn`qhrCJjZ?4f;AuG4Wt@bQ6ZGE}vAM9|H=8p8-`_6{9uYc8J0%5#AnQ9JgspSrrt-9tMl3t#jq%!-c4l#iO z5D&8D;Bw(3!x%piRUcjz?U9_>!=6Q#(aYU(PgE3-h`ptI=ta&4>n_PM>c_T#M@lXzxtUXg8;|?YURu1P#mBja zOEaF7a*H!JW68({;A3OrB_XVO(#uNrXNpY6gD@(~OPE8g0ZJD645SYWYW=IzW9&MX z(Gg9#AUBdo&Zo~_(~4I&$ver5I(%6_Nlc6=-HHF$xThl*){V_?*9LXZ}>E6fVToO%|N3%sD>OneNSn z`2>}gjU-FTKm?{e^9Zs*F-Mu2hDsyM@QmQ{TXIJ&@x(I|jj;~HCk~#QSQqHb;F+h8 zsG5mbpdB?Z zOCh>WEy;4rUj5RQCO)zDRh(H!y%|2RL~Kh2w#XYkvOs3v>yy12k%3$<-(QHlou}Rr za$Mp35FbzFC|UL^F##XcZ=3#P*CCVN(jT1L#2NE72ygYuha||YE*LMPSz|{%n7D9$ z0+U1_E^cVnnRG06kWlKSnE9=@dC+{-SIkQYM780-wp@`)28xoxug%6;wy>s5gdv9Ts>Pds1RvNBRcGf9 z);-po;>qi<(=cl?c3UHr)Qz)WZ?+v3-uP^z?#p`ocx2>d9K1i?K3z!9*8WrWqWF$5 zd#cUWQ(&6!*V_pe82A#3k$z~b(WZHgo zv#Q0qEf_haTmok|*tdlbVEl&~c~m{>n)GeE9v;L46LH5cGg<7?Y{C)b`> zr)zd-HfnmanpjfXW=$~d#dRk)FS1|kBlW6A8S=$bHuVB+lH)DM93~r*_tav>=mrOd z=82JUVqQD-5*^`V+AEMSwiLpvMuP{fTHrGa@zo4NqSZr(paWF$Uy>Sn{4~(X%1~Om zHAt-_zj@cD6l3Xzx_vTe7B=CTd00ksi80|TTZ7O9 z@tWItANA_iT20(Rw-9JGvVAiQnq?F`n6~nb*QW+1FzO65{t7mOjOiDXPvE|_ZD!4n z^$iTGl3g?ruOJ;oIxwE;F(a+nQr=M1 zk?tkUPY)$9$;rAlIy$YuewI@9vw7O_>0QRb!YXJqy57w`^3?qURLTd6?w*IxrWCdd zE5iyM+Qb}t96BJsJycba@?-b8G+O)`W3PeCW^HE3|5b4=d~YEljRj ziJ4Vyc!jJ^pDXN>w=|tL#j*R^!9m7Szghf3lWZRLEH{0EejR(LVu1(-_w>hBIQH-NDFA1Gh~-$ zkJQ-WOH@)#k9L-6@jvBp7Cvh`GqWDqPi%iXEFBN`9N3o-6}2LA;H-Orpc{d-@I#n(V}Jf@obXPq`Z`{H?M!ONgOp2 zj5oh2>-oK)-OXP2P#azS(*2kHR-`X@ow1~?mRR>&KjL@hq(fkUoX4Kktd%1 z0nuWO+uIr!t&&Paea{ZJ9=aVh)QpYIhGCMdx1C#_tkE5Y1S0&kUKI;wZww{z4;cRZD47|n@-7|HFs zVpWYgjn;Zx(}m8@7mf8(ts7soEBRnN$au%f3RGyHu<7>@6svQORQt2XHZupuzF4kE z)B0{cfFx=E+O&$z+YziN)9_p`b|&V{yNSeq z-1H9_Chws4QMHg=swQ3$Vo+7_>E`%e_)OlNoegRZi$gd*^d2yC(u)eq**o3r%O@|M zVPFy2EM>)8#K{|)TPJ)iI&4p-DDl6G9h%$t-7VQ__~5Y%YV3gJ=i{Q%9XBNJ58QB0 z`gtT@a}Mqhws==91|$crRJHMr71H49;tZlWzbF%8qwy|1zsL1xwx$F{nxbyl^hQ(4 zy6L{iMSr=7Y5ahvp$L0gAzM2lL8VZ?{xd@^TW4pQ7jJJPdT`b-z9jk}I+6)h{-o|D z*2_m(!{0gGW{U|Cb~-9@4~a=hrGy!ZLG7cJ(bLJFtYwQ5 z^&{3#%0@;rFzXLNbCgM&N5mj)ovL8WNI^iNwdz1>5~NDBv5%TSDfQS`B~4Q@v(PHt zT`IFtAVrPe&{?g?gw)zRv6GLs>TL3^CyB7tLn#z&P)1E?pe;k%c&3@g+N6GDw9@76 zdCxcjhP@DNWu?=pBfVN5mfuE{6-p{uh@V0UDqWaxO3)4SM=5CN1ifZR$a}G1y zkq{}ur(xL1G$LPPI%BGYuk;8S5C(?(d!xe#vANa>VGf&%*nZ=*% zl}3CkXvLX()zQ{Z_F|daKQB$M2~Txdr^l-U5l)CbrME9AtPrjXpw1dyWpZnSwrbkBgm<3wJqB@DerY?Z_{APoUa(K?M(eE_w~Ve!|cYh zNYqli@=9Vnw+2JDPP+eoFvAT{b0~1Y=O?nsHINln9Yow2sfe>(oqyc6o+sEqE( z+r;V(L`^h**EJ@=K?+9sCM$SCq>>F7*9|76>c+qq5UNH^dUV7a?;CmeM=_0p`rT@_ z_PgC8@N`M@$}#??pv5HjdJDk^H6bC*eRJ3RakYX?O)4L*J2j>}oIo1bJ5|cP{t{Op zH2dyWvGWD2YH?!>A9FJcmfEPYUT>v#Bn309z7DBWiMP=6{TBDtK224OUCp;4{V&PG zsUP_skljVQ!5{g7@RsOrQaH@d!ULJ`13hSLougFQgJkX|S)r4Sj_fozYf+UI9zB*C zN-|oQUVa|vzW&qgspgbkq~x=E^2sl*FX-;)-|MLS3l=QOWE8l*+Gwb>80LM`e0!#F zWs1U$%jD-mp5ux^Q7n@TbJ2W})FCWj^UZXzs6q$>J)*^29R&-7-0S5b-Be+d<1MFPskSiAoKHB_HM445a1=P%|dD#0k=l?MxF=e?)%T zNze@$4Usj;OHHJg6xMRsuIi>0-(%Q3)vI5qD79>pjnsZerG64MW9s%yWrDPT;Z!kO z*0qgr5w3R4aJD$>V+~FxXW(}DqL%lw=e1r@*CcVjEUYBk8FPH5m-u-z9`yyj(5YqLKe_2o zNMM*Wurq&^v~$9_-YBXT;IpL{3r}L1q6i7&%?>K&%xT(n$m@t}Td@U|$H`6Pp8UZlBp=YP8tnP=V5@*C)SasX$seJ5t&!ASbFLDUNBS;= z=_6hIEI4jGs{6(wa@i=x-=^R`Pybs%Dm2|$E@4LYnSDkP)YEF~JsH#+F9&xHC`Ht% z*&_RCU{Z^GK@y0F`-aU3X@8Rv+UMRA*I_V=^{u9*R*a}mZ}&d(`Gi!l`S$I##d`mI zOOn&GheG#CadHhK7mF&#G4TIG^q*xZE>B{kL_l1M&`cdp@Z2L>tzt?HF)&kxbYI_(D3kA=n{t; zXx*@2$WESf#@_{uaRNh|OwQ;5d<^>Rot8Tv8Kq&^Q0e5_F8_1Yi=cY5K$nPHvVw*( zOy=Km%K0%U4In#o$I8639a#4<1NC@1&S^nw~YA*I*d zJrKz3WORJGr#I#iS&m?khG+z7;-*& zj0}|WMA)J+i~f)dUVDf#Hh)_1@;L8N(CE%Uh{L3*SB>r7L#(iL-QLp+-fyL5ulD3x zWDrPN!`&2fyBl6~$2rSw)A9tN#r2hdI+FJVv0>`QmgD|H2s+LW5;mt3xG7HCgY~q6wv2ER@GDBY8%nR)d{omXqP;1#iKnw z!pY9!bb7UA1mQS5Cs{tc)Ty%D=yX$Fv8rw)Gh~zh^4(OU`dQ#9_qt?9+gTLk$wja! zCO7zVHgsZNW+MCCXv%!+`$A=}Vmwuh17a%qKxcGY*wdl7zSrYU_XKQ(ReG&p0SM(s zjZH0uwh>ADi@`FYex;g9W^k4Iv7X~!n)->660%jbTf1YpQ=Jq~}$X-9D z7Z$3J&Vu54T|Q&kd+kGCB|Ss}Pj-ke`9KYcNu5D7SG^fY>ha4m?oo`HO$t-7g*{&q z5DUh6BJ-8@czLf;qzljWI`;>#j4a=mU%1TIF9Zn1PvJBI*AVY}58BO;O&Y!`?Y+$O zyAgI})P(XF=;o`UHuZ_Jo6-+FS+FXZ;Nm?w*}fCS^u3Pf=(cs5n3iWJx-(~g6YEt; z@oTgs*%*Po7lZ@#wmIt>D;`drY%*AZG_~z=w$n-R^4#U>ik}bc+A-e}ry>tyO#ZIA_Ux21?o7yk8fbZ8xC{$A_?{^1|9$G_Rcp}O z`?;A?S2a4IiuJ#)m*~HiwyX}l0kRr%b90+b|GJ{;84Xawea0^M=GX133L9`~EB#ZP z1FCnJo%!|_pg5PLKXV#7h5+s*(fzM2z+KVz{|i3;Ki1@hyxLM1B?AN8X{v51!)NaM zHI)&L?7(=J;Hf$%bgcs>%HXPBiz4o-f8bmEr>H2D%<(iU6BCmWZzBUK(2m0!8=FFRcQz?=Sx2c$js z;$gx+(^gD4(n`ALZ+Hh(_D0i+!p5zEDO`8|>2K3#$y18a_4fAuvMo(diVZ&7q^?u| zZgbwbYP@)>ny(zk2-C?^`m(L8ps+gWP~P&1{W`F;3Sb%_kUk4ll%nPDXs#ORVyrVJ z(`RNam=m}S;wD23UA?5`HbvXm+B%FD>zHv~^#k&M_;;&oLpkqh#hn9Z&To;>ataBJ z+`XFi`^SH$eF4n*rQgOwYiiRQ>xD5je`3iWB^ZLKs67}AAb$W8r)!C_R*%^E2?FQV_zW47w zp#X*MjV1sWX$9YWsE@j8K6>-2#T59;+do!TR-D4ZdWH7ff`XM@ers!MJv}|#o>{$1 zy1KgYXVTSGRf$O_mik?s;OzP$>bsH&-Xgo9O7RNN)# zfoJ(Ww3CV&zAMF-8I-zti64-O74;P$Wx zO=jaWu2T=QDAvlBU1>3Va>5hQ?9Z)kCbQICn)lT2Fc={pPD*K1c=u+W^%p}eW@hH9 z2}fcQ60h+0eqI!`wA>l}g`YqF?bAR_LBR%$0d7n?!;ppgSAWHG0$OBfk8mzQsQO;h1Jiz;QAg7!E=AvU7#7I|aQOZ~pa5 z;fhuH`3>*e6B8)_R;|wcc;9iWnflMlN)aF;pVegQxlCyW3X1&CpFh7SdjIYn1u(9t zsCe;S53nx#E4MtS#UdkP8W9n(We7;Kb?xVJ_Sq$WO6kz{;qkE{z)Rp$fEXMp1u)Pr zH){z*-JBhCP}Hvge@6GPEBQ52wAVhpcJ10$MU83_H}$5^K4f+2qYogdo`C@aRn^G& z`1t7`dyOc+A0Cd5j;(9Wukq7VZ&#{b@e<#?tM}?v>d*FeN=C-5EZJl5`1#Jxj+&6f zEzq292jvqG$n0}`d~O1V7H+XSK`IDeqnRnSMA$CTD;mhiPE9Q?D=XV??R2iE_P3nL zS4r4+p57%LQb90mOKPZl6l0!V;(ELiB&miM3GMm7|-sobp z56P&p8_w-A;6TH`th_howjxc7G&6OB_(es*MOxY3L}x3v-2L`}P9%VznrkP!3jl{b zTL;5Gek?J7Lttht{;ePc;G$qg)rDs*ICN>s3hp|9E-(XnV^Sw1B;>g^pE#MNr0y@o55fV!7zOtU*85O1_8$(u9rGs5CDIZuiipxpE+YTe4OZ@!#+xYmaD~*@4 zt)9>a#B#+Kz;xWR&uB%Vi&+R@@-wbhpwG?ustF{84G2hpS+9K$y%8A`16Eg$e)HxH z1sz=}kX)<}<;b62uep-9Gqwq4-qwm}-cvnXLn(W$kM(2H7v7Nek%A~Yoa;-yHz#4H z$U1;z@J7|{?ygkuc>_}Ve7-&0>tx5G1JERZ!%~4KuC2*#fF(eOzJ&tJT9GYUZUM;3 z2X@}u*;$lp2n2}oM*#26?E%B>`2D-dI5+cbveqF5U?tm9tWPp#qzqUqGCH~sL!4)i zDkQ#hXLoNB^lVA5K$Q|)qN@n_FkbhxvNB!fH)`}4tjV>|~CsNKpyCjJm( z4&)a6>>M0w@wm~F1=11x!CJz$QgpH%NE?Vl8wF}9{3|UfU%!6cCgZ$XZVQX7wHsyu zG+aNP;{N?yKt)RcQ9-dywZLKpVosAgPi}#p6}$j8+_3spv&b7z?A28}!2C3j-kT#w zn^V{Yygp#aPLnk{ov{o6u@c8ISB9UWMhfC|0en8ddVMRaMZax!HnuA=PYEF55W@LB zp#>0e5~2ym?X@;?B?YT#anM`Ct4x4{?PtAAgEZT2rWXqb`+uH-UH&A?K z05UJbtF=s%(1NSg)S0b;jaH9GpIT=nD`mV}i>t&vlMPbw21!|18wp#=?EtCdx^ag%X!VtV@Pp zFv+0}`uP}>Ssf~Wk3B4tWGGzyPIg=pTQuu4@p3paoW*P5Pb;K(Qy#0G7Qk%yiUA2U zmht$nu4x^#AF-+qcy||dpdf7*lwONHyITA}V;LHZEfgCIsZ&}zf^=kk z!pI14a41WAZww8}U6uhn3%WSO_;1!?tkMf$wDoSd8)erUm)$4ny`C?7kgu(i$H zMDbnpX(Ymb z$A~0NO7$|(y5Lq@Napnx(|mWUS_t54qyd%HJe~c+-PNl8J$VXX z!?qyAj}-gIUui{6YL)nhD9M6KFIy%Jz0U17J?y*3@EnlRPGyoq0%F_KrKPttSOYa0 z%OG77c%qX5l(=B*}xUxYOKRx$>kSOA)rPM3K9rCP5+Y#DU2 z%zue&od3phWh35|paO2Qj&^QPzaiGGzqf|!G59=ryr%NqTcxTLwWeVmGjKkuyfq5q553Ekn_r0WQ=9~8RF|e6!1UB`k;|ME=KT_{o8->!!sJ6VPu~w zV+WfNXRot0srHng*lv{5?%Qn%)%VFThp!F!TxP8kOa=|>9JJB`*8crf$D4;j1I>YF z>gX@ItpY*i%QE6Gu;pbXf18`%#6Sx*Z!iueE`=w~o7LV(AH|9vpS@GE4=zk>Jkk7! z4Ed)^Ix+8=Bo&sNJIdTV$Wug~i0VR5o@JfuX?|1s;lrZi;Mf#Dn?_cLqV-1Uvl6|1 zDdNLwo(ibE;!&Hji$ILdk>qS@^_o#_)x;>iQA6smm#C>UfE9e_wl=yDvlc3F%vsI5 zo*bv9Ox-^WJ8(T@7&0ZSI8E02msNttY_#lxj!v^>{%c!Yfm!{Cb9MZfn|+FVnI^R z<&mP`!`E`BH_<~$GM#IOOmFVer`Q@9&*l?i)HhS<3N|Gbc|FsB!yyh;6gLyFEyM8C zS35(@Y2@v-8zNpf{i~pLadwQ^Tk35wRR=WS<63Bre0Vw#^bZ$#0K8xvi0^%*3{t7_ zeKRcq8Bx>%Vjk}9&8B6lUG{{7rNHVl`Raa$fo6Y=Q*5+HW}B5}pSQ;Wt8EI;32t{! z%%i@#Y^h76jmEG;giX)Z>}ZB5HT>?GtYLHGDx#?{&duBCxYy+fKK z&{j&E(RL`Mjd!R!Fa3izw5=`uDVO`=)s_8S$27BqOF^wBbyVM12hx3>^8gYDnf+Kl zj=pA%)$x<)i4mx%adgo4=E`*T%c>)>WjQgMq+slx$`f1CJ3=J0bbWDg)Vgkk?b9df zYA76PDA3i~hEVVBol)*xm6lAq(AUA`m+@_J-2mof5GlpqFV<^XS1grm@3dRf_~T3c z&h~PrxMFJZi@I|1Y4q0yW_SgONi4UrX(BY6*PcdbB_r=0ug&WuSJZajk*FiQPqqb` zA?uf2StpW>^(++n+S&K<5;J`uP{!|QnCdeiZ9^3-r6Nu}qthX>f$am@`%gc3sCY=w zFaP{_w;#wK)8hL8SCd|9SbpSP2?9~t4Sba>3>QpOQ4<4w_XAF*BK`+S!-}KT3Nif(u|_)j%P^}M?IJ0? zw0I^MkRky`%hhmi)h6fDx=v@OnJi|tm}RJ0LB-87sLvT^ZS5~8p$vTrX_xd-v>z}? z!QG?DlYMTxo(q&m_M8m_d7NyBey9t&XM4W8ljC7m^1 zGr~9;@0Y@ln{&L3K03)G5sqjDgHBDSNwd}KDKe#K*i~TZh@`v4t9I{59YIW5n%Moa~ z^cDbaXl=;54vDwo7uvop8tL4Nw5Cd?BMPDG><N0u+>I#pNm*#FjS9H^n2UmovBTWN7fH(K61!=+NGHAs4nm7Mc^7cxjE z6WtSx`EJ}N5H_hR_tOh^N7FJwY3q)b+v=#Nhdufj1eDUn97?Ox3eDh##ae@!(5c)sgd;{CbKIaizcl=Mcv7K07FYP4WyX6WQ29oA(2LUh zk#L@D5O0s8$bZ@V0^K|Pg`73qA<@rQB6tVjZoSDjjGrGgc&OD^Wzt*wt#nkR~5Wk)$ z^-WMif2OqIRJ{wP*>C^$IKLkix_ONcE?r~ZJIho~mrlgVkN41cIIbFllqs<;>Y1;b z(pB?vp!6#FQj)aYQXs0rjKl9x zE1X-e?HvnR+1PMbcY*7Rw>}Hv7*fBn3#V|-Bvy?Ljy22i0*+&|sVCDi(Hg5-)!&nw zKUGH2*X*)e*rQ9$KG9kr@ZLgFc<{{^#dr(By_wI<-D~bDg?gGrTxo@0HEp+gm*RVq zraHt^5a={frMSP`zkh{GCfx&0Bi}bu%!75T ztn$;voN$}Db1Z`A4y6|)tOxa-go0z+^5JriZQZ$LpToFI0>AB#kN;c8NBjrjFh$v9 z7U$Xizcy!?1cZIxX8AisBb@K&JTu<1Ryjg|Apt-4pSP&iI*$Rz^IjHAL>~86Q4H?A z$ntM(c7e=-4h~cebUCmCf?PcU`l_+cy$N1C&KY3hYcS(B7_TIB+gY zbBpf;nyJ}jqN=7L*LF8e3k<$1zBn$SB`ns-g-W3ByKSKGAv+PDN7$dI?Z{v3O7nvR zq-cyaeA%{(<8X55sdBEPiga8Tz(imt^;S7g{FV^8m$usj3%M|i+}jYFhhY{(&lR6d zyWW`oUaPIfjBz9Sg*CQMSIkf|YHKJkt#vl+!Tqe&pT5sa-KVe4qh0-rZ)n<@`sTF> z8k>P_GVANsbgxrsPMXcjxzZ#(mL;crTKB*1f88E8av!IrZ;y4Fst7WdQV;8YXWkik z_zq$smYI-yS^UK=#&N^w=;;_!Tg0^PO=8`U&SiFig^+svCjn5AFk_k1fuE+R*WCA? zTjAocF>d8S@)J~$s2{Y0H)G!i{kX=vA3`_v2Jy!^^t!d;@t0KB>u0Ty(6xnGW%VS( zs-!v9pI(VH$o6VOu!;msMMsAmVcr%ykSV<`MZv-{BCK9TPX1+bv_wzGz#xoWEm@ya zJa>{JTRN2dIdYDn=I)Hk?>SfUZw<8J-Q_a=TQPa#6AzbCYPC586<9u5J*W^he707Z zGVge>D!KM|p!>zJsL2aK-H@qRvgMc$;R&MZY`U*%nLahnn!G4Uqqb4^2-)M;G>$!b z_g{Jeew#fM%@uXsCTbpQ<5Q1O_&b8l)o;mGU+;^-jk_lV%g0g?Bjks_pLkXXh5Z_3 zl*tOXTXd`WuaKf!o;tW{$BJK1W`BI<8kqmJ{ZVvaX2eIVP=adV)s9f0&){|0>cVgC z4U@)VF?E&6uVR9zFX^qxz$0eOg1L7c)$=P{89$twcjk39%gsPU>A+wMm_Au`6-$I0 z<$7DNXCog@(^&A+Yw@KAfue(ylAV}0M-^Lk4$L1b(-@@ueJw}+)P8#ELly~3XY5e? zDscf)GF5)9piH12`k-3yvsa+)UXtoZ01U)Q(OgOW&*KGpY zYilK=>0YVYE}h*W-88x7$E5t~q1x-oNs>IX*;rQdE{oaEMi|?PM_~&eK=;IZe`pcC z(XmYL_vEy}d%gX93{n)ce4^%j`K$IN#8@oo(OTq3q~+h0LUR^td7}>t>lo&^LP+RB zWTTIX(!slrZ}OkJT_b*&v)@cThk8Y=MblL`1iANyLw&-Rn4{g0#kGNs9&Y_PeIn=m z87lQ0=f5F&&K#za9cXNH{`}lm?XP-!!fL<85RO+`nht=Te*+mT0NIc#D=V9ZD<~-V zE3t{p>E4u+lLL{tBx)!q^mleAa0mjYkz#%BGP4%i%acyIM^yXG_e6af$K}e23K{7m zJ~>Q1nSfw+MOdv@c3W3KWlh55#sW%j!rtfD(vlc)X|^tf&R=f69pq>D%|czbS*ydH zbh4b`QsKU&`jYK_AaHJdC}i~s(ayq>t3*Xk9{Tp}+gsp+9pj`=pR#sF^|a=JBT%Yb zh^$%gsdYnB6AK&LUpH^wY$A<$8C53ey;i!@Hx)HlKl7qJ-0Yh>Y)Y9tNHLPR*WgF? z9kI;&Ticys@cMb7ONKs+cW|YSj*i=8WZ_7pG?jo^G*C$}X$?%fBgVo4qiQpX_*ASa zCMNc(+J@xM!h#?GN;h8|O}Tn||5g)d`2ZZpce0#Dc~z!UUiU|By8fA6-_grLt8&;1FDs~~!?^FzAb0(8303I+0z(@)Jto_7q&q_V@$+el8 znI}N0&g&B(DOx@5q)!nMKY++xX*(T9# z_3}1X{DJ*f*&te=?9iFSWB4ecdv0?xhLXqN^`Bo6tCO{*2zkI{Z*;QJk&}zQtarv% zc&_R4nl@1=Dk@$v7XazWA3S&updop{dVm;#WsP~HGbR@B#HyjB$yM(<&f zK4d_V;P-ahB|@`oX?gj@xb$g+a*_NnRVxGNq&l~S!(0kyP*LW#}I z%}?<7>mUl^n}EwMB2-p;tY`-A`~Zr7hK7dsXhm)Vu(KjiMw;8&8W7S0yv$2?ccJ_D z@9(T+hg`8-A_y2DDlk4EYNu&Vc}h16&7aZ zYXDgFDK1X=#f$6i?(QH^Y;5c*7AX#tItp5$>9Y&~14jSZ(_?E}+i(EieX}uMiCfv4 zZ7V12>Ju(MeY$OBYkLDkPftHLKYvYJT-}5P2a7?6>gdAgoOQHWHkWRx-nHx20+sw(3h8g+1lEAu8rK}=jV5Ib9)W| z^Ya@U(E!5o4G?TaRn#>2~7miQLwws0#x zA;Ilv!-$Q8gH>GIPzGLG>s64SQC+?N;Ev>j2ML8HA}lN|U;tL&a!XE3{E(QKsH##+ z1pFC{qWa28o%%{=XXl4}Rc0vBK@nn~3U|_&;Z_Oa#<;%iprbM~^JOwIY5s@tD zdn|p{gBf=dkYS`1ac1Y>(0kNI6gNUbO3J{_jsqa~TK9EY(2L=izttQ}Oxf+#-Cw>u zn9TF^@s>0>(mr2B=;d7KZ>rbt`1{?4ZU3nU9N#A zV2I?qiK@iGN_7H0f#n<9^`5SbuU%a&pdF6uQ=*T|&EI5ZW)9qW{`~pS>hqu=blCe) z(5OOUt67$d0Hwh3_V%XXya5k5V}?9netMUGWnF#!sE8eekN`7aPc%+EGdPu%l~oq= zhf5|TCW`XVy_UbEr>7T12k4T9M(Vg|^*g5mzMwr~JdJ~=xw*ObRaF{i6Yx9i#QVxh^)ov=JG_Sv zA8KoBZ-mSay(1+fV-ypkJ2*Ii{gLqVyNivD9h;F6v$A5l(})AVr==Mz?)+W+dUmp} zVcyi*A7f{0YiwhKLq$cklMOv6Sc6{fd>0*QsqO3u9e9*&VnorTygEBOFGD{}0*b0j zJEzGnoHecyKs&Nfo>pCkgRJ%YcS=o7%~-iyQ0>6*a4b+2V|#o20(EQ~8=I?CR9Z*L zQjAZlsswUWZ*+hAW@2G+iI|vJTSuoEHt9r2<-@OErXXz+1?s=RtFhj=Az@;|01vOM zs~eG;N)3lFyet5AvZ3M5pFe-Z37=V;u_~!2v1ELWA0#6q0}0DdNG)O0v4FMM*eqYy zw}FQqQb$yT(K@8dzfVZr^Il12ChFF$Tb*COs$3XU(UtF}-zo#S0m$Gj0a9FQ_A ziEH|&JGV5_e*DniS%}m4)|j84uX>Pm?@c~GWvN{5J$w6-!}aoV-aoUm7f^ry{#Aw@ zUK9n38K7FAzJ43!c(iNx>h)`hT-7)4-xH$vzZ-RxUST)Nn4ReF>cYMfE|tdo1omZH z*Ry=At4l78zoN48Syfflp@TvV0TfcJIFKBPNTsUswKl8Leu7b&dV1dtrTm0514x-* z*FSf3{2n#(Z-dy1Ax?G!OmHSRCXMUXU`f(FN=(rn5Msb-s7K6iVh|6P6iQ`-*h(?> z6PIcw;<~!Hyi36S*4x{$vpfpM9haP({5h>CCHf`Y0MJY3E8!wenre(ud$e#wj0Pr% zGKRT;&Rnjpt`<;?kBtogBVYEtDcXO@N@4~1wxRLDW zxo9wce36zG1tRz?IGBNp3x9Q@rUS||?~R+$T-JVk`0~r+&TDnOd$E+E+8iM@H9`#^ zKbHBOcu@;GqD)LoE?>SJP+ZKJr;ri?WwVySLAB90S71jqTHb)>gvrh)$X$!36~*@V z_ZJoxHeMXab$s$f(#3@z++7(25!V&o$N6y!2n1(jWO#{bSKCs+<6gRX^JZ;*z1iyb zs#@>8l9E)r4yds3gmb8D6w;=y+y}&PAkT8MnQLonVbRg;u(dk)ayyI=iq6l60?Qlo_3Newe`oiB9FE6jLD}`zA^;Cu zXs#oYNq8=qaU%&yLDde^v=B_*)77OEv>gioSNSqXH5<%k3uH1*vt6VRre6X`+5Z0T zn3+F+Fd(vg78;uU{=M?32^@wBU;Fvi2^4mGmDpPu4}uta1_^nbm7hF$Vi5Y^+pDA` za=U>+l0$t8k61tZpEoI3kJGEBnLRx{9GKTW&Mhp2WoFW+iuuz*`1wlMiSg6%--k#n1v$%Q z92`jriKqXh0{!38ta2Kthd}rY@asG_OoY9k_x6(_$YpqWc%JHWh3a#4 zK<})~d(Zw^NJtB462x?XMf(gHnDAdtIIdx=QpCvx8HaX2 zZLKhv5hiYKLhu~-MoSEaXLG>%{gc1#tLM%|{ehR42pmCQsgW22oW`)P+6^8&5KJ{Y zD_awFeSH9kEJuLQK)CWNQOgi?MPG8Mt4w*uT(cIC2 z2}04?(<2Qh0LZbKHN;N_TM$JhR8F0+ zzyn4|`@Qi6Q!}$Tutcx1Ch-eLx)>2bH2fMHYcMAfHDkPRcpzJw$d5rx0B!GBCBm@Ksks194 z50!sXPke4q^C=)ol~h(HOifE;PYlY-%`Kx&$R`>f8#9Im8*+U>&up~d+%!HrJy{rh zOMom6(2Gz#Ms5dTHskr?l3)LI?gQ{pmjBtQu=AhGNb%rDgC|IwwS`6AGIw}#F|DW< zy|D0AByN|N$8Bh6Kq?Xtxk+**G4b-gn9YIZej_|T?r-94&dT5oFfh;4S0wX>#~_4d zLV5uDmPK|SB8rI#DzG+)3TbO!jfjjSA|ym1U%Gg42GYNO@j7@tSQe-=3bCo8?Ggd)AJV9L8?}0VmX;q-126Zrs2E zA#Q~{6>_FrE|zbdoqeva>a<+2LqmI7&ZZ*nhnJW^a^;2lFXqc@psoZM?80=T7|3uL zln6lT1g&r7|2p=}nTzaaGrnbUR!4d>Z7SqqJ^#lDJk)dJMn^od)TxjpMc)YEn7ikh z6#^w6&7}Bru+m_^_7}2ae*Ux4Qqs~8cB!AsYromKJ03uXl2X|7=g;)H5Jf;l0;0hp z8BBvuKDQu=rradHm@qddfkiw^d>{t%4KV@|VE}9< z3}Ao;p|Y@LF&pZ95cA{1^%?Hmp@ue$jG`iby6>)z58#S6wzh&GR?GHJUn;#Ub!sM7 zp6D;;zEgZqd$=&2xc((iyg68{wsMc;#TC`qlvBWhbK3s!&bIj}wN3d*>#V}HZL#)) zxeyoa369R<#ehUnTd(bVdreLYJWZdRG4ChFcw=Te+V@iAOxMP!zG}&mq1)c?;NPYg z3%DAF8wN>YMh4yS@iF3ppq}zO#o?0#oW9rY3KoP8-5FwZG$Jm2uK7_)uH@uKx*_r8 zU8jCrbB$yC3ym?Sgsm;>cHJIDmchH?PbZoqofh8qT&$;2{ zqh*nm6xx<8xQQ7ycS<}+=5~B^l56Q?+wJf8u|vQ6_y^|P?9KP0V2>y_yusJyhXcGC zsSkn+DN)By27oOW=CE|2f?4joqw|BSzwl4Py~D>@IW@9YB(C!+(XjQ=8wevVZKb%LR;+eMO|&HRdmMFg>;U5chT?P$7YSB zrQg%XRc!o(-ALVgas9dsXA~rqfq{XJGc6Yp4-YW${a=wc`vA^^L@{uN!NI|G_-aa* zS?^YnVrs6m7PIfEsP3aa%$sdYj5@K7RBjr3!^v0u^RNn;P{;xstMnZIa5l+E&~^h2 zFWh^d>1U1@>cYkPv*6VX=9Zm!ysI+?!`zuyUVR{Vnslt=LNP`(;>H^mu@&pROCR0n zO(>Mz^Si(@L?X{B*J>Q^N5r>Fwq9m^GFw0AOfa)sIQ|@y#<0f~aBKD|i9c9TJgyU? zS=i@K_cLSEKdKB4XQLZ$q-hVYQmL(*L3!O3CB`4PR7}PgG<#mdxcN@6245zcx*3wN zO^EA$4{I5kHC1G{)jk5(&Ac3=MX8^$B8B>H1Y43c7tS^{2be`BIBI*Ik4Mk zxQ%zvwwF}zG6o5(q^Dh4*CQ|U{+%IPm-|f8Jxa!g-da;$UYxT(MvL&Z8X5;e|cZTO@8Z}(giT=08j^Ypy zAVqo$;6|eW(%20C6_SxQh}Qe6?eq*nwJZx$LN|T~(#U=-%0tN&yfd;&P_f+h^CcM& z3u`%T`s(<11-CvL|NY_!de6!CW+~tGY>q$7NvJNV)Yk%ZmZ+coEZ8mCXJH)5u?1-R z*)L^_A>q4>jeQUNClnsvyDn?`9Ly%|Z?qEdJnAHbS{x>yWzVw@A3l^w4&TkLUf52x zY4zVe*B}w&^V$N@9YcI;D=RFpzdledR?x-&?m;3uv4cK- zBc%vuRI=VNN?_%d|C+XqL7drnv@Ia+d20d67O^QQk(L?0WPrVFg3y*Bk#Zx-^BLM5 zS-I$my$l*J(X0!xc~yZI7|k>ZhHyZFpC{nIt< zzc`76a$9j*M<&$Xdoq92_r7Y{UMSj~BP7-ToGV%b8)ljuI89Wsz<#{hdhJIqN1$Bp zvtMfpKSbQ{+5!X0vtG!Exhc#X>x*?p^v=|{Gdk`Z(~ywfc*FKZX6jo}?a3sURO2Si zfamPxIB9?(2Lm9Q9uiK7@R3k@?psQKv7V5Jw>J|92d<~5XB~cpwJA;~$A#h9X{XgL zbk=1*wJJ6k3JOZRp?thWAnY%QV=i|w!XdT2Kq-bvDSxb|3^Co`H|3CsQY2p+{}A}Q zT41=R>8pedQ_$A4x5ln3EX*ZmgyG9{qMW>evkN>6s?LO?MLrok-=lmfM=<@Qe!X2c z9ej(T4;)XJeO2}2_sqDc#M8iiVp(BtG^d!x$x58MXh%O6GG&Z$K0a>SA9lws^w=Fe zQlgtyjNWv$!^`cCQW(buigjK)*3@Et)R0qp%qD7Wz_69c@%|=6)KC2qSUb)?rXtE$ww z3$d4V2Ro9qS`(6!heFz&1yw89LMbOF9wcvtGATVjf931zW3(^uQ=d8&wP#Ce>0N#J zoW9KU-ceKA)y8*5QA`nXhw`oYdp&$>$fuJzWz~+s3cvHyM{YN+v394p2@x&#)8tmf_SUXsr<*obt##c zm>>(MhlCw+5bZ`kQK*za8p*Ett_dK)5JS@QO1{n1;_;Ao*Pcig*VmxRo;en2*kud3 zGm15r*B(12oOQC@cJ-(bGbg4uc@;^n?lG(T&-ds;(sTjE$izg&372Oyc=+H!Ga%K= zoPJB|H~P^)#rhSWCH`D*Mht?56}iKhb^Xb~U-OmkRX9+hmNqqIggA-Yd)Kx*U6cxN z!vn4!;2fi_!=pk{DuC;#cKbXLdKNmH85b89`*n5!;JqUy29YrNi%2RK7M3B!f{`C+X_)VB&?R3^=cw&lIv_LZ~VJ_%Z7pQBhYK zMxwZ$@HSIqqrir+tuh6I2}1>JRhiUwI3^kHBG1j6KCXrHx= zk5f2*poVRMqTPAsk^kVlqWewP^VVZd@Ybze(8&*cZ$&L3&~fudqNq$o2>OS%Qr{I_mVK*$0`0Z7FDkA#NKZ~sDi zdisAQG*C!CPiTe~&U*qDZ%l7JhWHuDlyJs@K!xzu{?upz61mLSaT zWkW-TgXJ1ymPIIXT)f(BiP&XjWlQ}MMQoU>qoo=i z84fv@l&$T%=xiuaSzkpK(D@T8Hg3x!cmUA5Z)?jj<-2D9#qIah31PCmualEopfCa} zs`Wh*1PEvJ?~WL=g}ygS&I1s=pRXW?=lnAC;5h4`(Ds~4s6_SFV>ljHq%dUFG(N5a z4r8vH0;ID$NLv1%A{KP|Xra-9>;PlXc|N$92H7t33^ds=XBRRdZOxFsu9g&k{rYuA z$Z(NPN(ep;HZ=1ffn|cJ!*iJueR_UZ9Jp)9vuDO&?1bIds1Z@nWrfNclC}cYRrtvq zu8*(pW*fOtCNyne;Vmuq&KHhxt}6p?po##6acE@+LcV>Tgb$$4&gOvpkJY>1w6(L_ zgmk?LDmq9x6WX*4mjjXW$s&_K&rI`YAIQnIKueW~j11{bwLqA({^^J--T%~cY+_;= z3@!Vg`;4rmzl7pSoqghD5s6BnuNWL@9<*^CZ# zHKdRVqdDqHLFr=th6eau7@(e3S^pxXmhlQpB09Pl7zvswD*Y4)ZA2)TB2RLB>~p?K zP7EXl%@H8|Dgh&b+5zMZcLdaR0y(qNFJ*9)}qmlhT}k*f^o!PylQ6hNsZBqfE) z(0f$?I$4)s&J|2kplZR;<8&Pzom$uBmoFK?6ic=uE1apB_6vfdZ{IYMf_9-kd3H2?20^bxk3ZabxM;Hs7Ekfr=?kCKFH zW}@YUg1o%4swx^%cLc8s3V$Ew>Rbtt?&B%)P++#oAgzEtHPS$XOhQTC($*FikW|zn zE@9IdQc{<|v?30Ach)#gb6@WM|sHu45rTxOy6H9{nwakT&(lMXkXAGki zeVz*V#)6XHyv!jUxU)0+MFu7VRz%Tv`dm4x`o25ahD}6mM>+cQwSSS~-iZ`f(I8&f z_Rc4nKU>5ato9p=>R_k=4*ww(0yae&ze(`{^#=Lz?&VMql08jxAI+3)&DgRjyo7yn8lld-f<+fpq&5%MSQ`hQ26+jMXJ`@A>@i{+7qX_x{xapLywyg^Di=?VZu(c~plMYf!mR2@+(SxSg#(^usAN z?LB(Bu{J{Xl9%|^gnN6y7YCECA5t6N-WU&}czMM|ciYJJ@`bC!otX03eB0HXraUY_N&+~oq&$<&2R(yy)ui65}Ag!mQOmGft`<@y+2$g59Vsm zk{e8UexO_X_6mlClfq@7^&^Z-Pkr0z{C=aE?qZC^-bcd63}oGU{w+hRn=_Mq*9chD zrwC6^Dl0=)jhQQrob_2bGs<`k#%eM{jdh2;xYwAwPq+;7sTC)YgCoU|V2vg@ex%XC z{-ToH!as>2pVo*W=B3(7t?YimpG|UsoJEmAW6{Uuwv@$u!@eeGK9^YePi}oLa0aDU))94?-_z=$B3>qtQd+S!B3A+;b9Jbi{j}0X(T@yb>~MDL$JPhR$)?sJwM;?{rJoiqy9o48u#Ynv7R6;kB~gyq z!GlaSj&n|mRFV+m<2#{J_l7R2vC28tx=&j)J-K(qA2{LLx7mKqdRk40?&0W2db+NB z{c{iT#%BL=t&x5p=7YEIozkfu9lR~GpB2o>=j%d~``*(`;EoepDT;zL;n=yCPG*ga z-n_#_IN$zfOf9yh^>R1HZ$Wd3YnKTl_ROlQ@?S0{*3y=d7iI?4zBCAYag;<<)<>h- zcFpH#bv$(3%E_{UNbj%X$)EMlBoy@V4idZBJJ&9o%N`z6TouzTNzI@2u=-6J=G7zB z8TYkTFF$}$jAp*k8?)8!N}1ibNGeCa&&kNGDVgOrlTG3cH3ux8-~P$0eTRFEU?7gL ziUeIj_5D#wQ0IWp>`0pQvi#sq=3@D%f`;hTju}%G8tRShXi@STUR9rx9q^^OALdXdha@bzD zM+PhIpu6IS<(w$wZ>&}EzVumb$yGPr2>yfDi>(@0%*`Srp9NDg|}-HMuYvu>{JhI(+*eCi#F!MBW{Q0 z9qAaH$U7`*GAOJj!MsU06;(vt8g%j`xcXN| zZ@#AT*HgVxx3@iZOVi~wofj!`THeP_?1&uUE{rDH`kR4mg_B8dLTqa?#{}`$F66fr zca5wdKP@%GFL^Ta%zCWh?AI~zpL!Z9d=>gxf@ zuVU_9cweykNb!ExBmEDCm*1@1xw~K)L4blTf_MNuDv%zPr75eRh2~+KRN{1EO;*Y{ zBNj_w5MfkabZb^EFJS6r*l0N|zfrY_z1bI3!)XTnv4{g+`lfJnTq|a#d$0Y%l>1em zP}k666<;#xpx-TXWd!-I{`nm|>Z)Okxu`h#OYuv)Hcl-2-?rPmA7;JZlI3bwD!U}- z-_aAsE*?|EWA<~J>CLdWSkd68l5pv}(NyJ=l<`xxIChn{U4Qsf4de-uX^6Z~?LuvR zy86@bAT>95l)m<1_0LDX^Tmm7^AC8{h*4+}*pbOUy%Q<%jtr{I8jEH?5yDQmAQn7# zX#zYseo0d=BduGfg(BW9GK5SX)Acu`K`i zVi<~5LxE?d__8YTq8U#l{gs*i+)TMMca$^Cidt)5*R z&$)Zsn`tnej8JllzLD$p;r;IIxZ5iJ8G^?Da}S>$LS&G8bK>6ZRmHU3>5$5Ts0SG0 z+kd9$hiv8xWgo8gm_!Y&SH5xyR9pC4`L$PoS7hB?ar1z;{Z5=oP7nHZ9uAY;_K!)HzSsO#a_ z#^2jx@v*^I?lY(=g_`sa7}QS0#;5Oq!L?U==y+{Lr0eu@6VlV zjuejf?UHUS1?e#d$_if4dvY;>)|;Y7Iwn0HznNuJKz6Uz5$rM9UC7 zyeyIl`0)=!UUpNv+pM*vANRzE(}G|*x|&S&YRdSV>`)N{gygFx=KueUVd-cZ=JrhOe#*6w1N;zqCwvyao9{g3rpI6X(td8%{B>he6lcFX zIk2lp%X{M9a)%Smkb8XrZDK?ib9IokK80IXC>dj z8??Kz55CRZdBo%AL>1EZ0Gz zND02$w_+aowi{w&^W>_Q>xa@YEH%OOmMYa~R^P9A<^y|Mb(!eKEl)W8!MU|!BV1;(bi%M}>qXI9R~`7k%s|sH z$u;lkA(vk9|78NvD|SwIaoCCB{}?KC@BRM_72e`0w!`sNE?v!Q3KruMzrf^@zK7;R zx%?;bsVz=%n{ugORa{MQo3f9~0_MB4-*{TL)jHqlb3dBW8l&UIuU42`dpwr+vW@O@ zOpTOd%3RD8^}RyR{-SkqPLsxmPgdL)`<;-8y=*#{2kgkxlMY~JZ9`cKGRgrmztLp^`^Zq-vT5_HDyqM4jN|za%2W+i-IeF& z_#@a@TNyl&r4v_svpJc3V&-$>=*-(Y1~P*_VFuGJUtXYn{xqE3zlL*fr`7HP3S0lj z2GOoD+FcJz;cizN{6(>U7qyDue$M}HX&+AERM{%15W`WGmF zT@sq4+{R6)?d4Gdm@=uukINtTSSom1DYy=u9V@#jDNYrD=$`KXCOF>gpsRITrGz;S z7+S=G5l5&+YmLr8NjseoE0BI@@7rv6#edbqw_5?vh5#f1?Dq@!#+2C-uio+C1_TDT zc^65g!?fWj1Aw%x6ad!_y=^C8mHph-_6C_|1L8Uue2RgQ*;Xi_A|F#>f+DZDxHyV3 z_WJqkjVqIfmHONw0+*xt<2$hVXNSVlv#Q8VH^In$XMCgTLBjy0z(3}KWd52 zj_1Yt%gkhu8B*vP0YTt=3?Y2sGL(1EXFI8d0{}4h3y9kQ?B5C~A>=l9PPPZ{z~rcu z>p5iGN{TTKL6;(cZY-<ic>*uqG z?3&fFGIXfp7jH8?23lJfFsj3U4IdD9MJJa}6G>!of`|NauAB;8ot^pLpAxsUw6G;^ z$tUw;!Mx1*r?+L>p5pi&Z_1F6k!1l6SiABu5kQPQZ=*qnj^bq`G`gnpJUIDf*B+pfI80WnIYk`1=0a9=e1PW#a&~jD7K^9xVIRS<*xT2z> zL=r#(amZkc`}!0cX7vnmoH+#ATyooItCOA{>cb};o4sk>@|yR?YTjch~HhSK3jZCuMmRDej-VipRODhVSde_Y@V<06SEDMi32{9)4mBfJUN z3+&x6U`~OE6aL`Q-qF@%rwd2@j2yrwue2RkMcBsybtgN620T?U4kW06Ekocbg??I| zNo$A|?EUu!&x#-RHLx?ti-QsFMT9D^b*2xPElZ*0=pTxU+PS9_jB|Rj?ypvhLiJ_F z0{16BE6`EH14l$hkAN6d!t@0^(g%Gm7*J3IqGd`EmwamcQ0W-j;+LFyhqoW0^3`Cb zGBF`R-Fha_>391g0O)qz!{C+17i|!(;Q^o_t%64R2mO*O!Nl-Y;43HsM$?1aQmBo{ z48Ow%AXP6XR!3mE)x1pGWmrS&KeV4P;JI^tp=T{Ho;?%`qwBsfsSOmM62&@f#cQg<_Kfs@805k;*DR6>Q9w6Hj`zsg+;e-sr zBp@T@LVA4^RX9M@!VPpSD7sX(RA^A=z^vL=pjN(qT^gPTNWD{aYwTCo2`^+mgF#xD z`BAj5(E|)UG(um3`!m@7k>z*+mC0bL01V?xvCy5?0bG(gFllR+=oi2U-Ol=SW4`L* zeR6|8a20-K@7;)+OKZ>EPlY9dzK(&*#EMiHj;XIJN+PfSRex7l6-_5D~orSw#*TaFUup!`mo% zNm&V|BH3=;(t-5?>GL&g@`l~-Hk91P0UTtJjjb4{ObKJ;k^Qy-WoQkUBH#)iKw8ZS zv5`?x=4s5RToLC#VSpdgQ?LCA9p{LMh!=y4>OMXV!0CTX!-zud;1AxvfD(>@5oq=s zH-HEFffq1$K2$&c9m&`C9)zfAd1onty(AXCrmvq0Iz10th{_+8MZ`J1>&q8J&q3gG zfY%`5722GO$ZNDq{3uH@uFLrNBe4vUtolH$iFYnA=&d z?08jqZdrg@aq8E`e!@qIb7pM-6Fi={PB1Y@jfSbqSBO~v(lE0gdqG`!v*jXJ(nD~= znQ-Q)9TFy>4$kk0C>^Y{_(th@Cq2tG*}it>5^$0pf=*eo8lHo#fcl4nP5-B(FmS&j zl7xEf_zF$|6|vDqlV1-Eu%MKMs>HuYxPUYM?B74}16?7QEXB7KpGhh`x8@ch=+XlE z&sj&|KaRru9vj9w#m9W8{0H^!>s-l~bQ>I-;UJOaHhG4{0qJw1Zxs^e$62W%mt7H!qYOsUt8luW26Re$Eq765OO`K?~SEZ|$X+UN!m zDlr~x|Lye#*myN$@MpsfG%#7r{}WhCumi=LzdLLIz~S{PoFySSxe)B46&M(jt6%{x zFo9V@F+Tt_3lPsE-$CFHc4O})04OmCMy$A~q)rY797h1G!BsO+4DgaCpL$_PFCinN z6b#GKi-%k$tC~X;(|$+AK>bk!$Nt5%?D3Zu*UYQM1Q)B_Z`11phodJjtTo)rfon!UPDLA0Tv?AkM>et~PBQ z9X6-O2QW3J9d4Jzp~DE=djMK6H#fflW9T(MoFicn{=|hppb`cO@5{*8fb0R?e8T85 zkS4r<$?3@N9v=;%ANT-fZIsp36S!)BqKWPdsVxoUMk3sdU@Sqc6jFs%AOrEcLkY&8 zTm`YS>Rr$+$OMIj5$^}fqoqm1*5`EFW@fIfctZJrRJ1I1H3t&i+=t|xh7H=H-n-js ze;$XyJ;AK=e*{n`FYfPmR=tQ3e*8EeJS!QeZVmvF$Vs^zg{cuO%6yt1y_5x*4wQHz z49>{@GueZ6uFE9As9afDsXFo9RuEWYVFY1mukp;6QzjzhF;Hbt-gbM3U`;@mD-3IZ zOcVzCVf?h@D_6Zdn8p#v8AwSz=3S_ zlZMt4fq`hV^YhEV4KoTu94-(->PC~{eSzsUgjyEh=|w{rDK5X#e*jr75}o>jH%z|k zD}o5LvibS*SHZmJAi&q&L72xo@j9;n0y%^R(+J!g=l#$SM&L-UnRH9B|G1+GGbD*R zIg&46;@uS}T!bZ7%E^y`Q&28AZIi9|H^6>;XcGZ?z{^pixD;}qn zW@d^bM8^o(FTzMHm{QzzbQ5PH<7@(^IQ8_ZZR19l6HTs?ej?x6+7bXd9fY3w?rI~4 zZSStDr#DJ`hXK)?!Zv{QnkQSRTx(au;)7R509Eo0;KX;UHvkDjUalP zuRoa(Gll#F#2{Ym{O2y?fA_(nMs!p_$aUcE2NS*rH$?rsBV^o!@(%?Qi~T?lm2b|K z^mUp6E_KNmh|y2+FQ6z%@*>gx`-@s`E;W6B`)6fBg6Qdm@~a`XDuX-sXFhzV8w`!F zzwv&HMmJt3#Ys~lU5}WF1DpEr#jdA~1-Zpx%AsrQ8)M5XXKM@cPes97%4eiYUuC?z zEUxk@o?=witI_?@a`8UAKbvtaU+izy?B-o;``Mw}nBCU4IH>$yWRYwv{xP&4#3Ur= zPV#@BM(2jS!kikQON+7UCokyg+$(G-l?tAiTYBS(N&4h0h?2~Zogw{Ym9h{96*lcH zBN-ddQDOAaER4q8NRh1a-YY{aZaa1JL{Y=eC$kRAi5hlL8rq?N4MFw9R?Ud|#+vk( z=Di!4vW<}+RBsmYA2iQR#Np+u{JZ{S+kF43LV91pn*aMtBK&9hTAs^m>h&UYws^np zs0V2HrvE5J8%$U{FubX6Rl1+Ijq@PU=Vj&;PqK&$ysYYQ@!Q)SP3v(EFn|q?e445^ zs-TxgHSfbV8guaWE$8cl?+1%ohg1AN{>yLc?B!a*rxhLyjor7pzbaaA_7+Qx*!61{)3ruflC{xzf@=#){%Zp> zSGMxl$Q(BJzU?pGKCyFqA^oFKc2w-(jfG!a0rpK&jffV7nEcRhu_oF>Ydr)B4z$C^ zaT51!R1!Z$ztB3;VqVBzxlTz&mHqQcLyZupPKDoX<;mnwHs-AOlH*C6@8$DR1EZW< zc{YP7gCwJkDY}B#s><5z2Kr2L|hgeHTpvT#;9*mGD&foxkjk> zNQQmsosWAd7Ud**ddvDYN@>Cbf;&Dg@qVW(8HWn;auaa&A;~T z&;0GG{GXnzWLpt>lwYs#%a^Y7_OFrp_mACO9cFIG<)RjT@Z>a!tAckk-KW?0i^D>{ zUTBfSo|`rGBdy03ic5UI+&;dyh*3IO_;caqfH1ZzhSxBT{Kx?NZ7P3I2Dmrk-S8n==buGe^G8slva0e@enJk@|$n<`RlKab~3)n*!X!_(R?5i za6X&k99D>Wx8&2oo_u@FRKV_sgk;88=JtvGao`7<9qy79Ki zQ+W1#AD^{!O6f=A6+5u|b+&{}&iiVHdu=giHhfhMysT+oMI@)*M1^Uhr6L9fk_MY^} zZ)x70Qd{@l(o5++qE!@Tn$nmn^1h>S*X*K^yioQ#-rNEIcjRKWeuj>7nljz)?IEwF ze^$*k;(I;x)*I|T%s6=Ll5a)gFUHig;@B#@C0E~_S&P9}zrat;opko5p29Zuqkrqj zc!TTaQ{vIqt&aySiMvnQ*)G0XT|e#XzKP8^DxW-5DTTYsaK^v)(z#AC6|=~5^K<>T zeT5VyJh`uys@yuB02AM&2Xx#D=}Z@7s4&N3B|SE ziVmOKCB+nkKfb2!I)!*fR~ST73KJxLBul(Sz+z^2m*EFLTIef(-;=$$i-Oso1)nk* z3K4j-drmqVh;a^f(AboV?Q3zuF6X;yz4p{ zdVh0UR_6wN?ecPmnFoE}6>e&-qzrTb`-m-C{`UOJiwGwonxHU^|_zL6& zDTL(w)@vMia^p3xuIo_3Z5h_Z0}tQrAFLk_CRhz9Q;Ub$-5%S#VP=vSQgmYT+!eki zHfj`P`{JXe*hew%AK}yAqDNoUzWnR?`i=81(dMBk&Z9^8l=;EE!aFmAOxu4Fqs(_K ztS6IX)Q6idj4U)->pi_OxXtRiHe61nz~7(7KFHtuFdhv@L z1dqS|TBIb|j3<*nsu!GlQ0~(2`^A&R^!8~<`=tjG(Ps;tU$hk}l)Vk9IrRFyF_%Ui zju}g>y1EVC=usw-**MXz%IRNUPe?pmlTH6SP{Y2tOIuFvlO2rHc;{ADYZ2YFhx(IG zJsKX31u2W<`NN%(BNq%;gc9?+5aF8ZR=!PKY z8^+Pq&z_H;9C(Nx&WYGfdaHj34(|GTC(~{X)5Y_rA?M+q+{v%6goVyW!5T?Z{Dw)| zs#Sj`V;(;Lv+Wa~|BTbHit4t6WUJ5fnjcRziBGbedtY=XDlOhzKjV1#M^?1PEotD% zR>B9C`=tsIs(qVEJ9{!@j~$}8rZu!mg4OxN26hHVn zFy(HqZnM63=%ZFAZ*-WqQkO5IrGry#H8`3pm~V^~{QF zHB-JrI{Af(k<^}KVN2mC!QLt>#o?oVPh<PYGp zjgGPKyZ_BCKdzKscPL7`C4B*%cEDyZw8-X}!S3}*QX#YQOW!|!uivUCPTcHvDK3!JCR$_^BalDnpvu)LDHkfL7pu?=9^2yTd#l6Di=y}&c*3e&X9hCQ@O07>t zRSG}fa~`(zjzZ;zr)6GCvpm?^ zM_N(q(>e-4tJv5M*|gE~+g2|NS|l-D#*#iOdY;@!CirWnRVj?Dw+IP909H7A{&}d90PatF_>MvoUAd zy_$77UgJ=>!n#Zo-t|E??0-gS?bGJi#!sN~r;lODsv>_efGEO>D62Z?Qeo&fc&F09 zpPj`6s!UBg-5X8a!=Hxu6DM8SkeG^2W#TfN{Yz=etCRW?xgY1gjik5IzMf+8`u*X) zm!t>}kMvyl0T1%t_*RNM5}h`OoM8%w|IyibMm3SHaXg?{uyA)5P*Gqhil9ghNN)-f zI-yrtf5hB$0#`6(ZzL^z7a*_wJr^ zpOX(WlR0zd%p`C5{h#;${9>j&rLDZ&`(oE_M<@<9cR-1H{|Q{tiym;I-+yqIu;clf zZL}{7{iA}4njV`dXLYdX`1}=4Lq|>8!}xcWV=ujF>U&6^ZqIZ_n`T)k2+pfSF`#E?p~klKjn_+Pbaa~HWM&dVnm#^e z2*ibbJmR~&XAf3i*2!^w(=_X3F}+R4JY!~YycMA?s@MGJJ}-a9=8a-F%1zrlp4{oA zGb3EwG*~^yHK9!wZlt!22XQ*vET0YMo8$!>BU^pE9@)k0kFLCRouPAcWHcY)@Y`2> zS%Z0*y8Iy-KVk4*FkCcZg=Z%A#{=EWYz0dHJkmb2R355B)1lW_xaK#5u5Y9OASu*P zBVkniyO&DA%7ZEx#0MYszF(6LwmUW4USFr8q|Ddz{fiuW7Q0x7CQ;?2Ea2DYK15!ao z%p-@BZEd4YY~;X-ik($owJkXtG`Fzem1?hRHn>=uu_}@ z(Z;mhPxAPlcs6e)EIV3^t`>WXWIA7~j$Mz>yd&jCij(Nl5M@bS+`6#-RJorK?pQzbQhQjuIBp( zASS4De-wrlxL@lL0s8QZRxtAz=DTl)`4+FJ%y0@ji-mU6_V!ohFRDKoU3IxoEIp`DJLkt{Z` z(3#Z3Gw7Ja%{a`o3p}3 zj#Ktrs79HWUqUa5gw)}fNqI5K*lTIH?BR3D9@{PZjb1ZVkgrSplU6=%l@}ud5hH0$ zgb!Zs9#&Vw7bl&}`{0|*u_1`c33nf#&f|A5HTCGX_N%obYcB!uC1zbfXn)b~fq+A| z5bhwd{X8|GCk3079<)H1U67V$xtPb_X#5garOocJT8&>)f^~&UI1PT9`F!)V(Bb5} zZ?7q1)3@w>>(8e3zO|ZtL3tU{>J-E*Gwc*T8EBLf49x%eki%FR@SuVMlxb5kS-$2n z-YOi`sVd34)LK&1}ih*#N+*gvGLP_m=9bqcbzVA1)1oqz!SNJA5i# z{32+k1vSuD2h%!M-CMMzUApsi_IMJKMS^7{@$jOxo2+%`Qgz1?>O{$q7)-<&#?#|CIMvo4 ze`M`RPgnlR%r6M-mkn5pbJ|mK2zxA9bd(w24%TzIaad~j+X6G3APilF&*_UoS;Xr1 z;CdH7@*GJU*L2yK#odARe{3`Uc8(FhxCvR{_N$xk-oe1$*2Z~u`L%0fG74yW51C(| zoO$H39!ONi`-&o>hLk0yv|OcMJgDkT>CwFS#4TrU8Fo1r-^Wf7-qpGFDmwPs)u^v0 zDRE|}?9Y^X``z1IE2NT?6xP(Id85ioA>vik7}-RGTgaG}axa=%{vNKhLwe&vH+AD9 z#abxraJfwJAPp5T>5>f;=LrHlUx|$>AP^r`g#*e70BHe`rEsuYBJ6`|ySfadhn$Ry z3cY?Yq7|U)^*0iunaIfRkh~Hd=(7gJ3ok&^G*XLfPid&DhW^sWlj1~49ot0Hbhx);Bg97GrdQda6RrzMmTKhE;D!W=NGG19 z3gVgXP9?Ww!9+p3jP?csWrFHRxV$ANVRRf4a(L%6sS#_*7_S<&Y+e}#haTd-g2&OCZ z(Q`8ht=iZ(ZYxJvwDeT=B}xNB;$O~o{6DdMZwIgx8RN2f{#*>WwC0sA!zT0h8R-0Uf zp_Wh^8~EXqS$9^g$2NqNN<&knm=sq#$s-t2N5n$aILCnEhNdGvY#M++<)jy^4#B5~ z%W9YR@LP_h6d<1g;Hs?8vOYOo6iE;WtvSvSv35cGKVN(AX>?zmtYntTmaMB$sT+0W zX5dOyJ$I7i;gfOrg`Gr7rk}yNhIjf-KApzSQ7A7Qk43=o2oFW=5{B?E*2D_9;~9*l z!i54i+mTvzK0iEQcMSxJg)Lqtm8=8i?<-GciL8lye8J#ky9e(vlS(Fd%h5v0;FOr6 z3#bxIZhAkb3$PiU`IY|6v+~>9Y(!uV_qXaBO(^+c=<`9=fodfyDwn9HV)g86q=E_C zRcIrcHA`@k<2+q6QED}g444!IF*FPd1wc#ga^42>dh{X1MdB`0@AX#_5)R; zi{=5CT>~tN=i}`l)qfz?e~-2Y7;He9DZBk=UjVT2)35d&RP6!~wsE2gBe)Ht{^=E1IBVc55}J5D<}L|`h`!I`SIfs z{@j$l_t%O13G0ve?@varname(x)) >, fontname="Courier"]; + iflinked_getindex [label=< if istrans(varinfo, varname) >, fontname="Courier"]; + without_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname)", fontname="Courier"]; + with_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname)", fontname="Courier"]; + return_getindex [shape=box, label=< return f(getindex_internal(varinfo, varname)) >, style=dashed, fontname="Courier"]; + + getindex -> iflinked_getindex; + iflinked_getindex -> without_linking_getindex [label=< false>, fontname="Courier"]; + iflinked_getindex -> with_linking_getindex [label=< true>, fontname="Courier"]; + without_linking_getindex -> return_getindex; + with_linking_getindex -> return_getindex; + } +} + diff --git a/docs/src/assets/images/transformations-getindex-without-dist.dot.png b/docs/src/assets/images/transformations-getindex-without-dist.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..e18e9686fe9e653086e21720b0bfcf8408068247 GIT binary patch literal 40798 zcmd431yq&aw>FAMNJvPxfPhGMg9r*pNq3iYcPU6o2}qaHNP~2<>6Gs7?vDMf?eG7; z_l`5ZbI!P9+&j+qI)u%;H!J3P=3H|<&wL5_ASdw@l^7Kc4(_Rxq_`p+9D+I=+@ty@ zNZ`)pE+sSghGZx$Ar1$H{rl0F8v_UT98OAHRM{crmqK#>4;1>%0*J z_$NOZM(`9Kz4d$ksCC|(X-ciEsKB!1WuE$k4&xKam+-x)e#-PCQNKQY`|vm`N;t9h zkmB37*YHxblqiCWLuT!@j+=vgm;H17-Ww^7Pe(ar#s76>jWOcsz+wM$Mg5X{6dL@m zD;c-+-!E&V{%3C&9tgV6)z}y6Hr5sPQvK^vq^xt=Zd;8M(cRgs$14SICUIF#{Hb*` zbLFP~_h$?y3N=Dm(U;WK{TwZ= zk`=VEh4o|Vs|9nAOewR9Q$z^B?6DuP~ZMuZhUs}T5Z$4o29hJ z=elzdzS1*i4*2bh9cA4`J0bZwk?6sqhc0TSy``^4JB^vwoK<|jdaO|bZRfMG!NJ|J z?-$|07>jwCSDHN1>`6wCLrEdu!zCp9-mf0b$ZwZ9^V?e!qpPsLzBH+mS}axm5U+ni+RjN87vI+}Q4h<8*1&!T+c z$~YqCJxACaxla|fYztKxR539H1sT;DVNk^mI(-rp!OJ4r$~0y@8Ig3|)JWLMg>s6( zAr$L`V@F1Lp^tG@^;7IiQA@@BjPoo1IwAxY&j5O;)zGntNLl^Cz1Q(D*+=7Vz~#1f zKY_2f15PVWN%~2=se_|D@o4e8XQk<*q{afwHL>i5`j@}cIncXWSa~@R=xwnHH!&yp zv=-(NvkSE*C>XVCLK$&V%A^yU`2whNhVWph4~98@6hnA6_tTx3=hlx4eEXP>KI{Zm z;u7u~6-kV~TTs74J~>C3OnknYlFSfbYjY~N$;GIS{+OdHjvh2+c?Zp1e zgs`mGxAXlk;)DglHdU{%&$o@q&F_#8!)wwL+L*mxz$}RSFGErXr}5~8Ryz_tw_$?-?<@0_s>*i`Mk;g7t zA#Zhjkc9Iazo(?TX6)`pf{F33d|CarhpTaP2g$s6H#AIbMWo`WQ^hPrRv+k^X%>p= zYJ-zqScX@9H-E9|9UU(Qp5CkxY)1>GdS)(~X-&9igAn+CStKH$j#_Wy(AryamN#lS zDwnRdXnGl{zHsAGT=|&LEO3QTnc@tCBGN);B$|h4c;BM!Zg0x#*XR!uH5<<*Swn8J zE_)jmJS~Bn3)FyBk)w~dtPMiw=#ck~L2=Q^YFWMlgtm|tR&?}|M_#E`(f6l>J8mf& zriX0;p6~gqLj!Iybyt3>roAZf2@+)zJ1W0b%u3 z-!U?6oOx=CF4!nIH0W%YKk#EntzxHOlW?LDarv|El=m+skrjR$-z#rZ7Pmy3P2D$L zPYjo82p5QlV>^E~O)EJe%;(F}crk~`0T{6qqRM>+gQZ{8M4wBF7 zFv3^o&Tm8kO%chLEfK~>RRi8D{{Ho1DlCjj(wg7mA3a?0HHgqI8bbV&9@8BeyOy9b zB*Jzym8^fY_&kHwYY5rMCPxSI!AsEliTPN702CWiPVcIuw}|z(L2)5tW}v|a=0No4Pzm+|cG>(j{oj`>w0~j$f7dxI zJp_%nBRVK~$$k8uQKoi{MVEbj0-rg8(IX%ajgd^6h2x70lbFWT`HYHW1cs`ty@jM6xk~|!XkSCRgK9!2C9aB-7 z_Tk>sKK*IMhJVuTHn=DRO#N{^icoZ309A}cie8{2L0YV z$MYoq3CDI?{JDqA`6$ih4ds`wf4%Qrt}syE9rC9pv0DnAyyib>(Iifi?Ha^ zngf>f3O*Ob<3u6%L0 zDk+N}q$;;^U4J|BQ^Oj*aqmji_O)@dr@N~G)8`^lJwK&gz19e+;EZOW(peSTLg**&QzNcD}z~g zvvt~_c;=kV%nEk{$BRUVdfui(`3E6)B42N3W^X9DSh`u$W}5s2hqamq<>&CwSiPZ@ zn`V~vdSkkvYU|bM>P`F6sY613117dxdD}^%OOoM8S+n*#@o3DuR?F4q+(jIYn(K9% zFs*cl!P}n+Cf$79p+>nlT;41k%~urnX-QqwBII;5*Q~3H$KALaSRBdo zmeog*mp)4qWo&0&4j2&H!u90&ZCs&_!MDOBHl%7rOtv|CVlmciqthNyd_Mnxmkj1V zzg|&)vni2VKesV&2U$3#OA23+eFxk5!mtCc-jitbnD ziKCMzD+4^omr@K~Dnf7dLNOn-)()j>j0wM^>)A zuj2(BrClGmcVEW8;&>P|3yeT(QODzcmCU9lw#_PT9Q1dfjMj@uXTd|7 zpSFBj3E?zO-Sz@K-PQ{N3KzW-zB3|kqaaV6t(pnCORGKbGc*N1Z+o%aB~fF4#L&Os z^es!N>)NOzKFoVtKrW8G_NA)Uckw}T;n1DlrTEOXke_%hlNP z?d(|_Hok|u7%16kre3ujJbFvN;chh7-C}6P)^$50Qlu6{9tFRDj4@j1m!naZNM{c( z{=NHP{i5^Nbe7tquq5Wr^K8v;Sk-|FtBwFmD z=-(7wY}U%4HfGnYPKp)fyfv;OI0U3rG>bU>UqsxuM`imL<2&htV!q=X_otPAWjV$4 zq&FaX#$&ynIY6J%D`U3W_EZo$?qPZ`^qhmie=*M5bmJ$7WpZL+N>c)I&Vk|fgK~7a zHg7pc$ZUPOyzqS@Pqi_=y;n`Q86AZ( zbGXTs|NRP)z+O*is3v^->TQHUZzpw@Nw&S)e(u?naQrx;j~vhc$Hz5YD@?v=j(Y*^ z$o|d9cE<8&F6{RUPxd0nWRk0n`&o{dFXTPaP_#Huw$2FInmb~moWItnPz+mlVHVO` z&;M|;-ka*_EFw~GOBycJG&rvq*(f&nUDeKJxpM;F7?>iEbGmOzP>K9@boOvhxM19# zaLBhT!5TTHyT6B$Vg1SMrjV4xkbnQx;a%HfoK0QY#E`polPh;}HaCh2;qrQW_wExyqvb&#( z#)T_A(&Dq)j^=`g4Lfq*{@tW8^zirvLf~*&+@qQ+$L;r>%#zvGW8q%Vo1zByM&xFz z_wD9MHLM>;nuNo{oZ_s*Sg`EsB09^izDad+ottka){psr^EfBuYCzU&<_xAed}`36 z^*Hg)ci^Lqtl9PqH6naW1&OrDLx#~9B=I>*NC(* zeI>A3&iZ-GTFcGT8TFmIH@W){ip6G*FDAB_YoVT+e>w$H5glF2ib~Tc3GS_yI_<{? z$Fal9_^T=mb~{!!qYGh~MgT0|dNuf;A6QvBE(T#SYhz_COY5lNYGaW7zdT->G3H*G z_mW0<{9evCk8!&SH2vbCRdzG6mL$q@ybQ4}B~|Z^oMn++^!U$?*Y-EBo}0o;6BN>n zW+GzBdd0Ldi5-{UD&(FvdYv46u&B|fepYQiNNE^ZQ~G6`sxY|ujC>%7tsyg(dQsNE z-eLh~T5w;hshk=l4hWjZotxL9rP<7mZl zR{K&L2<0=zR`lk2Y6Uf9EMtzNQqSS5q&UF^rYF*E3KQf^Sq>dnRM8tY(V=f8S2PI?1Uc4DN@^STDTB5EVYwduTbYn1ptPRKN*M9L4^K~of) zaEmTZ<|;?H9W3S4*b^-3vGpxPT?azj34F=!`9(Y|j}mBUX>(9=%>5L?#UxU`L(|#O zoM$UDO`Gn`5hh>sVLPRohK=vNMALiCK2cGACO@!x7`F?o#cyV@Jh; z&?UhfW|@r%(a+!R8I*Dl15JnQ)VeCH*QOrG?@N56O+wzLAZ_rXrk}RWB+Ldmw^j#J zeb;82$q;XqusP zXe=cBWKg|g@=5;lEiD}xvy2*lSyyU8u{dp`kiS&Rw*5O=TK%L?2Agi>D9(fKVz`?X z_BlS;e8nE_p769p+e8GZ&ABgjzXnT=f0%Ca0XvYn@qKGsKz%oRD^dOT>tv<(_W_OC zoBH(3qamZ_e&#Dq{PK(1%-Y4HLLcQEzj;eB7yLsDy=Vcn&}cFpj222SOJ%Y>@Kc;9 zdSSh{?Oz@AIWbYnG>SnZkK<*MZc|8qI(s?-PPwYQ(MCG{%$zg99O_iy7$ksw)WO{S>+tt8Qhr855(#aT6g7+Ed zEEWjb4KKJ0)*Gt2b@b!P(cc}Ib+eYPHT#-U)XLzn+9%N_EIB0R+UA7xy;$^Y3CJuJ zELBer=Xh_gJ<#8cfZpKUz;cuJs;3SBohuKp)siY^v}I@027=XXk$LhWSrf><>y6l# zqXy^3fn&wue06_GF#TC1RcPG6Ck!)B$t)Xn>CeN$6CRw_YvCq(EKPTg+{5Fc?l%Ty z`fTX^q;zsI(ykL$D7ijl3|6!4bz~SwzlCN)X^I$kR7be0CVuKNp%ohi@oo5-zIAG1 zwLpJU~d!Z~Wwo4mJ zQM~pk8hKo=*Upw$s>gWQ+Me(1E5?Rdo_GZ$#e!ItG>=lJT#$1`__et9i1`YO90J=91(N-igY6uCL>*&+d)9_ z8B-7U*X!w*^Y-IC!;mgJV%@P~tyQXK!?=H+yi?UeCUJ+MCIPWBUeG& zQXi zs0P|&8{%4Ell{?1%aKaUbCtg;Mu(tIJ5nJ$IrA1L`hr7=R(>OyPvU3~Usl=w&dBZ# z<>|TKSo-x)7ela6tUt%;i+X}^v#~#yQ!3fMp4FpGwzhAij4!5|j~=4EUziC<3Y;`*32?-a(5)^idwefd?ZXv@jF4p0yh{ewfJC9>Yu7@Hx zM;yzq9U-*B(@ulqKIj5E{9kYq$cE&1F-woCW%j~}|5UWNk&%e{kblD)x@;!hRF4bk zi;~?tSad({TjcvNLlrMx8{7IWsM`*{0+Zu7w1{!a^J}{mft&=lL}Z$NM{iNX(KL4z zF84z|aFr2qwTh`mZ%n?bCsbNGdt2(!&VcUu^#SnmiUI#E!5C|5*Zg0Qq>$;K&ydEE z8zEhp#4*3uWZyZ}C$WkMaHsEZew>b-C3lNSknOTh(c<~vQluL?2DFFmQ>?5FNJq}) zQWrx;d8_uOiH&H7d-P7GU(qIt9`fSd@?zecf0o<7=#w{hR`OjD18I7hXn z%nrA0?jSi~I+ZW`YN;n#WP1nkZ_40WXNzs#j8XaPnL_)?jB-c!L#F4>k4>v4EyYVh zb<6S@zv~gfmzYCvQtu@1jp$h96Xc?&Sp{hJ=T?RP9+_ zT)q0Ihf6o&7rx(z;+dBMB7_43ds^e1wd|HeAqu|PktAeTf^8)1HTnMO_y;tDh=ggs z52h1`9=N~(0mE=(Rj0=HDxA?hpKSjmm$B-Djn!&s;`;O1dVFtc!68eY#e!MZgM$ri zwOJBd%*XQ0aQx?6ZJr91(!4}=+p$_i=(cs3`w+Terma!lPgu5?RxScumVtkYJhgP| zoAIns$sQb+hXMt<##5|MuNTvin2<1kp1)PB{wL!PGDzdO)jfXu_U$7q92_N6#An#G zDOVBPPE@SaG$<0M%Lxy z|5j4nX{RQ8IfA-*+$bpI!_h@uVlua1`Q97FlD_!l%F4K6CnU}Znk9_m!npL{L2&c5 z;>B(p*}V|xu(X_4?iWn&)qc~{?{&sap3Ixh6r^O|-F{g0?^=MhEJvqPr%vGqK3~OT zU2iJf;_r#CZG27)sN}u7koBM%?jCG9NZPjq$D=j^I-N~zSUeCsxX3^68m@ec@+z(I ze6Q`OE2`-n80Yi38F);f)2uifC(Dz!`8+oo?{r_V(*NDbE4MrgVdMDUQ~6Zs^fYGS zZr^wwF7#|XE}8Hb@7Z{+^zQf^?BhD+OJD7Vu+Qh%Hs&;3C^M;+-1r~*YC;Gk6s3BR zT$;NBqW&R|{vltVp;B42U1p7ubP+(c zx(3C}uf}Qv7hHvUq=)9@yi-Nv*nBf=zauQ5L6%jNAsr_yD9nr6C1$QE(_zV{<-F|V4wec z{ZHREF4}?zYjJ=0 zJ1;N6%LPvsa&mHY#$17!q#TqMe@x*gpZeC@ww>5r59UxAa=`+2*Zv*2WVZdNEDtso z*6a80QRQ+gy3gmEC7;uZn?x)Kqr;|J&o0#f5~yQG2J2NC z1{@$nyn+Nir4|sV@9rYQKz5yXo}^)ng8h02y*}(?@3Xb4ZN3Bq;9a|Y{P=QG&-ea6 zyx(D1hDAoEvxW7%etvM?bupFa5zse+B~5^RM5uT1tM5zJ9u2!2jRqmkSbnq*41Z*x*|60x3(y%aeb`B0U*_f=X zEUNhI+}xPhSd}gI_`bIe4h{?473JlU^72FV=`k^oe5G8}1{ z!=o1BkdkUI`MiD&2e9FO>Fwopbap0qSSctZbbNeFU)tZ-hh}Sg_>2h$OtbfI@s=d$ zQBF}&QBq2Z5?KHHcU&;;p6>2$G}LD}IQG1xSvffyh*V%?Yin!YzJC{Yap4BTa~A~@ zVaEva|Ni~EVi0aEvr{>Sex>H||I@78&pg`K}Bsdtyn%hOrH#VczAFh65wHd z2IjZ&*Im@Y*4B)pqoXBdWoek0m{^|IPTl|h&8euQgf_>rgCNwrw6xUloSMh(=<;$% zpy&MjoQ9E+@v!XECpb`G0_+TjfP`dNrjB~%_0np3eWCBCT8S>4a*1xEpS0q|*V)~? zQWGgDDPB?xxm5mG0^(GD=dUR#q!)*azO8nLh7EvmIX@pj({_L)-(B9{-935o;{t;GTp8(Ue+eOufn)Hl9QWy__pvGEuT?D_Lg zEiEk#F+E^#J0LbJJ=8{SUXb%v^L!frB~oUx?trf9aD8>PhG;VWOQqhpBzcyDj;BUS zPL38Vtx~RBR7y$;!K;ntbQ1o~jt;z&^ngD@#*g@w~V6vL*VNGFI(AxTIP;QIw|_ZcVq&z}#P)i$ZF z&3SsYr}C1osPMWV5bBbk_vYEz*?7}9`)fTh?p@y+Juk~$_GcPadukia_FQh=yJF_x zXc-y9W-2X}GT%LIgFsNw&=`)rL_stN_=_~COuSSNC+6xc2V9FoBCLA{#J=wCB@m(Z zW~(E;Z!hexj+Ps4!B|Y85BG@7>$}_AQk53tPcSfO1qD-pL5D#pq{aBJw{C8{;64>E zZvybv_??kjDyr-#fxQx+`69FtaOt3NW@SululKf&s8h|pBD2-DW}Abld(#y`Fl3nxkpuhL@u$LpMfL9tCmF4=H$uW7?FRL#ag&e#b|OJEE{&_h!rba*nvmz!e)VF4kS8~AL1a03T?z!E(ghrKDbD(l%!@PDewTcG*=f&{?wISmcsCjjKlfR|u2%ttiZ zLLl*dRWcXSDB!%~v|t3{JYw6akmp6I_3kG ziltq^!_F6nt+Fy)?r6`&{;Wp53lq#RT@hqfU}v=@^#x805Rm~u8AiQLJG!-o%u z@!I0zy_#T;0_GNf_b%;H-O!Kob)bRu3ZT^9*$Fm-j}wJN0JMf%e||b0%q0TD==wCXZ;)=R`xY*3*6k-e zyf%8n;#R2`6I1Vc@^I;?;rh~_GdF~$kS|?$KFJiHr!R^tGRolbb=7>(-tG1yi~9OT zhEDo8LDC-C(;HUXzOFg-M2WOmEs}cP@+~fMpA8P58IR!aRs`e&=?3Q$lOIcL+=>wy+1*+qXDGmqY^EYSPG1ZK8bV1g$Rb{3F#1q9jeXfh%T_m;*pUsoT zO7uin>}nrwc0{VOA<}hyRx9+GwL6&bTDd^e`#-TZz#L)z2DW_ zo~z{m&hmnlwKOACGe3bI+0Ao=Th%lHPv`rJRB|U7B_zQY5>J{a5Z!0rqB?ki>3*{) zv3)bBbe7-0{GN|J(rdt5frXW)M`ieJ6ua4GQc#-FpLT=0)5>i0yB~B-csg0^#+8~N z2YYHg^Lx4J_TcezK0dw@4G|HatP;u zqHjU?*RSx;fkM)c`hrRkFCnJ~w#po47>*nL1|X`!R=UyC6<9{>aG??GxGk)|4oWiP z-*o1vpkJLlU!Jc!mV>gsv5xif)j4<+n%F<5Fi^AFMUrrQtA^h2=CoyOTP%;uiCazR z?lIJ}5wo)KoB_H{lRPe8l{HKE^h&=ga!&2C*FtZUbGpi!B$R-~bU5QJqgGYe*qE|r zrA5c5M~?}4?Vf|=esYloyLRRK+a`OH`?uKG_M;1pp6GJHR6nrsFO>RDQ^O4{0)-lzJ;WnZOh0~l!P7~R?q@}Z4J9*Dxc73=SN_c#_)vVN42ei` znKZ$~2r_|@-{wlX9@{j)m(_+lfO{*R6^V*x5dtS{m*trkkD-;lIvbTI0`nl@u`vcg zy#e}gd$u!ta_uMD!tg8=QJ z`@G(4Z?R%zfA>nz@OT*^QfFea*oM-Cc1~E{Rolgnr|8@vSEaP+^6tQ3&&;v^Ld3jXxOc_&v7x z^LzFsZJ|_)V8z1E*ci8mgC?hss>!ce1e(T4u%-`n7jy`vU7DCStlT1=B!|Zh*)Fx0 zP_Wfmiv7C2`p_sPPkv`gkeHpX^@tf`=swP5yY;NLl3`&JM3eu!un1avyRiE;Ucx!B z^F>^Z4b8k+hxKaoUd8AxF=1f94Cb9oRbso^uUh&L2Z|$AshJGp!ydypN=t6nYpvRy2l=a`!3|}!LW#q8)o*rg}Avs9nvX#d40<0_!nLcq^W4{{oln0apzC0ow@`18N*2|3c zhp}Zgpe`006j??jxcB$Z2Sl6?B{3DMR$O-@4MZvDV-WwJ>8wh+NuJKEdD zV?JYsaCGL5$T5}LbzP)i-WG$4DPT_z(p+&pQun*f#g6C~skK7JrLleAH@-g74Kp_H z0>orpvx^dgP*%^&CpgG34g zoO?R`t{2~LJ~?VQ@{(?(@HzIA3wb1bng%~I?8poD)N=mEAS+?BC>C3Fhm z6z~}=V&Yzq!D%$Ou~jd4?5?|xr?VR@&(zzPbzBK(TpygD36GhLo?L*9^_Obnqy3c>KqX%^p z(S62FmPN7H*GVC=93`x_9CNEIEk>@F7+{{1kgr6vhNyer{9>3>o`u6P`2GiSRhurP zHeSk>F{7t~W7_FuKwch3ozB^4Od6@B^Nm&Q2z%=M1of7rV;2DJmxc-+wxE2F{p8*&!> z)fIbJSooc}_IAA=Eb1=GZN%VdX{AonR+@%fjW{U=uL1-@hw}R^6FZbWvCI$5+Qf zSh^D(2e3cC8x~S0wd^mGEy^j{58a5{dqd75v!m?zjUY!2$}-T7)_k>14yQg)K%r%m zgI}B0Co?TP@5NRZKmE(5bla!`cPr7r5}H)IDif4H+K7lgS}+-8YSna86X9jCuwhWru7J&#nOrpw9B!>g(LmSHA4^S_rvT##N^R**aBA|8L{qef{AvS;1 zRjk6SrXX`Zye`!~%3IZ;W6rE2Zm_i`rlxRzYlJNUUjE!-sM31hi}ez#Od=IaIihj? zm8=%7Ei|qW9m(gRiSX0%YEX&6(&3@&MiYy0U{6#Sql_78{biQXhSf%P)j|V;eo{h0 z!p3&Ny1w%2A8b0i2Q^(5Tltn3)Iynen*+5n=!C4Fz)52SP{Kx za#e%Y98{QrptZz%w`mg86@+!TH`1 zUgv(Zv~QqqY8tI!JkB&mge%LkD83<|DDQ`&QIN}aDvxG;P#0=>GH4*JRK2oSG4k?2 znBftJD+6AB`R}*(7q&fp`s#y>73!sMfgRo6aF6ZkwDtMz`DIyUQ{*N1Py_u9UVHq__PNp}1NxP* zfqxY%Xu02ne=N2p{aUhrmcG76)~D{?e=jBYUYW;|Y-hZDiQgpwt?{>l5!^5jVsyFP ztJ8?wGONEu4>gvc=;3_QF4u-&#w^wk(|JdwE4m_Pe0QkU@WTd@s%2um}!_oSXl}O5)s?pYb|J?*3 zBfKsh<`KowAqws#h175QkGpOXWflnKa|DQjm*qd^5oA5|`wHud?&XuFB+^YceiY0M zZ0cfm^-*GxowphYvDy#cosx6G<6kwmsV%T~bn<;O;$zyo*5k#o`6O*VBgEpHZKC&ll{TdwOoZJ;KoyWLac4u`J}IkY=dUvg5w26DBV-aeEP?a~3GzL9CgK z8!APwtQBlbo~(5mRAWV72pI(lLe}t_and)Q(@}qZJnL&P*d@<#HbJ1S@DVeP(ZzOA zEP~0eTQvZY)q#?ThzNGp2MLy{lL9zepA-_GQgi6E%{?^>>4r?#MEvRcnF;^ z=}=t975nP7`k~w1HGQ6?5USWaek;QiO36bIgixLV)W< z%67ehsl&$so(9feJWQ6zE9R9rAF8Qcp{~f&!uvIrcJ0>{+*32d)oL$Ep`RsYgO-6) z5W+I$?^N$7wLekt%yFa?gi-fSntKoTGGDH+<@d=t3z^**A@lZ&c* zG(s;`F6?25o84>nw4H(arPW$^pEG|HM`6{Q3*T*JV=Y`g_t@4SH$Pmj+?0)?;~IK` zZPai!%(^y|QjwVI9cCZfKC9T~jcCIViYU4(=SGaAZQL4vtrsNSw(h`Y+E1v>OyD|a zFY%T@LOe@xSP!t1UVus)h}ZM!0sI{i=SM8S#^UL`plA8)dej=3ern}JK}8a@Y*xKY zG7}t8E!1fyy>aTa`-awr^ZzNAbMpkrBg`FPh{%yiLoM5mD>RNVtVI{QnhcgD?osT` zMzixt@Y|5*q+HWq;QKa5qrGEXnB)-rCA^W3%kFoBJukA>ApB%qwYs-u=8LCUq5h4d zYa021Vf8YZc6#?iNAvY2{{(Bo#%Z@jn) znmi&5=C!cMw*uyK`Om3~S1*DV%`W`nw@pQ$=6tgor_CZyHPOl6Sn8booB{-W?fu1U zBD)Eyp7%K(;K;zlffUohI)4?y4K^rVA^NP~?sa33M8FVOqs^@AeXep)yGhQ7Z!+Rg zD{Xp^XFA8$%rgp2=->a0r<2wvIcTaIyw}1Y0Ukx1AUyb`F|L&Ji!oV6{&&_b-FafE z(WA@B(Xx9(KB1fPwipr~2iJy%VpC5INBVMa2+nvGEYs2_q5@4>3*P5lw2EIF3Ao$(7hoQx6+9 zJqOHm7Z1IJZES3UBO>0ZsKicXV>?rl^t)ZZbz8hbqOhs`6yaJS9nM)RCE3s3krlg( z)o=G!xouo%>NgF+V5l1P_vK!leC1bwuxuS2#TgR1Bz^Pdjon5+Q6Pi3c+pZWDz*ZH zm9;fC9$o;Dc?55^GTqi=)t1)QzL=%CBjxAk?_~yfyf7Nd4)*YPuDIxld}|x5BRfRt zto2!}QbkgOEo1aX@8?BcSC6dis%3xs%(#+1y$hd*Bo+cti=Hzwq62EQy<$i>6i7oW z{fQi4B7xygS*)q4ES~yOiHle2*5~}Y7GUh1TCsNPX1Z_0kO-6%&{%DMeu^DmtwtJc z^d}yjp87{AOk(!E7Z)F!9<$(zj)^G)6r|Vv`DFLNKt}?b5jZ@2rAzmEc7Jn@Q!h@H z!8}QHjU=CegeLpofivTro zd2Q{5OD!O1i(b8a`C3u&84z4bZ5M^V$Hp4KM6{IQG@vwHma%5g)@a z>Wb)?Dl>xFvBGS~5HvkiZA+2neW8)3{0bzz@R&kpSO@io!ZInm*pML1An`UIg5co1 zTNJ-fi$1qcB_c>$0XJ=Tv?OY5Vj@Z#(>gSSS)f|f2KHiBKu4EJ(@?R0L{|?E=I@@L zCu3>8W7qY$HUVigj0Oc^LQ+ny(=6Q^DfT_0mX;Q*7r=P8e?b8ik1OI+v$LT({Mgv% zuj5(tIzc~2eJsr%#bUpHMeR0bck}c#y4afrqEQzJI|ep3%&_?jAw9;Y%8)a*v55c@ z4FUoJ9Nf#F(E;}?F&i*tAXD| zQ+YZ2K2Zm?M8wkl<)Va?6w0S$Y65q6cK~7~k~%=ZekDVHhJEe;!wUe`x5Pw{nV=gR z8$ajZz*%2khl8utjsQz!wA>ajQ$TY)Eh;|NNLY%6?OXvUbpznQ04{+SP2dqm3zETb zXk=t$SEc1799%?1#L>x#Ur-P-*!7Uq)YNKfYYn=;Q3{esiI0hmO-`=11)y~g4t8Fj zZrUHr@udiR6N7E?B@+`O+}Zvtfv&DDp^yhBKBGFEyu7@dyZcYu#U?5>d2w;r+MT@y zJQJHqFRn}?d-n7_sLmO96AxyoKU-y8Z$SezGiRo!~699Ne zjeQ?*x;bPi_B{tW)Ys>)S%hTgxW>=J!vjQJeJ~x^pyT5S)*NcP55e=^nh;=9$w+!! z)R(xZGRfRU)AztCuAGA$92{kymo~Tr1drifJVOB*+S1Mrs+pM?uYiEjpC6)j4i0TV zj9~lt=Q$wmB*LoSijNuTH2aVtA|iJ8^@)j!K0-x9i%L(AY-kWPHZ#l2%X{MY9InPa z%oe?a!ei^FDP?VVczE~Wm*8M0*ur+hP0!4GkWz54!#2o0qx@} zz=^W5^2glDvNCi08eTrWdF;-F#6;&!Eo!O}t;)*Efj2RCcV4*o_}pm9#d^)fsS6Vm zs-9uzM@KD2kwO!U;>%Bj0a(#sU?uNtYlF{x7v2UmCQUExY{j27HH3hAeaX!H4qynV z)!$N6!-0mnwz-*EUalz6C`0S7&D_=U`Eez*x}u_fRainoLNa^gz#~~)JT)gL2Z@G; zX2!_S-kuGx?k-*4KwbOukrNJ1NlEE!cM{W~mXw*9*=?_^>vxwp0RcfxU0oX(-kR$> zaq;<(M`+jzEYvYQODikz1O)M_xwUS`RH&$^AC#59WMoij*Eykp=4-%BpNDHRuq3PC z0ouHRf>GhFu`yf>42*Am>x8WOpSYd27n?-@D-!f@e`^V~wzVC4FbK};1J3`Wun-;Q z1M7Y9_UpZP`$fmS!V=^eRC*X!ouR$g|w_uS9mxh6O)0h zZ6p{5WMP5);^JcFC>!u?c6N3jUDAt+n8b)Obn}4K0M=$&vdlk@xanzV5HagDh4sa= zWP-7(sHhBV7#JI82@Ue{^BXR;d?w_!O5heZNB@G1P6+4e=^3>^kBv>lkHiR2ZxgPk z4&?qrK5JRor(gjl_R4QNn@MaF6BB1{SwO3Sfq?=Wz{2`wX1S+DU?h9;fs!v|FMV}v zU&Zc3Tyv2l6Z>WW>B_gJzTWckGDpPhcSQvR@Ynjr#xL3g=%_LlzJO)Ic1Ck_bWBK2 z7GsXDrvIi;Fg`FS7at#A?ZLpv2y~smi$kE9RPxG8OG{4yIQRCVs;a7%R#f^B#I*Or!OmX?nVOnzjpky8g@yh0 z#o_kKPiCk>o3X1;ztf z8P>$a1bD*yb6FW%VNuZ|IM|Qe5BHgiot-(;v2b9$z`Mf_);2Z^g^MIOI5_GW8kOp5 z(!~B5RdwDE)(RRLAvZTS-13WrGQV>>sQhmytd#$EYWAtLg%~^8w#7a97X7{qPGVbU0 zPEJ-A>P8=?a^)$k^$o49pOTP}>@U_8Q&9aIhr>Ef>+!vvEVtomG@L$(ws-9FSFVGw|h$1PCl%TYfbPEUwA|cWp zf~0hVfTW~K2$F)7v~&v6DAElg0@B@e=HhvO-}t_N&N%0carPL`GZZ&#uXW$^o^f5* zoEt8UDk&{3wisjvw})Kr$&)9tj~`39y9g0k}snq;~lDWtT42LnO1;9MnRufB!ZBl$ru);cwuml{GYi4-Q<<-e+KJ zZEikz{1_|r&6~|3ZQGe3YVTJNi$TPcs>+f=?Dm>kmy}KJ>Dm&kyBM-kJ4)P?RuP{D54g^H>8>D^9!-Z|&rN3WX&<2U| z!Lo`JBJPAxEY;cJ&E zHw;)4kWsXzNe2Dza!4X8btk>LT97eD4)VQ52-;$&I|NhS$JT^IM7Qbbufk0m#mTxi zLDmOjr5Kr+CBQmP^_Mqh(fL5N!j?YHmTxx)1U zDmVV4x5aW)D?RNhl|C_R==>A zfp8EM;Zab8fk1)|ZpY_rJH>9M3H{EUJB`!R)Hg$VPnHNJUrAWvmmNJ zoWBKOnU;nIb^ZEvyekh@ZUSZMj*SEuDcu!_s=?vm0kBq9p?~}-xQRT_+(@upoH?={=tWOrCIv?g-{C+NV)AqAkNzLMEqHBwXB*Na(WOW z!pFa4YikSgm&@RZuVG^cz%!5BRlzkjF?rqCD8AbB<9TKRiUTrYAZ)2eO5#0p}seal!J0&0v9__8N*ar+~ zYieuLLBR%fmjvkHKv+^_(s2{X3!&tOB>~HXZKd+PxI>JLtSmaXb4XT^8j92B%oDOf zI!;b}UtiyqudzFbkm1klY;;^)5WxC|kn+E@vwL7*Km+B}2ooDTBnF28_?$jaTS~wU zLk^3kudk2f(}}#+5>SM|To4YEcD!QwJJl>zIt8nQNY$Z~qdJyS_E1I!$=@!v5(@`8 zr0Kk%e4qe*5qf}xZ&48ka!{dqm6ey5)X*U9hWd~#T9yGaIye*(pwZ~-??>|R+M}gB zI5^jEab+ONkKBdrB*VHe4VSe44n1raaxJ;cn=I-DQig`N;i2r{iJ|yY&f3`9lSU4H zXy^vicC-QlWRQLO!IQt2yGu?^esQp53ptVJwD^`OA6Mjk>JG;Er&oqLKrXZCXOfWJ z-ijt97)?`Cdhj&WufxN!!9_(yNBe?03G%kruV0%BwcKN7zKN_pZQVu8&kotyaeN&V zWDJGLL7Ui_FK7*3G?-sSNB`E|E{j|zK$STF%30)I;R8T*W&~!9Y{1Kx9IyfZyD?+~ z()9|<%ZcIf-Z3Z!78bJi_4R=o^y>c(Jk%zH@WIMem6Zrd5D*Z6Am4}`;^CpFsi{Fi zlAfC@cd-#tZHPZ|&xdpJ1*B8pUo>9g6#S~e`aII>I`FVq;wZS76+gFjYQw` z5@a7yQF!0m+M2t%^3WWJluo;BGCV!;0u~C>3-n$&wm%lGzejrNw1lP}&ca#vC(nmO z!eN;zq)v?Lc%F&zv7i!F^fhv!mR+d5K^$G=w*4FwxI(VrXE+_^A0wKJ<^%$K<;YeR zBxh>N$;-}twxG2-5_C;dh!J4=FOWx=-`G(^@SADrxyIVI>6z7dVpWVz1;0au&aYu!kage zu&0RN!)m-7yP%)|39cXtx#Fwt4^hZ)HiwIrmP?k;W}*>7>L$c!BuK(aDQV zFYa35|8e)D=kvQbN#+Nq^?!Z-h!LE-E#oqM@zeWuLPVFi@@BVPcbnp@W*QyW`&6Aj zY1F^@{D?aA204ubHCiDFvBCSe!Ti}?mnh*w;`u<_!_z}289Rd`41Tdck5~_WQOAbk zZfJ;04cv1TzDrB%|28HDPV+TzL16O+kZK}04mbzE*v9V8g#KKLnEBPGk+Cik(B(F) zq@E?+yvtH)e}KYa*MCEet7kjy8NmGabXqLxAcd|nUf=iV=;}b6?)SL?sf=2}ppTvO z{T9qoHQx7}3y*OG$TA^q2&$eJWRAEy+rH&=uxNnpT89b&?e;Z@Cu1p_3MrzL5JUVW z(!@cM|0G$M$vK4fW~yK<`gFlq_zg{UWo<7?qPBON+w+duj+1t7tTePsytRJt3_HnM z`9sNzQy^f}(be4?vn+I4{$R5hY|t%4h$SvQ7k%~>XR>D*sID&TgBLP6-O zMzCNDh-;Am-f+5V3pg4Eugyd-r*SKiezDgd7{%#+7|P7RA*DPflSbncs!H0!-(nE; z|K)OO;9R(n%YJ)#yQ|2J`EozizQnXBxoXlOdX{(bYnvc!svD4%5FpX9Pn(2Y?>hpl zMv9%0f)__*SVe#X3Ga;;bT-i?8 zK7Nl&NtQT6=b;E%p19Ah9JdG+CB!z!?E^uXZ#7YgFCWYL0P>Gy&#gzo=O^2w+~!du zrp_X}JhCThv%OhCmbaFNjH#MVjcv}f`6op9&;HxLxyW=A>&ttuKl(O}qn`%yHw}4R z`Php6`a^j`0M~nxfy+z68TWL;xKWk6p?)uy#cr~w>h_{eR35r_0vHP#cLYa*gje+J zz_|8!ttMX7OAul*htrbkDUOy;V4)`2N4ltOeGf z_(eZGU_Eq#f@20vM5$CQLJg9SVg#;sdk~kpc!f1&Oi^?mei#_su>aRiB5|E{EhVze zr6IDB=qaap5t9Yi(~WI?x?<}N9KzV?Q0pt$SFlkJUZwUccD&x8ztT4*lijqN&U}rS zfWv0-@AFe~bGDPfw|SQh#`_2q3{0@Dl`(TqEN>Ct7aCEl#oZ&g^jk6B_21K#gJqvh z`6fvvR{PiA+o{T*&_@x5w=_?)`qwDnigYQ~h34ci`T6ZI7PelaTtWR`?V@rbW)=oF`8`f(Ei49p4XFK*{;BzQZ7k$_oO9Z1v-}Qm zHxCb_qG@SqIcPXP7Q4g102##%usOIzbMs%N;o27HA9zv)wr~72)eWj%P0Ic%%Vj`w z6pxlm-O#-G$<$nFSTe@zO3<`e@GXgcW2{JY1cFot$XmIBiJiR-Is8Cp(};;tM+$FU zgOWZ(z+t|zJv-~sni@U`9SPT7a^y{Z}0XO0f)SRUUOhdvNoH^KeU=I_(p za%M~LlIGq_g!;RY<7R*3(zm{)#l;UGBR|`UYXI~Q4T`UapsWUw(5tKzZ*t&$gpc;u z*_}VU+El_KpT;n&7mFBRNRebliDTxas&n|X$WHdSb_#P?h2AS^5DC{$32!V!(Qx|x z=*}T^yp4Z2cKyQjKe=%I)jCMY-`spVu4{8X(bj0}>kH@x5{LXBtjk@g(*E7z>%)WD z>NXD7c*@6N;RaS&jj2%!8zY@|E|dc8uH8DqN7Z74Pt-WxmUE;Hx*j|{N1y7iGgvo4 zBOst79mjsv{C+WhYqN`VVaP;4Zk1zakSG3hp0(1A#o&eM<(zKJA+|MAL0f#PNq!PE zY^wq7^dZ?y6;SZfiHcHz3SZ-^IX<8i!E26J8QBE3{X(}n4RqIA7>d6mDZX_^GP)^X&qO~H`i=#8}TTod^rYRY;gc7;_ z_QiO)G{A&VnT(9rT3abHB7%3+jk78NOMvRRX`;%Z7-~9znPB^7A&>(+A|-V-7%5@O z9_i{*LNNk3&~IoM121~Lu;EN_YkM0Kb~JW&cI0#WMxolm1J)2o4kRRkO;myod1iU} z4K#xQ!heB8kzGSp&QTrc0*wZ55hRWIPN4XI%giB)0*i}O{MpTrq9ZXDXgL~ywvfik zYDXqeGWkJI!EJVSMWk4h?LmstbmB~SFn)f7aykIqsL0^kWu3~Gm*C+5tP7H&MJk_k zwwc*k|H?{!AWdd7M02lyhLHU0HD$c732%r)zdQpZT%bWWw@At||804X0 zxA<_NS42ijiyU$Y8sNp?S!KClkkJ1M`X?!(;SgPoVGb#Ljx=<@{Ffwx8Dk*C->wRI z_*Hl?1UZB*pkm8mHl#`g3Jz+AW?%y^F1U_RwUAZ-wAJLXUKa#{_b{^iSL^HRkqX1h zs}3nEqh&8H$OU8&vKxUol%asDp`o1Dr^12!rG-)w?2M>Nf|mAl7_eUGB)tvAMn^{n zRN&vh-jQd5%}7g46-PEBI}zE8rMrMh-~;>!r4dqV!<7({F&bD>z#yEpJf(K&*$NnN-M`6}lFo5wD=`s|d7?LN6Io zZKYEI&0!2eOGZ}K@6dW@w>rjy;C%qS!J`91ft38TOA-jI<(>+(BLV=yk-v81hJQ_s z5MXd$lZLba{)EG&9BB&l&?+Gxm>RC=0gYOr9yTB_8yhcy{)KuUsFh2I?TWmX{`3Iu z@-iCQ%-UK6G^zdxAkh>54^QEA+cH6{X!DNh>i^LKEID~X9Zf*s|L2bp$g+q5K|?ss z|0lL0cpY&9XzF@;5&!;~BQ+Cvcz}pTsvH;qJ4A5D3+fy0v(OBf5HKX`X?0-jZr{F* z45UTUBVeWwuY>P|cX6%3kr(+AX^lv(0Tzxh%2515>m&zYw#c$nhyP#D3UmnuJbXiJ z*#B7pNnOjI>OhJ47+EzeS!?TiaP*s{f+k&KzfEEc03tzayvG&11qW>F= znj{nipLlR|L<=7^I5>FWN(2oMdBKmd`#NUI%J^`t8NE;pk0aCq=xFH1ve?E#H za}$zd4nS9+KmvV3Aao@H6lyS9VhRWs0F;dZH%J_l8G-`oX_vf_yR24*+8RNr)$-N_ z@@cNg%E?IpK7_amIKv3q4~PiZTTnxT51;B|$R~1&pH56n=+tg?vW(Pu@WG!E94J#| z^m7kxxU@eo@w6-~dAFG9Y15@IKA{7I3YP)rS($F2p7uHS%IinCWPl+4ht(k9u=P0a z-?_QlVDbPNO=|Gq+cmX0=NBnMEcMvq0 zknjp{LHNwjx6R4p9GVyvb)m((s%IG)S2Q&-Xp z4vFx^V~)Tn!Y3sS2JD=hn_E0z%^2Dh`91bnfp`H906Kh8(A9jh-*DctT-bo1{m_qO z09|uNai9!vZvv1D3<3D{W+?v=Bplq`lW`^+pgj>H!{^+4bwL44ippb!(-R;88Vx7rSm6{hgne$Wi?R;0ke$7MulDT(J`lI0nd3gnvE(B@evY z)(-lhhLPiMPJBRsue^Y``+*KvDnwTU!vNNk`>U#}CDN1}pPEAM@f1`Hm6es^nJQ(1 z;WXC{{=QS>05q!R3FH>52yB#1@JvnZPo&QYUH!>cmq+sL0dsC7oK+*3&(fkOV1ph{ zKB?txLZ?3BAhNTw{{`lPZ^K3GB`E{QBWNKK^FCT+$R1qsc*1_|G7?Q7_}XMcE4+IN zl&c>FopKdGwq|bL2svsPRDZY-IwqAU@s-ybFYe(ZR7zkcfFn!iHF(9zWg=uGWJTTG z-Cvv8d`>poCN-H62~LXV)^*gxrT|?``NTwFVWAB02J{LEuVKGj%yPh0#%|UzEK=MI z$*9qTK_BdsqHmQxLHnB_h(u^rrS2pHPzcNH+)?ZA=eIEzrw@M$+0nJU?IVBXfjHct~qfE#?l00Ha9O%SyvYu&NlEwp}OE$ydd=c_xjJj!jA#0$GNEC;WuD=A0Q{_zn9=e z-gNPHWemi+|9dUbAp7*+i}E7RxOjWEKN9Tzf4exDY-9R^M}2}%u~m(a`}2` zoRZQk_!w`TGO%hY+ZwB4(h%>`8>JHTo-1`B?ERQZ%2~ek>_+fz$7ezt3=Y@3&05oR zR7i$;Ov z_iky`r7zEWah zmpK-WAR)o&Jx8-7S#7&oF(u3Jw%%8RT=0rp$|;&Qru$yzmH+NW(6Htc_Q={lM*nfR z8I-}R<#0!%>{Xn)Y3fUthPyhIFj&G81<^4byx2#7K{~`2Sb=-}uAJxApecX1UhoTE zBXv6BW^LOHmZxF*cz)5j)cD&ekF#6Yvbspe%B{?IZ!X?yzPL1&U0rkmR_SC~g|?C8 zQQii3v^Z_F#OB1Mf$MX@iA?=x<*L)(`dJ=zWh15SS5-cUooVX$e-yDbh}fqY*|mFh z?5}7z+oSH&(4Bw8TJL@TiuW3t!M*$bnRPc>#p*AoP_vwSC5#nvPLGjFrkpbH{eah&ML(PESEJUett~Z^>hB*SaZ-}MWFLv{&N1u(# zbtiFS;S6!Lf4ORK@II?Zv^e$9QIK_-ci6)+`61K!w`E1q&`mnRqUaAzCDVAWinwha zVt+q%`7BK&n-0ydepy{ts4%^nDtMztF9oAvCgPv*^Q+6gYozBRDpq4DK?y?hdCj5& zv8NJobHCQSWc~k$2yH$sqdcMB_CKU#Px6b?2{&9hV9V=O`cjkbn#8HV?dQ@y z516YQl5Z{_zbGg1*h3%8Oh{qyIo_I#f&7Ni5Rq`q5+X|I%f5boIoRW&dag~#;?A6FE z7}vZtY^T1`rdvNHnCSiG4V%E)zaz5LZO$^~vLX*<{Rc*Zk6iaXyPkdxU|S7ZOAz~S zgL=1W$7;JBHE@vK>iuF=E@Wd=^9h~O)@5a)>M3fB*uzo!Sjx80fOGFqh-(@BnjblO zB2{3{cbv5A%-8%<;~EMrUFw#sP8gW4H><5T6{jy}KZ}`_*vfKs_M^-|6|Cz9@tt{~ zt7LsRXvo;E%*}1tJ!wAJVZv=^j{KdZUXiYph=rL=i5I%g_U!ig8@HFLW`DA}z8%b{ z5Mfq`T#`}X^;WR856u11z0r35{sG-JJXMdPc4_;_#X)}Rb8K#W54$z3dlrSs4?HUw z9xp3Gt@(+MKrZKoo?K2{l_ztBaRgQrx_5+SMQj`mH_dr-kb(WOR}6#HpAU+qG-Oq% zUa^ngGMC&3)MI}i6b!~2B?jfQOn*z$nvAneT&^plxccPUx8;;o8EHcSRc!3+Iz7Mc z*KwTRzToMfd9Zgn>Q`E4^>g(*3H&>n8msJHiZpPM;Cy}NBR=rWO6S?&rz3mbh_n5a z;0abO>toiUI-c8se3a^ux%gv~&-<7bvagTo{(#*v*O1Lk4^2?9rv2D-rV zArBV2*To#3hv%<5iL-Ku_QbOAGE{^N*Y;R`5OgI_2axE7T?v`9nt5aC$NO-l!nII| z<(l8TQ5c>9JrPCnl*{&ddSpP(bNZ&rhNPS1f?Ip49*j;KcKX{59kU5< zNpCkkrft7G``WTXEXTfvL+0uO;)BDa$m@PI*N7xMi^6F*$rx&#O4iET$X|ywxZR*{ zO|H0mVy#(UJ(P!EVAguWl30{BjNF7lBVIN!bBn&-_C}Vier1vyeNK)CTdB0QHcoXd zUs9~*X-|`!i}5#;3G$imt9b>__Q!%8N05^R2P-#5q9{{}gX|o4reesMZP#t4uH*TS z9u@N*i^}Uh3f9$j|EiP9oGAIc^tw~yllXv2=*F?JrJ?EAqNr<( zIXTvCAMWTr;%sljh)^hqd^;jw^ooo|RXtq`WwlOZ#Fxn?K6Get4ZE!vzOG>C@+%tg zS11?-^Z<3oOIbDj>dSF@ui?2o?Yi9$)HEBuahv^RYUyi%?*CR5(bkJ89DQu9b=q2I zi%FHMo%TgsbVsrLR9+>PIsr!^qd8@w34;?moFtIFA1+_EY$Sr#z}fiX=@J`6Gq&hYDufLS~=g) ze0mMOoE0~Yc>S>B)Az!s+O`PN& z-fXcd6-w~{qd}V?FDWgQDDuY2ODF#0U2#^#T~&2FkaTvPDKnpk<_FDv^w$Y;s!q>u zZR>TdIJ;=$ta{s8GjPTkPkeQAJ<6=un29j?sKORF*Ng4CI=wK)`Bw{L@6FFKsst`) zzin|Q?#!0T@$|R+-QSzUMt&dRjz_<&!W6SU#zb!=;5-kR`K2}%4d$|*|8%iuv)f?k zq55hfn2kjEo0d!^c@rfpdjq;AJmeJ0cof0^+bG!^on^Gix~n1ecuFb}o;1JzBzn!c zKXpaSM@l}g;`l?;es9jdNXrBFxtmt}@&2LJ?y5wmem$|Gkv_V=8vmS$8OTu=FR##i z@-C`)mnv#aQchKOWsvf?0Q+;m3I5)*X(Q7OmXQQc;<((UUL^-2w+}fZryb0~nj`;`D9&|lCct%j8?BwSK0?q*#D?%Jw6H<=br(5yE_2?vT~12!DhWN^ zjK~;8Lf`#^Rhv3&xg(?2Ch-w<%Et11?ah|8oADx*cn>5K^zudnmL(il^=$`*7MgTu zeZLEn>N~f-@Rrmh(N&X_iO4By?-Y)_N}%W(*%zrO=hf;Qlx$QZ>0;&bug4qK^H%X> znnx0Zkw877BGPox`+CjVkzMh-X~PfesQ6otXrf7*4?B zeC#Vkq26bgEmIOO*#oe98axVR&ZF}G7l+!R8t-o8(hUU zR*+}$uaUFt?2byLPEc==%tkvN?`?NlFaG+H$xD!HJ+X4(PI-%j3CAX988ePBW{>-` zhS$!y$Tkaq*0_C$&AHyadbL^TC7I}{9re^lgWJvllnli!U-n~M&7!F%jS zFuMz^;U39k&1JWq0BGS!Z~FvYbFaNQdXDMq^zKCU&` zuVQkpvmVzNHLc7Ai&T>2NmuY!&ekmsdfyK4oELsv%-D8j`}F1T=ZHfz4LokrrtszL z;-rWG-#wR)H~y74rQwd1O)c5|879aM!q-VYx^W`3{5QZCf@@w{QrkdaSlz*y-xVqc zf6j!-zx$8fojg}*!3x2>iHxz2teM}3m1=dCX?yf~+9S62|FB?F#{qY4Las^xXyU4u z04=S^GhY$K@WzzACl61m@?9sSSKQsB0%u+OJ%opJKL*@mh>2e}|CcW!sTL&3Qp+)y z;$OVEBPzT+apu5X{gryh?o3MR59GjIMU!bVf?1Qm8ApOJH|<`rjM-OeS9ECi2pTnL z?P6~_Zsj|8*@f=d7Ee|eK+ktQMDug}S9?mo(RTcpmrl66)*S)ce+I9uidTZi)^tVWd;`}UU`rB8fb>h3VYA;m5i+6kB`7pk0uCMZlx;o40K3%*cY3}tnWZ}EH) z>GRmd%QUp<5LQ!@yI+n|VrjVFDy<4GTu3htd1n^OxBp|t~eTP0EjgFGNqkBY-;7AHi+Knaf4{PpB zpsBoj?J^9>3jnSMbQ#8tuP((4yK&JgCSw8AUfe~u#Or?*d#S{&VW&?X$fl)|LxROb z5LF=BaULEbfX@J5M}FmOCrgae{0{As^dxZ+@PRJF^7w$OwXz(-NWU~y=3>(>V$a<{ z^^J`UabRj;644cCFpJK+!=(e$h?)RO`_AT{El)&DYy<=KfcS?wj{(3%zyl#l2+&yH z*SJl4^U^UOV>d&xvKSEh0-oYF=uSc3jR2#+=jX2i+%5(B*sotzSGWM|Rn)0-CjvGI zh_6?rrT1Z|$~{0+4S*O1@!dnv4+BX-T7ZWif;7TJw`Zs@!4F`%X*duscj)PXXhK71 zOF~R^g`Rg$FA6Jv2*zoC1X? zm=P6iU?A=7+n2Dhv1!$zeUuUr^uK-U2ZR7{#D>uOh%gN>WlKp##Se71@OEhNky}xL z8z-ubje;gN==z}rxpP`t8n8HlDmkxJa=wFN{}*WUP8Jn?n$jP>87YS05WXGV-QQt` z8luyF`}QU>a`kW_rR|-fXvhVEN(C123R!61=nGN?z9#m{*49>$aT|{3-l!Q6;sLNp z&wG-?kw#YFVT3l0oCegMlt%&r#(Gwt5!!d1%%kZxL&#;dqg zVIU`jkvmtRFHr;L=+vL?X#){tF2f|~b5=WI3VoCA(9Jid?2U|4UCc@`f>B~f`zY{3 zz(ltIo!{Pg!7bd@BHueqIX__lMENu2-r=8?H%4*L2WiPospbas{g1$BKfAj6ud=}o z+(*R3ONI$J$2zL216CtuaCmTzA4?4cj0t`z0WxPK6VPJ@V+8B>FXLV*ee>o4zk%&l+S^9g?XhQ z7WfSYgb*`IIyzy{MDY#01{^4$9ih%g?7Z$V$aTPJgD!ic5e*mVYzEzzG+Ym1>Y%9t z`U8u*?f`+^sIdYg#Iwy_Q0xgn%L4Rz92_kdmL*B!K$9bA9QnaHj40p1clrQ(0zS$U zX43KdoB{PJjCA+_ukP>{x~`VE5Vw@w1x*g{ZR(eSOBiDphS|@E3KyQo7dnJkVT1~( zPYA#zalM3NbFh?Gh#YRvr8RYQTt{Z|rOE^A7$}Ah#D-08Ur`SyC+ABSmmtJJgTMs5 z0CLZ0Dr#zhK&v8ie&OgK-P3UV5OyDwA2@lB;S!;UYQ(9g)iC7-wAg<9h$SW_hVaJF zlZ)`)AlpNFJdulus6w9{Y%Ma*7gSUl?FSt;{gx928oqKIdof z8`*Kt5jkcU11*}rmWn!KV`DF_eg$!4_={W^zen1#F5YWSp*9K>(8cv0%rL_B zGKRsRm3xNAiaIGw;V5BU(Dq!U_ppyoT1fN}>q8sh<`99Yw`lM^xLmG~UTo-mYDS@` zW*o3pQS)|mT|$T!<}Ug5H^ zsZ6*?-PR!T;RD6%WBThfezeLDG~OFvVY=R6^L-T^ymB;-e&vC#REv`nx-Frha--8j zbYBX;S0$b9uP)ItUt_(-NPFYaE2*juo)0Q`p$B4D7Du_)E4yL6P z6m+&5XquuL(L=7>K>hanyS}a;uk=V>o~2~P0Wt!OJ+f>AGi|lQs)b}@c%OK1#4g>F@8)6MUSiLR9g^ z9ul9Jj~{g^zi@RBp-`An2A5F3x#1|f&jgS_Q2z{4iruwIZCD)@=w_TL>49G0xijy1 zci%!chX_g-_Sat03A1OPLK6{Y+_Pl(6p(&ts*#~y^#%gOrsU9{oZ)90)8E&JNx;UMhlOe!jfT5QC48d0 zCqPFB1FL#s9p_7WU^>CVNiy)Xi!hT?MOF1Z^v$hyUZdn`AGdADx0$Sdfy^QR@svPU za`hK`cmia4>FP+aBy`>kfy_Yzxk)ZsSa5MNa5PNx^q*gjF2ENk{{x+5HFSjxLc#&d zB|_Ch<^-b)8G;a9V`;Z_0YS5V?kPD>>{Fo1&D zd2|63ivK`l!SuzrVtf#Y8LxwxAPdmYJqw7iV7-6;DM%3H9zBX4Uu8S-f?xRx<8~i0 zz~YN|l!Klw4aA3x^z?zIF_%#M1+-hY?r5M;8@_O7mT-*(KHFsEO(5ILgXx`M7V~7L zlsG|4gLzpXu7UhK9W(u*QyJX;E(|1SI6KPIfw>+jxVLy8v#5Rf1Ul2mTU;)izaFe|-`!fT^k#;RQCMyrk*}njx251}m z5^d`>(Rbxv`rC#o3F2Y>#*W7G*vw0~ulbwS?p0_OErgSH_C=6C-sOYQHPTUb~g zRM$sl;UgQ5qNYPO{;o!`Q2-YgmmI5En6js5Epl-fJn&4yB>M@-!-&YpazNx3m+Sfb z`Dd7a;za?s5+z%NLMbu9fMb@RX64)x2!>-Nrf&qDU{qaK+%YeQ>Me+(YM`_$1F0f> zZd`6e^UN=hE_zYG2S?rUcm_5A$xazGQpJ>dLo%5LEv)fJ1BKEy+vjX%X7M2|3=4!SO=}32*Ou z=-%OmLAXW>=Q(W0ie5{{>7poA?whPA)B#L$KK_|N^GH!~m~YzCVW=9M*jSZA7U&C7 zLBe#zEcLMs`7_MjA@=yBk+7WwZTMpqwkhC{Jjv)$C?2XVZZs6UUXh2JCR|cw0rQrV zmCat_K%vC3p=mb*R2;cDXyVuse(2(vnsx4F(AB9rXo&m<%-EHNl_`KX56~b+iM(p1 zz{eu5M*07bf25iK{cSDZzpFZtqEM+<|G{8XxWQaOEEG?0T`n3*$pLCkMzG{u2x%xD z7%HRkEC`hvXo%Plyk5m2TZp_|FCqaq)#{;Dx>0qxs)&!+D64z~&g`!4*=Q$qZ``Nx zj`MVUE$2$ES=OM5yW}c8^yhlEj6Y@*72;3V|LvjVeL1gp?){*zRlnYrcGNXo&A=Yl zTw+3bzImfcU;l4Rx#(&7%J5B9YB#O({hydtZwegATc$k$8$^9?x%7`HWfH?ZmNAgLEC(a{l@1LdF{oKxh_d^*Echaw+F?3sTfS} zI~Kj=3-A9${805p1HaP~;!fEFB`lge$3yg`>h*7zGu=THKBkuZqVaFMD zY>8KlNtlb}lrEr7?q3g0B%Y+`>x=trp_}AQPC0yZa{lShw9Go?4&(mzF_HZg#C_%|e}JRCkF}it^21(T1FlkkV2P312US1w?b~TL%m4Ovj^1#b)Q??mmbA&V9n08cXYai? zD>|V2bePf3Y^oz|iC%Gh$m*9qgU=VC*1JoAGQ>>IOXPJ8_v8cmuf%^!5WA}!ZTyGm zP&QPpGNV?((y=T4z;8PIk&>f|w4vMn+Bx%D+Lg)D9r50Csrf9+CgaWOncdY+RxY)A z$IaxUx}dZDcgcxc(KeKc8RPVxIqD5UUsM#amBUWT6d5NBJ4&oDtIluCeeicLv=yJ3 zAZY1H{H=WS$2Gq_w{Xfuig|-|dbL9%r^6>qrAE4pP_?kEpl*n6l+H1-QUiX7`2I~cS z@+2zDg2yWVX9Z3_OP90TU}VJl@2ca6YjPr<#kza)RLY%Xx3q{s@C@jNU_|I= zb&}gIMtmqddibI6uBMp9%+un~+Ke0vjDtM2PrOTGwqK@B&r;X*nFJNbniv*r*CZx& z#|z#_cmMkOATH;@WbD%Js|)5)MtIQ ziCf`h2fsWWR{y;d8}1n~#BmSfOtyL_?31%7@cG}ua73~#Ewz>reL)Yejd9o}1?RS& zJpZTejuh8^cDoci%4zqrg7D`X{+sk`v#0du14TM7=0C_RBra#2ns5o5M(%&#$4Ram z5lhxGW6h_fZgW}- zn#M6Fk4LiE74!`81m-`bVTXSIy>eLn=-&KGA_|O}Dyw!z2YEW}Ueb^1@eST_9wi&m zCa_tl_xJAY{4^&}p7Ofi8b6`%d<%OwT>mtAzU)U^#ln>76!ovB19WHZ*?K{fpY~tn ze}nl9l{0qP@l$Q5R|>Xnt-nY<9ZuDMw+L(v|Q)8_t|0d}4FnoY{E$-*)>1Ggu=ky{n<3pIja`HM=We`Vcf?@e-U zKWkv@FZ;gx?t`1Qu`$baf-KO@UGIt~*{8^2#2f1H1HSK*ZtaO8=4 z!xbFf3`-+I@eD&$wqacRKL|>?(>=&ZyT}abt=SiCl{7omupYs8g4C;&qo*F_1)p4{ z(?ID|3d@x3lzL&-ZEF2+|J+dTc~>jMxH?x~K8W>^44?I{E~X$;9_vib{ps9DdhCJF zxb*=cpOw@nzM4mln`+jbaVF@?1)Q8(0;W-F^%QATI2~?>JVz#-seaQ6WZCFw3GK@- z(B*ZL2SV?X^@|?UBv!9k)*Ht>U!o}5^KiO{+)t_5< zTi)IzOyIcodci%d3Qq0v&#Qy=zR$WV+`k^zG3+i?v!c6D-fMFuhthZgxr^$;l<~#z}GVkC8}>_Bes)d$VqpQ%4&0 zs^bbqLygLh1=QR=XVtTc_99~f%a0T$<39-w3uk;kf6gehAM9M{t*b2N`K7R{#`f)B ztl@v7_qjO^SVTk*nDPZb=23s1%8R~H>U6s8z%x@V$ov8~bO9{ub2+G-7;kp=B_peZj<jmz&Q@<}qtt&li7RST)9k2za@t1S!-`)z+7m3pvC6(H4?qh1uZ8SC2Z*~{MP zd{3a(G5u_>qp&1kZ?G%Hdid<44&_u#dPShKQe!WV`(lWI-0q?y<$}DnY`uU_xP2l{ zZyQb= z0irgWPBQCgQ(RpQ!N!hS5AJ0?sRE3CC!RoVE=cT`9@1yPhhN)|K zGK(qH?75m)j_D}8Ikq&yS?bEndp|dJCc{c)J~Q$ue;v=H4Y=kF8a8N?x<(t z;n?!a*fCD`g2e#yj^Dc&p!8oojzqJ!Y2UCGxtPDnmuD@K^>KIH zva$aY(cBKB)3_o!i$VOU*4ZNY+=0_CG~3$jOL2T^ukLH4O$NSbWpT*~i93FNBcyjn z05^Aa=u!3YPf>DyQB9{c_owS3zaE%^24}x5?QyO7>=_0{v~ z*8Bfv9WM@0Nay_7ZIC3$+J84~*2B#EhnDA>BXjB#hk$)IP0I+fx%Kf+k8N$1I# zQoydB&TVgfV9`q_H-i(&wQt%!JGeWnsF27PlhUC)wuY5dv;01j6{V`!J&W_Z$uuuN zn=PDBX~Ha-b+yy6%N56bsOwR}T_UW&SSC^e>pPlHOsTk3%mvn^nYpGUViR8TFm9#T zvoMNdue|N(QjduCV`0*nd(Jd&ZGwKKyx;3f@wA@_apV9q<^z&aH3u$@% zAz7OoD;JTc_o+*+Vpi&reai_2ku+DUtQuP8s$JKQ~9|ao)pB zQDUYRJI1c6d4*f!|2)}5>W{#ElN1oTj`mufz z8Li^d)AK>Qe0rD6)XX%F-}Z@S=VfuH@7IV?By23=3QHeXCFv?S-$n~lJ>4)46P|aq zfB5!qbe?eJ2x~`Q&a{KkJF2-9%f;6YtsZZiZG={dqGxZtXWke4a`pgor_{vW{{8G~ zV7Ru8%eQYXHIwLKRFT5Te#4lUmG523KM=FLznj&t9cFB$DR`D3Ia7FNxm!ri!M!=o zW!l~czBETW#`4a7!JJn0*~`1rQSZ0--^cPM?CG(!uaaArn1VXV$(L_Yl_8_i-q3xTA@Q zg~?icuS*5|6rFSts`8~~tX0bdO8%`?*+5Bjid6&eAc5k;rTIS5`7+kgJ7kg3*~+ly^T@ZJEO9X ztd*yDWAAj)i)o{C7YL-L_)M`s6_Spu!^>LB!XoK9HVjZ(+m;Ov&HGn~$}%a5T+(>- zX!z|>i=EoL`p}xpf>>`ZMR~5{+`|d2p7)ZvLKRArwT^Y0Kc&5nw(%+3x7$}c;rm}2nBC^2MRw=ut@*vahy2K? zeh@oRqf*AoBjeU0Tvj$%n2(p$n-;FkHWXtB^uePe=??iG11t@ z!*D2LO5wmH=FxCg;TA)z;PcpJa`XLLMIXPVVWsdTZF{eM-}x_HN>5Dd25bk1clq=` zq4seTcg{TLrAJ6x$9X1p4&B^F;Z&50o_Zx_aExdS0A@{4O za`@|_XVR^b@8yhUx76?Cm6P+~L`8U~`6wt370?~pFPwa~PEqLnRYPyNJj&%*E)7ts z?-$7wi}}Q9%k!$6nRj>JG-;6kE3Xu_a9p|ZBk8`(I{O&ic{YE`SJ7R{X!};}d;GgK zi5@w{IOa2z$2((jLXM0*nZ_@s|0OBY?e^#kV=j5WaVNZ=%g*Bc>he8#wcp}*pGo;- zvsurSlGZLJR*jXr)ofADEO`d_Tpwpb>fdC57qYnl>_omXb%_IWaCPL7apCo$-js6l zrh&|*ZP|49Fx~7+Jw63(1Qq|f1}5%LXI1e0;#ob({IrHvn)sdB zKZ}aU_mK1#B6(~-{$)93cQEF3V3O(4(5CifdF)RNFOGhd3m(k9sb*MG;lXz9#2-+~ zXOA3MU-wZCnKtWmIy%k$^u}gwRHO4m1NW>T=kh75W~}Z#Y7M>0CFz+T@unS^|MEtI z!nW#3bCeop8yuv~`}Q2_YMxOfYGHdxTS z^yOADoS}z9RQKHYnI;_V~h|$qmmekW&6FTn@xi6KZ(uCH%RRToCRvMJw#! zvD^P^=~Hx9#?J}*bgWwBu9ADajjf-23ft=Pm@Qpx0rx6^=Xif!DDvvgxA4DzpSgW{ z^DBV=`<{ref97)W?RBgFaa}F-`medi{PwsX-CGG9gG+g@vwQKOwxsaX=qdBoCw*9y zzsoG^>a4?`vabi*Mo-`H7ucOt0G=in4jd$3n!^n|>Q#q93wVTk3ve)Ql~vE=H4a~Q ztlpk>q%`3Dw$dva7yS8_2Lm^R28Hym{&J^7`^w8N7hg_up7?jyYhWKyL@H|cMnYlpsBn-XAd zyZ6ies?>}ndU*mX8g{J}F)Z07W+T1SA@URd2m7CAu8J*su{~(Zuf6Ibw-+}@UE!Tz zykcQgYy9fE#qpO9`KxQZ`oc0b)W3Y+tQFJWb%xwNw)M%1GvP{2>n|72uURAB?y@TP zumkIr%#|GrFK?|l^>$T#mBi9J=EV<+ZKZyPuPwQ=#&ffL@bg1Yz5iVKb+7)_TKkra z*HbKbXYc?0C%SR5Np9dgvvmojd2^QdKkE&i-={0`W`p*ILiW3VF8q%ARwZw0b^N8U zgry^QdhZq9T-X8Saj{nu#r}JN}T;1HY^2w#PeDf2k_FE-?)xU}jWL8||8T~`2 zt?1j?Kfc~y-^c5eN*Dx$Pn+;o%ll^6B;e>JaANq>O$i2%O=l!Gf1G-+Q~e@v)2FDI z0K)<+%OyvhZ?FD;DbDLhX5X)?y0L5bgw_YP%LPBJ`I4PmvyEH0=^<0_zPiSs6Y=Yp zTa{luvbQ|z+cB5Bch~GnpSfzY&04wYomFQtmaep~StY1@{_CA>I;o5cvO-+ynpaCR z9N;c3y{)|X>dzDHzmC6&?~)8H^pLmtQEmI>SWvEqy;gYLv~TPDFYIW)e@UM!@cg}p zD&Yl;S}Wrh?6I-7zg)k~Qq(YS5!KjInMg@*3k6j=W^Ri$_sX^ zckwKmY+~@?M~tNEzaQFJr~L9hY2V!WoAuP`Yh`P~p37Z6wehXzxiHP?W`5Ov+*iZT z_q<g#vzU(P)Kd!@_7UN7SLD{+U?=K;s}uMPfoUuzezF`N$?6t)w+v1_`U@9U+3p|U4h zkJL8^ZK~aMhYvV@d#WS79XycB5C@#d0go2Wiw6x+H*kjq1x*6p)VdlpD$Kzt#xOMv zcmd0ya&-KtJ-BMDV>+ z>YI*3(3Ak=6?&|g)!)1G>1_O7f z0}nS^5Ag=WiY!po*U_O-Ub`OVqXeLbz+gSm0Si`6F!+g3byHO}tQ0sk1p-%Z{%4Ng VTKPmKAlwzi_jL7hS?83{1OOC&nhF2_ literal 0 HcmV?d00001 diff --git a/docs/src/assets/images/transformations.dot b/docs/src/assets/images/transformations.dot new file mode 100644 index 000000000..2ca40ddd0 --- /dev/null +++ b/docs/src/assets/images/transformations.dot @@ -0,0 +1,28 @@ +digraph { + # Nodes. + tilde_node [shape=box, label="x ~ Normal()", fontname="Courier"]; + base_node [shape=box, label=< vn = @varname(x)
dist = Normal()
x, vi = ... >, fontname="Courier"]; + assume [shape=box, label="assume(vn, dist, vi)", fontname="Courier"]; + + iflinked [label=< if istrans(vi, vn) >, fontname="Courier"]; + + without_linking [shape=box, label="f = from_internal_transform(vi, vn, dist)", styled=dashed, fontname="Courier"]; + with_linking [shape=box, label="f = from_linked_internal_transform(vi, vn, dist)", styled=dashed, fontname="Courier"]; + + with_logabsdetjac [shape=box, label="x, logjac = with_logabsdet_jacobian(f, getindex_internal(vi, vn, dist))", styled=dashed, fontname="Courier"]; + return [shape=box, label=< return x, logpdf(dist, x) - logjac, vi >, styled=dashed, fontname="Courier"]; + + # Edges. + tilde_node -> base_node [style=dashed, label=< @model>, fontname="Courier"] + base_node -> assume [style=dashed, label=" tilde-pipeline", fontname="Courier"]; + + assume -> iflinked; + + iflinked -> without_linking [label=< false>, fontname="Courier"]; + iflinked -> with_linking [label=< true>, fontname="Courier"]; + + without_linking -> with_logabsdetjac; + with_linking -> with_logabsdetjac; + + with_logabsdetjac -> return; +} diff --git a/docs/src/assets/images/transformations.dot.png b/docs/src/assets/images/transformations.dot.png new file mode 100644 index 0000000000000000000000000000000000000000..1343a81e701af229c436f68d0704646923107450 GIT binary patch literal 56255 zcmcG$2UJsCv@RO^FGVaAK>?K>no^}BNbe*e(xpR?-lU@_C|!E*H31?e^r9lYmw2h0)a>sunKvx++ zpv&EVT?am)=tlv6gRUDZ$-M-f6aJ*v=S6`)k3b49UqIg`uTHvok-MC=ZtcHm&kgLm zlA*Y9{YBqjPgpbkSfniPG&{D9s`~mpd}#dLQ9$))&rKrDyX4noUU*PsD{v3oe1#=b~^Rbno|01&8-VOVpB;^NXT^5fIX z7guj7jILf>Y=U()s6e1^F`qtvX2ac8fB#ZOhQj402&4xmx&HMkNH3VAiIwoVef3q4 z`YB?am;}#6$n@T==w=qFEP>i! zn^Y1aF zAP^BbVTzhx4HqlWR6fed?_?Dytq8&P^@f8T`-pM3lSdwkBaU|MT?Y3(w(v5Fct>v# z==Z>FU>4N%wBM88wO5|@5{Ix9Ck37e$kf$na}7o>PZhb@;j4FLx*5BDtV4#cO`|O` zQH!2;OrqB_^NwI6XE*IE|GZFp2PGIld0{|IQGo{@kv){wYBhGr7TL{Ea&$2MHs)Nn ze4VJeFIgPXUs;K>;26l8tz6Ps?Ya#5h7SX#+y0a{s@&B6-r*=Rf4}_sd_$4oy#-vm z$E{J6jP=pXAJ-;3$f-~GvH<5`6*spdB&7nvE4!y#l8_}TWtGFnL zv_~xXKQ29R?PCcX#8@l@<0v&`GQQ8l^17R7cC6nkFOuMImKL;pQ{$_!@Q%4vfZRiK zz<$hVA6cv@5|I$W=h8=oafj9^4xKI5Kc#80nFeMb4~|&Vm0e_H3(+Zhqko zztx+Z+iaDA>l4!Sp-LDZUZQhBS|Oa?gb0}H<6AC)Brh=lTTAaeF)R8WL=Zq7b?_!hG}&9=tUOq>>FT;#=%SsLQTxaE<(Wf@T1oAodmWOGVfsq%}K)) z#ap${xTOscY??iKmt_TiV6Qx7@zdk%HJHdcL9 z27%HK^j$82)bD%I;1WKkv4ZVCpJCmyV_8HnE?v?dj^h}M7L1r3U7nHUmGKq=LLK@566p}XK%3Mnh)VqJl|Nedcs}c%~v3N>x-g> z^ci!bS-XFFSxj$kAS)M7Gd_^yMUoW`RRun16EusNq7hdJ+^(T_+Za9&TrAl8YF94K zrQH((8LM?pzFtAnT`l=^zOYYCf=&(l4(*A|^bDqKm9tvkg3=~VC{c*j;>X86Bu9MD z_!?W!Z~Vi{M%-DET3g$ZnZ7?(+aEjNAV1g`pK{M$@9~I0(Re(U(nk*@{7BOWB90vY z8vCZ0Q>XS>Bl!&wh@9fWKlxR@0`K>=M)?dbgl**adRtSJBMRPos`SIv1pCZLvfj!L zFZAM1zH*o2du>4l;~3KpSI&g(*i>=exQqd^X&+e8<;!DYxi_rCM*?RXu>HmLy$Oj0 z{7wsO2024(Mcli-Yn`TnV3^Kwf4Iaz3)33@`*vD6D zCbq2+8Wug5{J33R9u*eEvBK?nEHczahnluq&Tex&E3LH23fQwF_g$?L1#_da%4EHh zGxn$6s?^IBj;v=XD7v}#tX7`>y3_2LT{_Gz`j@tYu>DU4CI6?es%$wnE!bbEN2ux- z4mZL(vk;>1E`ieT&(x8CSYMEQET_(C=bp+wXl}HWi+Pv0Idi4v&lL6E)5*PEmLscE zgA`?z{?B5KRtv37yT55kq0jo%z8kr?=yF)Ded#cF%qNiIx& zFvFi^zJykr0!K`KZTfeAm@>lG@X}xaR%_lt;tWQ}b0~nhm)Gq~O zq^EyNPP;N^7w?!WVy5V5LtVu6rG9rH``VcEQw^)?_kSs9#wx-2o&9-LmTe&cX!OIl zUmu!BdLOq#J#>y>u6KZ_a4i(@QR=17-K^Fp)*KTfkH0L*<5OMx#$JE{x5@8io$jJO3Y_U`iMo}T3@R|BzE7yYhNGi|uzI z2}4)GdrI*?nz+2!UcYk8)#WIa=iz8cyJsrz=qVb>Q*PNMqLWBosMV|5MOXP`kaLjF z%$%z)YUY}K^~`2m8a5)~OGb#&$w#wn3wfTo#SOQ^Rv?~}gMoM@*>03NkZDxU?`4(l z;U%olJ3N-tPj(CXMsJJ$ci&$txvlHO*s#yD<sUv+A#1!zr>HPZ3s5Mac5qQS~r$95?o=1o$N}i&OG_+-iDuT6E&|o!{*(*Xt(iS zx^~HP3|8TXrk3d7-(!0T!A>$9s^`izE0dwQd5zGzK+F?0OSkGTiQosvWh!3W=^XU-L z8FQxl+MJKxZyZ({tiRGqy)wOgC}OM578riH$xiG1+(ygen+gpd?ghn5wa+Kla6NKI z$N-0zWvO;qi0`hWCmqNsegE>(=6gZNjTs|5qJ#kTEfA(GGDfX0eZu6?dbKNDKFiU_?x`^u7AW~YZA(*#ZNu~ z!0lroFCTEH`7avr96@@r!ELwcfs;Y^?wNs>60~(d4KBvW#s$L-y9V6qZx9WDC7Qe0 zt|qSSLd=cJuKeB(DFcSP<|@GFj{Q9U3rzk06fXkS{vUy7DFqOML3#x->~Kn#n^iqq zg4aQkGf$NV`GRWz+{zVabYDjQvf^#7I1q?cy`-FgN}{TS>bx{GG$OO(E1cH4sSgQ%mRrDgqP-MKU(c$*chv08sZdUe5g=bxJ_h>QZmU$`z6XY`{1k zpfl6>Tq{Z!0Dq@%%E-tlL;?#RRCO1)?ML&#|7->Rp15x!G=jEzLrmmsyf)~7kXJ>R zP}_dBQ&gLSyOQYijWKR(Jgv07><%!k@eXA9E&s@ zFG4f?-6se-V%%KB3Gf^i4qsq9^$_QvR^aq0+v+$ zwd%&N^vBOwppEbwp=Y`({OI58zO}H&^Xy2w#$lYfNP0MBYBVaJ7}<L9v8MdtG&p zf2~%mqpAdxYQ5llc8@G=0N2MMKUnz?`@PHA8^1v@<_K}ZD~=a9jF&?5^x5mj#&Wg{ z3;PQhG}m->mN3UZphvw1vflyT z0!CrlMpeD874DKK(6{5ZI4JhBYbXpm&ganl*yed;E_6_i0tJOGhJ|9wiY2I&fS7f; z2Mt&$S+%#0RcpPBEwE#{_8?L@Pk>7YT@p-Y4G)6mYQ~SnRJJKQfZMwam>o`Kz2j30 zlp#8TZbh=I9YaUn%iHo}6 z@a>vszW*6^C@wb}u=Xk8gsX#;*T&uTZa$_M;^~T*>z|V8woUL!MG zA!POeIvnMlRN$oIs|~q}dA)}SHSpFnJg-vQ7j^LOc#s=bIF`K}C`Od81?S!^M(cL! zy5N5Mc#B%4EVwgz79$}7=f#z9=FWU|m0nltbP?<8Y;DenjS77a3G<|du8CqU(xif$ zZ~(uPv%dEs{18!LXHgiKJ`+r-P&^x?AD>?O<&H<4QTpXo@tTb0P@1<{cE#)D0hdv5 zXcP$5fn`G{YDs_R|1reahb+Z(6a>J8V^*Wt!Yk}Vav|FqNyj=!c#7n;QI2hHm~4vY zM0RlwsoAPf<}@#3LV`}?X*N&jYq8tdmuo*BZkLQf8;(e6PAZW^Vkq-CAKeFeSC#UQ z#D_G+aMw}MlM&cfI3`hGV8=$WJ<=YE3QHYN20X4Frz2px;wP0P0fk7y5#D4M)E7=! z_=24-21#!-rVh}vX<>^Dz$488`gl}eTg?u-`NYeHhml%b;E*QrTw5YZczL@mT6Zgt zXTIbE(g;)58W14{4gJBH*dV0e8mZF$CBr07xm`mM&z^eP)r?l8#(rHL%#~=|%wCqP z)6Hekn4L~7-ijwUIin%;FR?e-U9V+F-^}Mv)+c0v1sHQ7M7sfRtN2yjSKyS~6 z4Cyu0@MqUZB6DX%rFGnR5IK_iOD#uc;mTOiQu3#W6N1F6L(FYKY{S;{!tiIS@w2r{ftJs(d* zmKQ3o>DL;h74}yB6u4)Bt~EF%0_P}mCRFP@tJRGDv&YK7_M$8gjV-(i38OJWEU>qK zU1>Qon=@fvN&2Ot+jW zQ#f$iph~uJr!20)xP*CQyx7R@O>aWG`k`zLc609WJWZ+8W0fSUzV6Q+W>|sx8mDph zam39$`#t(yom&q`Wsy1j34{8mc6L6e$|r%N9T#%*aOCb0Z5 z7fitx)e(J5E2ReJthRzf#L5Yh$aS}P)Q8J$oR%9VshW2*6AJ>Uh$FyiLB#kTa1{nH zYzp9?mpGWJXWguqjvtWII|NVFN+{>J-vnaIb3+ajAX58XHsN&hH#qV%`DkO1A=g?w z7{d8kICg=u2@HHWu>@cM$(#LE4kqX~Ea?c^`ZK9`t>Rxf^gf5Mdtup2(A>Gd{(`Rb zR9cy$BxU=qgVH}_B>{O1&^DQCtS&A<8~^u2UG!}LJPhQjBn8!w0G&|hS|!Tp15k(c zKOrCc3rs{9=Kn|nUgY*5qA4mvuZ%cDA8p&p<1V*?f`Z8G&CN~TY3(Ad(1${r;iB80 zMy!ERn5gUq0puoJ?tAWo&DW5o66t;rkWufskJP~+43L0DewDy)v)4yz6(#DFTRskE zI)~I7-A9a;GQ9wv-xMaqm}1E35kAW3{-J8+D-PTUI*p1Q*gkEtf&k}M4B6V+A|fUh zv7Y!eQK*HwMK4~sJ>ThB4ZtP!AN?vFJLY z20q_oIq;@%0`M*XgaJSw^bGK+*M2+=$>h3JWoz$#9aPnG0V3631G}Z=-lXvxAWh^U zk>=E`;J|4{gvZC<$FGi-<#z%T*Y`K_yDbGQ;Gg%vVfMTGz8J_Q(yM;k0J8$Lzu95Q zo6b7KX$72eq3;rtyf7;rLjiFNbMTBnW_qLF3IHy@HVHh%$~+KGtGW6^OsAaTMPQ^p z&*^0t`UR<&mzq95w+vPA`)G_=VStYh^vgzF`mOKqS^;`~Z;z@P?;&08h6ee4?>^N_ z*VKWZGs3Gwz|3=^!n@;W-1(Gn5uS7;N^PQOV`?>bxpyL8%gT4@<1-Xy4w*JJOTw#A zHcgY|@_m0LVF`E5@tNs=p8*DzZ2n5A)DJTK+);Fj(jS3VYC zdQ%AJnB1<%%;W5O(i@lS)@BRU_r%6{g*6+7d2kd-tPsuNu^EJAC93}uM!4ii8XsO* zFmQGThSa8{9(mOA1ihx>YD>wtnqJdr2^GXuo1ZPF#d|BFq{|-BPP;>I@Tx8O!;Oc` zhc?E%rY4IEnt~TllM)t|a{m4L{RJ3H*ti{Jnwo>;ik&GpJ}E(%Jt9hIx^;g^`nb;-yao z7$?T5T0QcN!Qd^a z%Za4}?t6Uus?EyUobc-6pmlQOL>Bn%E#*G{eetZDo%`TtV){3IIR@myUoK-Xi%c|B zF|_vgc|wT2IlzPGazfk%9Jc1=p^VAn-Tvg~Op7|-lz*Y%;`-=OPfeGxd9Cfwvx(2# zKQ?qts?6|M%j-32(A&xnFNg3ICDm#%H0Wg6?d7mHB^0X!3oa#9C0BQoa0Vx!eU(Rx zWsWtSJC?t&^Ge@v+Zbd?ryZskyce)*lEyPni&;?i)E|NNAYA@SZI4Dh_fj|uzSNq~Q4 zP*;WH27WD?NwJr|xt?EkP~a`JTDN@c#)3ghk%VzW)Z*yqSj+53;8%4tK_S571TQRme&xI9HdZ?2 zxHj#XABOBo{j2V+xbPj9!T^i!V){#0%ZevqTdPcTd*WF+107*%_l!J5$JQfPjjN6rp4`A}X) zYQn+i%*%00hTd`r_fm*%t7tZhMfS>`AoB`#{C&)RB3AULoES zW!I@eb24e#?YpSzaw|HU!zNx>B}t5b_sy=&B7bG&6c2PZzy(SX%!q~T$EEf!-b?Kp zpskHYYJpv-$1C}LRdn^x6hrHCfBG|K3p5WSR)zIjDQcg-Fk={B*$Q}9s&J7RRn2}n ztLOE+sq_(9gBqjsF5mJSo`P#sK4)wnDt5V1-b$h#p473dOlgG5Zq1rVJn9sv-!;kn zqgp%XoMfT%=iYk#yrpEDfG-w>SuF zC$IU{wz0OtyjUk|Iy_A)po4YO5fQ;iO;l*0p>nyfl#rGU`yXY=#wT9YF+;&r`3nP` z!J}*tt*{DQ3Oza8A6p?Tr}u=OX9 zQgp>vD9=Qf>X|J`hX?#|SkPqxkTye z%_cJ0c1uVZ5;>D~zz7i*46)B;o3!cJ#K+9O_q`GzH5M{3M-^8vziH@PwKte}JaeT3 zkJR9i^28@zdOFoWiP+zI;}WN8+xicU zm9B)zB7Cr$rAdtn>3GFJE5li?`-Y6w=&=}PwcZuyn!(qy2|N593^%US?h$G6^)WIq zA`LJ)IV96|JWqBC5zS?52V4jfWczK`YPeO033y~~|p3RP;@ zKUx6e!0~D&r$yVi!RUz_lvaBIbHQf;JOjLp)Z07zoq6m)5iG(I;1R22@6XqQkJfK@ zpScH_H@m@p&Dpv1xt66a!{Ec2T(f*+1iZ7Wf#h32(c`4 z*i6SUR=z*e;rLoAJUpFlSh+T`9jyp1+lvU(Rz!Ke>@!rO?ow_}vG3R|%3R{Y4*fbC zG1r$1x1P0v5VfJXq$#sNwmInB0OFFX`3P+k2MgHppiSPTF4^VR4;X*flirdh;e{`_ z`TbUte_tfgGNKpFB@{`5W1`4mLX8$ewzOhi%C=4L7*zD)XEG3o>YMvrkiE@Q#g)`0 zE^tzfWE6|az{oWHmn@ZX4=DEuM0+p(j=3r-=tP_@<)QC};Vi%2-C!h)C^9T{AeBR5 zZyVQNT>dAnXz;uu7tF~H@f;~0VOi;QU->wbHOvWSW8A6o(JnJ|5;SXD48S4Ww9;|gp~#K!TdK6_OZ#rfmZ*EXTeIHy zmh8A8VK&(-ndhkPLjhk0L~E=>w3%K0r<|_>e9@-VH8TB7&ebj`6}TKY(ppN0p97 zii#^9AM>x|O`G+WFXvLufZeoAt|#e~BXhr!`dS`_T+7tQstQAck{=}Su`oT10_uH}} zVyo1=fl#f<&h_WpI*mc#f7#>dC^K?4CVbAS7>Yx68o=7aV{S{gxIdTz2~Fp7$YoG5 ztxsivT{wbAjfK4)$|B>&=Gnd9S*e9ASa;DoGF!K$-IeEgh!`l25%;W4qHGvfnHHVa zxpF*}^AXJl-mw^&g<9JSg43%We)|uQ#F3Q-jQ}_DG>9KM4wQNfL z-H(Xu1hSFAD`n&C<|B-g^qRLB-(OzSExliIFdnd1P@W#D;$~Fgq zd*+DMpWetZ-}obaquapK3Q`O}auxK>59zXdL~Nu}Z5-!wFb`LG9+?SAh-7%AUL-P| z`=#jevaYt@8`89aq;TJi+G%g49r~kJZrAm_5>Bei9adKoGm2>h3)d*>A6*5~&HzHD zX8gnc>T;~>othkd4Y=mrs|Azs`i^02pIk<%ZlN$s@bUHgAQbd?-9Dx9?n!yb4Nx zdXbn{Yv$oDNl5$XHx3kJ7SG=`T}#Y%Al3}32{C4{?tDdhdXoR-Mm>-b0wt@gTs1Pd zpf{Sz&@dayJ#qt>K|ufyYWn!q1ravL(jr6W|obf?YbsmtgzmlAA#LGjBnjubt`hS!pC`<}{`RtYWM z?q8B{|`)ZxYmLdZ`w8hBlUIUu3sFNK|2Yv4 zy@B`QlCc21y|M zxZ?FJK9keODZ;d9eWk{s2zbMw0{r7T-!oR#yGsowSBh5P5D@G=5V_UoYjFyn-1Zum`d)I8`1oF@IrJ747CJK!DmE6mv_?1R zkGZ!yS!L(4Tle5dxhR2GZ3~>!SdagBhCU@FayeklS7aAE+LH6A3y>-3^c{$-a||hb zt}WtQu*^M)HjjBfH~W_czD0PU8ip(=G?63sRula?ZciW6ArJIcZWS_a1#c>_%AD2DyyuELHdS zA`deJT|BeKv9lZgc-&GiMVWozU38uX@!B2Sd7Fx3&x5@7+TO_$K3d_onsmBXFt*gl zvQpaVB>QS9W#zPQ+FYifKG^7hw9ZWa=y9_L^gN`-*t1E=>S3bx^LngNud?s7^|@3FYi z&!e+irZr`{0j7UTjoSbGJE;-I{69#I=7RrD`oIM`RVs?w=;?}7G0B&>G1!9neJ%dE zYVe>V)jE0;X=d6!mK$T;26 zp(l>-Djuq8<&NhJ2Ko$!EVGkuY_ak)t(`0!4)cH?Alfhs4^(vA^r+#_;{7UtrJ&5L zrQaw*2Ig6B|Cw0h@N$NU+Oj zQ|gPP3PyItVQ$e;e|(1|?2wNDtlRXp05=%yJQXY!L_Mubhm{WHiB4V>XE(jsusH2e zZKCad%PVhjqv}P@LvbRKM(xw{O6}!Kf>wR`H(WS(T z4K6<>Tk;3g%&(kp$P6LVCFyG~cTA?o`g*FSaV8gKsBFL0&eKrbucty)(uF{a950np zy%fA=SHeFe9WK+cF%fBl;eUOk*-WyLFe&q+m8-E25wx_e7OKl9$Dn&esgy8HBNaEHO~+s85uNb1=%Xsu4peAVxbADN2!p2(tew(fJI z1TIQBQ=8PHv^^R`pO~Ubv3Ud(sOZo&)g66l1k!M$`-uWJiJ6(1K{e#RFGuq2%y@Xb zcJ*vtP0T))-cI6%uwUob99KhHbqx#Ayuym~+Wjr1>b-nFo&tc%oUs?Wk*hcR#r)GW zelBn9e*anDVOK@QYn0ofI+2Gw4J=5Ddk@s86z}#F1ND!WCH!8BcH(ro^BWC0hj}}R zO*x?^h9e{{SoWqd@VD`606lcM?+PHE!VLO$v8wM2n?NWs5 zEzthfJCVEfXyf1=Pvnln$>XLU0o(#;D?)i~yx8bx8vE zKxh{t{{O5hr*r+T9FJmI5i&(n7A*%7J4jw%63|{qRzcrKR)HizP4+`Lks*C~AfM4YI|Bm@2=DgZw z4k;6_$NqFwpH|h*+|`3j=$i1^_{+uA%AmY|r%ekxZyZEz$LDs1BzQc{wkPzy$qY=M zL~$QT~3Ag@Z-%*t z^+~Xq<##3NT@|l%;g@cZd8eK>3#r>p5i{ekdr`_}TyHgJhLbnEbXHO%Cayt&60e^d zp#oIwY5vEnzVCQi$9yB-jrkVrX98U76%5D(_=|nbY3x)GiEtG6{xc1t+lHn9>PoBzB!hhQg0@tk-!*3cjzP%zZEDlk_KA|lWmjBf;yy~8 z;xmbQ)j#sgf2**KA-Ocr<+Tu=!MSAL+Fqh@Ts2L ze>ly+C!l}#CJ?aTg!W%4BmXJ%{~iMv4xrec1Jvg;xvo{2(+WGwH=Z4#{)MWrkADnS(Fxwra-5nHHT_K6+vS4?^wghDIOHgjM7qj&fpyqFYy zTRQL-wl70E_(Jb68dL*x`I{CP40u@zxMciSr7HD5;}MkNg!W%e|bD_sYs`Uowo6O3U`N>KxMRM7X;rJ-x@g zWMC-mBW4_JpzcCdPD~g0!;zx3QXY2=q{mH2VI;pRrXX*ekQ}<4D&_^%04$6;2f+-# z%=rsHKbL$?1{2@t_?i0&F4$?7$TL}sUoR4u(a3Pq+k8$Qi+bK5qsa|O2fh~o{tjq= zFHe<6s+b4&z!$M~$=7z?NsrCCJmfPhTAo}$lR4osKZwL0H-l1f2@w%u_0oUIP@ixdeRzzUs;TI>ao8dF^*ye_`_=j3{Oe*J?r`h&GNkQ}#?FX<mR zz=7!?hgW2B0rsbV%jOu|3`^M^-mvNjqu;Y`K42M|pYujK+S1JkArr2Q7mO^9$M~3! zIjrez=bX80+_g!nkur2fz5S?Wt1RT}37a%X$>{D$R`dzRvQG{#^$DVO-Qyg2h@9#_ z*4V|hI=y~RSGUh_>h*-B+ z;1IVEQlm6K>4~4;EIATe={@Ayf3DZzLF_cm)j0LPtEa{d{>k|2O9euYCND1zqR%&2U_q8OAOk)ML3m; zpTs=(EPGSMWb|ccwjTDslh)RhY-bunfbz6k)zgRn9P%$6AUhkNd_C_G- zjTWj?w1s^@;;IP*)9JK#u%XluvZ4U0F#ZvA|f18}v>0KEd(n zNsuVC9O~at-bud1pjahGBMV(b0u0@*baA2 zCpIpldU(0rfK0fZQS~2>OP~=R!X&GIEi{7)3`JOX3nrGYSMdAgFMM)o zJ{1Nx7jHd&{QL)7m`kmz+Oez%nLzgP*>CdEO0eI**{@%}0UI5n=VQbf zCYC$rh2_;m$E2p8>c*}dhX*3zi(hm+DAZ;Lsp<+_rxNqT6cRd=$FkhdsnbW|%-eKo z{e_3>2A=-W9QOQp>{>FbYe{tO2Ec>>Wx&8EyG#8|tNmGWGj?rJmOY<}U+XYJEX1|S zIv~W|UCG*-dE%Srzze!>p=M`iTb|5OQ}XgG=ng>+mkNM zX*C@+r2Vk>ztq*Q<|ux_qCY;#1YQhy>KCmpefRDou>{jdp+<>E3>gYiJW<;|`XkS0 zx)ONPoK#!doqBgfTQ|Ww2iFAmi(8a#1aXf!pXD~*@19gyEV34OLVwi3%Ax;w@xg4n z>r}w$038(5303nq(9Wec@yyrGM^ALeIE?m`)p?#svrl_M zD+P#S7jr6B+=9c6|cm`>m`RapSE5?b;2S7~PE69r9KS=og7)YR1Ntt|%~ z8{mF|1w&Fhi@hTZ;i8W1t?&K)J5P>71*iI&$Rt)nN`YG#6_}fzlEA5e0l*^#kUa7` zcQPU)BiV}=7Z!4dhM*rme0W~K1Kh}I(gXZ3L`Fo!ZT!lJnysyEoPKL}w+bMSWcUPp zuS!7HlAQeJUHA11u4EhCrx9^2yhjjAJ`Xkmba_LX?(XhHH*bEyBI@U60V{=FaM33x zC&_x$AW;77eZpMZ%&)AV-d(xTF*$jf*~&o~e4Ui$B~Xz9>0Rjjblw1G5zNKaVCk-) z@L4vTZtZ;#F!IHtMOsj3HelQQ++4Kw`o;!&ch}k7!=ol2uw6aynKGaV8tZCml5Pqh zW#@%9O-|}J1(LB44Cj)*FkBEZ#0qFATN`Zzl4L_^=6?WPOX{~i1$3SWq1wsfPoMtE z$;(rUl#-H)PD}&~3JOxeuY*81)&MJDgzTWy0zt_DW1L)W<{OKPi$`Wt^O{#x-GySzI%6-^znBmx zA1UZGcvaW9t|6C}Ea7qQE5Q1$|7Zcg1AwA8L75pI_YNE#9i0t`C94QK$>$z`F2Bue zwbGAl&NS)kUq`l0)q6520N*=1Fy(`l;<&J%?l9=-1Ew^7CPZW5_qTy{f>J5J2EaAY z{@p(N7ockYuR!&`+t!5VRsVUH|G(`BihO7j?^rL2_Izb%URx83G)i_i^xfoX_xzTF3l}B5Bdp*vHF87gJ#<&~@eWU~kqu=#s5tMQo?Cw3s+own3Q~_7 znwJ0$SUyy?gQvdwiN2A4>r#az9PXdkBrO<9R}edl#_xI7!v5eKfi=GjggeP=6u6(2 z8_Rb4xB65#X;}L@r5o@``LkKXB?}H4wZU*w-%WUUu51a>6qWC-~ z`-2y0t2$hQXVWyS-P|yTlZ|TbiJN`o( zJ#^a*OZMI<2|-MoW0N7~Fw@^;Bm?f*)>+iBt{wY0tE-`TWrz%IEfs$M+VMgD8NL!R z5ZKt}F}kB!t*Gjg1wR{768vz~k!NtvrEx=lt*d@$;`+xUXDROl73PGV$>QhmAJ}A* zX)M|@fiOr^?a%xiug8Msjs z^Yo6}3JqgyJjX1}O70h>g$lkgLmdCS|G<(*^r+63G3I`Nk-kyLF7Jbe?=J#Is@D=H zbRT$o%2C_qj8~NNq<)sDjEN*S!T+QT*k?jKtv0lnICcoqaO}U-{#2X>XAv)I$2+Bp zG2G2J`W0fMR>{L*s}j;~sM{$Hts6P$Rap-Z@~SIg&RcE3JQQp{@cvZ&hLp}G)S{BY z;ItZp< znld^=Mnm!#Uiy(Bnd#`|8I4klg4orssPBExa4T+SPKwU<%ivKHiNSXvQWUA9s2sA2 z*g%_bO71BGjM!ztA~4J-AM>>`tz02rU3-l7lgH27an2B1Uidez$d4~WM8CGx(EJTYPm#)SV&Bu@i>FAbO19NPyWlr&*Q@Z zYmFT}#zBz%z^xJ`mmi+#frnbV)eflTwiiG2#`hy~YFzq;nm4Qz&nb!U6>WbWWb8>B z_N}tfnu;Igh);XJFg*HmPdL^%PDoU9ChpML#ac)+tM*lNDUPA2(?wK6gpyn1{i!f1 zy;G-5lgNb6e)N>LhhHLgCqho0cv8$J$iw4TO!^`DG12HfZ^8q8rOui)SG7mm6{k@r%{{Ti8QlCNI4Nmj zqKhY~w@1y0U7`PTG;t7H7yQNN$5C-{qTD0uf`)lUS)Og|-l3G<2`;YM{bV}j&S^t` z=t6OS9A@qfeTA^dB=1|<5Cy4M<(bYcg3mP}f>27&pfDy1r3PAO$V#a<*@^IE;c%?= zFV~oqS+M5%j@C1SsnkdkiaDx$uo@=-EiVBP!Wk)-`8R8!M&&%CnJRzigDH%C$s*TdZ3 z%M5mcQ6wnXaAyT$cMzShthX2M?x!!}TZ6)IvLvk05G$rJ|D9cAU|ygy6$fvE zvu*s3d4s&jc*X=SRjm+fpPB6wap;YCF*^zq&aa9*?c=SwBLiE2A|TO^u;+bb1Mc4I zZQE>wsHxCrP4g4SppQqxqcg=G;Mdxf+;{}t<(6B`Gr?S)HR3bL^51eVmKbOnlQ=YbEh9v#Z9aTLDxA0ChwHL!-h2FuIaIK}PdGpYkyy^+CmIL;wT!BNm zOvkB$#5!A2-CVOB_G1?rsc1kS^(lL#u#vi*2YBdLF;5Dzx@z{AViS-fn8eZO7$S)_dK%@w0H z=!D$bugunFKGEWIDWfQ}`;w4Yqg4@BsHpH({#8$qTG@$9`p8;qTl;yjx-)mboynj^ zyDM(I+F_{H(NXEP-!(f1o#=!wZpuCZe(oh?s1eb#=TuJTymSXtfN^n3%b-euvP+}TqFuD(V}G1K;bKAHWx13 z#x;4(P1$lNxgJ^4LKC)c$y5{RbVK_2GT-Dor-;?=Rk6hW*0D@Q!?uCXc{V$n+EMgt zY8T$a4ToFj6a?Q3GacdYuU|9zarQe6C&HdgLUGF}45!#)|2a`biBi+ggqo^n7#Qqi z|3v$w?_4I;F8-m2$=RrUz{OGhEj&5@k%X@KSc;y_%OtLqx0T`tx0AJQPbiy_UTOuX z$J1NWcB7JyVS6b}&Y>hSoqFYHG&|&DYTDDgJEM64vkU9#UG}NFMzL~7RTd)q?P*6s z+WoOJu@7w;{Gr~btp1dDc3oaudU&U6QcSDwM`DkD1*&+pAmOy4wQVn~`k3n6uV8Y* zBiCD0wb*?)?1*bj=JfYGmRh zM`l-L!P2QBT8~w=&v1JnsxtfK9J!CSX1f#PfE8DwSyc5%3{%O5i4zV_YgXlKqHr!*snkFBfAfpg^; zKcTx>E02(9krqGk$au88!dogjD?s=}bP}h7CxLGZd>cbux z5|;PDxy0m8ZLmbl{o0Ipf%R}{bH6(tHAk8&B&+TvixY_+*(6Q7vV}+E+c<`4<}(w0 zbXFd+UB!eN2{sz7|3iwe%#aisB$UO#WAWOm-r%UF;X!e^OX;wplJf-&LH2j7`EJAv zkQ}~YjV{t_xrs~3KZ;K8d!u@!=X&jQtFT(l`AxF-rDun8{tk%XrB$*1h?~T{QPNhh_-e$l5sL5n8d1J06~YC$o55&+B$9>@N{Zm;4}GwW%&cv+z7GqFj=N zcTAEfVHpf0X~vO<=tT_K)Pt3N)hh~IZD_lscb3Y>3qNxvxs`z$^|a~!Pr`!hzlN#( zf0a@}{`J3t99Zvo>y*Mg3eOct8KF~9vAerFpBYOKiHdt*6+Z_C&R1Q>KoKk=+%NeP*8@xOo}P3`O@x#|%|t#i z_zxQ8|L3hC`5*H0|2*x45C5;u=>Ojb{R06oA~pbv=^HJx(JvVxZ`)fNp%aIO$`ak^ zA=7@h{jp3y9(gK&q?7fDzTWUy1L>bdY4A9#EBN4%=Npfa^Vp-1K2HsVaBN`x2S!{g zwwTarkL5gX-;H565k&x$IAqG3r<(EV;CE{Whnm%y_4W0b$2uSWJAekB_`2|2yKQ>V z(ZwrxDAmGuuW~^cJpaKrYCyh~Z~eZj&l=6_{6~)!u%cI&;H9;+2)u=Ag!v0^6cR-QK6(6a#j##qUReg6 z3Hr`rrDbJV|JiCY!$-0eSCbb zH^|D#g+wx`&pSS%r(fhOA-Zux_T5b>3kwTCyTn6X-Q4D;rl`cX_xGb3>k~)SzP#FlT*t zW?YgVg@qAeVX3RDrece-M;m`#Xfzg2yni&lec1bLZ)+-nCZ7y!1l zHZ_U$q~BeBSE%~_{rgZF2+%z|JjAU{|GJXwDU^oykV?3n9WJm&OOZZ)6xbRB;h!0u3`r7EG}5fQW%W6Vnf33u!(G*wkqL(@)9T#1N@8F+ZqIbx;Z zH-V(gM3WkWjI%Gs$H!qSmZK6vez1ze<738q_XOefICywUmX_b3Wm|i`y2t^FSN>Tg z!FHY7-aS1%;gOLU(EA+E>+~ArC(v`MFH4@p3opvpFpA^V#fukNj7$4-lsmsy+Nv6T z{Aug7#JR&RZS+R=mdma1y+*&yd4qM{6JYzjI$Ic2+MvJ^dj&yCU?n zOGrxEJd%`@ln0P=#tD#8Xfd~8dm!}U1+%e=*78Ek;O(YjMu0(44u8A^qQ)f}@&N3F z<9YT|=er6aymfQa5_#*RM^Xd?1b)80vMMTHfj}oDB1#8nvwH;uunifGsK6B4o&kTG zo|Xo5oRGw;SJjIP_r(Ia%J(Drw464KI&OJV__O0d4ZZ#qKk#(girGx-Gu+MkRjJ-=99m-?H5T-0{@ z7dAMmrN!=42?+_Lp}h4(gH+|K)B_P=;l=WmeEHrCX(05UvYQR}SzW`$wcDbAxrqM^ zi1Gs5t;EZhT?n##3IFrw&w(wte7wA^Y`I8v1iVy4Oe{yI!5dEMo}pm;yONQJn3(LD zrXbLHarn_6m-NA>w3oV03flc34klz}<=31!7aR48Qc+PckHesbd~768&i(S``r3Gv zIxzlLqm)qW02VdW);_@-aBTVq4@AL)8l$r#)Jn|rprf7LeDr{`U=vIN9HK2;fd9D1 z6+AM_OCHakKM&j=KisH;Pmw6&W7H}_GplELHYf)?TF%wA5~cwZyX>3O($Flz=0e@g zR>Q|7?(*T1vRPWq369xK|N6}OTxZD*j0Md@8X94<;Uc+}{ycP+Je}S8sA`^CF*VI4 z3}Cp!J;Nj;Z(K7DvkpV>m53W?_<$GWb%@VR``p?Fo|oJhL5kAAe(G<8YNr3A|{&bZxj;609$nEaj#^ax@GV z+g%@1A1k*DD=8_-k-Uu2M~*D}Zn!tTw#R8P@@z`?gXi!k#&9#vyUXdZ7S*cpJPt}D zKh3~8tuI?+mVy+AxSt&wn2r3DRxdF}gS)a@?4klYi2^*le79d6oukZRx6&^S3Y^bO zf+6by*u9{%9mr~)M%inV{@iY`^Lm9twx_!T$S{OubWW7>{(9a&U>-XuhR6M+^nNpV zzEnI4UV40dd?d&MKiU^eCDd1BB(EHWje*hxkCv1H5GI_&K(=B|kujP9E|DIt$YIR}ofkb#>_Ozs2ERQw8OTSUpY038^Si0R>d}*RAK4u@-`K4VcB5%8U>Ng* zZlTt1-n_Xp?y_NYxIOpGe)TIUyYZ|c;SS79c-IlC9hd9DTOq!3*S@DmEaqXKZF&T5 z{CJ7-)AJ}4r3@i`!LMNM%Jgyf?07Yewi6dG6EZ~kzY78;0f*H8UAL(Z zSc)--WRP$JBp;$mObqtkA8K1x_!z~~-~;B{V>6jG%Tcg7wKD5;@b)?IpNOja`TO_g zm5i%B{pf4W`XgvLeg9_BluEU=_cbZ$d(p)TA=6`pyL~)a%Uiz` zc5}YmOX>d~@qYQ|#Qq{~M?2k6_hUQTOn``j?(Z+%engCY8Pa!=>)NO}uLf(YsmXwm zEGe za<;6R)pG^Bzw|+qIOANHRrbQNGec}hy3FsAE=qxog1KXXt|UQa5`{4$qrciCs0DTD zo#BMeUD<}=zvRJV<^$fici9})HQY|Nn!t_q&$fiIJ8kQMZ>uQh#Sq*T14tKz2T{Vi zj<_u7AX|=5*6K!LMKIllQ}temG_RQ6pb~d-5@CoG8JM`CRQ1vKS?x>J3{K@VM$^sP zW1=^#h?Izszm%xD>4b+^aH#fgjj#6q_JvvK`D5tgm>>O7y zc69VST|31?t7R*ZeSWKO<+_&rYmSDD&I#3dN38;njUJXgzdl)%3|(K;kY4E0XnWzt z-Dg&1W^?}=Iz*0$%YG$TYiY{K-o^cXaXo|Tz-Z3BCKDATJO806+eD{(QGahavn2GI z?PAyhTam617S^XF?}Qn4leTD^200wN>5=`N5|=BB8N|4Ri*r;HN78*;dslWRSc0+^ zE18I!1XBCg*Ny*{IL04M*-0-Fq`sk~MAw`tNgr*Js51Vr&>xIAX}M|k2mf^qk4y7$ ziqLW1*=Qei8?Fa>KE*%LWf4uW4M>Yiq(0YPp2kATkBGu?Ysv|mM`r6?YfrpyMlzxE z)rR?l{QP=6&i+z@KQsp05%bJ0+r0Yl9oWD&bUG$R_7l)<%Vb)~NEBEtu*dx}0;g#( z8wP@hX0-nE&@b2XqWfg-pA{=itO;brrBKghP;~v2(H@xP$y}Vtm2e`6Kjc*^89SB9 zJ9#J`&As&On^ARwt4_qLyP-G(i(6+NJpuv;uSfQ187sOnPM(VG_Ac1$3<&IWoig?? z#|SmJuet@CHV{^wkLl%oZ3%?|fT@AM&?eT!oshj~V zQ$7J1S#Fe;vp6<3Hbir>QOue}Z|W{6M`2;0GQeckij2D-n<3+U!X8K?6NUOJootk& z6dL~=jiIQ92{~U(ob40imI}N43|DXXsdlh@)$J~)2ID$jSVfCaO(K`n#P5Kv607?~ zTT6ag;tld1jvGa*-c$`opMPK0N>();$wr%m>VG=Z>gA+a{({ap*%3B7KVfRzt9Zqq zF2mlm*f~Mm`uO^tGJm!{$!$_D?K|F9_R(^=x&^zD$LU(BG;$l#+BqwUXDR~m%}UAa z@;;nZIX)Ga;*UM|6!fA(jz*4G!h^#fnpl0uDU>gAY1c+KJ{50UaI@zqIk?_)f>%L$ojkSLorsG>F{N@>LfOr=ZBzDdf4koIHKmjM@+U325F-_v zk3Ex)VH4jUcPSWm+t*Z%x`cu9MvfSZ!(u$QC5&#xi$tsI%blls9Di$OjGM^Pj_Tf( zu`gziR5Z~y_fl^J-yY+deBQ3{R9=X2M&f28-K_nBD^nteFnP+!+|(V^1HSd%oL+DG zL7JAouTKBAQ(3DYvjnW#$A75enr-D^H)*!ZS(9LZD*U^+sFS6J>g`YEB4>f6O9d1wsZOW z3_%-Mm6d$7cNm&kP~%YILJb^|3VH;SfyKKPLu1MS9+|`Z2GN)dD=Bv5OC%* zU%`+K08-vF42mt6Hf=3hYRH<5-Rub z&gX_XC5uQ!V)O-~a~Z3O95?!7)pvEXwfRc{6nVHmowHSmN;(OM7)+@L91U#zj z=?rpEzcS*|D2uHR_az|5h>G>FigGDA!og;3`MIXcG@9^WrFa(W|B8snpX1r{Ta9&* zv@=~`I&P^tn%f}?C6@oTe*38`mD9>@cfHqCO(0#MY}*(n2&jji27(1}lg6vh34${r-ndH3gQ za&M+gMLAjK_p_b&qhHK&L4&l*TT#+SDS|)Nf(C+{I+%|Csxf7MNyb;6G;EIf-@xgY z{%-5yDnrL*IW zs91Ho)w>rlEF;SIv#w&GYi&CD=o}mza4C3IATk7bjkNDML=`^9VE}5uIzsIgjbD&EnXnsm3rv&UkaOZhL1(?*e=yhlrH7RoN2M z)+-u8b?fp(O%2SED|*|AGiKRqm_Y9;D08g2PQ9~p;}hLjSIO>nBcCxmzJ=3w7?(A?SC2_al9Ex=la z<;W_aFLTPk6024dQIR(aNCx<^&9s#ZK>1-!cAPX`- zm=p^odg?z-y8$6C2JU2N%|3uqKr4uE?<+iTez^xox{H84C@5#jB=x`reU2rO{4 zQZVpesUpk;)FH!&`)yug0!Tk}WJD8E00EW-iH_Hdy=(EYFU zxO9LR1CoNEBSTgb|I+nqwJngzBWdYFe*ST*#IUfLtCUE_59n!@VvZ8PdH;Zd3PcFZ z6iHt}HpF=ZWp_lBlm$>bj$oMxl7pauhCOMY0lJVChrDX_D!If*pFe-3R38x+gFXOK z(J#$8mX^Thk_`&;3C7$Q06OPc{_9<#(fQe_Ur>-TKt9#S>pTc9127nb$X&Yw+678^ zYFPkT1Rk&CR|I4jcAt?*K!v-JTT}e=R5|810$j^mpF4KQ}st&4^k?Mh0-GK6g0H zXRZg|Zr-}J+s0C3xG`RZ;A((aL;!*<4)_`UE$W{Di$SALa6v(Vf6f(*$xsBukFuZt za|gsS{P4FOEHE&*-joA@+yJpzqO|~AntA+U(tI<(ssPDFkJNp<67lsb0Q58<@8Yi= z9ZO*ivWL%}#RTI+ zO(#;k$shy3Q7VrU;Fd^A3@buJ5O67ZW8>`mxt0zN%6fWf2;6e@QA%nm5tPqD0_^4O z&A`g~8enMVamLAQ1mX`3O@UY;JtKpGoqac@mWrEO6>L;!#~wWb(1PTX*Zc*1Fz{Ql zc-RA|A+_An0swOkIlKTl^m|LA&TPxVa3XBa9BT#s(3BtI&y&Vc33KpjifI1-|4S4(ufjQx^v5Wwi z0}KRDpPQf0PD&C~%29+k0HE*I*4Edc?a`i709>)gwzqb4Fcq*NR{$C>Pi)*~Et^j<+_j>*(mL+}ua;&+HHni4G*^2?z2;-MlPaN{9<^ zRCo=;!-t3M15n-#)Qds%=8I4OQX$&BFOHBuQBhGeEx`FaF%UE=rHWtMvYPsoxKQB2 z4XaXi<-LD@#xIBjI1~Yanm?=%xx%}|4Sd*VXJ=nN<1u&%ToX+T%%_Dtg@z|47Sz|j z5NEi>tYJJ7H4UZU2)A-@!1aL|4hq0z@w2nD<*ww92vz|l;`CsIcbCyNm&DTdp*lRO zrsmwD8oI|UaUdPRkXW0j(f&K%-aBGZ13XtYNYc`0x&+w@`!(h=ccawGqN(_>u={Yd zU=+m~cv)Cv0L%^42VD$p<>-=hx%`shmNg8G%ihsFdYxQnfrY)`b1~|#`g`vdz1LOF zyUc1@q}Jhk^dauJxHJ(+}Xz)HQM;Ro1Rce zuc=9q{{rNdm_;kMw3KUYs89^%;3}J%=6YYhQ-1^MyR$R(H73b1Wa%LOf~xgb5F(&G zF)!*Bt09j3bi0iO&>n{8!nikY+(-gOgMo_bxwhMG1TZ0N*0V2x9#DZe0a2?Q$xAT$ zED7+)q*p`LZWRa^!lY3u3wY-Ks7(-s%j%P2)1e-8dh$)A#nNW}Pj|&ebq*N!F}ndx z=hg2`I~5bB9SER=f$*)%AD~|V+?X2j$>CxXbc)C|zMU?i^7FG}D8XM@-j?-3+&e6@ zYP*$jxXdOKamlbogg^#iv$t~s8VmV0HqT*3V6BiA1HhUQDvM{mROtKn@5>`41yT>f za@szIH8wRNXXsDDq9iLD{6Ok!5|V=lkg{9b!CLy*|2@S=lrH~MFtE*0x)7rZuDvN% z+pW7oiGps3WuO)IR)Fz|%8y1IxnW2rs>LRSlk?qJ$)ySlT}V|I&i-m25R5!fv$hw0Y@$7$?h zlYxKtDl3olILPvWzeI9rD=Vv`on9F<1TteD=cmq-@2?P&k*&hTUO~dq?i3Lu9JSvV z&s!g>VC3LX67s$t()iZu-@VdYhw;w}EN}{de9KEoeYt%7_C3&BP$m{KN5b`iNBsQA z>8VvZeuuP(O{e}6-^peJV6)$IB#~zzE|078FrWxRk$;gm%o|7C|8kF~L=@15hkqAL zKw5!I?I(8_O-7y!kotqL7W5o6J4+xf7AQ6F8kDY^?~xR!Bc8_u(y$mlm!E-T+$jL< z-V#pghnS!Xwt&b@qd!R-*kzCNGo%b>dU`rMDrx~aTgyAnV5YK#Av76xS|G1FUax3v zZ%1o7jHNeGdT0P)W@BgM*>(bQ;JlJoQe-*x0wT^a$Y2qR(Dy@MQaK7-(bIo|;{?vI zzd*YHq}j01j|g1rz(suCBoIKL4}Zbh(V{?jk9(ZEwa0PGBk@lnzZ;v?v@kCDGlhR~ zJU)hJ_HWRN+Tym3j{Z1q8=wut2u;$XHZvc_XGetR*evzL@d$Y>RPkMgUxh zaU3vghU=qcF;5Kwad2?hOa~wHZT-4sJdhv%@4kNg+Z{jD5g;}k8l}@AV9tWlmVapv ztp8uy!<+QGt)&ID!hdJjq~IYKVsOJf;Sc2zLtIu-f$%OE3`m|extx}szErh8-U}pS z7I4Hlh;?Aq_gk<0+aXaZm7IJ+skAb}$w9v~d537|dUWh)Iu#|ZvSFEbXVhZfNT*?A zCCyML(-W_q-uRr~IhbZ#eS#D2$9c%oAV)$<1t1vg1m-FB3O=OKjhx$B z7<3gWGeUyosRkbtU>Ne5JIio5R{Yg}FUHeVD<`T0%a#frbDc|i`_y< zdJ-gXhKhb_u>9-uErbjMr=pxyU@-Dsh|anlhvJE%ZjEY^&rjyQU$k>;L{#GDI8K)3 z56m@DJ{&bE6U8M8#_^sxZ17DFh(Ckim6%xrg&ZBoG-BL!1EGA(_$IvAQwWq(%XLen z@z*cI+2#=Ow{#>q1WFs+c-{wPc7^bW)SFHWvl+zOySbkdhy|JEmX@+0sxSf( zLRf6X4Iriy{y9UBoh{_&%2BW_MIAUuPh^rl2}vgC6(bjy3an!wHHbeO5(lLrlVK0l zY<-U-;XAZZ?v<-?1RQf~qq%%J94mwCH2Ud!ZRAr`Lswvj5rZI!9b4m{Zyb zt4?GS+e$pm-ATOz9c^+}dk^D>E45-NJ)Ub@H0KZ62k?@+EeRYgSLI3YMbgIRsr>BC zk~m*ybUR+ax24~&v|Ceswmp4K?X*+Ij(#c)2gj7*VY5=dOl;BamWT0-_oI;Y2BP|g z26k}oUHqTzJwh`Z_0T{#Us_%{F>jSxE>&!_aH6Po zyeFrh*r9B-RWKnG)2WwTD|^^*wXmS_!XH*APWe=Oo$aER+gIgZ4)EhFIiL9{(Yk%t zJe>bszNCE%{nKdaPt5ylkCm}yE61qFn&ZMia%MT%0)}&{txijW}-z{vu-4na(wtT7t8hdPz40ZJH=6&(>U49M(F|OlwwevHXII&b|PK zx$i9`NQl=)qBi7PK3w+K5w`=;IJJNJyz8Ywe7F4b*n^Nz8|Geu^0c(KnrUNi`_{KG zC7g5u`0tb{<39V1au2(Tu1NMuvG@|6shfLXq))*TE)i%v@)djdf%t+}_v4P!rrFF& zdHc`5uV#lp=GgfAH{y=eYTT>9HM70_brnq4;`=KUU8nmK1xLFp73F`jr-<`c%uVdq zJXW{Qu6Q!Y*__>ki5tbBWlKmiE>ir-5qmD+vQJKojyS_SamjovbW3hkMR!2B@#{U`#2( zeAlso{|V#uJf8g4u@lAF4Hk!S?)}RBIH!ei8-hD`J|7%7+O7?wyOULuFpR85LK4;j zMRn<@A5Y62un69QzXawOwWJrEwx%_Fi$6|&(hBco;c#Z zuEj-sW=AXd^_~>QC&9TDev*2HAyOA%ybM|T8)Rf(AVz{f;4-6f{x5(gA4B;1$;W2~ z9F|Cl`4|~_hTX&E`rZP+xo#Tlo9iTm61PNeOS0s)ozPMLz$U)$>uZc1*76CAzivw? z?O=%oB^Y=Ak_q>_Pw|sjmziU`4(c~r#y4GVV*H+LNh2COSbpVNvuniWZ+=~T$*26x_@#1%Zfz7xa#fJO zK(Z%=F?h#@lf%DT^RX6hfzmlxObvoDH4acva| z@WRfBUR+i|(Z97VxC}nc88nK{d0a#my{hRuR|GOy=#@MAAFldXzA#5p*qPoAiyg&_U?mMalSzKV*_e$ zZ)>QO;dSntDQj@fuc5X>!OT>CZX%AoVc;`O(0~`0x48jj9U7DerpGu@XQ#wUr*|=E z9KR%dL~(2SXK$ro3<*$*wtjDT$$Y-B9+WgDcz&$tf<2tslsfUvf3U=iagTEBs5VF3 zy>9W3!EuN`1Qjo2Wp6+c=r`9xYuD`-I+apODqsgbLrpU^7uR4n)rIVS!|@zE_rv8& zcrvb-KA0Fdj$Y@Kziax%&p1toI}8VEm^;jPK2v6N*5hN`dpPyua?aT zmp$YV`rAzM*wsFWAu%a=nb_N498dJA2MawBk?A7(+H?PNPtRn{sfCS;#cJ3Q7Wdg! z1qe?C+{pQcBh|jVn)2_r=x9JWh^CkcK6P}?O73#5tB`gnqPQDi5cw(j^DV8RS2SV} zf;|B}Ov}i43qt-3HmemixehD8O9_m1( z=eX5K(hW5p$j9)bfe6t7cSy2Ph|Sikup6trRn=UT?-vq1&bXl#8hDWR%34lX0P{H58VU;&0VYBi z$*k!IVf%|Lg||??^OfC%_?g4{>SC9#Z*$oe2QD?W7|;ZRH68+yEZTtp6C_*B{61)) zJo5zVs}>ld?kNI-XxgVwZvj&98`fa8H7#XDIC6A*CU(Yo_oSWhgS#Yl;s(D%!iUyy-y!}m%dhp(sD zSdu^_aFdOi`T~5F9z(g^%KX3GZT(wN-kktZfP^keMn0;mc02BZlus&}6$^w+2=Y&I zateYFX?=Zt9heeva4-{uSeIm2%~x)l2LL1|E9D>`A>h93ParJ()B5zt4&oae$kZMJ z-FA4Jm)dzzai??|Ijf0@iSA?qf)Kaww_&knkW{KvyK)1zlH6^W)Vs6^a9S#mGJ$vj zzReGja{*g(0I6?Cbo6J)8Q(7VWnIO?16w03tVj6>)Km3i-J4n{>HG$W4Gk?V)xCQd z$g&j_2zhyV$&i)HPnU)HE+#jEjLJ!Gj_^@auS_cH3k%~DA-?HR-Ejh_`V8P_Dpk(x z10z(_*<(96Vq${AvkZXvBgTbyZ%}t}q=cm-Q6LDUWE#F5?5ltW|25dHQWr|@J{NrWz+hkB+<%rtaR$IOmG5t{_|H#l^b8E{BbQOR zjf0>h$YoIAWWI{8hen+5)^if{gg8*R{T76Lu_xU>RR})Zwh5A$;s@2zfO}Gc@%asX zCf((BCQ|Xwh&(_XWdWNP^AQu?k!}6+M_5CH3^Fud5W!?fZV{OVawYk77IG*A(Lso+ zMMT1}0zH@t#k03r^{xV9euIR>2kLQ&08`YV0)zkF)z$Segz#x;X>S1FfU4<-aOK^} zLRTKCe8b?h7`L_cg}Zow2;U{G3sSmxd7?Pjg4 zRAawJO8iL0vp-K=w+|(U;Rz)gli={aJ~hBY?t>X%h--+}L9Xiy)MyGQ>sMBN9Ec&` zLwWN>D9rz{*OY00gJ6ib;LAPgDB4f8fYk*836rZ{l0Uo_78dp$t{m)-U=I34X73bK z9Es*A<(X1GA|M#jbas9Q!sZR-dxQY2A$OheD&B8$a?<$RzWx(75QNxghn*fTLTPw} zCE+5b;1p(Kz*lVo>#Pf=S~g1othGq0XaL1v$D7A*fXPMt7Jx<{0GyYK<-mnPeVDXo zDL4zqYSE}$2XIDkc=#s@KIcgpk0beX-8w)ECpR{N0Na=P`WO=yH=RczEZpBX!sgq& z3rxU08k$RYIZQDyUWR^x+l?A}0L}`M>W?)w0su=Re+$IsJ#Owha8mkk%ZpSg-G(

ClhR94d&9H~&2CoCLZM&o|B&wKp9> zAUyjTir*jGF3Qr-(WSyDo7sJLclVc)k_`?MZa#qL-Fxzc0Dkk8%ZeJ31Wf2Bxgshp zEZi{q+8^WI>2XErAT&*_faNsFwOaMV^p}fS~LH92m`m2h(9P{B@iY zt{;&T1t^4GKtw4A4PYwP-o``_L`q!rYSbxe9~-YyTjSEnB>YNx{?QVu?yukwU?eFp zAfj(N_ze@`?k`e<+fv@22@;@YWxWa1y?3{v>*P7wm>sm{o5O~d?Mi<;Cc#YtG^xhZ z5592|$Z-JsJ_)9sh?p9q){t!)@-=<)v3u0ip+Z;RhtbI>srX{}5zAEDEa1Vn{rN-k zG{pUkI%U`kS&uQ)!V5tPZ3UdI@ikoIVp7oDVtGAbVTBRboo+ESKRX=%4alKxb6MU4 zCU}+b4uRZS19DtO!13dFoXk*Cee!J4V_mUeBgF+~Z)#=5WMJNaVSBQ+@I)U70mYS0vv)%FO4+(L=TD=Ta zc%4SCiF$ndI$~?Q@hQzaEkEly0dixy_bCc8mDtrN)KA#J$VD&T?B8TmmpArybZI5#y$W=;5_)Nu~CtWUi~ADIL8nRxbTp4 zhEl4TI>!6igI|4{=dWMB2qc_riMggbDn+e@;&eE&5s%tA_{If?_0jh}K3CXC15z8P zsI5sAHCJVgo`7K^fbG*+3xnv=h)=5l9m%?7L@N9rKYGfDYx(yRqKSB_|M;;-e8_+N z3=tu|?%&V<+aHd=^&@zYwsCs5-lbSjGvFqU_=obp8e~jGXN;;oIgN}Tbr@~!h`O#H z5h_au;{8@7njRd{Opc^^x$hC)u2p;HD%Lb7q4C6LeX914eOFOJ;Fv>Rk>tF3fbU4o z6Dh_zKKNE#BD2Y^z4oTh4i_iK>B&}ly{fcFJ2aRQHJQe<``+;Wc;|lGqwYAU;`!1( zul=CTqg;;txk;zN>v8*iaf}HLTpLy#ar=bdou9l&Ynqms+L-iDRSu4}_)sV_qQRu) z(>Ct=Wc}u-*ssaJR9!2g;>vbd4~{(l@4JcT-bn-V_euj(Q>lEO@NkZGt--`(HjWvV z4UYiRW8sAx`H`LZ#4Ke=YW&_K!b}!Vm3cQT$AvKBtjBeC-wN;5tqmq-)utBOQ?WN) zrMiUksye7ivEQ9`#j@B@zx&g@`9N%MHcJcAKs0vaACyu*4>s$BX61=IdzV| z{>l*i{CJ0JBkN%l53FS-h9cA{C}JX}c1*n_7@uY5b{}Q?Xnd7Hxmu-g$M0S1muV~C zZF_Oq$cVhC_yYsIHGJ+mOZy6ERu+6WRmAxch}ZPs-%)FUOUri{pQ`kEq?_mzq{xbT zd?;-zCmBqG6;t(%@^BjOt*D{ox`@rsW7KSuNo}MdC4Eud!6PP;g_0Aj)Qd3-`H#_62fsJgDy7HNDZ3vs+Sl z#vh$2C{PpT81Hgp9cei?M-=srC#9Gg_FA-drGs-ebS z+q>N&peU{V&$_{U4X2aY<$DWkH^(XblkfuM#8TfogzYV|yI$rWcgXs4YWl;bIj%Ce z7ms)$7~4pFdQ&=(<%7?ay3_Yj{GXI5)a@6J-VIQNzY9@KOZaI z{6*Q`;~T+D@%i*b=f2qS<%CVt2##AuZteLGhVm}OcgHck9!#z6)YzH!!{y_jUiM%=I)DU)AR_B1Xg2UCn@&W@>{?E0)8SB%h5R}t11 zCKPmyPbj^1Ao!SI&Jf<{k-Ge7Tq<(Na9?+fBP$_TqUz^Jt&DK?_aFAMpI?>VnN}4L z8FCz>AhIB5Cp0tUr8`46<@wDd5w;tU-8gLI;mGiBm*>IR_ZxmbrY0rzl0?7ZNr78? zpz>9!?Hy_jb*swI>@we1r4OX$?#`YaE3m~g@7-xV&y=zrkrL%;cCz>}{Kr<^GlZ!`{L0^X&2rD6GAiQO+HzdJNV3C4k6f1u zE*2;9FNOL{WoGu#i%P}(FS5HQJg;#seR@3I+H`U9!QqP%cKjDtGb@8yryTs{fuykme*o=zjOx~n?@3GG3KjL(Gw954o;{!Wyjujr) z&3U!7NQIx5t_Ro-y7S$Q8VC^4)y1hUBX~bHP#99~PDUErG37J+dZPcU^hcc>^E#SK z!sEok$1l;Q7QcoLSaf2FFB-63e>{4Iwbff*n>R_rVa}_hKr3HV$LMoiWG$>1Gqu>0 zLl2jAy(H>}SLsQ2uPi{e4nI5U5|!e-`;0CSZ=7Kb*x<+pM+Dx&w&|`}r_GH&#SFc^ zJuI5?_j=$m`Cl4*sU*Bdp%Nunqy*$hsq-DV9G)kx%3H!JV#TX5@;NhUjt3h=)mNtfi0-vsp63%jercMDf2g zZZ%mNkT%SHt8>&&cs#w_AAOZ#4?hn3;I94p5rz;cNl>%(Zfr`TCWTlG2jfvKzWzg=qHVn-})ZhMF*d5zruRZS9u?NzV* z$*ffM=TY2o@z){IhcumXZ#BK?50%o-X36XCeR_QQQ|fv@y>NrTI>XUETBh+{fZC4Z z2A!NrSV>fg2y&~gLie!OYJ(c*UY}b?Xg|1YLX(X9nc&XcIlalp+j(AH-nmYj=bvu8 z(DOUEN=cRF;_Kk?NL!bYP@sn7USBrz97D-`QNgIA^t8D|EAMWk(c|e)gB; zrU??dHqLzZ8Y=p{VGhQ3TK}QExs&QoDyk$cogTpN>NkN zGE=&W3jGwrG}QD+_ww*S=$VPV`5CjWv~O3v^p3i#oQm^T%!P8+X$R2Lch{qMC{%4K zS;|=7R;m=e{a~0WceTbK(l?RvrPQ4nH)(5aJmfmSZ~mp|aA8*q*afSTf=tN=gP! zJzVOAZa$tRH9g70!JzM|G>~CqM&F{D_-&YmDg27qdiL_0$`8E;HurF3CJXpRA%#wQ z%>a;2k}TpNb?F~x%osakwIwMGRc}u-y;SuLPO|!1VlUuc9{N7+gLRM=!z0gIV;3}) z*w^lzS4BI0QLNhBu2)@*enu(d;6^$nuifNwO+f7AeQ)zOM>J`flbl5HY261h*4cXl z^!JR4yA+g6m5M|UKHW34Fm3c6ix#PAJEyoiEpw5)N$2F&tDoDYtB!ebf33gvSaBEM z*jx?N(roRYta9f6%*=kcH}>Fk|GAn2F2?)OI1Qe-#FlL1x!cS>m7^qgtQ_rPD55^q z#>)##+P*6*u%7#}|MQRA5X5)S;`T>_;D8Ik1g1s)af$nRd+Yec6U9?{S&wb|!L_%l zBDW4Wvr<~tDXDB&R6+^&_tRwsF$G`0Y&^%r_UzURLcgIYH5zSgeDplj^1O#Ux&ml{ zfD1P+h^BsDY|{JCqG0<_Hsr#rvaPg&-I|GgrPG0Pix1aV)hks7jnkXaE6uGe&O4#a z6G0@W@jpjMvVM?1aS~IJhrwPpRwiID5Jz%;G3VS%zU}_b)V~SsHNp44n0u>`DOfr z5){736bqMi(Wpe3Qwx_hFQui4_k2Kv{nb8iNTX=5Q@XC|inwW@FRW2o`P946ytRmt zSWjpCt6T1Gm&%5JW)`u=__4VbzTbhW(q|^%P|h*i?4o0~f4TEni-qN)n*J?e+#++& z=Jppq*%yM*#i~w<#UC3t+Fe6khF*pC<%K_6G>6cLA_887tjAdgDalsBp3!l-FI~^{wGl*8-wTX7*-pGpV?MQdmrq9Tx zyk(w8>W%Yi18>s1De=yy%gut*s}EIdCwW=_UJG{Hb;(WGqUNu8^Px$;GsS~ktVva< z^=R{NgsdeSrb&>d*!V>{4$AAF$u8HdntdhT@=$TvRMzNZ7Svfc(~#i)(k?ksTM9`` z5|grjNM(1@d*bHdpa@C)>Fuf)xBr???Tj~mWizLGrKmI@e~@f8;vCHCpfj{RQ)-si zB3Eqo;A8qmJDEdG?X`<3n;kKyFU2eTU3D?n)6YDAv4mPS=)JI43dD*s;8xW_@lZ`Z zarl~l`axbXZQd#eMRR%o$aAyCF>?ZAG)-z?Oa?+>+q>HA7zt?hrf4TK{%s0h4_aw^7n&M z?@vxy>b0HD9n9Y!Y?|Vl)4bmQ_4MLh49=f3=}mX*G*!Iuh{X}_=%};#_g0shL{q==QL;WcyTZErBG)WC+0rhZ zFk8;*J;N?#PRzA%&Tno?7|S`dFGFuiXKv&T{VmHed+Jij`P7$JT@SwpRLV`QN-GaL`Ec0y=Eu{Y_N(I|S)^H9J_L2=v2@9ponPZ+ z3g`99{g;-lHiY*Ax!Vkg;s*Ake7k9!1kX7e(Fob`*ODej+*#*ZLX?#98O)TWS0K@6 z%c|*XKU@_Juj$in(_$ zsz!YLll|>d3JXZWEd)T}-=5#3{(WeMfJvFfF`GrL{

>UETfORu|M?F5$kQ3mPal z_$fgS8AJiITTYSWILc@gCfznoRv=%tU)-bd za`uTy)Z`p06)w)Fv!=SZ5IK6idfY-Aw7LKOfYfF@7&!YB^D73f&gjmuf${zi59@fTNt2xqkAkSk=@q#<}Tw`7ULq{2h)@_1RoT z@#tgks962p4CI_p8OGz4QwlBEh{wMc^kWuXTJR<6 z%=q9q1V{crby;aDwq2OuI*1HEA*SMq`!Y{XvcccE%X|Dz#*7^f=EvdHds*; z{;=eeckSZ(T(X*@ZyRjHG4XjjJ$7BPZV46H4R;N(-tDXWQmfm@5f%P(vOe>{grR(O zl~Km}lM9314+^j?1q8FUpra{kprEZvf$!`bUxwiA^t%=0F>SgSJw@j?bUy^=0-=97 zkaov(dvvZ#Fko-Pbe`-eytvy!+6Irnr5-$~*0INM$#qVqDU*t<^Af=j?0MZRH;SXc z-r8k9SPJP$z-xZzx!=1gyfr5cA9{{Vf%bz6#Bc980!r!{=e zPJvfpvk3`PYp@=P1^Y2G)W5<>TEay_e(7zTmoDQSie!J8jR?0~=YxH~fLq^&+0nG$ z-8w(2H37!3a38^w*ktcdfmtiBC)34hS_p;r^$3X&!KNFCOH%G#&Vx_PZ3&;-(9(J# zyvdKl;nC55x7r8`R41G)mcJ(BT?BjH|1(=-|DQMA3VQCzLG@3n4G@fW!LBmEb275p zU)$n~hY{M?KNDeO07l?|>OTZfv^|Im#^M5Q$?VPmP+BrgfeU)0q8Mntp8#4IhVmV3 zQRXTAKojEP0rH|W@L_}LP8=NA1e{hPYHHY^Fd<`f>n*6o0=firnDiBTWE3s2@Tn_5 zATz^(V+i@%w`faCOSpobe4rw0aT6^tz>JO`>?f+2udc0q0xAno*E_)$amHW#;zf#_ zKajrK`uqF;ei(*?q~v=G3wlsu69I%;pfQmHr3-L@B4CpmCd3W5e+7HR1u1~1{0`)i zccqgng{944smlpm&wNW-d@Es=Df&FVSAV)BRE0;-Rdk*AO zdwXY~lMw)&Ebm+zs1`y(kHKw#3Yr&S2S0?EAD<@$1tnEj$R|8xroLyb@I|9(3p`t=TRZoVJYP}~fP+(9&JDnZ^E>}!O<^?c~2>>KU7SJLC zfT>+3Q1uvdD%igR>#SQpW@=e1Qc-U<^l#rMilT4U0s1CP*hC;aGcc+w7V17t*ayeS`By&Jpwwq_GJhV z*Ff1$`s7YIsOC=wt4(b8^)Y^~(Q4|dq z_Zf&vpwKzHx$(kUE?0eAlYfx!|Z%rg9NrgKA|vWux8Ig1h9{zcKV+&d&puymZ=m}`et1vzs*ZxN%bQX%Gc)^*!3}V67+p-x#)b(S44^Uy*76l~%T%9WG(np$GNpVQ zU>5)r1GE6pC(w_uuhb2qS_OB;<#o?TwF-uy>iN@5K%8a(K%yFyfdMK6`~#yu1Jn}YvKqetfiRpaPcokgsEBM>(5*^{WuXH%sCTen zwVd_=3e*c?K3v%72R{z#eu#i{ub3S)**y$yw9bwSyu=LXGjOAtudDw22Mb^TY^6Y~ z!M->k83Wd^1DU5QdtCWcWG1S&r?0_?^YM9#8`|}Q%~~&B<*2wQqM~iNRDs86bfSmfTU>$vV=-70E5Wiyg>l6%_U&*hwAK1Kp@E^ zbD@JGK1Wy{S*`y71Xo*1P*esRU!Zc~vh{0#1xx@72oHw>B;gAnHSD*?o&$*`{_Us7 z$mw8${`>89nafnRu&V@=MECFC?}7RSQY;!C-g8jL1RLUvtN$Df|DK*eJ3m80vW0!S z$_;D;u5Jh6LESg_`C z$vM#>k2NV&75h{$;t3^}k=b)Rwo-TyD)33@>YAl)z_ zekyR-v;!0lqhVlt2i(l+`nvwFua7`76EiiXw%!3Vk2ow!H$O%E`0*HYGbltz7W5(p zG`EbT{VZ)k&0?4z+j z)(Z}M04zd6LV{X@q8ijxz)}-VASLMg=depR^iVI+ZUt^VWybyZgo2)gz-mGqMCTj% z)DRFDu;6Q;(N6l$7wUO~hd0tNpU{7{FB_ zAP2#+!(VdHS%d>reg*?VBzCsz=YO{9LFY#Ldo3iL^3R5L&S1O^Obf06t!`+Tyvd5b4fAh!{BvA=g(B-az*!LI5nIR|tM@{R$v3 z;nGQRFeQUL4R8_IpaM0ji%Uz2djepl^;gUi`vg)P06l&ZW#;&Ig4Gs`y9L=6;A>^m zd(t@5|6GKe4A+2QQXTE3yTCw4*YFn0f;9(hkro^42Pljua4--PG$VjYK7o}QsD3T7 zCjpBfQAbA(5Clj24F}&K4h1x%PuB<2!T}IQ3V1RQs-?C!pscqqi4zrE2n>iT2Vj9r zng9`u2Lryc%)rzOn+z;UX@MLLfVIy6i+Eo;seTjf4L~|T;*s+562PS^P`@?oMuUYv z%-a$vP5T2_tAct-ACPtcQUv!kE)K_PzVUgem~zD&xZcS4_yB;5t^h&;#zp|j)fZF_ zlD&TYOK}O@U>*K^4$#F1wgB|>9-v0BwJo6e(g9HdF6ABIuR!sGH+agTE(|apKyw&? zH3qObjCw*&rEa%B%MH$NA_TXFlAEKs zs3cloBvJfbQmoGUpmEu*K7)NBfC^?CT`)kkS{n#TlG;$=Y^B!z{+FQDHh5kc#)6d; zEG>X~0c!9#@uH3So0_tv<3Q#L92!yph!sFZ&@iIA5$mqWg zM3MgXW>GLKN^NOCwqZ8s1pqZzXs0_J%!7qWV`vtcx(#T;)9rEOB;X2A@k~+X2St%i z!vsnsm^}HlVN@S_T?iny(*ej?LA;W$K)txO28sfnK1D>FdCOz#*tsJNP$3o9Bze(j zDM|o>41+=?0x>{a0ct4`y#%&I2Wub)dt4CK;6*f0N(z#$lRd7A+ooz**w@v-;9xuO zx047;6rjjJs8v=o(9`?0eso^E0CmM*?CdHXO;E;)6CzO|7U-=B14GKt$}Xr$J4F*@QNwqKWPWu@iL%Vm4vMaA--ZX!@Cxx zU2SulHXZMjrXJe;o>Dkx`QkOX-U{5oOJ67{j=@f347_E`H311)F3I(x7;554s{t6)U$kqex&d7 z;fU`IYor7cg(XR^=8T@|nW&B0bkEL4dqa!G3i@UJtE~m(8VTSo#Fjp!F&5P_MBc&H zq!W=%=mxWtwc!D+V5d~5y47HJgReJtp$CMnmheyHqFP6i0me&RebW z9{Kwbf||h2i=$dN*=PFu@gM(OJWk|Z!)wR8!TYTqY&VK$D`BV)MU?(hBu8y51}U&T z92BS}x?#m~wx3}9xsFTf^RD&x#ZhgK)i4jD376Ht*S(K^_#wh9w#wMVoHMpxZlunH z-U?45#i>0!DOWFOW)mK$2J5d;kf;z65rj945xfc>iydpK zShFis5#?{5PAgs5O2y2_S^fGiSqL#vFH#n{k8@fWhxY1Tm#(#0a-(6-UU>}P%2^3^W?Z% z&d2)qLjKV5pk7w`x58@jUmt5vA0zlV zn!z!J*P9O8vCYrgpSL=8bBFrWK*{oi4aSB|7w3NRyS_~G4!_WGUDQ?kKTJgn^rodq zim~dyfI)f=pnb~*7ARD|W%2GZ0=S|5eS|-*<~{YCDDgP^2S_nSsymjF2=$(#qJDFO z*TURW|3*PGwCj7D7Ptb6l&nqSPb~q7BO-l-OkHzR&(j z%+`VD-a-T+gCn*!p1pc+dD6Mvci2R3Jfq51XJ*iBmz!GTuQnXG3x3PClQtd)AQhaK zu5>?PGH$!_eLqCIV~{Mca$hGk-~vT3>@uIa>bve6lug((ANZs5omp-CjQfQp`C+91 zv7nlrXKW=yq|S5O=YaGOI$vVBrN>4&#FAR@)>jP5*mUaZ)z|r;oMC_P#`u9#IlhR* ztTSYzqvg8xx#;MYep<#4OQeASqd|5wr00*RZ^&IPu)Xd`Z!Rn=Td=GzPBE3X_9(JV znT?A!SN?-{Nb1-dZ}-hPtws_&x>d(|`Xd)-X&Om1gvHWWMixz(r#3Eh zh==)u-Ry}I= z*5yjJC^Ry8izJoOz8hPD%RmYC*qh25DCGD(iIBBcbI>>lVIg4~?wjmb zOLDTF$0IcI0@F+HO8h9G-XhG*uLP4*h9x|y??i6vLSv0fC9{5(9!-415u4VC?r5X5 zH}v{naTNY5Xg|klr}+uDH}~plPVzas({9NyZW1|%_$0;;t#qJ$I3Q{aRAV6{BVz!o zaO^t4#G!BhhY+zij7n!GRkiL(k zw$@S67j+ICRuq-v$nry+qx%)JjQ#VH%kslkj==NCg0vw+OSCKq+H?eaW<{d-@J*DD z<_?ozV~uk1@3+s|_w+<=s~CmIX+&ox!#-SQoD2LTTsDP^DJYB#m@k`BmSx0J-_CMV zoez-ECQ_3|@9ARs2d{9tQ|4UK_+&0s>fd|YEj#LZch}j-5zbsC#Iw7{N_5k^oGiEf z$n|abi6$W5Ob-|koH{?Z<(|mFj!Q>;(wM+W3YO-vz&A*e&i1LV0pB93JxGHFud%f( z(x!c6|HM~8;JAY8LvvJuO$ftA{486GU^k!DntmyZ|LQ4`v@NxuWaESyO*DgXkadCV z+pOB-BeaR_i3lGwV71P?ti8FwHnIvUJ$doxV>&w&_d> z#Q8i$cKLF=0iZzQ+FZ3lji&=|Y+DB)YLuFM{2%OX)B~|GPL~Ua5Z{pc;C-)rzw4Hfpg`>| z`o}NHsO59_oo1g!7UqwQyiDtJ*_w_-G*r@jo8)tQHcIEI0)T zj@6^y{*>W;wxU{Y5#S%8GS|`gfMtDK96zT^ImW*p_@K&8$2x~r=vRjo7|0 z=qw!c)E^2L*Zy3p@12df^KeP&U|rjPi>A9*Uglyo?VMbr#v82${_~S)ABQ~s6_M_J zX?MZuUU#s1mrHJZvc3;@-Df_!`*CsQeWZY8I<)B^b-JRjNd8#ls~sS1ymv5MGv*$L zl-3zIeZ~$c$1@l5!M4f-dQN72JH5$F5ZI*RKpSKyH{*XUaCh%_=H)`j**x`qv_IG2 zjLKkWY%9$DU*HxY+B>~z=#dI4=j#Vr#s}l4GN8Oy>e1p9hIe?ZPWd$5;&c1_)}RA; z8f5fWY$zua`%rmxmI!5q{c=ZNct1cLmCtS(w2GzNA;V_fAz8QW;biHsRto91?e%$YNF0!KJp>S*&B8LBN)sR$N zma_vqsRmRPGZmW`H|I#@3fbR(oLvCvHC~0+JMYoyI>A>4N=i!zg_K`(6 zM+d>(s%Kg6?tagFil{VjKs`l16`LO>=lUc8O--S3z8&XeUOl(k<@@}r&bkHaVB~bV zw`U2tP~v`WgNWaS)^Fzvk?!V<^l%ww}k#4f67xLPkWhuf5 zfvqm=yLg$m2_pHH$Pw041<@QI+h?oHydvnih&~+88?HxmJ=vFfPR4}lBIj}(_dEZ( zGy&Q+QMYwyGvBjumPlyumj~KUHkWIh5GYmuRWz&e%gY`U!PWR9^qQ!a0j@nVRH8Qh z-Nn($aQ_x@9UZ4G;^6bXe%UF*kU92Ab;JSzPRLYL2Z`W4_I1x^T{xc?hoXItNHF6CxFA_P(}8Q zIubnQkEe>JKEt2-m(o4oR32b5FCU`{zHaJ!bvYU7eo7Y7V{E!rN?!?CoyX+&3+W`$ zbHO38$%L0>N|GGwK(E4FdD$uM zldnI&Dt+<|*Tmj1c34rMF$M=i|5t=hgiBXBa$KOgoRwzYba{gepY5Qc*ln!iOEJk{ z<9-6QlMFdn(6={#C`|Zbw8AFBaF122ryZK@mHgGKD8>I*pR-J`rcWB(l^H)3nFmSUNI6$ausO!R3a8O~e)6cl+2-?~b_8 zq%mNSDSq*2ta4Ki@v92XlT_foX3MlG%;C+0VyS^>&yJKYWl2D66ZgQ!yp6y(a>Lo7q9zLpsMexn6Ci@F4nbv~zcUigLhKOnY+%1VwK=A!Gra68s zMo`(f_C9t$5*Bbz+YT;azw#5+An|N|g%tMY<6ikrw_OL{AH_QvtA#qIgQS)v)Q3Nz zb_rM1wQi3P1(N-oF4l#nT;C$pv#PzWe_=X9r0|UNA$WsP zQE3|_AR^uDe4Rj*!JgUyWf~CG?_t12vi)+_uoi#sN>~#28~OhQ*7Whpn&^Lr3ym=m zr&bfE^uLfu*lDlnj3Gxt6E1V(*QE@0qZKyuO9lMc*u)0r6>OY^ME+9DoSfON&WyBA zJ*RHUJ@+(=W|C24Tc~|%DhAl*w@(Dr1J%{6dJOmf=p^piMgJcRNsW^(8vJcUG-XJG zRKX*SUDkKEASfzG;(NG#mny-i;gPGL{efMHP037jQXM5AXu`3fMnJ{%5vfZkPN150 z3;o7oVWx0`xFwB9l;GZ#M{CD7y-pKhI+u=Ab*7^Mr)-isXOTuCg?scbw8K0sQm3Qu zOxu@5t$Y<#nk01B_i;oBRHvZc@09}8hATzw>9Q4^m~N_`8<>4_o(~E?ugY)a*Vk0` zji7*R-Wpd>qY2@bx|1Q)O9M~7jxtqFDQo~|afz80x@P*<*IaHS*cMtQu!!iZkZ0sA zN2+v&A8jA24evY9PRzhu|oZ3Hx%;9s{ z7xgim;Uqe(iGvHvHTsXA$B%>tT;{~?)CX*$9Ox~glVZb8#)*-%Am2j_NFQ)<1gGsH zKDa`TIT@&Q>%ii2JWQCAZj+34|R+pc|mTj-x#a%d6sL@WVDWJ*B&Pe1KX;_{Qp7c$ zJMIm_-zt13AKxDMKV-m3RVADvKGiCB-X=aJex=?N z@W^uKz49<+bb(|1dFT6%7VhdzTM16~Sr09;*Q=C8m##STE^R9pmi(>;^I@fjrIPQR zf2Q(Q2{52@eWu~KYa*E?eV;Y{;nYJvrIkc~!*A-@)&!=m8P+U;eHSZJY5Tss(4u-P z3Rb_EL1$!pKmsqy9S^&AxZ0D_uDzqxP0lh}V{BrF95-(E@GLKD`FF#wg#G!Eu;4^T zO_=%Kz09l6v`)9~Z~h+hZ&;OOXs;nKmG*p7a{%}C&7!P(`90PPD?QZxX`VyWXPpn7 zd(fVMH!XVA>#PVQd)nT0imlH~XP)U=nK^SfKa~%0#p^}g{Y4NSqiNq;$TYuDM}%zu z-c)OWXqtxs4cu5YJu$+%26UDG7|h+>vKP(qX!Ea@=XF*mh=TIa1nK53tg~Fw3OneiIM*Ur9dV*u!mc0A z_xTtLwjWk5Z&04R2za)Rzps%nZTvYwE2l1|E;G|`rqnt(M*4HMUS(&jwHjkWwa4g~ z%CD*lwtfakpOSKmW7OA3-52M-GDc}_2dM#?GtA97%oOC)FHNx#51R}HYI0Mpa=QD z1gGh-yA@4{Y!YRnQTQagCxn1= zmIdKEI?Gr$+qFFNxABDEH0c)X%8~6UIP=+)N?P3GIc@_DP*58#i^1{YOf|KoVSz(7^}+clnsjPKT-Ln6 zI|F!)kk_RNRr?4|H>0pgdZ3OfxbEkoKZQ%ESc*~&%y6+4A-!~P2h4idd4YY+j zMb2S+OcIb@9O~fhh)`hC{Z$EeLK*N)zqsVAnJEV7R1zIHM3Czr@4=jRfmnIwJR=9l zx;Yytk!~JvQGHGD_k&ehJny1`Q}PP8o}Qkszdtw|@t3qJ7#QG)4LAie*g2uVAeuY1 zx4T;b=)+Aa;AY-&{Ppam0lxuin}Lh{!I&MmAeiAIP9gYWm;q+GN-!|32F|oUcfVxn zwRqa{r0AA_uhriD+Yp_=G$8|UHPSuJB3ZyfLbn8V7U0kPiTpwyOUZ4s)t&d3^ zw9RW~d5m`%&rwk?zyRR`L{A(AJR&0Oh&A03gs)Vsy+uH<1K#?j))xY&kOy4r`x4aCalwIt#ukq|qm4A? z86Uw}K>NTIF+PTMUOE@d78LfV{*Uut|7V_HmdjY;PAN3UQtouLQ|&4;NMAVWT$zN) zj1w_RKUx}=I{MRAd?zhN*J$(7_wuIulg4qep!N>S$@DTagV(a{)ug9LFa~nIU)9w% z$#R5OlH~MGbD3o<}Uz&2YY@(0V%tQPFm8r;u9hy6RVUw69KODb$R~c z>elM$4{s$a$H{*w{n8n=ADe+B$EB5cxck)a*zsTVAW^zBfCH}=(LuPS2XK#XW;67> z6?kdX0X>C$!K#{7m zVE26r`(B>bXK51EKcd%k)-eY~(Cf*O{rplhR|^|{hz=ptR~Uj&qM|wR>8?iDCJ!o& zD-8D$xmh3hz+;L9dOZ}|!*5-46k}q4*-!gWDJT}TL9CWt5^1JBkn@TO-_vZCNk~1r zWXMgqnznal4aB21XpZQ&R>i3T>I|vc8H2AXs;qmlUy`uu5&3!K2w+Um8mnJ#qivJzZzn{<`9-zs8Az95Hk|w ze5*OPLC7;MKzHsHL`YcNfYY(T_v`m;Zd_`h(vH*p&=R5D2~*KLYfVy_TZ1m%LpF~9 zyLgJOW=YKiWx+3X>!hc}gs!2FlgF{pA?hZnKq$aAf5-22H4!qF?9sWqYjb>%$Vn^I zUz%4`!RQr}uOJ@Uz@HuNC6Kr?f_RZJj_>9rt)rU}@>tj`f9eT|*-9!+-GplQ2W|yO zvS8%3VLFRcxYP$*^^yy&okgi82QNAu#iLV4PNG4NCDHY3bhAbC z-xY*NZuSWy{fe>;y$)tXaD(PzwUC2gYL6tsf@&uY^>@V9?Y(n@^Z!wd-$Ac`)|{&(!Q5cK6^#O|1%|c^-&;Q(a%? zn(;LY;kh)5WLzTNO-_|X@k}sJzrrZ)cPW`tY{P7=jR;v5+^3(!^MLl)^#`!+>F|q^ zAABzDI%`uISj%OJ8&k>(L*44{bmQ}sPq+-dUR}7eH*YsXdig@(F#Bw*Y6i{Yuze}> zW#EXKdAE@qcPVZGdmWP9&R}`}+HswThyJpBqtg|kFx*$KzR?avIBN7)UUTWK!(L_w zf&9}eypEqoGNsHeN~dXaRf@xOOBBst#WhAXJX1dlegG01<6go#|6izC+-(+`I$`N^Os@1bqMq!lXd1;X} zafkl1ORyzxC&)~G?7{eK$1Z#Revq(VlT}kz>niOY{7D;>dX~jPd#-E<9a27f47aP= z=9TbByJ{8+`S|<2YC}^(PU9cp-i6(gc<<2hV!u*)NobA5NIZLB1?}Y6FLB=OUQXAo z8D<@}v-|6M#@8Cb&6l!TER-7Ww^`q67eX&4wwYzed(@B3ouitG7kF>#j+)qNOtQUF zXH#?MEyQdHE_$lKz$*1MzbYg;Q>wJJt;c}a3pXOU+fJ5xr^erxxWX5YRo;nhHFU7u zu`Ch4jE~FI(o4yY(|n3km~zwspLI4n>GfN2k1Bj+9g=@$1ov2ihm5t5^PsnCTI z(X5$#r3pC?KK)D%BV%i)$>_+$GH%sY&X;O^4=6O0d-3`$Ug8X1S5kt>5S@*eOy=f( zk<1!FDStFQ)$*20EGI!c|-&~x)0&V#KZY}KI4ka*?Ev(O>-un+wy5Ef{PNrh*i+L%~PTfKNe|8EK| zL4#*-3e>RKKxW^N8-Mm^9Ix8w+FJ6l!*zjK3Xf~10Xw`F-u34@T_PKwf(3qAOi!aS zA$NwW%Q}^dX+QiC7LhP=#H#bJp*45BCQZ3X+-8%bta}A{nqP&Ry<6)c?Q8V|P(WfE zNcFqzY^*&)o?ONIt^z!4$7-Za&hVS1T0Q0`tDrk+E5F(aMj%$huZYwF>YMK)Q!VTg zgzoNCs;~Q9KOR%?f0)Vmd4O$Zlu2Qy(u?DsYnbMWf4Hv|RNEZ%`0lYP zx(z-0(7KDq<>_8MeTUvXsl)jG)CGH{ue3%B!vrU9C54>`9d@y5fxHL5z?R_lR9U9B z=lcT36Jghjet`lNTqt#d8^okVE38LVb!wE!MsZ^C**N7I(ed*J4WguR#k>~fM_AjN z*XO98XK!t`bzdB8E0*2wWc8@h^+JD|S?`j@>`tAF3NkVVJfz_B*Cn~fHWmfsKvV^O z$(Jk^m<_pvOuWG}BAzX6r&`$Tux2yA|D+LjKItV$lGTtjK1|7|cLlGicvt-}q7_A~ z+D?|5wqvL2bIopJ_F%#aIUQs8!2+>yjIORVFH|=S=3|O{42Q`N)~%FLSw))j3hx=^ zz3iyW2bbvaoEmdN60%~yXN}t`RB56FDFwfu+F>zF#3Yg{V-Dd`Z_52ppwiShz^7;h zU#~JxlW;1NeV=8^5M$1*$(q&x%zf{~W(ou|^AAoHZ0dQ)wJMhNA7W=T^kkEg**&Hm zvaIUn<)s!1`4KB9E_C;T`XhQ~({t1}_$XeQjfuSEBi_8sQ(GahLx2mDc327hWLvvZ z#XG=IcVRBCoujhO!)_IE(&#nxl5?$m&Rt4cO)55W&Gz@gN=E=$QR~!)?@*Mnmo}wa z6Mi+cw|c^7BY#jao7|I@gZHGu;dfd};JhToXnxadH1%&^5sgc7c@w?#~~bD^+@4wF&8hu{Bw&khrOJwfe<#0 zbTFx1(tg5j4saB^EkQH2Q;vLHk~2%C#>@|JwSO=h<`m#+U&l(v9P@oQk2VT1q^OxO z6by-+kysx~t<=JU=TI*Rk+?jRQ@~zaXzw^ zVpW<*#nLQeE=NNd3i=h8X(8y&ZFAL{k5gl-{?XRs5Pp>;4~|6o4LkcmM1Hr~mprLl z&eOYnBF*b_ax--%1^)@T(XxS;_8OzPkkwNCDLSMvFJ)yVN)4oEFR0bx$gl!xx;km{ zHJO)Q7E3*w=zo&C)3E1n&&Pjr4te8|fS}TA5H})BQXMWlUN62~&2psq*3LT&m&UT+ z!lQmPW9RyO98#2w0H+HMu}U|rpyxiG{_PO07>Xodd+Gmqm5yrClWT{$#9vHjue(WB zvddDAY1iqz_nK#|;-^o?7$IG&$u4iE-~c4Ay>1)wox1?5ot#ZyHKJ`HDZ|^L#(Z}! zoVx2gd--ndI7+6+hSJ4->z!($O(0=Hi&=ktfny>NDly?PjGB z*WxR^fwklCQoX`rV!?3wz3U??BEA#I+zmL2OSjIWO@+Kq+*Z3FaXfkCDD7;NiJPfR zzQ?ZNYPBc_UzNy~WHzocHp(Ke6{%KbvSz0k(Z4ah#-w!QL4RL4;pNKdL3s5>bFX(h zslU;0L*q4Wjvy89YwK8)UEVzU-t4mN7^F}BR3djR65J4MiSzfD;jTHUgM#xF7tl`l zw@1_ED*-Z7yk~vpk@eYDqA^GPp6a7sS89cW+uZ`}B|}DxG%n+DnvP>`;K+w9Mo~xV z+>Xvlwudi2np3#d%LkIiY6fFyL*!B56uy8%L8M6a*sbjId$JEkf95AaOrgx3xeYNf zktI(M??(#8li0WK7^dyd(c{|X*B&z%?FEi5*Q#W@{qb`2iB*kHQ*2vbU^@vn%&wo4 z`a@DEY3F2`#_&+{?F*gbxo{BSLxUM=e8F$j^U*@qRP|9GQ+`ZtIj!RlGfdnCxsU0k z)!s4n(-fUO5Mrs+Sh;9C4?dm05!{g|c|tBY_oq3OmR9$~k3Zd}_d!L-)$hmoeO`X} z%OOPGIi$wy@0#$^6+&b6qYOyh*RmKS#6<~$Bq~y3K`13@nb5QpJ6F=C%AfRZ?;kF% zU)?1{l$`nqLRJM)w_;TDkLwzXrzeggixlRj@88r_cn)_wYG=3JS&-_Ln3#tg>_lcL z@_+=b+N_>G%>Kxh>?Tp~G3Tua<+4BA`epqDvP$zo5xPt~7v(uC45C&iPR z?AGNyNLhepMq@CSn^?XSp)?rv3Cn7M=EFnGNR{OdNzbWex|sUS#(17HHNwhrV^KxH zhha)0y_^R*-E4j+OH9mlO*I2vsoPD+xV6dP)U#$CkNqO*O6x@8O%@f_=7s#rDj6dR zl96gGWwzZwCrD?zMTOAAe0fiyvZ0Bpgl0|xtMO*k;6-nX(a$#Xa^x3qx+ZYx5fUt5 z`%CneR>OF_N?!Jaw70)8Yr}M14%mu#8d(u=&9J$VU1W83I~eP%Pd&VuD@s03_#ByK z8NG#VH)NPRY}c1W^5DvPVZm{SAezu!8WvAzWVFl9 zB(V3n0f*6O6nC^Dx&7(f?GKGK_ie2H0U8|{Ba*n!T!wXl7iD_}tPOZ^hDGVk#=2&$ zx-(I$s`80+9t>H#fg57B@dc<}1|xDD-IsHXGv%2!A9%84&)bpAEX0asd78?9vKt#_ zv2^iNrmV3+$jM}-z3|irO`~ixKVR3#$0rKNDYw|+*u}KO*fk6!<~-yg_T|?s{gGnb zP5fF}y<zqSW_Gpu%`cn4x|@@g)TGrc zzn@e5A<}gVrOK!aN&WGARIU@lTkDK*8;)uTNquxlBX*O^VVoyqbOhpI=K?v`2EC#F z6TZ_hC#mDZHJ`W~gcSz=j zhSBHdJ6ycAb7CM_ahpRnKEc!bs9}?qcvH$(h??fiMG(d&=ys_cwCYB6MR4+kon(cc zz@mriTfYto35$tSf_rEN*~h^HDmYBK;f^*qDJr^m#n)Bi4UuQ#)mCM+PcOSC2jx`@ zR1|vECU;`QOC}sl28SW;rrITU!N4z3MggmN}0&qt!*d{I!j~jsG0) z#~;l;-qIq&CV%6ZpBi>qqjpn0NidqGn58TYy~&&4C)sSjX*->{l;CIMJuhl_X=pks zpA=0>GG2G_FjkEwPmjpvC3%wbLSd(VZSCOjFo&CIQrJ$7##Q8CH-FKy8R=PZzE}$4 zRy$TOTY==_R<$?h&J{fJ*?LpIgT@tHUC+46s2i2}elG*oYw3H&2W|5&wTU4kv!CTR ztLrmt>w5C9U-Gl*4=6P)G#zhMzh^k@nbg}7yh|$Fr+j1!KdAtIdnfqFiMmQbz{H=i zsjx;9o_KtONu6#=IV!UtLmRc}?#v&bcuFmFTTLw@t+Yp_QAV~NW#Xn6XEVW^Z?y?2G86|TIWCMK3@YMJzZuVtg^q7y) zwr6A_E@|_617*9F(rY_qf@8rl^TR38!QutRFHvegJrlThjQcv$r~Mp)>_i-zO+jam z-v9TajQ^98<;;K8Uiuu2TYl1_ct8oSgfMasltOR(3LixAL@*7tYhxPSyx$B?XUdom z=_te?={EA&Bo1tIyUYAuUa=Z&c; zUECyX1KI#W_!b0FP|2TDZ*s)#&CYqne930oiRZ%aq4mI0)71SD|8}kX5FEOrRAdsI zEj_{XGG2iZz1)$v{zANG>YPkIT=NIKaB#e^qsyeq{#%))^Zuu^1|+pi=QBp5j`-*0 zLi4nl^yrO79e+-&kJ=wV3BT%CS-XMjNy?3bq8#-jxIaNaWx&KQae;?>{cw5r^b5Lv zKiEG7kB>~S&0^RSy8oL`a%+&`6zW5Wc-25%7)(0>+t$otV5K$$9QsFRD8c6TGI*v8 z%8)J`@xaCP;uYt literal 0 HcmV?d00001 diff --git a/docs/src/assets/images/transformations.dot.svg b/docs/src/assets/images/transformations.dot.svg new file mode 100644 index 000000000..1e98f612d --- /dev/null +++ b/docs/src/assets/images/transformations.dot.svg @@ -0,0 +1,124 @@ + + + + + + +%3 + + + +tilde_node + +x ~ Normal() + + + +base_node + + vn = +@varname +(x) +dist = Normal() +x, vi = ... + + + +tilde_node->base_node + + +   +@model + + + +assume + +assume(vn, dist, vi) + + + +base_node->assume + + +  tilde-pipeline + + + +iflinked + + +if + istrans(vi, vn) + + + +assume->iflinked + + + + + +without_linking + +f = from_internal_transform(vi, vn, dist) + + + +iflinked->without_linking + + +   +false + + + +with_linking + +f = from_linked_internal_transform(vi, vn, dist) + + + +iflinked->with_linking + + +   +true + + + +with_logabsdetjac + +x, logjac = with_logabsdet_jacobian(f, getindex_internal(vi, vn, dist)) + + + +without_linking->with_logabsdetjac + + + + + +with_linking->with_logabsdetjac + + + + + +return + + +return + x, logpdf(dist, x) - logjac, vi + + + +with_logabsdetjac->return + + + + + diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 7ee800556..45134a593 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -316,7 +316,7 @@ While if `dist` is not provided, we have: Notice that `dist` is not present here, but otherwise the diagrams are the same. !!! warning - This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occcurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. + This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. ## Other functionalities From 95dc8e37a1195ba87e900c957e6d9e578d7015d8 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 17:51:34 +0000 Subject: [PATCH 134/209] temporarily removed `VarNameVector` completely --- docs/make.jl | 2 +- docs/src/internals/varinfo.md | 309 -------------- src/DynamicPPL.jl | 2 - src/threadsafe.jl | 2 - src/varinfo.jl | 216 ---------- src/varnamevector.jl | 756 ---------------------------------- test/runtests.jl | 1 - test/test_util.jl | 1 - test/varinfo.jl | 6 - test/varnamevector.jl | 472 --------------------- 10 files changed, 1 insertion(+), 1766 deletions(-) delete mode 100644 docs/src/internals/varinfo.md delete mode 100644 src/varnamevector.jl delete mode 100644 test/varnamevector.jl diff --git a/docs/make.jl b/docs/make.jl index de557ef9f..65d43d524 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,7 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], - "Internals" => ["internals/varinfo.md", "internals/transformations.md"], + "Internals" => ["internals/transformations.md"], ], checkdocs=:exports, doctest=false, diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md deleted file mode 100644 index 8d74fd2fa..000000000 --- a/docs/src/internals/varinfo.md +++ /dev/null @@ -1,309 +0,0 @@ -# Design of `VarInfo` - -[`VarInfo`](@ref) is a fairly simple structure. - -```@docs; canonical=false -VarInfo -``` - -It contains - - - a `logp` field for accumulation of the log-density evaluation, and - - a `metadata` field for storing information about the realizations of the different variables. - -Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. - -**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding to a variable `@varname(x)`. - -!!! note - - We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. - -To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo`, and hence the underlying `metadata`, to replicate the following functionality of `Dict`: - - - `keys(::Dict)`: return all the `VarName`s present in `metadata`. - - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. - - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. - - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. - - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. - - `empty!(::Dict)`: delete all realizations in `metadata`. - - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. - -*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: - - - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. - - + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. - - - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - - `similar(::Vector{<:Real})`: return a new instance with the same `eltype` as the input. - -We also want some additional methods that are *not* part of the `Dict` or `Vector` interface: - - - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. - - - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. - -In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - - - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. - - - `getindex_internal` and `setindex_internal!` for extracting and mutating the internal representaton of a particular `VarName`. - -Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: - - 1. Type-stable when possible, but functional when not. - 2. Efficient storage and iteration when possible, but functional when not. - -The "but functional when not" is important as we want to support arbitrary models, which means that we can't always have these performance properties. - -In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). - -## Type-stability - -Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. - -Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form - -```@example varinfo-design -using DynamicPPL, Distributions, FillArrays - -@model function demo() - x ~ product_distribution(Fill(Bernoulli(0.5), 2)) - y ~ Normal(0, 1) - return nothing -end -``` - -then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where - - - `Vx` is a container with `eltype` `Bool`, and - - `Vy` is a container with `eltype` `Float64`. - -Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. - -For example, with the model above we have - -```@example varinfo-design -# Type-unstable `VarInfo` -varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) -typeof(varinfo_untyped.metadata) -``` - -```@example varinfo-design -# Type-stable `VarInfo` -varinfo_typed = DynamicPPL.typed_varinfo(demo()) -typeof(varinfo_typed.metadata) -``` - -But they both work as expected but one results in concrete typing and the other does not: - -```@example varinfo-design -varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] -``` - -```@example varinfo-design -varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] -``` - -Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entries while the typed uses `Vector{Bool}`. This is because the untyped version needs the underlying container to be able to handle both the `Bool` for `x` and the `Float64` for `y`, while the typed version can use a `Vector{Bool}` for `x` and a `Vector{Float64}` for `y` due to its usage of `NamedTuple`. - -!!! warning - - Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. - - ```julia - x[1] ~ Bernoulli(0.5) - x[2] ~ Normal(0, 1) - ``` - - In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. - - In practice, rarely observe such mixing of types, therefore in DynamicPPL, and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. - -!!! warning - - Another downside with such a `NamedTuple` approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. - - For these scenarios it can be useful to fall back to "untyped" representations. - -Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. - -## Efficient storage and iteration - -Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): - -```@docs -DynamicPPL.VarNameVector -``` - -In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve the desirata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. - -This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: - - - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. - - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - - `transforms::Vector`: the transforms associated with each `VarName`. - -Mutating functions, e.g. `setindex!(vnv::VarNameVector, val, vn::VarName)`, are then treated according to the following rules: - - 1. If `vn` is not already present: add it to the end of `vnv.varnames`, add the `val` to the underlying `vnv.vals`, etc. - - 2. If `vn` is already present in `vnv`: - - 1. If `val` has the *same length* as the existing value for `vn`: replace existing value. - 2. If `val` has a *smaller length* than the existing value for `vn`: replace existing value and mark the remaining indices as "inactive" by increasing the entry in `vnv.num_inactive` field. - 3. If `val` has a *larger length* than the existing value for `vn`: expand the underlying `vnv.vals` to accommodate the new value, update all `VarName`s occuring after `vn`, and update the `vnv.ranges` to point to the new range for `vn`. - -This means that `VarNameVector` is allowed to grow as needed, while "shrinking" (i.e. insertion of smaller elements) is handled by simply marking the redundant indices as "inactive". This turns out to be efficient for use-cases that we are generally interested in. - -For example, we want to optimize code-paths which effectively boil down to inner-loop in the following example: - -```julia -# Construct a `VarInfo` with types inferred from `model`. -varinfo = VarInfo(model) - -# Repeatedly sample from `model`. -for _ in 1:num_samples - rand!(rng, model, varinfo) - - # Do something with `varinfo`. - # ... -end -``` - -There are typically a few scenarios where we encounter changing representation sizes of a random variable `x`: - - 1. We're working with a transformed version `x` which is represented in a lower-dimensional space, e.g. transforming a `x ~ LKJ(2, 1)` to unconstrained `y = f(x)` takes us from 2-by-2 `Matrix{Float64}` to a 1-length `Vector{Float64}`. - 2. `x` has a random size, e.g. in a mixture model with a prior on the number of components. Here the size of `x` can vary widly between every realization of the `Model`. - -In scenario (1), we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". - -In scenario (2), we end up increasing the allocated memory for the randomly sized `x`, eventually leading to a vector that is large enough to hold realizations without needing to reallocate. But this can still lead to unnecessary memory usage, which might be undesirable. Hence one has to make a decision regarding the trade-off between memory usage and performance for the use-case at hand. - -To help with this, we have the following functions: - -```@docs -DynamicPPL.has_inactive -DynamicPPL.num_inactive -DynamicPPL.num_allocated -DynamicPPL.is_contiguous -DynamicPPL.contiguify! -``` - -For example, one might encounter the following scenario: - -```@example varinfo-design -vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") - -for i in 1:5 - x = fill(true, rand(1:100)) - DynamicPPL.update!(vnv, @varname(x), x) - println( - "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", - ) -end -``` - -We can then insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion whenever the allocation grows too large to reduce overall memory usage: - -```@example varinfo-design -vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) -println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") - -for i in 1:5 - x = fill(true, rand(1:100)) - DynamicPPL.update!(vnv, @varname(x), x) - if DynamicPPL.num_allocated(vnv) > 10 - DynamicPPL.contiguify!(vnv) - end - println( - "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", - ) -end -``` - -This does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. However, this also ensures that the the underlying `Vector{T}` is contiguous, which is important for performance. Hence, if we're about to do a lot of work with the `VarNameVector` without insertions, etc., it can be worth it to do a sweep to ensure that the underlying `Vector{T}` is contiguous. - -!!! note - - Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. - -Continuing from the example from the previous section, we can use a `VarInfo` with a `VarNameVector` as the `metadata` field: - -```@example varinfo-design -# Type-unstable -varinfo_untyped_vnv = DynamicPPL.VectorVarInfo(varinfo_untyped) -varinfo_untyped_vnv[@varname(x)], varinfo_untyped_vnv[@varname(y)] -``` - -```@example varinfo-design -# Type-stable -varinfo_typed_vnv = DynamicPPL.VectorVarInfo(varinfo_typed) -varinfo_typed_vnv[@varname(x)], varinfo_typed_vnv[@varname(y)] -``` - -If we now try to `delete!` `@varname(x)` - -```@example varinfo-design -haskey(varinfo_untyped_vnv, @varname(x)) -``` - -```@example varinfo-design -DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) -``` - -```@example varinfo-design -# `delete!` -DynamicPPL.delete!(varinfo_untyped_vnv.metadata, @varname(x)) -DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) -``` - -```@example varinfo-design -haskey(varinfo_untyped_vnv, @varname(x)) -``` - -Or insert a differently-sized value for `@varname(x)` - -```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 1)) -varinfo_untyped_vnv[@varname(x)] -``` - -```@example varinfo-design -DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) -``` - -```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 4)) -varinfo_untyped_vnv[@varname(x)] -``` - -```@example varinfo-design -DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) -``` - -### Performance summary - -In the end, we have the following "rough" performance characteristics for `VarNameVector`: - -| Method | Is blazingly fast? | -|:---------------------------------------:|:--------------------------------------------------------------------------------------------:| -| `getindex` | ${\color{green} \checkmark}$ | -| `setindex!` | ${\color{green} \checkmark}$ | -| `push!` | ${\color{green} \checkmark}$ | -| `delete!` | ${\color{red} \times}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | -| `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | - -## Other methods - -```@docs -DynamicPPL.replace_values(::VarNameVector, vals::AbstractVector) -``` - -```@docs; canonical=false -DynamicPPL.values_as(::VarNameVector) -``` diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index ec0b086f7..c099a1fe7 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -44,7 +44,6 @@ export AbstractVarInfo, UntypedVarInfo, TypedVarInfo, SimpleVarInfo, - VarNameVector, push!!, empty!!, subset, @@ -159,7 +158,6 @@ include("sampler.jl") include("varname.jl") include("distribution_wrappers.jl") include("contexts.jl") -include("varnamevector.jl") include("abstract_varinfo.jl") include("threadsafe.jl") include("varinfo.jl") diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 648a8ef1c..dd0357547 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -55,8 +55,6 @@ function setlogp!!(vi::ThreadSafeVarInfoWithRef, logp) return ThreadSafeVarInfo(setlogp!!(vi.varinfo, logp), vi.logps) end -has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) = has_varnamevector(vi.varinfo) - function BangBang.push!!( vi::ThreadSafeVarInfo, vn::VarName, r, dist::Distribution, gidset::Set{Selector} ) diff --git a/src/varinfo.jl b/src/varinfo.jl index a12a25cc3..eda8213b1 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -101,7 +101,6 @@ struct VarInfo{Tmeta,Tlogp} <: AbstractVarInfo logp::Base.RefValue{Tlogp} num_produce::Base.RefValue{Int} end -const VectorVarInfo = VarInfo{<:VarNameVector} const UntypedVarInfo = VarInfo{<:Metadata} const TypedVarInfo = VarInfo{<:NamedTuple} const VarInfoOrThreadSafeVarInfo{Tmeta} = Union{ @@ -132,46 +131,6 @@ function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) ) end -# No-op if we're already working with a `VarNameVector`. -metadata_to_varnamevector(vnv::VarNameVector) = vnv -function metadata_to_varnamevector(md::Metadata) - idcs = copy(md.idcs) - vns = copy(md.vns) - ranges = copy(md.ranges) - vals = copy(md.vals) - transforms = map(md.dists) do dist - # TODO: Handle linked distributions. - from_vec_transform(dist) - end - - return VarNameVector( - OrderedDict{eltype(keys(idcs)),Int}(idcs), vns, ranges, vals, transforms - ) -end - -function VectorVarInfo(vi::UntypedVarInfo) - md = metadata_to_varnamevector(vi.metadata) - lp = getlogp(vi) - return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) -end - -function VectorVarInfo(vi::TypedVarInfo) - md = map(metadata_to_varnamevector, vi.metadata) - lp = getlogp(vi) - return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) -end - -""" - has_varnamevector(varinfo::VarInfo) - -Returns `true` if `varinfo` uses `VarNameVector` as metadata. -""" -has_varnamevector(vi) = false -function has_varnamevector(vi::VarInfo) - return vi.metadata isa VarNameVector || - (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNameVector), values(vi.metadata))) -end - function untyped_varinfo( rng::Random.AbstractRNG, model::Model, @@ -373,10 +332,6 @@ function _merge(varinfo_left::VarInfo, varinfo_right::VarInfo) ) end -function merge_metadata(vnv_left::VarNameVector, vnv_right::VarNameVector) - return merge(vnv_left, vnv_right) -end - @generated function merge_metadata( metadata_left::NamedTuple{names_left}, metadata_right::NamedTuple{names_right} ) where {names_left,names_right} @@ -580,8 +535,6 @@ Return the distribution from which `vn` was sampled in `vi`. """ getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] -# HACK: we shouldn't need this -getdist(::VarNameVector, ::VarName) = nothing """ getindex_internal(vi::VarInfo, vn::VarName) @@ -592,8 +545,6 @@ The values may or may not be transformed to Euclidean space. """ getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) -# HACK: We shouldn't need this -getindex_internal(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) """ setval!(vi::VarInfo, val, vn::VarName) @@ -609,9 +560,6 @@ end function setval!(md::Metadata, val, vn::VarName) return md.vals[getrange(md, vn)] = vectorize(getdist(md, vn), val) end -function setval!(vnv::VarNameVector, val, vn::VarName) - return setindex_raw!(vnv, tovec(val), vn) -end """ getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) @@ -639,7 +587,6 @@ function getall(md::Metadata) Base.Fix1(getindex_internal, md), vcat, md.vns; init=similar(md.vals, 0) ) end -getall(vnv::VarNameVector) = vnv.vals """ setall!(vi::VarInfo, val) @@ -655,12 +602,6 @@ function _setall!(metadata::Metadata, val) metadata.vals[r] .= val[r] end end -function _setall!(vnv::VarNameVector, val) - # TODO: Do something more efficient here. - for i in 1:length(vnv) - vnv[i] = val[i] - end -end @generated function _setall!(metadata::NamedTuple{names}, val) where {names} expr = Expr(:block) start = :(1) @@ -693,10 +634,6 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) return metadata end -function settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) - settrans!(vnv, trans, vn) - return vnv -end function settrans!!(vi::VarInfo, trans::Bool) for vn in keys(vi) @@ -1044,8 +981,6 @@ end # X -> R for all variables associated with given sampler function link!!(t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model) - # If we're working with a `VarNameVector`, we always use immutable. - has_varnamevector(vi) && return link(t, vi, spl, model) # Call `_link!` instead of `link!` to avoid deprecation warning. _link!(vi, spl) return vi @@ -1141,8 +1076,6 @@ end function invlink!!( t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model ) - # If we're working with a `VarNameVector`, we always use immutable. - has_varnamevector(vi) && return invlink(t, vi, spl, model) # Call `_invlink!` instead of `invlink!` to avoid deprecation warning. _invlink!(vi, spl) return vi @@ -1377,66 +1310,6 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar ) end -function _link_metadata!( - model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns -) - # HACK: We ignore `target_vns` here. - vns = keys(metadata) - # Need to extract the priors from the model. - dists = extract_priors(model, varinfo) - - is_transformed = copy(metadata.is_transformed) - - # Construct the linking transformations. - link_transforms = map(vns) do vn - # If `vn` is not part of `target_vns`, the `identity` transformation is used. - if (target_vns !== nothing && vn ∉ target_vns) - return identity - end - - # Otherwise, we derive the transformation from the distribution. - is_transformed[getidx(metadata, vn)] = true - internal_to_linked_internal_transform(varinfo, vn, dists[vn]) - end - # Compute the transformed values. - ys = map(vns, link_transforms) do vn, f - # TODO: Do we need to handle scenarios where `vn` is not in `dists`? - dist = dists[vn] - x = getindex_internal(metadata, vn) - y, logjac = with_logabsdet_jacobian(f, x) - # Accumulate the log-abs-det jacobian correction. - acclogp!!(varinfo, -logjac) - # Return the transformed value. - return y - end - # Extract the from-vec transformations. - fromvec_transforms = map(from_vec_transform, ys) - # Compose the transformations to form a full transformation from - # unconstrained vector representation to constrained space. - transforms = map(∘, map(inverse, link_transforms), fromvec_transforms) - # Convert to vector representation. - yvecs = map(tovec, ys) - - # Determine new ranges. - ranges_new = similar(metadata.ranges) - offset = 0 - for (i, v) in enumerate(yvecs) - r_start, r_end = offset + 1, length(v) + offset - offset = r_end - ranges_new[i] = r_start:r_end - end - - # Now we just create a new metadata with the new `vals` and `ranges`. - return VarNameVector( - metadata.varname_to_index, - metadata.varnames, - ranges_new, - reduce(vcat, yvecs), - transforms, - is_transformed, - ) -end - function invlink( ::DynamicTransformation, varinfo::VarInfo, spl::AbstractSampler, model::Model ) @@ -1537,55 +1410,6 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe ) end -function _invlink_metadata!( - model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns -) - # HACK: We ignore `target_vns` here. - # TODO: Make use of `update!` to aovid copying values. - # => Only need to allocate for transformations. - - vns = keys(metadata) - is_transformed = copy(metadata.is_transformed) - - # Compute the transformed values. - xs = map(vns) do vn - f = gettransform(metadata, vn) - y = getindex_internal(metadata, vn) - # No need to use `with_reconstruct` as `f` will include this. - x, logjac = with_logabsdet_jacobian(f, y) - # Accumulate the log-abs-det jacobian correction. - acclogp!!(varinfo, -logjac) - # Mark as no longer transformed. - is_transformed[getidx(metadata, vn)] = false - # Return the transformed value. - return x - end - # Compose the transformations to form a full transformation from - # unconstrained vector representation to constrained space. - transforms = map(from_vec_transform, xs) - # Convert to vector representation. - xvecs = map(tovec, xs) - - # Determine new ranges. - ranges_new = similar(metadata.ranges) - offset = 0 - for (i, v) in enumerate(xvecs) - r_start, r_end = offset + 1, length(v) + offset - offset = r_end - ranges_new[i] = r_start:r_end - end - - # Now we just create a new metadata with the new `vals` and `ranges`. - return VarNameVector( - metadata.varname_to_index, - metadata.varnames, - ranges_new, - reduce(vcat, xvecs), - transforms, - is_transformed, - ) -end - """ islinked(vi::VarInfo, spl::Union{Sampler, SampleFromPrior}) @@ -1655,14 +1479,6 @@ function getindex(vi::VarInfo, vn::VarName, dist::Distribution) val = getindex_internal(vi, vn) return from_maybe_linked_internal(vi, vn, dist, val) end -# HACK: Allows us to also work with `VarNameVector` where `dist` is not used, -# but we instead use a transformation stored with the variable. -function getindex(vi::VarInfo, vn::VarName, ::Nothing) - if !haskey(vi, vn) - throw(KeyError(vn)) - end - return getmetadata(vi, vn)[vn] -end function getindex(vi::VarInfo, vns::Vector{<:VarName}) vals_linked = mapreduce(vcat, vns) do vn @@ -1680,15 +1496,6 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) end getindex_raw(vi::VarInfo, vn::VarName) = getindex_raw(vi, vn, getdist(vi, vn)) -function getindex_raw(vi::VarInfo, vn::VarName, ::Nothing) - # FIXME: This is too hacky. - # We know this will only be hit if we're working with `VarNameVector`, - # so we can just use the `getindex_raw` for `VarNameVector`. - # NOTE: This won't result in the same behavior as `getindex_raw` - # for the other `VarInfo`s since we don't have access to the `dist` - # and so can't call `reconstruct`. - return getindex_raw(getmetadata(vi, vn), vn) -end function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) f = from_internal_transform(vi, vn, dist) return f(getindex_internal(vi, vn)) @@ -1876,11 +1683,6 @@ function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) return meta end -function Base.push!(vnv::VarNameVector, vn, r, dist, gidset, num_produce) - f = from_vec_transform(dist) - return push!(vnv, vn, r, f) -end - """ setorder!(vi::VarInfo, vn::VarName, index::Int) @@ -1895,7 +1697,6 @@ function setorder!(metadata::Metadata, vn::VarName, index::Int) metadata.orders[metadata.idcs[vn]] = index return metadata end -setorder!(vnv::VarNameVector, ::VarName, ::Int) = vnv """ getorder(vi::VarInfo, vn::VarName) @@ -1921,8 +1722,6 @@ end function is_flagged(metadata::Metadata, vn::VarName, flag::String) return metadata.flags[flag][getidx(metadata, vn)] end -# HACK: This is bad. Should we always return `true` here? -is_flagged(::VarNameVector, ::VarName, flag::String) = flag == "del" ? true : false """ unset_flag!(vi::VarInfo, vn::VarName, flag::String) @@ -1937,7 +1736,6 @@ function unset_flag!(metadata::Metadata, vn::VarName, flag::String) metadata.flags[flag][getidx(metadata, vn)] = false return metadata end -unset_flag!(vnv::VarNameVector, ::VarName, ::String) = vnv """ set_retained_vns_del_by_spl!(vi::VarInfo, spl::Sampler) @@ -2295,8 +2093,6 @@ function values_from_metadata(md::Metadata) ) end -values_from_metadata(md::VarNameVector) = pairs(md) - # Transforming from internal representation to distribution representation. # Without `dist` argument: base on `dist` extracted from self. function from_internal_transform(vi::VarInfo, vn::VarName) @@ -2305,17 +2101,11 @@ end function from_internal_transform(md::Metadata, vn::VarName) return from_internal_transform(md, vn, getdist(md, vn)) end -function from_internal_transform(md::VarNameVector, vn::VarName) - return gettransform(md, vn) -end # With both `vn` and `dist` arguments: base on provided `dist`. function from_internal_transform(vi::VarInfo, vn::VarName, dist) return from_internal_transform(getmetadata(vi, vn), vn, dist) end from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) -function from_internal_transform(::VarNameVector, ::VarName, dist) - return from_vec_transform(dist) -end # Without `dist` argument: base on `dist` extracted from self. function from_linked_internal_transform(vi::VarInfo, vn::VarName) @@ -2324,9 +2114,6 @@ end function from_linked_internal_transform(md::Metadata, vn::VarName) return from_linked_internal_transform(md, vn, getdist(md, vn)) end -function from_linked_internal_transform(md::VarNameVector, vn::VarName) - return gettransform(md, vn) -end # With both `vn` and `dist` arguments: base on provided `dist`. function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) # Dispatch to metadata in case this alters the behavior. @@ -2335,6 +2122,3 @@ end function from_linked_internal_transform(::Metadata, ::VarName, dist) return from_linked_vec_transform(dist) end -function from_linked_internal_transform(::VarNameVector, ::VarName, dist) - return from_linked_vec_transform(dist) -end diff --git a/src/varnamevector.jl b/src/varnamevector.jl deleted file mode 100644 index ed919a20c..000000000 --- a/src/varnamevector.jl +++ /dev/null @@ -1,756 +0,0 @@ -""" - VarNameVector - -A container that works like a `Vector` and an `OrderedDict` but is neither. - -# Fields -$(FIELDS) -""" -struct VarNameVector{ - K<:VarName,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData -} - "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" - varname_to_index::OrderedDict{K,Int} - - "vector of identifiers for the random variables, where `varnames[varname_to_index[vn]] == vn`" - varnames::TVN # AbstractVector{<:VarName} - - "vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" - ranges::Vector{UnitRange{Int}} - - "vector of values of all variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" - vals::TVal # AbstractVector{<:Real} - - "vector of transformations whose inverse takes us back to the original space" - transforms::TTrans - - "specifies whether a variable is transformed or not " - is_transformed::BitVector - - "additional entries which are considered inactive" - num_inactive::OrderedDict{Int,Int} - - "metadata associated with the varnames" - metadata::MData -end - -function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) - return vnv_left.varname_to_index == vnv_right.varname_to_index && - vnv_left.varnames == vnv_right.varnames && - vnv_left.ranges == vnv_right.ranges && - vnv_left.vals == vnv_right.vals && - vnv_left.transforms == vnv_right.transforms && - vnv_left.is_transformed == vnv_right.is_transformed && - vnv_left.num_inactive == vnv_right.num_inactive && - vnv_left.metadata == vnv_right.metadata -end - -function VarNameVector( - varname_to_index, - varnames, - ranges, - vals, - transforms, - is_transformed=fill!(BitVector(undef, length(varnames)), 0), -) - return VarNameVector( - varname_to_index, - varnames, - ranges, - vals, - transforms, - is_transformed, - OrderedDict{Int,Int}(), - nothing, - ) -end -# TODO: Do we need this? -function VarNameVector{K,V}() where {K,V} - return VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) -end - -istrans(vnv::VarNameVector, vn::VarName) = vnv.is_transformed[vnv.varname_to_index[vn]] -function settrans!(vnv::VarNameVector, val::Bool, vn::VarName) - return vnv.is_transformed[vnv.varname_to_index[vn]] = val -end - -VarNameVector() = VarNameVector{VarName,Real}() -VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) -VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) -VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) -function VarNameVector( - varnames::AbstractVector, vals::AbstractVector, transforms=map(from_vec_transform, vals) -) - # TODO: Check uniqueness of `varnames`? - - # Convert `vals` into a vector of vectors. - vals_vecs = map(tovec, vals) - - # TODO: Is this really the way to do this? - if !(eltype(varnames) <: VarName) - varnames = convert(Vector{VarName}, varnames) - end - varname_to_index = OrderedDict{eltype(varnames),Int}() - ranges = Vector{UnitRange{Int}}() - offset = 0 - for (i, (vn, x)) in enumerate(zip(varnames, vals_vecs)) - # Add the varname index. - push!(varname_to_index, vn => length(varname_to_index) + 1) - # Add the range. - r = (offset + 1):(offset + length(x)) - push!(ranges, r) - # Update the offset. - offset = r[end] - end - - return VarNameVector( - varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms - ) -end - -""" - replace_values(vnv::VarNameVector, vals::AbstractVector) - -Replace the values in `vnv` with `vals`. - -This is useful when we want to update the entire underlying vector of values -in one go or if we want to change the how the values are stored, e.g. alter the `eltype`. - -!!! warning - This replaces the raw underlying values, and so care should be taken when using this - function. For example, if `vnv` has any inactive entries, then the provided `vals` - should also contain the inactive entries to avoid unexpected behavior. - -# Example - -```jldoctest varnamevector-replace-values -julia> using DynamicPPL: VarNameVector, replace_values - -julia> vnv = VarNameVector(@varname(x) => [1.0]); - -julia> replace_values(vnv, [2.0])[@varname(x)] == [2.0] -true -``` - -This is also useful when we want to differentiate wrt. the values -using automatic differentiation, e.g. ForwardDiff.jl. - -```jldoctest varnamevector-replace-values -julia> using ForwardDiff: ForwardDiff - -julia> f(x) = sum(abs2, replace_values(vnv, x)[@varname(x)]) -f (generic function with 1 method) - -julia> ForwardDiff.gradient(f, [1.0]) -1-element Vector{Float64}: - 2.0 -``` -""" -replace_values(vnv::VarNameVector, vals) = Setfield.@set vnv.vals = vals - -# Some `VarNameVector` specific functions. -getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] - -getrange(vnv::VarNameVector, idx::Int) = vnv.ranges[idx] -getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) - -gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] - -""" - has_inactive(vnv::VarNameVector) - -Returns `true` if `vnv` has inactive ranges. -""" -has_inactive(vnv::VarNameVector) = !isempty(vnv.num_inactive) - -""" - num_inactive(vnv::VarNameVector) - -Return the number of inactive entries in `vnv`. -""" -num_inactive(vnv::VarNameVector) = sum(values(vnv.num_inactive)) - -""" - num_inactive(vnv::VarNameVector, vn::VarName) - -Returns the number of inactive entries for `vn` in `vnv`. -""" -num_inactive(vnv::VarNameVector, vn::VarName) = num_inactive(vnv, getidx(vnv, vn)) -num_inactive(vnv::VarNameVector, idx::Int) = get(vnv.num_inactive, idx, 0) - -""" - num_allocated(vnv::VarNameVector) - -Returns the number of allocated entries in `vnv`. -""" -num_allocated(vnv::VarNameVector) = length(vnv.vals) - -""" - num_allocated(vnv::VarNameVector, vn::VarName) - -Returns the number of allocated entries for `vn` in `vnv`. -""" -num_allocated(vnv::VarNameVector, vn::VarName) = num_allocated(vnv, getidx(vnv, vn)) -function num_allocated(vnv::VarNameVector, idx::Int) - return length(getrange(vnv, idx)) + num_inactive(vnv, idx) -end - -# Basic array interface. -Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) -Base.length(vnv::VarNameVector) = - if isempty(vnv.num_inactive) - length(vnv.vals) - else - sum(length, vnv.ranges) - end -Base.size(vnv::VarNameVector) = (length(vnv),) -Base.isempty(vnv::VarNameVector) = isempty(vnv.varnames) - -# TODO: We should probably remove this -Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() - -# Dictionary interface. -Base.keys(vnv::VarNameVector) = vnv.varnames -Base.values(vnv::VarNameVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) -Base.pairs(vnv::VarNameVector) = (vn => vnv[vn] for vn in keys(vnv)) - -Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) - -# `getindex` & `setindex!` -Base.getindex(vnv::VarNameVector, i::Int) = getindex_raw(vnv, i) -function Base.getindex(vnv::VarNameVector, vn::VarName) - x = getindex_raw(vnv, vn) - f = gettransform(vnv, vn) - return f(x) -end - -function find_range_from_sorted(ranges::AbstractVector{<:AbstractRange}, x) - # TODO: Assume `ranges` to be sorted and contiguous, and use `searchsortedfirst` - # for a more efficient approach. - range_idx = findfirst(Base.Fix1(∈, x), ranges) - - # If we're out of bounds, we raise an error. - if range_idx === nothing - throw(ArgumentError("Value $x is not in any of the ranges.")) - end - - return range_idx -end - -function adjusted_ranges(vnv::VarNameVector) - # Every range following inactive entries needs to be shifted. - offset = 0 - ranges_adj = similar(vnv.ranges) - for (idx, r) in enumerate(vnv.ranges) - # Remove the `offset` in `r` due to inactive entries. - ranges_adj[idx] = r .- offset - # Update `offset`. - offset += get(vnv.num_inactive, idx, 0) - end - - return ranges_adj -end - -function index_to_raw_index(vnv::VarNameVector, i::Int) - # If we don't have any inactive entries, there's nothing to do. - has_inactive(vnv) || return i - - # Get the adjusted ranges. - ranges_adj = adjusted_ranges(vnv) - # Determine the adjusted range that the index corresponds to. - r_idx = find_range_from_sorted(ranges_adj, i) - r = vnv.ranges[r_idx] - # Determine how much of the index `i` is used to get to this range. - i_used = r_idx == 1 ? 0 : sum(length, ranges_adj[1:(r_idx - 1)]) - # Use remainder to index into `r`. - i_remainder = i - i_used - return r[i_remainder] -end - -getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] -getindex_raw(vnv::VarNameVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] - -# `getindex` for `Colon` -function Base.getindex(vnv::VarNameVector, ::Colon) - return if has_inactive(vnv) - mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) - else - vnv.vals - end -end - -function getindex_raw(vnv::VarNameVector, ::Colon) - return if has_inactive(vnv) - mapreduce(Base.Fix1(getindex_raw, vnv.vals), vcat, vnv.ranges) - else - vnv.vals - end -end - -# HACK: remove this as soon as possible. -Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] - -Base.setindex!(vnv::VarNameVector, val, i::Int) = setindex_raw!(vnv, val, i) -function Base.setindex!(vnv::VarNameVector, val, vn::VarName) - f = inverse(gettransform(vnv, vn)) - return setindex_raw!(vnv, f(val), vn) -end - -setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] = val -function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) - return vnv.vals[getrange(vnv, vn)] = val -end - -# `empty!(!)` -function Base.empty!(vnv::VarNameVector) - # TODO: Or should the semantics be different, e.g. keeping `varnames`? - empty!(vnv.varname_to_index) - empty!(vnv.varnames) - empty!(vnv.ranges) - empty!(vnv.vals) - empty!(vnv.transforms) - empty!(vnv.num_inactive) - return nothing -end -BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) - -function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) - # Return early if possible. - isempty(left_vnv) && return deepcopy(right_vnv) - isempty(right_vnv) && return deepcopy(left_vnv) - - # Determine varnames. - vns_left = left_vnv.varnames - vns_right = right_vnv.varnames - vns_both = union(vns_left, vns_right) - - # Determine `eltype` of `vals`. - T_left = eltype(left_vnv.vals) - T_right = eltype(right_vnv.vals) - T = promote_type(T_left, T_right) - # TODO: Is this necessary? - if !(T <: Real) - T = Real - end - - # Determine `eltype` of `varnames`. - V_left = eltype(left_vnv.varnames) - V_right = eltype(right_vnv.varnames) - V = promote_type(V_left, V_right) - if !(V <: VarName) - V = VarName - end - - # Determine `eltype` of `transforms`. - F_left = eltype(left_vnv.transforms) - F_right = eltype(right_vnv.transforms) - F = promote_type(F_left, F_right) - - # Allocate. - varnames_to_index = OrderedDict{V,Int}() - ranges = UnitRange{Int}[] - vals = T[] - transforms = F[] - is_transformed = BitVector(undef, length(vns_both)) - - # Range offset. - offset = 0 - - for (idx, vn) in enumerate(vns_both) - # Extract the necessary information from `left` or `right`. - if vn in vns_left && !(vn in vns_right) - # `vn` is only in `left`. - varnames_to_index[vn] = idx - val = getindex_raw(left_vnv, vn) - n = length(val) - r = (offset + 1):(offset + n) - f = gettransform(left_vnv, vn) - is_transformed[idx] = istrans(left_vnv, vn) - else - # `vn` is either in both or just `right`. - varnames_to_index[vn] = idx - val = getindex_raw(right_vnv, vn) - n = length(val) - r = (offset + 1):(offset + n) - f = gettransform(right_vnv, vn) - is_transformed[idx] = istrans(right_vnv, vn) - end - # Update. - append!(vals, val) - push!(ranges, r) - push!(transforms, f) - # Increment `offset`. - offset += n - end - - return VarNameVector(varnames_to_index, vns_both, ranges, vals, transforms) -end - -function subset(vnv::VarNameVector, vns::AbstractVector{<:VarName}) - # NOTE: This does not specialize types when possible. - vnv_new = similar(vnv) - # Return early if possible. - isempty(vnv) && return vnv_new - - for vn in vns - push!(vnv_new, vn, getindex_internal(vnv, vn), gettransform(vnv, vn)) - end - - return vnv_new -end - -# `similar` -similar_metadata(::Nothing) = nothing -similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) -function Base.similar(vnv::VarNameVector) - # NOTE: Whether or not we should empty the underlying containers or note - # is somewhat ambiguous. For example, `similar(vnv.varname_to_index)` will - # result in an empty `AbstractDict`, while the vectors, e.g. `vnv.ranges`, - # will result in non-empty vectors but with entries as `undef`. But it's - # much easier to write the rest of the code assuming that `undef` is not - # present, and so for now we empty the underlying containers, thus differing - # from the behavior of `similar` for `AbstractArray`s. - return VarNameVector( - similar(vnv.varname_to_index), - similar(vnv.varnames, 0), - similar(vnv.ranges, 0), - similar(vnv.vals, 0), - similar(vnv.transforms, 0), - BitVector(), - similar(vnv.num_inactive), - similar_metadata(vnv.metadata), - ) -end - -""" - is_contiguous(vnv::VarNameVector) - -Returns `true` if the underlying data of `vnv` is stored in a contiguous array. - -This is equivalent to negating [`has_inactive(vnv)`](@ref). -""" -is_contiguous(vnv::VarNameVector) = !has_inactive(vnv) - -function nextrange(vnv::VarNameVector, x) - # If `vnv` is empty, return immediately. - isempty(vnv) && return 1:length(x) - - # The offset will be the last range's end + its number of inactive entries. - vn_last = vnv.varnames[end] - idx = getidx(vnv, vn_last) - offset = last(getrange(vnv, idx)) + num_inactive(vnv, idx) - - return (offset + 1):(offset + length(x)) -end - -# `push!` and `push!!`: add a variable to the varname vector. -function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) - # Error if we already have the variable. - haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) - # NOTE: We need to compute the `nextrange` BEFORE we start mutating - # the underlying; otherwise we might get some strange behaviors. - val_vec = tovec(val) - r_new = nextrange(vnv, val_vec) - vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 - push!(vnv.varnames, vn) - push!(vnv.ranges, r_new) - append!(vnv.vals, val_vec) - push!(vnv.transforms, transform) - push!(vnv.is_transformed, false) - return nothing -end - -""" - shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) - -Shifts the elements of `x` starting from index `start` by `n` to the right. -""" -function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) - x[(start + n):end] = x[start:(end - n)] - return x -end - -""" - shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) - -Shifts the ranges of variables in `vnv` starting from index `idx` by `n`. -""" -function shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) - for i in (idx + 1):length(vnv.ranges) - vnv.ranges[i] = vnv.ranges[i] .+ n - end - return nothing -end - -# `update!` and `update!!`: update a variable in the varname vector. -""" - update!(vnv::VarNameVector, vn::VarName, val[, transform]) - -Either add a new entry or update existing entry for `vn` in `vnv` with the value `val`. - -If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). -""" -function update!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) - if !haskey(vnv, vn) - # Here we just add a new entry. - return push!(vnv, vn, val, transform) - end - - # Here we update an existing entry. - val_vec = tovec(val) - idx = getidx(vnv, vn) - # Extract the old range. - r_old = getrange(vnv, idx) - start_old, end_old = first(r_old), last(r_old) - n_old = length(r_old) - # Compute the new range. - n_new = length(val_vec) - start_new = start_old - end_new = start_old + n_new - 1 - r_new = start_new:end_new - - #= - Suppose we currently have the following: - - | x | x | o | o | o | y | y | y | <- Current entries - - where 'O' denotes an inactive entry, and we're going to - update the variable `x` to be of size `k` instead of 2. - - We then have a few different scenarios: - 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. - E.g. if `k = 7`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | x | x | x | x | y | y | y | <- New entries - - 2. `k = 5`: All inactive entries become active. - Then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | x | x | y | y | y | <- New entries - - 3. `k < 5`: Some inactive entries become active, some remain inactive. - E.g. if `k = 3`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | o | o | y | y | y | <- New entries - - 4. `k = 2`: No inactive entries become active. - Then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | o | o | o | y | y | y | <- New entries - - 5. `k < 2`: More entries become inactive. - E.g. if `k = 1`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | o | o | o | o | y | y | y | <- New entries - =# - - # Compute the allocated space for `vn`. - had_inactive = haskey(vnv.num_inactive, idx) - n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old - - if n_new > n_allocated - # Then we need to grow the underlying vector. - n_extra = n_new - n_allocated - # Allocate. - resize!(vnv.vals, length(vnv.vals) + n_extra) - # Shift current values. - shift_right!(vnv.vals, end_old + 1, n_extra) - # No more inactive entries. - had_inactive && delete!(vnv.num_inactive, idx) - # Update the ranges for all variables after this one. - shift_subsequent_ranges_by!(vnv, idx, n_extra) - elseif n_new == n_allocated - # => No more inactive entries. - had_inactive && delete!(vnv.num_inactive, idx) - else - # `n_new < n_allocated` - # => Need to update the number of inactive entries. - vnv.num_inactive[idx] = n_allocated - n_new - end - - # Update the range for this variable. - vnv.ranges[idx] = r_new - # Update the value. - vnv.vals[r_new] = val_vec - # Update the transform. - vnv.transforms[idx] = transform - - # TODO: Should we maybe sweep over inactive ranges and re-contiguify - # if we the total number of inactive elements is "large" in some sense? - - return nothing -end - -function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) - offset = 0 - for i in 1:length(ranges) - r_old = ranges[i] - ranges[i] = (offset + 1):(offset + length(r_old)) - offset += length(r_old) - end - - return ranges -end - -""" - contiguify!(vnv::VarNameVector) - -Re-contiguify the underlying vector and shrink if possible. -""" -function contiguify!(vnv::VarNameVector) - # Extract the re-contiguified values. - # NOTE: We need to do this before we update the ranges. - vals = vnv[:] - # And then we re-contiguify the ranges. - recontiguify_ranges!(vnv.ranges) - # Clear the inactive ranges. - empty!(vnv.num_inactive) - # Now we update the values. - for (i, r) in enumerate(vnv.ranges) - vnv.vals[r] = vals[r] - end - # And (potentially) shrink the underlying vector. - resize!(vnv.vals, vnv.ranges[end][end]) - # The rest should be left as is. - return vnv -end - -""" - group_by_symbol(vnv::VarNameVector) - -Return a dictionary mapping symbols to `VarNameVector`s with -varnames containing that symbol. -""" -function group_by_symbol(vnv::VarNameVector) - # Group varnames in `vnv` by the symbol. - d = OrderedDict{Symbol,Vector{VarName}}() - for vn in vnv.varnames - push!(get!(d, getsym(vn), Vector{VarName}()), vn) - end - - # Create a `NamedTuple` from the grouped varnames. - nt_vals = map(values(d)) do varnames - # TODO: Do we need to specialize the inputs here? - VarNameVector( - map(identity, varnames), - map(Base.Fix1(getindex, vnv), varnames), - map(Base.Fix1(gettransform, vnv), varnames), - ) - end - - return OrderedDict(zip(keys(d), nt_vals)) -end - -""" - shift_index_left!(vnv::VarNameVector, idx::Int) - -Shift the index `idx` to the left by one and update the relevant fields. - -!!! warning - This does not check if index we're shifting to is already occupied. -""" -function shift_index_left!(vnv::VarNameVector, idx::Int) - # Shift the index in the lookup table. - vn = vnv.varnames[idx] - vnv.varname_to_index[vn] = idx - 1 - # Shift the index in the inactive ranges. - if haskey(vnv.num_inactive, idx) - # Done in increasing order => don't need to worry about - # potentially shifting the same index twice. - vnv.num_inactive[idx - 1] = pop!(vnv.num_inactive, idx) - end -end - -""" - shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) - -Shift the indices for all variables after `idx` to the left by one and update -the relevant fields. - -This just -""" -function shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) - # Shift the indices for all variables after `idx`. - for idx_to_shift in (idx + 1):length(vnv.varnames) - shift_index_left!(vnv, idx_to_shift) - end -end - -function Base.delete!(vnv::VarNameVector, vn::VarName) - # Error if we don't have the variable. - !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) - - # Get the index of the variable. - idx = getidx(vnv, vn) - - # Delete the values. - r_start = first(getrange(vnv, idx)) - n_allocated = num_allocated(vnv, idx) - # NOTE: `deleteat!` also results in a `resize!` so we don't need to do that. - deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) - - # Delete `vn` from the lookup table. - delete!(vnv.varname_to_index, vn) - - # Delete any inactive ranges corresponding to `vn`. - haskey(vnv.num_inactive, idx) && delete!(vnv.num_inactive, idx) - - # Re-adjust the indices for varnames occuring after `vn` so - # that they point to the correct indices after the deletions below. - shift_subsequent_indices_left!(vnv, idx) - - # Re-adjust the ranges for varnames occuring after `vn`. - shift_subsequent_ranges_by!(vnv, idx, -n_allocated) - - # Delete references from vector fields, thus shifting the indices of - # varnames occuring after `vn` by one to the left, as we adjusted for above. - deleteat!(vnv.varnames, idx) - deleteat!(vnv.ranges, idx) - deleteat!(vnv.transforms, idx) - - return vnv -end - -""" - values_as(vnv::VarNameVector[, T]) - -Return the values/realizations in `vnv` as type `T`, if implemented. - -If no type `T` is provided, return values as stored in `vnv`. - -# Examples - -```jldoctest -julia> using DynamicPPL: VarNameVector - -julia> vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2.0]); - -julia> values_as(vnv) == [1.0, 2.0] -true - -julia> values_as(vnv, Vector{Float32}) == Vector{Float32}([1.0, 2.0]) -true - -julia> values_as(vnv, OrderedDict) == OrderedDict(@varname(x) => 1.0, @varname(y) => [2.0]) -true - -julia> values_as(vnv, NamedTuple) == (x = 1.0, y = [2.0]) -true -``` -""" -values_as(vnv::VarNameVector) = values_as(vnv, Vector) -values_as(vnv::VarNameVector, ::Type{Vector}) = vnv[:] -function values_as(vnv::VarNameVector, ::Type{Vector{T}}) where {T} - return convert(Vector{T}, values_as(vnv, Vector)) -end -function values_as(vnv::VarNameVector, ::Type{NamedTuple}) - return NamedTuple(zip(map(Symbol, keys(vnv)), values(vnv))) -end -function values_as(vnv::VarNameVector, ::Type{D}) where {D<:AbstractDict} - return ConstructionBase.constructorof(D)(pairs(vnv)) -end diff --git a/test/runtests.jl b/test/runtests.jl index ef7672db8..d15b17b42 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -37,7 +37,6 @@ include("test_util.jl") @testset "interface" begin include("utils.jl") include("compiler.jl") - include("varnamevector.jl") include("varinfo.jl") include("simple_varinfo.jl") include("model.jl") diff --git a/test/test_util.jl b/test/test_util.jl index 563e13df1..5267d6d2b 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -85,7 +85,6 @@ Return string representing a short description of `vi`. short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" function short_varinfo_name(vi::TypedVarInfo) - DynamicPPL.has_varnamevector(vi) && return "TypedVarInfo with VarNameVector" return "TypedVarInfo" end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" diff --git a/test/varinfo.jl b/test/varinfo.jl index 570f9623e..4237b291a 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -405,12 +405,6 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end - if DynamicPPL.has_varnamevector(varinfo) && mutating - # NOTE: Can't handle mutating `link!` and `invlink!` `VarNameVector`. - @test_broken false - continue - end - # Evaluate the model once to update the logp of the varinfo. varinfo = last(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())) diff --git a/test/varnamevector.jl b/test/varnamevector.jl deleted file mode 100644 index 6b7acfbeb..000000000 --- a/test/varnamevector.jl +++ /dev/null @@ -1,472 +0,0 @@ -replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) - -increase_size_for_test(x::Real) = [x] -increase_size_for_test(x::AbstractArray) = repeat(x, 2) - -decrease_size_for_test(x::Real) = x -decrease_size_for_test(x::AbstractVector) = first(x) -decrease_size_for_test(x::AbstractArray) = first(eachslice(x; dims=1)) - -function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) - if isconcretetype(eltype(vnv.varnames)) - # If the container is concrete, we need to make sure that the varname types match. - # E.g. if `vnv.varnames` has `eltype` `VarName{:x, IndexLens{Tuple{Int64}}}` then - # we need `vn` to also be of this type. - # => If the varname types don't match, we need to relax the container type. - return any(keys(vnv)) do vn_present - typeof(vn_present) !== typeof(val) - end - end - - return false -end -function need_varnames_relaxation(vnv::VarNameVector, vns, vals) - return any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) -end - -function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) - if isconcretetype(eltype(vnv.vals)) - return promote_type(eltype(vnv.vals), eltype(val)) != eltype(vnv.vals) - end - - return false -end -function need_values_relaxation(vnv::VarNameVector, vns, vals) - return any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) -end - -function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) - return if isconcretetype(eltype(vnv.transforms)) - # If the container is concrete, we need to make sure that the sizes match. - # => If the sizes don't match, we need to relax the container type. - any(keys(vnv)) do vn_present - size(vnv[vn_present]) != size(val) - end - elseif eltype(vnv.transforms) !== Any - # If it's not concrete AND it's not `Any`, then we should just make it `Any`. - true - else - # Otherwise, it's `Any`, so we don't need to relax the container type. - false - end -end -function need_transforms_relaxation(vnv::VarNameVector, vns, vals) - return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) -end - -""" - relax_container_types(vnv::VarNameVector, vn::VarName, val) - relax_container_types(vnv::VarNameVector, vns, val) - -Relax the container types of `vnv` if necessary to accommodate `vn` and `val`. - -This attempts to avoid unnecessary container type relaxations by checking whether -the container types of `vnv` are already compatible with `vn` and `val`. - -# Notes -For example, if `vn` is not compatible with the current keys in `vnv`, then -the underlying types will be changed to `VarName` to accommodate `vn`. - -Similarly: -- If `val` is not compatible with the current values in `vnv`, then - the underlying value type will be changed to `Real`. -- If `val` requires a transformation that is not compatible with the current - transformations type in `vnv`, then the underlying transformation type will - be changed to `Any`. -""" -function relax_container_types(vnv::VarNameVector, vn::VarName, val) - return relax_container_types(vnv, [vn], [val]) -end -function relax_container_types(vnv::VarNameVector, vns, vals) - if need_varnames_relaxation(vnv, vns, vals) - varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) - varnames_new = convert(Vector{VarName}, vnv.varnames) - else - varname_to_index_new = vnv.varname_to_index - varnames_new = vnv.varnames - end - - transforms_new = if need_transforms_relaxation(vnv, vns, vals) - convert(Vector{Any}, vnv.transforms) - else - vnv.transforms - end - - vals_new = if need_values_relaxation(vnv, vns, vals) - convert(Vector{Real}, vnv.vals) - else - vnv.vals - end - - return VarNameVector( - varname_to_index_new, - varnames_new, - vnv.ranges, - vals_new, - transforms_new, - vnv.is_transformed, - vnv.num_inactive, - vnv.metadata, - ) -end - -@testset "VarNameVector" begin - # Need to test element-related operations: - # - `getindex` - # - `setindex!` - # - `push!` - # - `update!` - # - # And these should all be tested for different types of values: - # - scalar - # - vector - # - matrix - - # Need to test operations on `VarNameVector`: - # - `empty!` - # - `iterate` - # - `convert` to - # - `AbstractDict` - test_pairs = OrderedDict( - @varname(x[1]) => rand(), - @varname(x[2]) => rand(2), - @varname(x[3]) => rand(2, 3), - @varname(y[1]) => rand(), - @varname(y[2]) => rand(2), - @varname(y[3]) => rand(2, 3), - @varname(z[1]) => rand(1:10), - @varname(z[2]) => rand(1:10, 2), - @varname(z[3]) => rand(1:10, 2, 3), - ) - test_vns = collect(keys(test_pairs)) - test_vals = collect(values(test_pairs)) - - @testset "constructor: no args" begin - # Empty. - vnv = VarNameVector() - @test isempty(vnv) - @test eltype(vnv) == Real - - # Empty with types. - vnv = VarNameVector{VarName,Float64}() - @test isempty(vnv) - @test eltype(vnv) == Float64 - end - - test_varnames_iter = combinations(test_vns, 2) - @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter - val_left = test_pairs[vn_left] - val_right = test_pairs[vn_right] - vnv_base = VarNameVector([vn_left, vn_right], [val_left, val_right]) - - # We'll need the transformations later. - # TODO: Should we test other transformations than just `FromVec`? - from_vec_left = DynamicPPL.from_vec_transform(val_left) - from_vec_right = DynamicPPL.from_vec_transform(val_right) - to_vec_left = inverse(from_vec_left) - to_vec_right = inverse(from_vec_right) - - # Compare to alternative constructors. - vnv_from_dict = VarNameVector( - OrderedDict(vn_left => val_left, vn_right => val_right) - ) - @test vnv_base == vnv_from_dict - - # We want the types of fields such as `varnames` and `transforms` to specialize - # whenever possible + some functionality, e.g. `push!`, is only sensible - # if the underlying containers can support it. - # Expected behavior - should_have_restricted_varname_type = typeof(vn_left) == typeof(vn_right) - should_have_restricted_transform_type = size(val_left) == size(val_right) - # Actual behavior - has_restricted_transform_type = isconcretetype(eltype(vnv_base.transforms)) - has_restricted_varname_type = isconcretetype(eltype(vnv_base.varnames)) - - @testset "type specialization" begin - @test !should_have_restricted_varname_type || has_restricted_varname_type - @test !should_have_restricted_transform_type || has_restricted_transform_type - end - - # `eltype` - @test eltype(vnv_base) == promote_type(eltype(val_left), eltype(val_right)) - # `length` - @test length(vnv_base) == length(val_left) + length(val_right) - - # `isempty` - @test !isempty(vnv_base) - - # `empty!` - @testset "empty!" begin - vnv = deepcopy(vnv_base) - empty!(vnv) - @test isempty(vnv) - end - - # `similar` - @testset "similar" begin - vnv = similar(vnv_base) - @test isempty(vnv) - @test typeof(vnv) == typeof(vnv_base) - end - - # `getindex` - @testset "getindex" begin - # With `VarName` index. - @test vnv_base[vn_left] == val_left - @test vnv_base[vn_right] == val_right - - # With `Int` index. - val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) - @test all(vnv_base[i] == val_vec[i] for i in 1:length(val_vec)) - end - - # `setindex!` - @testset "setindex!" begin - vnv = deepcopy(vnv_base) - vnv[vn_left] = val_left .+ 100 - @test vnv[vn_left] == val_left .+ 100 - vnv[vn_right] = val_right .+ 100 - @test vnv[vn_right] == val_right .+ 100 - end - - # `getindex_raw` - @testset "getindex_raw" begin - # With `VarName` index. - @test DynamicPPL.getindex_raw(vnv_base, vn_left) == to_vec_left(val_left) - @test DynamicPPL.getindex_raw(vnv_base, vn_right) == to_vec_right(val_right) - # With `Int` index. - val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) - @test all( - DynamicPPL.getindex_raw(vnv_base, i) == val_vec[i] for - i in 1:length(val_vec) - ) - end - - # `setindex_raw!` - @testset "setindex_raw!" begin - vnv = deepcopy(vnv_base) - DynamicPPL.setindex_raw!(vnv, to_vec_left(val_left .+ 100), vn_left) - @test vnv[vn_left] == val_left .+ 100 - DynamicPPL.setindex_raw!(vnv, to_vec_right(val_right .+ 100), vn_right) - @test vnv[vn_right] == val_right .+ 100 - end - - # `delete!` - @testset "delete!" begin - vnv = deepcopy(vnv_base) - delete!(vnv, vn_left) - @test !haskey(vnv, vn_left) - @test haskey(vnv, vn_right) - delete!(vnv, vn_right) - @test !haskey(vnv, vn_right) - end - - # `merge` - @testset "merge" begin - # When there are no inactive entries, `merge` on itself result in the same. - @test merge(vnv_base, vnv_base) == vnv_base - - # Merging with empty should result in the same. - @test merge(vnv_base, similar(vnv_base)) == vnv_base - @test merge(similar(vnv_base), vnv_base) == vnv_base - - # With differences. - vnv_left_only = deepcopy(vnv_base) - delete!(vnv_left_only, vn_right) - vnv_right_only = deepcopy(vnv_base) - delete!(vnv_right_only, vn_left) - - # `(x,)` and `(x, y)` should be `(x, y)`. - @test merge(vnv_left_only, vnv_base) == vnv_base - # `(x, y)` and `(x,)` should be `(x, y)`. - @test merge(vnv_base, vnv_left_only) == vnv_base - # `(x, y)` and `(y,)` should be `(x, y)`. - @test merge(vnv_base, vnv_right_only) == vnv_base - # `(y,)` and `(x, y)` should be `(y, x)`. - vnv_merged = merge(vnv_right_only, vnv_base) - @test vnv_merged != vnv_base - @test collect(keys(vnv_merged)) == [vn_right, vn_left] - end - - # `push!` & `update!` - @testset "push!" begin - vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn" for vn in test_vns - val = test_pairs[vn] - if vn == vn_left || vn == vn_right - # Should not be possible to `push!` existing varname. - @test_throws ArgumentError push!(vnv, vn, val) - else - push!(vnv, vn, val) - @test vnv[vn] == val - end - end - end - @testset "update!" begin - vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn" for vn in test_vns - val = test_pairs[vn] - expected_length = if haskey(vnv, vn) - # If it's already present, the resulting length will be unchanged. - length(vnv) - else - length(vnv) + length(val) - end - - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] - @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) - - # There should be no redundant values in the underlying vector. - @test !DynamicPPL.has_inactive(vnv) - - # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) - end - - vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn (increased size)" for vn in test_vns - val_original = test_pairs[vn] - val = increase_size_for_test(val_original) - vn_already_present = haskey(vnv, vn) - expected_length = if vn_already_present - # If it's already present, the resulting length will be altered. - length(vnv) + length(val) - length(val_original) - else - length(vnv) + length(val) - end - - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] - @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) - - # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) - end - - vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) - @testset "$vn (decreased size)" for vn in test_vns - val_original = test_pairs[vn] - val = decrease_size_for_test(val_original) - vn_already_present = haskey(vnv, vn) - expected_length = if vn_already_present - # If it's already present, the resulting length will be altered. - length(vnv) + length(val) - length(val_original) - else - length(vnv) + length(val) - end - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] - @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) - - # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) - end - end - end - - @testset "growing and shrinking" begin - @testset "deterministic" begin - n = 5 - vn = @varname(x) - vnv = VarNameVector(OrderedDict(vn => [true])) - @test !DynamicPPL.has_inactive(vnv) - # Growing should not create inactive ranges. - for i in 1:n - x = fill(true, i) - DynamicPPL.update!(vnv, vn, x) - @test !DynamicPPL.has_inactive(vnv) - end - - # Same size should not create inactive ranges. - x = fill(true, n) - DynamicPPL.update!(vnv, vn, x) - @test !DynamicPPL.has_inactive(vnv) - - # Shrinking should create inactive ranges. - for i in (n - 1):-1:1 - x = fill(true, i) - DynamicPPL.update!(vnv, vn, x) - @test DynamicPPL.has_inactive(vnv) - @test DynamicPPL.num_inactive(vnv, vn) == n - i - end - end - - @testset "random" begin - n = 5 - vn = @varname(x) - vnv = VarNameVector(OrderedDict(vn => [true])) - @test !DynamicPPL.has_inactive(vnv) - - # Insert a bunch of random-length vectors. - for i in 1:100 - x = fill(true, rand(1:n)) - DynamicPPL.update!(vnv, vn, x) - end - # Should never be allocating more than `n` elements. - @test DynamicPPL.num_allocated(vnv, vn) ≤ n - - # If we compaticfy, then it should always be the same size as just inserted. - for i in 1:10 - x = fill(true, rand(1:n)) - DynamicPPL.update!(vnv, vn, x) - DynamicPPL.contiguify!(vnv) - @test DynamicPPL.num_allocated(vnv, vn) == length(x) - end - end - end -end - -@testset "VarInfo + VarNameVector" begin - models = DynamicPPL.TestUtils.DEMO_MODELS - @testset "$(model.f)" for model in models - # NOTE: Need to set random seed explicitly to avoid using the same seed - # for initialization as for sampling in the inner testset below. - Random.seed!(42) - value_true = DynamicPPL.TestUtils.rand_prior_true(model) - vns = DynamicPPL.TestUtils.varnames(model) - varnames = DynamicPPL.TestUtils.varnames(model) - varinfos = DynamicPPL.TestUtils.setup_varinfos( - model, value_true, varnames; include_threadsafe=false - ) - # Filter out those which are not based on `VarNameVector`. - varinfos = filter(DynamicPPL.has_varnamevector, varinfos) - # Get the true log joint. - logp_true = DynamicPPL.TestUtils.logjoint_true(model, value_true...) - - @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos - # Need to make sure we're using a different random seed from the - # one used in the above call to `rand_prior_true`. - Random.seed!(43) - - # Are values correct? - DynamicPPL.TestUtils.test_values(varinfo, value_true, vns) - - # Is evaluation correct? - varinfo_eval = last( - DynamicPPL.evaluate!!(model, deepcopy(varinfo), DefaultContext()) - ) - # Log density should be the same. - @test getlogp(varinfo_eval) ≈ logp_true - # Values should be the same. - DynamicPPL.TestUtils.test_values(varinfo_eval, value_true, vns) - - # Is sampling correct? - varinfo_sample = last( - DynamicPPL.evaluate!!(model, deepcopy(varinfo), SamplingContext()) - ) - # Log density should be different. - @test getlogp(varinfo_sample) != getlogp(varinfo) - # Values should be different. - DynamicPPL.TestUtils.test_values( - varinfo_sample, value_true, vns; compare=!isequal - ) - end - end -end From 8930f9c2ea69a4ce106df2263bf3264315c3e5e7 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 18:11:07 +0000 Subject: [PATCH 135/209] formatting --- docs/src/internals/transformations.md | 88 +++++++++++++-------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 45134a593..8173a4abc 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -9,7 +9,7 @@ For example, consider the following model: ```julia @model function demo() s ~ InverseGamma(2, 3) - m ~ Normal(0, √s) + return m ~ Normal(0, √s) end ``` @@ -18,6 +18,7 @@ Here we have two variables `s` and `m`, where `s` is constrained to be positive, For certain inference methods, it's necessary / much more convenient to work with an equivalent model to `demo` but where all the variables can take any real values (they're "unconstrained"). !!! note + We write "unconstrained" with quotes because there are many ways to transform a constrained variable to an unconstrained one, *and* DynamicPPL can work with a much broader class of bijective transformations of variables, not just ones that go to the entire real line. But for MCMC, unconstraining is the most common transformation so we'll stick with that terminology. For a large family of constraints encoucntered in practice, it is indeed possible to transform a (partially) contrained model to a completely unconstrained one in such a way that sampling in the unconstrained space is equivalent to sampling in the constrained space. @@ -30,7 +31,7 @@ For example, the above model could be transformed into (the following psuedo-cod @model function demo() log_s ~ log(InverseGamma(2, 3)) s = exp(log_s) - m ~ Normal(0, √s) + return m ~ Normal(0, √s) end ``` @@ -51,12 +52,12 @@ Below we'll see how this is done. ## What do we need? There are two aspects to transforming from the internal representation of a variable in a `varinfo` to the representation wanted in the model: -1. Different implementations of [`AbstractVarInfo`](@ref) represent realizations of a model in different ways internally, so we need to transform from this internal representation to the desired representation in the model. For example, - - [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. - - [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). -2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. - + 1. Different implementations of [`AbstractVarInfo`](@ref) represent realizations of a model in different ways internally, so we need to transform from this internal representation to the desired representation in the model. For example, + + + [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. + + [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). + 2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. ## Working example @@ -70,11 +71,13 @@ using DynamicPPL, Distributions `LKJCholesky` is a `LKJ(2, 1.0)` distribution, a distribution over correlation matrices (covariance matrices but with unit diagonal), but working directly with the Cholesky factorization of the correlation matrix rather than the correlation matrix itself (this is more numerically stable and computationally efficient). !!! note + This is a particularly "annoying" case because the return-value is not a simple `Real` or `AbstractArray{<:Real}`, but rather a `LineraAlgebra.Cholesky` object which wraps a triangular matrix (whether it's upper- or lower-triangular depends on the instance). As mentioned, some implementations of `AbstractVarInfo`, e.g. [`VarInfo`](@ref), works with a "flattened" / vector representation of a variable, and so in this case we need two transformations: -1. From the `Cholesky` object to a vector representation. -2. From the `Cholesky` object to an "unconstrained" / linked vector representation. + + 1. From the `Cholesky` object to a vector representation. + 2. From the `Cholesky` object to an "unconstrained" / linked vector representation. And similarly, we'll need the inverses of these transformations. @@ -88,10 +91,12 @@ DynamicPPL.from_internal_transform ``` These methods allows us to extract the internal-to-model transformation function depending on the `varinfo`, the variable, and the distribution of the variable: -- `varinfo` + `vn` defines the internal representation of the variable. -- `dist` defines the representation expected within the model scope. + + - `varinfo` + `vn` defines the internal representation of the variable. + - `dist` defines the representation expected within the model scope. !!! note + If `vn` is not present in `varinfo`, then the internal representation is fully determined by `varinfo` alone. This is used when we're about to add a new variable to the `varinfo` and need to know how to represent it internally. Continuing from the example above, we can inspect the internal representation of `x` in `demo_lkj` with [`VarInfo`](@ref) using [`DynamicPPL.getindex_internal`](@ref): @@ -104,9 +109,7 @@ x_internal = DynamicPPL.getindex_internal(varinfo, @varname(x)) ```@example transformations-internal f_from_internal = DynamicPPL.from_internal_transform( - varinfo, - @varname(x), - LKJCholesky(2, 1.0) + varinfo, @varname(x), LKJCholesky(2, 1.0) ) f_from_internal(x_internal) ``` @@ -120,11 +123,7 @@ x_model = varinfo[@varname(x)] Similarly, we can go from the model representation to the internal representation: ```@example transformations-internal -f_to_internal = DynamicPPL.to_internal_transform( - varinfo, - @varname(x), - LKJCholesky(2, 1.0) -) +f_to_internal = DynamicPPL.to_internal_transform(varinfo, @varname(x), LKJCholesky(2, 1.0)) f_to_internal(x_model) ``` @@ -139,11 +138,7 @@ DynamicPPL.getindex_internal(simple_varinfo, @varname(x)) Here see that the internal representation is exactly the same as the model representation, and so we'd expect `from_internal_transform` to be the `identity` function: ```@example transformations-internal -DynamicPPL.from_internal_transform( - simple_varinfo, - @varname(x), - LKJCholesky(2, 1.0) -) +DynamicPPL.from_internal_transform(simple_varinfo, @varname(x), LKJCholesky(2, 1.0)) ``` Great! @@ -165,9 +160,7 @@ Continuing from the example above: ```@example transformations-internal f_to_linked_internal = DynamicPPL.to_linked_internal_transform( - varinfo, - @varname(x), - LKJCholesky(2, 1.0) + varinfo, @varname(x), LKJCholesky(2, 1.0) ) x_linked_internal = f_to_linked_internal(x_model) @@ -175,9 +168,7 @@ x_linked_internal = f_to_linked_internal(x_model) ```@example transformations-internal f_from_linked_internal = DynamicPPL.from_linked_internal_transform( - varinfo, - @varname(x), - LKJCholesky(2, 1.0) + varinfo, @varname(x), LKJCholesky(2, 1.0) ) f_from_linked_internal(x_linked_internal) @@ -216,7 +207,7 @@ Unfortunately, this is not possible in general. Consider for example the followi ```@example transformations-internal @model function demo_dynamic_constraint() m ~ Normal() - x ~ truncated(Normal(), lower=m) + x ~ truncated(Normal(); lower=m) return (m=m, x=x) end @@ -266,6 +257,7 @@ first(DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext())) we see that we indeed satisfy the constraint `m < x`, as desired. !!! warning + One shouldn't be setting variables in a linked `varinfo` willy-nilly directly like this unless one knows that the value will be compatible with the constraints of the model. The reason for this is that internally in a model evaluation, we construct the transformation from the internal to the model representation based on the *current* realizations in the model! That is, we take the `dist` in a `x ~ dist` expression _at model evaluation time_ and use that to construct the transformation, thus allowing it to change between model evaluations without invalidating the transformation. @@ -291,11 +283,13 @@ And so the earlier diagram becomes: ``` !!! note - If the support of `dist` was constant, this would not be necessary since we could just determine the transformation at the time of `varinfo_linked = link(varinfo, model)` and define this as the `from_internal_transform` for all subsequent evaluations. However, since the support of `dist` is *not* constant in general, we need to be able to determine the transformation at the time of the evaluation *and* thus whether we should construct the transformation from the linked internal representation or the non-linked internal representation. This is annoying, but necessary. + If the support of `dist` was constant, this would not be necessary since we could just determine the transformation at the time of `varinfo_linked = link(varinfo, model)` and define this as the `from_internal_transform` for all subsequent evaluations. However, since the support of `dist` is *not* constant in general, we need to be able to determine the transformation at the time of the evaluation *and* thus whether we should construct the transformation from the linked internal representation or the non-linked internal representation. This is annoying, but necessary. + This is also the reason why we have two definitions of `getindex`: -- [`getindex(::AbstractVarInfo, ::VarName, ::Distribution)`](@ref): used internally in model evaluations with the `dist` in a `x ~ dist` expression. -- [`getindex(::AbstractVarInfo, ::VarName)`](@ref): used externally by the user to get the realization of a variable. + + - [`getindex(::AbstractVarInfo, ::VarName, ::Distribution)`](@ref): used internally in model evaluations with the `dist` in a `x ~ dist` expression. + - [`getindex(::AbstractVarInfo, ::VarName)`](@ref): used externally by the user to get the realization of a variable. For `getindex` we have the following diagram: @@ -316,8 +310,9 @@ While if `dist` is not provided, we have: Notice that `dist` is not present here, but otherwise the diagrams are the same. !!! warning - This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. + This does mean that the `getindex(varinfo, varname)` might not be the same as the `getindex(varinfo, varname, dist)` that occurs within a model evaluation! This can be confusing, but as outlined above, we do want to allow the `dist` in a `x ~ dist` expression to "override" whatever transformation `varinfo` might have. + ## Other functionalities There are also some additional methods for transforming between representations that are all automatically implemented from [`DynamicPPL.from_internal_transform`](@ref), [`DynamicPPL.from_linked_internal_transform`](@ref) and their siblings, and thus don't need to be implemented manually. @@ -341,8 +336,9 @@ DynamicPPL.from_maybe_linked_internal # Supporting a new distribution To support a new distribution, one needs to implement for the desired `AbstractVarInfo` the following methods: -- [`DynamicPPL.from_internal_transform`](@ref) -- [`DynamicPPL.from_linked_internal_transform`](@ref) + + - [`DynamicPPL.from_internal_transform`](@ref) + - [`DynamicPPL.from_linked_internal_transform`](@ref) At the time of writing, [`VarInfo`](@ref) is the one that is most commonly used, whose internal representation is always a `Vector`. In this scenario, one can just implemente the following methods instead: @@ -354,8 +350,9 @@ DynamicPPL.from_linked_vec_transform(::Distribution) These are used internally by [`VarInfo`](@ref). Optionally, if `inverse` of the above is expensive to compute, one can also implement: -- [`DynamicPPL.to_internal_transform`](@ref) -- [`DynamicPPL.to_linked_internal_transform`](@ref) + + - [`DynamicPPL.to_internal_transform`](@ref) + - [`DynamicPPL.to_linked_internal_transform`](@ref) And similarly, there are corresponding to-methods for the `from_*_vec_transform` variants too @@ -365,13 +362,14 @@ DynamicPPL.to_linked_vec_transform ``` !!! warning + Whatever the resulting transformation is, it should be invertible, i.e. implement `InverseFunctions.inverse`, and have a well-defined log-abs-det Jacobian, i.e. implement `ChangesOfVariables.with_logabsdet_jacobian`. # TL;DR -- DynamicPPL.jl has three representations of a variable: the **model representation**, the **internal representation**, and the **linked internal representation**. - - The **model representation** is the representation of the variable as it appears in the model code / is expected by the `dist` on the right-hand-side of the `~` in the model code. - - The **internal representation** is the representation of the variable as it appears in the `varinfo`, which varies between implementations of [`AbstractVarInfo`](@ref), e.g. a `Vector` in [`VarInfo`](@ref). This can be converted to the model representation by [`DynamicPPL.from_internal_transform`](@ref). - - The **linked internal representation** is the representation of the variable as it appears in the `varinfo` after [`link`](@ref)ing. This can be converted to the model representation by [`DynamicPPL.from_linked_internal_transform`](@ref). -- Having separation between *internal* and *linked internal* is necessary because transformations might be constructed at the time of model evaluation, and thus we need to know whether to construct the transformation from the internal representation or the linked internal representation. - + - DynamicPPL.jl has three representations of a variable: the **model representation**, the **internal representation**, and the **linked internal representation**. + + + The **model representation** is the representation of the variable as it appears in the model code / is expected by the `dist` on the right-hand-side of the `~` in the model code. + + The **internal representation** is the representation of the variable as it appears in the `varinfo`, which varies between implementations of [`AbstractVarInfo`](@ref), e.g. a `Vector` in [`VarInfo`](@ref). This can be converted to the model representation by [`DynamicPPL.from_internal_transform`](@ref). + + The **linked internal representation** is the representation of the variable as it appears in the `varinfo` after [`link`](@ref)ing. This can be converted to the model representation by [`DynamicPPL.from_linked_internal_transform`](@ref). + - Having separation between *internal* and *linked internal* is necessary because transformations might be constructed at the time of model evaluation, and thus we need to know whether to construct the transformation from the internal representation or the linked internal representation. From e45b6681d28be4c866d94ccd25dcff9ddb7d281c Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 18:14:49 +0000 Subject: [PATCH 136/209] Update docs/src/internals/transformations.md Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/src/internals/transformations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 8173a4abc..ee2c652bf 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -57,6 +57,7 @@ There are two aspects to transforming from the internal representation of a vari + [`VarInfo`](@ref) represents a realization of a model as in a "flattened" / vector representation, regardless of form of the variable in the model. + [`SimpleVarInfo`](@ref) represents a realization of a model exactly as in the model (unless it has been transformed; we'll get to that later). + 2. We need the ability to transform from "constrained space" to "unconstrained space", as we saw in the previous section. ## Working example From 0e71092394ac0c6d9a228e6dd108b72812854928 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 18:14:55 +0000 Subject: [PATCH 137/209] Update docs/src/internals/transformations.md Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- docs/src/internals/transformations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index ee2c652bf..19fe9c1e0 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -373,4 +373,5 @@ DynamicPPL.to_linked_vec_transform + The **model representation** is the representation of the variable as it appears in the model code / is expected by the `dist` on the right-hand-side of the `~` in the model code. + The **internal representation** is the representation of the variable as it appears in the `varinfo`, which varies between implementations of [`AbstractVarInfo`](@ref), e.g. a `Vector` in [`VarInfo`](@ref). This can be converted to the model representation by [`DynamicPPL.from_internal_transform`](@ref). + The **linked internal representation** is the representation of the variable as it appears in the `varinfo` after [`link`](@ref)ing. This can be converted to the model representation by [`DynamicPPL.from_linked_internal_transform`](@ref). + - Having separation between *internal* and *linked internal* is necessary because transformations might be constructed at the time of model evaluation, and thus we need to know whether to construct the transformation from the internal representation or the linked internal representation. From 2de9ac98eb92f7bb14354e7fb9617e33b2c64ad2 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Wed, 31 Jan 2024 18:18:34 +0000 Subject: [PATCH 138/209] removed refs to VectorVarInfo --- src/test_utils.jl | 2 -- src/varinfo.jl | 37 ------------------------------------- test/test_util.jl | 1 - 3 files changed, 40 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index e79aad9d1..ba40d10bd 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -52,8 +52,6 @@ function setup_varinfos( vi_untyped = VarInfo() model(vi_untyped) vi_typed = DynamicPPL.TypedVarInfo(vi_untyped) - vi_vnv = DynamicPPL.VectorVarInfo(vi_untyped) - vi_vnv_typed = DynamicPPL.VectorVarInfo(vi_typed) # SimpleVarInfo svi_typed = SimpleVarInfo(example_values) svi_untyped = SimpleVarInfo(OrderedDict()) diff --git a/src/varinfo.jl b/src/varinfo.jl index eda8213b1..725dbdc86 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -118,19 +118,6 @@ function VarInfo(old_vi::UntypedVarInfo, spl, x::AbstractVector) return new_vi end -function VarInfo(old_vi::VectorVarInfo, spl, x::AbstractVector) - new_vi = deepcopy(old_vi) - new_vi[spl] = x - return new_vi -end - -function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) - md = newmetadata(old_vi.metadata, Val(getspace(spl)), x) - return VarInfo( - md, Base.RefValue{eltype(x)}(getlogp(old_vi)), Ref(get_num_produce(old_vi)) - ) -end - function untyped_varinfo( rng::Random.AbstractRNG, model::Model, @@ -252,11 +239,6 @@ function subset(varinfo::UntypedVarInfo, vns::AbstractVector{<:VarName}) return VarInfo(metadata, varinfo.logp, varinfo.num_produce) end -function subset(varinfo::VectorVarInfo, vns::AbstractVector{<:VarName}) - metadata = subset(varinfo.metadata, vns) - return VarInfo(metadata, varinfo.logp, varinfo.num_produce) -end - function subset(varinfo::TypedVarInfo, vns::AbstractVector{<:VarName{sym}}) where {sym} # If all the variables are using the same symbol, then we can just extract that field from the metadata. metadata = subset(getfield(varinfo.metadata, sym), vns) @@ -875,12 +857,6 @@ function TypedVarInfo(vi::UntypedVarInfo) return VarInfo(nt, Ref(logp), Ref(num_produce)) end TypedVarInfo(vi::TypedVarInfo) = vi -function TypedVarInfo(vi::VectorVarInfo) - logp = getlogp(vi) - num_produce = get_num_produce(vi) - nt = NamedTuple(group_by_symbol(vi.metadata)) - return VarInfo(nt, Ref(logp), Ref(num_produce)) -end function BangBang.empty!!(vi::VarInfo) _empty!(vi.metadata) @@ -1227,15 +1203,6 @@ function _link(model::Model, varinfo::UntypedVarInfo, spl::AbstractSampler) ) end -function _link(model::Model, varinfo::VectorVarInfo, spl::AbstractSampler) - varinfo = deepcopy(varinfo) - return VarInfo( - _link_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), - Base.Ref(getlogp(varinfo)), - Ref(get_num_produce(varinfo)), - ) -end - function _link(model::Model, varinfo::TypedVarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) md = _link_metadata_namedtuple!( @@ -2067,10 +2034,6 @@ end function values_as(vi::UntypedVarInfo, ::Type{D}) where {D<:AbstractDict} return ConstructionBase.constructorof(D)(values_from_metadata(vi.metadata)) end -values_as(vi::VectorVarInfo, ::Type{NamedTuple}) = values_as(vi.metadata, NamedTuple) -function values_as(vi::VectorVarInfo, ::Type{D}) where {D<:AbstractDict} - return values_as(vi.metadata, D) -end function values_as(vi::VarInfo{<:NamedTuple{names}}, ::Type{NamedTuple}) where {names} iter = Iterators.flatten(values_from_metadata(getfield(vi.metadata, n)) for n in names) diff --git a/test/test_util.jl b/test/test_util.jl index 5267d6d2b..1d23567da 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -88,7 +88,6 @@ function short_varinfo_name(vi::TypedVarInfo) return "TypedVarInfo" end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" -short_varinfo_name(::DynamicPPL.VectorVarInfo) = "VectorVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" From 9b7142850dedab31e25e66216e36c3239a3d7607 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:18:52 +0000 Subject: [PATCH 139/209] added impls of `from_internal_transform` for `ThreadSafeVarInfo` --- src/threadsafe.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index dd0357547..e3095b791 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -243,12 +243,16 @@ function invlink_with_logpdf(vi::ThreadSafeVarInfo, vn::VarName, dist, y) return invlink_with_logpdf(vi.varinfo, vn, dist, y) end -function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) +function from_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName) + return from_linked_internal_transform(varinfo.varinfo, vn) +end +function from_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) return from_linked_internal_transform(varinfo.varinfo, vn, dist) end -function from_linked_internal_transform( - varinfo::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}, dist -) - return from_linked_internal_transform(varinfo.varinfo, vns, dist) +function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName) + return from_linked_internal_transform(varinfo.varinfo, vn) +end +function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) + return from_linked_internal_transform(varinfo.varinfo, vn, dist) end From 786e9bf8f32c777cb66bd83e5a7243d2ef5ae791 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:21:58 +0000 Subject: [PATCH 140/209] reverted accidental removal of old `VarInfo` constructor --- src/varinfo.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index 725dbdc86..0d0bb1bad 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -118,6 +118,13 @@ function VarInfo(old_vi::UntypedVarInfo, spl, x::AbstractVector) return new_vi end +function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) + md = newmetadata(old_vi.metadata, Val(getspace(spl)), x) + return VarInfo( + md, Base.RefValue{eltype(x)}(getlogp(old_vi)), Ref(get_num_produce(old_vi)) + ) +end + function untyped_varinfo( rng::Random.AbstractRNG, model::Model, From f1fe42cc23b8f20aba6f7ccac80a2d77399a9afd Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:26:18 +0000 Subject: [PATCH 141/209] fixed incorrect `recombine` call --- src/varinfo.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 0d0bb1bad..c7bec1829 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1458,7 +1458,9 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}) vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn) end - return recombine(vi, vals_linked, length(vns)) + # HACK: I don't like this. + dist = getdist(vi, vns[1]) + return recombine(dist, vals_linked, length(vns)) end function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) @assert haskey(vi, vns[1]) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" From 2273954f12fb37f7fbad45903a1bb0927f4825e1 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:32:04 +0000 Subject: [PATCH 142/209] removed undefined refs to `VarNameVector` stuff in `setup_varinfos` --- src/test_utils.jl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/test_utils.jl b/src/test_utils.jl index ba40d10bd..172d68d4e 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -62,14 +62,7 @@ function setup_varinfos( lp = getlogp(vi_typed) varinfos = map(( - vi_untyped, - vi_typed, - vi_vnv, - vi_vnv_typed, - svi_typed, - svi_untyped, - svi_typed_ref, - svi_untyped_ref, + vi_untyped, vi_typed, svi_typed, svi_untyped, svi_typed_ref, svi_untyped_ref )) do vi # Set them all to the same values. DynamicPPL.setlogp!!(update_values!!(vi, example_values, varnames), lp) From ab7c18984bbf9d3ea32f3b9dd676853d85928291 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 00:40:13 +0000 Subject: [PATCH 143/209] bump minior version because Turing breaks --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c6d662c44..218a1af72 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "DynamicPPL" uuid = "366bfd00-2699-11ea-058f-f148b4cae6d8" -version = "0.24.5" +version = "0.25.0" [deps] AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" From 3a86601f199f844161a3975d5a883a52f926b4b4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 10:24:57 +0000 Subject: [PATCH 144/209] fix: was using `from_linked_internal_transform` in `from_internal_transform` for `ThreadSafeVarInfo` --- src/threadsafe.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index e3095b791..8769c7008 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -244,10 +244,10 @@ function invlink_with_logpdf(vi::ThreadSafeVarInfo, vn::VarName, dist, y) end function from_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName) - return from_linked_internal_transform(varinfo.varinfo, vn) + return from_internal_transform(varinfo.varinfo, vn) end function from_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName, dist) - return from_linked_internal_transform(varinfo.varinfo, vn, dist) + return from_internal_transform(varinfo.varinfo, vn, dist) end function from_linked_internal_transform(varinfo::ThreadSafeVarInfo, vn::VarName) From 28c7d85ea829feeb034d185ea86cebbaf849ff6f Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 10:48:09 +0000 Subject: [PATCH 145/209] removed `getindex_raw` --- docs/src/api.md | 1 - src/abstract_varinfo.jl | 20 +++++--------------- src/simple_varinfo.jl | 26 +++++--------------------- src/threadsafe.jl | 15 --------------- src/varinfo.jl | 32 ++++---------------------------- test/simple_varinfo.jl | 21 +++++++++++---------- test/varinfo.jl | 12 ++++++++---- 7 files changed, 33 insertions(+), 94 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index e07d762da..9d1d1501f 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -269,7 +269,6 @@ resetlogp!! ```@docs keys getindex -DynamicPPL.getindex_raw DynamicPPL.getindex_internal push!! empty!! diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index c048014b5..eb3c222b3 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -144,9 +144,7 @@ Return an iterator over all `vns` in `vi`. Return the current value(s) of `vn` (`vns`) in `vi` in the support of its (their) distribution(s). -If `dist` is specified, the value(s) will be reshaped accordingly. - -See also: [`getindex_raw(vi::AbstractVarInfo, vn::VarName, dist::Distribution)`](@ref) +If `dist` is specified, the value(s) will be massaged into the representation expected by `dist`. """ Base.getindex """ @@ -164,22 +162,14 @@ Base.getindex(vi::AbstractVarInfo, ::Colon) = values_as(vi, Vector) Base.getindex(vi::AbstractVarInfo, ::AbstractSampler) = vi[:] """ - getindex_raw(vi::AbstractVarInfo, vn::VarName[, dist::Distribution]) - getindex_raw(vi::AbstractVarInfo, vns::Vector{<:VarName}[, dist::Distribution]) - -Return the current value(s) of `vn` (`vns`) in `vi`. + getindex_internal(vi::AbstractVarInfo, vn::VarName) + getindex_internal(vi::AbstractVarInfo, vns::Vector{<:VarName}) -If `dist` is specified, the value(s) will be reshaped accordingly. +Return the current value(s) of `vn` (`vns`) in `vi` as represented internally in `vi`. See also: [`getindex(vi::AbstractVarInfo, vn::VarName, dist::Distribution)`](@ref) - -!!! note - The difference between `getindex(vi, vn, dist)` and `getindex_raw` is that - `getindex` will also transform the value(s) to the support of the distribution(s). - This is _not_ the case for `getindex_raw`. - """ -function getindex_raw end +function getindex_internal end """ push!!(vi::AbstractVarInfo, vn::VarName, r, dist::Distribution) diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index dc2780aae..a0d9e5925 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -290,7 +290,6 @@ function Base.show(io::IO, ::MIME"text/plain", svi::SimpleVarInfo) return print(io, "SimpleVarInfo(", svi.values, ", ", svi.logp, ")") end -# `NamedTuple` function Base.getindex(vi::SimpleVarInfo, vn::VarName, dist::Distribution) return from_maybe_linked_internal(vi, vn, dist, getindex(vi, vn)) end @@ -301,12 +300,7 @@ function Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribu return recombine(dist, vals_linked, length(vns)) end -Base.getindex(vi::SimpleVarInfo, vn::VarName) = get(vi.values, vn) - -# `AbstractDict` -function Base.getindex(vi::SimpleVarInfo{<:AbstractDict}, vn::VarName) - return nested_getindex(vi.values, vn) -end +Base.getindex(vi::SimpleVarInfo, vn::VarName) = getindex_internal(vi, vn) # `SimpleVarInfo` doesn't necessarily vectorize, so we can have arrays other than # just `Vector`. @@ -318,22 +312,12 @@ Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}) = map(Base.Fix1(getinde Base.getindex(svi::SimpleVarInfo, ::Colon) = values_as(svi, Vector) -# Since we don't perform any transformations in `getindex` for `SimpleVarInfo` -# we simply call `getindex` in `getindex_raw`. -getindex_raw(vi::SimpleVarInfo, vn::VarName) = vi[vn] -function getindex_raw(vi::SimpleVarInfo, vn::VarName, dist::Distribution) - f = from_internal_transform(vi, vn, dist) - return f(getindex_raw(vi, vn)) -end -getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}) = vi[vns] -function getindex_raw(vi::SimpleVarInfo, vns::Vector{<:VarName}, dist::Distribution) - vals = mapreduce(Base.Fix1(getindex_raw, vi), vcat, vns) - return recombine(dist, vals, length(vns)) +getindex_internal(vi::SimpleVarInfo, vn::VarName) = get(vi.values, vn) +# `AbstractDict` +function getindex_internal(vi::SimpleVarInfo{<:AbstractDict}, vn::VarName) + return nested_getindex(vi.values, vn) end -# HACK: because `VarInfo` isn't ready to implement a proper `getindex_raw`. -getindex_internal(vi::SimpleVarInfo, vn::VarName) = getindex_raw(vi, vn) - Base.haskey(vi::SimpleVarInfo, vn::VarName) = hasvalue(vi.values, vn) function BangBang.setindex!!(vi::SimpleVarInfo, val, vn::VarName) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 8769c7008..eb2d0246c 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -159,21 +159,6 @@ function getindex(vi::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}, dist::D end getindex(vi::ThreadSafeVarInfo, spl::AbstractSampler) = getindex(vi.varinfo, spl) -getindex_raw(vi::ThreadSafeVarInfo, ::Colon) = getindex_raw(vi.varinfo, Colon()) -getindex_raw(vi::ThreadSafeVarInfo, vn::VarName) = getindex_raw(vi.varinfo, vn) -function getindex_raw(vi::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}) - return getindex_raw(vi.varinfo, vns) -end -function getindex_raw(vi::ThreadSafeVarInfo, vn::VarName, dist::Distribution) - return getindex_raw(vi.varinfo, vn, dist) -end -function getindex_raw( - vi::ThreadSafeVarInfo, vns::AbstractVector{<:VarName}, dist::Distribution -) - return getindex_raw(vi.varinfo, vns, dist) -end -getindex_raw(vi::ThreadSafeVarInfo, spl::AbstractSampler) = getindex_raw(vi.varinfo, spl) - function BangBang.setindex!!(vi::ThreadSafeVarInfo, val, spl::AbstractSampler) return Setfield.@set vi.varinfo = BangBang.setindex!!(vi.varinfo, val, spl) end diff --git a/src/varinfo.jl b/src/varinfo.jl index c7bec1829..4e6779463 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -456,11 +456,6 @@ end const VarView = Union{Int,UnitRange,Vector{Int}} -""" - getindex_internal(vi::UntypedVarInfo, vview::Union{Int, UnitRange, Vector{Int}}) - -Return a view `vi.vals[vview]`. -""" getindex_internal(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, vview) """ @@ -535,6 +530,10 @@ The values may or may not be transformed to Euclidean space. getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) + return mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) +end + """ setval!(vi::VarInfo, val, vn::VarName) @@ -550,16 +549,6 @@ function setval!(md::Metadata, val, vn::VarName) return md.vals[getrange(md, vn)] = vectorize(getdist(md, vn), val) end -""" - getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) - -Return the value(s) of `vns`. - -The values may or may not be transformed to Euclidean space. -""" -getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) = - mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) - """ getall(vi::VarInfo) @@ -1471,19 +1460,6 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) return recombine(dist, vals_linked, length(vns)) end -getindex_raw(vi::VarInfo, vn::VarName) = getindex_raw(vi, vn, getdist(vi, vn)) -function getindex_raw(vi::VarInfo, vn::VarName, dist::Distribution) - f = from_internal_transform(vi, vn, dist) - return f(getindex_internal(vi, vn)) -end -function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}) - return getindex_raw(vi, vns, getdist(vi, first(vns))) -end -function getindex_raw(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) - # TODO: Replace when we have better dispatch for multiple vals. - return recombine(dist, getindex_internal(vi, vns), length(vns)) -end - """ getindex(vi::VarInfo, spl::Union{SampleFromPrior, Sampler}) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 223d0dd49..e2d87cbac 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -100,7 +100,8 @@ # Should result in same values. @test all( - DynamicPPL.getindex_raw(vi_invlinked, vn) ≈ get(values_constrained, vn) for + DynamicPPL.getindex_internal(vi_invlinked, vn) ≈ + vec(get(values_constrained, vn)) for vn in DynamicPPL.TestUtils.varnames(model) ) end @@ -252,11 +253,9 @@ model, deepcopy(vi_linked), DefaultContext() ) - @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≠ - retval.s # `s` is unconstrained in original - @test DynamicPPL.getindex_raw( - vi_linked_result, @varname(s), priors[@varname(s)] - ) == retval.s # `s` is constrained in result + @test DynamicPPL.getindex_internal(vi_linked, @varname(s)) ≠ vec(retval.s) # `s` is unconstrained in original + @test DynamicPPL.getindex_internal(vi_linked_result, @varname(s)) == + vec(retval.s) # `s` is constrained in result # `m` should not be transformed. @test vi_linked[@varname(m)] == retval.m @@ -267,10 +266,12 @@ model, retval.s, retval.m ) - @test DynamicPPL.getindex_raw(vi_linked, @varname(s), priors[@varname(s)]) ≈ - retval_unconstrained.s - @test DynamicPPL.getindex_raw(vi_linked, @varname(m), priors[@varname(m)]) ≈ - retval_unconstrained.m + @test DynamicPPL.getindex_internal( + vi_linked, @varname(s), priors[@varname(s)] + ) ≈ vec(retval_unconstrained.s) + @test DynamicPPL.getindex_internal( + vi_linked, @varname(m), priors[@varname(m)] + ) ≈ vec(retval_unconstrained.m) # The resulting varinfo should hold the correct logp. lp = getlogp(vi_linked_result) diff --git a/test/varinfo.jl b/test/varinfo.jl index 4237b291a..5c23f75a6 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -309,7 +309,8 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) vi = DynamicPPL.settrans!!(vi, true, vn) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) - x = Bijectors.invlink(dist, DynamicPPL.getindex_raw(vi, vn)) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) ## `TypedVarInfo` @@ -317,7 +318,8 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) vi = DynamicPPL.settrans!!(vi, true, vn) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) - x = Bijectors.invlink(dist, DynamicPPL.getindex_raw(vi, vn)) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) ### `SimpleVarInfo` @@ -325,14 +327,16 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) vi = DynamicPPL.settrans!!(SimpleVarInfo(), true) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) - x = Bijectors.invlink(dist, DynamicPPL.getindex_raw(vi, vn)) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) ## `SimpleVarInfo{<:Dict}` vi = DynamicPPL.settrans!!(SimpleVarInfo(Dict()), true) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) - x = Bijectors.invlink(dist, DynamicPPL.getindex_raw(vi, vn)) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) end From 59514d6c5378a2402a562e41a575e8ee5a950949 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 10:48:19 +0000 Subject: [PATCH 146/209] removed redundant docstrings --- src/varinfo.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 4e6779463..369c68ca9 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -520,13 +520,6 @@ Return the distribution from which `vn` was sampled in `vi`. getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] -""" - getindex_internal(vi::VarInfo, vn::VarName) - -Return the value(s) of `vn`. - -The values may or may not be transformed to Euclidean space. -""" getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) From cdc882b086f663b237a8b539b89328e5327ff6b3 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 10:49:25 +0000 Subject: [PATCH 147/209] fixed tests --- test/simple_varinfo.jl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index e2d87cbac..98fe0d8a9 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -101,7 +101,7 @@ # Should result in same values. @test all( DynamicPPL.getindex_internal(vi_invlinked, vn) ≈ - vec(get(values_constrained, vn)) for + DynamicPPLL.tovec(get(values_constrained, vn)) for vn in DynamicPPL.TestUtils.varnames(model) ) end @@ -253,9 +253,10 @@ model, deepcopy(vi_linked), DefaultContext() ) - @test DynamicPPL.getindex_internal(vi_linked, @varname(s)) ≠ vec(retval.s) # `s` is unconstrained in original + @test DynamicPPL.getindex_internal(vi_linked, @varname(s)) ≠ + DynamicPPL.tovec(retval.s) # `s` is unconstrained in original @test DynamicPPL.getindex_internal(vi_linked_result, @varname(s)) == - vec(retval.s) # `s` is constrained in result + DynamicPPL.tovec(retval.s) # `s` is constrained in result # `m` should not be transformed. @test vi_linked[@varname(m)] == retval.m @@ -268,10 +269,10 @@ @test DynamicPPL.getindex_internal( vi_linked, @varname(s), priors[@varname(s)] - ) ≈ vec(retval_unconstrained.s) + ) ≈ DynamicPPL.tovec(retval_unconstrained.s) @test DynamicPPL.getindex_internal( vi_linked, @varname(m), priors[@varname(m)] - ) ≈ vec(retval_unconstrained.m) + ) ≈ DynamicPPL.tovec(retval_unconstrained.m) # The resulting varinfo should hold the correct logp. lp = getlogp(vi_linked_result) From 57ba7c0245999c9f055f95af577bfc73fd10aa75 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 11:16:18 +0000 Subject: [PATCH 148/209] fixed comparisons in tests --- test/simple_varinfo.jl | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 98fe0d8a9..74dcdd842 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -100,8 +100,8 @@ # Should result in same values. @test all( - DynamicPPL.getindex_internal(vi_invlinked, vn) ≈ - DynamicPPLL.tovec(get(values_constrained, vn)) for + DynamicPPL.tovec(DynamicPPL.getindex_internal(vi_invlinked, vn)) ≈ + DynamicPPL.tovec(get(values_constrained, vn)) for vn in DynamicPPL.TestUtils.varnames(model) ) end @@ -253,10 +253,11 @@ model, deepcopy(vi_linked), DefaultContext() ) - @test DynamicPPL.getindex_internal(vi_linked, @varname(s)) ≠ + @test DynamicPPL.tovec(DynamicPPL.getindex_internal(vi_linked, @varname(s))) ≠ DynamicPPL.tovec(retval.s) # `s` is unconstrained in original - @test DynamicPPL.getindex_internal(vi_linked_result, @varname(s)) == - DynamicPPL.tovec(retval.s) # `s` is constrained in result + @test DynamicPPL.tovec( + DynamicPPL.getindex_internal(vi_linked_result, @varname(s)) + ) == DynamicPPL.tovec(retval.s) # `s` is constrained in result # `m` should not be transformed. @test vi_linked[@varname(m)] == retval.m @@ -267,12 +268,10 @@ model, retval.s, retval.m ) - @test DynamicPPL.getindex_internal( - vi_linked, @varname(s), priors[@varname(s)] - ) ≈ DynamicPPL.tovec(retval_unconstrained.s) - @test DynamicPPL.getindex_internal( - vi_linked, @varname(m), priors[@varname(m)] - ) ≈ DynamicPPL.tovec(retval_unconstrained.m) + @test DynamicPPL.tovec(DynamicPPL.getindex_internal(vi_linked, @varname(s))) ≈ + DynamicPPL.tovec(retval_unconstrained.s) + @test DynamicPPL.tovec(DynamicPPL.getindex_internal(vi_linked, @varname(m))) ≈ + DynamicPPL.tovec(retval_unconstrained.m) # The resulting varinfo should hold the correct logp. lp = getlogp(vi_linked_result) From 902e59cb65599b67f8cf7c8b320c8df68eb4cbf4 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Thu, 1 Feb 2024 11:22:49 +0000 Subject: [PATCH 149/209] try relative references for images in transformation docs --- docs/src/internals/transformations.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 19fe9c1e0..6fb18a392 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -43,7 +43,7 @@ In the end, we'll end up with something that looks like this: ```@raw html

- +
``` @@ -199,7 +199,7 @@ That is, why can't we just do ```@raw html
- +
``` @@ -279,7 +279,7 @@ And so the earlier diagram becomes: ```@raw html
- +
``` @@ -296,7 +296,7 @@ For `getindex` we have the following diagram: ```@raw html
- +
``` @@ -304,7 +304,7 @@ While if `dist` is not provided, we have: ```@raw html
- +
``` From d7aba554290d6cd8ea9b3884dea8ec29c5010d45 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 10:33:11 +0000 Subject: [PATCH 150/209] another attempt at fixing asset-references --- docs/src/internals/transformations.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/src/internals/transformations.md b/docs/src/internals/transformations.md index 6fb18a392..d4721bddb 100644 --- a/docs/src/internals/transformations.md +++ b/docs/src/internals/transformations.md @@ -43,7 +43,7 @@ In the end, we'll end up with something that looks like this: ```@raw html
- +
``` @@ -199,7 +199,7 @@ That is, why can't we just do ```@raw html
- +
``` @@ -279,7 +279,7 @@ And so the earlier diagram becomes: ```@raw html
- +
``` @@ -296,7 +296,7 @@ For `getindex` we have the following diagram: ```@raw html
- +
``` @@ -304,7 +304,7 @@ While if `dist` is not provided, we have: ```@raw html
- +
``` @@ -341,7 +341,7 @@ To support a new distribution, one needs to implement for the desired `AbstractV - [`DynamicPPL.from_internal_transform`](@ref) - [`DynamicPPL.from_linked_internal_transform`](@ref) -At the time of writing, [`VarInfo`](@ref) is the one that is most commonly used, whose internal representation is always a `Vector`. In this scenario, one can just implemente the following methods instead: +At the time of writing, [`VarInfo`](@ref) is the one that is most commonly used, whose internal representation is always a `Vector`. In this scenario, one can just implement the following methods instead: ```@docs DynamicPPL.from_vec_transform(::Distribution) From 1f51203120523473a73c603dc618253ed11ad59e Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:13:17 +0000 Subject: [PATCH 151/209] fixed getindex diagrams in docs --- .../transformations-getindex-with-dist.dot | 2 +- ...transformations-getindex-with-dist.dot.png | Bin 42305 -> 43833 bytes .../transformations-getindex-without-dist.dot | 2 +- ...nsformations-getindex-without-dist.dot.png | Bin 40798 -> 41849 bytes 4 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/assets/images/transformations-getindex-with-dist.dot b/docs/src/assets/images/transformations-getindex-with-dist.dot index 8c6826e3f..e44fc0ce6 100644 --- a/docs/src/assets/images/transformations-getindex-with-dist.dot +++ b/docs/src/assets/images/transformations-getindex-with-dist.dot @@ -7,7 +7,7 @@ digraph { getindex [shape=box, label=< x = getindex(varinfo, @varname(x), Normal()) >, fontname="Courier"]; iflinked_getindex [label=< if istrans(varinfo, varname) >, fontname="Courier"]; without_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; - with_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname, dist)", fontname="Courier"]; + with_linking_getindex [shape=box, label="f = from_linked_internal_transform(varinfo, varname, dist)", fontname="Courier"]; return_getindex [shape=box, label=< return f(getindex_internal(varinfo, varname)) >, style=dashed, fontname="Courier"]; getindex -> iflinked_getindex; diff --git a/docs/src/assets/images/transformations-getindex-with-dist.dot.png b/docs/src/assets/images/transformations-getindex-with-dist.dot.png index 621b10a51498ec87050960725b6806bab2913416..381ba45a2c9af4ee4a9728c25f31bf225a865fb1 100644 GIT binary patch literal 43833 zcmd431z4Ng*Dpvd1b$Auf6=Y_W7i#u1HKkO@N1oN38VxnKmBY zbv8V_EAX2)z?ol<{k_504GR^;XLuKv|1ui#zT)9M#8Y}EtLvMxf$|9;gPgbTVx3e< znctIWlihq65-D&+o=i5HLiX9V#FFmgynNrZ^B=-aAD3#H|1i9L{WkOAbMlW~*&Gj< zWgf~vXEyq_qrS*Ie8bG_^m=rEcPw;kV`Iy9S)9M8aX)qYey)AeUx0+mKhht*b^jdU z;U)0=rTnjBIysqJ{~Z3ZyvcUupF>}wD=h!I9>tJx?VrP0-fOY{Iw=wI`tCo6uiI~H z{_71p_x~RaTD9-D|Nhp!3Md902D|>xsK4ZMb@J4q8$a8-&d&D<*bGX|iUl1<@-5pV zSN3`TMFjU#$B`n@?S7Hc*V-{+-W&Oe{I+R64FA5${SKd0YP(f_@f{u*EEL3F$f zuZOA)$@(k>d`LVha5)rODKaqD?1Wvt?~mW1!&6Egst|@|X`}M;oDDx8_Le7mc~nA3 zf4jBLvW-} z^$riUnml+3k#wz7GNSjc{xkOYUe&=pR-?{#eS<-B+TjA0q)&tx86BSS^{9c-P!4-M zH87hLO72}C-T4lDmItS5!su2ERaeyJp(8T(MDU^Y7j#jG~$TMA^X$d81I`KQ#w#;BkSR$?G2nh6h1XHjPPFWI> zS86tsVrVt)A}W<}+HG^KU{b{#9fvW+9IZy9$M*0gr=}Qc7Y>#JebyJoY;f&wmFgNO zf|Kz9VPPUFSHYL|RM^R>q1xCN_KSRc0cW{-J`X-Gwq;Aqn20yLq_#tV7IL@U+<~C zO`65bHiQ41t^e9Gd;5}*TH?4T#P*AOWS!}Rj7RL(8ae*0@Yv>BQ$(Jt z$l&B)aiC5b(%-m@d~8mu_moT4P}Dxm81tsi`NGVx3+@!N_;L6fkJQlZizO)Z$P>pR zs5m(TS0U}`)Q&R9UHee-Mr`y_@qY3Dt*WrQ{Dvya7n!P#I+8ns*W}H0L&lFf-tCEe;DMcQ+9pMSu)8&6+nM@<~F ztI>@rM!ci-crSp=SZU8sZZ5tGKi3RlPW)t4_fTPcWEpBVkeea#FLJT6o8g|yLC(Vk z)`Ks4rC|+>8nhYh^WPuq%ICk=uFU{TV z6V}5aL~^X$3_K~!RJLYLfo0QgZ!MWrrASw}&8aVai{bPxSawy`yHz~{#*-nJy|kg4 zaW5PSWliC+Kg|k*ySetXak%^BwMLWr8U{jp(JU`tH;ODv-MN-_=$3~Lu7;)_(oz@_ z$=Rf-ln^FP8#)vO(I&?>YB^R@s!{NBSd8~ranVv1+v*=V?zHP*+!mw{4pXxP%WI6I zz)Ui_9$uQ|eC5wCO?`B1k88Xg=Fg%WCfb;~JZ#oFaO}|4wNgm^^Njo1G?}jYFWP*l z#TV_>On2dT`BJ*Nm50xrn^q@D(}#Wp_1m|uH4qge$tWy3YF*-UU7l+me(2i%d>gZh zkex2sOgj1{(mU>LBHE`)`}z1;R{D|ht(8hAmsYhV4W{WcQv->e0;zhR8nZ-Py;x4q zKp@lf;7AHT?Ek=+oEN%2i-fp#!N#t!CrZs(d#*7$S*V$cdZaUY;|udbOU`fzj5t7t zKGLCRk)?F4&&7cK&kO_HRZ!4wP`>*7Ot5##SGN!0uQV~4uhH+ewu*&jJHHPo3L$FF zg_3gV$xbI6BOe~$2fx3}q0G&*zjd7*UqSv%`?kEAE$ov3aZ(9KM^TrC8bR5uG0D_p zjH=Tl(#W*~A1=pf4cc%b6~ij)?rA<*CWYYQSnqxZbYTOT#q((c=@OplP1aKKem3Zo zHj8JmFj(!lXMe9Y^2mF5e?NG@IIAvSD?h6K9l8*aPbzql!9JTQJi^PBp~t6a0D@^I?VlTH^=Pt^|P zg&94bi=zfJnsNDv!s8C{A8BF{I8D!)21ACt1g$P>{;w44ovXnPISoEZel ze-DXT1!FS@rpelwOV6bX3+upNlf9JYrEI$qrfT${(thQ!W?hP3qFB9sNSl<9VKA%; zdKk%^0_5)(%R5hFJiON&?YeE#*P$yYS4`j4JXN7081Jm4#Iq5o50$DN(xM@{7vZBVKtwpx->`sAK8vUfTp;6}fu{!&1zVYYqew?}(* z78Mk^W;;AQ$;ys|)$^1M`M)>O>|@;=;jcQT15!qo^cH<)=XIk`6ibB8%wq!>Z#Wv& z8{TP+lP#0l-ezikV&#g6RHSTb^&!;X?RC!U7 z?%99V7qH^6P*#!5I(Z-OHDNdiNnh_r^O{zl+}~TfsEv^{RU!XU?VOp0!@h#}oho5S z4!i`@UbO{(Kf#yID;;fRXBjPqJ>ayW70u|+;~AB_z(%+nwX2$5^wS>=(1$-PCje8v@mas<`DO=xNOQC(JHf#H8*kBv*EH^JXY4Y5x zdUfBb+JoZ4w34SX_(LyQZ}6F<^UZoSO9g>^94l4aFa;H7<7huSMP-2R8xpHJtEAkN zNppQ;#Wc;MY87btebzg%LC8JW^wmQPom(fN zcc9^P55{Fqqyy#6TlBOOHp+iqW9!)dXmjRY5^2hpSf!@L;}nMwu&z5?Eq0{M33gtT!)S3@3Z7%Z^oYDduSx?-Dg z70hm|bKNYZ?qtcxk5^o=y9-u<u%UWNmoPQ~db{k@+^ zy#oV$2)bX3-~BtC9h$}e><$j6iNt+JtG_6y-d@Xn^eSkN*d@d9g3mD7B9Ba3Jxg3` zU7EH~HZ>_9`ntOQjLAz6s|DT6tNHpW!oT#9*KQYmde8qBFK4^;@Fk-qD-L4aSQTex zOx5Cb+5|Z5r<&_A{cJ-n2I01&>tm(#t6SW}|`MI9clY{X?bM28s0H! zq}FWnD7A)JOcCEeeX!gIqXo~cBwyV8VqSS84DFvjZFIkoKlEMq;FS0~tKF^QC=zS? z?WthPrqmM&#z{*iw;6Rcm7Sk!;wPZ`usWvOX%ULx8r zU)UXYhhXP0#{7j`C4AF8u!^^KkLQ*T%UYr9A;(FQr)*{tPZPj=8xt(!{pSW_oEWgX9qrYD_M}F zM*i||zYxW7<~{nO!H zAZ<9^@*jn=JwU`hJHjj7oD9xJ#=JFstQYq#@RG6}OriauGCkUU+`%E^5&1b6M% z+y3b(h)ecmNt<|RQ{vs~G$K87Zg$qzHw@7TC|t!_95k6Gik;lfS{ak>TNrb2bpMJI z5~zeM6FJw;E~HEhJ1v?c^*9BO%RS%QT2Cyi{FGkl8Kk`;WXg+UQhP9=`dtK3oQQY?4gRQD!*pQm^iT zotI={?~TiaN+hVif2-D z(??mmmduRxwbQ=lMf1|jQwAT-z$raUeWiq2??6{sYoAC|Dx2TkDK?k8tSKP z)VWV`Jbi~QR6UYpV?0lZkc0yhEBx(Ku)}FPjbN~A$WRJ*4xP$Cihbe1ctJvA{Lhf= z2;+m_xw<*lwhqmIFwnz&NKz$_%DAW@&UyS%ub8ie3%@a^?cpaxDu9q z)vLDp*mJW#H?r@39;?lkuC_x;e7%T~T*yr6C&&4?=pfCe`jhvMT*RVR<4(S58?D$a zAmVaR5>x+nHLuIupE>kU(%XN$@Q{=|6+Med%G%sU<=`({6|{3A4xWzwI?c#bcH`XI zw-tOPmwTE;H&8*m+ttxI@cmm9(sI(In0qL-TX(OX6>_m?%9lhu3MceoV?eOmLhFTk zsq?mep9n%W){musaS>z2x*jCvvh$y*5Ff;#`EW0TI%y8y)``ABrD!(>5X&GNS{<6s`=?}J9p~83{@1| zufArzl+Esf#79@1`4_WqmnkhdTCR{dkHxU4Jkt>*=5jf9atKtcFfyDMIO$%JhZfgW z^$!>%Pr)SJ-b0VmO>6}x71zf#C>__|SXr+uv9k)AO)g7*KQzNXzyz;Q(u>p`+I1C% z4_i$=c9!@COWCkGtFIaX?C{g(_tpEc z1^ed1O+gZmR8r5OmV+)VHY;p`-j$D{9Jsd+?l+wG=+fp_5d9f$oVgc!pX~dgAhD9m zS*=%28~5{3!FR3$KBWf;N7G`QwPlP+E_*eRpuMqq#Aj8m$uF9Vq}R5Z-HieUbjOew zWLx1&nBFT4+J1P44rg&H!{z7yF}_|(M(1Kgx7R7ri29y6vzx^6tF@%2)sJYC;NslR zO{OZ-)cJ{|)d*+IMxK*nE{T&ETe-imS}^2CeEpCCj|mNPvbbKjFmvkAyPuB*zj@pc z7@*-H9@>b;kfGHFa}VmECpof7{AFoRd%ZS4c1N?b2d>*R{1nqucorymifzPg^)eV8 zw&4XrH9RX1JBQ!aidR-xY(vIXzDHOdfG%-%oezOV?66I~RK{Xwfvh zNFPpLQ!zDpDvGt$5xv8N349u2aUrZLXfj!`v3i?JwPWCoeY0+JgQDXo%-qxwgwW= z)w?H#ZBMm&1g^|n3$hKXc6n|ToU@IAy<~`&(h|OL24v+Evq`D0O!YToH!}6onf(nDtk7p`|Or2 z<%rE7_4IV^k!!BK`msx+8dqM08hH~mJC z+^?G^tn*2n53`wKIHABEN!P?Sf8g1vvx6z83`yxj!PPy=2Jn+Z&qmV(wc-%Y*NLpV z9`2?4yAFf7K+`u+v$u4Ae_cn5Ng!oio92Anb+i$6|r=!`GmvT zGy}wHyi%@T<48S_KD{8#vI+T7!`s2XWKp?Ws@$iPxt8Dh(?&)DZba+r!{iMixPy>W zK5yXqB6ja_{!&;j6dghhTd<5q32REaIAxCL>)g6!&&pHFqeC2WN;9mSiJJ-=ZkUtjzl1XsHXBEjx_oc1yWSn+ z`+_%A<#~|BlY$V1l2j@nIU)DY$tmkP&Ro^M+D5$}al1jN{j^*B!R>3CcwR~K1;4O(W( zz1A5N=>ne*(z#DNBaf*=r_=p4e}TJ!nTYXb{=xUN5<`ubiygJKPb0KIKBz z@wbO-r&^~3jR3%%Nta@-ApjS_R+6`*OHCa6@haY-vv>zrZpCzhHPqKY93Rhtf3 zG+wMNSWvH4^~tq}sR$JYG|R&3 zs03ym!$iY4=@5JO4XwJ9w5p_?4J)-t{%6Et^}cj}+bOy8D?!JbOY1z+&6+Z=FFvgV z?Fp*3+Ad*Vb73zf0QHXqOkYYs8SGH!E}q-Zj$=7O$}QbU_F9Vshx6QFj|)54FfW@* zw^!#99}6P3d#Nk92IT?Y_jg`O&yC5FL9!?;LsFUYH>@x?jY4F%-v zZ_r0Ol*g7MOLcn*l2}_w%@rrp0SB0RZjcu?aoRKZgViSIA5BHAT+2}p-I>rihrEU2=^Hd?VpCp-wUx@g3DEU{DDyWRxyw6 z=wp6)8dy{t_SuEMy=5cn_=Mn)q4~?w@Cxfl^rQewqDNw`m9k59;>8oSTD8)+O)t`bhR1y?7z1?;v441_^71$0hL6)u%c#Q7*(9mcQdzwlU+TEWP+!X^4Ow>=q5I!z5g6;MD zm?H}e)Z*)DjT*Kp;FPwdqv(j*n5-)FNdEijuQ66*j#F}E2|gXp z(!lqFS2@mKT@bLHbYBwThdf!n7ZEhi$&xZ!7MB&V*-M7(Qruz^uh@6;NqX`Wg7|1m z7Qb7btCkA82&CaDbzu^y`CTxoiO(i6WvqN*cya9Z!2+?j*dhp+IE`dEQ-as?-&H)X0PI~=`P3KjasPiHMMX5Do>{~;OeGKKXJj# zIc3CBiW>Z4Hjsixb(*xMwXzrEY2SP5P0nhAvZN+_Cs0+yp< zE`x6+?UlPA-Ag^lx#so?izNz4Y^T06_aP1v=W)Td{Zmj8UbU&8CaJB()gSb}=$mss zvuxe&v>j6XpfZ?SkgABcZTS$62>*6x`cS} z@F-aU{40}a<(imrC#y+c8Ez|xL7EZpGH(~sk5XT|{6ra3o*LpK_c!x{3F zC>(ZFz;j`cpU1SKS}Q$mZfI~D^L+B=@e5+A5H$>)k&$oc${)>NPpo{gqm=Bq9>GB} zkRsMxQL(jN6blOt3y<-c)&fRTg(s&dNaw2Y`_9R^Jh9aC66@ZR0?X)E?A=vPOO6Y8 z5O`x&eJTy6yEM_ZSYj3@F8D*gDkvzZ$c7eUuzlyM9=wC%ohD1%gA5jwW?{ZRe+<|Lz4q zouwCd?Ku{Y#inb{BywrC42N zZ9*(`?|C`vepKJ?U-s9#3L*8dKO;@96hP%xZK6}Wv)Gd-AVsA~T|+#cK_i=%w)v9_ zo?h*t$rZ(BOc;q3S_J3QCFE@{h@<56Z}YX_jRC^#-cc@%9$jqviroNbIOd>-*2v@G z%45s>Y5o^%*q_K+`KFfau8HhCsF27>qZaR^W@o+lg}hH?hrR=>@cA#(Y3IiISnRj? zoD!zc!cUm|#$XX;c%|4!k{02c-IQ0iyhj@PDB`Q*G zj($W>wYPUDHty4=ZG4rrbOhF)uH}VYhLJCFg}MqNqw4jmBm5LSDe5y^@@AArZ;rga z#gD9#O-x@%dck&1h2reba+Nl;^CrpG5doZnhL7`no3c@ zoQlx(kdYuh&y+4qL4j1XnQcY#j9#(eT!~FxRaF##%IZ(ABEKuLHb~ZZQ@K9U4T{r$ zHsCYCw%{i4{VOw9&T*hyFG+P?3Jt?xDvwaWm%@Q-!i z1UMA)*?+{LW>0&*@yf}Cl1T0=u(L5@6N2;ou-HXkkBF-J;oPC^?m)ligHu{+kdsrh zq}_I16vg5il$*gX`M^Y1yusJ^b>kSHj?b`2Q$;_-D=H1@^EFZdGV*@XfufdQh(pDC zgTB@X<4Xv+$ZwpUY?R4bPtDnuz3wW&F@L>OKx&FalN`Uz9Jn@qXoe zXWWy8f-r;IojnQDQ4IAO3UHQ+<+e4;C2>vq{xeE#Bpp_q*1&X&l~eVL3)JFs=1}9{ ziTSA^1UhK;*VGBcTAnEhg!&{iL|5jA6|rmf%EgGbV0Fom`6HKs6lW!|n6=Da4!zcG zdYgF@wNs09G)hvPssE0N-ht5j`Ph<(?jaZJL`uBZ;Z%}rS7z_Zk#-?i*7wf^A*1yh znO}(u9f*#iMtbU0Fvqvy=P&W%6$$wGFBnX0QFoORTfH^?O|59xIf3Y`A zNP`m0?RD3kd-t+>lZ7c0t9-W2l!jQ-1GKH60A(HiY1vMwPXq4Je0|xGHhVYj%a@Fk zEx3k$p&o_G`3L@7d*v9qfE`(-n90rf?xmrrT2D!RR`39~>))VfLOzXaz1EvIZ1>3R{qkz0q#m^xqpdVC$Pa$C9j%8I_KO{AZe?stL1&52?I~}(PhWm zYn2yXUKJjmo+<&TH7|6e_*Ttp@=J00SD)CQI>>tN$B!THnv;@}8jO~h^1PG>S7o&R zA>c`PYvvs$1v39z)y;DQramY=->U$X!~1nlga)+3)wINUCHlCylw@XQNow8!7u@b% z0SNgomJ+pOA&24oK`O-hXlZGFJ{zg-Wv5=s-|Y!*RQGu;-fwvO&v}M6e}`{3PQLq_ zrsKamak#HCDSZBdSbH4Mw}b-EWhaFuIPhAEGl14T2DkjfUk_f7 z#pi~_`ryBa2ckoWC2$2V=AVaY1_uW{;3$t;6KXN9{Jic<7BDOQvxII8*4EbS;^M}+ z_8g+3wJr9EiHTjEo$TI8lbia=%F1)L%ZrO_{QTuRer9HALqkJh0j}EG+R-Zo1qEQh z5B>ane6h*N{t69wLBWREd3lsno_k)9^$|(SB$P`+Q&Vta6Q_vCi;!0-yr?<1j{g4A zom!-W-piLSqy3&feHtb~N<B?C$PXX>4qi&-L^7?{06;Q$+xJ zXZQq3!>v>{?nH3&Cfi;b;$1Zkvrc(y`t@n*b-u6$yiufIVB}csi>$vTv@11v$e8flay@e*pKUzxICxKVP@~&Kq1TpIHR7YV{dQ&;k%=eQPSStUWMySV|mEJ%KYXgm$Y=# zpI>DUPbDz5O-)T;?qaK*dZpE@i7m)!N49Qf_vlV`Fs}{oD~6 zXTjBzrW-4`qa(1^8bE&mH5^2iU=Y1(7ZhW^MsPz0c4Dnz&`C($kfST-zAD8I2E#>^ zBqt}wGlr!&AmX|yrK7D4gsQNjLLetQn=E86Pm1T}=ED4Zm;}#FjK-!PlmqDI?CeWG z4kt77GX@5R5B(Z}WWtin%!@HZs4JiZ7*!q!Y{?ZcMKS0>ek7eFY@t1Bee9>@hYLyX zjWH_3*RNmWzJB#=TM>B>X(1QU7D1DK2TFDtEW2Iu`t|ESbf11LwE%$yhQ%G^m?Ds| zcb`8K<>cg$pR~VwD^1p@|(kv^D2>qYO8z?9!e3JYgMK5jUOEg|?8)ac( zK@i27hn@Bkd0_iMgW#{8j~{O$OH7TCg$4%2MwMiD?r1IVY6I>6d+A=^y?Zwn7-jJG zwVkaNtEN z9aO&)#h4e8cpXvgJedpnfBzmN=+rkFi;s_AXw#bnGi!iG@02M-P{Xvc6y_WO8KoVq z*Fi$cdug3Hp6lz!&o3-g0?)CDf*XXA&~gL>tqw43PF8CGEA(OC-_MU-KtLC)E?{ar z8d;bZNh=P!92xv{7%3|F^5r(N)ZG01{G85&4=>^C+gCK`(Nc5Rv8SgeWMhn%g@uJ( zN~&>Z%wT7x(s7i7on6V@-CbElg)<->=pk4wz}lP;Vj2p%6KWx+7f1DAa6Y1}G756w zL+)H!oY=R(H=UoITvFNE+R82@r2q5h&#>@t#G9-56~PzhjB}Qi4oKZRby_e#4Fgk1 z9O#$M5rd$~*5thyT<5Rg#kJ14u@UpTp=B23Z7X!^d!Mno3!Z?Ob@|3cN<0RDt_K8}NhC zVB*VjEWjNnBqSKT>Ua_JHu;3?L6gt+bn#5kiJIqf&){H|BFoZ@T|N*Q@PyXfF5GW_ zIqdW2$C#ycVEq4FHdUIGMLrPCEH*ImZUyRZKQyc*TB;d1ULM!I=JcjK4{rrxO=aqJYPq~aL zG_9<%jjEhFBmD93c5VX!{UzVO~gcctT;EoFP9UMga z#XOenty=H`kv5r!=LbGWshNJb(`6GH8%m_KdfEA0Q&@K?`YK0p1S-t@psY>e>{HN$ zhh{;oi(#^4G1Ayf-?;A?k4NpqK`<@SUYqI67D0s4+AqS-=P3Tin1n#FTc#phJUkl4 z)h@^+K}RwWhPpN$E0i|kV5$h+GaQESYIlh#oP&cynSu~s0S~V$9Pt!%h?d*-({a7{ zArnr?GXfy&Sm3opM#ju9a0b9$XaUP(psD#KPo-G9XZQ?i*`;6H!zXE~yk@03y7_7_ zDfrAbEL~hXVg5SSZ}JhkAedf1K{#D^8}>k5pOOEZ5G8#i+NJC!tCglo1f(ku&TJkCL*I68pw)dO&`Hd5|H{EFC+${1W(< z7*KeXK-_`oI&lyPdlHEd0^MhvHb7-#vcYZ&{TH%$ue+n{lZ@cr#f@i&#(^iB)t4#( zf{Nh=e;>>i${47EHV`9fE3uT@0iEKh`E}Inn_( z8C-8>esNxV!z9-aiu=p}6LpF#xxe2KQZ;0 zi_wY)D4!X2US6$m3NGy^I!OTtTDQ`5#>~LXFQ+YvzO=y~3#~H&dE?{0yI1fir!wWk zy%r*bVSq~9(L$8q%U^iTV?Sj5v0F&-?fNV*i7p#>iXwx%BeFsijoUFmd(69|@Zpe( zZ;o^(?oIYBS;zZRV*4Lca&@H7{cYw>>YD21#k&Koog*8Y;7N}|7#3QZ3rCr0AFu86 z`Ph$?%ORIw)C326+A_VI$dZ|NNj1JrWdxMYe>ciqNL<+U_QjX2pmM(a+5S;5EvK?1 zQ5;$8bE9(}WlT@oH8fp&G4w0Hj^I{N2`4;a$lvj*)DeIVTqK(XhqfAdfL6h z5gDz(ZG;GgUN`mcpnA`fmRB6Ry{$~v*km!D@Lhl?pN2{-!U91&i*dFXw zAUwRj7%AXnqn(&gT(n+P;cOBQiU050;6fXYMT)z{A!q+dE1Y4R)Js`%`|vX7uQ^2w zwvk&mf2@TK(%a7u*borI^2`#(Pp_i$yBGE$N5hBStPW>+mV)^@_w-12 z;;Vz-m!PGTOV7xBCW8E8id)stlLNyeiTo8TrSB8D?}30E=);izc0*p^$)P*czcV*^XX-}WmnA-s78+uI zv)F9eLjuGZ2L$S&NjzsOX64Z+wmrJi#tYu4IyFR3nmx%O_qivrvFU!Fp<>Tly;T$8 zJ|~y*#9jp{yXpHj>|&)^Imi7FQET%m#+V-at7mG>>o@innp*`@)&xc)99<@gh0KZA$4XRULo(JN5S3 zj-=ZW8Pp$d&LkyCwN`#OBiJu7+_l*a{i?b@DK4khJlGlF=~GK8k|2ULe@VUeb(W&aGdzYP?@ET@KhNjdox2XJ=%~9*V(ObUUn`yfdQ>6W) z^h%;#N>ksdu<~7VQzdH%;gk(7{RwN%GAVDn=J>~kj7cziJ9qvufI{0s1uW=b%1w zBrH~9OR7<$xrPhFlDhC@O&R!1~l3rf`)mn^x=+I(gi_)cec|^>VU|N$m@P8U;aMd zC|`bLf)trqZUb5zyKAnh-yrJJhTeSb5b@31gGW%f=;q=pw)-w~xP-_Gq za3YJeFLa`->%V^^((zNAha6O7bpK~PNt$iPkBz5gTif8)pYjVn-tDU@>dh3aZ&2Hv zQAN!E31HMn&JE5f-r^LndN%qsAtb?aT)DEZ>$~oH(XdCQ%Wp!#j~Ymr8+D@IA74QY`0Fgfup@%P8Ayx#|2t+-ijS+B|I@lt|4`}HYh%LZxwRS zHNub$$&8yWw30CotA`E{6|Ru62_p^l^i1ASVZE9tHK8WzI*@LvJwO=6pAljfs^&}P zsyH%j-WSCUuyq2YLVlO}D1r8%n;&1! z4N|(t%}*-4H;w+@Si$nWY;0m}u$q2kPpcHB7|w!5Hl!K}!nF-5w{q73a`+`|mCezy zXfjj0v-EvJ>g4nYmudPyS@zxeXoo8Vz+%v zT#P~Q4}B)1E5@a;Z&c*qeCl18_9-xa+UhA2TL^6d-Rp9)G*^C;zlkXx1i!Uf6q6l{mfnGbsEd*Z9a7Z;bP zcf1f{#)!jimpYczDgdv81iMKp0**b#H(L+s@xkU(MX>o)CMQwUGq=iVJQL*Fcht@H z9>d%b$wrmIQ4~$mM#2IZy{A1Z)cz{j)DWn4c%+_;)vJO6F+mkENax59c6vB0Rt6Gq zM0mER_;$$C^f-ZR<-7o3CsLbHXH7B6qDxu2mAK1Rj%C8@u?NTOS{%baBn8i;f$P#}FMK;e2bj zc(_455V2rpqW@ElFV4i@ipMDBY-TIpljC_U8rS z1YxjRo#@l;+ANM+3BLHUevruZ@11!=*Nj8t^;K>Sh9+i9`roy_ae&Y%J(u?Pp`+Am zxNtueZ07D@ym_;R!R4jCQPQW0w_MY1#A=Anx}42n=bw&{g(~0T$>MyMoKY9i_yQ|# z-6rKi|Eq%C!^WA3@UolSvu`&%gM!(<*6=x9JqnVd)8PYY>Wb6ManDLX|8e9>SN@1E zoL0kAS@=BRU~0nL3q4R)t1wU!L1pYBw1TtK+0ks9nQfmdm12aLGhC#e-10X$;oFKY z89N~yu^I2H?4$lb@vMTd#ia3*o})U|Pxuk5elh2DtlLD$$VG!ga!*x$X&|bS)>O>z zm5b;Qv>2N$jZPe_ z!8*%oA$bNGJWjRAN0`q{xw>trFouPNTO6k$we$hMLRCC=?u(?oCt)>TFplb8_S!P> zKC7&FlrA525k1<7>ia&#M^GL#dv>zDU@q=dQL}gBM(UhYdi(Os=Y-DposI(ZqpdZM z)vS$&jfsLxgbzutkq31xWL@;&+c!lUyio8Pl=ZeQdf7Rt;nuCz|HZqFE|$fqZ%nz? zn5cnUOy`PDUrv7hN06x8rJ`Ef(0umnS&G7S;`&J&YD;eEha-CdlZ0SMeGKo0 zT(_A1EVX!Uo%#B$e``C|IHefacexajf$e54k$ zjB_+q{4qzI%SDt$aCk_4sG?&V5}TE{l?JrD6v85!#YzeTOoH- z^59(Mb3cbNrwXgtS8Uv$mU4`GBUx+jgmk~SoXnoYc6I13w51R0yT|qXj=;j))#t=ub z)s+uSPC_(Tln=d7as(*&iHbkCbLUQ;)(r+mM$97#i4;%-+5e4C{{#RPpeCF`8#7GV zq*cG$`ZzZ-D$79krqq^;Z}rx*^f^-g{mFj6%oVB|K~IkPH{!_5l5VT$I=W6d_#GE+ z&n6ttXSIK75+R}&HmEmJ^gL^(-$P0-+=BHMIhHwnyG0`wBFjZ^ghP1s8D<_(|0=$< z$s73~x8p*gtsOsg1ixfDzL?i-vu801@nUE3+d={-96m=rkY=_^%DXOYzNeZ_Ve6|y zb#byA96R@Pu|@n-VWGOen$@KQF1so88MP{}fBj5LruHqJb&KmSdwN9r9n!|OOrCVh zjFUB2&Y2z2i|XcQ2fbNq1q>pEpqiczV5BD?Dk;+EeD&&8ABobXf~qQT)m1PvGvl@6 z%ig|oM>IK((}2%mm<=$Ir+()zpDA~IoASPBB~PD^Irrea=zT}SA4^vijzq=NY_bC+ z`w!|;RH^#B&OtqGC|XNDo-F3vu38L!kv;s3O1eojLY`=8 zyx`rAx3@^dmsj@z`owQP#5_@9-z&{h6ey*klau`UEMG7@ohsnDuxwycf@_R z7U#ox+AMy$O*&WKxqp9k<2D!)t!p}r4;eB9W`&WB?V}={-}}oCW_^_3by@<%<$nNk zpT2zgPj}=Taosz2q94g!+DCicI|Wb!TmD?UGiIc1;Ig1cD0qE*6^yGUtQM z$pooD+~q6ZzI~fQTlev!fZWO9rX}!2ryvYJ%)A5@b`B1*!T&6Th+ZlMFc?HdQJ@%H z-b@Z&6mvPdXaxZKh5vqs&+ohQ60fVP>jkKRo`udU(g(Jb_e&AV=<1Sz7-}g-g?lbD zoSX5{BZY(zu$Xe73@xXiaHHvTj}V}(w&0~slU2kbL4Or%q)SO$obDfP(+CR-vnoDj zWE_&nmtkjQR0~N?rUAvZ@8FKZ6_N)cFSiHF0AI}fmdHV2{PhA;D0X%59Vm?Y5AK1N z4dGp9WTdPCWvzX{?k~~NzX4=0_T$x6fX|VUk>TB@pilrSNd+*GhU&&fao}UnQ3SEz zec9J<+~Bk7xE(@7Wf8nJ`}^G?eOlaw`Hz=Zj*pM=-iL$?Whs&Z{CI7umOoVt!pOl9 zc3H3${`~Pdzm~p!8z@(gRXNjuDkrF8aXC%ZL=)2peMwFxyxhsWI+%qo>3_I5V(iND z=urrrBy<3#$H}V5!^r6IO6v{Kx1>}61@PJpJiI|nt-EwwR8+(v-wlL-AK0S9?HSV7 zg@v2|(qZD@=rXI2IKD_rOY3_9thQ+Q{Jce#3(A4#Y1-+9DcoC2OY4e^rsi$%{Y+7j zAIu1Puqq4E^P7Z(o_luz0D1Fo&QgBa*>U^R++4d=w+@pq*VJ^Ih?)ej8PJ2L-rhoZ zkG_5aw*qkF8Qgo*ZnT6Jl+ali8Q%hE0RgIo0fB+3KVL91_KwUohXM^H1$S8h&RW^n zl=)*_yf-IWtDGl0y1M>6A)582G56D6gB6y*JYqWg`};dpfDvI_$^0wl<42R7)3dWR z6kHe-_M|o|K_y#{-OOdODgyB92S`wW&@=;~+(6)tkM-DJC;;u%?>X4nvFnh(cgMa0 z;NCDYT7XZe+1S{`yeg~OB2NuxE*0vlV24&^tot>SpfIa}Ri~!2xy*=Fv(2h-nKNNZw2oA7~8yBQEZ;EWz z0(7Rw##4?f_v7e*$U(?frCp-a?J?12YH-$poA|hgb{P;7#E&wTy z{FCofQ7PrAr|w{}K6^2M!OsK#Uj1we2}wy105bFO?G+(5Q1ApW`!8{E^7{G|etv$7 zi;F)1sMUCJ?te*ScXw7r#qc?ukB`sWckgZi$d8%1#jL^!RVN5Q+zJ=e#QAnc#zPdn z&NH+cFbDV~4_*S;dtCkV8W1a5g zY_8hakjGCUF8tW9auvjcewJN57vGcz;z%kt^Y`ueZ2u^2$&hfa4kH{(EO?CtHn zHoswrNp*@D1eYa$m>48U}=umEX(Cij76nUpgmuX^k zRT72^xz!y(b907QCDth$YBN*Q#JweB8L;6wA`uPu5C;th2L}-tbo9*5!w&i1JhBsy z+1Yn}KZfNpMz-zk?@uf(`IMG&LsHY;-v0NGmp34XF}t|v9TP*c zva(|DxzJ1)-7S8v_QkQv~Y{myv*J{r&(E~^!Y| zvzoPqOy#keZ@>*HD8?HDNp~L3f@0vXoW2UFenKwmPhDLy@Ho`V7|3rTvyuew~piJEuV;7(ko=ih8IUUZ8yz_-;GWUprV% zA)`nYMi>?vDv!54s-J5CdGfd=ffEfb87E!dh=*A6tWjO1A5FiXFtLb;yD5T0iz*z`&R0<`vuA{S z*_1bR=6FWosP^(e_Re{Q=N5Up<4x_`LxXZV4A!+xAIlfH@{{I^)wwn3yAF05Ndu(h z*T)lgZX5clZzuApW3>jzDX=gK-3^0IlC7>Y&$+*KQaV8;#Y~YSlD`q}={P-he zqyxjk1XoAP9af6xie|p3UcGkBLd7Hrds2cfQKV*lW8>k22X8-oxCjd^EH19s9>wX4 z$9x%RGl-rBx&)wN`$`dY- za)Yh;PG16cf4WyC9Ed}BCs4n^`2=E?7ccNVL6b6A&5FT87k+=g(7|^|0|2}xa2J+` z@_nYKb+O13`rs{uMMShg?9-KVJrJKxr~W3mqatErH|Z<(9&&PWwxi1;LN+Lc=)V{E z`H-U4%2)9U2uOlBglFsb^A8_CX6rWH-B}s&={0&xN7v5Rl`HgW5llLmQ<#dNFJA;q zOlZ0j#Yw?`7#L6r3JxBubgRs*L2Qg+*O?Lz6iAAX|IpJTM}vI>cr;>PS+2PQ%$997 z#)nO#^u*cuNLerdJH;5R0?>JwV3a@>LKETxsM4X%DfnQq--3vqcX|NC}n2=C6 zXbzz?8OrklGiDC)W$@RpA|QmHyu2PzQhI^=$H^^*h?v0Bvw2Nn2gewKN5GQBz-Iz6 zddS2Slpq?GV>Kt?^h+k5GmSz&p^K?`2`qq~2Xqm(G>@3V5$Xi3Nd|~yM8$N?3WNd6-K!Cv) zQZ6vTgZJXu?vVvCWQ2eO0(U`wLJQkuWO8qb{2gk(xA*lg^cwzk+7BK*`T!yRXIN*L zh^Ol6#6U@b(0&|@+OpJ3d_;Z0Xn(3?OwY`$cM6hBEG#quq4Xsf*7f!E`1p7!6_rJ} z66SdE+nHUJAq5l#^67UmF;V4}Y;XQFT6ORWg69T82XgEMMjj+GP@oOm22cXPi5x7o zqi5D{jfF^%QK$YkKE5{$4Bj0^Y2Zl(fnoPm0pZu-c)NdL zw46ImFhKuLk%jd_SKZ)uL_PL+)2(B}1DJskg~n{q{s`6vT$qjJ+>Wj!NfMx;E|Z$4 zaZq+K{M9v+g=4NdH6FKa5^;s#G3)=%H&z7r!G66C2egHSZ8)-QtY?wIrvc#pH1aju zzJ5)C!GFX6TZ=|Hr`FZ140MPXS#a)itYS@|;V=L|0hB5sQBhKOwqbW72&opyV`p%A z|IuATXDX(+Dl{bI^VhG)WP?u*gn@HwE2XPJVPOsLF5UP90@l{n))eU7*wm`e4AC~! zRlEl=#imym)75VXK&9QBk3ka+ktI-bt*v6fHWC&SdqhFeIH=_@67nL9?iCW3iiXkYZA>?rjFv^fBNN3Uo1K-NsW18d^jO4z z1dq;j@2SK(%xnF$bT9G6Ko?R{Qi6u|1bu}`L?r&~*?CMdu~=B0c63QtkguOhk}*=R zfN>RK4vdKj=rz&h(!V0ggS^Mcup2=X>-~^&U znG?<|iMZIymzP??=o_@NZxa%>0$c{{-Z|h+;XouApO_f*`LmQkoiGChul0xPJXcG~ zRB63~A9jTo7Zu&c!SMtWPEQ{IEb2SAZ&&?i#TOqTjR8J+AX$}uZnou&2>3bS>}JGp z`>mCc$hjD|XecSTOGv1;8?(Q3US>r}#IMf*3}{IYWLhfV(+2oR2j z(kHuMf@GxQ8DPS05Z9P_f(`Bh5x4DaBBC}xJ-hcV=TN+AXg)N^K~4M1Cmaq|E)wjV zuJNOntr7OCp_cZX{hrw#ADht}F5lnLI!0gC8}lw-K3qpzIo%~M*^9Ctwf*#szM^X5 z@nFcXS+H{bcN@cW*FDAS(UOy?X~$Bhl6M*)-N1@a|=Otx4)W zt(m~|lW96&sc_H_USCof&AZRWDEos2;8XJ zqy5bYZaX@}Nd^<535xuWMFY`ao7H#AihuB1#=ryK|BHs);Q&K-1Bb5GC_e{;UnSqD4w$8T z{go##!QwDR;5$^}?usrjC-|PWskd(6m2xHhFS!;KJqGai?=@UkDT z`|rl@!rkog5i zqb?MO44A8rnGY>|>5@J*Y_!AMiuHn3X2`m<3o*`duUUQ3meAYHG+S@PK z^bqH0EWTKmgt49H>B~z74l{q$9xm*!ZBO$=Hg{!5UR}i!*7TG*Dzzq^>|(PYSZ#yV zdbg zrHsP6_wG$EOA+{sUM0%%Nj=7(%Qpl3?%Y`*5`PX02PfBLkO`EJQjz%sP`fvYdBW|+ zT*43rI|_wbzHJ=qySyJ0C_DeMF?@KQ{ovudXR4EnM(;NK;sf>w2#k5mFljRh8$LZ| z&v84}-!W~Od?LVA*=`@u<^LS_Xc70}k+|jngNeJACawYX$1gh6{4JsnWyt?}2M~p9 zz$RGlRn!O!3aW+(xd|o=m&cv~jG%Z?xt04M$!e_C%~v{1TIVHcI1xy7Zp_`Ue2B<7qrB z>(8y%ji&-#|#dmVzoJacBj0C~+2;)HQe`u*9I)3|Yy%E>mqy@Ks} zxjkkA98`S#uSRCP%PB)G@9E#D^LXkY-Q-2Gqy9q8jxxG# zY0q`>ysK(u*pDCAO&$w!<{Y!z(nWRT@Y6mLu!8H$3quF6hoxTOj*;^ z_rH+VH=z=T*db^=>3NSJqv)o9^Iy7wjbsdg81ig)Wdw1$rt1BS`QnD6>w`aiSDZX- z>tA+6aoU#KP8tT^pfxKBlz1GIyY0Jb71#FpWY6WmZP{^#a@6Tk)eE0MSzP{vhV|TJ z0lbd{3ZJuWFY$jnu+5UNt{|JN9aO-KR`k;uC$ZhwPD*jkCEL!EWWH2)XXKMH$wM6P zzusDHym0V9xf46kXSD%4M>Yjux=h-&SKpMP*r4EHW?3rD5+yX;;xf49wxnz^T%L|r z0z^JFfFBfrXbFpo>cHy*2*WxxIc>4xQorswXZFKo4(*THEAv}dlTZUr#S81A z9h{bhOLgp4$K4hm1oy_Z`D#A!g<(cIlb63)ek%2W1Q6z;%O7s@%nMvD-^L9S_{MVo z1E;&cBpKsTgw~7g>kY#Vw{nv(Xh}QqbgeD{`oRaH~xSsEhj+`O71JXA- zr0`e)kNsoaEe^_z0kSY^YAivipM=EaMhn^h2SODAs6pW$%Nu|;x1&qLeI#UXwx4bz zX~Yyj2p@uigV=-|;#P?D2_z*YXFFm#M$4T5>!@vLfXM2h%#Bdex8&sH!4TBu=(T); zC=s|{NFE=&NB#DwCO}-H&TyxmJ7*z4`}ZF|-a#f)CtK+%I=VjKalkrZHtwSVOa;)e z#}6Kyv$NaX>=1kLf)Ik?XKHGM;JJYB3ndOOl$3BWc-GK<{rUxu4-N`?W@wlKp&0ms zDrcH``W{#&igdqznew`}BSRWuRfmwi<97g&cEZj=@t~yvbfPT)Leu=AB zuU2VigP#NWrpc)QelnO~fQAvc2TZe&kWegC|AdR>6ShM72*P$mej6{DLK#Nc-lUK9 z>gYGf$gPb!t==Ia>455#CP4b2BBV|jzz!OiMsO=2>KZJvAVt(BVj&?0x({dvd9?jI zC=cO214cNnAd@@&4wgY2a`F&CS}zX-R-PWO-X!Gsj6g0ce}v(SFm=b2Gwkb%4Llhv zI3##mAw0lk*1rkU43G(w>OKc32=dt0X92P0^}lzPl9JLS(JO`ID$iys^9uoi9KhWU z%Uc2Bv!RRyzYTs^1E5Y*YwIIgXk$~vakAgx)|Dblen>F^7>(80-QJz$JAJ`^Lg0x6 z!|;ax6%2=l>d`Belq`N}1@9nq(4+w85i>LMKR_GL{&aAj=`f@!kRTXPDw{Ky1&0P1 zUc7Qja`Js%-fvC88T&*}PT-u&ywG zD*!Bz0R)nh!$d9!5XisMq90>i54>Rh!Gp&APhl_v!*2+K!B?G_oAdmOq5vSL?u23m zbO-_606GVJ1Ob1va1qf1B(s~<6o#Rzr>pxJEH@M}Gy-mhfE&T&lCzMaO#pW4nP7eW z^na&QK?)&ndG=lp+k(y}HXhh>o|1g~9{`C#!d^gA0Q7$Za5L;i15a`Y1=PTi7J?X_ zk+IYD;qrS7zHuN|kaKZ;1$YQ*93-JE>1;OaN~U6`FM*H84mh z3)t7+P*xcv^-hWvBO@c|-E5mBg}!v<=dfv4%Qn%!aof%QgSnjruYkFAO`l~#{bV3F z2+=fbV+3)7EP)@;3^wOF3}8nBum?MzA#z(Dto?QtBv>DTf(Z?+`;Q6+vaa6%G591D z9Is&m4|omPp8>%^^#7q;AOQq0#b4!OcZPG}Q2aaEmi@Z(VRd!2{rKA}fYxUR+;xTg z(b;ufx@ReC38%O>uigu>lqjf3l6RfWRtp`9**I$yuJ-pv&fGZx|6yyRN2qa7#Vt~3#$TebfCQM0rv25)u-Aqfkuc4xv8;&VVz5a_P8@hX+h< z(Sg5E<$Wlo3@h>it%>j#;9X_+P7fByXOxUK=3MB+>=Z6W&|2ZR}fL4)h@ z7Kj=oE?b?3-N*mYGuG1!1nLGjX^=RzsYdu1EU$o(a3AyoWEvoe(qf!dE&vP+z(`41 zSnfd*U}L6*@n&4};cWXiAGo`R)=&L5e8-*TCaAOhmUjcP>LYVq_jts@i2rmfeF$sV``x0?~fr`^ksPXeabn^5&@HY4n zt_$D6JSp%l#OK<-L8hw3uscGG>}Jh4)Y(l!4zP9vC>5Pxyg$a$Ext$gn&r9SN;m#~ zbtl@>y#{`Un^V4Z!oZ6FP6^_P!d=8-_+BCE@eVnVw8*=YBn{vn;}a5)Tz6#Tuwl2n zYf>eU-w+!Os+`RlNgVVd>AA*BMk6IQYR>C*=_>i&puKV>6jfA2U}=Cw(V)x3s`&vf zwPXXP45V{{p|e8!IcrF=qQ>@1=wWBEC?H{;ntQu-Ws_ACz`jr$e(QDwthii%b0{?! zX`WTL?IE{EB|F4}}XW{35;}H8QKO?mlNP z_lUxk%%aKb#IAc29tRw-N9&+a4zlv*doGY+gLL`%-w-u0kABj4K?s?OaK}?$Ph!VU zuEz(xhjX7kS%rj#W}f?fxKcLZ1e`RD!pZT`0&T5<24w~ zCzi*z0@r|QCs`bZ{Iy5uISz#lf+GI&vA{>0FReJs9?%T-;7VOt+J#p`S{Fqj=BSWeU+Zc!?%qeJK%!*Bm} zO+@KQX9St~;;t0s(&^BRtj`x`XK`;GPuxmQA!-q)9koh%i7p-1AW%n^_-fW)^f!_7 zDB6CDLx7VYg<{6#nbiFLd&5LCsp3{}&p5UG6Be)Y34zR>rI^tZQ>;!Laz=64S+j}Q zS>0IeIw4`#vc4+-y;oGWv$bDYmdB%+d34fS}Zld^W7iyMFLwfC_2Me|s3o4Ls= z(YlXB_geB!r3M%}S&I^FY1s4a6RsJ}Iw+REo?S|(EZ}aIkDRr|JVzI$ zROEAgI+3HGRvj!L8VuGOs?9DgH@2sz)Ggf(uus3lCQ?xmwnmn{%kiGNJbL&^gqEh9 z$rQ=)$TuRX;O=Sv2UE9WDz~_dT?E(0_t5<+wK9^XikeK>4ZlVm@#-33bsT>|Uv@kN zwre*J^vcWEDNd<#dXwMyAS8|`XyrE2{ijH4}>G}6HKr-^i z2vg_&WS&(|Y$@xb7@uwE*zh?pxOzBdnbdyIK6BH`mZiOGs~}RSt;lLWLuWz#h(y0E z?CHncniMN>l+qicb#I~W{;eUH?cetb;WCyHwfs)v&>4PSVc*DW!A)X&c3l76RA_u-JC_#&lg$$YgvwX5ywNa<*GKx zaa1`gRczSWAnI+nWQk=N>fSkKIQz~+(r6Gou!)` z1rZ-510EC@Q&bQW=b3bWY%wc|VXqvOpNS?yCwbn>Nk^!R!OG`hqD%j+!!6*_Z97>S z^&}T-F1N*LZ{x5AC7U4$ZJookZ2OXnmZBmRHN(}NrfPEy7UgDzTzx_XFU#||SqI&S zjAWMtU#{4dSvK4+$=gZ%=yv7c)B^jI+cA;%`@tc-RJm=Efa#(|Ca;INPW&&AkV{8@ zK3N71Nsab~Z_o+$NDjN|Ul#1EsQ=8sUKV%S_>PF2hzLE8yqAB4=(}3JWpuNAXd0EG zd}gXz{tAVnsipw!ALq@sdN53&1iq!4w8@bUVO?lHGDn>lOf@hyO!AKL1n`Rqq@& zxsxVKzk2826ps}>XjEfW-}UiL>I)%^`KcYb9&oZUH-JtjFnn0aDnf0b7N|oL(bKlZ z)$M4$^!iT$LqjsY#CD%q{uJxY8VU)vh zX^KmyezbcBt%jl5msK|ImSPxs{e@G`1UtRq)7mUF%y^jSCXdbWV=;Ssl$>S~VOLb(%{hk`>UU`AtiuD?07J7e;EK zmyPVD*D~C!(O*7Y?G`Yh#;M8bh^r6dBzJKqMR`4MC?^d`BaTV1l=m#i)}m14B{qp` z3sy5zHsvL_`aou=Rr&B_9%J01sQEskbAbkHg;!JIhE9^Cq~l3 zSsC#=;?f)znlKvk6lZRUJfm^`qS&~C(QsQO@R!@J-)vghGp4!GXUV1n{3q5j7;lX> z$1zrks`PPj>qH~Q4Y)eCuuNY4#8sV~Gijp#K0Qu`E$08L+=O&MN7ck5!u-r#hMl7@$L`wDwzHLBJUD_&w^MLCkurZRLoi!IPD-8UzWGhKrt^yx|I z-uQZ$lJexAS>M`-{V`n%XHYKE?n;NJvO+TI_x;w?yH5vb^DNv^|d#XBcTk5?| zvI(ZK8(lSFm*1O=c59O&%ci32Etb`Z;DCxWOR@^>sq?d0g?IQYdHN6J_*83-)A5Ik z+5@IMhThmDW}DndjfzGSnRAldJm{#~+i?jG(b%A%;;aUax9)g-$KFYv z>@W1?%3f1Ds70NW=ntn*Yhl`3m~(t3eM{g>Y6wAOoUjBgDW_oE-Sw=@PUzosUY`+F z!%-Rh%Gnjo;rNH!ftNzjCdyKNCKU`%3Orm2R7EePk?cr%2XRPc8Ux5f{=}Q>3^XbJ z(HeFhi}g;E%a<|*`aLU|{i7|u-kn-?V4|l%YiZUZmfTx^XoN#&9rUGdVh>0FTd0q z6Z_1tiCKKB=E3U4bPd6&d^}HPY+T`~EJivb0!={i6*-?z+77blOr=m4(shW$ESD{% z%WU_(CvBPi%|(4UYG3A)HeqWRv|h;^GsO9Rlju1*A)*dI8#KuHc8-$Opf)P;Dx*>3 zsRKqwag~C-%+Wi{@zV9A)Dokdgd-iPrGApGqP(C3az+-PlK#~Jd8d8x;u$&xF7-}s zYfNI9yuL}UsrWS$359qMX05j3xjIf%<7|_LOcz%-3>pZciwG_R+x5-g zjm%laQRloEt@9dN#OuMXx!SSOJO;e!d!ap>5zc7LrcS$LTk$8wiSAO*c&a#TLv$|h z#jJi9XrkKr>9EA;yRpsVl!DW79{2L)%O2-EwA@p$lsU1gC`-3E`?GY`zvz{T9Jx2_ znVl$0M23cSzBXGmkKP{N%aOMtC{=H-$s*ersa;U(ol?ytvgYJ1wfnyG_yQi!PwI1Y ztjf4o{BNAf@{N}rKELJvJxJ%CJZ?A-?>fh@tLchRbw)OiiyIY*i_ND%B*6WUo?%3ALD=c5EpoQWfr-}(5wMfH;m zIyVZmHeC?q=!#ZT3uq{>2}xt)YL*X>&0Iu7wp3_Zw5V*WJOS$Q==Q`e!F2SJ7TMZP z$LpUGKWEuqD+tIEPi>fvZ3Iq15w`k};AYf};~_ z@Yf(7mYf`}CNA)83Ec)v222`C*+>)x79!|wag8Agt?p=UJ4~2+x&_P2N-6tsKwJV= zg9iPv$KgZ#{qNT8+fu=LX)A1Om|W9(3M)hP{bqk^Y(`W*7Y;Q~tutCLvVWX8G#-%S zvR50B*Ksr7!`*oOHq7T*Tv7OBE}ehGr@*P=dhVExIoI~57eo}=bSBW5!+Z%E>ULIp zRhA8Uhx&=68yXmUoJ)IC>T_rplSWedPE;e=^64Vbx)+?@OUHd%(%tS^iTY8LjLR_5 zTiw!hXg!rsmi8U|7>>RIf<3DlS&bamUoVFAT^gskPwKt|yP>#-qI-rdRw-su{roL^ zF()8m;6vuMBlju@dHyff4YSl6?oGZv`zEELHy#dM(xw}$O=V5;#aB4OOzR7M=fq+9 zd};lr#B+v7#z-;T87#S|1tn`!!7vH+a8;;pbB#Q9eIQR*$*XI5N}k^fB+Q3I7wGHR~&J^AsO#_6cU3mzL$`R4`c83l^Cj zK3Xi;=oBqUy3P1G7#;;#a*rBoOr^M)eXpJSz@1)N z<}hx0{5cj4Q_mQCPxJp+^J~`pBhuxTj&TgP3l{dyu2L~ohGlx)(4qJ(8!WVLK)MB0$y7jF# zgz0J1hhE~~+9&nlQ*Yvn^sA0*F#0`G7D$Bi zW@Mqa`-bWYnJaJqw>j|)TqeHg*lM%z!(UgNiymDGcsnILZ@yuJxm`pXl{q_%8aRqv zN2+lBW~WN#^?T*OX|I#oWXo?a_(zm%oTs)V(KixZ9 zYau>F1Pm(9Q=;Sa5cIu5J9?{`L-QF+nr(C0lLn!?U?v`s{~>i3Bebm(ZvqgPKn zagvE?aB%^>!2z<jOKREeOn&HjilI{&c1nx^}V0$1tsRBo;M#gYhy2(_!F2Am1`*XF?3 zKKPP>@2j0tar)VSTwOH4oS#Q+WZiVwbNHk@unW5!&(2EBZw0>#*WYI>|KAIcotl4P z;^HZ^vI_^~n#W*RObN=^Z1;^7B=1fY;Tx5Vn2^)SGZmNg4!+WlDyB`%s?bh$h*Yr~ zt`e#4yNld7+%|z;SabjUb{6(eHzS)+znnknu2Xs^=tDzmCscg%?)frQS8cE>y z@3!n)eY$Kq%u7Ejy+r+IzVIVnq2z2FFB_q;Bq28vSz(2w^?kPNO_fDrtHs+hsiE)> zeuEVO%^P%s>2&$@JiexZr=1`69VSN~e%@g`##N>Ac-x|E}G_vXOd+6T94wL{TumIxdsWIJuq6>WO$5s_aN$y&fkw zoQuz2UHZXZQR&1;38sq&e`~85|D0De>k(^Lvi}Wct>+C4$uG}oxP5Ks?B>fl_vhS*pn=tF=aZz z>~5&xT<`Nm>Bsr$clq7hWzLA{P1o|h_pO{(u5&zdQJkBRa}KU9 z+tt~fj&6$Y-k41-NLCR078J!kNy#mtUi9<-y4xlqzA9Bb<|s}U43*N!UYO(xR?hjG zCunGr)wxD~6Z^?4v9xr_(FXDw3m(OlNZ=&K!YVVw{ zf8q&)T|`39k%?(kZ~oBi*3+5a*gX)ti9vuIsJANrv_ zp`5S3<}7Ci*Cp|%vHh{V;BM5b^nUI*_Q*k?;au$OUUCETaOk0Lrv)@{mgY%2za3-1 zpB5Br^4gi4g0E0O^y-hY*U-syk)3ae+FwTj?u6#9QhVDBnwugYj|a#}#mJ|M;SuzS zJ^SE#TzvfZ|IlhaLkp*~k0$P2q>nT5x%vDZbc#kk?pi|7?ELJ*aPu1U3O)P$zq#q^ z(2x}LlM^G8H8n*?LyLXtMkVsD@+?qH~!1Q*$(M{=r9(A-O!pt&T}FF^yq zz?umuBcoI(9K*uHqfF~f1^WK{!BQ4Gml({jA5`4Do@ow+3p>5BsI6E2Z~{>rS_-!q z z50{7x8k5&7SQ|mw!jgj!c4-&90@801N?xExp&vAr1?q@^=QXq^*LnjghG^i z5*N@|l%V^XG&Fez(#95ad1{%lOaTH$-Sfi3tpy$HPcQ=n&&$rC*{DlM_$y?oX8;F+ z_Em`-l*A#8vss+?%m-1nNXIZ}HN%KZt&tcs?}qH@Bk02o)MThzC13g!p&WK@y=3c4^wSw71gjbW)oK z?k(VX6>Uuu3V!89M7#;nJt0w;05z#lzn^CBKMm>XMqMU6DRz+_?HwV*ECa^>l6<6LQ*%b2Kx9({18G%i~VH`30bda+>ooJkx;1E z?o-{^-R*})Ot5G+(8qxiik#;^gVa<3!2~*Ua={wVD(Ad_%KpVa#knSIk?PLtFKsxa;$7qrhQ*A3+mev{426>F??h;?tLW&>f&6su|FVf+K$k3ZL^3IofHz<|l!xpX zr2i-s#$mx~!G93JfQAN8`o;=^`GmE(y}b{*ngZVe>K+~uaaz8B{xu5F%N|*CU)=9d zvv;=U#D1^{Xd%JSq&r<94gNqfHx2TQ*cg{8p^R|ave^3u3P=~wHopOR%TRN}w~Ipc^rQZWD&gad4Y{_fM@F zC>1ua4o7pBfl^RIDDp$J2VA%X`QKEikpx0`G7JT1nmvi)1ZQh8s{ILOuebHD%Bw4< zE;JjXF0HMOX5eZsD^CjgC5tpF@j%@cnE? ze2sF^@R^djQ-IxrjWIF`O{F2@4II1&ppGE14!yX0kokY|xbr<|01W6hkp?dbD>Ia% zC%P`R28*BazXbs1jta_KeL*L`Qj=TWTb}|hZ!l2P691==vFPD{Nl;KUDjlAI;3p0vsz2>KSSJ ziB8Ou4Cek8F^>|6BCws17RZR8xZ~F&{>)+^qV}J02j#(HO@1Fo^nRVTG$>0L2V}xC zpv^EoA)~v8J`nX0dkg9eh+&z)L<2i*2t`nE&H%7@K7(0Btj@R=qW0adaM&z{^ku%H z_N;jT_6Il%UnugJq5ros4L0lz0(U;sw_ncq4eTv(G*Ja`JEFPZVnpv|r#u9rP7 zJw5$P0Rbu|CYc~0W$-~ksR(E&NYMWV)*mweAT2O>*sSi)xY05f=O`b84Teq(T*5M7 z{Z&qwS<#+!;0tv#wC=>8P`x$GXHq_@B&$DAcijFN9NH z81BKjJZP`)OT_SHup0N>!zl&gIDNEYyM8WX79q?ip9TsXlkhBT*k;^_thrnMV;;g3 zW?i7WVM^aaxDC#<7IXMW?O`*#=w$Dr8mm(XQ*D((&qKy#0C6+kN z|L8vA=1M*LZ;u3;nJz)Y7&uj+5B`H3?EpRoC@WAizcqoeWelbBh~|a%L^gYCT1Z!E zs9A;8{x@BL7W-Bc(GAJAz@&J59R)o=5El!|gh<0$5`bUQa^^;+LLVFf&zn6?Vc12& zAi>hofk5d_a9SxAMEd4B?Iu>;f~M$7NZ(a7IMl8c4`~<<+EOJ)*9(+XqQo~YL+FRo zEAtxR!7ReQW)--J8<#p~3AaLe8zR0KSXJoup7E94EE8z7%|`iDVDi8mrqqMeghu6w zXum3kFq|PE_x>-%llNzJ&axQkC=Iq)%B*}FyQXm>usxbL4Y+lR(DM+9rJ+OI0;uG846PRMeTiYdClUMM&NT(92(d9iK|#n*KV@{d(roN(gkXVM zd@O8i{GaSk_Te2D1`T46;d=)MgQ!6f=5bvP^ke{4SF&1hc<=7rABaZ?Y)nNsivwP# zdeYbciV7c*!@hn06xvauNlnqw=H)_Thrd2TuKm0b5I1{s4J3gApH}M#zG4Y5@{!Yg zpg#caJzjG?=(8dPp%pS&#cG*%vqpfQegN@Hi_M8FqG#->5ce4^AUHZY6PypkDThkJ ze(-zm!I1<|^ch-G2KDmJ&dyuAm8Y*Nk5OjEoGRnZK!db~c5>AG{HNZ$E^d5<@87>i zY^@(Ys|biX8~~wAD+ocT6f^E5$S03Pu6`R(&Xb^YVjkLO!MQ=b;!ufFr~MF2RC8Nf zFYx=au3?geaPbf(i$fV{$&^1gGQF^yG#EJz*&=(P8x!cPMfkPKPtCW$RTYweI8^N{ zaAoj?j#W@NJA*>K($5E(%7E?%Vo);G54OyR@>y&=nEPi?1{lY;KaD9o*U>`F7$H_S zR|JP1n0~wpn&09qs(p@i?;Zu5zXF5Oqf>=A3eF`lhCt47(u8mcxC5iHr!$L#>6FU?Uwb}OQN4I_33?kiZgmPO$80aP9>8W$ zgibtI{l!t;3n7o%BA5jU*x&R3BRLuNDn2&X<1X(db>Y9K7(vWJ z9Wdh7XK=icj)C?>{wzrM0JW{0;>I4?2D^ZYE@*0MDh`UzQ-L-KUHoLg93tl^$fqeR zK-n{$)h7-qyf=la4FM#Iz+kk2ChChNOPR88R>^uL=ut{I)dmDoF)CmK%7o$cjnH$Q z@G|z7QxOq)4&jYB%nO`ZAqVl~R~Ew`(Cp`~|C$h-Faa((Y-c$*Gz99hr%|X37&|Tz zRAfByO9uT3y1(i<^lYJIWtE3oLKd!ju_8i$V)>v64M+-JXo>VHR1~E`$awwQHAb#` zXis=w^j}0~f*A3D4q z3mPo7;+dxleg);IO*;1Rtc%i?qaa58?GU7j|1eHt5rj3jMtLJ1fM-TM6%QA z{Eb8L3eUcuiVdlLzf};jr)1a376z>q;IcpN`KEW zhM`6&c9adJ+x7(`jk)(3PYFFO*A!Og?9zUtz6|dNtK(@>o~Ekd1X0=3zinV-EnNC; z*gs!Y>ehManK`;$@UZ1P+BoIn5i{6^=$4Gu?j-rxC)@ZCTAwJM4%m>0|2~qGOzjO% z@kUQgxc!#Pf9YB3JWNsJj|y3`C66+jgMMzO>GZR)-{ctqha66rkp*=|Ytoe9x9XOv z>Q*V?!yLtx<5U>!-1PlU0DWJ)gKFr{%JE65=XP%^#t!mNmdr%;oOqtTCpi=go-^(B z9GgqF|H@+bg)bs9wlU%c*8YILi#i3?@@VQEW}N8L;Z}P;Q6A~Cvd!P8hKr=@B|6HE zP4rqX1-FhcZ?Nvg@Gt6mW&6+g>~`(8ajd;R;b9jhHF&&OFxS7>o5du*lCmx$orj{d z*<@_eWlB4~U@SMzmv)KKwa#ial+P?mRPYN;&m3o)ZvM;W)9b4!u54+yQm5F=TW{js z6Z>CRnT__fzh4dElZ;#PEhs-MBKx3`rr>uxjWeLqFF2CInTDS7(>|iyk+o~r+2n2T zwqbP0++O}ytPZUC40MdF(xS4B{H;uECJm;r@zB{JZZ|^bp5p~muI)O08%13MR*b7s zKE_v;zGGC%j_S2=2W0i9*$FO`e8Jg8Eu~Z6)TQmHC}z$($&Q>}B}#3&fw9zed&D4U zTT@YE-#RL_mNazXt=xr%@71Q#?NjE$t)eSeg9bZN+5M-DR)Y*!GVPMTvKl5VT^>zw zP;GB~6=+XiHaFq7I=`TSp*WU_!kRXaj>GOx-fcYG6504g^-2)Oc5Si$<%t0A0kxTf z#%?&R2`Ys7&Rx%(H`!hjK5cXvY}SQkH&H?*>JkzV2LZ@-EBF$3NUpl{Pon zCgK)dMRBNZKCfO9QtfHnkE|3bWZzLpS}>c>FCysSNq+CaSNPPxpJS_Z&Gxzp;g3e; z7VG7%oy_0)4$*eI!z?+us->8d`DwnzFR61(Y@k_#oQjGH%H$^6?{P#=V<1POL1QtM z5&hZ1;qAuLmCl1aLcN(eYE;s(qsxqzUdiL)%{)feeB!-5?u_A>gxq~PEI79Fx8&h= zeP!S4_I`&{)Z5pJe!J&)BU~I;G`RCj%nI7ExI!HkKVZ9rr0yEI45%gcWexq{i5bN= zZ^7yc=2YEYHYMFUv}2oZ=;my0_N$v^nv4vd7%JG&`M!x6Lcxj1F&&~2u!!7fD{rT#&75pwKF}^#LdEwcywGXzmmewz8xKKjf63j-DD$#WAa1*ESPMyr&^@Q?z3CC}5KR z;K1#l7^Ct<6O&3sv@+GJQFJ z!6o?emhr9U?GuqyDyxrcl;Xoq)zCB_fe?15m##9y7Ix!l4Rk#o4}Bz4^r{ zA$B$kzuzToFYXXt+HsX4G38HPJ~AO1JfOcdl=~u5)Vj%HDSZl~i+5av9O3JQ9d;&N+T2`7ms! zax-rh)hrwA8bC2|DnY@18ik&4B731yIh8hagk3nty4!bm?00R|_Q6!h;7~|Jmf7sF z3I<(sZ)~M={G6HTkDQ%YYgThYsvc}C9`wi0_Jvo=!haaY92MFUw#&JWDx-$i3KGY< zmV>U;Y~&5^PyOEc(&TtB(mV9cR>1-M)DE6hUjGJbAG5D_PwkWwFzthyEneBD&7p#A zg7-i39oxHj5!bexP3D^?iJ}_Bdw*9^)vL}jrg<&>(B2Jd)uW$}z)I3Z_v#d-(>HAA(L`^Yc5={jMI^10N8uG5+o0dBQMjQh+1-UX$hD^xHN>$yzZr}f>ikgE zkVIUDpkMO9au}~T&C;~_$5N?Hpxp$q>9G9b`A~2L+)Dxga*1_T<7Bd1O5*ps*_r=z zbYp!-dbs@?tB&rcui@9XX9{Y#blxOVR#v>s?q`>lB?&I0ab4_+Dl&t2w8eqd2Negvs|^T&fP#RG zabiX$2?zwxW{?B{1(|1$fWTo41Ob^r2!TXF2uXl6C_{(=MM5-;${dDZNWd^8w@}aN z{&WA_yVk8$|8}iid+(}MRp0wP^?pt~ZotBuq(JiVN3inb!wVBOxo_)+Yl7g#eaj6_ zqU-q&e(T{j@_QfEPPAuIWRG^;#+$6j-m`47F$6D z<#XZS$oN^Zyf2+5$ZGN;K_6~aTiVMh+7AVtp)4d&@&i)vX(|E}OhdB6V4u}71RjHw zw`}EodAHKr<=p3P{~JBR;K&>%M^Oj;{NyS_NM|Ps%xIBdgcQ?_oF^nMFL~InJ)d?e zb|l=QVpJLpdBtRTQQm26&xA<)LFuw(122vA(NMQZu&V8$HEkC*d{s<6~Mw+ z7Vn%zDMOy{Q>@4upTVe){UP5d=Ii>!yiV~?D_W~u$dydai0y4~JZ4L3T4z*^&!^xA z4`(Mf3av{AtJ6FDmGkTmEJmde!^oH~aUp%zHU!;Lgp^Y_0{3S>1@6>Z91EIU`s)HUESg+LjlwhG|uAm~l}X=z{eK z=!h2b(cZR_^rBDI&P7g@zcUPKr()6^z^RLEB~sm`XJ?(D%#Wm$F3ZCX<%lQv`~Bk{ zWZs3SqUd<8iOrqLh_mLPO=5;@;JHrAq`?p3+S~^L3HK%+RXkGVKG<5fWk6VK&MK~| zIca!UckS64ex!BUz~Mp^l>|%;ZeQ>4Vw{#d=lLFjk{W7vw?v~Ic#BuzEDPtsH+&!4 z+ml9Wz$1B2O(q2>P_$chi=4U3!cUq;%OxeRyFp^yO#P;3rcm1YR}E33A|=?J2%)EiLbce$&VvK@e>9I-#+ zOAX(nY}FFxq{a61?eQ)XsE^pI*TMME-+}kD^dS0jn;&~;H5xYb1Y^>*L@P`xl zY%*VBJEGW8-=FNFpB#7@`sCjWnvv{(tC3e=HqSD19(>F9QFHrN@JYEr7Ws+0dO*!I zfCz)O31_BfOw}lmKgn;KYQNvSK|3l|b~;5iP;tMkX%-<pUmE>+ddbT} zjiO9h3ZTy0qsMD+7>se$O}o`rr?R<$);!K~w7z)NGLs$@kw3tUR8LIcKqeB^H9GKCV3xK{Fq zkeAOCOv-F8rOmTQ{M|S1xf#b`z!)%SxGT#;jY@?bN?ovISV0OiNe^m^LLbzeRhh@a zR%o-#&=EP&+@>nvpML!!EI96W&#^ zkitHZlkPwdd5f@!y{hvngD~gpD`3?|GYI_7oAUkznrwv(<)x#{=X)cpTqiF0+TMP{comj@QLLKYUN5qkMxF|qTMu=6)4@Gy`V88_Hr=Bb&QTD(dv5+~Qq17KQ-X4dPw}8${qo%rn)3~OEAK4dCQaDbKf5vUj|%ea zzR}6N_BDxKOr0ssf_Bk66vEdA-ZtU}Xu}9gCGvUWM0^{1J{>%m1X?OEPc$G{-JBH| z7s^CBtg;Ks$dN0x9qM0##s|qhAue@b8!ux1-i<$9%*B)pXFjF|R zt{50JTS1;DDS0;2+JS-|1d9_&Je*d@+*^9Ix55HZA+3_&SAcN1zjiW;+rDpOrJKKU&fROt}b z*Rh|YmFK}Ju71O6meH#OBa{BvNXa4z^)&}Zj;g9QP48;6@X&B|Vtka9~ z>8jR_kVwqzEEu`{a*rX+sbYVOV;w|364MChlb>0=9f56Qc{%lI-pkZtcn|v+D8Q?w zORlK7xz@_(5Q*Z}t6hp0e$L=!?n%vWFf14jwi7aI@7qqE^qF`e;3$oh-f)}B<`5lA zKNK<85uyqan+y2^n~|B%G^1iLJgrpdW5X;9$u`@9!9PZ*`7w7T3te=LLk_mM*6jv! z_IsE55RDOUcL!#_n5+;}?O*@KeZO)BT!}R2kqVrt<5XEB#aUX$)d9w9x|DZEjWQ)! zzOFt3y0uzzwMAu%_77}sObhel=D<_>*S=ZqN@`p0$F2IVAaPW2W#-k6)9+^QX!2Aq zfIU0c!2S$OzK*`liKybCeX56{>c}aEJ8G=h2gy#Ty2z&l={P5Sex(s8rSkK_LrW4y z^$iOxjHg2%mgr~#C8lFu5RLi=X4=->Dnn{;eubKbjPSz2fxK3FM>jA~^QGri)f19b zRFDg-e(HtpMXXEFkltpqAWF@82t8@1_89m8XgUcw-CIiW6;=vmWRw~0o#79AaD#)4 zvOhDTi3>5oO8ED*d|rn%$am@$HOOGom$Qa$Duo&~8S?3N2cgZH2SVTT`92oS*Kps^k-a{2WRn4@P02vvX^YU<(1vq*B8yd$q`@Vq1GqRk6(ZlvFVh_A1G~a~`V3 z5(}4Wgtw197ZSQ5+Q965qX?R)wLj-;hekh*6@&+W3TO?uTMAxV3yPT@s@xPqZon$+`;)ES)(QEfX)V#;Z z&*4I@*O*g{IB43Lx;Q+mUkQcidu0*?|Ex-ump%e?49?Rvqp66OKiizKbgP3{2vmK! zC$PEWF;`ntsQV)3^WDgmcs2sZp0T7y6-(7b9KBEda4ZKti5T+uNOk)9b+UflyXx|3 ze?I=;hZSQ3I}T(?tgUx`@-)BYSnF!Z-VnITIHu_CbhAqGIM@D0(vreD!D3od8jW6? zaO1{d?zPCYZ#s?UHedN_)u~u!Zmv=Pt%vMr%xe-_`s19@pYVIK>upQh1RS5Ez2=4N z7ULln>&fgHUmN!eb{wWMv(pNQ z)IR>j-6AD@zYh4O)2!KnDZKRc^_&5W>NTNY^te7}LoF&w);!)&Tc=6gH660iw1#d? zxJAl*c3t0YC@8Ks;BUm7tTtR>Jx?xtX?vS>pMlJ`H|lu#L$CXG9XQ9~ughy0t}XaF z%~lS;ormx-ex7U>G#PTqOn+Dl)mfFvTuc5iqS`?36^w@~_u^jRuRC%iG~Nq4U~!&; zRr;TVP17uTJ;qJ4S~)P+(TvtV)aB^HfAHEmp2=etH=0}sU#2K%)cg=8^-_RrL)FEh zi7<#lj4K?c0bOj7^$ZS$vQ>n(<1&5rmrcYEwQgU~U+;d&>gi9b#$$tBM2h;1^@~1X zNAA`{01KqIZ%0;jhFF0Ztz^2#F1j>@@ec2DF+jc0{@VHMZObrq1AxP zOJ{fYDQO^=;w=K$BITe^UJs80#6U6-hJ{uGfA|Al=_h_P16nhn1R?&@{G+G;J#Qxi z0xEC=t}{l7z}Qa4ex2-A`i=nr`gk_}{Z8WU+>c7u2Aa+k3Q^TJ=m6eVtct5Ub# P8J?-3<#p`U+mHSmlqq^^ literal 42305 zcmd432UwF^_$L_kDvAOZrAU`19i%q_l_tFgBs7s8dhZ|>kgoJ1ARPh;y@sM9(rXBz zg(5xl4gtb^+&i=LpJ#VxXZG3M+0Vx(C;9T7^Pab#-}{CLZB1noVp?Jl2t=atQb894 z`fCCNx`w=c3;2Yc{P6=D+r)>M9ltiSvyrq+Y>-V>n#FR)ir?=i8Yl{Y0N{EnRPBrWcL@5P z&A?3e?|6>+h1kEN@A4lU{~f&zzYT2RpNovV>e{~}Mi#3Z|Bjw2bY1NYxO_i-Ci!=? z^Y{P!gjN1~^C}6C?Jw|$OJJ&h=lSliJdkO;Jm^0f?*x2~KV8X|Ilba7aD6W&Ws)r` zZ^ifIZSNGbXRvu~PNMS?$^N~&Q&?Qg8S_x^XWsT}_5YbI72m6(a&SylYz->JeJXtu z{4@GeU;K<_kBO=vI`Z@mz`0(ktIa)Ae3=2CIwv_L!<$6R@ZSZN<^GDEx(U3#Wt$Df zKOHU@Y%wJT-X`AR`)~O7wl|r_89vj93{N+>tN-kkyytN0x2 zSAVKruj51WzL8ISjqPKD2j{}i;fEs70rE^^1>xY5Etf4G)knk-Wp78sfMD4f~#$EK+;yL4c@)eKC%*FW02A*6!Sk_H zj;+S5waG)x65Gt0-pHaE#I?{qMFSs<&J3V9Y&+#lV4m5=>%q*FCWFDF^>XpLf)(TX z&?2IgAZ)bup|4-E8P*fd$RuZAI(4o^R$rrrc-7yu!XHGBdJ%FN40A5+pLM+k`o0uV z`UnJarE%vv+Fuc|_h@>@vW@BM?@UZiD$E;ZD#Z*aLbfG*jPxR+vPWA`M?H3ekc5;i z1Cu8C*ip+#sco9Wd(!zd4`zm%cvX^>{2iT=)lrFQQfl(&Y;B?0Tt7rSR?;LR{n}C- zx3-Va{V1(o>vSPu>a|$yh3iY+Q1elYyG*k}8;2DaPjG$imOVStGR0^w@7#Ydu)ugSFGnjyo9;>w&X z=DLZwOg8+ZZrR@c?nj<7oZ6I_liluiC9P9w9y*a*9*27AN9uB- zcdC0CXH4YHeP7CK@H5PuAxGG{qcJ_(c}Z(1T{6ueQ+Z?Mu0h89Ce!U%-kIadC=NQ5x{dn z&`(#02X2>_(6+{>Wx+i4;@4B(kgmK8-5f1R;+rjcFXhPqZ-3Kn0*w)yr1^0_x{sY=3m`2?^F{vn{?y%sAjLXS$yeuVzn%> z)SP-Jv7nB2v{tpmxW>Ye;K>VrI@Yf~+GDG%=Ds9i$|w#Ki|fe;;R+O z|6Nh=rzcj!uy?jmS#K*yU%39Ob{MNXhnNQDy;ev8>hwjWHuee52CR0b^dL+QPU8@) zXOb>2NwX_ev(LeIzQYwmztcQM;kdb^2S)!SkrW$1Mfbb)G{2%rBuAKCG<6L&s_AWo zDBbnWGDv1MRnC)7MFMZo^_^)ZU+lBQksnNI|~lp#5)$APVP~!G5A7)* zuR>b(u=V~yjy|F#IDxj$_^Y}3(gUl1-lfX>ux|617n6`j$f>Oh_Fs=GqFBp5z9|d! zU1<~DL`Y2v`$cc1G4M#mUI)nhVv!|&-MO2DPE0*s#Aj>To#$`BeTZ9M9{AL5oMvrk z5z00Nd_EukWCDWv_iPh={Y0hchi-)TxvHVbK=7XvO3L?(o6c1(;vcTdD{)`UUq2b+ z)bMpoCulPqZcRnLy$bj2ziU~qSd*qIz0)$2*{r=i6Efh~t}IH_R>a|t_k&vjUpm>( zVZU-(>L2aL17=q*&RlfEmB@*ANVIEzV@3WjdDKFeGojzX_kr*J=f$g*W-Hu@;{v7l-2{`KLyzyPUe7Z@|xiX@}ZN2COobbT<>h^LVW|o z9{?m%bJpu)aMUMq)@IXtAkYqnsOjvt0>xLpP3~u#}UVOFqWivO9;3(_Q=PI$zdR3?VsJ2m-JR-FH(c% zwU(Q-8Bz*onvFU^Z+qSXKE~Bgg-Ovnx1!2#{6`Oy_AtBsGqPm@Neaj(efs4?;gt z;j+ui^dXXo<-G2}**W%dh5l<8p6t6AH+vl1eN+(iJ>V)m=8WBb7L>KBJGOVGzoL6) zh*BSk7oR~be>OfGN)1c8%>DKb$Kupqj~DFD6(b8NZ(q|)E9B}#7>*7@g4^aLQ%GG- zjC?dE%QS+$ye?bBaqF0{)za|EQU|=O%;}5(#B{=#g)pK2eb+*trkLor*l3QHIhwkX za#W;FBKh^lTUkTRAp#>MYL8QDx%WC4+*1qHSAtuDVpb6m{}(XR1Q19pn#bC zCbe@J*M7K$Z_W7iu z=M=B^b?gq)X=Um%!d_X5cy-uIHk%Hqz3iZ8^7f*YQRNdd#4W5zdkst5Ldr+FzQB`? zup<2t%ph*Re<(2dhk~z$@u0VR7LV-@A2=9P|Vppe)x&NDNbE#>WHZ0zVAEm#71`(SpsPJu3kP_zO4}Pmn5!f?L)uJh zh_in%QHXlB#P#^0tN}X2bxHD?VsdLsDU5d|dRK2$;3QI_d7Sh&pR*! zv_rn#7ns=qkJg)3{0WvujBn`qhrCJjZ?4f;AuG4Wt@bQ6ZGE}vAM9|H=8p8-`_6{9uYc8J0%5#AnQ9JgspSrrt-9tMl3t#jq%!-c4l#iO z5D&8D;Bw(3!x%piRUcjz?U9_>!=6Q#(aYU(PgE3-h`ptI=ta&4>n_PM>c_T#M@lXzxtUXg8;|?YURu1P#mBja zOEaF7a*H!JW68({;A3OrB_XVO(#uNrXNpY6gD@(~OPE8g0ZJD645SYWYW=IzW9&MX z(Gg9#AUBdo&Zo~_(~4I&$ver5I(%6_Nlc6=-HHF$xThl*){V_?*9LXZ}>E6fVToO%|N3%sD>OneNSn z`2>}gjU-FTKm?{e^9Zs*F-Mu2hDsyM@QmQ{TXIJ&@x(I|jj;~HCk~#QSQqHb;F+h8 zsG5mbpdB?Z zOCh>WEy;4rUj5RQCO)zDRh(H!y%|2RL~Kh2w#XYkvOs3v>yy12k%3$<-(QHlou}Rr za$Mp35FbzFC|UL^F##XcZ=3#P*CCVN(jT1L#2NE72ygYuha||YE*LMPSz|{%n7D9$ z0+U1_E^cVnnRG06kWlKSnE9=@dC+{-SIkQYM780-wp@`)28xoxug%6;wy>s5gdv9Ts>Pds1RvNBRcGf9 z);-po;>qi<(=cl?c3UHr)Qz)WZ?+v3-uP^z?#p`ocx2>d9K1i?K3z!9*8WrWqWF$5 zd#cUWQ(&6!*V_pe82A#3k$z~b(WZHgo zv#Q0qEf_haTmok|*tdlbVEl&~c~m{>n)GeE9v;L46LH5cGg<7?Y{C)b`> zr)zd-HfnmanpjfXW=$~d#dRk)FS1|kBlW6A8S=$bHuVB+lH)DM93~r*_tav>=mrOd z=82JUVqQD-5*^`V+AEMSwiLpvMuP{fTHrGa@zo4NqSZr(paWF$Uy>Sn{4~(X%1~Om zHAt-_zj@cD6l3Xzx_vTe7B=CTd00ksi80|TTZ7O9 z@tWItANA_iT20(Rw-9JGvVAiQnq?F`n6~nb*QW+1FzO65{t7mOjOiDXPvE|_ZD!4n z^$iTGl3g?ruOJ;oIxwE;F(a+nQr=M1 zk?tkUPY)$9$;rAlIy$YuewI@9vw7O_>0QRb!YXJqy57w`^3?qURLTd6?w*IxrWCdd zE5iyM+Qb}t96BJsJycba@?-b8G+O)`W3PeCW^HE3|5b4=d~YEljRj ziJ4Vyc!jJ^pDXN>w=|tL#j*R^!9m7Szghf3lWZRLEH{0EejR(LVu1(-_w>hBIQH-NDFA1Gh~-$ zkJQ-WOH@)#k9L-6@jvBp7Cvh`GqWDqPi%iXEFBN`9N3o-6}2LA;H-Orpc{d-@I#n(V}Jf@obXPq`Z`{H?M!ONgOp2 zj5oh2>-oK)-OXP2P#azS(*2kHR-`X@ow1~?mRR>&KjL@hq(fkUoX4Kktd%1 z0nuWO+uIr!t&&Paea{ZJ9=aVh)QpYIhGCMdx1C#_tkE5Y1S0&kUKI;wZww{z4;cRZD47|n@-7|HFs zVpWYgjn;Zx(}m8@7mf8(ts7soEBRnN$au%f3RGyHu<7>@6svQORQt2XHZupuzF4kE z)B0{cfFx=E+O&$z+YziN)9_p`b|&V{yNSeq z-1H9_Chws4QMHg=swQ3$Vo+7_>E`%e_)OlNoegRZi$gd*^d2yC(u)eq**o3r%O@|M zVPFy2EM>)8#K{|)TPJ)iI&4p-DDl6G9h%$t-7VQ__~5Y%YV3gJ=i{Q%9XBNJ58QB0 z`gtT@a}Mqhws==91|$crRJHMr71H49;tZlWzbF%8qwy|1zsL1xwx$F{nxbyl^hQ(4 zy6L{iMSr=7Y5ahvp$L0gAzM2lL8VZ?{xd@^TW4pQ7jJJPdT`b-z9jk}I+6)h{-o|D z*2_m(!{0gGW{U|Cb~-9@4~a=hrGy!ZLG7cJ(bLJFtYwQ5 z^&{3#%0@;rFzXLNbCgM&N5mj)ovL8WNI^iNwdz1>5~NDBv5%TSDfQS`B~4Q@v(PHt zT`IFtAVrPe&{?g?gw)zRv6GLs>TL3^CyB7tLn#z&P)1E?pe;k%c&3@g+N6GDw9@76 zdCxcjhP@DNWu?=pBfVN5mfuE{6-p{uh@V0UDqWaxO3)4SM=5CN1ifZR$a}G1y zkq{}ur(xL1G$LPPI%BGYuk;8S5C(?(d!xe#vANa>VGf&%*nZ=*% zl}3CkXvLX()zQ{Z_F|daKQB$M2~Txdr^l-U5l)CbrME9AtPrjXpw1dyWpZnSwrbkBgm<3wJqB@DerY?Z_{APoUa(K?M(eE_w~Ve!|cYh zNYqli@=9Vnw+2JDPP+eoFvAT{b0~1Y=O?nsHINln9Yow2sfe>(oqyc6o+sEqE( z+r;V(L`^h**EJ@=K?+9sCM$SCq>>F7*9|76>c+qq5UNH^dUV7a?;CmeM=_0p`rT@_ z_PgC8@N`M@$}#??pv5HjdJDk^H6bC*eRJ3RakYX?O)4L*J2j>}oIo1bJ5|cP{t{Op zH2dyWvGWD2YH?!>A9FJcmfEPYUT>v#Bn309z7DBWiMP=6{TBDtK224OUCp;4{V&PG zsUP_skljVQ!5{g7@RsOrQaH@d!ULJ`13hSLougFQgJkX|S)r4Sj_fozYf+UI9zB*C zN-|oQUVa|vzW&qgspgbkq~x=E^2sl*FX-;)-|MLS3l=QOWE8l*+Gwb>80LM`e0!#F zWs1U$%jD-mp5ux^Q7n@TbJ2W})FCWj^UZXzs6q$>J)*^29R&-7-0S5b-Be+d<1MFPskSiAoKHB_HM445a1=P%|dD#0k=l?MxF=e?)%T zNze@$4Usj;OHHJg6xMRsuIi>0-(%Q3)vI5qD79>pjnsZerG64MW9s%yWrDPT;Z!kO z*0qgr5w3R4aJD$>V+~FxXW(}DqL%lw=e1r@*CcVjEUYBk8FPH5m-u-z9`yyj(5YqLKe_2o zNMM*Wurq&^v~$9_-YBXT;IpL{3r}L1q6i7&%?>K&%xT(n$m@t}Td@U|$H`6Pp8UZlBp=YP8tnP=V5@*C)SasX$seJ5t&!ASbFLDUNBS;= z=_6hIEI4jGs{6(wa@i=x-=^R`Pybs%Dm2|$E@4LYnSDkP)YEF~JsH#+F9&xHC`Ht% z*&_RCU{Z^GK@y0F`-aU3X@8Rv+UMRA*I_V=^{u9*R*a}mZ}&d(`Gi!l`S$I##d`mI zOOn&GheG#CadHhK7mF&#G4TIG^q*xZE>B{kL_l1M&`cdp@Z2L>tzt?HF)&kxbYI_(D3kA=n{t; zXx*@2$WESf#@_{uaRNh|OwQ;5d<^>Rot8Tv8Kq&^Q0e5_F8_1Yi=cY5K$nPHvVw*( zOy=Km%K0%U4In#o$I8639a#4<1NC@1&S^nw~YA*I*d zJrKz3WORJGr#I#iS&m?khG+z7;-*& zj0}|WMA)J+i~f)dUVDf#Hh)_1@;L8N(CE%Uh{L3*SB>r7L#(iL-QLp+-fyL5ulD3x zWDrPN!`&2fyBl6~$2rSw)A9tN#r2hdI+FJVv0>`QmgD|H2s+LW5;mt3xG7HCgY~q6wv2ER@GDBY8%nR)d{omXqP;1#iKnw z!pY9!bb7UA1mQS5Cs{tc)Ty%D=yX$Fv8rw)Gh~zh^4(OU`dQ#9_qt?9+gTLk$wja! zCO7zVHgsZNW+MCCXv%!+`$A=}Vmwuh17a%qKxcGY*wdl7zSrYU_XKQ(ReG&p0SM(s zjZH0uwh>ADi@`FYex;g9W^k4Iv7X~!n)->660%jbTf1YpQ=Jq~}$X-9D z7Z$3J&Vu54T|Q&kd+kGCB|Ss}Pj-ke`9KYcNu5D7SG^fY>ha4m?oo`HO$t-7g*{&q z5DUh6BJ-8@czLf;qzljWI`;>#j4a=mU%1TIF9Zn1PvJBI*AVY}58BO;O&Y!`?Y+$O zyAgI})P(XF=;o`UHuZ_Jo6-+FS+FXZ;Nm?w*}fCS^u3Pf=(cs5n3iWJx-(~g6YEt; z@oTgs*%*Po7lZ@#wmIt>D;`drY%*AZG_~z=w$n-R^4#U>ik}bc+A-e}ry>tyO#ZIA_Ux21?o7yk8fbZ8xC{$A_?{^1|9$G_Rcp}O z`?;A?S2a4IiuJ#)m*~HiwyX}l0kRr%b90+b|GJ{;84Xawea0^M=GX133L9`~EB#ZP z1FCnJo%!|_pg5PLKXV#7h5+s*(fzM2z+KVz{|i3;Ki1@hyxLM1B?AN8X{v51!)NaM zHI)&L?7(=J;Hf$%bgcs>%HXPBiz4o-f8bmEr>H2D%<(iU6BCmWZzBUK(2m0!8=FFRcQz?=Sx2c$js z;$gx+(^gD4(n`ALZ+Hh(_D0i+!p5zEDO`8|>2K3#$y18a_4fAuvMo(diVZ&7q^?u| zZgbwbYP@)>ny(zk2-C?^`m(L8ps+gWP~P&1{W`F;3Sb%_kUk4ll%nPDXs#ORVyrVJ z(`RNam=m}S;wD23UA?5`HbvXm+B%FD>zHv~^#k&M_;;&oLpkqh#hn9Z&To;>ataBJ z+`XFi`^SH$eF4n*rQgOwYiiRQ>xD5je`3iWB^ZLKs67}AAb$W8r)!C_R*%^E2?FQV_zW47w zp#X*MjV1sWX$9YWsE@j8K6>-2#T59;+do!TR-D4ZdWH7ff`XM@ers!MJv}|#o>{$1 zy1KgYXVTSGRf$O_mik?s;OzP$>bsH&-Xgo9O7RNN)# zfoJ(Ww3CV&zAMF-8I-zti64-O74;P$Wx zO=jaWu2T=QDAvlBU1>3Va>5hQ?9Z)kCbQICn)lT2Fc={pPD*K1c=u+W^%p}eW@hH9 z2}fcQ60h+0eqI!`wA>l}g`YqF?bAR_LBR%$0d7n?!;ppgSAWHG0$OBfk8mzQsQO;h1Jiz;QAg7!E=AvU7#7I|aQOZ~pa5 z;fhuH`3>*e6B8)_R;|wcc;9iWnflMlN)aF;pVegQxlCyW3X1&CpFh7SdjIYn1u(9t zsCe;S53nx#E4MtS#UdkP8W9n(We7;Kb?xVJ_Sq$WO6kz{;qkE{z)Rp$fEXMp1u)Pr zH){z*-JBhCP}Hvge@6GPEBQ52wAVhpcJ10$MU83_H}$5^K4f+2qYogdo`C@aRn^G& z`1t7`dyOc+A0Cd5j;(9Wukq7VZ&#{b@e<#?tM}?v>d*FeN=C-5EZJl5`1#Jxj+&6f zEzq292jvqG$n0}`d~O1V7H+XSK`IDeqnRnSMA$CTD;mhiPE9Q?D=XV??R2iE_P3nL zS4r4+p57%LQb90mOKPZl6l0!V;(ELiB&miM3GMm7|-sobp z56P&p8_w-A;6TH`th_howjxc7G&6OB_(es*MOxY3L}x3v-2L`}P9%VznrkP!3jl{b zTL;5Gek?J7Lttht{;ePc;G$qg)rDs*ICN>s3hp|9E-(XnV^Sw1B;>g^pE#MNr0y@o55fV!7zOtU*85O1_8$(u9rGs5CDIZuiipxpE+YTe4OZ@!#+xYmaD~*@4 zt)9>a#B#+Kz;xWR&uB%Vi&+R@@-wbhpwG?ustF{84G2hpS+9K$y%8A`16Eg$e)HxH z1sz=}kX)<}<;b62uep-9Gqwq4-qwm}-cvnXLn(W$kM(2H7v7Nek%A~Yoa;-yHz#4H z$U1;z@J7|{?ygkuc>_}Ve7-&0>tx5G1JERZ!%~4KuC2*#fF(eOzJ&tJT9GYUZUM;3 z2X@}u*;$lp2n2}oM*#26?E%B>`2D-dI5+cbveqF5U?tm9tWPp#qzqUqGCH~sL!4)i zDkQ#hXLoNB^lVA5K$Q|)qN@n_FkbhxvNB!fH)`}4tjV>|~CsNKpyCjJm( z4&)a6>>M0w@wm~F1=11x!CJz$QgpH%NE?Vl8wF}9{3|UfU%!6cCgZ$XZVQX7wHsyu zG+aNP;{N?yKt)RcQ9-dywZLKpVosAgPi}#p6}$j8+_3spv&b7z?A28}!2C3j-kT#w zn^V{Yygp#aPLnk{ov{o6u@c8ISB9UWMhfC|0en8ddVMRaMZax!HnuA=PYEF55W@LB zp#>0e5~2ym?X@;?B?YT#anM`Ct4x4{?PtAAgEZT2rWXqb`+uH-UH&A?K z05UJbtF=s%(1NSg)S0b;jaH9GpIT=nD`mV}i>t&vlMPbw21!|18wp#=?EtCdx^ag%X!VtV@Pp zFv+0}`uP}>Ssf~Wk3B4tWGGzyPIg=pTQuu4@p3paoW*P5Pb;K(Qy#0G7Qk%yiUA2U zmht$nu4x^#AF-+qcy||dpdf7*lwONHyITA}V;LHZEfgCIsZ&}zf^=kk z!pI14a41WAZww8}U6uhn3%WSO_;1!?tkMf$wDoSd8)erUm)$4ny`C?7kgu(i$H zMDbnpX(Ymb z$A~0NO7$|(y5Lq@Napnx(|mWUS_t54qyd%HJe~c+-PNl8J$VXX z!?qyAj}-gIUui{6YL)nhD9M6KFIy%Jz0U17J?y*3@EnlRPGyoq0%F_KrKPttSOYa0 z%OG77c%qX5l(=B*}xUxYOKRx$>kSOA)rPM3K9rCP5+Y#DU2 z%zue&od3phWh35|paO2Qj&^QPzaiGGzqf|!G59=ryr%NqTcxTLwWeVmGjKkuyfq5q553Ekn_r0WQ=9~8RF|e6!1UB`k;|ME=KT_{o8->!!sJ6VPu~w zV+WfNXRot0srHng*lv{5?%Qn%)%VFThp!F!TxP8kOa=|>9JJB`*8crf$D4;j1I>YF z>gX@ItpY*i%QE6Gu;pbXf18`%#6Sx*Z!iueE`=w~o7LV(AH|9vpS@GE4=zk>Jkk7! z4Ed)^Ix+8=Bo&sNJIdTV$Wug~i0VR5o@JfuX?|1s;lrZi;Mf#Dn?_cLqV-1Uvl6|1 zDdNLwo(ibE;!&Hji$ILdk>qS@^_o#_)x;>iQA6smm#C>UfE9e_wl=yDvlc3F%vsI5 zo*bv9Ox-^WJ8(T@7&0ZSI8E02msNttY_#lxj!v^>{%c!Yfm!{Cb9MZfn|+FVnI^R z<&mP`!`E`BH_<~$GM#IOOmFVer`Q@9&*l?i)HhS<3N|Gbc|FsB!yyh;6gLyFEyM8C zS35(@Y2@v-8zNpf{i~pLadwQ^Tk35wRR=WS<63Bre0Vw#^bZ$#0K8xvi0^%*3{t7_ zeKRcq8Bx>%Vjk}9&8B6lUG{{7rNHVl`Raa$fo6Y=Q*5+HW}B5}pSQ;Wt8EI;32t{! z%%i@#Y^h76jmEG;giX)Z>}ZB5HT>?GtYLHGDx#?{&duBCxYy+fKK z&{j&E(RL`Mjd!R!Fa3izw5=`uDVO`=)s_8S$27BqOF^wBbyVM12hx3>^8gYDnf+Kl zj=pA%)$x<)i4mx%adgo4=E`*T%c>)>WjQgMq+slx$`f1CJ3=J0bbWDg)Vgkk?b9df zYA76PDA3i~hEVVBol)*xm6lAq(AUA`m+@_J-2mof5GlpqFV<^XS1grm@3dRf_~T3c z&h~PrxMFJZi@I|1Y4q0yW_SgONi4UrX(BY6*PcdbB_r=0ug&WuSJZajk*FiQPqqb` zA?uf2StpW>^(++n+S&K<5;J`uP{!|QnCdeiZ9^3-r6Nu}qthX>f$am@`%gc3sCY=w zFaP{_w;#wK)8hL8SCd|9SbpSP2?9~t4Sba>3>QpOQ4<4w_XAF*BK`+S!-}KT3Nif(u|_)j%P^}M?IJ0? zw0I^MkRky`%hhmi)h6fDx=v@OnJi|tm}RJ0LB-87sLvT^ZS5~8p$vTrX_xd-v>z}? z!QG?DlYMTxo(q&m_M8m_d7NyBey9t&XM4W8ljC7m^1 zGr~9;@0Y@ln{&L3K03)G5sqjDgHBDSNwd}KDKe#K*i~TZh@`v4t9I{59YIW5n%Moa~ z^cDbaXl=;54vDwo7uvop8tL4Nw5Cd?BMPDG><N0u+>I#pNm*#FjS9H^n2UmovBTWN7fH(K61!=+NGHAs4nm7Mc^7cxjE z6WtSx`EJ}N5H_hR_tOh^N7FJwY3q)b+v=#Nhdufj1eDUn97?Ox3eDh##ae@!(5c)sgd;{CbKIaizcl=Mcv7K07FYP4WyX6WQ29oA(2LUh zk#L@D5O0s8$bZ@V0^K|Pg`73qA<@rQB6tVjZoSDjjGrGgc&OD^Wzt*wt#nkR~5Wk)$ z^-WMif2OqIRJ{wP*>C^$IKLkix_ONcE?r~ZJIho~mrlgVkN41cIIbFllqs<;>Y1;b z(pB?vp!6#FQj)aYQXs0rjKl9x zE1X-e?HvnR+1PMbcY*7Rw>}Hv7*fBn3#V|-Bvy?Ljy22i0*+&|sVCDi(Hg5-)!&nw zKUGH2*X*)e*rQ9$KG9kr@ZLgFc<{{^#dr(By_wI<-D~bDg?gGrTxo@0HEp+gm*RVq zraHt^5a={frMSP`zkh{GCfx&0Bi}bu%!75T ztn$;voN$}Db1Z`A4y6|)tOxa-go0z+^5JriZQZ$LpToFI0>AB#kN;c8NBjrjFh$v9 z7U$Xizcy!?1cZIxX8AisBb@K&JTu<1Ryjg|Apt-4pSP&iI*$Rz^IjHAL>~86Q4H?A z$ntM(c7e=-4h~cebUCmCf?PcU`l_+cy$N1C&KY3hYcS(B7_TIB+gY zbBpf;nyJ}jqN=7L*LF8e3k<$1zBn$SB`ns-g-W3ByKSKGAv+PDN7$dI?Z{v3O7nvR zq-cyaeA%{(<8X55sdBEPiga8Tz(imt^;S7g{FV^8m$usj3%M|i+}jYFhhY{(&lR6d zyWW`oUaPIfjBz9Sg*CQMSIkf|YHKJkt#vl+!Tqe&pT5sa-KVe4qh0-rZ)n<@`sTF> z8k>P_GVANsbgxrsPMXcjxzZ#(mL;crTKB*1f88E8av!IrZ;y4Fst7WdQV;8YXWkik z_zq$smYI-yS^UK=#&N^w=;;_!Tg0^PO=8`U&SiFig^+svCjn5AFk_k1fuE+R*WCA? zTjAocF>d8S@)J~$s2{Y0H)G!i{kX=vA3`_v2Jy!^^t!d;@t0KB>u0Ty(6xnGW%VS( zs-!v9pI(VH$o6VOu!;msMMsAmVcr%ykSV<`MZv-{BCK9TPX1+bv_wzGz#xoWEm@ya zJa>{JTRN2dIdYDn=I)Hk?>SfUZw<8J-Q_a=TQPa#6AzbCYPC586<9u5J*W^he707Z zGVge>D!KM|p!>zJsL2aK-H@qRvgMc$;R&MZY`U*%nLahnn!G4Uqqb4^2-)M;G>$!b z_g{Jeew#fM%@uXsCTbpQ<5Q1O_&b8l)o;mGU+;^-jk_lV%g0g?Bjks_pLkXXh5Z_3 zl*tOXTXd`WuaKf!o;tW{$BJK1W`BI<8kqmJ{ZVvaX2eIVP=adV)s9f0&){|0>cVgC z4U@)VF?E&6uVR9zFX^qxz$0eOg1L7c)$=P{89$twcjk39%gsPU>A+wMm_Au`6-$I0 z<$7DNXCog@(^&A+Yw@KAfue(ylAV}0M-^Lk4$L1b(-@@ueJw}+)P8#ELly~3XY5e? zDscf)GF5)9piH12`k-3yvsa+)UXtoZ01U)Q(OgOW&*KGpY zYilK=>0YVYE}h*W-88x7$E5t~q1x-oNs>IX*;rQdE{oaEMi|?PM_~&eK=;IZe`pcC z(XmYL_vEy}d%gX93{n)ce4^%j`K$IN#8@oo(OTq3q~+h0LUR^td7}>t>lo&^LP+RB zWTTIX(!slrZ}OkJT_b*&v)@cThk8Y=MblL`1iANyLw&-Rn4{g0#kGNs9&Y_PeIn=m z87lQ0=f5F&&K#za9cXNH{`}lm?XP-!!fL<85RO+`nht=Te*+mT0NIc#D=V9ZD<~-V zE3t{p>E4u+lLL{tBx)!q^mleAa0mjYkz#%BGP4%i%acyIM^yXG_e6af$K}e23K{7m zJ~>Q1nSfw+MOdv@c3W3KWlh55#sW%j!rtfD(vlc)X|^tf&R=f69pq>D%|czbS*ydH zbh4b`QsKU&`jYK_AaHJdC}i~s(ayq>t3*Xk9{Tp}+gsp+9pj`=pR#sF^|a=JBT%Yb zh^$%gsdYnB6AK&LUpH^wY$A<$8C53ey;i!@Hx)HlKl7qJ-0Yh>Y)Y9tNHLPR*WgF? z9kI;&Ticys@cMb7ONKs+cW|YSj*i=8WZ_7pG?jo^G*C$}X$?%fBgVo4qiQpX_*ASa zCMNc(+J@xM!h#?GN;h8|O}Tn||5g)d`2ZZpce0#Dc~z!UUiU|By8fA6-_grLt8&;1FDs~~!?^FzAb0(8303I+0z(@)Jto_7q&q_V@$+el8 znI}N0&g&B(DOx@5q)!nMKY++xX*(T9# z_3}1X{DJ*f*&te=?9iFSWB4ecdv0?xhLXqN^`Bo6tCO{*2zkI{Z*;QJk&}zQtarv% zc&_R4nl@1=Dk@$v7XazWA3S&updop{dVm;#WsP~HGbR@B#HyjB$yM(<&f zK4d_V;P-ahB|@`oX?gj@xb$g+a*_NnRVxGNq&l~S!(0kyP*LW#}I z%}?<7>mUl^n}EwMB2-p;tY`-A`~Zr7hK7dsXhm)Vu(KjiMw;8&8W7S0yv$2?ccJ_D z@9(T+hg`8-A_y2DDlk4EYNu&Vc}h16&7aZ zYXDgFDK1X=#f$6i?(QH^Y;5c*7AX#tItp5$>9Y&~14jSZ(_?E}+i(EieX}uMiCfv4 zZ7V12>Ju(MeY$OBYkLDkPftHLKYvYJT-}5P2a7?6>gdAgoOQHWHkWRx-nHx20+sw(3h8g+1lEAu8rK}=jV5Ib9)W| z^Ya@U(E!5o4G?TaRn#>2~7miQLwws0#x zA;Ilv!-$Q8gH>GIPzGLG>s64SQC+?N;Ev>j2ML8HA}lN|U;tL&a!XE3{E(QKsH##+ z1pFC{qWa28o%%{=XXl4}Rc0vBK@nn~3U|_&;Z_Oa#<;%iprbM~^JOwIY5s@tD zdn|p{gBf=dkYS`1ac1Y>(0kNI6gNUbO3J{_jsqa~TK9EY(2L=izttQ}Oxf+#-Cw>u zn9TF^@s>0>(mr2B=;d7KZ>rbt`1{?4ZU3nU9N#A zV2I?qiK@iGN_7H0f#n<9^`5SbuU%a&pdF6uQ=*T|&EI5ZW)9qW{`~pS>hqu=blCe) z(5OOUt67$d0Hwh3_V%XXya5k5V}?9netMUGWnF#!sE8eekN`7aPc%+EGdPu%l~oq= zhf5|TCW`XVy_UbEr>7T12k4T9M(Vg|^*g5mzMwr~JdJ~=xw*ObRaF{i6Yx9i#QVxh^)ov=JG_Sv zA8KoBZ-mSay(1+fV-ypkJ2*Ii{gLqVyNivD9h;F6v$A5l(})AVr==Mz?)+W+dUmp} zVcyi*A7f{0YiwhKLq$cklMOv6Sc6{fd>0*QsqO3u9e9*&VnorTygEBOFGD{}0*b0j zJEzGnoHecyKs&Nfo>pCkgRJ%YcS=o7%~-iyQ0>6*a4b+2V|#o20(EQ~8=I?CR9Z*L zQjAZlsswUWZ*+hAW@2G+iI|vJTSuoEHt9r2<-@OErXXz+1?s=RtFhj=Az@;|01vOM zs~eG;N)3lFyet5AvZ3M5pFe-Z37=V;u_~!2v1ELWA0#6q0}0DdNG)O0v4FMM*eqYy zw}FQqQb$yT(K@8dzfVZr^Il12ChFF$Tb*COs$3XU(UtF}-zo#S0m$Gj0a9FQ_A ziEH|&JGV5_e*DniS%}m4)|j84uX>Pm?@c~GWvN{5J$w6-!}aoV-aoUm7f^ry{#Aw@ zUK9n38K7FAzJ43!c(iNx>h)`hT-7)4-xH$vzZ-RxUST)Nn4ReF>cYMfE|tdo1omZH z*Ry=At4l78zoN48Syfflp@TvV0TfcJIFKBPNTsUswKl8Leu7b&dV1dtrTm0514x-* z*FSf3{2n#(Z-dy1Ax?G!OmHSRCXMUXU`f(FN=(rn5Msb-s7K6iVh|6P6iQ`-*h(?> z6PIcw;<~!Hyi36S*4x{$vpfpM9haP({5h>CCHf`Y0MJY3E8!wenre(ud$e#wj0Pr% zGKRT;&Rnjpt`<;?kBtogBVYEtDcXO@N@4~1wxRLDW zxo9wce36zG1tRz?IGBNp3x9Q@rUS||?~R+$T-JVk`0~r+&TDnOd$E+E+8iM@H9`#^ zKbHBOcu@;GqD)LoE?>SJP+ZKJr;ri?WwVySLAB90S71jqTHb)>gvrh)$X$!36~*@V z_ZJoxHeMXab$s$f(#3@z++7(25!V&o$N6y!2n1(jWO#{bSKCs+<6gRX^JZ;*z1iyb zs#@>8l9E)r4yds3gmb8D6w;=y+y}&PAkT8MnQLonVbRg;u(dk)ayyI=iq6l60?Qlo_3Newe`oiB9FE6jLD}`zA^;Cu zXs#oYNq8=qaU%&yLDde^v=B_*)77OEv>gioSNSqXH5<%k3uH1*vt6VRre6X`+5Z0T zn3+F+Fd(vg78;uU{=M?32^@wBU;Fvi2^4mGmDpPu4}uta1_^nbm7hF$Vi5Y^+pDA` za=U>+l0$t8k61tZpEoI3kJGEBnLRx{9GKTW&Mhp2WoFW+iuuz*`1wlMiSg6%--k#n1v$%Q z92`jriKqXh0{!38ta2Kthd}rY@asG_OoY9k_x6(_$YpqWc%JHWh3a#4 zK<})~d(Zw^NJtB462x?XMf(gHnDAdtIIdx=QpCvx8HaX2 zZLKhv5hiYKLhu~-MoSEaXLG>%{gc1#tLM%|{ehR42pmCQsgW22oW`)P+6^8&5KJ{Y zD_awFeSH9kEJuLQK)CWNQOgi?MPG8Mt4w*uT(cIC2 z2}04?(<2Qh0LZbKHN;N_TM$JhR8F0+ zzyn4|`@Qi6Q!}$Tutcx1Ch-eLx)>2bH2fMHYcMAfHDkPRcpzJw$d5rx0B!GBCBm@Ksks194 z50!sXPke4q^C=)ol~h(HOifE;PYlY-%`Kx&$R`>f8#9Im8*+U>&up~d+%!HrJy{rh zOMom6(2Gz#Ms5dTHskr?l3)LI?gQ{pmjBtQu=AhGNb%rDgC|IwwS`6AGIw}#F|DW< zy|D0AByN|N$8Bh6Kq?Xtxk+**G4b-gn9YIZej_|T?r-94&dT5oFfh;4S0wX>#~_4d zLV5uDmPK|SB8rI#DzG+)3TbO!jfjjSA|ym1U%Gg42GYNO@j7@tSQe-=3bCo8?Ggd)AJV9L8?}0VmX;q-126Zrs2E zA#Q~{6>_FrE|zbdoqeva>a<+2LqmI7&ZZ*nhnJW^a^;2lFXqc@psoZM?80=T7|3uL zln6lT1g&r7|2p=}nTzaaGrnbUR!4d>Z7SqqJ^#lDJk)dJMn^od)TxjpMc)YEn7ikh z6#^w6&7}Bru+m_^_7}2ae*Ux4Qqs~8cB!AsYromKJ03uXl2X|7=g;)H5Jf;l0;0hp z8BBvuKDQu=rradHm@qddfkiw^d>{t%4KV@|VE}9< z3}Ao;p|Y@LF&pZ95cA{1^%?Hmp@ue$jG`iby6>)z58#S6wzh&GR?GHJUn;#Ub!sM7 zp6D;;zEgZqd$=&2xc((iyg68{wsMc;#TC`qlvBWhbK3s!&bIj}wN3d*>#V}HZL#)) zxeyoa369R<#ehUnTd(bVdreLYJWZdRG4ChFcw=Te+V@iAOxMP!zG}&mq1)c?;NPYg z3%DAF8wN>YMh4yS@iF3ppq}zO#o?0#oW9rY3KoP8-5FwZG$Jm2uK7_)uH@uKx*_r8 zU8jCrbB$yC3ym?Sgsm;>cHJIDmchH?PbZoqofh8qT&$;2{ zqh*nm6xx<8xQQ7ycS<}+=5~B^l56Q?+wJf8u|vQ6_y^|P?9KP0V2>y_yusJyhXcGC zsSkn+DN)By27oOW=CE|2f?4joqw|BSzwl4Py~D>@IW@9YB(C!+(XjQ=8wevVZKb%LR;+eMO|&HRdmMFg>;U5chT?P$7YSB zrQg%XRc!o(-ALVgas9dsXA~rqfq{XJGc6Yp4-YW${a=wc`vA^^L@{uN!NI|G_-aa* zS?^YnVrs6m7PIfEsP3aa%$sdYj5@K7RBjr3!^v0u^RNn;P{;xstMnZIa5l+E&~^h2 zFWh^d>1U1@>cYkPv*6VX=9Zm!ysI+?!`zuyUVR{Vnslt=LNP`(;>H^mu@&pROCR0n zO(>Mz^Si(@L?X{B*J>Q^N5r>Fwq9m^GFw0AOfa)sIQ|@y#<0f~aBKD|i9c9TJgyU? zS=i@K_cLSEKdKB4XQLZ$q-hVYQmL(*L3!O3CB`4PR7}PgG<#mdxcN@6245zcx*3wN zO^EA$4{I5kHC1G{)jk5(&Ac3=MX8^$B8B>H1Y43c7tS^{2be`BIBI*Ik4Mk zxQ%zvwwF}zG6o5(q^Dh4*CQ|U{+%IPm-|f8Jxa!g-da;$UYxT(MvL&Z8X5;e|cZTO@8Z}(giT=08j^Ypy zAVqo$;6|eW(%20C6_SxQh}Qe6?eq*nwJZx$LN|T~(#U=-%0tN&yfd;&P_f+h^CcM& z3u`%T`s(<11-CvL|NY_!de6!CW+~tGY>q$7NvJNV)Yk%ZmZ+coEZ8mCXJH)5u?1-R z*)L^_A>q4>jeQUNClnsvyDn?`9Ly%|Z?qEdJnAHbS{x>yWzVw@A3l^w4&TkLUf52x zY4zVe*B}w&^V$N@9YcI;D=RFpzdledR?x-&?m;3uv4cK- zBc%vuRI=VNN?_%d|C+XqL7drnv@Ia+d20d67O^QQk(L?0WPrVFg3y*Bk#Zx-^BLM5 zS-I$my$l*J(X0!xc~yZI7|k>ZhHyZFpC{nIt< zzc`76a$9j*M<&$Xdoq92_r7Y{UMSj~BP7-ToGV%b8)ljuI89Wsz<#{hdhJIqN1$Bp zvtMfpKSbQ{+5!X0vtG!Exhc#X>x*?p^v=|{Gdk`Z(~ywfc*FKZX6jo}?a3sURO2Si zfamPxIB9?(2Lm9Q9uiK7@R3k@?psQKv7V5Jw>J|92d<~5XB~cpwJA;~$A#h9X{XgL zbk=1*wJJ6k3JOZRp?thWAnY%QV=i|w!XdT2Kq-bvDSxb|3^Co`H|3CsQY2p+{}A}Q zT41=R>8pedQ_$A4x5ln3EX*ZmgyG9{qMW>evkN>6s?LO?MLrok-=lmfM=<@Qe!X2c z9ej(T4;)XJeO2}2_sqDc#M8iiVp(BtG^d!x$x58MXh%O6GG&Z$K0a>SA9lws^w=Fe zQlgtyjNWv$!^`cCQW(buigjK)*3@Et)R0qp%qD7Wz_69c@%|=6)KC2qSUb)?rXtE$ww z3$d4V2Ro9qS`(6!heFz&1yw89LMbOF9wcvtGATVjf931zW3(^uQ=d8&wP#Ce>0N#J zoW9KU-ceKA)y8*5QA`nXhw`oYdp&$>$fuJzWz~+s3cvHyM{YN+v394p2@x&#)8tmf_SUXsr<*obt##c zm>>(MhlCw+5bZ`kQK*za8p*Ett_dK)5JS@QO1{n1;_;Ao*Pcig*VmxRo;en2*kud3 zGm15r*B(12oOQC@cJ-(bGbg4uc@;^n?lG(T&-ds;(sTjE$izg&372Oyc=+H!Ga%K= zoPJB|H~P^)#rhSWCH`D*Mht?56}iKhb^Xb~U-OmkRX9+hmNqqIggA-Yd)Kx*U6cxN z!vn4!;2fi_!=pk{DuC;#cKbXLdKNmH85b89`*n5!;JqUy29YrNi%2RK7M3B!f{`C+X_)VB&?R3^=cw&lIv_LZ~VJ_%Z7pQBhYK zMxwZ$@HSIqqrir+tuh6I2}1>JRhiUwI3^kHBG1j6KCXrHx= zk5f2*poVRMqTPAsk^kVlqWewP^VVZd@Ybze(8&*cZ$&L3&~fudqNq$o2>OS%Qr{I_mVK*$0`0Z7FDkA#NKZ~sDi zdisAQG*C!CPiTe~&U*qDZ%l7JhWHuDlyJs@K!xzu{?upz61mLSaT zWkW-TgXJ1ymPIIXT)f(BiP&XjWlQ}MMQoU>qoo=i z84fv@l&$T%=xiuaSzkpK(D@T8Hg3x!cmUA5Z)?jj<-2D9#qIah31PCmualEopfCa} zs`Wh*1PEvJ?~WL=g}ygS&I1s=pRXW?=lnAC;5h4`(Ds~4s6_SFV>ljHq%dUFG(N5a z4r8vH0;ID$NLv1%A{KP|Xra-9>;PlXc|N$92H7t33^ds=XBRRdZOxFsu9g&k{rYuA z$Z(NPN(ep;HZ=1ffn|cJ!*iJueR_UZ9Jp)9vuDO&?1bIds1Z@nWrfNclC}cYRrtvq zu8*(pW*fOtCNyne;Vmuq&KHhxt}6p?po##6acE@+LcV>Tgb$$4&gOvpkJY>1w6(L_ zgmk?LDmq9x6WX*4mjjXW$s&_K&rI`YAIQnIKueW~j11{bwLqA({^^J--T%~cY+_;= z3@!Vg`;4rmzl7pSoqghD5s6BnuNWL@9<*^CZ# zHKdRVqdDqHLFr=th6eau7@(e3S^pxXmhlQpB09Pl7zvswD*Y4)ZA2)TB2RLB>~p?K zP7EXl%@H8|Dgh&b+5zMZcLdaR0y(qNFJ*9)}qmlhT}k*f^o!PylQ6hNsZBqfE) z(0f$?I$4)s&J|2kplZR;<8&Pzom$uBmoFK?6ic=uE1apB_6vfdZ{IYMf_9-kd3H2?20^bxk3ZabxM;Hs7Ekfr=?kCKFH zW}@YUg1o%4swx^%cLc8s3V$Ew>Rbtt?&B%)P++#oAgzEtHPS$XOhQTC($*FikW|zn zE@9IdQc{<|v?30Ach)#gb6@WM|sHu45rTxOy6H9{nwakT&(lMXkXAGki zeVz*V#)6XHyv!jUxU)0+MFu7VRz%Tv`dm4x`o25ahD}6mM>+cQwSSS~-iZ`f(I8&f z_Rc4nKU>5ato9p=>R_k=4*ww(0yae&ze(`{^#=Lz?&VMql08jxAI+3)&DgRjyo7yn8lld-f<+fpq&5%MSQ`hQ26+jMXJ`@A>@i{+7qX_x{xapLywyg^Di=?VZu(c~plMYf!mR2@+(SxSg#(^usAN z?LB(Bu{J{Xl9%|^gnN6y7YCECA5t6N-WU&}czMM|ciYJJ@`bC!otX03eB0HXraUY_N&+~oq&$<&2R(yy)ui65}Ag!mQOmGft`<@y+2$g59Vsm zk{e8UexO_X_6mlClfq@7^&^Z-Pkr0z{C=aE?qZC^-bcd63}oGU{w+hRn=_Mq*9chD zrwC6^Dl0=)jhQQrob_2bGs<`k#%eM{jdh2;xYwAwPq+;7sTC)YgCoU|V2vg@ex%XC z{-ToH!as>2pVo*W=B3(7t?YimpG|UsoJEmAW6{Uuwv@$u!@eeGK9^YePi}oLa0aDU))94?-_z=$B3>qtQd+S!B3A+;b9Jbi{j}0X(T@yb>~MDL$JPhR$)?sJwM;?{rJoiqy9o48u#Ynv7R6;kB~gyq z!GlaSj&n|mRFV+m<2#{J_l7R2vC28tx=&j)J-K(qA2{LLx7mKqdRk40?&0W2db+NB z{c{iT#%BL=t&x5p=7YEIozkfu9lR~GpB2o>=j%d~``*(`;EoepDT;zL;n=yCPG*ga z-n_#_IN$zfOf9yh^>R1HZ$Wd3YnKTl_ROlQ@?S0{*3y=d7iI?4zBCAYag;<<)<>h- zcFpH#bv$(3%E_{UNbj%X$)EMlBoy@V4idZBJJ&9o%N`z6TouzTNzI@2u=-6J=G7zB z8TYkTFF$}$jAp*k8?)8!N}1ibNGeCa&&kNGDVgOrlTG3cH3ux8-~P$0eTRFEU?7gL ziUeIj_5D#wQ0IWp>`0pQvi#sq=3@D%f`;hTju}%G8tRShXi@STUR9rx9q^^OALdXdha@bzD zM+PhIpu6IS<(w$wZ>&}EzVumb$yGPr2>yfDi>(@0%*`Srp9NDg|}-HMuYvu>{JhI(+*eCi#F!MBW{Q0 z9qAaH$U7`*GAOJj!MsU06;(vt8g%j`xcXN| zZ@#AT*HgVxx3@iZOVi~wofj!`THeP_?1&uUE{rDH`kR4mg_B8dLTqa?#{}`$F66fr zca5wdKP@%GFL^Ta%zCWh?AI~zpL!Z9d=>gxf@ zuVU_9cweykNb!ExBmEDCm*1@1xw~K)L4blTf_MNuDv%zPr75eRh2~+KRN{1EO;*Y{ zBNj_w5MfkabZb^EFJS6r*l0N|zfrY_z1bI3!)XTnv4{g+`lfJnTq|a#d$0Y%l>1em zP}k666<;#xpx-TXWd!-I{`nm|>Z)Okxu`h#OYuv)Hcl-2-?rPmA7;JZlI3bwD!U}- z-_aAsE*?|EWA<~J>CLdWSkd68l5pv}(NyJ=l<`xxIChn{U4Qsf4de-uX^6Z~?LuvR zy86@bAT>95l)m<1_0LDX^Tmm7^AC8{h*4+}*pbOUy%Q<%jtr{I8jEH?5yDQmAQn7# zX#zYseo0d=BduGfg(BW9GK5SX)Acu`K`i zVi<~5LxE?d__8YTq8U#l{gs*i+)TMMca$^Cidt)5*R z&$)Zsn`tnej8JllzLD$p;r;IIxZ5iJ8G^?Da}S>$LS&G8bK>6ZRmHU3>5$5Ts0SG0 z+kd9$hiv8xWgo8gm_!Y&SH5xyR9pC4`L$PoS7hB?ar1z;{Z5=oP7nHZ9uAY;_K!)HzSsO#a_ z#^2jx@v*^I?lY(=g_`sa7}QS0#;5Oq!L?U==y+{Lr0eu@6VlV zjuejf?UHUS1?e#d$_if4dvY;>)|;Y7Iwn0HznNuJKz6Uz5$rM9UC7 zyeyIl`0)=!UUpNv+pM*vANRzE(}G|*x|&S&YRdSV>`)N{gygFx=KueUVd-cZ=JrhOe#*6w1N;zqCwvyao9{g3rpI6X(td8%{B>he6lcFX zIk2lp%X{M9a)%Smkb8XrZDK?ib9IokK80IXC>dj z8??Kz55CRZdBo%AL>1EZ0Gz zND02$w_+aowi{w&^W>_Q>xa@YEH%OOmMYa~R^P9A<^y|Mb(!eKEl)W8!MU|!BV1;(bi%M}>qXI9R~`7k%s|sH z$u;lkA(vk9|78NvD|SwIaoCCB{}?KC@BRM_72e`0w!`sNE?v!Q3KruMzrf^@zK7;R zx%?;bsVz=%n{ugORa{MQo3f9~0_MB4-*{TL)jHqlb3dBW8l&UIuU42`dpwr+vW@O@ zOpTOd%3RD8^}RyR{-SkqPLsxmPgdL)`<;-8y=*#{2kgkxlMY~JZ9`cKGRgrmztLp^`^Zq-vT5_HDyqM4jN|za%2W+i-IeF& z_#@a@TNyl&r4v_svpJc3V&-$>=*-(Y1~P*_VFuGJUtXYn{xqE3zlL*fr`7HP3S0lj z2GOoD+FcJz;cizN{6(>U7qyDue$M}HX&+AERM{%15W`WGmF zT@sq4+{R6)?d4Gdm@=uukINtTSSom1DYy=u9V@#jDNYrD=$`KXCOF>gpsRITrGz;S z7+S=G5l5&+YmLr8NjseoE0BI@@7rv6#edbqw_5?vh5#f1?Dq@!#+2C-uio+C1_TDT zc^65g!?fWj1Aw%x6ad!_y=^C8mHph-_6C_|1L8Uue2RgQ*;Xi_A|F#>f+DZDxHyV3 z_WJqkjVqIfmHONw0+*xt<2$hVXNSVlv#Q8VH^In$XMCgTLBjy0z(3}KWd52 zj_1Yt%gkhu8B*vP0YTt=3?Y2sGL(1EXFI8d0{}4h3y9kQ?B5C~A>=l9PPPZ{z~rcu z>p5iGN{TTKL6;(cZY-<ic>*uqG z?3&fFGIXfp7jH8?23lJfFsj3U4IdD9MJJa}6G>!of`|NauAB;8ot^pLpAxsUw6G;^ z$tUw;!Mx1*r?+L>p5pi&Z_1F6k!1l6SiABu5kQPQZ=*qnj^bq`G`gnpJUIDf*B+pfI80WnIYk`1=0a9=e1PW#a&~jD7K^9xVIRS<*xT2z> zL=r#(amZkc`}!0cX7vnmoH+#ATyooItCOA{>cb};o4sk>@|yR?YTjch~HhSK3jZCuMmRDej-VipRODhVSde_Y@V<06SEDMi32{9)4mBfJUN z3+&x6U`~OE6aL`Q-qF@%rwd2@j2yrwue2RkMcBsybtgN620T?U4kW06Ekocbg??I| zNo$A|?EUu!&x#-RHLx?ti-QsFMT9D^b*2xPElZ*0=pTxU+PS9_jB|Rj?ypvhLiJ_F z0{16BE6`EH14l$hkAN6d!t@0^(g%Gm7*J3IqGd`EmwamcQ0W-j;+LFyhqoW0^3`Cb zGBF`R-Fha_>391g0O)qz!{C+17i|!(;Q^o_t%64R2mO*O!Nl-Y;43HsM$?1aQmBo{ z48Ow%AXP6XR!3mE)x1pGWmrS&KeV4P;JI^tp=T{Ho;?%`qwBsfsSOmM62&@f#cQg<_Kfs@805k;*DR6>Q9w6Hj`zsg+;e-sr zBp@T@LVA4^RX9M@!VPpSD7sX(RA^A=z^vL=pjN(qT^gPTNWD{aYwTCo2`^+mgF#xD z`BAj5(E|)UG(um3`!m@7k>z*+mC0bL01V?xvCy5?0bG(gFllR+=oi2U-Ol=SW4`L* zeR6|8a20-K@7;)+OKZ>EPlY9dzK(&*#EMiHj;XIJN+PfSRex7l6-_5D~orSw#*TaFUup!`mo% zNm&V|BH3=;(t-5?>GL&g@`l~-Hk91P0UTtJjjb4{ObKJ;k^Qy-WoQkUBH#)iKw8ZS zv5`?x=4s5RToLC#VSpdgQ?LCA9p{LMh!=y4>OMXV!0CTX!-zud;1AxvfD(>@5oq=s zH-HEFffq1$K2$&c9m&`C9)zfAd1onty(AXCrmvq0Iz10th{_+8MZ`J1>&q8J&q3gG zfY%`5722GO$ZNDq{3uH@uFLrNBe4vUtolH$iFYnA=&d z?08jqZdrg@aq8E`e!@qIb7pM-6Fi={PB1Y@jfSbqSBO~v(lE0gdqG`!v*jXJ(nD~= znQ-Q)9TFy>4$kk0C>^Y{_(th@Cq2tG*}it>5^$0pf=*eo8lHo#fcl4nP5-B(FmS&j zl7xEf_zF$|6|vDqlV1-Eu%MKMs>HuYxPUYM?B74}16?7QEXB7KpGhh`x8@ch=+XlE z&sj&|KaRru9vj9w#m9W8{0H^!>s-l~bQ>I-;UJOaHhG4{0qJw1Zxs^e$62W%mt7H!qYOsUt8luW26Re$Eq765OO`K?~SEZ|$X+UN!m zDlr~x|Lye#*myN$@MpsfG%#7r{}WhCumi=LzdLLIz~S{PoFySSxe)B46&M(jt6%{x zFo9V@F+Tt_3lPsE-$CFHc4O})04OmCMy$A~q)rY797h1G!BsO+4DgaCpL$_PFCinN z6b#GKi-%k$tC~X;(|$+AK>bk!$Nt5%?D3Zu*UYQM1Q)B_Z`11phodJjtTo)rfon!UPDLA0Tv?AkM>et~PBQ z9X6-O2QW3J9d4Jzp~DE=djMK6H#fflW9T(MoFicn{=|hppb`cO@5{*8fb0R?e8T85 zkS4r<$?3@N9v=;%ANT-fZIsp36S!)BqKWPdsVxoUMk3sdU@Sqc6jFs%AOrEcLkY&8 zTm`YS>Rr$+$OMIj5$^}fqoqm1*5`EFW@fIfctZJrRJ1I1H3t&i+=t|xh7H=H-n-js ze;$XyJ;AK=e*{n`FYfPmR=tQ3e*8EeJS!QeZVmvF$Vs^zg{cuO%6yt1y_5x*4wQHz z49>{@GueZ6uFE9As9afDsXFo9RuEWYVFY1mukp;6QzjzhF;Hbt-gbM3U`;@mD-3IZ zOcVzCVf?h@D_6Zdn8p#v8AwSz=3S_ zlZMt4fq`hV^YhEV4KoTu94-(->PC~{eSzsUgjyEh=|w{rDK5X#e*jr75}o>jH%z|k zD}o5LvibS*SHZmJAi&q&L72xo@j9;n0y%^R(+J!g=l#$SM&L-UnRH9B|G1+GGbD*R zIg&46;@uS}T!bZ7%E^y`Q&28AZIi9|H^6>;XcGZ?z{^pixD;}qn zW@d^bM8^o(FTzMHm{QzzbQ5PH<7@(^IQ8_ZZR19l6HTs?ej?x6+7bXd9fY3w?rI~4 zZSStDr#DJ`hXK)?!Zv{QnkQSRTx(au;)7R509Eo0;KX;UHvkDjUalP zuRoa(Gll#F#2{Ym{O2y?fA_(nMs!p_$aUcE2NS*rH$?rsBV^o!@(%?Qi~T?lm2b|K z^mUp6E_KNmh|y2+FQ6z%@*>gx`-@s`E;W6B`)6fBg6Qdm@~a`XDuX-sXFhzV8w`!F zzwv&HMmJt3#Ys~lU5}WF1DpEr#jdA~1-Zpx%AsrQ8)M5XXKM@cPes97%4eiYUuC?z zEUxk@o?=witI_?@a`8UAKbvtaU+izy?B-o;``Mw}nBCU4IH>$yWRYwv{xP&4#3Ur= zPV#@BM(2jS!kikQON+7UCokyg+$(G-l?tAiTYBS(N&4h0h?2~Zogw{Ym9h{96*lcH zBN-ddQDOAaER4q8NRh1a-YY{aZaa1JL{Y=eC$kRAi5hlL8rq?N4MFw9R?Ud|#+vk( z=Di!4vW<}+RBsmYA2iQR#Np+u{JZ{S+kF43LV91pn*aMtBK&9hTAs^m>h&UYws^np zs0V2HrvE5J8%$U{FubX6Rl1+Ijq@PU=Vj&;PqK&$ysYYQ@!Q)SP3v(EFn|q?e445^ zs-TxgHSfbV8guaWE$8cl?+1%ohg1AN{>yLc?B!a*rxhLyjor7pzbaaA_7+Qx*!61{)3ruflC{xzf@=#){%Zp> zSGMxl$Q(BJzU?pGKCyFqA^oFKc2w-(jfG!a0rpK&jffV7nEcRhu_oF>Ydr)B4z$C^ zaT51!R1!Z$ztB3;VqVBzxlTz&mHqQcLyZupPKDoX<;mnwHs-AOlH*C6@8$DR1EZW< zc{YP7gCwJkDY}B#s><5z2Kr2L|hgeHTpvT#;9*mGD&foxkjk> zNQQmsosWAd7Ud**ddvDYN@>Cbf;&Dg@qVW(8HWn;auaa&A;~T z&;0GG{GXnzWLpt>lwYs#%a^Y7_OFrp_mACO9cFIG<)RjT@Z>a!tAckk-KW?0i^D>{ zUTBfSo|`rGBdy03ic5UI+&;dyh*3IO_;caqfH1ZzhSxBT{Kx?NZ7P3I2Dmrk-S8n==buGe^G8slva0e@enJk@|$n<`RlKab~3)n*!X!_(R?5i za6X&k99D>Wx8&2oo_u@FRKV_sgk;88=JtvGao`7<9qy79Ki zQ+W1#AD^{!O6f=A6+5u|b+&{}&iiVHdu=giHhfhMysT+oMI@)*M1^Uhr6L9fk_MY^} zZ)x70Qd{@l(o5++qE!@Tn$nmn^1h>S*X*K^yioQ#-rNEIcjRKWeuj>7nljz)?IEwF ze^$*k;(I;x)*I|T%s6=Ll5a)gFUHig;@B#@C0E~_S&P9}zrat;opko5p29Zuqkrqj zc!TTaQ{vIqt&aySiMvnQ*)G0XT|e#XzKP8^DxW-5DTTYsaK^v)(z#AC6|=~5^K<>T zeT5VyJh`uys@yuB02AM&2Xx#D=}Z@7s4&N3B|SE ziVmOKCB+nkKfb2!I)!*fR~ST73KJxLBul(Sz+z^2m*EFLTIef(-;=$$i-Oso1)nk* z3K4j-drmqVh;a^f(AboV?Q3zuF6X;yz4p{ zdVh0UR_6wN?ecPmnFoE}6>e&-qzrTb`-m-C{`UOJiwGwonxHU^|_zL6& zDTL(w)@vMia^p3xuIo_3Z5h_Z0}tQrAFLk_CRhz9Q;Ub$-5%S#VP=vSQgmYT+!eki zHfj`P`{JXe*hew%AK}yAqDNoUzWnR?`i=81(dMBk&Z9^8l=;EE!aFmAOxu4Fqs(_K ztS6IX)Q6idj4U)->pi_OxXtRiHe61nz~7(7KFHtuFdhv@L z1dqS|TBIb|j3<*nsu!GlQ0~(2`^A&R^!8~<`=tjG(Ps;tU$hk}l)Vk9IrRFyF_%Ui zju}g>y1EVC=usw-**MXz%IRNUPe?pmlTH6SP{Y2tOIuFvlO2rHc;{ADYZ2YFhx(IG zJsKX31u2W<`NN%(BNq%;gc9?+5aF8ZR=!PKY z8^+Pq&z_H;9C(Nx&WYGfdaHj34(|GTC(~{X)5Y_rA?M+q+{v%6goVyW!5T?Z{Dw)| zs#Sj`V;(;Lv+Wa~|BTbHit4t6WUJ5fnjcRziBGbedtY=XDlOhzKjV1#M^?1PEotD% zR>B9C`=tsIs(qVEJ9{!@j~$}8rZu!mg4OxN26hHVn zFy(HqZnM63=%ZFAZ*-WqQkO5IrGry#H8`3pm~V^~{QF zHB-JrI{Af(k<^}KVN2mC!QLt>#o?oVPh<PYGp zjgGPKyZ_BCKdzKscPL7`C4B*%cEDyZw8-X}!S3}*QX#YQOW!|!uivUCPTcHvDK3!JCR$_^BalDnpvu)LDHkfL7pu?=9^2yTd#l6Di=y}&c*3e&X9hCQ@O07>t zRSG}fa~`(zjzZ;zr)6GCvpm?^ zM_N(q(>e-4tJv5M*|gE~+g2|NS|l-D#*#iOdY;@!CirWnRVj?Dw+IP909H7A{&}d90PatF_>MvoUAd zy_$77UgJ=>!n#Zo-t|E??0-gS?bGJi#!sN~r;lODsv>_efGEO>D62Z?Qeo&fc&F09 zpPj`6s!UBg-5X8a!=Hxu6DM8SkeG^2W#TfN{Yz=etCRW?xgY1gjik5IzMf+8`u*X) zm!t>}kMvyl0T1%t_*RNM5}h`OoM8%w|IyibMm3SHaXg?{uyA)5P*Gqhil9ghNN)-f zI-yrtf5hB$0#`6(ZzL^z7a*_wJr^ zpOX(WlR0zd%p`C5{h#;${9>j&rLDZ&`(oE_M<@<9cR-1H{|Q{tiym;I-+yqIu;clf zZL}{7{iA}4njV`dXLYdX`1}=4Lq|>8!}xcWV=ujF>U&6^ZqIZ_n`T)k2+pfSF`#E?p~klKjn_+Pbaa~HWM&dVnm#^e z2*ibbJmR~&XAf3i*2!^w(=_X3F}+R4JY!~YycMA?s@MGJJ}-a9=8a-F%1zrlp4{oA zGb3EwG*~^yHK9!wZlt!22XQ*vET0YMo8$!>BU^pE9@)k0kFLCRouPAcWHcY)@Y`2> zS%Z0*y8Iy-KVk4*FkCcZg=Z%A#{=EWYz0dHJkmb2R355B)1lW_xaK#5u5Y9OASu*P zBVkniyO&DA%7ZEx#0MYszF(6LwmUW4USFr8q|Ddz{fiuW7Q0x7CQ;?2Ea2DYK15!ao z%p-@BZEd4YY~;X-ik($owJkXtG`Fzem1?hRHn>=uu_}@ z(Z;mhPxAPlcs6e)EIV3^t`>WXWIA7~j$Mz>yd&jCij(Nl5M@bS+`6#-RJorK?pQzbQhQjuIBp( zASS4De-wrlxL@lL0s8QZRxtAz=DTl)`4+FJ%y0@ji-mU6_V!ohFRDKoU3IxoEIp`DJLkt{Z` z(3#Z3Gw7Ja%{a`o3p}3 zj#Ktrs79HWUqUa5gw)}fNqI5K*lTIH?BR3D9@{PZjb1ZVkgrSplU6=%l@}ud5hH0$ zgb!Zs9#&Vw7bl&}`{0|*u_1`c33nf#&f|A5HTCGX_N%obYcB!uC1zbfXn)b~fq+A| z5bhwd{X8|GCk3079<)H1U67V$xtPb_X#5garOocJT8&>)f^~&UI1PT9`F!)V(Bb5} zZ?7q1)3@w>>(8e3zO|ZtL3tU{>J-E*Gwc*T8EBLf49x%eki%FR@SuVMlxb5kS-$2n z-YOi`sVd34)LK&1}ih*#N+*gvGLP_m=9bqcbzVA1)1oqz!SNJA5i# z{32+k1vSuD2h%!M-CMMzUApsi_IMJKMS^7{@$jOxo2+%`Qgz1?>O{$q7)-<&#?#|CIMvo4 ze`M`RPgnlR%r6M-mkn5pbJ|mK2zxA9bd(w24%TzIaad~j+X6G3APilF&*_UoS;Xr1 z;CdH7@*GJU*L2yK#odARe{3`Uc8(FhxCvR{_N$xk-oe1$*2Z~u`L%0fG74yW51C(| zoO$H39!ONi`-&o>hLk0yv|OcMJgDkT>CwFS#4TrU8Fo1r-^Wf7-qpGFDmwPs)u^v0 zDRE|}?9Y^X``z1IE2NT?6xP(Id85ioA>vik7}-RGTgaG}axa=%{vNKhLwe&vH+AD9 z#abxraJfwJAPp5T>5>f;=LrHlUx|$>AP^r`g#*e70BHe`rEsuYBJ6`|ySfadhn$Ry z3cY?Yq7|U)^*0iunaIfRkh~Hd=(7gJ3ok&^G*XLfPid&DhW^sWlj1~49ot0Hbhx);Bg97GrdQda6RrzMmTKhE;D!W=NGG19 z3gVgXP9?Ww!9+p3jP?csWrFHRxV$ANVRRf4a(L%6sS#_*7_S<&Y+e}#haTd-g2&OCZ z(Q`8ht=iZ(ZYxJvwDeT=B}xNB;$O~o{6DdMZwIgx8RN2f{#*>WwC0sA!zT0h8R-0Uf zp_Wh^8~EXqS$9^g$2NqNN<&knm=sq#$s-t2N5n$aILCnEhNdGvY#M++<)jy^4#B5~ z%W9YR@LP_h6d<1g;Hs?8vOYOo6iE;WtvSvSv35cGKVN(AX>?zmtYntTmaMB$sT+0W zX5dOyJ$I7i;gfOrg`Gr7rk}yNhIjf-KApzSQ7A7Qk43=o2oFW=5{B?E*2D_9;~9*l z!i54i+mTvzK0iEQcMSxJg)Lqtm8=8i?<-GciL8lye8J#ky9e(vlS(Fd%h5v0;FOr6 z3#bxIZhAkb3$PiU`IY|6v+~>9Y(!uV_qXaBO(^+c=<`9=fodfyDwn9HV)g86q=E_C zRcIrcHA`@k<2+q6QED}g444!IF*FPd1wc#ga^42>dh{X1MdB`0@AX#_5)R; zi{=5CT>~tN=i}`l)qfz?e~-2Y7;He9DZBk=UjVT2)35d&RP6!~wsE2gBe)Ht{^=E1IBVc55}J5D<}L|`h`!I`SIfs z{@j$l_t%O13G0ve?@varname(x)) >, fontname="Courier"]; iflinked_getindex [label=< if istrans(varinfo, varname) >, fontname="Courier"]; without_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname)", fontname="Courier"]; - with_linking_getindex [shape=box, label="f = from_internal_transform(varinfo, varname)", fontname="Courier"]; + with_linking_getindex [shape=box, label="f = from_linked_internal_transform(varinfo, varname)", fontname="Courier"]; return_getindex [shape=box, label=< return f(getindex_internal(varinfo, varname)) >, style=dashed, fontname="Courier"]; getindex -> iflinked_getindex; diff --git a/docs/src/assets/images/transformations-getindex-without-dist.dot.png b/docs/src/assets/images/transformations-getindex-without-dist.dot.png index e18e9686fe9e653086e21720b0bfcf8408068247..a869326d3cf8275692d38ae5525b404627954fa5 100644 GIT binary patch literal 41849 zcmc$`2UL^avo4Iifr=Cb1e7KnrFT?7l-@!w0@8c$V5LeC=~X%eNTh@onjpOiA%u=} zNTd^5D0k!UoO}Ogt$WWp=UaDu_Z1g0Z`reF&pb1G=6MZORhA>UPJNw-h=}CXOBr<{ zqKmmiMCXuKFN1IH=(1b^zb>09$jK0$p8fe&pZ}GJ=mF6ynP-|_sedLty~r1)+BSVX zgi@3qeq#D~=Q;eGW;FYI^47}?>YvG(yagXF^VzvpYw0qN#)P|J>4{S6-?pm`0e}=Oz7;`5Pr_~&nK_@ndJX`5|30p zn>IM;gKgiFgm9^vz9T@ss&%-nQXeY!L*0#4tt5osCsg5DQpa!jWxrX5P)1<{o zh%rkw-rO1k)aUdmGygxRCst%_pZReAod=W*`)K+bwsz*nre2lf^OJ4vnkZZR&xhF2 z+DP_T*Nz!_n9u4J?6I-wLXWOx!ix%gyvaS@qig{VL-`EX#`h$1L%c0{9gir}bd9MF zO;kfTMXG2n^|8ev4k={4y$c75irn26EUAYo^STI!>^aI;Bu?;@xfzHC+{QZf=2KrZ zi5RM~F{byths?t<3l{9&-QLOo#e-c+5%Wj7E9}=6C z+HMa??KIL+NNjPW`cBK>8_hlki28^gjh4PNqgyWdeOR=k#YZSK?^M{4PZ;0%VO;1v zHjE!JygMz(sJc3pp5>`MU9dz?`%Nn%a7cq>HAYNJgN`o8nLxcNhk`W$bbcjIVmZPZ7SIkc~W=7d7+MiKMG?unWe0#NdcNXgkn59f1yTi{DR@vIxK}4I-O>vvwg1ZG8=HQFb;a+V8uIM<8!CL|t*{n;T?2J0C`1l^-;iBM_iCajDiKpRwnhLQ6G7<8(yj9 zuSq6_l>0meCk#Z`n^C+)V7%U127cT z(=$}GuYfeY_zuKERsHM!9OzEJ``t@4655gy4tyDKC5jT48M{S{kf`~y1 zzt4uMfB9x2Z)&0F_t|Nmoln+bjX9My6k>Q2JWY*~z-5)8k-!}^^6Qi;?&m#;iFGpj z4&i)iV4&)Mk-zJ9IP|8MLcifsQLKR3P}NFdXdj1Q;LAp#8h!geF>g}oSS(p{+MOb3 zb@Z>@nASe^z>}{zCL}n9K!t3I)}rd+E4EP`-X4yua0~3<6GhRj59tk(29jFRK_(jW zWZ|)Cn>orjHu{!m0Ezb}cOY4XJ(oUEm`K$3OzE|FlDos$!=WU88Y=v|TC-&{*a23} zo5&0XJ*m8D>Zw94@4O^-z41Xzlxm}#C2Q$HQSy$_qf0a~bID71bcfhN{XT{4EyJ9P zZJWT^c!SQOS}a#X$xhn@J$|665(=4;i)^y7k#H)usGn4pSN4iOG3I%3s-aLdF){fPUf}xV^gd@u z#mn-t{)InW<;pb7LZD?Hw_5Z|uLQl6fx?f^6Ce>PJm6C8i~n5Op6~NWLbCQN9nb1> z@U^PxehDe~4UYd0>BRr#?BstwF14frghFyy0p|6KlwYHQT2XbS+k=j;_k!Jpg znt(b^<1(pKjPkV##dc;3;tuPJ_dJlPu}#}8tkaOFTLrLsvT3w->C*a|cC7n3BG&M( zBTKNdsy;o{E7W4R5O%lqeEstZ$mBs&_LY$*=3T}ZcN;jnr1{<)sKG42ceKyed}^M>_C^1{{vsY04d3^xX-EI*o>S@Sf0BwR;XN% z;r9WG3%b&7JK1pUpdUFe2}6?w7g*v%p{jikg9EFv2>OD(z)*?pg^79v#xbAn$HY`8 z1IcmRx&1Z9KZ2ZXlKuj)i+@u8uUj9Uzv--*WObs;E=I#yySf!3j0{V%IiyhEDg_`@ z810AdqRX4`D%Rs-ykT%4CgjRkk;6z*<#C)r)Ayq0iq)*BQvEBIDk#UoA6P$2{e#rw z5h{fpJFXNvcjI4WJ+9YLu&D7a6Ow^?lu9byT72dO1(szr0a@UE%zoW%WZ|y5p^~Go z4t?)3hYiNNd1vOq+i40uJvMH$>{od8`fx zvK;4~`ubMZiVGR08J`&4@@tUSsLEL+bE!eh7wOh+*2o?T4&;8I5wuzkc6Gx=JG!qI zP$V}j4S5)r1{O{8J;6%teY~^hfDvPC#Mt-u|5|nA8VZ5DvOHEIFAFL@_Az_;*{Ai- zP*127w({=Ma$!a!E8FI`7Xjk>X~8|i+0$(+UTYsrEo^XMjXV8OfxptE^NLz-Otbk6 zUH7jSfLv0c@p!k?&RkbHMm-4|w@Ds*Eb#)z^Vm^WZ5X(4G1%?AElMLT@NiTaK(4Cu z9}k>6bVIGc->vB=d3!~-t=fLGd=xPt5f{E2@4R^9_<)=86Y=3QD%lbd_fJ#R$669X zCZ!(t6(D-EM#oQPNGI`s`sKw5=quS)-H$j5I!sw+2uPRXg|d9EaDmn>!iD z8uo9HcyH^|B0DPbRok;z%RH8+>&Ku=`Wgr~-?|t+-Z&q-DlLD)m&=Gor53GRqY01j zrW)U>0P#QY@iBnXRD>KxY+ ziY?y~awv->xf22@+^t_rx&JAcR1TGJ$5G}FP5`vP(!1_ep}gJ(4k?9z)|1y zo6CBeT4K8~;?O@33M8orT%wCOICvZ5GWkQiLEkY_zo-VO?l!ohJ~-gz&v$Uk|BLTS zJyKiYQjdB`{e+yaD}{P|jtHY;mpF^HR~kMql21noFB%ulxQBe7h16_0eU zQ8|e4p(v29QIhmkPuEZ5jGAnJUP`A=*c!93&7#q}+<~xfp!`$c#G(1ul0Kj9n_up- zN*thJ{pTa-6v-YOm-K?pQC(r!7eyMQ-l+;#zVcP9G3p`Xq z{zceL*uDG|!M@=#aYI2x<{odgsL#{v+0Tq4Qmb?~3zbE_&Q&-psv>TU6}^&{{6$X{ z!zqCO6GNPo$HwO!<|E7`hv9K+=2zk3E3a5brOd~b+dED^nMs@g=lv@WuNYa~s;OXH z)UU}%E?RxW8|&;>Zb}y!GgbB}XU}GAJE@o?7T1i_RBhR+RO+W{;Gei$E zMlIv?WN_%vZG=hS`@!b1!WkCP0*3K}N`b zA9s^hrC)Wh!JJy^d*=O4j-4MIUn?4Aw$E$$R<`22URUdB@-o#Rdh$U>Fql^avlBVR`MxmpdL|2SjifsN3Zm>#R%+STEeOiE-QI$({UXY1m{Vi4Ls$1 z77l&+5P_5Dl2*s9vA*>c8B(ntV>S!`6K8rRaX^2*x*+3`<4mAze4wy zS6N91NJMk*AN@MRI0z54(ylK;um0qDssCTQwEJt9F`N(|>pwa95Z`X6Nl$HUJYB8M zxxEv8W61&4-SCmAB%YlmWVO3pGwoKx#$w-NvI?g)^;|ui?Zy%#xxb<8!I(d!@ZbaW zu?hXBg@YhZ^+v@L<=|orwzZIKXTBFtv1s%4UbYIBxh3f#a-N+H ziVN2JV#?w_k86^SuJ~8NY(xr132J#`*g>w-;v^lSWa% z2}F)OXN~?yrQaQz``QuN1+mY)(jGY=Wup^Yw$pwm{q;o2@g|%4r>79aOP4m=!{)i` z_T!qTAzKL<35JMd<9s=|>-@(D@f$BE)q8)H;t3U^kGdZt{Cg!=d5?!S55M#EE9sjb zXM29+)EB}r?27DGC&;KUWV@6u~cHFNiu^)0!se7iiZLUGk5B#D=|upbH9wA z3$B<|m5ku{xh-8gg3&5!=J?cYlZSh1cXY|emD2nyH;d=>?7Igt{U`sFB`RgOj1FY4 z;KVb%GbXbcBF8wq*Dj2a;bM=H3*G1Hj8~3QDph0cvramUeutrlX$yJFhgW)Wemi+g zNqLG%yh4(xefJrc+M>|-w!1syq30U}Ek(978r!w>J>{z6jn6TY8$OJb{n8B6T#b@osVYg* ztGe1ZzTN6vKW1Qw$uGZUHT^Jw{0&z+OKPth_6<~D%b4eYDuy8Q-h)XW0{2mcHQMnA z6wtBaDpA@bQtoz?d7FGUmlL>@5jAZJF0?2fTNQZ<>g9MFOApoI%o1weEuInuuGi7& z-D#&x=}=8%X}iJ0R6zy0xxS)US>? zNnSw-`8)NEtnOV} z)o#}Dyzxrto=HkW-)a9^mPe$oFj;CX2;nDBOVSUq zLv;#CFb(^DgKs}bmvA=fuSnyb zNqiTS(&VdPF{q%WVm&64rqn?dtul+wO#dSThwi+jpK9==`tfN&BheIWWo5=dpsEWqMs( z+V;k>GC>_N2;FWi=H->-h5nZb#+;;X<@DW# zfZW+jmoXQ(E_wWpM?rs=sVmpTa&6IA){wdwwbkp3;IAoruC8^XOY|;h%5ZL7|Z*9Xl7S%%h`TlSuTBG9uXY3sEC?DlkNQM=Icvl6;rK}na=zb zyDpHk*PUG9l`^Z5+9fXrtEWl8B1?Hzro@afv*o^Iy3*v-b{#b~c`PNN+O_#C)=%EM zYb3bYbb)Dbyh4tHVR2uy++M-w5p*6~&SN{c&(?-P2QAqz*yZqBAZ)Q9qEj@ZDZOF;{~7 z=}jbTeejiuXr^P3c!3J#n!0J82HO=ao7uS!_>4)1gUTAn>h{W$SkaPvKVs=1702G{ z9PSU4kRk(R(b#W>x-oU^1=_(|qGfL)PLOP}g}O;&^*$w%#=gV4Tf|5`A$W09kJoXT z4Sex9joY^bi;cGAOiOrwRHFW(uEE$xCYVpLCM{{K$~|T3#585sNPhGa#8kkm&>Pit zy+^#@2^j{rl=jG-0mYTTrxARU7Cp<1!yh&ZEyIu6SSYav&ZY1Ysg=GbEt3OXlD)^ZN zy)ZTCiaB66@4bh&_p!;^4(p{KTvuAtqCvV@T>Y9EP1hpe@!Fo5)6e*L9Ia8^XlT^% z5IR**QY*FFkO!AwSQ@q8w#nnBZOLJjT3*4mESL$rxhE(qS<1T9V<%$r-lW6H4W%8L z$9tDp>mz=L*VmS0`pY_}*pj_I;+-Qlv~5b>K+r&f`cs$5o`wvreMi4V49(upuOVwY z-by>?V&mz`A(Co~a5KkxbMSb&2I31+VV4|<2YMJ!uoLj$Esyh?nBH4E5E4dZ*L3@Q zn{!5G2Ops)T>ZAX+j886LPnkU!?UhcA7>nDC#cr!`4?G=4Xa028J>`G2`o3##4dW) zr_&D`A4I6SAyt^ugL-jykZ>C4;I|kCjkz5HcLGzm&BTxvz1t=VYb}L~yxWM`ZZuyB z%KIUTl#X(&$J)>@4`S(02eaO!z#QqEhp%0*vzdY>v$H8FH0ply4fQQ~ zQWh_2v|$(AnmKQCwQf;QVtUZHb6f~lqUMUme_F9QJrcqmtohZ8`+lq%Abo% zaUUYkHAV{BOr3i9*`ivFO|IFjxbsA+Z5w@;h_pqvfBMzmJ67S^#AeMH*XQJks z506g=_Mbs2%-={!hznHyNkw;7#Z4s=vL&fUv=Ci{4pJSZk}Hwu;;QjZxu81QDa*r2 z9cV%OzWL!?w>J4?Lniwb(KmY}YKQ_Fx4o|!%PASt>d11HcjgyYV5BJgv(ri_?cyUI zm7BEN|9LTbbvCM0HV%nb^PRrfdm0ehN272sQP5Bi?G#VSsrWWN1h-ul@y|H#KnKgn zBUZ!MvUg8+vt)b^!fy3olJ|7AKNj>(DvU9LEG>iHePE zy5s{pTZyqfN-n?hq~rzVyt-z$ZbSetUuxG*VGg(2)Y5&iMjwpL&tnodX$txS9TIQC zm4XGO61+2KSgK7?)t1^Qme2Y3S4R&P}gYu)U8K}Ld>ix&s`+WanVo)Gfs`uUFURCcZ z?#b;hG$E*!HC7worXAXThPE=Om0uagNVtlzS+MD)dHy>WfWS0Kt22f-cvd#Ng3s#} z`18x~(B{}}QhRd{MT_~8=3_lCN^3p#f^iKNyY$(E^&0c$QO6K{zmPhizsPzhF9UBi ze-I$e-BsHfyvGPaKX|DSI4qI~yU(wAf%`Tu#oxr3L9+Y;5w~=n zy2o*`%8P(}mCH+db~=zvn3_*!eRh*?*0P;i6s|t-NTn)Z=iTD8l$KBea>L)rBz~+P z7xj55+#eb6#U0-l9#`=vYW46WVIlcmLh(n%&hdWZlJz<@c-7XDcBiRkUH`58;iv$) z>4KEDTlRtP$gcmBn6rQyeUmxs-@Zgn2Wj&DT*jE+1K9>rOaaQDXSp|FT}z?AZBFZU6r|S>k`TBmS%0i&1$bXgkbs=~ot?^&Nn8rq4jNJmHIqkilZdO`|Re`w^Mb?DC6HovdqQ7Td^-8_x42?^SL zyhUi}H8K{f*`J{?ObOl0Qc_Zrfv3m0n#`cZjSipj1$Tsz)6#OiauH1vxtq|PTU=b6 ziqk5BoHYfjsyx}WZva{f{vR4kPG6c>k>A1Al41Lb8@7S4wyi1)Bv&%0vN#C`+o%sqX5I?qmtI`J&=Zgjgx@3!8<{~?a^nQ@g|C5 zfP7Gc8kdGDa|?^b9x-qyQU70hkv3<`WB%7oZ~cNdASHT`-V`C8Pf&37!C!gG7lEyHT$v; znSbyq>EcObH`8dZQ?!0Me{-eTBs?lAs=1|Q+fGPWShyem$A0P2eq$DS2;^EsOHFKa^tYcsWtf?n!Aj@?)yvDvckTJe%ZJ5$ z{d%L$NVVV#m)QI!9v@d#RdxQt1@rPz@_Y9ZUG=@Zs{eQ(!8Z#F3vrL$bWFCeh&gq@ zF-ig5Fn){?NcV>dHO9kWhGB|zx9h5|T)a3xj=R{pBu|pV6sHHz0V{guRV%p($;nXg zk2G;_^)rP&vs=S~S&t!XwHux}JbPBsGNTHPbOk3Tr-tq={Q&06CBU`&yeuyh84OQ2 zR7KKBK+W1CS4K+BL{1~4qWV1#2D0UDq&A+-V4A1q%a<=poHuUX>?318AP|;zc+bOF zMl-Ra(!4yfK0ZF_Qy)7!JC{g4SJ{oa^pzd$69^o_!a5HhK7^`fzJD$)O>yT=R&;c9 zYG6ja_}(u~&>F2fg9mfv=g;5I&d$2+ZuJ3Zap+!S?Nj*h%9664t6RtZ-5)2B~t z{QUe&-R5bc?k*>3Vz0Z7*C)QJUtpEE87vKcGBY#p)<Bq^hc{k-~JR2T0fmm4;G&eVE*0`7hSbYY8YN~y@PC-sbXEcA%5Bxd#3~|I6fR~;L z;I;AXCcR{_bnwkIGI8_ic;79{l-|7S6zm12tzm!X^ZdDH_KS#ABVrR1lj}EbbfZw2 z1(vE8oEAFinzc1+F0B7KrI+y0l#9IIsat7_;ii@J({Xlo_FhDn>6M7gXnMSc08u^# z2N$X5%0o2^)s;iy2ycM18kC%Ar4^t88Cbi7n{?s@)z!kQV+f9ZyoB%W{K~uEPQrhu z0o8o<2=w{9)%;EBXpkiq5v<*y?G^z@RIQ-w3DtE*En$cfYG z27e^bb%o=!q`A5I!UPXN;JZa{vIyL}=G3st!7%9J1tORi0QI*wc(U4z0M?$eYp6%pw#bF;JLckhDdjHF_QfM44HvE&sLx@%Oe8@ueeTnEN^ zz%0eiE}tgm*>~`%5ljt8G*7HeVw-zr#Ml;dxFWumGm=Hcgju=gZ4Mf0hL^Xm&NJ~>QmRi6j z2gK1X3>GsRGv+0$ZI_ zo(Fg=dXCtg$mvoS9m5FB5b5eLUTzr$?nvC0Zaq5K7Vz0w7GJ+Rx=se300C+}vnX|| zOGLkIOcL{~jsB$h;<3!Hk|q6pN+~W9u_lhM4lnkmAWU1Ya~U@naA+0(YKx#*aH#<< z#4KPvC<6rVJGfN$?8D_$kNZGvK-s_=uZP3s-J%mkeJeNMy z8kM(G0Gg69rlzJt`6~2=&x>VL*oauWL8QQsJ2qM$Y%KuJag7$tpfD$g352%--ExcC zgT)j)wg?V2?~aFpM_DK6AC=w2mIQJSzlZjcZ|m59qFT})9V#tJ!FTS7;g{a==T`r7WeVYI~F#Mg3h5Pw1P(sgidzU7wG&`e7(2zWy|ufC>6LjlZbFA{DMH8JkL zyhvmo(!DUawa|?RndUOaI1n91!2#l_CV&To-KhF{0|9oXI2NU3ryyi`;wxYL%tsQv z7>~`yAuq))%}Fxqk{Y4F%<9n#Zp_LesR-u^qr?7(2-=e@(S(Jk(2GgV#w%Zp;Ifv zBq#+AleO51nsQNh>{;H*@!&VR|KJ6pu+NVCM5KndCL&L z{}vl|aAfE&kW)`r=(9jS4ZNt}MFe$NWMn*3Zf>si%3yA?RA9qme+JxnuC3Dsakd1t zI2dxML|%WGX$#lt!j&+ zhc7QL7YRkPXT`=Uns?@jx^s{lKll`G+NOjV(UsH8^p%>tqcet%rPO%U9?7P0QVY}Y zJzvNXW7tc0QS|JPAtL3w8f=g|pL`z}w(C-#&)%db!ev4JEM(2KM!=13>-z`G|>V<1FEcpyGQ0Yz@*LNs9oWL_#B|c;yz|&|t2Dz?+WS z=pt=+MK`Cp!|{^?n#uhkeuRYo%EstT+D=Q&bdFs|yzk9JjxB5f2s%=Mg-4c z+iuy2)ULneq9?>u`8-o;ZOc!ejmu9$li<5*<31D=zgJ6K;;_(zv?$x1qk8#n>tXH) zE1bWL@BbnyiA4RSDb0%)MUEa{Fn4wvUb=#+u-&!A4pczjUtlkAK)$0!998$AaI`wS z&)`$jvb{KMwQnuljnP;epYfJMt0_tI#WB-itTy?=LhcJVY}zobE;5|i$|NVPgzo?p>@5wvv{nU}%m(Q?avh0%*d1{7dFanSiFBN60e zd0>-p(7_m$NF(IHxmwo6nozTY)3q3Xl%Fh=+AzWIgQQxQ^z1G6$MrU-WazH{Q9xFm z@(Llj^Mc&A3GWt^q{V&&<~-Kb=0SO)YB<(q|`#7be+ibyP3SguT5fa zHCBx7eMFhNRL^Exe&FEBgSsR6E7g_P_bz#Zz3iB!tZLoE+QY6CLK^1N-5l5+hsrkLPIbS(%CvMB<@J*aPJ?82+A zmny&3TIcXq;lELa>GhC8*$SgHVPl%zN30SY3RJJ0$Ap(IAplQ?>NKUu%)557`v1(~ zL2=}Kx9={O6143-*des=NSI*>6cd_2CTdI4Z~kZj9KP3cSZ=CNLTT)&G_(`Be# zX)*R(_VNje+2um*zav?AX_R@(C@JTdSyawPSAYuRD2 zh{M&Fj=kY311Ft(&@A8l69a#Py#ETx>0nhFiIqji_tvNtVV02Rnc{eijrt|_Qa%2R zWS$Lj%+`(B1!l>;IeRQ{Wh_-4^Xk9giS6sA7 z1Zc*HQ3K8uH9XqB6V7uzDfJ|%0em-~fT7~JdVWg%F)^y>0J#`ne16z@^)c{xF-M-% zR3kA&?^66CmP+GeC53&)LPPgKl=GL}(WDp2rP*SM9*iURA3r)y)mP$cNj=q}K$%_> z^i>u~t_oHgW7zIY)uT=@&llp-vx)xbfD6wDsbuhkD#K|A29ZuI4eI+Fx{h=fyh^5G;tTyT) za@1iy{k)IHj2+m2FLJ=^O!YV^x4+2IJu&3oy-#Ei8J3+-ZaID7?VU0ZD)Fo1+83_e z94b)Dep9{~<11SxfZ?N;%ALN_ke@=$rJ29mi_Upwh(=bXIXGym{1~3~`aD)ER_p#Q zKh@LWNXO98?^$CS>o+F$IS?6Pu z%+#un^(&}}4-&Qk@nQp?kYQ3_AaaVr9M9V(hADJMul(m+mIcpQo8YJ_CJpOCYi6li zBNWXTjCM;9iAFdj*AUXv5nZ4rEgeM6wpvC+l=~LcHdUV+`K}5G*pF#u1RS!x8=~mV z3ah$>q4)K)ImwU0#ve9>8E@tWxkad#4D(L{D^#^}h@jkU zs&y96lSFo`2L%0@+O2yQLQEB;e2WkHvC{j0F*KB_ShD~i?1*L7d|kp;vzc?IK$DPM zwYv=jH>Ez*g6Wu6qQA$J>&X7Frk*KzYj%$ItjC0^Wr2I~sBTV5T_!3@M#5#P7z2l@ zH?{LFNio)43odE3)srqr6_Oh9*213i7Gf$Y3IG_}G5N7k3k&)BRSwjGc8O>xLl^L;ici8hw0=>SG=a{#q>i z>PHwp%#WZj{~tORDsJO<0Y~4L$1L)I{(T+ILw`Y@j~R&Cy|`_ONZ+ttz(|N z3a}lsu|wQTlDa)J`fcne zr;Zh6Ak#4RbHWS^m5oPlCmszlckR#4sP#}JJ$vzHUz^L{Hr#8iDgeNQ*Ca4eY-Q{E%xMY+E|!1%bnek1_B!|Iq~cn+e!d~#~9p6AFniQ&Ue z=vvh!7b>(rUg@~yb9m?k`lwVvN?%ks)z2}0>}xR=lKQ1B@L9OmyD|-W$*Xgb40>24 z4maeM(wmQ0x3);s@GtDq$EAA${;Q-S+rFdDFw7vuE6`mnh(sUg7-|OJq92}% zl7b=EY85Crv^ZB?&LGEAg^2YbDB55?+>$H+FY@3(E2dXK=o*am-QF@Pqs`W;vCl?N z%;3a>=in1sJ$$WydHTi@HkZX{X{!C|pdn)(?!$>aM65@}C_%V_?f2r$p>@}7_xilD zqk6loYqka4{eC6h8SaqfT)RWlF_v(4Ph9qH!?ono<2z|W@mn=&{=Sf}-n_4@pH&r3 zpO$+rIjhAH=$oYYh#zxaXJs$@Q2oyOy2Gbrk+1dz-j@brE6l6ogrLTkg;p(apRii7 zlN?_yGIQT9Jl7Spn&cQ|NGyB+3*lt{_Mt=a@zP!}@4GDCCMS!H?ST3hA4Oajebbl3 z`^C2<`b8e~;Gs3#EaQT%4`@CwMT;WuWNznO2N)cz9R`VG}npbDQ+bOQz?K8koV>{Q?o_ zsnl{lV8}|Bxqb0xd;45&f>Cp|^IUXdVhBiO!)PhLgc>#te`?uCut!DvxY5!%-4RO% zuL;u|(eY8jGm=6P1k~gH-a$Us_~LWO4ElF$9R3^6);z>#dd=!r)r^cHK}p91lpIxA z$#oh7Bn%7;e%=fZ2@!oDAV2}i;Ax`!eJUyp3}i7cnf3|Or;#9jf76B=-4k-S4+`C1 z($o8rG=@Hi%uF$>nDw;w?mO8_iUtcs5ED@iIVD-)lXT?S2y&BYYQSb?0?}4o;BOj@}@U3i4x5e)BK@1Hez((HWaA)NnDDn=XP@8Quj=wv; zFsc@6M#~>SK87q?w4?SyLS(*!Dsr(=JvEJ>UDLt#k~3ko%51SGDN)3g1vEN--i!(f zDUb%G)bIA=f_9^2mq3+L+TUMdt!9ai)1dk`pGD8xxVX5Hhj&QT?t<5R_Yz|n818^L zf1%-cSKseoAz>fP#p&e8`{L!xvunuppy!!Q^POL3em9wI$jH1&-|>+EyO;3KpNXyb zssD^2ZV){-?;`I>7PtY{$jRNELPVZ{f%g;arM&Z^5b2i?J3BjKz8q>Ano>}Y_dCX8 z_Ca}Q6Fj)Nt?e=yi((U~hCL2A^7>3KMF(8{u9zJIL+vZ{@UZFnRQ*04V_fRH=Ww(? z7bWbuc7I}G;xTxs&u@3svU(}q_xGPaoE`;0s&i+v;3eZtBG$-PZUqZ*j>xhv_1TLT zmq9N64iqP6dxfSyKDfHEz1{Nh8rk3RXoPN>wS9h9Vy``UH$5a|`OdT3PV*h4M~66g zP&?R&XQEbq!HuQPCFTl-XbJEDzih^Wc0Dc|Vkt@x=pH)~ld8OX+sQAS3B=&QPn3}tg` z>m|==f1$MofLq5i1vP;&{S zk1gMxeYk!5_FGWBAigiG`ey??AY4;RYZmO6g#(U+L4PBNQ7-Z~s0()V_R1-*=jP_jL``750s;c=@mt*njR4|Xpqt?3}40?hk zux5Qlc~a8JpG(eYE0fWC&Rv9Y60R7#3|$~6wq zuJ!WuZRzce@U#H4D9*|{FDom{%F6oG#N;8k)(JbrmMZGrwc!iA{jGxkpQd*gW>!~8 zR8&+ZoZD%=b_V1~LLWIgI_l`^hSte~j#8VMI(WGjR%qN9u#ADzvnZ#IEN<(SP=fZZ zu>T=H=yAOxd!+m=C+8w4JN_Il(N|MdeVdoZ(z3A}S0SOdzahbaor23Y3kU~t*4AvG(WPJCH!wg-M@P3j-6(bW=KZTeLPCJEKo3ACxTuK3z|e4@%F(z* zrSJ9w293hvVi|B-S9iBFaOReNKFhv)bb#^TplwWf?_NluMt(Dp^ulH0w^da_b@lb$ z_5c)Wt;fQ$>>nH$dWxhe2M-Rar%L*dMSo{!V|&6>3Y=X{PcIBeM-6jD zt9M_7bY}n|9JfWl{}jVg7UJ+60CYh?!BbsbDqyjC%t=5DH7%{6t*tGsK-7v`$*>8y za@qK75@68*Mr1cYaOFN|`PqP^7%VATviasdrslZ3Na7#PA)M+@KUu~@8LuOFC* z)pjP@=1WG#G}aLUfnYxW7#*dGjEn^C`Zhb85yW}oQ%^THe(~wL&*kNOfM8;-)7IH} z4S?lhF&B0qd0friT>uJ&0)!aBcK#~T=IK~)85te@ z0WJV3laq%>5CVZ0$oSg%CWvQJSy?$+K9`nT8tct%R2Kq(bZB@u3kK`kKx!4qnwc?z zZ%1a90DFaabylyqq-0!isub6xtPiyQrv0T30t|rlvB6Xw}@-)^`8S`kz0-$OP@JEiRVmV+V4s=EP!omZ=@ zrmmimme#cHX6M13mzygJ3C6Z-{dBKoPtglAGj<(o;8u-_D=U$^rVRLdlACe#$j$?T z9fX-C5G<^}PfQ$B?JJin$wrO)v5D*E-VH!=+0cmeef9tMK?8m%VFi`H5KcLV*yv zJ$bJrBQ%M2;QLOp`J=9QiFQ^;N7XJ(;}w@x$ zI}9;cSpMlce~rxVzkjJfl5gJ>`~m`~C@Il$b8|JSZD}D>0rEJ&|0?kez+Vg7_i~Y^NNQtD#>d0kF#zYH zkT-Z48~dSx!gVgoQ9s&8v8mvKNy08pPfyc5k&beHpOloQlw0hyrf2-@*)$Yo&Dz%W zm=Gu6IuwF}^@E|Duu95`unedHz}j?SRc)$H({S z?v`WHuDAxFO#1O-f3Y-g2`W5(IsAAXW+`W99teXtWPE;sDH9 zm?Tf{_ztdSPjDVFI(-&)T#b#5DHs^Az!z$KGQL75_s(0l){zpjh?0_$`LW8P5yLm} zBoKN)&7!EG5d^}x7Kk&TR%r1?L%&W&76uMBh>QyCamV1?EHzW#yuqiTp)pwh)ug}> z1L;f~Y-a>KCJiku<%0(*%h$j7Kq^SV#B>dIhsR};=BbfUTK{ua+uzcDetseaY8f>( zTFb1#uKL!T>8PVr$F(IwWKR^2ymaea|My%#l3O&kqB%k0E4qa7UV~bXjml zyA{CVp&*2{X0xAHa6jJYc*ku`4q_&(FiGE>oV(x`gURS7Xuw&fD&<})-GmBl12XHD zA3q4-GM$~B-V~Rg2Fk!a5(UWIgVV%eJO7QcwXLlQ{sQPwOjKrOCJ44rAPj=^kC<1i z<|pQ*OCV^w($>}n2l$&=dBE=lV9aeGtvNqGR9Bl2LyhW09PaPm#&=_6ks2nisG(W; ziJ(8Yb*NbU0t3bA-om{pGJ;o^R8b*>6J%>=$L6+c;SL!Pd{>}Su7s8ri9KJ^h#^=^ z)UER@NSfy7&!0iz;Q?UnDhxM}2kU?S668}%`VFLddU~*o1I#UfxKi%!0^o0pJx^Rw zBF4rvNHR`LjHIB!q?GXJ@zm!lSfdCwJz&OZRllPKD{L@asH;(7aSJ{fnUH|xFNPK@ zK?UcR?5VkV$ol$AxN}I|u-MqxMv4v5FXEDiMn+!G88jOh7zjGwnD{H_mz6_Od=3tR z{{H^;_4Uud$VWhsCXsi09Tw&fVFI4W;T{-rYkNDQbpk_$2kw|@3z=SB4F?-OhT`ns z(V;qj>P`vy6rnIU0oQJ^W4xt%EavFQHR-k{7#$r=0lo=tboqL)oIFE}SegP-r+<)7 zY6k4x2h;;b-S5~gU%4Xs^eI(uiZIK$G16T4_wbQ=MN&|l!M6XJo%P+{ca}@!!amwv z5m!?qg2g2zBWuHXD@|x+Wfc@0-1G|lQe;ew|F2&LOu9879wUbnC;}hH5WUVXheQmz zb7ORPm7Y!o#`w9f5BJB1CsL3FLZSZ(u~1rCLxB3>!Hm@qPZS;PhnOfG-OT>REZ8lQ z<^%nzprys&7z9Vs`02k3{?93gJUI+>jonH7#2_vcH8G)s+gx5=ZpC@qnUBP^EG0^Q z{@X|zoUd8B4i*ASo?~jxM1jBqf_WIxc~f4P=2}3&=yup5J4+xUO!Jq1piOkQ;5p`oF;A zi+Xwr!J!7FsG+6heOPxvpa1rikU|z4G+l5%T=;PL@?~#0>0t9_=H{^6u&8{&h$*Ho zb@uj3LPB=9w-$kUmFP40K$5U_R>rSmW7@FYu4Ry{P|v%DHcVKKL#nR%Ui~I+#3o zV{j0k3o0jH|2ySS)q=%>@!S%nvY%4lifcO~~78NBVdGb$U^U#C4+ zr<*t~F3@EUFCmCSzHTiMz(KxH+5+h74_T~ynKmrDSp?`VU^~(YUeNYJf}Nf?1_?AM zEWMk2FHu7k2RGxqR=)>l$?V;YLBl`s`%@wMGp%WM&%Zeww?$*__Fb8+jg77N9AbVV_#Vd2Rvd4L615P=mdC3J%*sg; zxvc~0>jkN(sUhG6K!`5ynv|-vzVkXSbDrQpVAahGgVciG^=rxvos8ME*B4V++D;oA zyi-DIndH*Rkt`hY4u+VotV@U{c@%n!!^EJ;2eWE#(#`PMvw;T>;gQ)X|2?Rr zq9Ptx2`k8Az_q^TvAF;`FcGjaP*&GGpv1iVyy_+1<6{4%5yK>DX}qYC|Ckz-KgeqS zA3+~9v5*&lsbQR7o4hbB4Gl_pQ8i}R@smpjWg(V96H`3JgXv)$Nav~PjB_YTx!%!0Vg`ETT_*@QW1;tv#qAL zON6O$=j7GV>vg9EmWi$z*@p$o6#sW!JhOyvK)7rO9Eu>9j7@_eu3La+fU}cobeN>*%Ys_ za(wglqPpV{!tiOYj=H(P%I8yj?{!XT8pft0z9gRFCI?o`%tf39zG;^13=C}!Y83R- zmgLb}oo#sads>~9esr&In0vXc*KM+;#1MBj4fn>&=bk<%SD^;DBzy@DfE1tmJ}cx+ zzn}oSbjSU=gM$MCmxBJ$9~|jhn#|Clb$&+{HEOppwCEeq`TNbU{i|*CSsj6}@DYRO zaoQ)Nr5hg=*LQ`}G`ARg-`Aj$3h`>}Oci$DUX1Ufot5ai9lgNv%eXmjWb=@l$7Y?$ z%-rGm$AVV>DW~aSQ~eGvuPT%tc8IGFP(qa82B7F3sdM1~H^Az&ssj)NHDrO{2?_Zn zI4DzB=gRl_tgde2rbu}_re$6p^Ve51&FCHJ^+uj)0%Z#2jbenNx_aRDwmn3&<&hGk zSk)+dj+g8_a|P5<_K&W8H^d7XCqokv*=hHh*>b|VzTS&9ert3{?%J(oUpgut3xES=;6#z#2LNFu0(m9O$dwC-~gQ@xbX*BE>+{%}&1 zRGu-Q+|c8~BFFHom-Z9%hRWI9ITI>_0o2H1Uh2s+eotBHj;I^@M*AzcO+DGlZDPh; zfM;GOA_^!jW(D&#ny7I=Qf{IBU+5T@E{SI;O{WEt!@2hzNppN(CM@|;xJ~-kcmLkC z!^Rbeh*r^!^35Lz#`kvOMs9# zNUR}|K~iQYoDql@l(0`vi8(87h1rzw9dGv5iB7l#+3)G;SD-j%()`r=a?7{zWW^fEn91GOMRQab;7LQsO!myai{T$q zWeW=ptUpW}&f$MKmhFe!X?Y=>mwkhNdAa^)SD zB?%4wqnef^9NK$RzwFD&E=mZ62IEzJ9xMPr;L5jmA>EfwgU4}-v@aZu{hfYwe=pU2 z$}pOYrM>#S+;7?fQQxV0g$Cj)PfV{+%u~*zUA}-Zt+h}jIe)uogHRFo_sq5&*gLW0(5@2P)KkX0xW zVcFe!IEaOg*Tn&py$%4b^K2M5669eJp49R>pJa~YGaF8fBb%G``@3qxD*bbF)7vg+ zFRhmRhl<~ziFuHbSU(8+I)qs`OaIwH0CkKx>r~VnaCU^#m^FnKfoT<~QQiFW2g#cm zZ8nE3j+-+B;*G`x4hjIag5yV09U}mJr2$M(%54QymCbrmK;Upb13*@M{_dr>vMjXU z61L;;Ww_1$;$O)e?vnZWmGbSp&4$iWNc2)f(&y>`A}_-;F#vr(*RDant@(Xv2jag7 z1P3S|y@~al(JCh#Zv$)=1z8|aHC>OT(Yz&as!KT)-iC1Lb^zw;y~$!5*eMpsBK&!@F(X6A{hMMceR1X3?Iy9l2rAK`!Cef!GF7RrVI*(@FJ; zE0GDTnQfezbMpH9Fzs1m+rQo9fZD1v|G^YCBbdvVUB5WB;Bt@?jj_r9*KpH=zGQaEez{w=adCpC5DQ~7c}3zP zkOn~!5gkY-U!YwnR-i`B&W?{{m*~h0~;m+Q^-B>h0#!% zFRz{P{}mJ--=`kf6uFi>6FRFu_q30IZ3G>K&&bpB?qZQUw)`z3V`RIz+vB0!r>Q{z zGbV4mU}0$qQ6C3tSs$MZt*xyF zCx<&X*^Mtl6`Q7(2e1;g<2&zGa!rcC7ViNZuXw^V~PqJd~HmK{AhpgIa?UBLNV zhYTL!X`nUhI>36Mdc*@%=wH<)`5~MMAb=eVqB!e_R@{4mHHr3+K|D=*gE%*PiYD@ojlR8kMLfa%#(1Nc3L_|EJpMr%27r}Ut zkOR!813v(Uu*AlurnBI&kTh=6b?H77WQfcEUnN_*RSNk`<7qcIl?fz4Vh{GRna}z`Vg%I+OQ1C+ajC}nk9#C|#btjMhE8JMY zSyxq8QvzrNy+*|R5>RO{w;|75{zzGw0J%XtJTxG+pst39YH)UX()snPk8IX?jsx(B zrojJA>Bjn=ya3EZRh9O<>Q8D%E~q;UjADuVfi3VN2+<7yjz+yJCjd@JJr3OAL#Qo- zT$!AloDg&Z5Gz0@fR}6K=o3JPC6ajf5MVKJ{{c&ZmIr{&@O!3=GmzZVU?R{@1%RW-b&Qf0KsReBk!xGo60_(b zH8+5?K0ZF{gX;RzxgV0n{<~zSeES73g`l8O#BiWSjE;?c3ScoT7T^>J9;l;3iiL%B zUSZECAZcQa47_0gLH)qQpeA`~WAjQfqX}9~khO**9gF~$1d;7A}P5o}D6{@QJ> zYk<(*WnyXt%o*B#*#4&k09fQ4O2Y7lEG3=%$vHCh13QpepC<~D3GuT-`q)n zI`{ttgZ%|+3djWlp&#X*7%ku(BB09vb|5l7J{W`m2p@qM6!>a1h%{wo4<2xFRk(Qo z@v{$YT5>np05oPtj=lthJFzNIR06QsHi9!Tpz%_iJm9|y#AhcV8-A{ z!13@vkxlM(Y?H6wNDfj1M2`VzEg)rM?v}8sCW%lY<@wtSor?;Pq=PI8mYGf+TQ`N6O8KW0`ed16m1y7XYUsR8c`ym_SvO zl7RRpmPJ1tSakz{Lti>M{V!~Gd91RpcsZDW9+v)!=yi(A7lfcrkxLf11^$c53JtJj zg5)JpiGjxn2?^H;2z(=i4lkpwIX(EkU=OzQj?aN%C|}bTI{unQM@c}fgH*1d^k6X= z0U$=b7+GA7H@Kkh)rNP-5yNusLkZJcPjQetIxB zfxz4}f10K^h#SKaXWM}k1X&1RDOa>k%ZK%5 zlMcX3C?Q8D{&V0R&mhp>5`$%|kVh}s^T93aZV?It1osH~g-PIU%H3S*^&-m6hmlj0 zH<-Z64kwR$!f{VY$)SI9jZJ-IK0Ua+cFZYK%fan2(EDWrz zaUt9Txr6YyLqlrI3c(Uvzz4)13nFZqX^O|t?=^9W_7@-ulK%kEG#MNf1lxfeANX&v zqG6!BNMOgsfSn&n!^fK_1Dmd0sjuO@$7wUuYW1ZIlG&&O_Q7kpu>yjEyniX+=@-3M zSEzB`!>sQ^u-2Anh=>{ zuhNM85%=qzr4< zyXVz8;WiK3E6dHwieo=xZ1RV2Nb?+$X?OWa^E6HB6^}GlHF~JtC^avX@0;&;jl5G0 zx{B@k&J9mnt1dln(z|g)#(lZPu;%sw$$7V*5bju4@Qc7!Om) zWKO*W>!S4|w%gk45!_x9Yv$Qrz+A$~`4e}H&eimt)N8OHugAYWnW6ax5Nd(C)caECC8ntm<`*Oo|V9gnyoTy|kNHl=0h|Ev^ zI>iIBjz+6WJbsE0BO6QUDsM|`5AXW=JD-1SdGs_IaVqjxRJl@TNFVwc+C;O!#6S8W zqTS^M;@Aonc%tFq;68Q8B%XP^=1XZ)qG&}Z>RoTU>4w@c%wI<+nYOJGINJEe=}h-_Y~ zp02aDuyhJ)g*AF?$BGyxj7K`#%0dN2d} zCvi)bf<7DOhL7lr-|qy3ObZ&XpN>U*E7z$$75F6Vfo4aCBYCE0+evoGU_RPk`Ws<| zUT@H*rBhwpu;%;fi!6cyWJZB5pJlAj0~;D{+C6!2x^r^x-7z}`*}YQ7HPe+TuGz2G zM0yO}50q%c0&CJL3+}sOllqkGv9!M?!z?IFG_=2M_N10Io0M-nVEgXR1fR2^DiIs6 zyUT^HOL79-(ieY5((WI8Op=HoSo0UN8YITeei2r9)o7`zYpmqUQb9<(jPAS4sfEr{ zbCnd=V2jrJst*0nO=T*}+!=kp3P0Nf@`mw;8YVOpSYBXW^|-TFquoNeu9*3^^7%c< zvJcs%XDzi6a!rw*IK@u8Z7UIl&39CjR?v9W%=PFy%t-TW^4m*DI?RZ;)Cl+tL1fna zp4V=)oC!rqOPe*ZH1w?gF2+|1l%=q7%Ng@>|1ILRupyTPO1wj+W33zxt3k4$O8F7? z#!8BkdSz#$o!MpzbD6w9mTQ{#qZcqmzmeT|v${erO{-p{{N3DVeKC2lEAL45-O}9~ zq=!|PH@muI^L_nZI8|L$>nV!D#ClX3##Gi-;#prDLGfZ^lFP}HhLBS?kJPw%J*TaC zqVW6B#^^W|Q`g7FGJ_vB)B2TTIv>dMF^bxRFJt#GTJhbtP{bx!9%Z;>E<8QW^5kkk z%M1O33VuD&>wZA!=uA4+;m5}n|LQ9BIsJN}x-Lm+kKT`LM_uRzjZs&3kS(jpSSSO> ziHqS9)sv==VXnUJ0mT(FeGkLT#8)hd+oQ!TrN$!F_McW{z~erW=Cd0lr?~7R-|5Px z5UF{9<#n|2Iyq}2a=ou)$jAP(80E=RyPWn_)0KQ9C(1p;`MY0?+i2Dbi9h~Pc_pm6`F7KgPuUvPbfPPG zmwJWMsW`uHLTWxgs^Cec$x2bPownaotOD^m;WjeljfBRd%b=0h6<=Hi^X}K7T5LYI*H9<3%`MfOD)59qm5zJK+0RE z*YO*L{l1fs*{gmiS?eNZYpX^r={aAmT?%8Fa_n%cCcPM*7!oHm@yhq%jU00e36DQ7 zerK{Y@r+Sf92?*6y`^8ZVyY5a_4K0w zwGCeS_b|cmy8*lHZoH4K#?EGCkQdZZGcd1L;)WzJ3FnPBUp9ZJY zudm#Pa%t#njpz%;8;8OTC*RlF&WN&qYF*9H=cjn!XtK3ffpzsg4R`10 zBaDM2MnC2I$-^n~_|X;kqU5nm&x_m@2VD&-NPPsBOY>3)Ko`M$ITugfr`5G={p)D} zW3o9RP6L-@aV(~;jY_8AA9IU3*EDu|RSlst`Dr2+;-*Qh(M9P1ht5Q?)foS=7^x|( z8pC)_bA8^MrIwO0@vFqs{*>%ZuG_ZL5sZ1O304CEq0bQmml{)eoN#@b0x!SgRJpTO zZPtvB?s&;@Y)H!Snv~B4zq6ytO$){wIv>0!NbAf< z-r}vu=+os__sK_gN?z{zq|KsTy<=$TUBy_y&yk4l(-*6i;|wcL3*u{XltE@+3Kdj)ytf!Ie4hK+q9?J zW(=*9i4BWFwibwT)qI+8pVR|%#X*Y!jY^Z?S}TYye=19!?&@&8b{4Z z$)682Sv(o57v;U#XbJxBr-s0h-nG+8R&uA%eY79hp>3Vh;;h?4UlioYv551+Ekf~D znh#!cS52ROd&Q-q?>kC|hF?uuF+5t7<$2lbE+NZ(`HsUimS3ko%~`>Qf{u#ElFHC+dH7G&YoDq`mpgw?@1+x2Lq})={^droU?pl-9=2dP%0ck zrOp)HbLDlfezoFH_^EwkzAePtm(5&k42(v9n-OEJLD9LOQM53O5cj_phq%AHfuck7&LW8f9OlS6c(U0x{=M`^?)ZE;K z@8U&)dR{NSF3|ElDC1l?k^D2Z>b$f*duh#J*njff^->~XvilEJTe8M-u(0_zqCdoQ zXD!8wOH~xqJ23f=)vp;iHQ6lhuU{GG>?Y4@q?`3Grnr`HGbD5q$MLOX#Z_O&Wn9Th zN^Q3hO&N-T!L-$cL~8%J<%6mpq9K<07A-DlIcF~D4c&1wLoQpaU)MFi2cP8Y9@Vhb zRFR>R6;(1nV|Fl1@cf{AWNmt=f^sN#K+zPJRuWV*_uT0qo8SbuWaE*DG99n5;NQ(+ z_EX=O3-bJJL~40}-7&IDgGNzZWABgS@G+_B-)h?OKoGivc%gE#>a4KA?mDN2E7F<8 zZ}iO7O9DSQ=}X_nou12QTy<243rq2`Mp+2_`eV{8=in46O9}Mh1Fp2(f?nmnVU)hh z6f2wr=3P4hTPfOt+cEp@u`7c`^;QQYqTfIhwRPQfBBE;lpz40=9nyRj&Xj=hGpeoS z@{1D{kz*(^%NrkSZFM{|*lWMwmCcl}Q-t`w@PygB7yJ+q=$@oXetcs97dMy#Kvz(u}APKqU- znUs4~*+M&xJ>|6f>0+~rWhorYgqPXBnG$Ghot<~)D4Ov#2dY{mQqm+NYq~|oL!1ou zL-#g*a){=Po_f7IlO!@rAZE=Ll$n^Gy@&qMvm<$YXo}`zsn^5spYqda9cE-r$dOd9 zhhJBou{kuAXvG!#-K_2YcQzyJ%vZ&=2ggnT;ZE8tln!XKQynqzTuQV=75nk z$oF6v_D`Q9)rU-c?8kgMOryS! zZ#!kk6-ha$KE9M4ouFA;Odifgy^<>IcG+dP*7aj1?rPGVi^Gda`%fqM+#CKpu%Pdg zs*>gvG6@^c8=n!ReM>{3_N-ZjbY7 zujtE)@>dyi&P-Y!*UNDzolkc#S$8aq#p$V|M5t0;l@nt;Sa!!sR*O3`;lNuXzHb5j?aW}g3u?AX*0#o#IZ+2@Nm+vom^L%SZ+ z{8WUc$#Np2YtO#L+q2GH{zHx1^u(5Aq3hYja8iPxXSQxk5PW;kpl z6f(|{3zVCz$ru6{fHPQT{!t;3BoHqaBR4~5>uit!dcb-@M zhUxRlUCT{HXQ2b#@mYI9nUM671_j*N(YJ&`SM}SbtV-JaQ>L1h@d_iycs|?Byxq6{ zb@J}p-}^sDp>Xv!D#w}8DA;*})Hx=E;e$0k&m>$p-eOB`x9dvllDNF!OAX^()^TU0 zTfUWlXv42ZZpz~5mqc&V-tF{beaJ@vW82Q#L0kD=*8eyCr;Iyl4Zdqbjo!tIt1ru9 zn+o+QvW1U$8_w=tD;$>b8S`%lcx;~BJm4)o;C|H#kP_?Y6i2pEV7& z9{r*^{i}Ps!>oN)qbKwb;^8PHCmDdqxZtM|sHW zHnYJ}v*CVM#S3Y{8XR1nV6rE;->Yz0B5s8Y_jR|9p1gQBXR?qX_-M#`qA%iebd1nG zg%f}B`s%*TeJG6nR(V42$OF>6_-h?zCYej^W-C2YfHqUHvwu?NUj8KqjEuZpbN!>_ z_IDwP8`Goi@5bu0nc`24FFpvgWZm!|#%mG{&+*%W5veOO&MdBYib*$r0~hEZ96l)L zZ^XG}m}|t@6&zh7c_Jw>GBLk9(1WWjCok|teoRG!SnxH|bGDf~cY;#jrN@psg!~D~ z;w0+=j>GEITL*@^j$s!TFO%&SITIwZq8ByrVB$Ot6p!^yacDP+kH;N)7|~pk@B(ln zW+tcn_LG31oJZTu1GwSor|2cK6zrJhp@$uM1E=E4JOJGBy*A=p{DS;p=0H!oBaV`J z&p~nvH{K8?kj(bS`5fe3-R{NA0N#dw0Cma#^SIytgV3U;Q-dZD#t0<3bszY{&TBpK z937cUQ&l$9K)K0G`65Uq--od`N&wwX?5Kfy{slDn?k{{ud3n5VGcr2*6lpqzj%Xn9 zvHAJ=OIoNGl|Y?Nr@d#_Cm0B{|J#C;pu)#_gEKH>u8pPf*as*dXk-Fnh8nonZIcIT3T9h=Ntt9z%7+io*_W>AXqju5`iwf`DAY@0_GqA zmwe7QMj=85q`L+F${i+~ffl2^?8T4kAf;?P{=)*TU)K??nf=xOVEKB=uY)?r5#=Ye zb4fpW(gC{KRJFY6nHis~M{&?mNdVKWq~+u!6cwWo=(zv!$GSRFXeL4=BhbpYG2D21 z16Az0V+ImhN{|o0WzNYp2LuQ1fe-H7n|+|_jzm>{G- z0_4%H2yy4iLzfO5eV)S-()XG(yS%c3_fPaUFg$DoT7BgHU>L~8SH>a=R@UneRW%U%rgaazL^MgdM}>ER1Qggs1=1qJtkJH;m;KnN}9JP}P(_yr364(PM^ zH8&Rs{2vglfRTrgF$14)y*CJ)!D=(Y(D|1_!;_Bb$KvAah{PO*(`~?l8v)sTlhwd` znnd+n3Ghfl!Xzm8E)X&3Po}7;77c-6zBZij^~;wg*z7RSS9ZVWC4d$=rpbgm1iakvl!-L)7-R03lPjZwic(~<(D2GE>J0je16%H?E`A-;eAz7^ry z-39`(d>H1=O*p?9{-KEOF2qMW+?)p1RFf`Iykr=H2ZKm>AV85P&p`kMU?lb zcP1g~{VXu%4nH1qgR#lfI<9Ii{!IE-!W?Ozw{;Vq{PpP0I) zDA7OjhLIrO9vTC~VDX`0_ysf;C2*LABSVg3W3xe$3Hu=jY9!!>|5*-@bT@T90n$)ZKo=f!dzs~ z1=PU9@rFyNyOq%XVO@|5M5{SW7?qWm9|Ex^i~yp>CKu2E@eGNOdll%PPRN*11G3`@ zmryxN8ZeJOCmKFC0S)Q0Po4}v5kAR;-dHoBh)IM!>tL#DHjuS2``QeMwNsoMC^G>y z2{cqMEe%ZDRm}z)kd>DH1~Y7-g{;73>#xikppjC6x^=cj>+bE{waY{)*dh1$K2I7Z z1%HI@TEEI(6?yq^qWfBzuz<*TJY@W>oLnd}n+3Sd1AZ0MIxp-Y3R#GK5g06uOxq?P zh)GP$gV7=&VvvO)2ns;z)$QbRS&aNzQ$R}{MZV;hF>+f%Ffw_jfTJ9m0FbGX$Ogb1 zFHjUYOuEe7qofpFSXf|h--i{?Aw<6B{Vf^T>8X(wN9&|q7=X-p|Ndk756IZ@#o?k9 zn5vu#s{X9(?2lDdBU95CMBFi-L{95l!IF2L3U5U;r!lX99Kq><$(S~B}c~!7!OM? zlfY`|11wt|=SQI1Pe|{1*Z4@lWbi>>u%^@@=x$pdiU)TD>faE=c%W-f8hGgcvOt?z zNwC1wSV}xhl*kS>a<~Y<0{_66^mp%aU{pbYMu`}B^#nGffL`Gm@2)VISapciIU}9i6 zFUdh`>%ISD??x-Fi~f(jTSr4SeBc^#Oy;irzwcex?GF&o&dp4#Z2iB@?BjLh?##$* z8_{7fzdjtL`tocp>&bl3cEf04Y%)F-2p}Zv#zD{=2xEX*Ve;r_I4XBPKpPb!E$u_* zxJPSsTj8WQfiPwv6B!~1VmS&lOq6#@2|NHw1{P(KPZ=60`oWEr0W}|XM_3Qq7x#v8 z;2~1LExjblM~>Yt1+41ld!rSWdGKTfTII6PuL7@EOQw0z?04mSgL04&%PG{D6j8_r zDyWbTXaXq;E3`6Rjm!p9f)T~!uJe@TFMdeI>DUH?YA+jRY#j2d{CY12KFE7C0yYBm z3|g+Jq5mBjeb3FU4l-4?a_L%gQ zV+Al}1QzJNMzKhQz-ACKnE8BUbS{YFzJ6lz3cNNTOUQT$mAd*g3%)Rh@cQ*%1suR! zXF=Kl-&}vPJBAn`vXP0_4T!BUFf;EuRf&XkyoN35#YT2AM~o}+F?7}?@Y-d2?oVHW z!R|2L+XnGxK+)%cr*Sq{xj{$>CZF0)0cZ1bbs+M5gY(0D%E(OU2;tM5d&=3*p-~jE zLlSPQ{C{3=5yXWzT+e_OR|9wbaVF>0CMbRsVc>D~5wg0!#wC&MyA%BSwGzx@hp{i^ z&$@^Nq1h9LJPZT-KA5k`0bN!v-~fN^>AC;o2pJ0QH0ko8#$hQN>82Va6@1AT(UAd8 zmw}5*4Wr-jEy$%-L&Eo}Zh zGq|#MAX(6@43uWzK3>2JHOziwhcQ?G#s^RQSonCoMdk#gE)0o1bg-Kc0#6S5ywu#? z>mboOE~DpBht>vpIOM|}cf6P)I;a<(H=RMU=Q?)&2w5eeY}x5aN%9|Ppj%&6Qv_bF zkpq)M?s9UfB7=#c`4glN%nl3RAA-XLU1LAA|H?DQuD;}ffxX(uq-tVa`=}HuDcx9b zAn@Lf&e@Ik+Ye;AKEa%9Cpfb5U|HbKKk7F&fa!swao8DDXXN1-iHJf+dBb4;=u8kT z=Hb7feWL`1e-T<;4nZje6PelTVM5PnwcQ7%E{2T=RP6*axC2mPnSmL31`#z1)#me8LfN!F83!XL1w>GH9tPIXLGmH!y8R98RlY{Ih{mci@y0d^mD-K?bA>L* zj29u~KQq(T(aBHoI@JW7#&_tR)#`~#W45WCNZ@n$7y!o@=A0OeB0_?s*16vM_Q%tvg=FB}`qLr|&y&D$npf5ABDUIE5g1I=t_B_@j|-=hDAm zi!zIcGvk)>bk|&Q3%b8SeJJ+^`a+olTPOot(x4$;<~uanKhunbH>g0A$^sMKzwbF< z(SEZ)r&E;<3z%9$Sfp|7fgh;i6qPGvHpPE@H!C(k4S%g#38hPA|al z!wctca{;ddZz5T&0@w7+*m$4+8Y-t8)-T5nMonkI+9UshQt0SO7Sx6bAG=OBQ3~c! zkcpgMsPF&x-+rEufmQo(LI!L+4!1X)E~6Ck;Cc#kRXWUGfpGm^G^oTj z&;Q-#i2VE;QSRZ0t3rOxfBTFp@KrvE*|e|1jF6Bb@!6ODKcXIr?%BJScBWh9u@+y3 zO!EKg;$cjs(B$U)bnp2$yV2F$S(f&iV#VIe@F@u<<_O#&4IC}fSjyJ98~E%C z-!8$oLp2WtjiJ--kWKwZ3T}jpc~Y)-Gpm)0o+UC^Tt(%`zzT^xR<2TgyQY`^>TEP! z{4_i(;oY+OHu0Rw{S$awAF->-s8rC*j-A66+~ZI_uv0n{oXWNw^DT%$}Cd=v@Ut>zf%@ zH`UEgA1(bU{8WMRPVkwUL@kp3dtZ>*D|)0pz7^0eu$8NKa#~#X>`6D}{p49Cxn}&f@waIa zuQQvwpX}87(Mp}MP|CE#{dFCytvq?M@%D|q(bvhhXY0(9X&tNj>5Rz}nY#279ZD6> zPu!Ks3%lf^Q1We-()rizt52_C-aX1)-u zbE!Rp%$m7hDzJrN;_bnqIN8G5-#;_TS*G%$ z>iKgueGM=JxrVJTh7RVdj;Z{%EY&PE@(FJz;*;#!%sT!p%rLHf)KOUBlrJ=?hv{mr zL-12R2irb-lvgUkSDaX0GjAeAFwcff(M+c(Il;b0tg`ViY4gd8waJ4Lld~7zz1FVo zQw>7)0g2IHKUteJ3N_4CVwF_(zoQ*n>T#pdpwM)U^H z`|DeY+yP&R1T?lBN&EkD-x*pbw-c6n|pBlHK?-~PWJj0j1;Neox-lGn<| za8G_BD_u+={?iakdMeuWAzZQhpl6VRM|y`keZX#H;;=97=;N7VBMCdvpzSFU)9?=V z^5V-bfgUoCS0sVugQ-$K%1os`tu9ZdN^W$9kb`yl_wbRtj>S_t+QbEYGBzut&<)ZDA*WIH zT8Xp5kGffh<(2I%mfEx%*|^86TU#5;I|x&5s`Wc=&3lCO{ITOs(0Ot&d$QwU?fk5u zdg&GIht{JC&Ei4bX==6YHs9c`(*+{bC$LGGot7PD`kUMAho|H3f+l1vcC(XQ!nW#ZJs5&3Y+b*imz#F!no z@5piNITSdJHax^3vvO6ppMrFR;)j68!>TKn?>s*(IUU&@*Y3%q){gg%Ec`nev6sj} zx}t;}p*4-@IUdrGgRuR8b}W+Tr@J#4cF8S#xlg_gvLxsx+lFg5bn*B;3LpMd$;RE# zu(dtwW+7}@V8ymZ{QhtzS2mIO;op!;wKv^F-0V6QkKgJuoDOx*FR)hpHF~_+^txTU z!bnYbD_v*!O?sHgDd*#`g3PV7re}Zq`?h5HM`Bbs+S9dqr;eYQ$mR45Sd&`$6zI=s zl-;s-VI){LzvD*C;j&WFba!w3c+haG9n`2dM#coHbIYbg;h1pl3t;@xsem>dr37CsiQgULjmDaD{p17^Dj z=mQr22sE8@O;J))-!ksjH~kP(zMbjNyd8}x)}Mlxwohy7KhDSX?S`CX?>v-juPV1(y@6HM z?4WGh5Xxp+h_1X{Fyvt;QSmovu)0LGxz@GpzRhcsBL4huq-h%I^G zTd@O0mL56P50~_Z#aAV3`+jmtE~AHw&&yvH4NKKZF4T_;Jgaw1(sK43m^0t&R57t)P-qNZz3bNR zT5yuxdhrI5DO^D^g=fw)H%9s8Ryh84J1DdmflifpUrK^pf{Heq(YzNG9y1a&E zCN6~9+TkWUx@nd(OcoaWB>$OEsj~IP( zpM>6+`lZjqCCb$Ez~B!Vly#$mNtfcYA(Oh~WG*@SzaC!+trhaKg@xtNa=m`MnbtLpw;T4@lG_f=Ht?Uh@;z5^Z=9_>QGGDUPF}j1 zGrE{kfDMemGozJT@zYgcM3Nx<1l{JseDF zjBRGFm1V=*A>@6Y%4|O}P}Mj1`i0h$f{WYgWi1)H{C0nG2X?qcdbGLJy}u0aT2}KN z4cvR8jS_kMxl{&~%5H%@a5i%pGj2O!uDD{=w##{UF^fKSk#=D|42xZAO`Y{kzR1d` zb!I1)r%&Ej=Xxei^B?<}xa?~62Ama-vWqniB(ZiBs~C9lsja9Pc{D_6nWIMw?~lFC zJS5KBdr1F5Nz83qsqExg!NHF3_}KPIb_;u({zCsMrK`%DP(Y;E-mpo2A~AQX&>lQn z>d)pP*3T!RryoLcO=@FQh*3Z%>v|4}ATguO2ha6FsMs*L`!0-)!XF5xYN*!Lwi{Q<@c#JI|INU=lAQ3ZVWsE3 zho5LPcf?euJ`JRb&X;%%s|$ABJ(7MUM_bg`L@D|PDl{P_GqPeNv)#YLm<4;0Z@_8p zoJ|}y-YzY<>D=9a$LKd(2for9nuym7o2rGhntU4`o)ei;ZKiI&zk${D8dfIR9u1XX z%Ql=)3+E(@U0{kocK@U9*wTK#@@T%vsdhrjOMu)(K+5`n_oKAG>BkD~EGEp0n8}Wz z_aAjwiEf_887H0I8-LHYeq*B2C(L!Neoe)IW^ZUv_F&N)hReik@Iin{;k zs5FOZuVxexd!~+2<^6h6#NKl?kqZ2o!AZTW%l(IL zp(>Fj8bxQu%9Tq+>gsEAr?Rcu*+y#Qs_sJ4lUHMlHs@*c?incx&J{i5-}T~3w0Ogm zxU+Qi6HT>XvaG(>R>bc|p9qy$G6zbS;Lv)XAs8G(%O_(-$`=$ep9a;(f>1vlxrcw} zvekUd@4CYDh)bs3=r|<$@7KJUnH)%1QS0LG=Z_!YmRMJ?J|rDY1+V zGPfdpd+4B+BdqP?`M-)guc#)sERN^u6&w_~(v%XoDj-~yro0wEBB)N26+ z6i`AD44{a(D54^X6zLKOB=jO(RH7g~Boc&(NDG}L1Va+$3p#7onzvc=F#Bn*^Kj1j zzK8F8`}f~_9~Gd3to7pIMZ`3YcgNxhX%UExoIH^WnIrgNsYe6fYT!mQeaLPgy0>S9 zCr0>aQ;MnjY*^`FXMX-=Z7ZzpFLjWE?$sz$VZ+&?yN=wxBvg(JR$V?|=YT z#qr9hMwMK2X_DUhTyJqF**=|L+qF>zm$*(e&>`GRqa3gG(T2#p839g(E@BoKHdS|T zxY4#6(dwZ0bnk!%*vp54jf&NqH4v4(Lnc`+IZyX&3U@8(?_bWz`1A&5(H^J}HL z$co6=>GAbkZ26YuF3b2@!O*V8sciKwBBk|q&5hNbgdA2Z^z(XEc1}>Nk9Jnk_+~0P zm~Z;*g5&&798_LTd6IBNA3jv{dN5Cl{gvfQkEn22;(M_%Oy%U3C0|TfQCX3WIT&Vu zt=;nI@mhJgo~853o!-3CIp{)rOFnjZd1p&qP@X5|h?|*Ua_V~YSAm;cQoZwTiaJUX z;kY=w7D=90RSlpDI}T6X*gCJrh{_alQqOXDe7l|2F2)hzAI`+(*hBdM_i3{B_Q6gDc_%mIRHi;DVG3!1_3Z)g>az-G;R*U!SoHHN~DJ6 zO>V=eAGs_~)wfg^T&Tp;eT)5BM;ZO?Fr22wcGRj8IDd_Fs7><8 zz8Pj*w1)qR8ZOFozOC!ph#l^Gm32t1f3DC%*D+`S$NN_OM!h+}F*8$T$g7FwINmse zwn!<6>vd!EO)>n@i%c0!D{5tm12$uCG@>tMu^6M;JD4npE||`{3;ZhNuOH}~m8$Q3 zI^XR9fpSi?PG0iRuPCXR*BQHQ6&&}gOOe_lq%-Ns7^g6I*+PMWz28RMeu;f$QF2Mb;O;#nN^nVmMgY4$gFO3Q8(BN67o*=GqUe`o&cqF*wBUMtWMe()fm!<2hyuFH^tj8*^Am-UbEi|p2MbXrR=RLvs zeNIe=BDSQ*j0I=X(5NhO@X1Nv5il~;?XD>~e}SO&!1`lPejxh;UR<`{H&?7q&wCRH zBP;>Ey}$&++qmKef*<%7r{XM)jWRAOd&gaC4~f;+YfTgjyI%b8G014gPT;2B`#h;x z5;^eIxiPzz;J3FleOy$>=qJpdYjN3NV0SOLdjYARvpPXCBCgp}Re9sB|1BXt`iAV+wb6Ew^I0-TS-X&m8 zU=~m@^sQB|h>Pz27{jBUhI-@9tZa8AwzByBK~&`fi);!kaS7>8bCwACBsW--NXlRn z#v={v_Fpx^m|s#ETEhhl$$^!$X3J(SC``w3Z8zt{uENq|#nFWW!wD&gr*05!EGLO_ z3op>D!ISE?Dnt`-CLt=<;?Tq*<;apA-_DZ9W0fNdL6*saYBALCy-ZgC zEXZ5zn(?{mlQ1s)fWSQ|V2>jHxG&-bfDQ|!CTM*3xeS7fUzo1(EAgH|L)F0o&~8AV z`&4d74y8VPW@b%qfWPVJqL!|twh_8xuF8fpqB%Vb-J*wwna>k?_B=CNMP<+n@I^Cw zhqV|DXiOqKJH50=qIP;|kN#htFyrb8kFmk^>kK>&4=_9kQ1<8xWU=5)@8gvlTz7)} z8vgDP+;^IDs=<8g&J_l(=)ADv*q2Dm~E)%niPCk5S3gy$6nX%a=Ukm2u20+w5IAx2-TdbA=D7UlvEQ6FT z6qC?QwQTb1iSWu>P<`$OwIqc(#vO~&?|!%cqu9PaFZ z_=6I#S6gl`bhH`d2a9hsX|_A7LSl*@<37N7bE$iLP0!W$feS|k;By|*ClOX>37q30 z0R25hUKJeQ=D^z^I0j0dx^+xTtDA8V;M-7Lps|92f`W)H`{@8u2Qv#OclYhd(ErJs z0g_Dxs@(-(F%ZR7q?w8AOYnd1g+>EN9eD%5P9mYG?F}$Zh{bKmr=`3i=T&HNr%F^yM$^5T-{{)(I+4=wg literal 40798 zcmd431yq&aw>FAMNJvPxfPhGMg9r*pNq3iYcPU6o2}qaHNP~2<>6Gs7?vDMf?eG7; z_l`5ZbI!P9+&j+qI)u%;H!J3P=3H|<&wL5_ASdw@l^7Kc4(_Rxq_`p+9D+I=+@ty@ zNZ`)pE+sSghGZx$Ar1$H{rl0F8v_UT98OAHRM{crmqK#>4;1>%0*J z_$NOZM(`9Kz4d$ksCC|(X-ciEsKB!1WuE$k4&xKam+-x)e#-PCQNKQY`|vm`N;t9h zkmB37*YHxblqiCWLuT!@j+=vgm;H17-Ww^7Pe(ar#s76>jWOcsz+wM$Mg5X{6dL@m zD;c-+-!E&V{%3C&9tgV6)z}y6Hr5sPQvK^vq^xt=Zd;8M(cRgs$14SICUIF#{Hb*` zbLFP~_h$?y3N=Dm(U;WK{TwZ= zk`=VEh4o|Vs|9nAOewR9Q$z^B?6DuP~ZMuZhUs}T5Z$4o29hJ z=elzdzS1*i4*2bh9cA4`J0bZwk?6sqhc0TSy``^4JB^vwoK<|jdaO|bZRfMG!NJ|J z?-$|07>jwCSDHN1>`6wCLrEdu!zCp9-mf0b$ZwZ9^V?e!qpPsLzBH+mS}axm5U+ni+RjN87vI+}Q4h<8*1&!T+c z$~YqCJxACaxla|fYztKxR539H1sT;DVNk^mI(-rp!OJ4r$~0y@8Ig3|)JWLMg>s6( zAr$L`V@F1Lp^tG@^;7IiQA@@BjPoo1IwAxY&j5O;)zGntNLl^Cz1Q(D*+=7Vz~#1f zKY_2f15PVWN%~2=se_|D@o4e8XQk<*q{afwHL>i5`j@}cIncXWSa~@R=xwnHH!&yp zv=-(NvkSE*C>XVCLK$&V%A^yU`2whNhVWph4~98@6hnA6_tTx3=hlx4eEXP>KI{Zm z;u7u~6-kV~TTs74J~>C3OnknYlFSfbYjY~N$;GIS{+OdHjvh2+c?Zp1e zgs`mGxAXlk;)DglHdU{%&$o@q&F_#8!)wwL+L*mxz$}RSFGErXr}5~8Ryz_tw_$?-?<@0_s>*i`Mk;g7t zA#Zhjkc9Iazo(?TX6)`pf{F33d|CarhpTaP2g$s6H#AIbMWo`WQ^hPrRv+k^X%>p= zYJ-zqScX@9H-E9|9UU(Qp5CkxY)1>GdS)(~X-&9igAn+CStKH$j#_Wy(AryamN#lS zDwnRdXnGl{zHsAGT=|&LEO3QTnc@tCBGN);B$|h4c;BM!Zg0x#*XR!uH5<<*Swn8J zE_)jmJS~Bn3)FyBk)w~dtPMiw=#ck~L2=Q^YFWMlgtm|tR&?}|M_#E`(f6l>J8mf& zriX0;p6~gqLj!Iybyt3>roAZf2@+)zJ1W0b%u3 z-!U?6oOx=CF4!nIH0W%YKk#EntzxHOlW?LDarv|El=m+skrjR$-z#rZ7Pmy3P2D$L zPYjo82p5QlV>^E~O)EJe%;(F}crk~`0T{6qqRM>+gQZ{8M4wBF7 zFv3^o&Tm8kO%chLEfK~>RRi8D{{Ho1DlCjj(wg7mA3a?0HHgqI8bbV&9@8BeyOy9b zB*Jzym8^fY_&kHwYY5rMCPxSI!AsEliTPN702CWiPVcIuw}|z(L2)5tW}v|a=0No4Pzm+|cG>(j{oj`>w0~j$f7dxI zJp_%nBRVK~$$k8uQKoi{MVEbj0-rg8(IX%ajgd^6h2x70lbFWT`HYHW1cs`ty@jM6xk~|!XkSCRgK9!2C9aB-7 z_Tk>sKK*IMhJVuTHn=DRO#N{^icoZ309A}cie8{2L0YV z$MYoq3CDI?{JDqA`6$ih4ds`wf4%Qrt}syE9rC9pv0DnAyyib>(Iifi?Ha^ zngf>f3O*Ob<3u6%L0 zDk+N}q$;;^U4J|BQ^Oj*aqmji_O)@dr@N~G)8`^lJwK&gz19e+;EZOW(peSTLg**&QzNcD}z~g zvvt~_c;=kV%nEk{$BRUVdfui(`3E6)B42N3W^X9DSh`u$W}5s2hqamq<>&CwSiPZ@ zn`V~vdSkkvYU|bM>P`F6sY613117dxdD}^%OOoM8S+n*#@o3DuR?F4q+(jIYn(K9% zFs*cl!P}n+Cf$79p+>nlT;41k%~urnX-QqwBII;5*Q~3H$KALaSRBdo zmeog*mp)4qWo&0&4j2&H!u90&ZCs&_!MDOBHl%7rOtv|CVlmciqthNyd_Mnxmkj1V zzg|&)vni2VKesV&2U$3#OA23+eFxk5!mtCc-jitbnD ziKCMzD+4^omr@K~Dnf7dLNOn-)()j>j0wM^>)A zuj2(BrClGmcVEW8;&>P|3yeT(QODzcmCU9lw#_PT9Q1dfjMj@uXTd|7 zpSFBj3E?zO-Sz@K-PQ{N3KzW-zB3|kqaaV6t(pnCORGKbGc*N1Z+o%aB~fF4#L&Os z^es!N>)NOzKFoVtKrW8G_NA)Uckw}T;n1DlrTEOXke_%hlNP z?d(|_Hok|u7%16kre3ujJbFvN;chh7-C}6P)^$50Qlu6{9tFRDj4@j1m!naZNM{c( z{=NHP{i5^Nbe7tquq5Wr^K8v;Sk-|FtBwFmD z=-(7wY}U%4HfGnYPKp)fyfv;OI0U3rG>bU>UqsxuM`imL<2&htV!q=X_otPAWjV$4 zq&FaX#$&ynIY6J%D`U3W_EZo$?qPZ`^qhmie=*M5bmJ$7WpZL+N>c)I&Vk|fgK~7a zHg7pc$ZUPOyzqS@Pqi_=y;n`Q86AZ( zbGXTs|NRP)z+O*is3v^->TQHUZzpw@Nw&S)e(u?naQrx;j~vhc$Hz5YD@?v=j(Y*^ z$o|d9cE<8&F6{RUPxd0nWRk0n`&o{dFXTPaP_#Huw$2FInmb~moWItnPz+mlVHVO` z&;M|;-ka*_EFw~GOBycJG&rvq*(f&nUDeKJxpM;F7?>iEbGmOzP>K9@boOvhxM19# zaLBhT!5TTHyT6B$Vg1SMrjV4xkbnQx;a%HfoK0QY#E`polPh;}HaCh2;qrQW_wExyqvb&#( z#)T_A(&Dq)j^=`g4Lfq*{@tW8^zirvLf~*&+@qQ+$L;r>%#zvGW8q%Vo1zByM&xFz z_wD9MHLM>;nuNo{oZ_s*Sg`EsB09^izDad+ottka){psr^EfBuYCzU&<_xAed}`36 z^*Hg)ci^Lqtl9PqH6naW1&OrDLx#~9B=I>*NC(* zeI>A3&iZ-GTFcGT8TFmIH@W){ip6G*FDAB_YoVT+e>w$H5glF2ib~Tc3GS_yI_<{? z$Fal9_^T=mb~{!!qYGh~MgT0|dNuf;A6QvBE(T#SYhz_COY5lNYGaW7zdT->G3H*G z_mW0<{9evCk8!&SH2vbCRdzG6mL$q@ybQ4}B~|Z^oMn++^!U$?*Y-EBo}0o;6BN>n zW+GzBdd0Ldi5-{UD&(FvdYv46u&B|fepYQiNNE^ZQ~G6`sxY|ujC>%7tsyg(dQsNE z-eLh~T5w;hshk=l4hWjZotxL9rP<7mZl zR{K&L2<0=zR`lk2Y6Uf9EMtzNQqSS5q&UF^rYF*E3KQf^Sq>dnRM8tY(V=f8S2PI?1Uc4DN@^STDTB5EVYwduTbYn1ptPRKN*M9L4^K~of) zaEmTZ<|;?H9W3S4*b^-3vGpxPT?azj34F=!`9(Y|j}mBUX>(9=%>5L?#UxU`L(|#O zoM$UDO`Gn`5hh>sVLPRohK=vNMALiCK2cGACO@!x7`F?o#cyV@Jh; z&?UhfW|@r%(a+!R8I*Dl15JnQ)VeCH*QOrG?@N56O+wzLAZ_rXrk}RWB+Ldmw^j#J zeb;82$q;XqusP zXe=cBWKg|g@=5;lEiD}xvy2*lSyyU8u{dp`kiS&Rw*5O=TK%L?2Agi>D9(fKVz`?X z_BlS;e8nE_p769p+e8GZ&ABgjzXnT=f0%Ca0XvYn@qKGsKz%oRD^dOT>tv<(_W_OC zoBH(3qamZ_e&#Dq{PK(1%-Y4HLLcQEzj;eB7yLsDy=Vcn&}cFpj222SOJ%Y>@Kc;9 zdSSh{?Oz@AIWbYnG>SnZkK<*MZc|8qI(s?-PPwYQ(MCG{%$zg99O_iy7$ksw)WO{S>+tt8Qhr855(#aT6g7+Ed zEEWjb4KKJ0)*Gt2b@b!P(cc}Ib+eYPHT#-U)XLzn+9%N_EIB0R+UA7xy;$^Y3CJuJ zELBer=Xh_gJ<#8cfZpKUz;cuJs;3SBohuKp)siY^v}I@027=XXk$LhWSrf><>y6l# zqXy^3fn&wue06_GF#TC1RcPG6Ck!)B$t)Xn>CeN$6CRw_YvCq(EKPTg+{5Fc?l%Ty z`fTX^q;zsI(ykL$D7ijl3|6!4bz~SwzlCN)X^I$kR7be0CVuKNp%ohi@oo5-zIAG1 zwLpJU~d!Z~Wwo4mJ zQM~pk8hKo=*Upw$s>gWQ+Me(1E5?Rdo_GZ$#e!ItG>=lJT#$1`__et9i1`YO90J=91(N-igY6uCL>*&+d)9_ z8B-7U*X!w*^Y-IC!;mgJV%@P~tyQXK!?=H+yi?UeCUJ+MCIPWBUeG& zQXi zs0P|&8{%4Ell{?1%aKaUbCtg;Mu(tIJ5nJ$IrA1L`hr7=R(>OyPvU3~Usl=w&dBZ# z<>|TKSo-x)7ela6tUt%;i+X}^v#~#yQ!3fMp4FpGwzhAij4!5|j~=4EUziC<3Y;`*32?-a(5)^idwefd?ZXv@jF4p0yh{ewfJC9>Yu7@Hx zM;yzq9U-*B(@ulqKIj5E{9kYq$cE&1F-woCW%j~}|5UWNk&%e{kblD)x@;!hRF4bk zi;~?tSad({TjcvNLlrMx8{7IWsM`*{0+Zu7w1{!a^J}{mft&=lL}Z$NM{iNX(KL4z zF84z|aFr2qwTh`mZ%n?bCsbNGdt2(!&VcUu^#SnmiUI#E!5C|5*Zg0Qq>$;K&ydEE z8zEhp#4*3uWZyZ}C$WkMaHsEZew>b-C3lNSknOTh(c<~vQluL?2DFFmQ>?5FNJq}) zQWrx;d8_uOiH&H7d-P7GU(qIt9`fSd@?zecf0o<7=#w{hR`OjD18I7hXn z%nrA0?jSi~I+ZW`YN;n#WP1nkZ_40WXNzs#j8XaPnL_)?jB-c!L#F4>k4>v4EyYVh zb<6S@zv~gfmzYCvQtu@1jp$h96Xc?&Sp{hJ=T?RP9+_ zT)q0Ihf6o&7rx(z;+dBMB7_43ds^e1wd|HeAqu|PktAeTf^8)1HTnMO_y;tDh=ggs z52h1`9=N~(0mE=(Rj0=HDxA?hpKSjmm$B-Djn!&s;`;O1dVFtc!68eY#e!MZgM$ri zwOJBd%*XQ0aQx?6ZJr91(!4}=+p$_i=(cs3`w+Terma!lPgu5?RxScumVtkYJhgP| zoAIns$sQb+hXMt<##5|MuNTvin2<1kp1)PB{wL!PGDzdO)jfXu_U$7q92_N6#An#G zDOVBPPE@SaG$<0M%Lxy z|5j4nX{RQ8IfA-*+$bpI!_h@uVlua1`Q97FlD_!l%F4K6CnU}Znk9_m!npL{L2&c5 z;>B(p*}V|xu(X_4?iWn&)qc~{?{&sap3Ixh6r^O|-F{g0?^=MhEJvqPr%vGqK3~OT zU2iJf;_r#CZG27)sN}u7koBM%?jCG9NZPjq$D=j^I-N~zSUeCsxX3^68m@ec@+z(I ze6Q`OE2`-n80Yi38F);f)2uifC(Dz!`8+oo?{r_V(*NDbE4MrgVdMDUQ~6Zs^fYGS zZr^wwF7#|XE}8Hb@7Z{+^zQf^?BhD+OJD7Vu+Qh%Hs&;3C^M;+-1r~*YC;Gk6s3BR zT$;NBqW&R|{vltVp;B42U1p7ubP+(c zx(3C}uf}Qv7hHvUq=)9@yi-Nv*nBf=zauQ5L6%jNAsr_yD9nr6C1$QE(_zV{<-F|V4wec z{ZHREF4}?zYjJ=0 zJ1;N6%LPvsa&mHY#$17!q#TqMe@x*gpZeC@ww>5r59UxAa=`+2*Zv*2WVZdNEDtso z*6a80QRQ+gy3gmEC7;uZn?x)Kqr;|J&o0#f5~yQG2J2NC z1{@$nyn+Nir4|sV@9rYQKz5yXo}^)ng8h02y*}(?@3Xb4ZN3Bq;9a|Y{P=QG&-ea6 zyx(D1hDAoEvxW7%etvM?bupFa5zse+B~5^RM5uT1tM5zJ9u2!2jRqmkSbnq*41Z*x*|60x3(y%aeb`B0U*_f=X zEUNhI+}xPhSd}gI_`bIe4h{?473JlU^72FV=`k^oe5G8}1{ z!=o1BkdkUI`MiD&2e9FO>Fwopbap0qSSctZbbNeFU)tZ-hh}Sg_>2h$OtbfI@s=d$ zQBF}&QBq2Z5?KHHcU&;;p6>2$G}LD}IQG1xSvffyh*V%?Yin!YzJC{Yap4BTa~A~@ zVaEva|Ni~EVi0aEvr{>Sex>H||I@78&pg`K}Bsdtyn%hOrH#VczAFh65wHd z2IjZ&*Im@Y*4B)pqoXBdWoek0m{^|IPTl|h&8euQgf_>rgCNwrw6xUloSMh(=<;$% zpy&MjoQ9E+@v!XECpb`G0_+TjfP`dNrjB~%_0np3eWCBCT8S>4a*1xEpS0q|*V)~? zQWGgDDPB?xxm5mG0^(GD=dUR#q!)*azO8nLh7EvmIX@pj({_L)-(B9{-935o;{t;GTp8(Ue+eOufn)Hl9QWy__pvGEuT?D_Lg zEiEk#F+E^#J0LbJJ=8{SUXb%v^L!frB~oUx?trf9aD8>PhG;VWOQqhpBzcyDj;BUS zPL38Vtx~RBR7y$;!K;ntbQ1o~jt;z&^ngD@#*g@w~V6vL*VNGFI(AxTIP;QIw|_ZcVq&z}#P)i$ZF z&3SsYr}C1osPMWV5bBbk_vYEz*?7}9`)fTh?p@y+Juk~$_GcPadukia_FQh=yJF_x zXc-y9W-2X}GT%LIgFsNw&=`)rL_stN_=_~COuSSNC+6xc2V9FoBCLA{#J=wCB@m(Z zW~(E;Z!hexj+Ps4!B|Y85BG@7>$}_AQk53tPcSfO1qD-pL5D#pq{aBJw{C8{;64>E zZvybv_??kjDyr-#fxQx+`69FtaOt3NW@SululKf&s8h|pBD2-DW}Abld(#y`Fl3nxkpuhL@u$LpMfL9tCmF4=H$uW7?FRL#ag&e#b|OJEE{&_h!rba*nvmz!e)VF4kS8~AL1a03T?z!E(ghrKDbD(l%!@PDewTcG*=f&{?wISmcsCjjKlfR|u2%ttiZ zLLl*dRWcXSDB!%~v|t3{JYw6akmp6I_3kG ziltq^!_F6nt+Fy)?r6`&{;Wp53lq#RT@hqfU}v=@^#x805Rm~u8AiQLJG!-o%u z@!I0zy_#T;0_GNf_b%;H-O!Kob)bRu3ZT^9*$Fm-j}wJN0JMf%e||b0%q0TD==wCXZ;)=R`xY*3*6k-e zyf%8n;#R2`6I1Vc@^I;?;rh~_GdF~$kS|?$KFJiHr!R^tGRolbb=7>(-tG1yi~9OT zhEDo8LDC-C(;HUXzOFg-M2WOmEs}cP@+~fMpA8P58IR!aRs`e&=?3Q$lOIcL+=>wy+1*+qXDGmqY^EYSPG1ZK8bV1g$Rb{3F#1q9jeXfh%T_m;*pUsoT zO7uin>}nrwc0{VOA<}hyRx9+GwL6&bTDd^e`#-TZz#L)z2DW_ zo~z{m&hmnlwKOACGe3bI+0Ao=Th%lHPv`rJRB|U7B_zQY5>J{a5Z!0rqB?ki>3*{) zv3)bBbe7-0{GN|J(rdt5frXW)M`ieJ6ua4GQc#-FpLT=0)5>i0yB~B-csg0^#+8~N z2YYHg^Lx4J_TcezK0dw@4G|HatP;u zqHjU?*RSx;fkM)c`hrRkFCnJ~w#po47>*nL1|X`!R=UyC6<9{>aG??GxGk)|4oWiP z-*o1vpkJLlU!Jc!mV>gsv5xif)j4<+n%F<5Fi^AFMUrrQtA^h2=CoyOTP%;uiCazR z?lIJ}5wo)KoB_H{lRPe8l{HKE^h&=ga!&2C*FtZUbGpi!B$R-~bU5QJqgGYe*qE|r zrA5c5M~?}4?Vf|=esYloyLRRK+a`OH`?uKG_M;1pp6GJHR6nrsFO>RDQ^O4{0)-lzJ;WnZOh0~l!P7~R?q@}Z4J9*Dxc73=SN_c#_)vVN42ei` znKZ$~2r_|@-{wlX9@{j)m(_+lfO{*R6^V*x5dtS{m*trkkD-;lIvbTI0`nl@u`vcg zy#e}gd$u!ta_uMD!tg8=QJ z`@G(4Z?R%zfA>nz@OT*^QfFea*oM-Cc1~E{Rolgnr|8@vSEaP+^6tQ3&&;v^Ld3jXxOc_&v7x z^LzFsZJ|_)V8z1E*ci8mgC?hss>!ce1e(T4u%-`n7jy`vU7DCStlT1=B!|Zh*)Fx0 zP_Wfmiv7C2`p_sPPkv`gkeHpX^@tf`=swP5yY;NLl3`&JM3eu!un1avyRiE;Ucx!B z^F>^Z4b8k+hxKaoUd8AxF=1f94Cb9oRbso^uUh&L2Z|$AshJGp!ydypN=t6nYpvRy2l=a`!3|}!LW#q8)o*rg}Avs9nvX#d40<0_!nLcq^W4{{oln0apzC0ow@`18N*2|3c zhp}Zgpe`006j??jxcB$Z2Sl6?B{3DMR$O-@4MZvDV-WwJ>8wh+NuJKEdD zV?JYsaCGL5$T5}LbzP)i-WG$4DPT_z(p+&pQun*f#g6C~skK7JrLleAH@-g74Kp_H z0>orpvx^dgP*%^&CpgG34g zoO?R`t{2~LJ~?VQ@{(?(@HzIA3wb1bng%~I?8poD)N=mEAS+?BC>C3Fhm z6z~}=V&Yzq!D%$Ou~jd4?5?|xr?VR@&(zzPbzBK(TpygD36GhLo?L*9^_Obnqy3c>KqX%^p z(S62FmPN7H*GVC=93`x_9CNEIEk>@F7+{{1kgr6vhNyer{9>3>o`u6P`2GiSRhurP zHeSk>F{7t~W7_FuKwch3ozB^4Od6@B^Nm&Q2z%=M1of7rV;2DJmxc-+wxE2F{p8*&!> z)fIbJSooc}_IAA=Eb1=GZN%VdX{AonR+@%fjW{U=uL1-@hw}R^6FZbWvCI$5+Qf zSh^D(2e3cC8x~S0wd^mGEy^j{58a5{dqd75v!m?zjUY!2$}-T7)_k>14yQg)K%r%m zgI}B0Co?TP@5NRZKmE(5bla!`cPr7r5}H)IDif4H+K7lgS}+-8YSna86X9jCuwhWru7J&#nOrpw9B!>g(LmSHA4^S_rvT##N^R**aBA|8L{qef{AvS;1 zRjk6SrXX`Zye`!~%3IZ;W6rE2Zm_i`rlxRzYlJNUUjE!-sM31hi}ez#Od=IaIihj? zm8=%7Ei|qW9m(gRiSX0%YEX&6(&3@&MiYy0U{6#Sql_78{biQXhSf%P)j|V;eo{h0 z!p3&Ny1w%2A8b0i2Q^(5Tltn3)Iynen*+5n=!C4Fz)52SP{Kx za#e%Y98{QrptZz%w`mg86@+!TH`1 zUgv(Zv~QqqY8tI!JkB&mge%LkD83<|DDQ`&QIN}aDvxG;P#0=>GH4*JRK2oSG4k?2 znBftJD+6AB`R}*(7q&fp`s#y>73!sMfgRo6aF6ZkwDtMz`DIyUQ{*N1Py_u9UVHq__PNp}1NxP* zfqxY%Xu02ne=N2p{aUhrmcG76)~D{?e=jBYUYW;|Y-hZDiQgpwt?{>l5!^5jVsyFP ztJ8?wGONEu4>gvc=;3_QF4u-&#w^wk(|JdwE4m_Pe0QkU@WTd@s%2um}!_oSXl}O5)s?pYb|J?*3 zBfKsh<`KowAqws#h175QkGpOXWflnKa|DQjm*qd^5oA5|`wHud?&XuFB+^YceiY0M zZ0cfm^-*GxowphYvDy#cosx6G<6kwmsV%T~bn<;O;$zyo*5k#o`6O*VBgEpHZKC&ll{TdwOoZJ;KoyWLac4u`J}IkY=dUvg5w26DBV-aeEP?a~3GzL9CgK z8!APwtQBlbo~(5mRAWV72pI(lLe}t_and)Q(@}qZJnL&P*d@<#HbJ1S@DVeP(ZzOA zEP~0eTQvZY)q#?ThzNGp2MLy{lL9zepA-_GQgi6E%{?^>>4r?#MEvRcnF;^ z=}=t975nP7`k~w1HGQ6?5USWaek;QiO36bIgixLV)W< z%67ehsl&$so(9feJWQ6zE9R9rAF8Qcp{~f&!uvIrcJ0>{+*32d)oL$Ep`RsYgO-6) z5W+I$?^N$7wLekt%yFa?gi-fSntKoTGGDH+<@d=t3z^**A@lZ&c* zG(s;`F6?25o84>nw4H(arPW$^pEG|HM`6{Q3*T*JV=Y`g_t@4SH$Pmj+?0)?;~IK` zZPai!%(^y|QjwVI9cCZfKC9T~jcCIViYU4(=SGaAZQL4vtrsNSw(h`Y+E1v>OyD|a zFY%T@LOe@xSP!t1UVus)h}ZM!0sI{i=SM8S#^UL`plA8)dej=3ern}JK}8a@Y*xKY zG7}t8E!1fyy>aTa`-awr^ZzNAbMpkrBg`FPh{%yiLoM5mD>RNVtVI{QnhcgD?osT` zMzixt@Y|5*q+HWq;QKa5qrGEXnB)-rCA^W3%kFoBJukA>ApB%qwYs-u=8LCUq5h4d zYa021Vf8YZc6#?iNAvY2{{(Bo#%Z@jn) znmi&5=C!cMw*uyK`Om3~S1*DV%`W`nw@pQ$=6tgor_CZyHPOl6Sn8booB{-W?fu1U zBD)Eyp7%K(;K;zlffUohI)4?y4K^rVA^NP~?sa33M8FVOqs^@AeXep)yGhQ7Z!+Rg zD{Xp^XFA8$%rgp2=->a0r<2wvIcTaIyw}1Y0Ukx1AUyb`F|L&Ji!oV6{&&_b-FafE z(WA@B(Xx9(KB1fPwipr~2iJy%VpC5INBVMa2+nvGEYs2_q5@4>3*P5lw2EIF3Ao$(7hoQx6+9 zJqOHm7Z1IJZES3UBO>0ZsKicXV>?rl^t)ZZbz8hbqOhs`6yaJS9nM)RCE3s3krlg( z)o=G!xouo%>NgF+V5l1P_vK!leC1bwuxuS2#TgR1Bz^Pdjon5+Q6Pi3c+pZWDz*ZH zm9;fC9$o;Dc?55^GTqi=)t1)QzL=%CBjxAk?_~yfyf7Nd4)*YPuDIxld}|x5BRfRt zto2!}QbkgOEo1aX@8?BcSC6dis%3xs%(#+1y$hd*Bo+cti=Hzwq62EQy<$i>6i7oW z{fQi4B7xygS*)q4ES~yOiHle2*5~}Y7GUh1TCsNPX1Z_0kO-6%&{%DMeu^DmtwtJc z^d}yjp87{AOk(!E7Z)F!9<$(zj)^G)6r|Vv`DFLNKt}?b5jZ@2rAzmEc7Jn@Q!h@H z!8}QHjU=CegeLpofivTro zd2Q{5OD!O1i(b8a`C3u&84z4bZ5M^V$Hp4KM6{IQG@vwHma%5g)@a z>Wb)?Dl>xFvBGS~5HvkiZA+2neW8)3{0bzz@R&kpSO@io!ZInm*pML1An`UIg5co1 zTNJ-fi$1qcB_c>$0XJ=Tv?OY5Vj@Z#(>gSSS)f|f2KHiBKu4EJ(@?R0L{|?E=I@@L zCu3>8W7qY$HUVigj0Oc^LQ+ny(=6Q^DfT_0mX;Q*7r=P8e?b8ik1OI+v$LT({Mgv% zuj5(tIzc~2eJsr%#bUpHMeR0bck}c#y4afrqEQzJI|ep3%&_?jAw9;Y%8)a*v55c@ z4FUoJ9Nf#F(E;}?F&i*tAXD| zQ+YZ2K2Zm?M8wkl<)Va?6w0S$Y65q6cK~7~k~%=ZekDVHhJEe;!wUe`x5Pw{nV=gR z8$ajZz*%2khl8utjsQz!wA>ajQ$TY)Eh;|NNLY%6?OXvUbpznQ04{+SP2dqm3zETb zXk=t$SEc1799%?1#L>x#Ur-P-*!7Uq)YNKfYYn=;Q3{esiI0hmO-`=11)y~g4t8Fj zZrUHr@udiR6N7E?B@+`O+}Zvtfv&DDp^yhBKBGFEyu7@dyZcYu#U?5>d2w;r+MT@y zJQJHqFRn}?d-n7_sLmO96AxyoKU-y8Z$SezGiRo!~699Ne zjeQ?*x;bPi_B{tW)Ys>)S%hTgxW>=J!vjQJeJ~x^pyT5S)*NcP55e=^nh;=9$w+!! z)R(xZGRfRU)AztCuAGA$92{kymo~Tr1drifJVOB*+S1Mrs+pM?uYiEjpC6)j4i0TV zj9~lt=Q$wmB*LoSijNuTH2aVtA|iJ8^@)j!K0-x9i%L(AY-kWPHZ#l2%X{MY9InPa z%oe?a!ei^FDP?VVczE~Wm*8M0*ur+hP0!4GkWz54!#2o0qx@} zz=^W5^2glDvNCi08eTrWdF;-F#6;&!Eo!O}t;)*Efj2RCcV4*o_}pm9#d^)fsS6Vm zs-9uzM@KD2kwO!U;>%Bj0a(#sU?uNtYlF{x7v2UmCQUExY{j27HH3hAeaX!H4qynV z)!$N6!-0mnwz-*EUalz6C`0S7&D_=U`Eez*x}u_fRainoLNa^gz#~~)JT)gL2Z@G; zX2!_S-kuGx?k-*4KwbOukrNJ1NlEE!cM{W~mXw*9*=?_^>vxwp0RcfxU0oX(-kR$> zaq;<(M`+jzEYvYQODikz1O)M_xwUS`RH&$^AC#59WMoij*Eykp=4-%BpNDHRuq3PC z0ouHRf>GhFu`yf>42*Am>x8WOpSYd27n?-@D-!f@e`^V~wzVC4FbK};1J3`Wun-;Q z1M7Y9_UpZP`$fmS!V=^eRC*X!ouR$g|w_uS9mxh6O)0h zZ6p{5WMP5);^JcFC>!u?c6N3jUDAt+n8b)Obn}4K0M=$&vdlk@xanzV5HagDh4sa= zWP-7(sHhBV7#JI82@Ue{^BXR;d?w_!O5heZNB@G1P6+4e=^3>^kBv>lkHiR2ZxgPk z4&?qrK5JRor(gjl_R4QNn@MaF6BB1{SwO3Sfq?=Wz{2`wX1S+DU?h9;fs!v|FMV}v zU&Zc3Tyv2l6Z>WW>B_gJzTWckGDpPhcSQvR@Ynjr#xL3g=%_LlzJO)Ic1Ck_bWBK2 z7GsXDrvIi;Fg`FS7at#A?ZLpv2y~smi$kE9RPxG8OG{4yIQRCVs;a7%R#f^B#I*Or!OmX?nVOnzjpky8g@yh0 z#o_kKPiCk>o3X1;ztf z8P>$a1bD*yb6FW%VNuZ|IM|Qe5BHgiot-(;v2b9$z`Mf_);2Z^g^MIOI5_GW8kOp5 z(!~B5RdwDE)(RRLAvZTS-13WrGQV>>sQhmytd#$EYWAtLg%~^8w#7a97X7{qPGVbU0 zPEJ-A>P8=?a^)$k^$o49pOTP}>@U_8Q&9aIhr>Ef>+!vvEVtomG@L$(ws-9FSFVGw|h$1PCl%TYfbPEUwA|cWp zf~0hVfTW~K2$F)7v~&v6DAElg0@B@e=HhvO-}t_N&N%0carPL`GZZ&#uXW$^o^f5* zoEt8UDk&{3wisjvw})Kr$&)9tj~`39y9g0k}snq;~lDWtT42LnO1;9MnRufB!ZBl$ru);cwuml{GYi4-Q<<-e+KJ zZEikz{1_|r&6~|3ZQGe3YVTJNi$TPcs>+f=?Dm>kmy}KJ>Dm&kyBM-kJ4)P?RuP{D54g^H>8>D^9!-Z|&rN3WX&<2U| z!Lo`JBJPAxEY;cJ&E zHw;)4kWsXzNe2Dza!4X8btk>LT97eD4)VQ52-;$&I|NhS$JT^IM7Qbbufk0m#mTxi zLDmOjr5Kr+CBQmP^_Mqh(fL5N!j?YHmTxx)1U zDmVV4x5aW)D?RNhl|C_R==>A zfp8EM;Zab8fk1)|ZpY_rJH>9M3H{EUJB`!R)Hg$VPnHNJUrAWvmmNJ zoWBKOnU;nIb^ZEvyekh@ZUSZMj*SEuDcu!_s=?vm0kBq9p?~}-xQRT_+(@upoH?={=tWOrCIv?g-{C+NV)AqAkNzLMEqHBwXB*Na(WOW z!pFa4YikSgm&@RZuVG^cz%!5BRlzkjF?rqCD8AbB<9TKRiUTrYAZ)2eO5#0p}seal!J0&0v9__8N*ar+~ zYieuLLBR%fmjvkHKv+^_(s2{X3!&tOB>~HXZKd+PxI>JLtSmaXb4XT^8j92B%oDOf zI!;b}UtiyqudzFbkm1klY;;^)5WxC|kn+E@vwL7*Km+B}2ooDTBnF28_?$jaTS~wU zLk^3kudk2f(}}#+5>SM|To4YEcD!QwJJl>zIt8nQNY$Z~qdJyS_E1I!$=@!v5(@`8 zr0Kk%e4qe*5qf}xZ&48ka!{dqm6ey5)X*U9hWd~#T9yGaIye*(pwZ~-??>|R+M}gB zI5^jEab+ONkKBdrB*VHe4VSe44n1raaxJ;cn=I-DQig`N;i2r{iJ|yY&f3`9lSU4H zXy^vicC-QlWRQLO!IQt2yGu?^esQp53ptVJwD^`OA6Mjk>JG;Er&oqLKrXZCXOfWJ z-ijt97)?`Cdhj&WufxN!!9_(yNBe?03G%kruV0%BwcKN7zKN_pZQVu8&kotyaeN&V zWDJGLL7Ui_FK7*3G?-sSNB`E|E{j|zK$STF%30)I;R8T*W&~!9Y{1Kx9IyfZyD?+~ z()9|<%ZcIf-Z3Z!78bJi_4R=o^y>c(Jk%zH@WIMem6Zrd5D*Z6Am4}`;^CpFsi{Fi zlAfC@cd-#tZHPZ|&xdpJ1*B8pUo>9g6#S~e`aII>I`FVq;wZS76+gFjYQw` z5@a7yQF!0m+M2t%^3WWJluo;BGCV!;0u~C>3-n$&wm%lGzejrNw1lP}&ca#vC(nmO z!eN;zq)v?Lc%F&zv7i!F^fhv!mR+d5K^$G=w*4FwxI(VrXE+_^A0wKJ<^%$K<;YeR zBxh>N$;-}twxG2-5_C;dh!J4=FOWx=-`G(^@SADrxyIVI>6z7dVpWVz1;0au&aYu!kage zu&0RN!)m-7yP%)|39cXtx#Fwt4^hZ)HiwIrmP?k;W}*>7>L$c!BuK(aDQV zFYa35|8e)D=kvQbN#+Nq^?!Z-h!LE-E#oqM@zeWuLPVFi@@BVPcbnp@W*QyW`&6Aj zY1F^@{D?aA204ubHCiDFvBCSe!Ti}?mnh*w;`u<_!_z}289Rd`41Tdck5~_WQOAbk zZfJ;04cv1TzDrB%|28HDPV+TzL16O+kZK}04mbzE*v9V8g#KKLnEBPGk+Cik(B(F) zq@E?+yvtH)e}KYa*MCEet7kjy8NmGabXqLxAcd|nUf=iV=;}b6?)SL?sf=2}ppTvO z{T9qoHQx7}3y*OG$TA^q2&$eJWRAEy+rH&=uxNnpT89b&?e;Z@Cu1p_3MrzL5JUVW z(!@cM|0G$M$vK4fW~yK<`gFlq_zg{UWo<7?qPBON+w+duj+1t7tTePsytRJt3_HnM z`9sNzQy^f}(be4?vn+I4{$R5hY|t%4h$SvQ7k%~>XR>D*sID&TgBLP6-O zMzCNDh-;Am-f+5V3pg4Eugyd-r*SKiezDgd7{%#+7|P7RA*DPflSbncs!H0!-(nE; z|K)OO;9R(n%YJ)#yQ|2J`EozizQnXBxoXlOdX{(bYnvc!svD4%5FpX9Pn(2Y?>hpl zMv9%0f)__*SVe#X3Ga;;bT-i?8 zK7Nl&NtQT6=b;E%p19Ah9JdG+CB!z!?E^uXZ#7YgFCWYL0P>Gy&#gzo=O^2w+~!du zrp_X}JhCThv%OhCmbaFNjH#MVjcv}f`6op9&;HxLxyW=A>&ttuKl(O}qn`%yHw}4R z`Php6`a^j`0M~nxfy+z68TWL;xKWk6p?)uy#cr~w>h_{eR35r_0vHP#cLYa*gje+J zz_|8!ttMX7OAul*htrbkDUOy;V4)`2N4ltOeGf z_(eZGU_Eq#f@20vM5$CQLJg9SVg#;sdk~kpc!f1&Oi^?mei#_su>aRiB5|E{EhVze zr6IDB=qaap5t9Yi(~WI?x?<}N9KzV?Q0pt$SFlkJUZwUccD&x8ztT4*lijqN&U}rS zfWv0-@AFe~bGDPfw|SQh#`_2q3{0@Dl`(TqEN>Ct7aCEl#oZ&g^jk6B_21K#gJqvh z`6fvvR{PiA+o{T*&_@x5w=_?)`qwDnigYQ~h34ci`T6ZI7PelaTtWR`?V@rbW)=oF`8`f(Ei49p4XFK*{;BzQZ7k$_oO9Z1v-}Qm zHxCb_qG@SqIcPXP7Q4g102##%usOIzbMs%N;o27HA9zv)wr~72)eWj%P0Ic%%Vj`w z6pxlm-O#-G$<$nFSTe@zO3<`e@GXgcW2{JY1cFot$XmIBiJiR-Is8Cp(};;tM+$FU zgOWZ(z+t|zJv-~sni@U`9SPT7a^y{Z}0XO0f)SRUUOhdvNoH^KeU=I_(p za%M~LlIGq_g!;RY<7R*3(zm{)#l;UGBR|`UYXI~Q4T`UapsWUw(5tKzZ*t&$gpc;u z*_}VU+El_KpT;n&7mFBRNRebliDTxas&n|X$WHdSb_#P?h2AS^5DC{$32!V!(Qx|x z=*}T^yp4Z2cKyQjKe=%I)jCMY-`spVu4{8X(bj0}>kH@x5{LXBtjk@g(*E7z>%)WD z>NXD7c*@6N;RaS&jj2%!8zY@|E|dc8uH8DqN7Z74Pt-WxmUE;Hx*j|{N1y7iGgvo4 zBOst79mjsv{C+WhYqN`VVaP;4Zk1zakSG3hp0(1A#o&eM<(zKJA+|MAL0f#PNq!PE zY^wq7^dZ?y6;SZfiHcHz3SZ-^IX<8i!E26J8QBE3{X(}n4RqIA7>d6mDZX_^GP)^X&qO~H`i=#8}TTod^rYRY;gc7;_ z_QiO)G{A&VnT(9rT3abHB7%3+jk78NOMvRRX`;%Z7-~9znPB^7A&>(+A|-V-7%5@O z9_i{*LNNk3&~IoM121~Lu;EN_YkM0Kb~JW&cI0#WMxolm1J)2o4kRRkO;myod1iU} z4K#xQ!heB8kzGSp&QTrc0*wZ55hRWIPN4XI%giB)0*i}O{MpTrq9ZXDXgL~ywvfik zYDXqeGWkJI!EJVSMWk4h?LmstbmB~SFn)f7aykIqsL0^kWu3~Gm*C+5tP7H&MJk_k zwwc*k|H?{!AWdd7M02lyhLHU0HD$c732%r)zdQpZT%bWWw@At||804X0 zxA<_NS42ijiyU$Y8sNp?S!KClkkJ1M`X?!(;SgPoVGb#Ljx=<@{Ffwx8Dk*C->wRI z_*Hl?1UZB*pkm8mHl#`g3Jz+AW?%y^F1U_RwUAZ-wAJLXUKa#{_b{^iSL^HRkqX1h zs}3nEqh&8H$OU8&vKxUol%asDp`o1Dr^12!rG-)w?2M>Nf|mAl7_eUGB)tvAMn^{n zRN&vh-jQd5%}7g46-PEBI}zE8rMrMh-~;>!r4dqV!<7({F&bD>z#yEpJf(K&*$NnN-M`6}lFo5wD=`s|d7?LN6Io zZKYEI&0!2eOGZ}K@6dW@w>rjy;C%qS!J`91ft38TOA-jI<(>+(BLV=yk-v81hJQ_s z5MXd$lZLba{)EG&9BB&l&?+Gxm>RC=0gYOr9yTB_8yhcy{)KuUsFh2I?TWmX{`3Iu z@-iCQ%-UK6G^zdxAkh>54^QEA+cH6{X!DNh>i^LKEID~X9Zf*s|L2bp$g+q5K|?ss z|0lL0cpY&9XzF@;5&!;~BQ+Cvcz}pTsvH;qJ4A5D3+fy0v(OBf5HKX`X?0-jZr{F* z45UTUBVeWwuY>P|cX6%3kr(+AX^lv(0Tzxh%2515>m&zYw#c$nhyP#D3UmnuJbXiJ z*#B7pNnOjI>OhJ47+EzeS!?TiaP*s{f+k&KzfEEc03tzayvG&11qW>F= znj{nipLlR|L<=7^I5>FWN(2oMdBKmd`#NUI%J^`t8NE;pk0aCq=xFH1ve?E#H za}$zd4nS9+KmvV3Aao@H6lyS9VhRWs0F;dZH%J_l8G-`oX_vf_yR24*+8RNr)$-N_ z@@cNg%E?IpK7_amIKv3q4~PiZTTnxT51;B|$R~1&pH56n=+tg?vW(Pu@WG!E94J#| z^m7kxxU@eo@w6-~dAFG9Y15@IKA{7I3YP)rS($F2p7uHS%IinCWPl+4ht(k9u=P0a z-?_QlVDbPNO=|Gq+cmX0=NBnMEcMvq0 zknjp{LHNwjx6R4p9GVyvb)m((s%IG)S2Q&-Xp z4vFx^V~)Tn!Y3sS2JD=hn_E0z%^2Dh`91bnfp`H906Kh8(A9jh-*DctT-bo1{m_qO z09|uNai9!vZvv1D3<3D{W+?v=Bplq`lW`^+pgj>H!{^+4bwL44ippb!(-R;88Vx7rSm6{hgne$Wi?R;0ke$7MulDT(J`lI0nd3gnvE(B@evY z)(-lhhLPiMPJBRsue^Y``+*KvDnwTU!vNNk`>U#}CDN1}pPEAM@f1`Hm6es^nJQ(1 z;WXC{{=QS>05q!R3FH>52yB#1@JvnZPo&QYUH!>cmq+sL0dsC7oK+*3&(fkOV1ph{ zKB?txLZ?3BAhNTw{{`lPZ^K3GB`E{QBWNKK^FCT+$R1qsc*1_|G7?Q7_}XMcE4+IN zl&c>FopKdGwq|bL2svsPRDZY-IwqAU@s-ybFYe(ZR7zkcfFn!iHF(9zWg=uGWJTTG z-Cvv8d`>poCN-H62~LXV)^*gxrT|?``NTwFVWAB02J{LEuVKGj%yPh0#%|UzEK=MI z$*9qTK_BdsqHmQxLHnB_h(u^rrS2pHPzcNH+)?ZA=eIEzrw@M$+0nJU?IVBXfjHct~qfE#?l00Ha9O%SyvYu&NlEwp}OE$ydd=c_xjJj!jA#0$GNEC;WuD=A0Q{_zn9=e z-gNPHWemi+|9dUbAp7*+i}E7RxOjWEKN9Tzf4exDY-9R^M}2}%u~m(a`}2` zoRZQk_!w`TGO%hY+ZwB4(h%>`8>JHTo-1`B?ERQZ%2~ek>_+fz$7ezt3=Y@3&05oR zR7i$;Ov z_iky`r7zEWah zmpK-WAR)o&Jx8-7S#7&oF(u3Jw%%8RT=0rp$|;&Qru$yzmH+NW(6Htc_Q={lM*nfR z8I-}R<#0!%>{Xn)Y3fUthPyhIFj&G81<^4byx2#7K{~`2Sb=-}uAJxApecX1UhoTE zBXv6BW^LOHmZxF*cz)5j)cD&ekF#6Yvbspe%B{?IZ!X?yzPL1&U0rkmR_SC~g|?C8 zQQii3v^Z_F#OB1Mf$MX@iA?=x<*L)(`dJ=zWh15SS5-cUooVX$e-yDbh}fqY*|mFh z?5}7z+oSH&(4Bw8TJL@TiuW3t!M*$bnRPc>#p*AoP_vwSC5#nvPLGjFrkpbH{eah&ML(PESEJUett~Z^>hB*SaZ-}MWFLv{&N1u(# zbtiFS;S6!Lf4ORK@II?Zv^e$9QIK_-ci6)+`61K!w`E1q&`mnRqUaAzCDVAWinwha zVt+q%`7BK&n-0ydepy{ts4%^nDtMztF9oAvCgPv*^Q+6gYozBRDpq4DK?y?hdCj5& zv8NJobHCQSWc~k$2yH$sqdcMB_CKU#Px6b?2{&9hV9V=O`cjkbn#8HV?dQ@y z516YQl5Z{_zbGg1*h3%8Oh{qyIo_I#f&7Ni5Rq`q5+X|I%f5boIoRW&dag~#;?A6FE z7}vZtY^T1`rdvNHnCSiG4V%E)zaz5LZO$^~vLX*<{Rc*Zk6iaXyPkdxU|S7ZOAz~S zgL=1W$7;JBHE@vK>iuF=E@Wd=^9h~O)@5a)>M3fB*uzo!Sjx80fOGFqh-(@BnjblO zB2{3{cbv5A%-8%<;~EMrUFw#sP8gW4H><5T6{jy}KZ}`_*vfKs_M^-|6|Cz9@tt{~ zt7LsRXvo;E%*}1tJ!wAJVZv=^j{KdZUXiYph=rL=i5I%g_U!ig8@HFLW`DA}z8%b{ z5Mfq`T#`}X^;WR856u11z0r35{sG-JJXMdPc4_;_#X)}Rb8K#W54$z3dlrSs4?HUw z9xp3Gt@(+MKrZKoo?K2{l_ztBaRgQrx_5+SMQj`mH_dr-kb(WOR}6#HpAU+qG-Oq% zUa^ngGMC&3)MI}i6b!~2B?jfQOn*z$nvAneT&^plxccPUx8;;o8EHcSRc!3+Iz7Mc z*KwTRzToMfd9Zgn>Q`E4^>g(*3H&>n8msJHiZpPM;Cy}NBR=rWO6S?&rz3mbh_n5a z;0abO>toiUI-c8se3a^ux%gv~&-<7bvagTo{(#*v*O1Lk4^2?9rv2D-rV zArBV2*To#3hv%<5iL-Ku_QbOAGE{^N*Y;R`5OgI_2axE7T?v`9nt5aC$NO-l!nII| z<(l8TQ5c>9JrPCnl*{&ddSpP(bNZ&rhNPS1f?Ip49*j;KcKX{59kU5< zNpCkkrft7G``WTXEXTfvL+0uO;)BDa$m@PI*N7xMi^6F*$rx&#O4iET$X|ywxZR*{ zO|H0mVy#(UJ(P!EVAguWl30{BjNF7lBVIN!bBn&-_C}Vier1vyeNK)CTdB0QHcoXd zUs9~*X-|`!i}5#;3G$imt9b>__Q!%8N05^R2P-#5q9{{}gX|o4reesMZP#t4uH*TS z9u@N*i^}Uh3f9$j|EiP9oGAIc^tw~yllXv2=*F?JrJ?EAqNr<( zIXTvCAMWTr;%sljh)^hqd^;jw^ooo|RXtq`WwlOZ#Fxn?K6Get4ZE!vzOG>C@+%tg zS11?-^Z<3oOIbDj>dSF@ui?2o?Yi9$)HEBuahv^RYUyi%?*CR5(bkJ89DQu9b=q2I zi%FHMo%TgsbVsrLR9+>PIsr!^qd8@w34;?moFtIFA1+_EY$Sr#z}fiX=@J`6Gq&hYDufLS~=g) ze0mMOoE0~Yc>S>B)Az!s+O`PN& z-fXcd6-w~{qd}V?FDWgQDDuY2ODF#0U2#^#T~&2FkaTvPDKnpk<_FDv^w$Y;s!q>u zZR>TdIJ;=$ta{s8GjPTkPkeQAJ<6=un29j?sKORF*Ng4CI=wK)`Bw{L@6FFKsst`) zzin|Q?#!0T@$|R+-QSzUMt&dRjz_<&!W6SU#zb!=;5-kR`K2}%4d$|*|8%iuv)f?k zq55hfn2kjEo0d!^c@rfpdjq;AJmeJ0cof0^+bG!^on^Gix~n1ecuFb}o;1JzBzn!c zKXpaSM@l}g;`l?;es9jdNXrBFxtmt}@&2LJ?y5wmem$|Gkv_V=8vmS$8OTu=FR##i z@-C`)mnv#aQchKOWsvf?0Q+;m3I5)*X(Q7OmXQQc;<((UUL^-2w+}fZryb0~nj`;`D9&|lCct%j8?BwSK0?q*#D?%Jw6H<=br(5yE_2?vT~12!DhWN^ zjK~;8Lf`#^Rhv3&xg(?2Ch-w<%Et11?ah|8oADx*cn>5K^zudnmL(il^=$`*7MgTu zeZLEn>N~f-@Rrmh(N&X_iO4By?-Y)_N}%W(*%zrO=hf;Qlx$QZ>0;&bug4qK^H%X> znnx0Zkw877BGPox`+CjVkzMh-X~PfesQ6otXrf7*4?B zeC#Vkq26bgEmIOO*#oe98axVR&ZF}G7l+!R8t-o8(hUU zR*+}$uaUFt?2byLPEc==%tkvN?`?NlFaG+H$xD!HJ+X4(PI-%j3CAX988ePBW{>-` zhS$!y$Tkaq*0_C$&AHyadbL^TC7I}{9re^lgWJvllnli!U-n~M&7!F%jS zFuMz^;U39k&1JWq0BGS!Z~FvYbFaNQdXDMq^zKCU&` zuVQkpvmVzNHLc7Ai&T>2NmuY!&ekmsdfyK4oELsv%-D8j`}F1T=ZHfz4LokrrtszL z;-rWG-#wR)H~y74rQwd1O)c5|879aM!q-VYx^W`3{5QZCf@@w{QrkdaSlz*y-xVqc zf6j!-zx$8fojg}*!3x2>iHxz2teM}3m1=dCX?yf~+9S62|FB?F#{qY4Las^xXyU4u z04=S^GhY$K@WzzACl61m@?9sSSKQsB0%u+OJ%opJKL*@mh>2e}|CcW!sTL&3Qp+)y z;$OVEBPzT+apu5X{gryh?o3MR59GjIMU!bVf?1Qm8ApOJH|<`rjM-OeS9ECi2pTnL z?P6~_Zsj|8*@f=d7Ee|eK+ktQMDug}S9?mo(RTcpmrl66)*S)ce+I9uidTZi)^tVWd;`}UU`rB8fb>h3VYA;m5i+6kB`7pk0uCMZlx;o40K3%*cY3}tnWZ}EH) z>GRmd%QUp<5LQ!@yI+n|VrjVFDy<4GTu3htd1n^OxBp|t~eTP0EjgFGNqkBY-;7AHi+Knaf4{PpB zpsBoj?J^9>3jnSMbQ#8tuP((4yK&JgCSw8AUfe~u#Or?*d#S{&VW&?X$fl)|LxROb z5LF=BaULEbfX@J5M}FmOCrgae{0{As^dxZ+@PRJF^7w$OwXz(-NWU~y=3>(>V$a<{ z^^J`UabRj;644cCFpJK+!=(e$h?)RO`_AT{El)&DYy<=KfcS?wj{(3%zyl#l2+&yH z*SJl4^U^UOV>d&xvKSEh0-oYF=uSc3jR2#+=jX2i+%5(B*sotzSGWM|Rn)0-CjvGI zh_6?rrT1Z|$~{0+4S*O1@!dnv4+BX-T7ZWif;7TJw`Zs@!4F`%X*duscj)PXXhK71 zOF~R^g`Rg$FA6Jv2*zoC1X? zm=P6iU?A=7+n2Dhv1!$zeUuUr^uK-U2ZR7{#D>uOh%gN>WlKp##Se71@OEhNky}xL z8z-ubje;gN==z}rxpP`t8n8HlDmkxJa=wFN{}*WUP8Jn?n$jP>87YS05WXGV-QQt` z8luyF`}QU>a`kW_rR|-fXvhVEN(C123R!61=nGN?z9#m{*49>$aT|{3-l!Q6;sLNp z&wG-?kw#YFVT3l0oCegMlt%&r#(Gwt5!!d1%%kZxL&#;dqg zVIU`jkvmtRFHr;L=+vL?X#){tF2f|~b5=WI3VoCA(9Jid?2U|4UCc@`f>B~f`zY{3 zz(ltIo!{Pg!7bd@BHueqIX__lMENu2-r=8?H%4*L2WiPospbas{g1$BKfAj6ud=}o z+(*R3ONI$J$2zL216CtuaCmTzA4?4cj0t`z0WxPK6VPJ@V+8B>FXLV*ee>o4zk%&l+S^9g?XhQ z7WfSYgb*`IIyzy{MDY#01{^4$9ih%g?7Z$V$aTPJgD!ic5e*mVYzEzzG+Ym1>Y%9t z`U8u*?f`+^sIdYg#Iwy_Q0xgn%L4Rz92_kdmL*B!K$9bA9QnaHj40p1clrQ(0zS$U zX43KdoB{PJjCA+_ukP>{x~`VE5Vw@w1x*g{ZR(eSOBiDphS|@E3KyQo7dnJkVT1~( zPYA#zalM3NbFh?Gh#YRvr8RYQTt{Z|rOE^A7$}Ah#D-08Ur`SyC+ABSmmtJJgTMs5 z0CLZ0Dr#zhK&v8ie&OgK-P3UV5OyDwA2@lB;S!;UYQ(9g)iC7-wAg<9h$SW_hVaJF zlZ)`)AlpNFJdulus6w9{Y%Ma*7gSUl?FSt;{gx928oqKIdof z8`*Kt5jkcU11*}rmWn!KV`DF_eg$!4_={W^zen1#F5YWSp*9K>(8cv0%rL_B zGKRsRm3xNAiaIGw;V5BU(Dq!U_ppyoT1fN}>q8sh<`99Yw`lM^xLmG~UTo-mYDS@` zW*o3pQS)|mT|$T!<}Ug5H^ zsZ6*?-PR!T;RD6%WBThfezeLDG~OFvVY=R6^L-T^ymB;-e&vC#REv`nx-Frha--8j zbYBX;S0$b9uP)ItUt_(-NPFYaE2*juo)0Q`p$B4D7Du_)E4yL6P z6m+&5XquuL(L=7>K>hanyS}a;uk=V>o~2~P0Wt!OJ+f>AGi|lQs)b}@c%OK1#4g>F@8)6MUSiLR9g^ z9ul9Jj~{g^zi@RBp-`An2A5F3x#1|f&jgS_Q2z{4iruwIZCD)@=w_TL>49G0xijy1 zci%!chX_g-_Sat03A1OPLK6{Y+_Pl(6p(&ts*#~y^#%gOrsU9{oZ)90)8E&JNx;UMhlOe!jfT5QC48d0 zCqPFB1FL#s9p_7WU^>CVNiy)Xi!hT?MOF1Z^v$hyUZdn`AGdADx0$Sdfy^QR@svPU za`hK`cmia4>FP+aBy`>kfy_Yzxk)ZsSa5MNa5PNx^q*gjF2ENk{{x+5HFSjxLc#&d zB|_Ch<^-b)8G;a9V`;Z_0YS5V?kPD>>{Fo1&D zd2|63ivK`l!SuzrVtf#Y8LxwxAPdmYJqw7iV7-6;DM%3H9zBX4Uu8S-f?xRx<8~i0 zz~YN|l!Klw4aA3x^z?zIF_%#M1+-hY?r5M;8@_O7mT-*(KHFsEO(5ILgXx`M7V~7L zlsG|4gLzpXu7UhK9W(u*QyJX;E(|1SI6KPIfw>+jxVLy8v#5Rf1Ul2mTU;)izaFe|-`!fT^k#;RQCMyrk*}njx251}m z5^d`>(Rbxv`rC#o3F2Y>#*W7G*vw0~ulbwS?p0_OErgSH_C=6C-sOYQHPTUb~g zRM$sl;UgQ5qNYPO{;o!`Q2-YgmmI5En6js5Epl-fJn&4yB>M@-!-&YpazNx3m+Sfb z`Dd7a;za?s5+z%NLMbu9fMb@RX64)x2!>-Nrf&qDU{qaK+%YeQ>Me+(YM`_$1F0f> zZd`6e^UN=hE_zYG2S?rUcm_5A$xazGQpJ>dLo%5LEv)fJ1BKEy+vjX%X7M2|3=4!SO=}32*Ou z=-%OmLAXW>=Q(W0ie5{{>7poA?whPA)B#L$KK_|N^GH!~m~YzCVW=9M*jSZA7U&C7 zLBe#zEcLMs`7_MjA@=yBk+7WwZTMpqwkhC{Jjv)$C?2XVZZs6UUXh2JCR|cw0rQrV zmCat_K%vC3p=mb*R2;cDXyVuse(2(vnsx4F(AB9rXo&m<%-EHNl_`KX56~b+iM(p1 zz{eu5M*07bf25iK{cSDZzpFZtqEM+<|G{8XxWQaOEEG?0T`n3*$pLCkMzG{u2x%xD z7%HRkEC`hvXo%Plyk5m2TZp_|FCqaq)#{;Dx>0qxs)&!+D64z~&g`!4*=Q$qZ``Nx zj`MVUE$2$ES=OM5yW}c8^yhlEj6Y@*72;3V|LvjVeL1gp?){*zRlnYrcGNXo&A=Yl zTw+3bzImfcU;l4Rx#(&7%J5B9YB#O({hydtZwegATc$k$8$^9?x%7`HWfH?ZmNAgLEC(a{l@1LdF{oKxh_d^*Echaw+F?3sTfS} zI~Kj=3-A9${805p1HaP~;!fEFB`lge$3yg`>h*7zGu=THKBkuZqVaFMD zY>8KlNtlb}lrEr7?q3g0B%Y+`>x=trp_}AQPC0yZa{lShw9Go?4&(mzF_HZg#C_%|e}JRCkF}it^21(T1FlkkV2P312US1w?b~TL%m4Ovj^1#b)Q??mmbA&V9n08cXYai? zD>|V2bePf3Y^oz|iC%Gh$m*9qgU=VC*1JoAGQ>>IOXPJ8_v8cmuf%^!5WA}!ZTyGm zP&QPpGNV?((y=T4z;8PIk&>f|w4vMn+Bx%D+Lg)D9r50Csrf9+CgaWOncdY+RxY)A z$IaxUx}dZDcgcxc(KeKc8RPVxIqD5UUsM#amBUWT6d5NBJ4&oDtIluCeeicLv=yJ3 zAZY1H{H=WS$2Gq_w{Xfuig|-|dbL9%r^6>qrAE4pP_?kEpl*n6l+H1-QUiX7`2I~cS z@+2zDg2yWVX9Z3_OP90TU}VJl@2ca6YjPr<#kza)RLY%Xx3q{s@C@jNU_|I= zb&}gIMtmqddibI6uBMp9%+un~+Ke0vjDtM2PrOTGwqK@B&r;X*nFJNbniv*r*CZx& z#|z#_cmMkOATH;@WbD%Js|)5)MtIQ ziCf`h2fsWWR{y;d8}1n~#BmSfOtyL_?31%7@cG}ua73~#Ewz>reL)Yejd9o}1?RS& zJpZTejuh8^cDoci%4zqrg7D`X{+sk`v#0du14TM7=0C_RBra#2ns5o5M(%&#$4Ram z5lhxGW6h_fZgW}- zn#M6Fk4LiE74!`81m-`bVTXSIy>eLn=-&KGA_|O}Dyw!z2YEW}Ueb^1@eST_9wi&m zCa_tl_xJAY{4^&}p7Ofi8b6`%d<%OwT>mtAzU)U^#ln>76!ovB19WHZ*?K{fpY~tn ze}nl9l{0qP@l$Q5R|>Xnt-nY<9ZuDMw+L(v|Q)8_t|0d}4FnoY{E$-*)>1Ggu=ky{n<3pIja`HM=We`Vcf?@e-U zKWkv@FZ;gx?t`1Qu`$baf-KO@UGIt~*{8^2#2f1H1HSK*ZtaO8=4 z!xbFf3`-+I@eD&$wqacRKL|>?(>=&ZyT}abt=SiCl{7omupYs8g4C;&qo*F_1)p4{ z(?ID|3d@x3lzL&-ZEF2+|J+dTc~>jMxH?x~K8W>^44?I{E~X$;9_vib{ps9DdhCJF zxb*=cpOw@nzM4mln`+jbaVF@?1)Q8(0;W-F^%QATI2~?>JVz#-seaQ6WZCFw3GK@- z(B*ZL2SV?X^@|?UBv!9k)*Ht>U!o}5^KiO{+)t_5< zTi)IzOyIcodci%d3Qq0v&#Qy=zR$WV+`k^zG3+i?v!c6D-fMFuhthZgxr^$;l<~#z}GVkC8}>_Bes)d$VqpQ%4&0 zs^bbqLygLh1=QR=XVtTc_99~f%a0T$<39-w3uk;kf6gehAM9M{t*b2N`K7R{#`f)B ztl@v7_qjO^SVTk*nDPZb=23s1%8R~H>U6s8z%x@V$ov8~bO9{ub2+G-7;kp=B_peZj<jmz&Q@<}qtt&li7RST)9k2za@t1S!-`)z+7m3pvC6(H4?qh1uZ8SC2Z*~{MP zd{3a(G5u_>qp&1kZ?G%Hdid<44&_u#dPShKQe!WV`(lWI-0q?y<$}DnY`uU_xP2l{ zZyQb= z0irgWPBQCgQ(RpQ!N!hS5AJ0?sRE3CC!RoVE=cT`9@1yPhhN)|K zGK(qH?75m)j_D}8Ikq&yS?bEndp|dJCc{c)J~Q$ue;v=H4Y=kF8a8N?x<(t z;n?!a*fCD`g2e#yj^Dc&p!8oojzqJ!Y2UCGxtPDnmuD@K^>KIH zva$aY(cBKB)3_o!i$VOU*4ZNY+=0_CG~3$jOL2T^ukLH4O$NSbWpT*~i93FNBcyjn z05^Aa=u!3YPf>DyQB9{c_owS3zaE%^24}x5?QyO7>=_0{v~ z*8Bfv9WM@0Nay_7ZIC3$+J84~*2B#EhnDA>BXjB#hk$)IP0I+fx%Kf+k8N$1I# zQoydB&TVgfV9`q_H-i(&wQt%!JGeWnsF27PlhUC)wuY5dv;01j6{V`!J&W_Z$uuuN zn=PDBX~Ha-b+yy6%N56bsOwR}T_UW&SSC^e>pPlHOsTk3%mvn^nYpGUViR8TFm9#T zvoMNdue|N(QjduCV`0*nd(Jd&ZGwKKyx;3f@wA@_apV9q<^z&aH3u$@% zAz7OoD;JTc_o+*+Vpi&reai_2ku+DUtQuP8s$JKQ~9|ao)pB zQDUYRJI1c6d4*f!|2)}5>W{#ElN1oTj`mufz z8Li^d)AK>Qe0rD6)XX%F-}Z@S=VfuH@7IV?By23=3QHeXCFv?S-$n~lJ>4)46P|aq zfB5!qbe?eJ2x~`Q&a{KkJF2-9%f;6YtsZZiZG={dqGxZtXWke4a`pgor_{vW{{8G~ zV7Ru8%eQYXHIwLKRFT5Te#4lUmG523KM=FLznj&t9cFB$DR`D3Ia7FNxm!ri!M!=o zW!l~czBETW#`4a7!JJn0*~`1rQSZ0--^cPM?CG(!uaaArn1VXV$(L_Yl_8_i-q3xTA@Q zg~?icuS*5|6rFSts`8~~tX0bdO8%`?*+5Bjid6&eAc5k;rTIS5`7+kgJ7kg3*~+ly^T@ZJEO9X ztd*yDWAAj)i)o{C7YL-L_)M`s6_Spu!^>LB!XoK9HVjZ(+m;Ov&HGn~$}%a5T+(>- zX!z|>i=EoL`p}xpf>>`ZMR~5{+`|d2p7)ZvLKRArwT^Y0Kc&5nw(%+3x7$}c;rm}2nBC^2MRw=ut@*vahy2K? zeh@oRqf*AoBjeU0Tvj$%n2(p$n-;FkHWXtB^uePe=??iG11t@ z!*D2LO5wmH=FxCg;TA)z;PcpJa`XLLMIXPVVWsdTZF{eM-}x_HN>5Dd25bk1clq=` zq4seTcg{TLrAJ6x$9X1p4&B^F;Z&50o_Zx_aExdS0A@{4O za`@|_XVR^b@8yhUx76?Cm6P+~L`8U~`6wt370?~pFPwa~PEqLnRYPyNJj&%*E)7ts z?-$7wi}}Q9%k!$6nRj>JG-;6kE3Xu_a9p|ZBk8`(I{O&ic{YE`SJ7R{X!};}d;GgK zi5@w{IOa2z$2((jLXM0*nZ_@s|0OBY?e^#kV=j5WaVNZ=%g*Bc>he8#wcp}*pGo;- zvsurSlGZLJR*jXr)ofADEO`d_Tpwpb>fdC57qYnl>_omXb%_IWaCPL7apCo$-js6l zrh&|*ZP|49Fx~7+Jw63(1Qq|f1}5%LXI1e0;#ob({IrHvn)sdB zKZ}aU_mK1#B6(~-{$)93cQEF3V3O(4(5CifdF)RNFOGhd3m(k9sb*MG;lXz9#2-+~ zXOA3MU-wZCnKtWmIy%k$^u}gwRHO4m1NW>T=kh75W~}Z#Y7M>0CFz+T@unS^|MEtI z!nW#3bCeop8yuv~`}Q2_YMxOfYGHdxTS z^yOADoS}z9RQKHYnI;_V~h|$qmmekW&6FTn@xi6KZ(uCH%RRToCRvMJw#! zvD^P^=~Hx9#?J}*bgWwBu9ADajjf-23ft=Pm@Qpx0rx6^=Xif!DDvvgxA4DzpSgW{ z^DBV=`<{ref97)W?RBgFaa}F-`medi{PwsX-CGG9gG+g@vwQKOwxsaX=qdBoCw*9y zzsoG^>a4?`vabi*Mo-`H7ucOt0G=in4jd$3n!^n|>Q#q93wVTk3ve)Ql~vE=H4a~Q ztlpk>q%`3Dw$dva7yS8_2Lm^R28Hym{&J^7`^w8N7hg_up7?jyYhWKyL@H|cMnYlpsBn-XAd zyZ6ies?>}ndU*mX8g{J}F)Z07W+T1SA@URd2m7CAu8J*su{~(Zuf6Ibw-+}@UE!Tz zykcQgYy9fE#qpO9`KxQZ`oc0b)W3Y+tQFJWb%xwNw)M%1GvP{2>n|72uURAB?y@TP zumkIr%#|GrFK?|l^>$T#mBi9J=EV<+ZKZyPuPwQ=#&ffL@bg1Yz5iVKb+7)_TKkra z*HbKbXYc?0C%SR5Np9dgvvmojd2^QdKkE&i-={0`W`p*ILiW3VF8q%ARwZw0b^N8U zgry^QdhZq9T-X8Saj{nu#r}JN}T;1HY^2w#PeDf2k_FE-?)xU}jWL8||8T~`2 zt?1j?Kfc~y-^c5eN*Dx$Pn+;o%ll^6B;e>JaANq>O$i2%O=l!Gf1G-+Q~e@v)2FDI z0K)<+%OyvhZ?FD;DbDLhX5X)?y0L5bgw_YP%LPBJ`I4PmvyEH0=^<0_zPiSs6Y=Yp zTa{luvbQ|z+cB5Bch~GnpSfzY&04wYomFQtmaep~StY1@{_CA>I;o5cvO-+ynpaCR z9N;c3y{)|X>dzDHzmC6&?~)8H^pLmtQEmI>SWvEqy;gYLv~TPDFYIW)e@UM!@cg}p zD&Yl;S}Wrh?6I-7zg)k~Qq(YS5!KjInMg@*3k6j=W^Ri$_sX^ zckwKmY+~@?M~tNEzaQFJr~L9hY2V!WoAuP`Yh`P~p37Z6wehXzxiHP?W`5Ov+*iZT z_q<g#vzU(P)Kd!@_7UN7SLD{+U?=K;s}uMPfoUuzezF`N$?6t)w+v1_`U@9U+3p|U4h zkJL8^ZK~aMhYvV@d#WS79XycB5C@#d0go2Wiw6x+H*kjq1x*6p)VdlpD$Kzt#xOMv zcmd0ya&-KtJ-BMDV>+ z>YI*3(3Ak=6?&|g)!)1G>1_O7f z0}nS^5Ag=WiY!po*U_O-Ub`OVqXeLbz+gSm0Si`6F!+g3byHO}tQ0sk1p-%Z{%4Ng VTKPmKAlwzi_jL7hS?83{1OOC&nhF2_ From 0eb79b16e83e82550abc2a8771312982b8a2aabc Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:18:46 +0000 Subject: [PATCH 152/209] minor changes to comments --- src/simple_varinfo.jl | 1 + src/varinfo.jl | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index a0d9e5925..19589613b 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -676,6 +676,7 @@ end # With `SimpleVarInfo`, when we're not working with linked variables, there's no need to do anything. from_internal_transform(vi::SimpleVarInfo, ::VarName) = identity from_internal_transform(vi::SimpleVarInfo, ::VarName, dist) = identity +# TODO: Should the following methods specialize on the case where we have a `StaticTransformation{<:Bijectors.NamedTransform}`? from_linked_internal_transform(vi::SimpleVarInfo, ::VarName) = identity function from_linked_internal_transform(vi::SimpleVarInfo, ::VarName, dist) return invlink_transform(dist) diff --git a/src/varinfo.jl b/src/varinfo.jl index 369c68ca9..3b930eebb 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1449,7 +1449,6 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) end - # TODO: Replace when we have better dispatch for multiple vals. return recombine(dist, vals_linked, length(vns)) end From 071bebfb4e7fd669994ae5dc7aecf1cfcbc92f6b Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:20:35 +0000 Subject: [PATCH 153/209] remove Combinatorics as a test dep, as it's not needed for this PR --- test/Project.toml | 2 -- test/runtests.jl | 2 -- 2 files changed, 4 deletions(-) diff --git a/test/Project.toml b/test/Project.toml index e2bd0d7e5..80c227920 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,7 +2,6 @@ AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" -Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -27,7 +26,6 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" AbstractMCMC = "5" AbstractPPL = "0.7" Bijectors = "0.13" -Combinatorics = "1" Compat = "4.3.0" Distributions = "0.25" DistributionsAD = "0.6.3" diff --git a/test/runtests.jl b/test/runtests.jl index d15b17b42..43d68386c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,8 +20,6 @@ using Random using Serialization using Test -using Combinatorics: combinations - using DynamicPPL: getargs_dottilde, getargs_tilde, Selector const DIRECTORY_DynamicPPL = dirname(dirname(pathof(DynamicPPL))) From bbdc0603a521ad40a0cc4063f3f8f50df2dd0030 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:22:01 +0000 Subject: [PATCH 154/209] reverted unnecessary change --- test/test_util.jl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/test_util.jl b/test/test_util.jl index 1d23567da..64832f51e 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -84,9 +84,7 @@ Return string representing a short description of `vi`. """ short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" -function short_varinfo_name(vi::TypedVarInfo) - return "TypedVarInfo" -end +short_varinfo_name(::TypedVarInfo) = "TypedVarInfo" short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" From e2f4d185f686d6343634ee18d28b75fb4ad5fedb Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:32:40 +0000 Subject: [PATCH 155/209] disable type-stability tests for models on older Julia versions --- test/model.jl | 50 +++++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/test/model.jl b/test/model.jl index f8303e260..6e1a052aa 100644 --- a/test/model.jl +++ b/test/model.jl @@ -348,30 +348,34 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true end end - @testset "Type stability of models" begin - models_to_test = [ - # FIXME: Fix issues with type-stability in `DEMO_MODELS`. - # DynamicPPL.TestUtils.DEMO_MODELS..., - DynamicPPL.TestUtils.demo_lkjchol(2), - ] - @testset "$(model.f)" for model in models_to_test - vns = DynamicPPL.TestUtils.varnames(model) - example_values = DynamicPPL.TestUtils.rand(model) - varinfos = filter( - is_typed_varinfo, - DynamicPPL.TestUtils.setup_varinfos(model, example_values, vns), - ) - @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos - @test (@inferred(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())); - true) - - varinfo_linked = DynamicPPL.link(varinfo, model) - @test ( - @inferred( - DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext()) - ); - true + if VERSION >= v"1.8" + @testset "Type stability of models" begin + models_to_test = [ + # FIXME: Fix issues with type-stability in `DEMO_MODELS`. + # DynamicPPL.TestUtils.DEMO_MODELS..., + DynamicPPL.TestUtils.demo_lkjchol(2), + ] + @testset "$(model.f)" for model in models_to_test + vns = DynamicPPL.TestUtils.varnames(model) + example_values = DynamicPPL.TestUtils.rand(model) + varinfos = filter( + is_typed_varinfo, + DynamicPPL.TestUtils.setup_varinfos(model, example_values, vns), ) + @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos + @test ( + @inferred(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())); + true + ) + + varinfo_linked = DynamicPPL.link(varinfo, model) + @test ( + @inferred( + DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext()) + ); + true + ) + end end end end From 3d823ac119921cb5079b742ab2ee611a963cc3e0 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:32:53 +0000 Subject: [PATCH 156/209] removed seemingly completely unused impl of `setval!` --- src/varinfo.jl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 3b930eebb..bab89f118 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -464,12 +464,6 @@ getindex_internal(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, v Set the value of `vi.vals[vview]` to `val`. """ setval!(vi::UntypedVarInfo, val, vview::VarView) = vi.metadata.vals[vview] = val -function setval!(vi::UntypedVarInfo, val, vview::Vector{UnitRange}) - if length(vview) > 0 - vi.metadata.vals[[i for arr in vview for i in arr]] = val - end - return val -end """ getmetadata(vi::VarInfo, vn::VarName) From 54792f41abf6f06b2f9f240feeeb9734ff4381e8 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Sat, 3 Feb 2024 17:41:52 +0000 Subject: [PATCH 157/209] Revert "temporarily removed `VarNameVector` completely" This reverts commit 95dc8e37a1195ba87e900c957e6d9e578d7015d8. --- docs/make.jl | 2 +- docs/src/internals/varinfo.md | 309 ++++++++++++++ src/DynamicPPL.jl | 2 + src/threadsafe.jl | 2 + src/varinfo.jl | 207 ++++++++++ src/varnamevector.jl | 756 ++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + test/test_util.jl | 5 +- test/varinfo.jl | 6 + test/varnamevector.jl | 472 +++++++++++++++++++++ 10 files changed, 1760 insertions(+), 2 deletions(-) create mode 100644 docs/src/internals/varinfo.md create mode 100644 src/varnamevector.jl create mode 100644 test/varnamevector.jl diff --git a/docs/make.jl b/docs/make.jl index 65d43d524..de557ef9f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -20,7 +20,7 @@ makedocs(; "Home" => "index.md", "API" => "api.md", "Tutorials" => ["tutorials/prob-interface.md"], - "Internals" => ["internals/transformations.md"], + "Internals" => ["internals/varinfo.md", "internals/transformations.md"], ], checkdocs=:exports, doctest=false, diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md new file mode 100644 index 000000000..8d74fd2fa --- /dev/null +++ b/docs/src/internals/varinfo.md @@ -0,0 +1,309 @@ +# Design of `VarInfo` + +[`VarInfo`](@ref) is a fairly simple structure. + +```@docs; canonical=false +VarInfo +``` + +It contains + + - a `logp` field for accumulation of the log-density evaluation, and + - a `metadata` field for storing information about the realizations of the different variables. + +Representing `logp` is fairly straight-forward: we'll just use a `Real` or an array of `Real`, depending on the context. + +**Representing `metadata` is a bit trickier**. This is supposed to contain all the necessary information for each `VarName` to enable the different executions of the model + extraction of different properties of interest after execution, e.g. the realization / value corresponding to a variable `@varname(x)`. + +!!! note + + We want to work with `VarName` rather than something like `Symbol` or `String` as `VarName` contains additional structural information, e.g. a `Symbol("x[1]")` can be a result of either `var"x[1]" ~ Normal()` or `x[1] ~ Normal()`; these scenarios are disambiguated by `VarName`. + +To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo`, and hence the underlying `metadata`, to replicate the following functionality of `Dict`: + + - `keys(::Dict)`: return all the `VarName`s present in `metadata`. + - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. + - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. + - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. + - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. + - `empty!(::Dict)`: delete all realizations in `metadata`. + - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. + +*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: + + - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. + + + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. + + - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. + - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. + - `similar(::Vector{<:Real})`: return a new instance with the same `eltype` as the input. + +We also want some additional methods that are *not* part of the `Dict` or `Vector` interface: + + - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + + - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. + +In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: + + - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. + + - `getindex_internal` and `setindex_internal!` for extracting and mutating the internal representaton of a particular `VarName`. + +Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: + + 1. Type-stable when possible, but functional when not. + 2. Efficient storage and iteration when possible, but functional when not. + +The "but functional when not" is important as we want to support arbitrary models, which means that we can't always have these performance properties. + +In the following sections, we'll outline how we achieve this in [`VarInfo`](@ref). + +## Type-stability + +Ensuring type-stability is somewhat non-trivial to address since we want this to be the case even when models mix continuous (typically `Float64`) and discrete (typically `Int`) variables. + +Suppose we have an implementation of `metadata` which implements the functionality outlined in the previous section. The way we approach this in `VarInfo` is to use a `NamedTuple` with a separate `metadata` *for each distinct `Symbol` used*. For example, if we have a model of the form + +```@example varinfo-design +using DynamicPPL, Distributions, FillArrays + +@model function demo() + x ~ product_distribution(Fill(Bernoulli(0.5), 2)) + y ~ Normal(0, 1) + return nothing +end +``` + +then we construct a type-stable representation by using a `NamedTuple{(:x, :y), Tuple{Vx, Vy}}` where + + - `Vx` is a container with `eltype` `Bool`, and + - `Vy` is a container with `eltype` `Float64`. + +Since `VarName` contains the `Symbol` used in its type, something like `getindex(varinfo, @varname(x))` can be resolved to `getindex(varinfo.metadata.x, @varname(x))` at compile-time. + +For example, with the model above we have + +```@example varinfo-design +# Type-unstable `VarInfo` +varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) +typeof(varinfo_untyped.metadata) +``` + +```@example varinfo-design +# Type-stable `VarInfo` +varinfo_typed = DynamicPPL.typed_varinfo(demo()) +typeof(varinfo_typed.metadata) +``` + +But they both work as expected but one results in concrete typing and the other does not: + +```@example varinfo-design +varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] +``` + +```@example varinfo-design +varinfo_typed[@varname(x)], varinfo_typed[@varname(y)] +``` + +Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entries while the typed uses `Vector{Bool}`. This is because the untyped version needs the underlying container to be able to handle both the `Bool` for `x` and the `Float64` for `y`, while the typed version can use a `Vector{Bool}` for `x` and a `Vector{Float64}` for `y` due to its usage of `NamedTuple`. + +!!! warning + + Of course, this `NamedTuple` approach is *not* necessarily going to help us in scenarios where the `Symbol` does not correspond to a unique type, e.g. + + ```julia + x[1] ~ Bernoulli(0.5) + x[2] ~ Normal(0, 1) + ``` + + In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. + + In practice, rarely observe such mixing of types, therefore in DynamicPPL, and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. + +!!! warning + + Another downside with such a `NamedTuple` approach is that if we have a model with lots of tilde-statements, e.g. `a ~ Normal()`, `b ~ Normal()`, ..., `z ~ Normal()` will result in a `NamedTuple` with 27 entries, potentially leading to long compilation times. + + For these scenarios it can be useful to fall back to "untyped" representations. + +Hence we obtain a "type-stable when possible"-representation by wrapping it in a `NamedTuple` and partially resolving the `getindex`, `setindex!`, etc. methods at compile-time. When type-stability is *not* desired, we can simply use a single `metadata` for all `VarName`s instead of a `NamedTuple` wrapping a collection of `metadata`s. + +## Efficient storage and iteration + +Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): + +```@docs +DynamicPPL.VarNameVector +``` + +In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve the desirata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. + +This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: + + - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. + - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. + - `transforms::Vector`: the transforms associated with each `VarName`. + +Mutating functions, e.g. `setindex!(vnv::VarNameVector, val, vn::VarName)`, are then treated according to the following rules: + + 1. If `vn` is not already present: add it to the end of `vnv.varnames`, add the `val` to the underlying `vnv.vals`, etc. + + 2. If `vn` is already present in `vnv`: + + 1. If `val` has the *same length* as the existing value for `vn`: replace existing value. + 2. If `val` has a *smaller length* than the existing value for `vn`: replace existing value and mark the remaining indices as "inactive" by increasing the entry in `vnv.num_inactive` field. + 3. If `val` has a *larger length* than the existing value for `vn`: expand the underlying `vnv.vals` to accommodate the new value, update all `VarName`s occuring after `vn`, and update the `vnv.ranges` to point to the new range for `vn`. + +This means that `VarNameVector` is allowed to grow as needed, while "shrinking" (i.e. insertion of smaller elements) is handled by simply marking the redundant indices as "inactive". This turns out to be efficient for use-cases that we are generally interested in. + +For example, we want to optimize code-paths which effectively boil down to inner-loop in the following example: + +```julia +# Construct a `VarInfo` with types inferred from `model`. +varinfo = VarInfo(model) + +# Repeatedly sample from `model`. +for _ in 1:num_samples + rand!(rng, model, varinfo) + + # Do something with `varinfo`. + # ... +end +``` + +There are typically a few scenarios where we encounter changing representation sizes of a random variable `x`: + + 1. We're working with a transformed version `x` which is represented in a lower-dimensional space, e.g. transforming a `x ~ LKJ(2, 1)` to unconstrained `y = f(x)` takes us from 2-by-2 `Matrix{Float64}` to a 1-length `Vector{Float64}`. + 2. `x` has a random size, e.g. in a mixture model with a prior on the number of components. Here the size of `x` can vary widly between every realization of the `Model`. + +In scenario (1), we're usually *shrinking* the representation of `x`, and so we end up not making any allocations for the underlying `Vector{T}` but instead just marking the redundant part as "inactive". + +In scenario (2), we end up increasing the allocated memory for the randomly sized `x`, eventually leading to a vector that is large enough to hold realizations without needing to reallocate. But this can still lead to unnecessary memory usage, which might be undesirable. Hence one has to make a decision regarding the trade-off between memory usage and performance for the use-case at hand. + +To help with this, we have the following functions: + +```@docs +DynamicPPL.has_inactive +DynamicPPL.num_inactive +DynamicPPL.num_allocated +DynamicPPL.is_contiguous +DynamicPPL.contiguify! +``` + +For example, one might encounter the following scenario: + +```@example varinfo-design +vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") + +for i in 1:5 + x = fill(true, rand(1:100)) + DynamicPPL.update!(vnv, @varname(x), x) + println( + "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", + ) +end +``` + +We can then insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion whenever the allocation grows too large to reduce overall memory usage: + +```@example varinfo-design +vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") + +for i in 1:5 + x = fill(true, rand(1:100)) + DynamicPPL.update!(vnv, @varname(x), x) + if DynamicPPL.num_allocated(vnv) > 10 + DynamicPPL.contiguify!(vnv) + end + println( + "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", + ) +end +``` + +This does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. However, this also ensures that the the underlying `Vector{T}` is contiguous, which is important for performance. Hence, if we're about to do a lot of work with the `VarNameVector` without insertions, etc., it can be worth it to do a sweep to ensure that the underlying `Vector{T}` is contiguous. + +!!! note + + Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. + +Continuing from the example from the previous section, we can use a `VarInfo` with a `VarNameVector` as the `metadata` field: + +```@example varinfo-design +# Type-unstable +varinfo_untyped_vnv = DynamicPPL.VectorVarInfo(varinfo_untyped) +varinfo_untyped_vnv[@varname(x)], varinfo_untyped_vnv[@varname(y)] +``` + +```@example varinfo-design +# Type-stable +varinfo_typed_vnv = DynamicPPL.VectorVarInfo(varinfo_typed) +varinfo_typed_vnv[@varname(x)], varinfo_typed_vnv[@varname(y)] +``` + +If we now try to `delete!` `@varname(x)` + +```@example varinfo-design +haskey(varinfo_untyped_vnv, @varname(x)) +``` + +```@example varinfo-design +DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) +``` + +```@example varinfo-design +# `delete!` +DynamicPPL.delete!(varinfo_untyped_vnv.metadata, @varname(x)) +DynamicPPL.has_inactive(varinfo_untyped_vnv.metadata) +``` + +```@example varinfo-design +haskey(varinfo_untyped_vnv, @varname(x)) +``` + +Or insert a differently-sized value for `@varname(x)` + +```@example varinfo-design +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 1)) +varinfo_untyped_vnv[@varname(x)] +``` + +```@example varinfo-design +DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) +``` + +```@example varinfo-design +DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 4)) +varinfo_untyped_vnv[@varname(x)] +``` + +```@example varinfo-design +DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) +``` + +### Performance summary + +In the end, we have the following "rough" performance characteristics for `VarNameVector`: + +| Method | Is blazingly fast? | +|:---------------------------------------:|:--------------------------------------------------------------------------------------------:| +| `getindex` | ${\color{green} \checkmark}$ | +| `setindex!` | ${\color{green} \checkmark}$ | +| `push!` | ${\color{green} \checkmark}$ | +| `delete!` | ${\color{red} \times}$ | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | +| `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | + +## Other methods + +```@docs +DynamicPPL.replace_values(::VarNameVector, vals::AbstractVector) +``` + +```@docs; canonical=false +DynamicPPL.values_as(::VarNameVector) +``` diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index c099a1fe7..ec0b086f7 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -44,6 +44,7 @@ export AbstractVarInfo, UntypedVarInfo, TypedVarInfo, SimpleVarInfo, + VarNameVector, push!!, empty!!, subset, @@ -158,6 +159,7 @@ include("sampler.jl") include("varname.jl") include("distribution_wrappers.jl") include("contexts.jl") +include("varnamevector.jl") include("abstract_varinfo.jl") include("threadsafe.jl") include("varinfo.jl") diff --git a/src/threadsafe.jl b/src/threadsafe.jl index eb2d0246c..7999e060c 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -55,6 +55,8 @@ function setlogp!!(vi::ThreadSafeVarInfoWithRef, logp) return ThreadSafeVarInfo(setlogp!!(vi.varinfo, logp), vi.logps) end +has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) = has_varnamevector(vi.varinfo) + function BangBang.push!!( vi::ThreadSafeVarInfo, vn::VarName, r, dist::Distribution, gidset::Set{Selector} ) diff --git a/src/varinfo.jl b/src/varinfo.jl index bab89f118..4078c464d 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -101,6 +101,7 @@ struct VarInfo{Tmeta,Tlogp} <: AbstractVarInfo logp::Base.RefValue{Tlogp} num_produce::Base.RefValue{Int} end +const VectorVarInfo = VarInfo{<:VarNameVector} const UntypedVarInfo = VarInfo{<:Metadata} const TypedVarInfo = VarInfo{<:NamedTuple} const VarInfoOrThreadSafeVarInfo{Tmeta} = Union{ @@ -125,6 +126,46 @@ function VarInfo(old_vi::TypedVarInfo, spl, x::AbstractVector) ) end +# No-op if we're already working with a `VarNameVector`. +metadata_to_varnamevector(vnv::VarNameVector) = vnv +function metadata_to_varnamevector(md::Metadata) + idcs = copy(md.idcs) + vns = copy(md.vns) + ranges = copy(md.ranges) + vals = copy(md.vals) + transforms = map(md.dists) do dist + # TODO: Handle linked distributions. + from_vec_transform(dist) + end + + return VarNameVector( + OrderedDict{eltype(keys(idcs)),Int}(idcs), vns, ranges, vals, transforms + ) +end + +function VectorVarInfo(vi::UntypedVarInfo) + md = metadata_to_varnamevector(vi.metadata) + lp = getlogp(vi) + return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) +end + +function VectorVarInfo(vi::TypedVarInfo) + md = map(metadata_to_varnamevector, vi.metadata) + lp = getlogp(vi) + return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) +end + +""" + has_varnamevector(varinfo::VarInfo) + +Returns `true` if `varinfo` uses `VarNameVector` as metadata. +""" +has_varnamevector(vi) = false +function has_varnamevector(vi::VarInfo) + return vi.metadata isa VarNameVector || + (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNameVector), values(vi.metadata))) +end + function untyped_varinfo( rng::Random.AbstractRNG, model::Model, @@ -321,6 +362,10 @@ function _merge(varinfo_left::VarInfo, varinfo_right::VarInfo) ) end +function merge_metadata(vnv_left::VarNameVector, vnv_right::VarNameVector) + return merge(vnv_left, vnv_right) +end + @generated function merge_metadata( metadata_left::NamedTuple{names_left}, metadata_right::NamedTuple{names_right} ) where {names_left,names_right} @@ -513,9 +558,13 @@ Return the distribution from which `vn` was sampled in `vi`. """ getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] +# HACK: we shouldn't need this +getdist(::VarNameVector, ::VarName) = nothing getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +# HACK: We shouldn't need this +getindex_internal(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) return mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) @@ -535,6 +584,9 @@ end function setval!(md::Metadata, val, vn::VarName) return md.vals[getrange(md, vn)] = vectorize(getdist(md, vn), val) end +function setval!(vnv::VarNameVector, val, vn::VarName) + return setindex_raw!(vnv, tovec(val), vn) +end """ getall(vi::VarInfo) @@ -552,6 +604,7 @@ function getall(md::Metadata) Base.Fix1(getindex_internal, md), vcat, md.vns; init=similar(md.vals, 0) ) end +getall(vnv::VarNameVector) = vnv.vals """ setall!(vi::VarInfo, val) @@ -567,6 +620,12 @@ function _setall!(metadata::Metadata, val) metadata.vals[r] .= val[r] end end +function _setall!(vnv::VarNameVector, val) + # TODO: Do something more efficient here. + for i in 1:length(vnv) + vnv[i] = val[i] + end +end @generated function _setall!(metadata::NamedTuple{names}, val) where {names} expr = Expr(:block) start = :(1) @@ -599,6 +658,10 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) return metadata end +function settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) + settrans!(vnv, trans, vn) + return vnv +end function settrans!!(vi::VarInfo, trans::Bool) for vn in keys(vi) @@ -940,6 +1003,8 @@ end # X -> R for all variables associated with given sampler function link!!(t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model) + # If we're working with a `VarNameVector`, we always use immutable. + has_varnamevector(vi) && return link(t, vi, spl, model) # Call `_link!` instead of `link!` to avoid deprecation warning. _link!(vi, spl) return vi @@ -1035,6 +1100,8 @@ end function invlink!!( t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model ) + # If we're working with a `VarNameVector`, we always use immutable. + has_varnamevector(vi) && return invlink(t, vi, spl, model) # Call `_invlink!` instead of `invlink!` to avoid deprecation warning. _invlink!(vi, spl) return vi @@ -1260,6 +1327,66 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar ) end +function _link_metadata!( + model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns +) + # HACK: We ignore `target_vns` here. + vns = keys(metadata) + # Need to extract the priors from the model. + dists = extract_priors(model, varinfo) + + is_transformed = copy(metadata.is_transformed) + + # Construct the linking transformations. + link_transforms = map(vns) do vn + # If `vn` is not part of `target_vns`, the `identity` transformation is used. + if (target_vns !== nothing && vn ∉ target_vns) + return identity + end + + # Otherwise, we derive the transformation from the distribution. + is_transformed[getidx(metadata, vn)] = true + internal_to_linked_internal_transform(varinfo, vn, dists[vn]) + end + # Compute the transformed values. + ys = map(vns, link_transforms) do vn, f + # TODO: Do we need to handle scenarios where `vn` is not in `dists`? + dist = dists[vn] + x = getindex_internal(metadata, vn) + y, logjac = with_logabsdet_jacobian(f, x) + # Accumulate the log-abs-det jacobian correction. + acclogp!!(varinfo, -logjac) + # Return the transformed value. + return y + end + # Extract the from-vec transformations. + fromvec_transforms = map(from_vec_transform, ys) + # Compose the transformations to form a full transformation from + # unconstrained vector representation to constrained space. + transforms = map(∘, map(inverse, link_transforms), fromvec_transforms) + # Convert to vector representation. + yvecs = map(tovec, ys) + + # Determine new ranges. + ranges_new = similar(metadata.ranges) + offset = 0 + for (i, v) in enumerate(yvecs) + r_start, r_end = offset + 1, length(v) + offset + offset = r_end + ranges_new[i] = r_start:r_end + end + + # Now we just create a new metadata with the new `vals` and `ranges`. + return VarNameVector( + metadata.varname_to_index, + metadata.varnames, + ranges_new, + reduce(vcat, yvecs), + transforms, + is_transformed, + ) +end + function invlink( ::DynamicTransformation, varinfo::VarInfo, spl::AbstractSampler, model::Model ) @@ -1360,6 +1487,55 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe ) end +function _invlink_metadata!( + model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns +) + # HACK: We ignore `target_vns` here. + # TODO: Make use of `update!` to aovid copying values. + # => Only need to allocate for transformations. + + vns = keys(metadata) + is_transformed = copy(metadata.is_transformed) + + # Compute the transformed values. + xs = map(vns) do vn + f = gettransform(metadata, vn) + y = getindex_internal(metadata, vn) + # No need to use `with_reconstruct` as `f` will include this. + x, logjac = with_logabsdet_jacobian(f, y) + # Accumulate the log-abs-det jacobian correction. + acclogp!!(varinfo, -logjac) + # Mark as no longer transformed. + is_transformed[getidx(metadata, vn)] = false + # Return the transformed value. + return x + end + # Compose the transformations to form a full transformation from + # unconstrained vector representation to constrained space. + transforms = map(from_vec_transform, xs) + # Convert to vector representation. + xvecs = map(tovec, xs) + + # Determine new ranges. + ranges_new = similar(metadata.ranges) + offset = 0 + for (i, v) in enumerate(xvecs) + r_start, r_end = offset + 1, length(v) + offset + offset = r_end + ranges_new[i] = r_start:r_end + end + + # Now we just create a new metadata with the new `vals` and `ranges`. + return VarNameVector( + metadata.varname_to_index, + metadata.varnames, + ranges_new, + reduce(vcat, xvecs), + transforms, + is_transformed, + ) +end + """ islinked(vi::VarInfo, spl::Union{Sampler, SampleFromPrior}) @@ -1429,6 +1605,14 @@ function getindex(vi::VarInfo, vn::VarName, dist::Distribution) val = getindex_internal(vi, vn) return from_maybe_linked_internal(vi, vn, dist, val) end +# HACK: Allows us to also work with `VarNameVector` where `dist` is not used, +# but we instead use a transformation stored with the variable. +function getindex(vi::VarInfo, vn::VarName, ::Nothing) + if !haskey(vi, vn) + throw(KeyError(vn)) + end + return getmetadata(vi, vn)[vn] +end function getindex(vi::VarInfo, vns::Vector{<:VarName}) vals_linked = mapreduce(vcat, vns) do vn @@ -1621,6 +1805,11 @@ function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) return meta end +function Base.push!(vnv::VarNameVector, vn, r, dist, gidset, num_produce) + f = from_vec_transform(dist) + return push!(vnv, vn, r, f) +end + """ setorder!(vi::VarInfo, vn::VarName, index::Int) @@ -1635,6 +1824,7 @@ function setorder!(metadata::Metadata, vn::VarName, index::Int) metadata.orders[metadata.idcs[vn]] = index return metadata end +setorder!(vnv::VarNameVector, ::VarName, ::Int) = vnv """ getorder(vi::VarInfo, vn::VarName) @@ -1660,6 +1850,8 @@ end function is_flagged(metadata::Metadata, vn::VarName, flag::String) return metadata.flags[flag][getidx(metadata, vn)] end +# HACK: This is bad. Should we always return `true` here? +is_flagged(::VarNameVector, ::VarName, flag::String) = flag == "del" ? true : false """ unset_flag!(vi::VarInfo, vn::VarName, flag::String) @@ -1674,6 +1866,7 @@ function unset_flag!(metadata::Metadata, vn::VarName, flag::String) metadata.flags[flag][getidx(metadata, vn)] = false return metadata end +unset_flag!(vnv::VarNameVector, ::VarName, ::String) = vnv """ set_retained_vns_del_by_spl!(vi::VarInfo, spl::Sampler) @@ -2027,6 +2220,8 @@ function values_from_metadata(md::Metadata) ) end +values_from_metadata(md::VarNameVector) = pairs(md) + # Transforming from internal representation to distribution representation. # Without `dist` argument: base on `dist` extracted from self. function from_internal_transform(vi::VarInfo, vn::VarName) @@ -2035,11 +2230,17 @@ end function from_internal_transform(md::Metadata, vn::VarName) return from_internal_transform(md, vn, getdist(md, vn)) end +function from_internal_transform(md::VarNameVector, vn::VarName) + return gettransform(md, vn) +end # With both `vn` and `dist` arguments: base on provided `dist`. function from_internal_transform(vi::VarInfo, vn::VarName, dist) return from_internal_transform(getmetadata(vi, vn), vn, dist) end from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) +function from_internal_transform(::VarNameVector, ::VarName, dist) + return from_vec_transform(dist) +end # Without `dist` argument: base on `dist` extracted from self. function from_linked_internal_transform(vi::VarInfo, vn::VarName) @@ -2048,6 +2249,9 @@ end function from_linked_internal_transform(md::Metadata, vn::VarName) return from_linked_internal_transform(md, vn, getdist(md, vn)) end +function from_linked_internal_transform(md::VarNameVector, vn::VarName) + return gettransform(md, vn) +end # With both `vn` and `dist` arguments: base on provided `dist`. function from_linked_internal_transform(vi::VarInfo, vn::VarName, dist) # Dispatch to metadata in case this alters the behavior. @@ -2056,3 +2260,6 @@ end function from_linked_internal_transform(::Metadata, ::VarName, dist) return from_linked_vec_transform(dist) end +function from_linked_internal_transform(::VarNameVector, ::VarName, dist) + return from_linked_vec_transform(dist) +end diff --git a/src/varnamevector.jl b/src/varnamevector.jl new file mode 100644 index 000000000..ed919a20c --- /dev/null +++ b/src/varnamevector.jl @@ -0,0 +1,756 @@ +""" + VarNameVector + +A container that works like a `Vector` and an `OrderedDict` but is neither. + +# Fields +$(FIELDS) +""" +struct VarNameVector{ + K<:VarName,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData +} + "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" + varname_to_index::OrderedDict{K,Int} + + "vector of identifiers for the random variables, where `varnames[varname_to_index[vn]] == vn`" + varnames::TVN # AbstractVector{<:VarName} + + "vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" + ranges::Vector{UnitRange{Int}} + + "vector of values of all variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" + vals::TVal # AbstractVector{<:Real} + + "vector of transformations whose inverse takes us back to the original space" + transforms::TTrans + + "specifies whether a variable is transformed or not " + is_transformed::BitVector + + "additional entries which are considered inactive" + num_inactive::OrderedDict{Int,Int} + + "metadata associated with the varnames" + metadata::MData +end + +function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) + return vnv_left.varname_to_index == vnv_right.varname_to_index && + vnv_left.varnames == vnv_right.varnames && + vnv_left.ranges == vnv_right.ranges && + vnv_left.vals == vnv_right.vals && + vnv_left.transforms == vnv_right.transforms && + vnv_left.is_transformed == vnv_right.is_transformed && + vnv_left.num_inactive == vnv_right.num_inactive && + vnv_left.metadata == vnv_right.metadata +end + +function VarNameVector( + varname_to_index, + varnames, + ranges, + vals, + transforms, + is_transformed=fill!(BitVector(undef, length(varnames)), 0), +) + return VarNameVector( + varname_to_index, + varnames, + ranges, + vals, + transforms, + is_transformed, + OrderedDict{Int,Int}(), + nothing, + ) +end +# TODO: Do we need this? +function VarNameVector{K,V}() where {K,V} + return VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) +end + +istrans(vnv::VarNameVector, vn::VarName) = vnv.is_transformed[vnv.varname_to_index[vn]] +function settrans!(vnv::VarNameVector, val::Bool, vn::VarName) + return vnv.is_transformed[vnv.varname_to_index[vn]] = val +end + +VarNameVector() = VarNameVector{VarName,Real}() +VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) +VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) +VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) +function VarNameVector( + varnames::AbstractVector, vals::AbstractVector, transforms=map(from_vec_transform, vals) +) + # TODO: Check uniqueness of `varnames`? + + # Convert `vals` into a vector of vectors. + vals_vecs = map(tovec, vals) + + # TODO: Is this really the way to do this? + if !(eltype(varnames) <: VarName) + varnames = convert(Vector{VarName}, varnames) + end + varname_to_index = OrderedDict{eltype(varnames),Int}() + ranges = Vector{UnitRange{Int}}() + offset = 0 + for (i, (vn, x)) in enumerate(zip(varnames, vals_vecs)) + # Add the varname index. + push!(varname_to_index, vn => length(varname_to_index) + 1) + # Add the range. + r = (offset + 1):(offset + length(x)) + push!(ranges, r) + # Update the offset. + offset = r[end] + end + + return VarNameVector( + varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms + ) +end + +""" + replace_values(vnv::VarNameVector, vals::AbstractVector) + +Replace the values in `vnv` with `vals`. + +This is useful when we want to update the entire underlying vector of values +in one go or if we want to change the how the values are stored, e.g. alter the `eltype`. + +!!! warning + This replaces the raw underlying values, and so care should be taken when using this + function. For example, if `vnv` has any inactive entries, then the provided `vals` + should also contain the inactive entries to avoid unexpected behavior. + +# Example + +```jldoctest varnamevector-replace-values +julia> using DynamicPPL: VarNameVector, replace_values + +julia> vnv = VarNameVector(@varname(x) => [1.0]); + +julia> replace_values(vnv, [2.0])[@varname(x)] == [2.0] +true +``` + +This is also useful when we want to differentiate wrt. the values +using automatic differentiation, e.g. ForwardDiff.jl. + +```jldoctest varnamevector-replace-values +julia> using ForwardDiff: ForwardDiff + +julia> f(x) = sum(abs2, replace_values(vnv, x)[@varname(x)]) +f (generic function with 1 method) + +julia> ForwardDiff.gradient(f, [1.0]) +1-element Vector{Float64}: + 2.0 +``` +""" +replace_values(vnv::VarNameVector, vals) = Setfield.@set vnv.vals = vals + +# Some `VarNameVector` specific functions. +getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] + +getrange(vnv::VarNameVector, idx::Int) = vnv.ranges[idx] +getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) + +gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] + +""" + has_inactive(vnv::VarNameVector) + +Returns `true` if `vnv` has inactive ranges. +""" +has_inactive(vnv::VarNameVector) = !isempty(vnv.num_inactive) + +""" + num_inactive(vnv::VarNameVector) + +Return the number of inactive entries in `vnv`. +""" +num_inactive(vnv::VarNameVector) = sum(values(vnv.num_inactive)) + +""" + num_inactive(vnv::VarNameVector, vn::VarName) + +Returns the number of inactive entries for `vn` in `vnv`. +""" +num_inactive(vnv::VarNameVector, vn::VarName) = num_inactive(vnv, getidx(vnv, vn)) +num_inactive(vnv::VarNameVector, idx::Int) = get(vnv.num_inactive, idx, 0) + +""" + num_allocated(vnv::VarNameVector) + +Returns the number of allocated entries in `vnv`. +""" +num_allocated(vnv::VarNameVector) = length(vnv.vals) + +""" + num_allocated(vnv::VarNameVector, vn::VarName) + +Returns the number of allocated entries for `vn` in `vnv`. +""" +num_allocated(vnv::VarNameVector, vn::VarName) = num_allocated(vnv, getidx(vnv, vn)) +function num_allocated(vnv::VarNameVector, idx::Int) + return length(getrange(vnv, idx)) + num_inactive(vnv, idx) +end + +# Basic array interface. +Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) +Base.length(vnv::VarNameVector) = + if isempty(vnv.num_inactive) + length(vnv.vals) + else + sum(length, vnv.ranges) + end +Base.size(vnv::VarNameVector) = (length(vnv),) +Base.isempty(vnv::VarNameVector) = isempty(vnv.varnames) + +# TODO: We should probably remove this +Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() + +# Dictionary interface. +Base.keys(vnv::VarNameVector) = vnv.varnames +Base.values(vnv::VarNameVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) +Base.pairs(vnv::VarNameVector) = (vn => vnv[vn] for vn in keys(vnv)) + +Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) + +# `getindex` & `setindex!` +Base.getindex(vnv::VarNameVector, i::Int) = getindex_raw(vnv, i) +function Base.getindex(vnv::VarNameVector, vn::VarName) + x = getindex_raw(vnv, vn) + f = gettransform(vnv, vn) + return f(x) +end + +function find_range_from_sorted(ranges::AbstractVector{<:AbstractRange}, x) + # TODO: Assume `ranges` to be sorted and contiguous, and use `searchsortedfirst` + # for a more efficient approach. + range_idx = findfirst(Base.Fix1(∈, x), ranges) + + # If we're out of bounds, we raise an error. + if range_idx === nothing + throw(ArgumentError("Value $x is not in any of the ranges.")) + end + + return range_idx +end + +function adjusted_ranges(vnv::VarNameVector) + # Every range following inactive entries needs to be shifted. + offset = 0 + ranges_adj = similar(vnv.ranges) + for (idx, r) in enumerate(vnv.ranges) + # Remove the `offset` in `r` due to inactive entries. + ranges_adj[idx] = r .- offset + # Update `offset`. + offset += get(vnv.num_inactive, idx, 0) + end + + return ranges_adj +end + +function index_to_raw_index(vnv::VarNameVector, i::Int) + # If we don't have any inactive entries, there's nothing to do. + has_inactive(vnv) || return i + + # Get the adjusted ranges. + ranges_adj = adjusted_ranges(vnv) + # Determine the adjusted range that the index corresponds to. + r_idx = find_range_from_sorted(ranges_adj, i) + r = vnv.ranges[r_idx] + # Determine how much of the index `i` is used to get to this range. + i_used = r_idx == 1 ? 0 : sum(length, ranges_adj[1:(r_idx - 1)]) + # Use remainder to index into `r`. + i_remainder = i - i_used + return r[i_remainder] +end + +getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] +getindex_raw(vnv::VarNameVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] + +# `getindex` for `Colon` +function Base.getindex(vnv::VarNameVector, ::Colon) + return if has_inactive(vnv) + mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) + else + vnv.vals + end +end + +function getindex_raw(vnv::VarNameVector, ::Colon) + return if has_inactive(vnv) + mapreduce(Base.Fix1(getindex_raw, vnv.vals), vcat, vnv.ranges) + else + vnv.vals + end +end + +# HACK: remove this as soon as possible. +Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] + +Base.setindex!(vnv::VarNameVector, val, i::Int) = setindex_raw!(vnv, val, i) +function Base.setindex!(vnv::VarNameVector, val, vn::VarName) + f = inverse(gettransform(vnv, vn)) + return setindex_raw!(vnv, f(val), vn) +end + +setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] = val +function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) + return vnv.vals[getrange(vnv, vn)] = val +end + +# `empty!(!)` +function Base.empty!(vnv::VarNameVector) + # TODO: Or should the semantics be different, e.g. keeping `varnames`? + empty!(vnv.varname_to_index) + empty!(vnv.varnames) + empty!(vnv.ranges) + empty!(vnv.vals) + empty!(vnv.transforms) + empty!(vnv.num_inactive) + return nothing +end +BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) + +function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) + # Return early if possible. + isempty(left_vnv) && return deepcopy(right_vnv) + isempty(right_vnv) && return deepcopy(left_vnv) + + # Determine varnames. + vns_left = left_vnv.varnames + vns_right = right_vnv.varnames + vns_both = union(vns_left, vns_right) + + # Determine `eltype` of `vals`. + T_left = eltype(left_vnv.vals) + T_right = eltype(right_vnv.vals) + T = promote_type(T_left, T_right) + # TODO: Is this necessary? + if !(T <: Real) + T = Real + end + + # Determine `eltype` of `varnames`. + V_left = eltype(left_vnv.varnames) + V_right = eltype(right_vnv.varnames) + V = promote_type(V_left, V_right) + if !(V <: VarName) + V = VarName + end + + # Determine `eltype` of `transforms`. + F_left = eltype(left_vnv.transforms) + F_right = eltype(right_vnv.transforms) + F = promote_type(F_left, F_right) + + # Allocate. + varnames_to_index = OrderedDict{V,Int}() + ranges = UnitRange{Int}[] + vals = T[] + transforms = F[] + is_transformed = BitVector(undef, length(vns_both)) + + # Range offset. + offset = 0 + + for (idx, vn) in enumerate(vns_both) + # Extract the necessary information from `left` or `right`. + if vn in vns_left && !(vn in vns_right) + # `vn` is only in `left`. + varnames_to_index[vn] = idx + val = getindex_raw(left_vnv, vn) + n = length(val) + r = (offset + 1):(offset + n) + f = gettransform(left_vnv, vn) + is_transformed[idx] = istrans(left_vnv, vn) + else + # `vn` is either in both or just `right`. + varnames_to_index[vn] = idx + val = getindex_raw(right_vnv, vn) + n = length(val) + r = (offset + 1):(offset + n) + f = gettransform(right_vnv, vn) + is_transformed[idx] = istrans(right_vnv, vn) + end + # Update. + append!(vals, val) + push!(ranges, r) + push!(transforms, f) + # Increment `offset`. + offset += n + end + + return VarNameVector(varnames_to_index, vns_both, ranges, vals, transforms) +end + +function subset(vnv::VarNameVector, vns::AbstractVector{<:VarName}) + # NOTE: This does not specialize types when possible. + vnv_new = similar(vnv) + # Return early if possible. + isempty(vnv) && return vnv_new + + for vn in vns + push!(vnv_new, vn, getindex_internal(vnv, vn), gettransform(vnv, vn)) + end + + return vnv_new +end + +# `similar` +similar_metadata(::Nothing) = nothing +similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) +function Base.similar(vnv::VarNameVector) + # NOTE: Whether or not we should empty the underlying containers or note + # is somewhat ambiguous. For example, `similar(vnv.varname_to_index)` will + # result in an empty `AbstractDict`, while the vectors, e.g. `vnv.ranges`, + # will result in non-empty vectors but with entries as `undef`. But it's + # much easier to write the rest of the code assuming that `undef` is not + # present, and so for now we empty the underlying containers, thus differing + # from the behavior of `similar` for `AbstractArray`s. + return VarNameVector( + similar(vnv.varname_to_index), + similar(vnv.varnames, 0), + similar(vnv.ranges, 0), + similar(vnv.vals, 0), + similar(vnv.transforms, 0), + BitVector(), + similar(vnv.num_inactive), + similar_metadata(vnv.metadata), + ) +end + +""" + is_contiguous(vnv::VarNameVector) + +Returns `true` if the underlying data of `vnv` is stored in a contiguous array. + +This is equivalent to negating [`has_inactive(vnv)`](@ref). +""" +is_contiguous(vnv::VarNameVector) = !has_inactive(vnv) + +function nextrange(vnv::VarNameVector, x) + # If `vnv` is empty, return immediately. + isempty(vnv) && return 1:length(x) + + # The offset will be the last range's end + its number of inactive entries. + vn_last = vnv.varnames[end] + idx = getidx(vnv, vn_last) + offset = last(getrange(vnv, idx)) + num_inactive(vnv, idx) + + return (offset + 1):(offset + length(x)) +end + +# `push!` and `push!!`: add a variable to the varname vector. +function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) + # Error if we already have the variable. + haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) + # NOTE: We need to compute the `nextrange` BEFORE we start mutating + # the underlying; otherwise we might get some strange behaviors. + val_vec = tovec(val) + r_new = nextrange(vnv, val_vec) + vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 + push!(vnv.varnames, vn) + push!(vnv.ranges, r_new) + append!(vnv.vals, val_vec) + push!(vnv.transforms, transform) + push!(vnv.is_transformed, false) + return nothing +end + +""" + shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) + +Shifts the elements of `x` starting from index `start` by `n` to the right. +""" +function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) + x[(start + n):end] = x[start:(end - n)] + return x +end + +""" + shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) + +Shifts the ranges of variables in `vnv` starting from index `idx` by `n`. +""" +function shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) + for i in (idx + 1):length(vnv.ranges) + vnv.ranges[i] = vnv.ranges[i] .+ n + end + return nothing +end + +# `update!` and `update!!`: update a variable in the varname vector. +""" + update!(vnv::VarNameVector, vn::VarName, val[, transform]) + +Either add a new entry or update existing entry for `vn` in `vnv` with the value `val`. + +If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). +""" +function update!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) + if !haskey(vnv, vn) + # Here we just add a new entry. + return push!(vnv, vn, val, transform) + end + + # Here we update an existing entry. + val_vec = tovec(val) + idx = getidx(vnv, vn) + # Extract the old range. + r_old = getrange(vnv, idx) + start_old, end_old = first(r_old), last(r_old) + n_old = length(r_old) + # Compute the new range. + n_new = length(val_vec) + start_new = start_old + end_new = start_old + n_new - 1 + r_new = start_new:end_new + + #= + Suppose we currently have the following: + + | x | x | o | o | o | y | y | y | <- Current entries + + where 'O' denotes an inactive entry, and we're going to + update the variable `x` to be of size `k` instead of 2. + + We then have a few different scenarios: + 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. + E.g. if `k = 7`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | x | x | y | y | y | <- New entries + + 2. `k = 5`: All inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | y | y | y | <- New entries + + 3. `k < 5`: Some inactive entries become active, some remain inactive. + E.g. if `k = 3`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | o | o | y | y | y | <- New entries + + 4. `k = 2`: No inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | o | o | o | y | y | y | <- New entries + + 5. `k < 2`: More entries become inactive. + E.g. if `k = 1`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | o | o | o | o | y | y | y | <- New entries + =# + + # Compute the allocated space for `vn`. + had_inactive = haskey(vnv.num_inactive, idx) + n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old + + if n_new > n_allocated + # Then we need to grow the underlying vector. + n_extra = n_new - n_allocated + # Allocate. + resize!(vnv.vals, length(vnv.vals) + n_extra) + # Shift current values. + shift_right!(vnv.vals, end_old + 1, n_extra) + # No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) + # Update the ranges for all variables after this one. + shift_subsequent_ranges_by!(vnv, idx, n_extra) + elseif n_new == n_allocated + # => No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) + else + # `n_new < n_allocated` + # => Need to update the number of inactive entries. + vnv.num_inactive[idx] = n_allocated - n_new + end + + # Update the range for this variable. + vnv.ranges[idx] = r_new + # Update the value. + vnv.vals[r_new] = val_vec + # Update the transform. + vnv.transforms[idx] = transform + + # TODO: Should we maybe sweep over inactive ranges and re-contiguify + # if we the total number of inactive elements is "large" in some sense? + + return nothing +end + +function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) + offset = 0 + for i in 1:length(ranges) + r_old = ranges[i] + ranges[i] = (offset + 1):(offset + length(r_old)) + offset += length(r_old) + end + + return ranges +end + +""" + contiguify!(vnv::VarNameVector) + +Re-contiguify the underlying vector and shrink if possible. +""" +function contiguify!(vnv::VarNameVector) + # Extract the re-contiguified values. + # NOTE: We need to do this before we update the ranges. + vals = vnv[:] + # And then we re-contiguify the ranges. + recontiguify_ranges!(vnv.ranges) + # Clear the inactive ranges. + empty!(vnv.num_inactive) + # Now we update the values. + for (i, r) in enumerate(vnv.ranges) + vnv.vals[r] = vals[r] + end + # And (potentially) shrink the underlying vector. + resize!(vnv.vals, vnv.ranges[end][end]) + # The rest should be left as is. + return vnv +end + +""" + group_by_symbol(vnv::VarNameVector) + +Return a dictionary mapping symbols to `VarNameVector`s with +varnames containing that symbol. +""" +function group_by_symbol(vnv::VarNameVector) + # Group varnames in `vnv` by the symbol. + d = OrderedDict{Symbol,Vector{VarName}}() + for vn in vnv.varnames + push!(get!(d, getsym(vn), Vector{VarName}()), vn) + end + + # Create a `NamedTuple` from the grouped varnames. + nt_vals = map(values(d)) do varnames + # TODO: Do we need to specialize the inputs here? + VarNameVector( + map(identity, varnames), + map(Base.Fix1(getindex, vnv), varnames), + map(Base.Fix1(gettransform, vnv), varnames), + ) + end + + return OrderedDict(zip(keys(d), nt_vals)) +end + +""" + shift_index_left!(vnv::VarNameVector, idx::Int) + +Shift the index `idx` to the left by one and update the relevant fields. + +!!! warning + This does not check if index we're shifting to is already occupied. +""" +function shift_index_left!(vnv::VarNameVector, idx::Int) + # Shift the index in the lookup table. + vn = vnv.varnames[idx] + vnv.varname_to_index[vn] = idx - 1 + # Shift the index in the inactive ranges. + if haskey(vnv.num_inactive, idx) + # Done in increasing order => don't need to worry about + # potentially shifting the same index twice. + vnv.num_inactive[idx - 1] = pop!(vnv.num_inactive, idx) + end +end + +""" + shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) + +Shift the indices for all variables after `idx` to the left by one and update +the relevant fields. + +This just +""" +function shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) + # Shift the indices for all variables after `idx`. + for idx_to_shift in (idx + 1):length(vnv.varnames) + shift_index_left!(vnv, idx_to_shift) + end +end + +function Base.delete!(vnv::VarNameVector, vn::VarName) + # Error if we don't have the variable. + !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) + + # Get the index of the variable. + idx = getidx(vnv, vn) + + # Delete the values. + r_start = first(getrange(vnv, idx)) + n_allocated = num_allocated(vnv, idx) + # NOTE: `deleteat!` also results in a `resize!` so we don't need to do that. + deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) + + # Delete `vn` from the lookup table. + delete!(vnv.varname_to_index, vn) + + # Delete any inactive ranges corresponding to `vn`. + haskey(vnv.num_inactive, idx) && delete!(vnv.num_inactive, idx) + + # Re-adjust the indices for varnames occuring after `vn` so + # that they point to the correct indices after the deletions below. + shift_subsequent_indices_left!(vnv, idx) + + # Re-adjust the ranges for varnames occuring after `vn`. + shift_subsequent_ranges_by!(vnv, idx, -n_allocated) + + # Delete references from vector fields, thus shifting the indices of + # varnames occuring after `vn` by one to the left, as we adjusted for above. + deleteat!(vnv.varnames, idx) + deleteat!(vnv.ranges, idx) + deleteat!(vnv.transforms, idx) + + return vnv +end + +""" + values_as(vnv::VarNameVector[, T]) + +Return the values/realizations in `vnv` as type `T`, if implemented. + +If no type `T` is provided, return values as stored in `vnv`. + +# Examples + +```jldoctest +julia> using DynamicPPL: VarNameVector + +julia> vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2.0]); + +julia> values_as(vnv) == [1.0, 2.0] +true + +julia> values_as(vnv, Vector{Float32}) == Vector{Float32}([1.0, 2.0]) +true + +julia> values_as(vnv, OrderedDict) == OrderedDict(@varname(x) => 1.0, @varname(y) => [2.0]) +true + +julia> values_as(vnv, NamedTuple) == (x = 1.0, y = [2.0]) +true +``` +""" +values_as(vnv::VarNameVector) = values_as(vnv, Vector) +values_as(vnv::VarNameVector, ::Type{Vector}) = vnv[:] +function values_as(vnv::VarNameVector, ::Type{Vector{T}}) where {T} + return convert(Vector{T}, values_as(vnv, Vector)) +end +function values_as(vnv::VarNameVector, ::Type{NamedTuple}) + return NamedTuple(zip(map(Symbol, keys(vnv)), values(vnv))) +end +function values_as(vnv::VarNameVector, ::Type{D}) where {D<:AbstractDict} + return ConstructionBase.constructorof(D)(pairs(vnv)) +end diff --git a/test/runtests.jl b/test/runtests.jl index 43d68386c..7ee2696f0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -35,6 +35,7 @@ include("test_util.jl") @testset "interface" begin include("utils.jl") include("compiler.jl") + include("varnamevector.jl") include("varinfo.jl") include("simple_varinfo.jl") include("model.jl") diff --git a/test/test_util.jl b/test/test_util.jl index 64832f51e..021da2598 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -84,7 +84,10 @@ Return string representing a short description of `vi`. """ short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" -short_varinfo_name(::TypedVarInfo) = "TypedVarInfo" +function short_varinfo_name(vi::TypedVarInfo) + DynamicPPL.has_varnamevector(vi) && return "TypedVarInfo with VarNameVector" + return "TypedVarInfo" +end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" diff --git a/test/varinfo.jl b/test/varinfo.jl index 5c23f75a6..026b10975 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -409,6 +409,12 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end + if DynamicPPL.has_varnamevector(varinfo) && mutating + # NOTE: Can't handle mutating `link!` and `invlink!` `VarNameVector`. + @test_broken false + continue + end + # Evaluate the model once to update the logp of the varinfo. varinfo = last(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())) diff --git a/test/varnamevector.jl b/test/varnamevector.jl new file mode 100644 index 000000000..6b7acfbeb --- /dev/null +++ b/test/varnamevector.jl @@ -0,0 +1,472 @@ +replace_sym(vn::VarName, sym_new::Symbol) = VarName{sym_new}(vn.lens) + +increase_size_for_test(x::Real) = [x] +increase_size_for_test(x::AbstractArray) = repeat(x, 2) + +decrease_size_for_test(x::Real) = x +decrease_size_for_test(x::AbstractVector) = first(x) +decrease_size_for_test(x::AbstractArray) = first(eachslice(x; dims=1)) + +function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) + if isconcretetype(eltype(vnv.varnames)) + # If the container is concrete, we need to make sure that the varname types match. + # E.g. if `vnv.varnames` has `eltype` `VarName{:x, IndexLens{Tuple{Int64}}}` then + # we need `vn` to also be of this type. + # => If the varname types don't match, we need to relax the container type. + return any(keys(vnv)) do vn_present + typeof(vn_present) !== typeof(val) + end + end + + return false +end +function need_varnames_relaxation(vnv::VarNameVector, vns, vals) + return any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end + +function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) + if isconcretetype(eltype(vnv.vals)) + return promote_type(eltype(vnv.vals), eltype(val)) != eltype(vnv.vals) + end + + return false +end +function need_values_relaxation(vnv::VarNameVector, vns, vals) + return any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end + +function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) + return if isconcretetype(eltype(vnv.transforms)) + # If the container is concrete, we need to make sure that the sizes match. + # => If the sizes don't match, we need to relax the container type. + any(keys(vnv)) do vn_present + size(vnv[vn_present]) != size(val) + end + elseif eltype(vnv.transforms) !== Any + # If it's not concrete AND it's not `Any`, then we should just make it `Any`. + true + else + # Otherwise, it's `Any`, so we don't need to relax the container type. + false + end +end +function need_transforms_relaxation(vnv::VarNameVector, vns, vals) + return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) +end + +""" + relax_container_types(vnv::VarNameVector, vn::VarName, val) + relax_container_types(vnv::VarNameVector, vns, val) + +Relax the container types of `vnv` if necessary to accommodate `vn` and `val`. + +This attempts to avoid unnecessary container type relaxations by checking whether +the container types of `vnv` are already compatible with `vn` and `val`. + +# Notes +For example, if `vn` is not compatible with the current keys in `vnv`, then +the underlying types will be changed to `VarName` to accommodate `vn`. + +Similarly: +- If `val` is not compatible with the current values in `vnv`, then + the underlying value type will be changed to `Real`. +- If `val` requires a transformation that is not compatible with the current + transformations type in `vnv`, then the underlying transformation type will + be changed to `Any`. +""" +function relax_container_types(vnv::VarNameVector, vn::VarName, val) + return relax_container_types(vnv, [vn], [val]) +end +function relax_container_types(vnv::VarNameVector, vns, vals) + if need_varnames_relaxation(vnv, vns, vals) + varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) + varnames_new = convert(Vector{VarName}, vnv.varnames) + else + varname_to_index_new = vnv.varname_to_index + varnames_new = vnv.varnames + end + + transforms_new = if need_transforms_relaxation(vnv, vns, vals) + convert(Vector{Any}, vnv.transforms) + else + vnv.transforms + end + + vals_new = if need_values_relaxation(vnv, vns, vals) + convert(Vector{Real}, vnv.vals) + else + vnv.vals + end + + return VarNameVector( + varname_to_index_new, + varnames_new, + vnv.ranges, + vals_new, + transforms_new, + vnv.is_transformed, + vnv.num_inactive, + vnv.metadata, + ) +end + +@testset "VarNameVector" begin + # Need to test element-related operations: + # - `getindex` + # - `setindex!` + # - `push!` + # - `update!` + # + # And these should all be tested for different types of values: + # - scalar + # - vector + # - matrix + + # Need to test operations on `VarNameVector`: + # - `empty!` + # - `iterate` + # - `convert` to + # - `AbstractDict` + test_pairs = OrderedDict( + @varname(x[1]) => rand(), + @varname(x[2]) => rand(2), + @varname(x[3]) => rand(2, 3), + @varname(y[1]) => rand(), + @varname(y[2]) => rand(2), + @varname(y[3]) => rand(2, 3), + @varname(z[1]) => rand(1:10), + @varname(z[2]) => rand(1:10, 2), + @varname(z[3]) => rand(1:10, 2, 3), + ) + test_vns = collect(keys(test_pairs)) + test_vals = collect(values(test_pairs)) + + @testset "constructor: no args" begin + # Empty. + vnv = VarNameVector() + @test isempty(vnv) + @test eltype(vnv) == Real + + # Empty with types. + vnv = VarNameVector{VarName,Float64}() + @test isempty(vnv) + @test eltype(vnv) == Float64 + end + + test_varnames_iter = combinations(test_vns, 2) + @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter + val_left = test_pairs[vn_left] + val_right = test_pairs[vn_right] + vnv_base = VarNameVector([vn_left, vn_right], [val_left, val_right]) + + # We'll need the transformations later. + # TODO: Should we test other transformations than just `FromVec`? + from_vec_left = DynamicPPL.from_vec_transform(val_left) + from_vec_right = DynamicPPL.from_vec_transform(val_right) + to_vec_left = inverse(from_vec_left) + to_vec_right = inverse(from_vec_right) + + # Compare to alternative constructors. + vnv_from_dict = VarNameVector( + OrderedDict(vn_left => val_left, vn_right => val_right) + ) + @test vnv_base == vnv_from_dict + + # We want the types of fields such as `varnames` and `transforms` to specialize + # whenever possible + some functionality, e.g. `push!`, is only sensible + # if the underlying containers can support it. + # Expected behavior + should_have_restricted_varname_type = typeof(vn_left) == typeof(vn_right) + should_have_restricted_transform_type = size(val_left) == size(val_right) + # Actual behavior + has_restricted_transform_type = isconcretetype(eltype(vnv_base.transforms)) + has_restricted_varname_type = isconcretetype(eltype(vnv_base.varnames)) + + @testset "type specialization" begin + @test !should_have_restricted_varname_type || has_restricted_varname_type + @test !should_have_restricted_transform_type || has_restricted_transform_type + end + + # `eltype` + @test eltype(vnv_base) == promote_type(eltype(val_left), eltype(val_right)) + # `length` + @test length(vnv_base) == length(val_left) + length(val_right) + + # `isempty` + @test !isempty(vnv_base) + + # `empty!` + @testset "empty!" begin + vnv = deepcopy(vnv_base) + empty!(vnv) + @test isempty(vnv) + end + + # `similar` + @testset "similar" begin + vnv = similar(vnv_base) + @test isempty(vnv) + @test typeof(vnv) == typeof(vnv_base) + end + + # `getindex` + @testset "getindex" begin + # With `VarName` index. + @test vnv_base[vn_left] == val_left + @test vnv_base[vn_right] == val_right + + # With `Int` index. + val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) + @test all(vnv_base[i] == val_vec[i] for i in 1:length(val_vec)) + end + + # `setindex!` + @testset "setindex!" begin + vnv = deepcopy(vnv_base) + vnv[vn_left] = val_left .+ 100 + @test vnv[vn_left] == val_left .+ 100 + vnv[vn_right] = val_right .+ 100 + @test vnv[vn_right] == val_right .+ 100 + end + + # `getindex_raw` + @testset "getindex_raw" begin + # With `VarName` index. + @test DynamicPPL.getindex_raw(vnv_base, vn_left) == to_vec_left(val_left) + @test DynamicPPL.getindex_raw(vnv_base, vn_right) == to_vec_right(val_right) + # With `Int` index. + val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) + @test all( + DynamicPPL.getindex_raw(vnv_base, i) == val_vec[i] for + i in 1:length(val_vec) + ) + end + + # `setindex_raw!` + @testset "setindex_raw!" begin + vnv = deepcopy(vnv_base) + DynamicPPL.setindex_raw!(vnv, to_vec_left(val_left .+ 100), vn_left) + @test vnv[vn_left] == val_left .+ 100 + DynamicPPL.setindex_raw!(vnv, to_vec_right(val_right .+ 100), vn_right) + @test vnv[vn_right] == val_right .+ 100 + end + + # `delete!` + @testset "delete!" begin + vnv = deepcopy(vnv_base) + delete!(vnv, vn_left) + @test !haskey(vnv, vn_left) + @test haskey(vnv, vn_right) + delete!(vnv, vn_right) + @test !haskey(vnv, vn_right) + end + + # `merge` + @testset "merge" begin + # When there are no inactive entries, `merge` on itself result in the same. + @test merge(vnv_base, vnv_base) == vnv_base + + # Merging with empty should result in the same. + @test merge(vnv_base, similar(vnv_base)) == vnv_base + @test merge(similar(vnv_base), vnv_base) == vnv_base + + # With differences. + vnv_left_only = deepcopy(vnv_base) + delete!(vnv_left_only, vn_right) + vnv_right_only = deepcopy(vnv_base) + delete!(vnv_right_only, vn_left) + + # `(x,)` and `(x, y)` should be `(x, y)`. + @test merge(vnv_left_only, vnv_base) == vnv_base + # `(x, y)` and `(x,)` should be `(x, y)`. + @test merge(vnv_base, vnv_left_only) == vnv_base + # `(x, y)` and `(y,)` should be `(x, y)`. + @test merge(vnv_base, vnv_right_only) == vnv_base + # `(y,)` and `(x, y)` should be `(y, x)`. + vnv_merged = merge(vnv_right_only, vnv_base) + @test vnv_merged != vnv_base + @test collect(keys(vnv_merged)) == [vn_right, vn_left] + end + + # `push!` & `update!` + @testset "push!" begin + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) + @testset "$vn" for vn in test_vns + val = test_pairs[vn] + if vn == vn_left || vn == vn_right + # Should not be possible to `push!` existing varname. + @test_throws ArgumentError push!(vnv, vn, val) + else + push!(vnv, vn, val) + @test vnv[vn] == val + end + end + end + @testset "update!" begin + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) + @testset "$vn" for vn in test_vns + val = test_pairs[vn] + expected_length = if haskey(vnv, vn) + # If it's already present, the resulting length will be unchanged. + length(vnv) + else + length(vnv) + length(val) + end + + DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(x) == length(vnv) + + # There should be no redundant values in the underlying vector. + @test !DynamicPPL.has_inactive(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) + end + + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) + @testset "$vn (increased size)" for vn in test_vns + val_original = test_pairs[vn] + val = increase_size_for_test(val_original) + vn_already_present = haskey(vnv, vn) + expected_length = if vn_already_present + # If it's already present, the resulting length will be altered. + length(vnv) + length(val) - length(val_original) + else + length(vnv) + length(val) + end + + DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(x) == length(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) + end + + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) + @testset "$vn (decreased size)" for vn in test_vns + val_original = test_pairs[vn] + val = decrease_size_for_test(val_original) + vn_already_present = haskey(vnv, vn) + expected_length = if vn_already_present + # If it's already present, the resulting length will be altered. + length(vnv) + length(val) - length(val_original) + else + length(vnv) + length(val) + end + DynamicPPL.update!(vnv, vn, val .+ 1) + x = vnv[:] + @test vnv[vn] == val .+ 1 + @test length(vnv) == expected_length + @test length(x) == length(vnv) + + # `getindex` with `Int` index. + @test all(vnv[i] == x[i] for i in 1:length(x)) + end + end + end + + @testset "growing and shrinking" begin + @testset "deterministic" begin + n = 5 + vn = @varname(x) + vnv = VarNameVector(OrderedDict(vn => [true])) + @test !DynamicPPL.has_inactive(vnv) + # Growing should not create inactive ranges. + for i in 1:n + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test !DynamicPPL.has_inactive(vnv) + end + + # Same size should not create inactive ranges. + x = fill(true, n) + DynamicPPL.update!(vnv, vn, x) + @test !DynamicPPL.has_inactive(vnv) + + # Shrinking should create inactive ranges. + for i in (n - 1):-1:1 + x = fill(true, i) + DynamicPPL.update!(vnv, vn, x) + @test DynamicPPL.has_inactive(vnv) + @test DynamicPPL.num_inactive(vnv, vn) == n - i + end + end + + @testset "random" begin + n = 5 + vn = @varname(x) + vnv = VarNameVector(OrderedDict(vn => [true])) + @test !DynamicPPL.has_inactive(vnv) + + # Insert a bunch of random-length vectors. + for i in 1:100 + x = fill(true, rand(1:n)) + DynamicPPL.update!(vnv, vn, x) + end + # Should never be allocating more than `n` elements. + @test DynamicPPL.num_allocated(vnv, vn) ≤ n + + # If we compaticfy, then it should always be the same size as just inserted. + for i in 1:10 + x = fill(true, rand(1:n)) + DynamicPPL.update!(vnv, vn, x) + DynamicPPL.contiguify!(vnv) + @test DynamicPPL.num_allocated(vnv, vn) == length(x) + end + end + end +end + +@testset "VarInfo + VarNameVector" begin + models = DynamicPPL.TestUtils.DEMO_MODELS + @testset "$(model.f)" for model in models + # NOTE: Need to set random seed explicitly to avoid using the same seed + # for initialization as for sampling in the inner testset below. + Random.seed!(42) + value_true = DynamicPPL.TestUtils.rand_prior_true(model) + vns = DynamicPPL.TestUtils.varnames(model) + varnames = DynamicPPL.TestUtils.varnames(model) + varinfos = DynamicPPL.TestUtils.setup_varinfos( + model, value_true, varnames; include_threadsafe=false + ) + # Filter out those which are not based on `VarNameVector`. + varinfos = filter(DynamicPPL.has_varnamevector, varinfos) + # Get the true log joint. + logp_true = DynamicPPL.TestUtils.logjoint_true(model, value_true...) + + @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos + # Need to make sure we're using a different random seed from the + # one used in the above call to `rand_prior_true`. + Random.seed!(43) + + # Are values correct? + DynamicPPL.TestUtils.test_values(varinfo, value_true, vns) + + # Is evaluation correct? + varinfo_eval = last( + DynamicPPL.evaluate!!(model, deepcopy(varinfo), DefaultContext()) + ) + # Log density should be the same. + @test getlogp(varinfo_eval) ≈ logp_true + # Values should be the same. + DynamicPPL.TestUtils.test_values(varinfo_eval, value_true, vns) + + # Is sampling correct? + varinfo_sample = last( + DynamicPPL.evaluate!!(model, deepcopy(varinfo), SamplingContext()) + ) + # Log density should be different. + @test getlogp(varinfo_sample) != getlogp(varinfo) + # Values should be different. + DynamicPPL.TestUtils.test_values( + varinfo_sample, value_true, vns; compare=!isequal + ) + end + end +end From ff68206d4b34e230c7e444e91fc60297dd5a5bd0 Mon Sep 17 00:00:00 2001 From: Tor Erlend Fjelde Date: Tue, 6 Feb 2024 22:40:21 +0000 Subject: [PATCH 158/209] Revert "remove Combinatorics as a test dep, as it's not needed for this PR" This reverts commit 071bebfb4e7fd669994ae5dc7aecf1cfcbc92f6b. --- test/Project.toml | 2 ++ test/runtests.jl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/Project.toml b/test/Project.toml index 80c227920..e2bd0d7e5 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,6 +2,7 @@ AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" +Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -26,6 +27,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" AbstractMCMC = "5" AbstractPPL = "0.7" Bijectors = "0.13" +Combinatorics = "1" Compat = "4.3.0" Distributions = "0.25" DistributionsAD = "0.6.3" diff --git a/test/runtests.jl b/test/runtests.jl index 7ee2696f0..ef7672db8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,6 +20,8 @@ using Random using Serialization using Test +using Combinatorics: combinations + using DynamicPPL: getargs_dottilde, getargs_tilde, Selector const DIRECTORY_DynamicPPL = dirname(dirname(pathof(DynamicPPL))) From 19978ec21ba26defe9d5f92c7ba61a116033bf52 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 3 Sep 2024 11:01:06 +0200 Subject: [PATCH 159/209] More work on `VarNameVector` (#637) * Update test/model.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Markus Hauru * Type-stability tests are now correctly using `rand_prior_true` instead of `rand` * `getindex_internal` now calls `getindex` instead of `view`, as the latter can result in type-instability since transformed variables typically result in non-view even if input is a view * Removed seemingly unnecessary definition of `getindex_internal` * Fixed references to `newmetadata` which has been replaced by `replace_values` * Made implementation of `recombine` more explicit * Added docstrings for `untyped_varinfo` and `typed_varinfo` * Added TODO comment about implementing `view` for `VarInfo` * Fixed potential infinite recursion as suggested by @mhauru * added docstring to `from_vec_trnasform_for_size * Replaced references to `vectorize(dist, x)` with `tovec(x)` * Fixed docstring * Update src/extract_priors.jl Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Bump minor version since this is a breaking change * Apply suggestions from code review Co-authored-by: Markus Hauru * Update src/varinfo.jl Co-authored-by: Tor Erlend Fjelde * Apply suggestions from code review * Apply suggestions from code review * Update src/extract_priors.jl Co-authored-by: Xianda Sun <5433119+sunxd3@users.noreply.github.com> * Added fix for product distributions of targets with changing support + tests * Addeed tests for product of distributions with dynamic support * Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Fix typos, improve docstrings * Use Accessors rather than Setfield * Simplify group_by_symbol * Add short_varinfo_name(::VectorVarInfo) * Add tests for subset * Export VectorVarInfo * Tighter type bound for has_varnamevector * Add some VectorVarName methods * Add todo notes, remove dead code, fix a typo. * Bug fixes and small improvements * VarNameVector improvements * Improve generated_quantities and its tests * Improvement to VarNameVector * Fix a test to work with VectorVarName * Fix generated_quantities * Fix type stability issues * Various VarNameVector fixes and improvements * Bump version number * Improvements to generated_quantities * Code formatting * Code style * Add fallback implementation of findinds for VarNameVector * Rename VarNameVector to VarNamedVector * More renaming of VNV. Remove unused VarNamedVector.metadata field. * Rename FromVec to ReshapeTransform * Progress towards having VarNamedVector as storage for SimpleVarInfo * Fix unflatten(vnv::VarNamedVector, vals) * More work on SimpleVarInfo{VarNamedVector} * More tests for SimpleVarInfo{VarNamedVector} * More tests for SimpleVarInfo{VarNamedVector} * Respond to review feedback * Add float_type_with_fallback(::Type{Union{}}) * Move some VNV functions to the correct file * Fix push! for VNV * Rename VNV.is_transformed to VNV.is_unconstrained * Improve VNV docstring * Add VNV inner constructor checks * Reorganise parts of VNV code * Documentation and small fixes for VNV * Rename loosen_types!! and tighten_types, add docstrings and doctests * Rename VarNameVector to VarNamedVector in docs * Documentation and small fixes to VNV * Fix subset(::VarNamedVector, args...) for unconstrained variables. --------- Co-authored-by: Tor Erlend Fjelde Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Xianda Sun <5433119+sunxd3@users.noreply.github.com> --- docs/src/internals/varinfo.md | 42 +- ext/DynamicPPLMCMCChainsExt.jl | 145 ++- ext/DynamicPPLReverseDiffExt.jl | 6 +- src/DynamicPPL.jl | 5 +- src/abstract_varinfo.jl | 6 +- src/context_implementations.jl | 15 +- src/model.jl | 79 +- src/sampler.jl | 2 +- src/simple_varinfo.jl | 33 +- src/test_utils.jl | 25 +- src/threadsafe.jl | 2 +- src/utils.jl | 39 +- src/values_as_in_model.jl | 2 +- src/varinfo.jl | 330 +++-- src/varnamedvector.jl | 1217 ++++++++++++++++++ src/varnamevector.jl | 756 ----------- test/compiler.jl | 6 +- test/model.jl | 34 +- test/runtests.jl | 2 +- test/simple_varinfo.jl | 84 +- test/test_util.jl | 4 +- test/varinfo.jl | 40 +- test/{varnamevector.jl => varnamedvector.jl} | 68 +- 23 files changed, 1835 insertions(+), 1107 deletions(-) create mode 100644 src/varnamedvector.jl delete mode 100644 src/varnamevector.jl rename test/{varnamevector.jl => varnamedvector.jl} (89%) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 8d74fd2fa..d1d2e7116 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -132,13 +132,13 @@ Hence we obtain a "type-stable when possible"-representation by wrapping it in a ## Efficient storage and iteration -Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNameVector`](@ref): +Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNamedVector`](@ref): ```@docs -DynamicPPL.VarNameVector +DynamicPPL.VarNamedVector ``` -In a [`VarNameVector{<:VarName,Vector{T}}`](@ref), we achieve the desirata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. +In a [`VarNamedVector{<:VarName,Vector{T}}`](@ref), we achieve the desirata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: @@ -146,7 +146,7 @@ This does require a bit of book-keeping, in particular when it comes to insertio - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - `transforms::Vector`: the transforms associated with each `VarName`. -Mutating functions, e.g. `setindex!(vnv::VarNameVector, val, vn::VarName)`, are then treated according to the following rules: +Mutating functions, e.g. `setindex!(vnv::VarNamedVector, val, vn::VarName)`, are then treated according to the following rules: 1. If `vn` is not already present: add it to the end of `vnv.varnames`, add the `val` to the underlying `vnv.vals`, etc. @@ -156,7 +156,7 @@ Mutating functions, e.g. `setindex!(vnv::VarNameVector, val, vn::VarName)`, are 2. If `val` has a *smaller length* than the existing value for `vn`: replace existing value and mark the remaining indices as "inactive" by increasing the entry in `vnv.num_inactive` field. 3. If `val` has a *larger length* than the existing value for `vn`: expand the underlying `vnv.vals` to accommodate the new value, update all `VarName`s occuring after `vn`, and update the `vnv.ranges` to point to the new range for `vn`. -This means that `VarNameVector` is allowed to grow as needed, while "shrinking" (i.e. insertion of smaller elements) is handled by simply marking the redundant indices as "inactive". This turns out to be efficient for use-cases that we are generally interested in. +This means that `VarNamedVector` is allowed to grow as needed, while "shrinking" (i.e. insertion of smaller elements) is handled by simply marking the redundant indices as "inactive". This turns out to be efficient for use-cases that we are generally interested in. For example, we want to optimize code-paths which effectively boil down to inner-loop in the following example: @@ -195,7 +195,7 @@ DynamicPPL.contiguify! For example, one might encounter the following scenario: ```@example varinfo-design -vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +vnv = DynamicPPL.VarNamedVector(@varname(x) => [true]) println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 @@ -210,7 +210,7 @@ end We can then insert a call to [`DynamicPPL.contiguify!`](@ref) after every insertion whenever the allocation grows too large to reduce overall memory usage: ```@example varinfo-design -vnv = DynamicPPL.VarNameVector(@varname(x) => [true]) +vnv = DynamicPPL.VarNamedVector(@varname(x) => [true]) println("Before insertion: number of allocated entries $(DynamicPPL.num_allocated(vnv))") for i in 1:5 @@ -225,13 +225,13 @@ for i in 1:5 end ``` -This does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. However, this also ensures that the the underlying `Vector{T}` is contiguous, which is important for performance. Hence, if we're about to do a lot of work with the `VarNameVector` without insertions, etc., it can be worth it to do a sweep to ensure that the underlying `Vector{T}` is contiguous. +This does incur a runtime cost as it requires re-allocation of the `ranges` in addition to a `resize!` of the underlying `Vector{T}`. However, this also ensures that the the underlying `Vector{T}` is contiguous, which is important for performance. Hence, if we're about to do a lot of work with the `VarNamedVector` without insertions, etc., it can be worth it to do a sweep to ensure that the underlying `Vector{T}` is contiguous. !!! note - Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing he `VarName`'s transformation with a `DynamicPPL.FromVec`. + Higher-dimensional arrays, e.g. `Matrix`, are handled by simply vectorizing them before storing them in the `Vector{T}`, and composing the `VarName`'s transformation with a `DynamicPPL.ReshapeTransform`. -Continuing from the example from the previous section, we can use a `VarInfo` with a `VarNameVector` as the `metadata` field: +Continuing from the example from the previous section, we can use a `VarInfo` with a `VarNamedVector` as the `metadata` field: ```@example varinfo-design # Type-unstable @@ -287,23 +287,23 @@ DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) ### Performance summary -In the end, we have the following "rough" performance characteristics for `VarNameVector`: +In the end, we have the following "rough" performance characteristics for `VarNamedVector`: -| Method | Is blazingly fast? | -|:---------------------------------------:|:--------------------------------------------------------------------------------------------:| -| `getindex` | ${\color{green} \checkmark}$ | -| `setindex!` | ${\color{green} \checkmark}$ | -| `push!` | ${\color{green} \checkmark}$ | -| `delete!` | ${\color{red} \times}$ | -| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | -| `values_as(::VarNameVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | +| Method | Is blazingly fast? | +|:----------------------------------------:|:--------------------------------------------------------------------------------------------:| +| `getindex` | ${\color{green} \checkmark}$ | +| `setindex!` | ${\color{green} \checkmark}$ | +| `push!` | ${\color{green} \checkmark}$ | +| `delete!` | ${\color{red} \times}$ | +| `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | +| `values_as(::VarNamedVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | ## Other methods ```@docs -DynamicPPL.replace_values(::VarNameVector, vals::AbstractVector) +DynamicPPL.replace_values(::VarNamedVector, vals::AbstractVector) ``` ```@docs; canonical=false -DynamicPPL.values_as(::VarNameVector) +DynamicPPL.values_as(::VarNamedVector) ``` diff --git a/ext/DynamicPPLMCMCChainsExt.jl b/ext/DynamicPPLMCMCChainsExt.jl index 8c598a6a8..f362b02cc 100644 --- a/ext/DynamicPPLMCMCChainsExt.jl +++ b/ext/DynamicPPLMCMCChainsExt.jl @@ -11,7 +11,7 @@ end _has_varname_to_symbol(info::NamedTuple{names}) where {names} = :varname_to_symbol in names function _check_varname_indexing(c::MCMCChains.Chains) return DynamicPPL.supports_varname_indexing(c) || - error("Chains do not support indexing using $vn.") + error("$(typeof(c)) do not support indexing using varnmes.") end # Load state from a `Chains`: By convention, it is stored in `:samplerstate` metadata @@ -41,6 +41,65 @@ function DynamicPPL.varnames(c::MCMCChains.Chains) return keys(c.info.varname_to_symbol) end +""" + generated_quantities(model::Model, chain::MCMCChains.Chains) + +Execute `model` for each of the samples in `chain` and return an array of the values +returned by the `model` for each sample. + +# Examples +## General +Often you might have additional quantities computed inside the model that you want to +inspect, e.g. +```julia +@model function demo(x) + # sample and observe + θ ~ Prior() + x ~ Likelihood() + return interesting_quantity(θ, x) +end +m = demo(data) +chain = sample(m, alg, n) +# To inspect the `interesting_quantity(θ, x)` where `θ` is replaced by samples +# from the posterior/`chain`: +generated_quantities(m, chain) # <= results in a `Vector` of returned values + # from `interesting_quantity(θ, x)` +``` +## Concrete (and simple) +```julia +julia> using DynamicPPL, Turing + +julia> @model function demo(xs) + s ~ InverseGamma(2, 3) + m_shifted ~ Normal(10, √s) + m = m_shifted - 10 + + for i in eachindex(xs) + xs[i] ~ Normal(m, √s) + end + + return (m, ) + end +demo (generic function with 1 method) + +julia> model = demo(randn(10)); + +julia> chain = sample(model, MH(), 10); + +julia> generated_quantities(model, chain) +10×1 Array{Tuple{Float64},2}: + (2.1964758025119338,) + (2.1964758025119338,) + (0.09270081916291417,) + (0.09270081916291417,) + (0.09270081916291417,) + (0.09270081916291417,) + (0.09270081916291417,) + (0.043088571494005024,) + (-0.16489786710222099,) + (-0.16489786710222099,) +``` +""" function DynamicPPL.generated_quantities( model::DynamicPPL.Model, chain_full::MCMCChains.Chains ) @@ -48,14 +107,86 @@ function DynamicPPL.generated_quantities( varinfo = DynamicPPL.VarInfo(model) iters = Iterators.product(1:size(chain, 1), 1:size(chain, 3)) return map(iters) do (sample_idx, chain_idx) - # Update the varinfo with the current sample and make variables not present in `chain` - # to be sampled. - DynamicPPL.setval_and_resample!(varinfo, chain, sample_idx, chain_idx) + if DynamicPPL.supports_varname_indexing(chain) + varname_pairs = _varname_pairs_with_varname_indexing( + chain, varinfo, sample_idx, chain_idx + ) + else + varname_pairs = _varname_pairs_without_varname_indexing( + chain, varinfo, sample_idx, chain_idx + ) + end + fixed_model = DynamicPPL.fix(model, Dict(varname_pairs)) + return fixed_model() + end +end + +""" + _varname_pairs_with_varname_indexing( + chain::MCMCChains.Chains, varinfo, sample_idx, chain_idx + ) - # TODO: Some of the variables can be a view into the `varinfo`, so we need to - # `deepcopy` the `varinfo` before passing it to `model`. - model(deepcopy(varinfo)) +Get pairs of `VarName => value` for all the variables in the `varinfo`, picking the values +from the chain. + +This implementation assumes `chain` can be indexed using variable names, and is the +preffered implementation. +""" +function _varname_pairs_with_varname_indexing( + chain::MCMCChains.Chains, varinfo, sample_idx, chain_idx +) + vns = DynamicPPL.varnames(chain) + vn_parents = Iterators.map(vns) do vn + # The call nested_setindex_maybe! is used to handle cases where vn is not + # the variable name used in the model, but rather subsumed by one. Except + # for the subsumption part, this could be + # vn => getindex_varname(chain, sample_idx, vn, chain_idx) + # TODO(mhauru) This call to nested_setindex_maybe! is unintuitive. + DynamicPPL.nested_setindex_maybe!( + varinfo, DynamicPPL.getindex_varname(chain, sample_idx, vn, chain_idx), vn + ) end + varname_pairs = Iterators.map(Iterators.filter(!isnothing, vn_parents)) do vn_parent + vn_parent => varinfo[vn_parent] + end + return varname_pairs +end + +""" +Check which keys in `key_strings` are subsumed by `vn_string` and return the their values. + +The subsumption check is done with `DynamicPPL.subsumes_string`, which is quite weak, and +won't catch all cases. We should get rid of this if we can. +""" +# TODO(mhauru) See docstring above. +function _vcat_subsumed_values(vn_string, values, key_strings) + indices = findall(Base.Fix1(DynamicPPL.subsumes_string, vn_string), key_strings) + return !isempty(indices) ? reduce(vcat, values[indices]) : nothing +end + +""" + _varname_pairs_without_varname_indexing( + chain::MCMCChains.Chains, varinfo, sample_idx, chain_idx + ) + +Get pairs of `VarName => value` for all the variables in the `varinfo`, picking the values +from the chain. + +This implementation does not assume that `chain` can be indexed using variable names. It is +thus not guaranteed to work in cases where the variable names have complex subsumption +patterns, such as if the model has a variable `x` but the chain stores `x.a[1]`. +""" +function _varname_pairs_without_varname_indexing( + chain::MCMCChains.Chains, varinfo, sample_idx, chain_idx +) + values = chain.value[sample_idx, :, chain_idx] + keys = Base.keys(chain) + keys_strings = map(string, keys) + varname_pairs = [ + vn => _vcat_subsumed_values(string(vn), values, keys_strings) for + vn in Base.keys(varinfo) + ] + return varname_pairs end end diff --git a/ext/DynamicPPLReverseDiffExt.jl b/ext/DynamicPPLReverseDiffExt.jl index 3fd174ed1..b2b378d45 100644 --- a/ext/DynamicPPLReverseDiffExt.jl +++ b/ext/DynamicPPLReverseDiffExt.jl @@ -9,12 +9,12 @@ else end function LogDensityProblemsAD.ADgradient( - ad::ADTypes.AutoReverseDiff{Tcompile}, ℓ::DynamicPPL.LogDensityFunction -) where {Tcompile} + ad::ADTypes.AutoReverseDiff, ℓ::DynamicPPL.LogDensityFunction +) return LogDensityProblemsAD.ADgradient( Val(:ReverseDiff), ℓ; - compile=Val(Tcompile), + compile=Val(ad.compile), # `getparams` can return `Vector{Real}`, in which case, `ReverseDiff` will initialize the gradients to Integer 0 # because at https://github.com/JuliaDiff/ReverseDiff.jl/blob/c982cde5494fc166965a9d04691f390d9e3073fd/src/tracked.jl#L473 # `zero(D)` will return 0 when D is Real. diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index 255c4b51b..969d69936 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -45,8 +45,9 @@ export AbstractVarInfo, VarInfo, UntypedVarInfo, TypedVarInfo, + VectorVarInfo, SimpleVarInfo, - VarNameVector, + VarNamedVector, push!!, empty!!, subset, @@ -176,7 +177,7 @@ include("sampler.jl") include("varname.jl") include("distribution_wrappers.jl") include("contexts.jl") -include("varnamevector.jl") +include("varnamedvector.jl") include("abstract_varinfo.jl") include("threadsafe.jl") include("varinfo.jl") diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 7ddd09b2e..551bf87d3 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -295,7 +295,7 @@ julia> # Just use an example model to construct the `VarInfo` because we're lazy julia> vi[@varname(s)] = 1.0; vi[@varname(m)] = 2.0; julia> # For the sake of brevity, let's just check the type. - md = values_as(vi); md.s isa DynamicPPL.Metadata + md = values_as(vi); md.s isa Union{DynamicPPL.Metadata, DynamicPPL.VarNamedVector} true julia> values_as(vi, NamedTuple) @@ -321,7 +321,7 @@ julia> # Just use an example model to construct the `VarInfo` because we're lazy julia> vi[@varname(s)] = 1.0; vi[@varname(m)] = 2.0; julia> # For the sake of brevity, let's just check the type. - values_as(vi) isa DynamicPPL.Metadata + values_as(vi) isa Union{DynamicPPL.Metadata, Vector} true julia> values_as(vi, NamedTuple) @@ -349,7 +349,7 @@ Determine the default `eltype` of the values returned by `vi[spl]`. This should generally not be called explicitly, as it's only used in [`matchingvalue`](@ref) to determine the default type to use in place of type-parameters passed to the model. - + This method is considered legacy, and is likely to be deprecated in the future. """ function Base.eltype(vi::AbstractVarInfo, spl::Union{AbstractSampler,SampleFromPrior}) diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 13231837f..1961965ca 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -240,7 +240,10 @@ function assume( if haskey(vi, vn) # Always overwrite the parameters with new ones for `SampleFromUniform`. if sampler isa SampleFromUniform || is_flagged(vi, vn, "del") - unset_flag!(vi, vn, "del") + # TODO(mhauru) Is it important to unset the flag here? The `true` allows us + # to ignore the fact that for VarNamedVector this does nothing, but I'm unsure if + # that's okay. + unset_flag!(vi, vn, "del", true) r = init(rng, dist, sampler) f = to_maybe_linked_internal_transform(vi, vn, dist) BangBang.setindex!!(vi, f(r), vn) @@ -516,7 +519,10 @@ function get_and_set_val!( if haskey(vi, vns[1]) # Always overwrite the parameters with new ones for `SampleFromUniform`. if spl isa SampleFromUniform || is_flagged(vi, vns[1], "del") - unset_flag!(vi, vns[1], "del") + # TODO(mhauru) Is it important to unset the flag here? The `true` allows us + # to ignore the fact that for VarNamedVector this does nothing, but I'm unsure if + # that's okay. + unset_flag!(vi, vns[1], "del", true) r = init(rng, dist, spl, n) for i in 1:n vn = vns[i] @@ -554,7 +560,10 @@ function get_and_set_val!( if haskey(vi, vns[1]) # Always overwrite the parameters with new ones for `SampleFromUniform`. if spl isa SampleFromUniform || is_flagged(vi, vns[1], "del") - unset_flag!(vi, vns[1], "del") + # TODO(mhauru) Is it important to unset the flag here? The `true` allows us + # to ignore the fact that for VarNamedVector this does nothing, but I'm unsure if + # that's okay. + unset_flag!(vi, vns[1], "del", true) f = (vn, dist) -> init(rng, dist, spl) r = f.(vns, dists) for i in eachindex(vns) diff --git a/src/model.jl b/src/model.jl index 09c0c1be1..1003efaf6 100644 --- a/src/model.jl +++ b/src/model.jl @@ -1201,74 +1201,6 @@ function Distributions.loglikelihood(model::Model, chain::AbstractMCMC.AbstractC end end -""" - generated_quantities(model::Model, chain::AbstractChains) - -Execute `model` for each of the samples in `chain` and return an array of the values -returned by the `model` for each sample. - -# Examples -## General -Often you might have additional quantities computed inside the model that you want to -inspect, e.g. -```julia -@model function demo(x) - # sample and observe - θ ~ Prior() - x ~ Likelihood() - return interesting_quantity(θ, x) -end -m = demo(data) -chain = sample(m, alg, n) -# To inspect the `interesting_quantity(θ, x)` where `θ` is replaced by samples -# from the posterior/`chain`: -generated_quantities(m, chain) # <= results in a `Vector` of returned values - # from `interesting_quantity(θ, x)` -``` -## Concrete (and simple) -```julia -julia> using DynamicPPL, Turing - -julia> @model function demo(xs) - s ~ InverseGamma(2, 3) - m_shifted ~ Normal(10, √s) - m = m_shifted - 10 - - for i in eachindex(xs) - xs[i] ~ Normal(m, √s) - end - - return (m, ) - end -demo (generic function with 1 method) - -julia> model = demo(randn(10)); - -julia> chain = sample(model, MH(), 10); - -julia> generated_quantities(model, chain) -10×1 Array{Tuple{Float64},2}: - (2.1964758025119338,) - (2.1964758025119338,) - (0.09270081916291417,) - (0.09270081916291417,) - (0.09270081916291417,) - (0.09270081916291417,) - (0.09270081916291417,) - (0.043088571494005024,) - (-0.16489786710222099,) - (-0.16489786710222099,) -``` -""" -function generated_quantities(model::Model, chain::AbstractChains) - varinfo = VarInfo(model) - iters = Iterators.product(1:size(chain, 1), 1:size(chain, 3)) - return map(iters) do (sample_idx, chain_idx) - setval_and_resample!(varinfo, chain, sample_idx, chain_idx) - model(varinfo) - end -end - """ generated_quantities(model::Model, parameters::NamedTuple) generated_quantities(model::Model, values, keys) @@ -1295,7 +1227,7 @@ demo (generic function with 2 methods) julia> model = demo(randn(10)); -julia> parameters = (; s = 1.0, m_shifted=10); +julia> parameters = (; s = 1.0, m_shifted=10.0); julia> generated_quantities(model, parameters) (0.0,) @@ -1305,13 +1237,10 @@ julia> generated_quantities(model, values(parameters), keys(parameters)) ``` """ function generated_quantities(model::Model, parameters::NamedTuple) - varinfo = VarInfo(model) - setval_and_resample!(varinfo, values(parameters), keys(parameters)) - return model(varinfo) + fixed_model = fix(model, parameters) + return fixed_model() end function generated_quantities(model::Model, values, keys) - varinfo = VarInfo(model) - setval_and_resample!(varinfo, values, keys) - return model(varinfo) + return generated_quantities(model, NamedTuple{keys}(values)) end diff --git a/src/sampler.jl b/src/sampler.jl index cfc58942e..833aaf7e2 100644 --- a/src/sampler.jl +++ b/src/sampler.jl @@ -150,7 +150,7 @@ function set_values!!( flattened_param_vals = varinfo[spl] length(flattened_param_vals) == length(initial_params) || throw( DimensionMismatch( - "Provided initial value size ($(length(initial_params))) doesn't match the model size ($(length(theta)))", + "Provided initial value size ($(length(initial_params))) doesn't match the model size ($(length(flattened_param_vals)))", ), ) diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index d8afb9cec..06a151f82 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -322,15 +322,17 @@ Base.getindex(vi::SimpleVarInfo, vn::VarName) = getindex_internal(vi, vn) function Base.getindex(vi::SimpleVarInfo, vns::AbstractArray{<:VarName}) return map(Base.Fix1(getindex, vi), vns) end -# HACK: Needed to disambiguiate. +# HACK: Needed to disambiguate. Base.getindex(vi::SimpleVarInfo, vns::Vector{<:VarName}) = map(Base.Fix1(getindex, vi), vns) Base.getindex(svi::SimpleVarInfo, ::Colon) = values_as(svi, Vector) getindex_internal(vi::SimpleVarInfo, vn::VarName) = get(vi.values, vn) # `AbstractDict` -function getindex_internal(vi::SimpleVarInfo{<:AbstractDict}, vn::VarName) - return nested_getindex(vi.values, vn) +function getindex_internal( + vi::SimpleVarInfo{<:Union{AbstractDict,VarNamedVector}}, vn::VarName +) + return getvalue(vi.values, vn) end Base.haskey(vi::SimpleVarInfo, vn::VarName) = hasvalue(vi.values, vn) @@ -399,14 +401,28 @@ end function BangBang.push!!( vi::SimpleVarInfo{<:AbstractDict}, vn::VarName, - r, + value, dist::Distribution, gidset::Set{Selector}, ) - vi.values[vn] = r + vi.values[vn] = value return vi end +function BangBang.push!!( + vi::SimpleVarInfo{<:VarNamedVector}, + vn::VarName, + value, + dist::Distribution, + gidset::Set{Selector}, +) + # The semantics of push!! for SimpleVarInfo and VarNamedVector are different. For + # SimpleVarInfo, push!! allows the key to exist already, for VarNamedVector it does not. + # Hence we need to call update!! here, which has the same semantics as push!! does for + # SimpleVarInfo. + return Accessors.@set vi.values = update!!(vi.values, vn, value) +end + const SimpleOrThreadSafeSimple{T,V,C} = Union{ SimpleVarInfo{T,V,C},ThreadSafeVarInfo{<:SimpleVarInfo{T,V,C}} } @@ -456,6 +472,8 @@ function _subset(x::NamedTuple, vns) return NamedTuple{Tuple(syms)}(Tuple(map(Base.Fix1(getindex, x), syms))) end +_subset(x::VarNamedVector, vns) = subset(x, vns) + # `merge` function Base.merge(varinfo_left::SimpleVarInfo, varinfo_right::SimpleVarInfo) values = merge(varinfo_left.values, varinfo_right.values) @@ -563,6 +581,9 @@ end function values_as(vi::SimpleVarInfo{<:AbstractDict}, ::Type{NamedTuple}) return NamedTuple((Symbol(k), v) for (k, v) in vi.values) end +function values_as(vi::SimpleVarInfo, ::Type{T}) where {T} + return values_as(vi.values, T) +end """ logjoint(model::Model, θ) @@ -708,3 +729,5 @@ end function ThreadSafeVarInfo(vi::SimpleVarInfo{<:Any,<:Ref}) return ThreadSafeVarInfo(vi, [Ref(zero(getlogp(vi))) for _ in 1:Threads.nthreads()]) end + +has_varnamedvector(vi::SimpleVarInfo) = vi.values isa VarNamedVector diff --git a/src/test_utils.jl b/src/test_utils.jl index 6f7481c40..9a606b4ef 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -37,20 +37,35 @@ function setup_varinfos( model::Model, example_values::NamedTuple, varnames; include_threadsafe::Bool=false ) # VarInfo - vi_untyped = VarInfo() - model(vi_untyped) - vi_typed = DynamicPPL.TypedVarInfo(vi_untyped) + vi_untyped_metadata = VarInfo(DynamicPPL.Metadata()) + vi_untyped_vnv = VarInfo(DynamicPPL.VarNamedVector()) + model(vi_untyped_metadata) + model(vi_untyped_vnv) + vi_typed_metadata = DynamicPPL.TypedVarInfo(vi_untyped_metadata) + vi_typed_vnv = DynamicPPL.TypedVarInfo(vi_untyped_vnv) + # SimpleVarInfo svi_typed = SimpleVarInfo(example_values) svi_untyped = SimpleVarInfo(OrderedDict()) + svi_vnv = SimpleVarInfo(VarNamedVector()) # SimpleVarInfo{<:Any,<:Ref} svi_typed_ref = SimpleVarInfo(example_values, Ref(getlogp(svi_typed))) svi_untyped_ref = SimpleVarInfo(OrderedDict(), Ref(getlogp(svi_untyped))) + svi_vnv_ref = SimpleVarInfo(VarNamedVector(), Ref(getlogp(svi_vnv))) - lp = getlogp(vi_typed) + lp = getlogp(vi_typed_metadata) varinfos = map(( - vi_untyped, vi_typed, svi_typed, svi_untyped, svi_typed_ref, svi_untyped_ref + vi_untyped_metadata, + vi_untyped_vnv, + vi_typed_metadata, + vi_typed_vnv, + svi_typed, + svi_untyped, + svi_vnv, + svi_typed_ref, + svi_untyped_ref, + svi_vnv_ref, )) do vi # Set them all to the same values. DynamicPPL.setlogp!!(update_values!!(vi, example_values, varnames), lp) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index f3bc84935..196f243bf 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -55,7 +55,7 @@ function setlogp!!(vi::ThreadSafeVarInfoWithRef, logp) return ThreadSafeVarInfo(setlogp!!(vi.varinfo, logp), vi.logps) end -has_varnamevector(vi::DynamicPPL.ThreadSafeVarInfo) = has_varnamevector(vi.varinfo) +has_varnamedvector(vi::DynamicPPL.ThreadSafeVarInfo) = has_varnamedvector(vi.varinfo) function BangBang.push!!( vi::ThreadSafeVarInfo, vn::VarName, r, dist::Distribution, gidset::Set{Selector} diff --git a/src/utils.jl b/src/utils.jl index 9ddeb6247..75bcef327 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -48,7 +48,7 @@ true i.e., regardless of whether you evaluate the log prior, the log likelihood or the log joint density. If you would like to avoid this behaviour you should check the evaluation context. It can be accessed with the internal variable `__context__`. - For instance, in the following example the log density is not accumulated when only the log prior is computed: + For instance, in the following example the log density is not accumulated when only the log prior is computed: ```jldoctest; setup = :(using Distributions) julia> myloglikelihood(x, μ) = loglikelihood(Normal(μ, 1), x); @@ -225,21 +225,30 @@ invlink_transform(dist) = inverse(link_transform(dist)) # Helper functions for vectorize/reconstruct values # ##################################################### -# Useful transformation going from the flattened representation. -struct FromVec{Size} <: Bijectors.Bijector +""" + ReshapeTransform(size::Size) + +A `Bijector` that transforms an `AbstractVector` to a realization of size `size`. As a +special case, if `size` is an empty tuple it transforms a singleton vector into a scalar. + +This transformation can be inverted by calling `tovec`. +""" +struct ReshapeTransform{Size} <: Bijectors.Bijector size::Size end -FromVec(x::Union{Real,AbstractArray}) = FromVec(size(x)) +ReshapeTransform(x::Union{Real,AbstractArray}) = ReshapeTransform(size(x)) # TODO: Should we materialize the `reshape`? -(f::FromVec)(x) = reshape(x, f.size) -(f::FromVec{Tuple{}})(x) = only(x) +(f::ReshapeTransform)(x::AbstractVector) = reshape(x, f.size) +(f::ReshapeTransform{Tuple{}})(x::AbstractVector) = only(x) # TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. -Bijectors.with_logabsdet_jacobian(f::FromVec, x) = (f(x), 0) -# We want to use the inverse of `FromVec` so it preserves the size information. -Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:FromVec}, x) = (tovec(x), 0) +Bijectors.with_logabsdet_jacobian(f::ReshapeTransform, x) = (f(x), 0) +# We want to use the inverse of `ReshapeTransform` so it preserves the size information. +function Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ReshapeTransform}, x) + return (tovec(x), 0) +end struct ToChol <: Bijectors.Bijector uplo::Char @@ -254,15 +263,16 @@ Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = Return the transformation from the vector representation of `x` to original representation. """ from_vec_transform(x::Union{Real,AbstractArray}) = from_vec_transform_for_size(size(x)) -from_vec_transform(C::Cholesky) = ToChol(C.uplo) ∘ FromVec(size(C.UL)) +from_vec_transform(C::Cholesky) = ToChol(C.uplo) ∘ ReshapeTransform(size(C.UL)) """ from_vec_transform_for_size(sz::Tuple) -Return the transformation from the vector representation of a realization of size `sz` to original representation. +Return the transformation from the vector representation of a realization of size `sz` to +original representation. """ -from_vec_transform_for_size(sz::Tuple) = FromVec(sz) -from_vec_transform_for_size(::Tuple{()}) = FromVec(()) +from_vec_transform_for_size(sz::Tuple) = ReshapeTransform(sz) +from_vec_transform_for_size(::Tuple{()}) = ReshapeTransform(()) from_vec_transform_for_size(::Tuple{<:Any}) = identity """ @@ -272,7 +282,7 @@ Return the transformation from the vector representation of a realization from distribution `dist` to the original representation compatible with `dist`. """ from_vec_transform(dist::Distribution) = from_vec_transform_for_size(size(dist)) -from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ FromVec(size(dist)) +from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ ReshapeTransform(size(dist)) """ from_vec_transform(f, size::Tuple) @@ -854,6 +864,7 @@ end Return type corresponding to `float(typeof(x))` if possible; otherwise return `Real`. """ float_type_with_fallback(::Type) = Real +float_type_with_fallback(::Type{Union{}}) = Real float_type_with_fallback(::Type{T}) where {T<:Real} = float(T) """ diff --git a/src/values_as_in_model.jl b/src/values_as_in_model.jl index 52ba6eb61..c5003d53a 100644 --- a/src/values_as_in_model.jl +++ b/src/values_as_in_model.jl @@ -177,7 +177,7 @@ julia> # Approach 1: Convert back to constrained space using `invlink` and extra julia> # (×) Fails! Because `VarInfo` _saves_ the original distributions # used in the very first model evaluation, hence the support of `y` # is not updated even though `x` has changed. - lb ≤ varinfo_invlinked[@varname(y)] ≤ ub + lb ≤ first(varinfo_invlinked[@varname(y)]) ≤ ub false julia> # Approach 2: Extract realizations using `values_as_in_model`. diff --git a/src/varinfo.jl b/src/varinfo.jl index 19c178a03..a6a5c0400 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -101,7 +101,7 @@ struct VarInfo{Tmeta,Tlogp} <: AbstractVarInfo logp::Base.RefValue{Tlogp} num_produce::Base.RefValue{Int} end -const VectorVarInfo = VarInfo{<:VarNameVector} +const VectorVarInfo = VarInfo{<:VarNamedVector} const UntypedVarInfo = VarInfo{<:Metadata} const TypedVarInfo = VarInfo{<:NamedTuple} const VarInfoOrThreadSafeVarInfo{Tmeta} = Union{ @@ -120,9 +120,9 @@ function VarInfo(old_vi::VarInfo, spl, x::AbstractVector) ) end -# No-op if we're already working with a `VarNameVector`. -metadata_to_varnamevector(vnv::VarNameVector) = vnv -function metadata_to_varnamevector(md::Metadata) +# No-op if we're already working with a `VarNamedVector`. +metadata_to_varnamedvector(vnv::VarNamedVector) = vnv +function metadata_to_varnamedvector(md::Metadata) idcs = copy(md.idcs) vns = copy(md.vns) ranges = copy(md.ranges) @@ -132,32 +132,32 @@ function metadata_to_varnamevector(md::Metadata) from_vec_transform(dist) end - return VarNameVector( + return VarNamedVector( OrderedDict{eltype(keys(idcs)),Int}(idcs), vns, ranges, vals, transforms ) end function VectorVarInfo(vi::UntypedVarInfo) - md = metadata_to_varnamevector(vi.metadata) + md = metadata_to_varnamedvector(vi.metadata) lp = getlogp(vi) return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end function VectorVarInfo(vi::TypedVarInfo) - md = map(metadata_to_varnamevector, vi.metadata) + md = map(metadata_to_varnamedvector, vi.metadata) lp = getlogp(vi) return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end """ - has_varnamevector(varinfo::VarInfo) + has_varnamedvector(varinfo::VarInfo) -Returns `true` if `varinfo` uses `VarNameVector` as metadata. +Returns `true` if `varinfo` uses `VarNamedVector` as metadata. """ -has_varnamevector(vi) = false -function has_varnamevector(vi::VarInfo) - return vi.metadata isa VarNameVector || - (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNameVector), values(vi.metadata))) +has_varnamedvector(vi::AbstractVarInfo) = false +function has_varnamedvector(vi::VarInfo) + return vi.metadata isa VarNamedVector || + (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNamedVector), values(vi.metadata))) end """ @@ -170,8 +170,9 @@ function untyped_varinfo( model::Model, sampler::AbstractSampler=SampleFromPrior(), context::AbstractContext=DefaultContext(), + metadata_type::Type=VarNamedVector, ) - varinfo = VarInfo() + varinfo = VarInfo(metadata_type()) return last(evaluate!!(model, varinfo, SamplingContext(rng, sampler, context))) end function untyped_varinfo(model::Model, args::Union{AbstractSampler,AbstractContext}...) @@ -190,8 +191,9 @@ function VarInfo( model::Model, sampler::AbstractSampler=SampleFromPrior(), context::AbstractContext=DefaultContext(), + metadata_type::Type=VarNamedVector, ) - return typed_varinfo(rng, model, sampler, context) + return typed_varinfo(rng, model, sampler, context, metadata_type) end VarInfo(model::Model, args...) = VarInfo(Random.default_rng(), model, args...) @@ -300,6 +302,11 @@ function subset(varinfo::UntypedVarInfo, vns::AbstractVector{<:VarName}) return VarInfo(metadata, varinfo.logp, varinfo.num_produce) end +function subset(varinfo::VectorVarInfo, vns::AbstractVector{<:VarName}) + metadata = subset(varinfo.metadata, vns) + return VarInfo(metadata, varinfo.logp, varinfo.num_produce) +end + function subset(varinfo::TypedVarInfo, vns::AbstractVector{<:VarName{sym}}) where {sym} # If all the variables are using the same symbol, then we can just extract that field from the metadata. metadata = subset(getfield(varinfo.metadata, sym), vns) @@ -379,7 +386,7 @@ function _merge(varinfo_left::VarInfo, varinfo_right::VarInfo) ) end -function merge_metadata(vnv_left::VarNameVector, vnv_right::VarNameVector) +function merge_metadata(vnv_left::VarNamedVector, vnv_right::VarNamedVector) return merge(vnv_left, vnv_right) end @@ -518,8 +525,6 @@ end const VarView = Union{Int,UnitRange,Vector{Int}} -getindex_internal(vi::UntypedVarInfo, vview::VarView) = view(vi.metadata.vals, vview) - """ setval!(vi::UntypedVarInfo, val, vview::Union{Int, UnitRange, Vector{Int}}) @@ -576,12 +581,20 @@ Return the distribution from which `vn` was sampled in `vi`. getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] # HACK: we shouldn't need this -getdist(::VarNameVector, ::VarName) = nothing +function getdist(::VarNamedVector, ::VarName) + throw(ErrorException("getdist does not exist for VarNamedVector")) +end getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, vn), vn) -getindex_internal(md::Metadata, vn::VarName) = view(md.vals, getrange(md, vn)) +# TODO(torfjelde): Use `view` instead of `getindex`. Requires addressing type-stability issues though, +# since then we might be returning a `SubArray` rather than an `Array`, which is typically +# what a bijector would result in, even if the input is a view (`SubArray`). +# TODO(torfjelde): An alternative is to implement `view` directly instead. +getindex_internal(md::Metadata, vn::VarName) = getindex(md.vals, getrange(md, vn)) # HACK: We shouldn't need this -getindex_internal(vnv::VarNameVector, vn::VarName) = view(vnv.vals, getrange(vnv, vn)) +# TODO(mhauru) This seems to return an array always for VarNamedVector, but a scalar for +# Metadata. What's the right thing to do here? +getindex_internal(vnv::VarNamedVector, vn::VarName) = getindex(vnv.vals, getrange(vnv, vn)) function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) return mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) @@ -601,9 +614,6 @@ end function setval!(md::Metadata, val, vn::VarName) return md.vals[getrange(md, vn)] = tovec(val) end -function setval!(vnv::VarNameVector, val, vn::VarName) - return setindex_raw!(vnv, tovec(val), vn) -end """ getall(vi::VarInfo) @@ -621,7 +631,7 @@ function getall(md::Metadata) Base.Fix1(getindex_internal, md), vcat, md.vns; init=similar(md.vals, 0) ) end -getall(vnv::VarNameVector) = vnv.vals +getall(vnv::VarNamedVector) = vnv.vals """ setall!(vi::VarInfo, val) @@ -637,7 +647,7 @@ function _setall!(metadata::Metadata, val) metadata.vals[r] .= val[r] end end -function _setall!(vnv::VarNameVector, val) +function _setall!(vnv::VarNamedVector, val) # TODO: Do something more efficient here. for i in 1:length(vnv) vnv[i] = val[i] @@ -675,7 +685,10 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) return metadata end -function settrans!!(vnv::VarNameVector, trans::Bool, vn::VarName) + +# TODO(mhauru) Isn't this infinite recursion? Shouldn't rather change the `transforms` +# field? +function settrans!!(vnv::VarNamedVector, trans::Bool, vn::VarName) settrans!(vnv, trans, vn) return vnv end @@ -759,7 +772,7 @@ end length(exprs) == 0 && return :(NamedTuple()) return :($(exprs...),) end -@inline function findinds(f_meta, s, ::Val{space}) where {space} +@inline function findinds(f_meta::Metadata, s, ::Val{space}) where {space} # Get all the idcs of the vns in `space` and that belong to the selector `s` return filter( (i) -> @@ -768,11 +781,27 @@ end 1:length(f_meta.gids), ) end -@inline function findinds(f_meta) +@inline function findinds(f_meta::Metadata) # Get all the idcs of the vns return filter((i) -> isempty(f_meta.gids[i]), 1:length(f_meta.gids)) end +function findinds(vnv::VarNamedVector, ::Selector, ::Val{space}) where {space} + # New Metadata objects are created with an empty list of gids, which is intrepreted as + # all Selectors applying to all variables. We assume the same behavior for + # VarNamedVector, and thus ignore the Selector argument. + if space !== () + msg = "VarNamedVector does not support selecting variables based on samplers" + throw(ErrorException(msg)) + else + return findinds(vnv) + end +end + +function findinds(vnv::VarNamedVector) + return 1:length(vnv.varnames) +end + # Get all vns of variables belonging to spl _getvns(vi::VarInfo, spl::Sampler) = _getvns(vi, spl.selector, Val(getspace(spl))) function _getvns(vi::VarInfo, spl::Union{SampleFromPrior,SampleFromUniform}) @@ -788,7 +817,7 @@ end @generated function _getvns(metadata, idcs::NamedTuple{names}) where {names} exprs = [] for f in names - push!(exprs, :($f = metadata.$f.vns[idcs.$f])) + push!(exprs, :($f = Base.keys(metadata.$f)[idcs.$f])) end length(exprs) == 0 && return :(NamedTuple()) return :($(exprs...),) @@ -848,13 +877,30 @@ function set_flag!(md::Metadata, vn::VarName, flag::String) return md.flags[flag][getidx(md, vn)] = true end +function set_flag!(vnv::VarNamedVector, ::VarName, flag::String) + if flag == "del" + # The "del" flag is effectively always set for a VarNamedVector, so this is a no-op. + else + throw(ErrorException("Flag $flag not valid for VarNamedVector")) + end + return vnv +end + #### #### APIs for typed and untyped VarInfo #### # VarInfo -VarInfo(meta=Metadata()) = VarInfo(meta, Ref{Float64}(0.0), Ref(0)) +VarInfo(meta=VarNamedVector()) = VarInfo(meta, Ref{Float64}(0.0), Ref(0)) + +function TypedVarInfo(vi::VectorVarInfo) + new_metas = group_by_symbol(vi.metadata) + logp = getlogp(vi) + num_produce = get_num_produce(vi) + nt = NamedTuple(new_metas) + return VarInfo(nt, Ref(logp), Ref(num_produce)) +end """ TypedVarInfo(vi::UntypedVarInfo) @@ -966,8 +1012,14 @@ end Add `gid` to the set of sampler selectors associated with `vn` in `vi`. """ -function setgid!(vi::VarInfo, gid::Selector, vn::VarName) - return push!(getmetadata(vi, vn).gids[getidx(vi, vn)], gid) +setgid!(vi::VarInfo, gid::Selector, vn::VarName) = setgid!(getmetadata(vi, vn), gid, vn) + +function setgid!(m::Metadata, gid::Selector, vn::VarName) + return push!(m.gids[getidx(m, vn)], gid) +end + +function setgid!(vnv::VarNamedVector, gid::Selector, vn::VarName) + throw(ErrorException("Calling setgid! on a VarNamedVector isn't valid.")) end istrans(vi::VarInfo, vn::VarName) = istrans(getmetadata(vi, vn), vn) @@ -1014,20 +1066,18 @@ and parameters sampled in `vi` to 0. """ reset_num_produce!(vi::VarInfo) = set_num_produce!(vi, 0) -isempty(vi::UntypedVarInfo) = isempty(vi.metadata.idcs) -isempty(vi::TypedVarInfo) = _isempty(vi.metadata) +# Need to introduce the _isempty to avoid type piracy of isempty(::NamedTuple). +isempty(vi::VarInfo) = _isempty(vi.metadata) +_isempty(metadata::Metadata) = isempty(metadata.idcs) +_isempty(vnv::VarNamedVector) = isempty(vnv) @generated function _isempty(metadata::NamedTuple{names}) where {names} - expr = Expr(:&&, :true) - for f in names - push!(expr.args, :(isempty(metadata.$f.idcs))) - end - return expr + return Expr(:&&, (:(isempty(metadata.$f)) for f in names)...) end # X -> R for all variables associated with given sampler function link!!(t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model) - # If we're working with a `VarNameVector`, we always use immutable. - has_varnamevector(vi) && return link(t, vi, spl, model) + # If we're working with a `VarNamedVector`, we always use immutable. + has_varnamedvector(vi) && return link(t, vi, spl, model) # Call `_link!` instead of `link!` to avoid deprecation warning. _link!(vi, spl) return vi @@ -1070,10 +1120,8 @@ function _link!(vi::UntypedVarInfo, spl::AbstractSampler) vns = _getvns(vi, spl) if ~istrans(vi, vns[1]) for vn in vns - dist = getdist(vi, vn) - _inner_transform!( - vi, vn, dist, internal_to_linked_internal_transform(vi, vn, dist) - ) + f = internal_to_linked_internal_transform(vi, vn) + _inner_transform!(vi, vn, f) settrans!!(vi, true, vn) end else @@ -1100,13 +1148,8 @@ end if ~istrans(vi, f_vns[1]) # Iterate over all `f_vns` and transform for vn in f_vns - dist = getdist(vi, vn) - _inner_transform!( - vi, - vn, - dist, - internal_to_linked_internal_transform(vi, vn, dist), - ) + f = internal_to_linked_internal_transform(vi, vn) + _inner_transform!(vi, vn, f) settrans!!(vi, true, vn) end else @@ -1123,8 +1166,8 @@ end function invlink!!( t::DynamicTransformation, vi::VarInfo, spl::AbstractSampler, model::Model ) - # If we're working with a `VarNameVector`, we always use immutable. - has_varnamevector(vi) && return invlink(t, vi, spl, model) + # If we're working with a `VarNamedVector`, we always use immutable. + has_varnamedvector(vi) && return invlink(t, vi, spl, model) # Call `_invlink!` instead of `invlink!` to avoid deprecation warning. _invlink!(vi, spl) return vi @@ -1176,10 +1219,8 @@ function _invlink!(vi::UntypedVarInfo, spl::AbstractSampler) vns = _getvns(vi, spl) if istrans(vi, vns[1]) for vn in vns - dist = getdist(vi, vn) - _inner_transform!( - vi, vn, dist, linked_internal_to_internal_transform(vi, vn, dist) - ) + f = linked_internal_to_internal_transform(vi, vn) + _inner_transform!(vi, vn, f) settrans!!(vi, false, vn) end else @@ -1206,13 +1247,8 @@ end if istrans(vi, f_vns[1]) # Iterate over all `f_vns` and transform for vn in f_vns - dist = getdist(vi, vn) - _inner_transform!( - vi, - vn, - dist, - linked_internal_to_internal_transform(vi, vn, dist), - ) + f = linked_internal_to_internal_transform(vi, vn) + _inner_transform!(vi, vn, f) settrans!!(vi, false, vn) end else @@ -1225,11 +1261,11 @@ end return expr end -function _inner_transform!(vi::VarInfo, vn::VarName, dist, f) - return _inner_transform!(getmetadata(vi, vn), vi, vn, dist, f) +function _inner_transform!(vi::VarInfo, vn::VarName, f) + return _inner_transform!(getmetadata(vi, vn), vi, vn, f) end -function _inner_transform!(md::Metadata, vi::VarInfo, vn::VarName, dist, f) +function _inner_transform!(md::Metadata, vi::VarInfo, vn::VarName, f) # TODO: Use inplace versions to avoid allocations yvec, logjac = with_logabsdet_jacobian(f, getindex_internal(vi, vn)) # Determine the new range. @@ -1267,7 +1303,9 @@ function link( return Accessors.@set varinfo.varinfo = link(varinfo.varinfo, spl, model) end -function _link(model::Model, varinfo::UntypedVarInfo, spl::AbstractSampler) +function _link( + model::Model, varinfo::Union{UntypedVarInfo,VectorVarInfo}, spl::AbstractSampler +) varinfo = deepcopy(varinfo) return VarInfo( _link_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), @@ -1322,7 +1360,7 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar yvec = tovec(y) # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) - # Mark as no longer transformed. + # Mark as transformed. settrans!!(varinfo, true, vn) # Return the vectorized transformed value. return yvec @@ -1351,14 +1389,14 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar end function _link_metadata!( - model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns + model::Model, varinfo::VarInfo, metadata::VarNamedVector, target_vns ) # HACK: We ignore `target_vns` here. vns = keys(metadata) # Need to extract the priors from the model. dists = extract_priors(model, varinfo) - is_transformed = copy(metadata.is_transformed) + is_unconstrained = copy(metadata.is_unconstrained) # Construct the linking transformations. link_transforms = map(vns) do vn @@ -1368,13 +1406,12 @@ function _link_metadata!( end # Otherwise, we derive the transformation from the distribution. - is_transformed[getidx(metadata, vn)] = true + # TODO(mhauru) Could move the mutation outside of the map, just for style. + is_unconstrained[getidx(metadata, vn)] = true internal_to_linked_internal_transform(varinfo, vn, dists[vn]) end # Compute the transformed values. ys = map(vns, link_transforms) do vn, f - # TODO: Do we need to handle scenarios where `vn` is not in `dists`? - dist = dists[vn] x = getindex_internal(metadata, vn) y, logjac = with_logabsdet_jacobian(f, x) # Accumulate the log-abs-det jacobian correction. @@ -1400,13 +1437,13 @@ function _link_metadata!( end # Now we just create a new metadata with the new `vals` and `ranges`. - return VarNameVector( + return VarNamedVector( metadata.varname_to_index, metadata.varnames, ranges_new, reduce(vcat, yvecs), transforms, - is_transformed, + is_unconstrained, ) end @@ -1467,7 +1504,7 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe # Construct the new transformed values, and keep track of their lengths. vals_new = map(vns) do vn # Return early if we're already in constrained space OR if we're not - # supposed to touch this `vn`, e.g. when `vn` does not belong to the current sampler. + # supposed to touch this `vn`, e.g. when `vn` does not belong to the current sampler. # HACK: if `target_vns` is `nothing`, we ignore the `target_vns` check. if !istrans(varinfo, vn) || (target_vns !== nothing && vn ∉ target_vns) return metadata.vals[getrange(metadata, vn)] @@ -1511,14 +1548,14 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe end function _invlink_metadata!( - model::Model, varinfo::VarInfo, metadata::VarNameVector, target_vns + model::Model, varinfo::VarInfo, metadata::VarNamedVector, target_vns ) # HACK: We ignore `target_vns` here. # TODO: Make use of `update!` to aovid copying values. # => Only need to allocate for transformations. vns = keys(metadata) - is_transformed = copy(metadata.is_transformed) + is_unconstrained = copy(metadata.is_unconstrained) # Compute the transformed values. xs = map(vns) do vn @@ -1529,7 +1566,7 @@ function _invlink_metadata!( # Accumulate the log-abs-det jacobian correction. acclogp!!(varinfo, -logjac) # Mark as no longer transformed. - is_transformed[getidx(metadata, vn)] = false + is_unconstrained[getidx(metadata, vn)] = false # Return the transformed value. return x end @@ -1549,13 +1586,13 @@ function _invlink_metadata!( end # Now we just create a new metadata with the new `vals` and `ranges`. - return VarNameVector( + return VarNamedVector( metadata.varname_to_index, metadata.varnames, ranges_new, reduce(vcat, xvecs), transforms, - is_transformed, + is_unconstrained, ) end @@ -1564,9 +1601,9 @@ end Check whether `vi` is in the transformed space for a particular sampler `spl`. -Turing's Hamiltonian samplers use the `link` and `invlink` functions from +Turing's Hamiltonian samplers use the `link` and `invlink` functions from [Bijectors.jl](https://github.com/TuringLang/Bijectors.jl) to map a constrained variable -(for example, one bounded to the space `[0, 1]`) from its constrained space to the set of +(for example, one bounded to the space `[0, 1]`) from its constrained space to the set of real numbers. `islinked` checks if the number is in the constrained space or the real space. """ function islinked(vi::UntypedVarInfo, spl::Union{Sampler,SampleFromPrior}) @@ -1597,9 +1634,11 @@ function nested_setindex_maybe!( nothing end end -function _nested_setindex_maybe!(vi::VarInfo, md::Metadata, val, vn::VarName) +function _nested_setindex_maybe!( + vi::VarInfo, md::Union{Metadata,VarNamedVector}, val, vn::VarName +) # If `vn` is in `vns`, then we can just use the standard `setindex!`. - vns = md.vns + vns = Base.keys(md) if vn in vns setindex!(vi, val, vn) return vn @@ -1610,8 +1649,7 @@ function _nested_setindex_maybe!(vi::VarInfo, md::Metadata, val, vn::VarName) i === nothing && return nothing vn_parent = vns[i] - dist = getdist(md, vn_parent) - val_parent = getindex(vi, vn_parent, dist) # TODO: Ensure that we're working with a view here. + val_parent = getindex(vi, vn_parent) # TODO: Ensure that we're working with a view here. # Split the varname into its tail optic. optic = remove_parent_optic(vn_parent, vn) # Update the value for the parent. @@ -1622,30 +1660,47 @@ end # The default getindex & setindex!() for get & set values # NOTE: vi[vn] will always transform the variable to its original space and Julia type -getindex(vi::VarInfo, vn::VarName) = getindex(vi, vn, getdist(vi, vn)) +function getindex(vi::VarInfo, vn::VarName) + return from_maybe_linked_internal_transform(vi, vn)(getindex_internal(vi, vn)) +end + function getindex(vi::VarInfo, vn::VarName, dist::Distribution) @assert haskey(vi, vn) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" val = getindex_internal(vi, vn) return from_maybe_linked_internal(vi, vn, dist, val) end -# HACK: Allows us to also work with `VarNameVector` where `dist` is not used, -# but we instead use a transformation stored with the variable. -function getindex(vi::VarInfo, vn::VarName, ::Nothing) - if !haskey(vi, vn) - throw(KeyError(vn)) - end - return getmetadata(vi, vn)[vn] -end function getindex(vi::VarInfo, vns::Vector{<:VarName}) - vals_linked = mapreduce(vcat, vns) do vn - getindex(vi, vn) + vals = map(vn -> getindex(vi, vn), vns) + + et = eltype(vals) + # This will catch type unstable cases, where vals has mixed types. + if !isconcretetype(et) + throw(ArgumentError("All variables must have the same type.")) + end + + if et <: Vector + all_of_equal_dimension = all(x -> length(x) == length(vals[1]), vals) + if !all_of_equal_dimension + throw(ArgumentError("All variables must have the same dimension.")) + end + end + + # TODO(mhauru) I'm not very pleased with the return type varying like this, even though + # this should be type stable. + vec_vals = reduce(vcat, vals) + if et <: Vector + # The individual variables are multivariate, and thus we return the values as a + # matrix. + return reshape(vec_vals, (:, length(vns))) + else + # The individual variables are univariate, and thus we return a vector of scalars. + return vec_vals end - # HACK: I don't like this. - dist = getdist(vi, vns[1]) - return recombine(dist, vals_linked, length(vns)) end + function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) + # TODO(mhauru) Does this ever get called? @assert haskey(vi, vns[1]) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) @@ -1828,11 +1883,6 @@ function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) return meta end -function Base.push!(vnv::VarNameVector, vn, r, dist, gidset, num_produce) - f = from_vec_transform(dist) - return push!(vnv, vn, r, f) -end - """ setorder!(vi::VarInfo, vn::VarName, index::Int) @@ -1847,7 +1897,7 @@ function setorder!(metadata::Metadata, vn::VarName, index::Int) metadata.orders[metadata.idcs[vn]] = index return metadata end -setorder!(vnv::VarNameVector, ::VarName, ::Int) = vnv +setorder!(vnv::VarNamedVector, ::VarName, ::Int) = vnv """ getorder(vi::VarInfo, vn::VarName) @@ -1873,23 +1923,44 @@ end function is_flagged(metadata::Metadata, vn::VarName, flag::String) return metadata.flags[flag][getidx(metadata, vn)] end -# HACK: This is bad. Should we always return `true` here? -is_flagged(::VarNameVector, ::VarName, flag::String) = flag == "del" ? true : false +function is_flagged(::VarNamedVector, ::VarName, flag::String) + if flag == "del" + return true + else + throw(ErrorException("Flag $flag not valid for VarNamedVector")) + end +end +# TODO(mhauru) The "ignorable" argument is a temporary hack while developing VarNamedVector, +# but still having to support the interface based on Metadata too """ - unset_flag!(vi::VarInfo, vn::VarName, flag::String) + unset_flag!(vi::VarInfo, vn::VarName, flag::String, ignorable::Bool=false Set `vn`'s value for `flag` to `false` in `vi`. + +If `ignorable` is `false`, as it is by default, then this will error if setting the flag is +not possible. """ -function unset_flag!(vi::VarInfo, vn::VarName, flag::String) - unset_flag!(getmetadata(vi, vn), vn, flag) +function unset_flag!(vi::VarInfo, vn::VarName, flag::String, ignorable::Bool=false) + unset_flag!(getmetadata(vi, vn), vn, flag, ignorable) return vi end -function unset_flag!(metadata::Metadata, vn::VarName, flag::String) +function unset_flag!(metadata::Metadata, vn::VarName, flag::String, ignorable::Bool=false) metadata.flags[flag][getidx(metadata, vn)] = false return metadata end -unset_flag!(vnv::VarNameVector, ::VarName, ::String) = vnv + +function unset_flag!(vnv::VarNamedVector, ::VarName, flag::String, ignorable::Bool=false) + if ignorable + return vnv + end + if flag == "del" + throw(ErrorException("The \"del\" flag cannot be unset for VarNamedVector")) + else + throw(ErrorException("Flag $flag not valid for VarNamedVector")) + end + return vnv +end """ set_retained_vns_del_by_spl!(vi::VarInfo, spl::Sampler) @@ -1995,7 +2066,7 @@ end ) where {names} updates = map(names) do n quote - for vn in metadata.$n.vns + for vn in Base.keys(metadata.$n) indices_found = kernel!(vi, vn, values, keys_strings) if indices_found !== nothing num_indices_seen += length(indices_found) @@ -2077,14 +2148,6 @@ julia> DynamicPPL.setval!(var_info, (m = 100.0, )); # set `m` and and keep `x[1] julia> var_info[@varname(m)] # [✓] changed 100.0 -julia> var_info[@varname(x[1])] # [✓] unchanged --0.22312984965118443 - -julia> m(rng, var_info); # rerun model - -julia> var_info[@varname(m)] # [✓] unchanged -100.0 - julia> var_info[@varname(x[1])] # [✓] unchanged -0.22312984965118443 ``` @@ -2136,7 +2199,7 @@ julia> rng = StableRNG(42); julia> m = demo([missing]); -julia> var_info = DynamicPPL.VarInfo(rng, m); +julia> var_info = DynamicPPL.VarInfo(rng, m, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata); julia> var_info[@varname(m)] -0.6702516921145671 @@ -2234,6 +2297,9 @@ function values_as( return ConstructionBase.constructorof(D)(iter) end +values_as(vi::VectorVarInfo, args...) = values_as(vi.metadata, args...) +values_as(vi::VectorVarInfo, T::Type{Vector}) = values_as(vi.metadata, T) + function values_from_metadata(md::Metadata) return ( # `copy` to avoid accidentally mutation of internal representation. @@ -2243,7 +2309,7 @@ function values_from_metadata(md::Metadata) ) end -values_from_metadata(md::VarNameVector) = pairs(md) +values_from_metadata(md::VarNamedVector) = pairs(md) # Transforming from internal representation to distribution representation. # Without `dist` argument: base on `dist` extracted from self. @@ -2253,7 +2319,7 @@ end function from_internal_transform(md::Metadata, vn::VarName) return from_internal_transform(md, vn, getdist(md, vn)) end -function from_internal_transform(md::VarNameVector, vn::VarName) +function from_internal_transform(md::VarNamedVector, vn::VarName) return gettransform(md, vn) end # With both `vn` and `dist` arguments: base on provided `dist`. @@ -2261,7 +2327,7 @@ function from_internal_transform(vi::VarInfo, vn::VarName, dist) return from_internal_transform(getmetadata(vi, vn), vn, dist) end from_internal_transform(::Metadata, ::VarName, dist) = from_vec_transform(dist) -function from_internal_transform(::VarNameVector, ::VarName, dist) +function from_internal_transform(::VarNamedVector, ::VarName, dist) return from_vec_transform(dist) end @@ -2272,7 +2338,7 @@ end function from_linked_internal_transform(md::Metadata, vn::VarName) return from_linked_internal_transform(md, vn, getdist(md, vn)) end -function from_linked_internal_transform(md::VarNameVector, vn::VarName) +function from_linked_internal_transform(md::VarNamedVector, vn::VarName) return gettransform(md, vn) end # With both `vn` and `dist` arguments: base on provided `dist`. @@ -2283,6 +2349,6 @@ end function from_linked_internal_transform(::Metadata, ::VarName, dist) return from_linked_vec_transform(dist) end -function from_linked_internal_transform(::VarNameVector, ::VarName, dist) +function from_linked_internal_transform(::VarNamedVector, ::VarName, dist) return from_linked_vec_transform(dist) end diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl new file mode 100644 index 000000000..e9a7126ae --- /dev/null +++ b/src/varnamedvector.jl @@ -0,0 +1,1217 @@ +""" + VarNamedVector + +A container that stores values in a vectorised form, but indexable by variable names. + +When indexed by integers or `Colon`s, e.g. `vnv[2]` or `vnv[:]`, `VarNamedVector` behaves +like a `Vector`, and returns the values as they are stored. The stored form is always +vectorised, for instance matrix variables have been flattened, and may be further +transformed to achieve linking. + +When indexed by `VarName`s, e.g. `vnv[@varname(x)]`, `VarNamedVector` returns the values +in the original space. For instance, a linked matrix variable is first inverse linked and +then reshaped to its original form before returning it to the caller. + +`VarNamedVector` also stores a boolean for whether a variable has been transformed to +unconstrained Euclidean space or not. + +# Fields +$(FIELDS) +""" +struct VarNamedVector{ + K<:VarName,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector +} + """ + mapping from a `VarName` to its integer index in `varnames`, `ranges` and `transforms` + """ + varname_to_index::OrderedDict{K,Int} + + """ + vector of `VarNames` for the variables, where `varnames[varname_to_index[vn]] == vn` + """ + varnames::TVN # AbstractVector{<:VarName} + + """ + vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has + a single index or a set of contiguous indices, such that the values of `vn` can be found + at `vals[ranges[varname_to_index[vn]]]` + """ + ranges::Vector{UnitRange{Int}} + + """ + vector of values of all variables; the value(s) of `vn` is/are + `vals[ranges[varname_to_index[vn]]]` + """ + vals::TVal # AbstractVector{<:Real} + + """ + vector of transformations, so that `transforms[varname_to_index[vn]]` is a callable + that transformes the value of `vn` back to its original space, undoing any linking and + vectorisation + """ + transforms::TTrans + + """ + vector of booleans indicating whether a variable has been transformed to unconstrained + Euclidean space or not, i.e. whether its domain is all of `ℝ^ⁿ`. Having + `is_unconstrained[varname_to_index[vn]] == false` does not necessarily mean that a + variable is constrained, but rather that it's not guaranteed to not be. + """ + is_unconstrained::BitVector + + """ + mapping from a variable index to the number of inactive entries for that variable. + Inactive entries are elements in `vals` that are not part of the value of any variable. + They arise when transformations change the dimension of the value stored. In active + entries always come after the last active entry for the given variable. + """ + num_inactive::OrderedDict{Int,Int} + + function VarNamedVector( + varname_to_index, + varnames::TVN, + ranges, + vals::TVal, + transforms::TTrans, + is_unconstrained, + num_inactive, + ) where {K,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector} + if length(varnames) != length(ranges) || + length(varnames) != length(transforms) || + length(varnames) != length(is_unconstrained) || + length(varnames) != length(varname_to_index) + msg = """ + Inputs to VarNamedVector have inconsistent lengths. Got lengths \ + varnames: $(length(varnames)), \ + ranges: $(length(ranges)), \ + transforms: $(length(transforms)), \ + is_unconstrained: $(length(is_unconstrained)), \ + varname_to_index: $(length(varname_to_index)).""" + throw(ArgumentError(msg)) + end + + num_vals = mapreduce(length, (+), ranges; init=0) + sum(values(num_inactive)) + if num_vals != length(vals) + msg = """ + The total number of elements in `vals` does not match the sum of the \ + lengths of the ranges and the number of inactive entries.""" + throw(ArgumentError(msg)) + end + + if Set(values(varname_to_index)) != Set(1:length(varnames)) + msg = "The values of `varname_to_index` are not valid indices." + throw(ArgumentError(msg)) + end + + if !issubset(Set(keys(num_inactive)), Set(values(varname_to_index))) + msg = "The keys of `num_inactive` are not valid indices." + throw(ArgumentError(msg)) + end + + # Check that the varnames don't overlap. The time cost is quadratic in number of + # variables. If this ever becomes an issue, we should be able to go down to at least + # N log N by sorting based on subsumes-order. + for vn1 in keys(varname_to_index) + for vn2 in keys(varname_to_index) + vn1 === vn2 && continue + if subsumes(vn1, vn2) + msg = """ + Variables in a VarNamedVector should not subsume each other, \ + but $vn1 subsumes $vn2""" + throw(ArgumentError(msg)) + end + end + end + + # We could also have a test to check that the ranges don't overlap, but that sounds + # unlikely to occur, and implementing it in linear time would require a tiny bit of + # thought. + + return new{K,V,TVN,TVal,TTrans}( + varname_to_index, + varnames, + ranges, + vals, + transforms, + is_unconstrained, + num_inactive, + ) + end +end + +# Default values for is_unconstrained (all false) and num_inactive (empty). +function VarNamedVector( + varname_to_index, + varnames, + ranges, + vals, + transforms, + is_unconstrained=fill!(BitVector(undef, length(varnames)), 0), +) + return VarNamedVector( + varname_to_index, + varnames, + ranges, + vals, + transforms, + is_unconstrained, + OrderedDict{Int,Int}(), + ) +end + +# TODO(mhauru) Are we sure we want the last one to be of type Any[]? Might this call +# unnecessary type instability? +function VarNamedVector{K,V}() where {K,V} + return VarNamedVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) +end + +# TODO(mhauru) I would like for this to be VarNamedVector(Union{}, Union{}). This would +# allow expanding the VarName and element types only as necessary, which would help keep +# them concrete. However, making that change here opens some other cans of worms related to +# how VarInfo uses BangBang, that I don't want to deal with right now. +VarNamedVector() = VarNamedVector{VarName,Real}() +VarNamedVector(xs::Pair...) = VarNamedVector(OrderedDict(xs...)) +VarNamedVector(x::AbstractDict) = VarNamedVector(keys(x), values(x)) +function VarNamedVector(varnames, vals) + return VarNamedVector(collect_maybe(varnames), collect_maybe(vals)) +end +function VarNamedVector( + varnames::AbstractVector, vals::AbstractVector, transforms=map(from_vec_transform, vals) +) + # Convert `vals` into a vector of vectors. + vals_vecs = map(tovec, vals) + + # TODO: Is this really the way to do this? + if !(eltype(varnames) <: VarName) + varnames = convert(Vector{VarName}, varnames) + end + varname_to_index = OrderedDict{eltype(varnames),Int}( + vn => i for (i, vn) in enumerate(varnames) + ) + vals = reduce(vcat, vals_vecs) + # Make the ranges. + ranges = Vector{UnitRange{Int}}() + offset = 0 + for x in vals_vecs + r = (offset + 1):(offset + length(x)) + push!(ranges, r) + offset = r[end] + end + + return VarNamedVector(varname_to_index, varnames, ranges, vals, transforms) +end + +function ==(vnv_left::VarNamedVector, vnv_right::VarNamedVector) + return vnv_left.varname_to_index == vnv_right.varname_to_index && + vnv_left.varnames == vnv_right.varnames && + vnv_left.ranges == vnv_right.ranges && + vnv_left.vals == vnv_right.vals && + vnv_left.transforms == vnv_right.transforms && + vnv_left.is_unconstrained == vnv_right.is_unconstrained && + vnv_left.num_inactive == vnv_right.num_inactive +end + +# Some `VarNamedVector` specific functions. +getidx(vnv::VarNamedVector, vn::VarName) = vnv.varname_to_index[vn] + +getrange(vnv::VarNamedVector, idx::Int) = vnv.ranges[idx] +getrange(vnv::VarNamedVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) + +gettransform(vnv::VarNamedVector, idx::Int) = vnv.transforms[idx] +gettransform(vnv::VarNamedVector, vn::VarName) = gettransform(vnv, getidx(vnv, vn)) + +# TODO(mhauru) Eventually I would like to rename the istrans function to is_unconstrained, +# but that's significantly breaking. +""" + istrans(vnv::VarNamedVector, vn::VarName) + +Return a boolean for whether `vn` is guaranteed to have been transformed so that all of +Euclidean space is its domain. +""" +istrans(vnv::VarNamedVector, vn::VarName) = vnv.is_unconstrained[getidx(vnv, vn)] + +""" + settrans!(vnv::VarNamedVector, val::Bool, vn::VarName) + +Set the value for whether `vn` is guaranteed to have been transformed so that all of +Euclidean space is its domain. +""" +function settrans!(vnv::VarNamedVector, val::Bool, vn::VarName) + return vnv.is_unconstrained[vnv.varname_to_index[vn]] = val +end + +""" + has_inactive(vnv::VarNamedVector) + +Returns `true` if `vnv` has inactive ranges. +""" +has_inactive(vnv::VarNamedVector) = !isempty(vnv.num_inactive) + +""" + num_inactive(vnv::VarNamedVector) + +Return the number of inactive entries in `vnv`. +""" +num_inactive(vnv::VarNamedVector) = sum(values(vnv.num_inactive)) + +""" + num_inactive(vnv::VarNamedVector, vn::VarName) + +Returns the number of inactive entries for `vn` in `vnv`. +""" +num_inactive(vnv::VarNamedVector, vn::VarName) = num_inactive(vnv, getidx(vnv, vn)) +num_inactive(vnv::VarNamedVector, idx::Int) = get(vnv.num_inactive, idx, 0) + +""" + num_allocated(vnv::VarNamedVector) + +Returns the number of allocated entries in `vnv`, both active and inactive. +""" +num_allocated(vnv::VarNamedVector) = length(vnv.vals) + +""" + num_allocated(vnv::VarNamedVector, vn::VarName) + +Returns the number of allocated entries for `vn` in `vnv`, both active and inactive. +""" +num_allocated(vnv::VarNamedVector, vn::VarName) = num_allocated(vnv, getidx(vnv, vn)) +function num_allocated(vnv::VarNamedVector, idx::Int) + return length(getrange(vnv, idx)) + num_inactive(vnv, idx) +end + +# Basic array interface. +Base.eltype(vnv::VarNamedVector) = eltype(vnv.vals) +Base.length(vnv::VarNamedVector) = + if !has_inactive(vnv) + length(vnv.vals) + else + sum(length, vnv.ranges) + end +Base.size(vnv::VarNamedVector) = (length(vnv),) +Base.isempty(vnv::VarNamedVector) = isempty(vnv.varnames) + +# TODO: We should probably remove this +Base.IndexStyle(::Type{<:VarNamedVector}) = IndexLinear() + +# Dictionary interface. +Base.keys(vnv::VarNamedVector) = vnv.varnames +Base.values(vnv::VarNamedVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) +Base.pairs(vnv::VarNamedVector) = (vn => vnv[vn] for vn in keys(vnv)) + +Base.haskey(vnv::VarNamedVector, vn::VarName) = haskey(vnv.varname_to_index, vn) + +# `getindex` & `setindex!` +Base.getindex(vnv::VarNamedVector, i::Int) = getindex_raw(vnv, i) +function Base.getindex(vnv::VarNamedVector, vn::VarName) + x = getindex_raw(vnv, vn) + f = gettransform(vnv, vn) + return f(x) +end + +""" + find_containing_range(ranges::AbstractVector{<:AbstractRange}, x) + +Find the first range in `ranges` that contains `x`. + +Throw an `ArgumentError` if `x` is not in any of the ranges. +""" +function find_containing_range(ranges::AbstractVector{<:AbstractRange}, x) + # TODO: Assume `ranges` to be sorted and contiguous, and use `searchsortedfirst` + # for a more efficient approach. + range_idx = findfirst(Base.Fix1(∈, x), ranges) + + # If we're out of bounds, we raise an error. + if range_idx === nothing + throw(ArgumentError("Value $x is not in any of the ranges.")) + end + + return range_idx +end + +""" + adjusted_ranges(vnv::VarNamedVector) + +Return what `vnv.ranges` would be if there were no inactive entries. +""" +function adjusted_ranges(vnv::VarNamedVector) + # Every range following inactive entries needs to be shifted. + offset = 0 + ranges_adj = similar(vnv.ranges) + for (idx, r) in enumerate(vnv.ranges) + # Remove the `offset` in `r` due to inactive entries. + ranges_adj[idx] = r .- offset + # Update `offset`. + offset += get(vnv.num_inactive, idx, 0) + end + + return ranges_adj +end + +""" + index_to_vals_index(vnv::VarNamedVector, i::Int) + +Convert an integer index that ignores inactive entries to an index that accounts for them. + +This is needed when the user wants to index `vnv` like a vector, but shouldn't have to care +about inactive entries in `vnv.vals`. +""" +function index_to_vals_index(vnv::VarNamedVector, i::Int) + # If we don't have any inactive entries, there's nothing to do. + has_inactive(vnv) || return i + + # Get the adjusted ranges. + ranges_adj = adjusted_ranges(vnv) + # Determine the adjusted range that the index corresponds to. + r_idx = find_containing_range(ranges_adj, i) + r = vnv.ranges[r_idx] + # Determine how much of the index `i` is used to get to this range. + i_used = r_idx == 1 ? 0 : sum(length, ranges_adj[1:(r_idx - 1)]) + # Use remainder to index into `r`. + i_remainder = i - i_used + return r[i_remainder] +end + +""" + getindex_raw(vnv::VarNamedVector, i::Int) + getindex_raw(vnv::VarNamedVector, vn::VarName) + +Like `getindex`, but returns the values as they are stored in `vnv` without transforming. + +For integer indices this is the same as `getindex`, but for `VarName`s this is different. +""" +getindex_raw(vnv::VarNamedVector, i::Int) = vnv.vals[index_to_vals_index(vnv, i)] +getindex_raw(vnv::VarNamedVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] + +# `getindex` for `Colon` +function Base.getindex(vnv::VarNamedVector, ::Colon) + return if has_inactive(vnv) + mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) + else + vnv.vals + end +end + +getindex_raw(vnv::VarNamedVector, ::Colon) = getindex(vnv, Colon()) + +# TODO(mhauru): Remove this as soon as possible. Only needed because of the old Gibbs +# sampler. +function Base.getindex(vnv::VarNamedVector, spl::AbstractSampler) + throw(ErrorException("Cannot index a VarNamedVector with a sampler.")) +end + +Base.setindex!(vnv::VarNamedVector, val, i::Int) = setindex_raw!(vnv, val, i) +function Base.setindex!(vnv::VarNamedVector, val, vn::VarName) + f = inverse(gettransform(vnv, vn)) + return setindex_raw!(vnv, f(val), vn) +end + +""" + setindex_raw!(vnv::VarNamedVector, val, i::Int) + setindex_raw!(vnv::VarNamedVector, val, vn::VarName) + +Like `setindex!`, but sets the values as they are stored in `vnv` without transforming. + +For integer indices this is the same as `setindex!`, but for `VarName`s this is different. +""" +function setindex_raw!(vnv::VarNamedVector, val, i::Int) + return vnv.vals[index_to_vals_index(vnv, i)] = val +end + +function setindex_raw!(vnv::VarNamedVector, val::AbstractVector, vn::VarName) + return vnv.vals[getrange(vnv, vn)] = val +end + +function Base.empty!(vnv::VarNamedVector) + # TODO: Or should the semantics be different, e.g. keeping `varnames`? + empty!(vnv.varname_to_index) + empty!(vnv.varnames) + empty!(vnv.ranges) + empty!(vnv.vals) + empty!(vnv.transforms) + empty!(vnv.is_unconstrained) + empty!(vnv.num_inactive) + return nothing +end +BangBang.empty!!(vnv::VarNamedVector) = (empty!(vnv); return vnv) + +""" + replace_values(vnv::VarNamedVector, vals::AbstractVector) + +Replace the values in `vnv` with `vals`, as they are stored internally. + +This is useful when we want to update the entire underlying vector of values in one go or if +we want to change the how the values are stored, e.g. alter the `eltype`. + +!!! warning + This replaces the raw underlying values, and so care should be taken when using this + function. For example, if `vnv` has any inactive entries, then the provided `vals` + should also contain the inactive entries to avoid unexpected behavior. + +# Examples + +```jldoctest varnamedvector-replace-values +julia> using DynamicPPL: VarNamedVector, replace_values + +julia> vnv = VarNamedVector(@varname(x) => [1.0]); + +julia> replace_values(vnv, [2.0])[@varname(x)] == [2.0] +true +``` + +This is also useful when we want to differentiate wrt. the values using automatic +differentiation, e.g. ForwardDiff.jl. + +```jldoctest varnamedvector-replace-values +julia> using ForwardDiff: ForwardDiff + +julia> f(x) = sum(abs2, replace_values(vnv, x)[@varname(x)]) +f (generic function with 1 method) + +julia> ForwardDiff.gradient(f, [1.0]) +1-element Vector{Float64}: + 2.0 +``` +""" +replace_values(vnv::VarNamedVector, vals) = Accessors.@set vnv.vals = vals + +# TODO(mhauru) The space argument is used by the old Gibbs sampler. To be removed. +function replace_values(vnv::VarNamedVector, ::Val{space}, vals) where {space} + if length(space) > 0 + msg = "Selecting values in a VarNamedVector with a space is not supported." + throw(ArgumentError(msg)) + end + return replace_values(vnv, vals) +end + +""" + unflatten(vnv::VarNamedVector, vals::AbstractVector) + +Return a new instance of `vnv` with the values of `vals` assigned to the variables. + +This assumes that `vals` have been transformed by the same transformations that that the +values in `vnv` have been transformed by. However, unlike [`replace_values`](@ref), +`unflatten` does account for inactive entries in `vnv`, so that the user does not have to +care about them. + +This is in a sense the reverse operation of `vnv[:]`. + +Unflatten recontiguifies the internal storage, getting rid of any inactive entries. + +# Examples + +```jldoctest varnamedvector-unflatten +julia> using DynamicPPL: VarNamedVector, unflatten + +julia> vnv = VarNamedVector(@varname(x) => [1.0, 2.0], @varname(y) => [3.0]); + +julia> unflatten(vnv, vnv[:]) == vnv +true +""" +function unflatten(vnv::VarNamedVector, vals::AbstractVector) + new_ranges = deepcopy(vnv.ranges) + recontiguify_ranges!(new_ranges) + return VarNamedVector( + vnv.varname_to_index, vnv.varnames, new_ranges, vals, vnv.transforms + ) +end + +function Base.merge(left_vnv::VarNamedVector, right_vnv::VarNamedVector) + # Return early if possible. + isempty(left_vnv) && return deepcopy(right_vnv) + isempty(right_vnv) && return deepcopy(left_vnv) + + # Determine varnames. + vns_left = left_vnv.varnames + vns_right = right_vnv.varnames + vns_both = union(vns_left, vns_right) + + # Determine `eltype` of `vals`. + T_left = eltype(left_vnv.vals) + T_right = eltype(right_vnv.vals) + T = promote_type(T_left, T_right) + + # Determine `eltype` of `varnames`. + V_left = eltype(left_vnv.varnames) + V_right = eltype(right_vnv.varnames) + V = promote_type(V_left, V_right) + if !(V <: VarName) + V = VarName + end + + # Determine `eltype` of `transforms`. + F_left = eltype(left_vnv.transforms) + F_right = eltype(right_vnv.transforms) + F = promote_type(F_left, F_right) + + # Allocate. + varname_to_index = OrderedDict{V,Int}() + ranges = UnitRange{Int}[] + vals = T[] + transforms = F[] + is_unconstrained = BitVector(undef, length(vns_both)) + + # Range offset. + offset = 0 + + for (idx, vn) in enumerate(vns_both) + varname_to_index[vn] = idx + # Extract the necessary information from `left` or `right`. + if vn in vns_left && !(vn in vns_right) + # `vn` is only in `left`. + val = getindex_raw(left_vnv, vn) + f = gettransform(left_vnv, vn) + is_unconstrained[idx] = istrans(left_vnv, vn) + else + # `vn` is either in both or just `right`. + # Note that in a `merge` the right value has precedence. + val = getindex_raw(right_vnv, vn) + f = gettransform(right_vnv, vn) + is_unconstrained[idx] = istrans(right_vnv, vn) + end + n = length(val) + r = (offset + 1):(offset + n) + # Update. + append!(vals, val) + push!(ranges, r) + push!(transforms, f) + # Increment `offset`. + offset += n + end + + return VarNamedVector( + varname_to_index, vns_both, ranges, vals, transforms, is_unconstrained + ) +end + +""" + subset(vnv::VarNamedVector, vns::AbstractVector{<:VarName}) + +Return a new `VarNamedVector` containing the values from `vnv` for variables in `vns`. + +Which variables to include is determined by the `VarName`'s `subsumes` relation, meaning +that e.g. `subset(vnv, [@varname(x)])` will include variables like `@varname(x.a[1])`. + +# Examples + +```jldoctest varnamedvector-subset +julia> using DynamicPPL: VarNamedVector, @varname, subset + +julia> vnv = VarNamedVector(@varname(x) => [1.0, 2.0], @varname(y) => [3.0]); + +julia> subset(vnv, [@varname(x)]) == VarNamedVector(@varname(x) => [1.0, 2.0]) +true + +julia> subset(vnv, [@varname(x[2])]) == VarNamedVector(@varname(x[2]) => [2.0]) +true +""" +function subset(vnv::VarNamedVector, vns_given::AbstractVector{VN}) where {VN<:VarName} + # NOTE: This does not specialize types when possible. + vns = mapreduce(vcat, vns_given; init=VN[]) do vn + filter(Base.Fix1(subsumes, vn), vnv.varnames) + end + vnv_new = similar(vnv) + # Return early if possible. + isempty(vnv) && return vnv_new + + for vn in vns + push!(vnv_new, vn, getindex_raw(vnv, vn), gettransform(vnv, vn)) + settrans!(vnv_new, istrans(vnv, vn), vn) + end + + return vnv_new +end + +""" + similar(vnv::VarNamedVector) + +Return a new `VarNamedVector` with the same structure as `vnv`, but with empty values. + +In this respect `vnv` behaves more like a dictionary than an array: `similar(vnv)` will +be entirely empty, rather than have `undef` values in it. + +# Examples + +```julia-doctest-varnamedvector-similar +julia> using DynamicPPL: VarNamedVector, @varname, similar + +julia> vnv = VarNamedVector(@varname(x) => [1.0, 2.0], @varname(x[3]) => [3.0]); + +julia> similar(vnv) == VarNamedVector{VarName{:x}, Float64}() +true +""" +function Base.similar(vnv::VarNamedVector) + # NOTE: Whether or not we should empty the underlying containers or not + # is somewhat ambiguous. For example, `similar(vnv.varname_to_index)` will + # result in an empty `AbstractDict`, while the vectors, e.g. `vnv.ranges`, + # will result in non-empty vectors but with entries as `undef`. But it's + # much easier to write the rest of the code assuming that `undef` is not + # present, and so for now we empty the underlying containers, thus differing + # from the behavior of `similar` for `AbstractArray`s. + return VarNamedVector( + similar(vnv.varname_to_index), + similar(vnv.varnames, 0), + similar(vnv.ranges, 0), + similar(vnv.vals, 0), + similar(vnv.transforms, 0), + BitVector(), + similar(vnv.num_inactive), + ) +end + +""" + is_contiguous(vnv::VarNamedVector) + +Returns `true` if the underlying data of `vnv` is stored in a contiguous array. + +This is equivalent to negating [`has_inactive(vnv)`](@ref). +""" +is_contiguous(vnv::VarNamedVector) = !has_inactive(vnv) + +""" + nextrange(vnv::VarNamedVector, x) + +Return the range of `length(x)` from the end of current data in `vnv`. +""" +function nextrange(vnv::VarNamedVector, x) + offset = length(vnv.vals) + return (offset + 1):(offset + length(x)) +end + +""" + push!(vnv::VarNamedVector, vn::VarName, val[, transform]) + +Add a variable with given value to `vnv`. + +By default `transform` is the one that converts the value to a vector, which is how it is +stored in `vnv`. +""" +function Base.push!( + vnv::VarNamedVector, vn::VarName, val, transform=from_vec_transform(val) +) + # Error if we already have the variable. + haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) + # NOTE: We need to compute the `nextrange` BEFORE we start mutating the underlying + # storage. + val_vec = tovec(val) + r_new = nextrange(vnv, val_vec) + vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 + push!(vnv.varnames, vn) + push!(vnv.ranges, r_new) + append!(vnv.vals, val_vec) + push!(vnv.transforms, transform) + push!(vnv.is_unconstrained, false) + return nothing +end + +# TODO(mhauru) The gidset and num_produce arguments are used by the old Gibbs sampler. +# Remove this method as soon as possible. +function Base.push!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) + f = from_vec_transform(dist) + return push!(vnv, vn, val, f) +end + +""" + loosen_types!!(vnv::VarNamedVector{K,V,TVN,TVal,TTrans}, ::Type{KNew}, ::Type{TransNew}) + +Loosen the types of `vnv` to allow varname type `KNew` and transformation type `TransNew`. + +If `KNew` is a subtype of `K` and `TransNew` is a subtype of the element type of the +`TTrans` then this is a no-op and `vnv` is returned as is. Otherwise a new `VarNamedVector` +is returned with the same data but more abstract types, so that variables of type `KNew` and +transformations of type `TransNew` can be pushed to it. Some of the underlying storage is +shared between `vnv` and the return value, and thus mutating one may affect the other. + +# See also +[`tighten_types`](@ref) + +# Examples + +```jldoctest varnamedvector-loosen-types +julia> using DynamicPPL: VarNamedVector, @varname, loosen_types!! + +julia> vnv = VarNamedVector(@varname(x) => [1.0]); + +julia> vnv_new = loosen_types!!(vnv, VarName{:x}, Real); + +julia> push!(vnv, @varname(y), Float32[2.0]) +ERROR: MethodError: Cannot `convert` an object of type + VarName{y,typeof(identity)} to an object of type + VarName{x,typeof(identity)} +[...] + +julia> vnv_loose = DynamicPPL.loosen_types!!(vnv, typeof(@varname(y)), Float32); + +julia> push!(vnv_loose, @varname(y), Float32[2.0]); vnv_loose # Passes without issues. +VarNamedVector{VarName{sym, typeof(identity)} where sym, Float64, Vector{VarName{sym, typeof(identity)} where sym}, Vector{Float64}, Vector{Any}}(OrderedDict{VarName{sym, typeof(identity)} where sym, Int64}(x => 1, y => 2), VarName{sym, typeof(identity)} where sym[x, y], UnitRange{Int64}[1:1, 2:2], [1.0, 2.0], Any[identity, identity], Bool[0, 0], OrderedDict{Int64, Int64}()) +""" +function loosen_types!!( + vnv::VarNamedVector, ::Type{KNew}, ::Type{TransNew} +) where {KNew,TransNew} + K = eltype(vnv.varnames) + Trans = eltype(vnv.transforms) + if KNew <: K && TransNew <: Trans + return vnv + else + vn_type = promote_type(K, KNew) + transform_type = promote_type(Trans, TransNew) + return VarNamedVector( + OrderedDict{vn_type,Int}(vnv.varname_to_index), + Vector{vn_type}(vnv.varnames), + vnv.ranges, + vnv.vals, + Vector{transform_type}(vnv.transforms), + vnv.is_unconstrained, + vnv.num_inactive, + ) + end +end + +""" + tighten_types(vnv::VarNamedVector) + +Return a copy of `vnv` with the most concrete types possible. + +For instance, if `vnv` has element type `Real`, but all the values are actually `Float64`s, +then `tighten_types(vnv)` will have element type `Float64`. + +# See also +[`loosen_types!!`](@ref) +""" +function tighten_types(vnv::VarNamedVector) + return VarNamedVector( + OrderedDict(vnv.varname_to_index...), + [vnv.varnames...], + copy(vnv.ranges), + [vnv.vals...], + [vnv.transforms...], + copy(vnv.is_unconstrained), + copy(vnv.num_inactive), + ) +end + +function BangBang.push!!( + vnv::VarNamedVector, vn::VarName, val, transform=from_vec_transform(val) +) + vnv = loosen_types!!(vnv, typeof(vn), typeof(transform)) + push!(vnv, vn, val, transform) + return vnv +end + +# TODO(mhauru) The gidset and num_produce arguments are used by the old Gibbs sampler. +# Remove this method as soon as possible. +function BangBang.push!!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) + f = from_vec_transform(dist) + return push!!(vnv, vn, val, f) +end + +""" + shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) + +Shifts the elements of `x` starting from index `start` by `n` to the right. +""" +function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) + x[(start + n):end] = x[start:(end - n)] + return x +end + +""" + shift_subsequent_ranges_by!(vnv::VarNamedVector, idx::Int, n) + +Shifts the ranges of variables in `vnv` starting from index `idx` by `n`. +""" +function shift_subsequent_ranges_by!(vnv::VarNamedVector, idx::Int, n) + for i in (idx + 1):length(vnv.ranges) + vnv.ranges[i] = vnv.ranges[i] .+ n + end + return nothing +end + +""" + update!(vnv::VarNamedVector, vn::VarName, val[, transform]) + +Either add a new entry or update existing entry for `vn` in `vnv` with the value `val`. + +If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). + +By default `transform` is the one that converts the value to a vector, which is how it is +stored in `vnv`. +""" +function update!(vnv::VarNamedVector, vn::VarName, val, transform=from_vec_transform(val)) + if !haskey(vnv, vn) + # Here we just add a new entry. + return push!(vnv, vn, val, transform) + end + + # Here we update an existing entry. + val_vec = tovec(val) + idx = getidx(vnv, vn) + # Extract the old range. + r_old = getrange(vnv, idx) + start_old, end_old = first(r_old), last(r_old) + n_old = length(r_old) + # Compute the new range. + n_new = length(val_vec) + start_new = start_old + end_new = start_old + n_new - 1 + r_new = start_new:end_new + + #= + Suppose we currently have the following: + + | x | x | o | o | o | y | y | y | <- Current entries + + where 'O' denotes an inactive entry, and we're going to + update the variable `x` to be of size `k` instead of 2. + + We then have a few different scenarios: + 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. + E.g. if `k = 7`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | x | x | y | y | y | <- New entries + + 2. `k = 5`: All inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | y | y | y | <- New entries + + 3. `k < 5`: Some inactive entries become active, some remain inactive. + E.g. if `k = 3`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | o | o | y | y | y | <- New entries + + 4. `k = 2`: No inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | o | o | o | y | y | y | <- New entries + + 5. `k < 2`: More entries become inactive. + E.g. if `k = 1`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | o | o | o | o | y | y | y | <- New entries + =# + + # Compute the allocated space for `vn`. + had_inactive = haskey(vnv.num_inactive, idx) + n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old + + if n_new > n_allocated + # Then we need to grow the underlying vector. + n_extra = n_new - n_allocated + # Allocate. + resize!(vnv.vals, length(vnv.vals) + n_extra) + # Shift current values. + shift_right!(vnv.vals, end_old + 1, n_extra) + # No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) + # Update the ranges for all variables after this one. + shift_subsequent_ranges_by!(vnv, idx, n_extra) + elseif n_new == n_allocated + # => No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) + else + # `n_new < n_allocated` + # => Need to update the number of inactive entries. + vnv.num_inactive[idx] = n_allocated - n_new + end + + # Update the range for this variable. + vnv.ranges[idx] = r_new + # Update the value. + vnv.vals[r_new] = val_vec + # Update the transform. + vnv.transforms[idx] = transform + + # TODO: Should we maybe sweep over inactive ranges and re-contiguify + # if the total number of inactive elements is "large" in some sense? + + return nothing +end + +function update!!(vnv::VarNamedVector, vn::VarName, val, transform=from_vec_transform(val)) + vnv = loosen_types!!(vnv, typeof(vn), typeof(transform)) + update!(vnv, vn, val, transform) + return vnv +end + +# set!! is the function defined in utils.jl that tries to do fancy stuff with optics when +# setting the value of a generic container using a VarName. We can bypass all that because +# VarNamedVector handles VarNames natively. +set!!(vnv::VarNamedVector, vn::VarName, val) = update!!(vnv, vn, val) + +function setval!(vnv::VarNamedVector, val, vn::VarName) + return setindex_raw!(vnv, tovec(val), vn) +end + +function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) + offset = 0 + for i in 1:length(ranges) + r_old = ranges[i] + ranges[i] = (offset + 1):(offset + length(r_old)) + offset += length(r_old) + end + + return ranges +end + +""" + contiguify!(vnv::VarNamedVector) + +Re-contiguify the underlying vector and shrink if possible. + +# Examples + +```jldoctest varnamedvector-contiguify +julia> using DynamicPPL: VarNamedVector, @varname, contiguify!, update!, has_inactive + +julia> vnv = VarNamedVector(@varname(x) => [1.0, 2.0, 3.0], @varname(y) => [3.0]); + +julia> update!(vnv, @varname(x), [23.0, 24.0]); + +julia> has_inactive(vnv) +true + +julia> length(vnv.vals) +4 + +julia> contiguify!(vnv); + +julia> has_inactive(vnv) +false + +julia> length(vnv.vals) +3 + +julia> vnv[@varname(x)] # All the values are still there. +2-element Vector{Float64}: + 23.0 + 24.0 +``` +""" +function contiguify!(vnv::VarNamedVector) + # Extract the re-contiguified values. + # NOTE: We need to do this before we update the ranges. + old_vals = copy(vnv.vals) + old_ranges = copy(vnv.ranges) + # And then we re-contiguify the ranges. + recontiguify_ranges!(vnv.ranges) + # Clear the inactive ranges. + empty!(vnv.num_inactive) + # Now we update the values. + for (old_range, new_range) in zip(old_ranges, vnv.ranges) + vnv.vals[new_range] = old_vals[old_range] + end + # And (potentially) shrink the underlying vector. + resize!(vnv.vals, vnv.ranges[end][end]) + # The rest should be left as is. + return vnv +end + +""" + group_by_symbol(vnv::VarNamedVector) + +Return a dictionary mapping symbols to `VarNamedVector`s with varnames containing that +symbol. + +# Examples + +```jldoctest varnamedvector-group-by-symbol +julia> using DynamicPPL: VarNamedVector, @varname, group_by_symbol + +julia> vnv = VarNamedVector(@varname(x) => [1.0], @varname(y) => [2.0], @varname(x[1]) => [3.0]); + +julia> d = group_by_symbol(vnv); + +julia> collect(keys(d)) +[Symbol("x"), Symbol("y")] + +julia> d[@varname(x)] == VarNamedVector(@varname(x) => [1.0], @varname(x[1]) => [3.0]) +true + +julia> d[@varname(y)] == VarNamedVector(@varname(y) => [2.0]) +true +""" +function group_by_symbol(vnv::VarNamedVector) + symbols = unique(map(getsym, vnv.varnames)) + nt_vals = map(s -> tighten_types(subset(vnv, [VarName(s)])), symbols) + return OrderedDict(zip(symbols, nt_vals)) +end + +""" + shift_index_left!(vnv::VarNamedVector, idx::Int) + +Shift the index `idx` to the left by one and update the relevant fields. + +This only affects `vnv.varname_to_index` and `vnv.num_inactive` and is only valid as a +helper function for [`shift_subsequent_indices_left!`](@ref). + +!!! warning + This does not check if index we're shifting to is already occupied. +""" +function shift_index_left!(vnv::VarNamedVector, idx::Int) + # Shift the index in the lookup table. + vn = vnv.varnames[idx] + vnv.varname_to_index[vn] = idx - 1 + # Shift the index in the inactive ranges. + if haskey(vnv.num_inactive, idx) + # Done in increasing order => don't need to worry about + # potentially shifting the same index twice. + vnv.num_inactive[idx - 1] = pop!(vnv.num_inactive, idx) + end +end + +""" + shift_subsequent_indices_left!(vnv::VarNamedVector, idx::Int) + +Shift the indices for all variables after `idx` to the left by one and update the relevant + fields. + +This only affects `vnv.varname_to_index` and `vnv.num_inactive` and is only valid as a +helper function for [`delete!`](@ref). +""" +function shift_subsequent_indices_left!(vnv::VarNamedVector, idx::Int) + # Shift the indices for all variables after `idx`. + for idx_to_shift in (idx + 1):length(vnv.varnames) + shift_index_left!(vnv, idx_to_shift) + end +end + +function Base.delete!(vnv::VarNamedVector, vn::VarName) + # Error if we don't have the variable. + !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) + + # Get the index of the variable. + idx = getidx(vnv, vn) + + # Delete the values. + r_start = first(getrange(vnv, idx)) + n_allocated = num_allocated(vnv, idx) + # NOTE: `deleteat!` also results in a `resize!` so we don't need to do that. + deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) + + # Delete `vn` from the lookup table. + delete!(vnv.varname_to_index, vn) + + # Delete any inactive ranges corresponding to `vn`. + haskey(vnv.num_inactive, idx) && delete!(vnv.num_inactive, idx) + + # Re-adjust the indices for varnames occuring after `vn` so + # that they point to the correct indices after the deletions below. + shift_subsequent_indices_left!(vnv, idx) + + # Re-adjust the ranges for varnames occuring after `vn`. + shift_subsequent_ranges_by!(vnv, idx, -n_allocated) + + # Delete references from vector fields, thus shifting the indices of + # varnames occuring after `vn` by one to the left, as we adjusted for above. + deleteat!(vnv.varnames, idx) + deleteat!(vnv.ranges, idx) + deleteat!(vnv.transforms, idx) + + return vnv +end + +""" + values_as(vnv::VarNamedVector[, T]) + +Return the values/realizations in `vnv` as type `T`, if implemented. + +If no type `T` is provided, return values as stored in `vnv`. + +# Examples + +```jldoctest +julia> using DynamicPPL: VarNamedVector + +julia> vnv = VarNamedVector(@varname(x) => 1, @varname(y) => [2.0]); + +julia> values_as(vnv) == [1.0, 2.0] +true + +julia> values_as(vnv, Vector{Float32}) == Vector{Float32}([1.0, 2.0]) +true + +julia> values_as(vnv, OrderedDict) == OrderedDict(@varname(x) => 1.0, @varname(y) => [2.0]) +true + +julia> values_as(vnv, NamedTuple) == (x = 1.0, y = [2.0]) +true +``` +""" +values_as(vnv::VarNamedVector) = values_as(vnv, Vector) +values_as(vnv::VarNamedVector, ::Type{Vector}) = vnv[:] +function values_as(vnv::VarNamedVector, ::Type{Vector{T}}) where {T} + return convert(Vector{T}, values_as(vnv, Vector)) +end +function values_as(vnv::VarNamedVector, ::Type{NamedTuple}) + return NamedTuple(zip(map(Symbol, keys(vnv)), values(vnv))) +end +function values_as(vnv::VarNamedVector, ::Type{D}) where {D<:AbstractDict} + return ConstructionBase.constructorof(D)(pairs(vnv)) +end + +# TODO(mhauru) This is tricky to implement in the general case, and the below implementation +# only covers some simple cases. It's probably sufficient in most situations though. +function hasvalue(vnv::VarNamedVector, vn::VarName) + haskey(vnv, vn) && return true + any(subsumes(vn, k) for k in keys(vnv)) && return true + # Handle the easy case where the right symbol isn't even present. + !any(k -> getsym(k) == getsym(vn), keys(vnv)) && return false + + optic = getoptic(vn) + if optic isa Accessors.IndexLens || optic isa Accessors.ComposedOptic + # If vn is of the form @varname(somesymbol[someindex]), we check whether we store + # @varname(somesymbol) and can index into it with someindex. If we rather have a + # composed optic with the last part being an index lens, we do a similar check but + # stripping out the last index lens part. If these pass, the answer is definitely + # "yes". If not, we still don't know for sure. + # TODO(mhauru) What about casese where vnv stores both @varname(x) and + # @varname(x[1]) or @varname(x.a)? Those should probably be banned, but currently + # aren't. + head, tail = if optic isa Accessors.ComposedOptic + decomp_optic = Accessors.decompose(optic) + first(decomp_optic), Accessors.compose(decomp_optic[2:end]...) + else + optic, identity + end + parent_varname = VarName{getsym(vn)}(tail) + if haskey(vnv, parent_varname) + valvec = getindex(vnv, parent_varname) + return canview(head, valvec) + end + end + throw(ErrorException("hasvalue has not been fully implemented for this VarName: $(vn)")) +end + +# TODO(mhauru) Like hasvalue, this is only partially implemented. +function getvalue(vnv::VarNamedVector, vn::VarName) + !hasvalue(vnv, vn) && throw(KeyError(vn)) + haskey(vnv, vn) && getindex(vnv, vn) + + subsumed_keys = filter(k -> subsumes(vn, k), keys(vnv)) + if length(subsumed_keys) > 0 + # TODO(mhauru) What happens if getindex returns e.g. matrices, and we vcat them? + return mapreduce(k -> getindex(vnv, k), vcat, subsumed_keys) + end + + optic = getoptic(vn) + # See hasvalue for some comments on the logic of this if block. + if optic isa Accessors.IndexLens || optic isa Accessors.ComposedOptic + head, tail = if optic isa Accessors.ComposedOptic + decomp_optic = Accessors.decompose(optic) + first(decomp_optic), Accessors.compose(decomp_optic[2:end]...) + else + optic, identity + end + parent_varname = VarName{getsym(vn)}(tail) + valvec = getindex(vnv, parent_varname) + return head(valvec) + end + throw(ErrorException("getvalue has not been fully implemented for this VarName: $(vn)")) +end + +Base.get(vnv::VarNamedVector, vn::VarName) = getvalue(vnv, vn) diff --git a/src/varnamevector.jl b/src/varnamevector.jl deleted file mode 100644 index ed919a20c..000000000 --- a/src/varnamevector.jl +++ /dev/null @@ -1,756 +0,0 @@ -""" - VarNameVector - -A container that works like a `Vector` and an `OrderedDict` but is neither. - -# Fields -$(FIELDS) -""" -struct VarNameVector{ - K<:VarName,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector,MData -} - "mapping from the `VarName` to its integer index in `varnames`, `ranges` and `dists`" - varname_to_index::OrderedDict{K,Int} - - "vector of identifiers for the random variables, where `varnames[varname_to_index[vn]] == vn`" - varnames::TVN # AbstractVector{<:VarName} - - "vector of index ranges in `vals` corresponding to `varnames`; each `VarName` `vn` has a single index or a set of contiguous indices in `vals`" - ranges::Vector{UnitRange{Int}} - - "vector of values of all variables; the value(s) of `vn` is/are `vals[ranges[varname_to_index[vn]]]`" - vals::TVal # AbstractVector{<:Real} - - "vector of transformations whose inverse takes us back to the original space" - transforms::TTrans - - "specifies whether a variable is transformed or not " - is_transformed::BitVector - - "additional entries which are considered inactive" - num_inactive::OrderedDict{Int,Int} - - "metadata associated with the varnames" - metadata::MData -end - -function ==(vnv_left::VarNameVector, vnv_right::VarNameVector) - return vnv_left.varname_to_index == vnv_right.varname_to_index && - vnv_left.varnames == vnv_right.varnames && - vnv_left.ranges == vnv_right.ranges && - vnv_left.vals == vnv_right.vals && - vnv_left.transforms == vnv_right.transforms && - vnv_left.is_transformed == vnv_right.is_transformed && - vnv_left.num_inactive == vnv_right.num_inactive && - vnv_left.metadata == vnv_right.metadata -end - -function VarNameVector( - varname_to_index, - varnames, - ranges, - vals, - transforms, - is_transformed=fill!(BitVector(undef, length(varnames)), 0), -) - return VarNameVector( - varname_to_index, - varnames, - ranges, - vals, - transforms, - is_transformed, - OrderedDict{Int,Int}(), - nothing, - ) -end -# TODO: Do we need this? -function VarNameVector{K,V}() where {K,V} - return VarNameVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) -end - -istrans(vnv::VarNameVector, vn::VarName) = vnv.is_transformed[vnv.varname_to_index[vn]] -function settrans!(vnv::VarNameVector, val::Bool, vn::VarName) - return vnv.is_transformed[vnv.varname_to_index[vn]] = val -end - -VarNameVector() = VarNameVector{VarName,Real}() -VarNameVector(xs::Pair...) = VarNameVector(OrderedDict(xs...)) -VarNameVector(x::AbstractDict) = VarNameVector(keys(x), values(x)) -VarNameVector(varnames, vals) = VarNameVector(collect_maybe(varnames), collect_maybe(vals)) -function VarNameVector( - varnames::AbstractVector, vals::AbstractVector, transforms=map(from_vec_transform, vals) -) - # TODO: Check uniqueness of `varnames`? - - # Convert `vals` into a vector of vectors. - vals_vecs = map(tovec, vals) - - # TODO: Is this really the way to do this? - if !(eltype(varnames) <: VarName) - varnames = convert(Vector{VarName}, varnames) - end - varname_to_index = OrderedDict{eltype(varnames),Int}() - ranges = Vector{UnitRange{Int}}() - offset = 0 - for (i, (vn, x)) in enumerate(zip(varnames, vals_vecs)) - # Add the varname index. - push!(varname_to_index, vn => length(varname_to_index) + 1) - # Add the range. - r = (offset + 1):(offset + length(x)) - push!(ranges, r) - # Update the offset. - offset = r[end] - end - - return VarNameVector( - varname_to_index, varnames, ranges, reduce(vcat, vals_vecs), transforms - ) -end - -""" - replace_values(vnv::VarNameVector, vals::AbstractVector) - -Replace the values in `vnv` with `vals`. - -This is useful when we want to update the entire underlying vector of values -in one go or if we want to change the how the values are stored, e.g. alter the `eltype`. - -!!! warning - This replaces the raw underlying values, and so care should be taken when using this - function. For example, if `vnv` has any inactive entries, then the provided `vals` - should also contain the inactive entries to avoid unexpected behavior. - -# Example - -```jldoctest varnamevector-replace-values -julia> using DynamicPPL: VarNameVector, replace_values - -julia> vnv = VarNameVector(@varname(x) => [1.0]); - -julia> replace_values(vnv, [2.0])[@varname(x)] == [2.0] -true -``` - -This is also useful when we want to differentiate wrt. the values -using automatic differentiation, e.g. ForwardDiff.jl. - -```jldoctest varnamevector-replace-values -julia> using ForwardDiff: ForwardDiff - -julia> f(x) = sum(abs2, replace_values(vnv, x)[@varname(x)]) -f (generic function with 1 method) - -julia> ForwardDiff.gradient(f, [1.0]) -1-element Vector{Float64}: - 2.0 -``` -""" -replace_values(vnv::VarNameVector, vals) = Setfield.@set vnv.vals = vals - -# Some `VarNameVector` specific functions. -getidx(vnv::VarNameVector, vn::VarName) = vnv.varname_to_index[vn] - -getrange(vnv::VarNameVector, idx::Int) = vnv.ranges[idx] -getrange(vnv::VarNameVector, vn::VarName) = getrange(vnv, getidx(vnv, vn)) - -gettransform(vnv::VarNameVector, vn::VarName) = vnv.transforms[getidx(vnv, vn)] - -""" - has_inactive(vnv::VarNameVector) - -Returns `true` if `vnv` has inactive ranges. -""" -has_inactive(vnv::VarNameVector) = !isempty(vnv.num_inactive) - -""" - num_inactive(vnv::VarNameVector) - -Return the number of inactive entries in `vnv`. -""" -num_inactive(vnv::VarNameVector) = sum(values(vnv.num_inactive)) - -""" - num_inactive(vnv::VarNameVector, vn::VarName) - -Returns the number of inactive entries for `vn` in `vnv`. -""" -num_inactive(vnv::VarNameVector, vn::VarName) = num_inactive(vnv, getidx(vnv, vn)) -num_inactive(vnv::VarNameVector, idx::Int) = get(vnv.num_inactive, idx, 0) - -""" - num_allocated(vnv::VarNameVector) - -Returns the number of allocated entries in `vnv`. -""" -num_allocated(vnv::VarNameVector) = length(vnv.vals) - -""" - num_allocated(vnv::VarNameVector, vn::VarName) - -Returns the number of allocated entries for `vn` in `vnv`. -""" -num_allocated(vnv::VarNameVector, vn::VarName) = num_allocated(vnv, getidx(vnv, vn)) -function num_allocated(vnv::VarNameVector, idx::Int) - return length(getrange(vnv, idx)) + num_inactive(vnv, idx) -end - -# Basic array interface. -Base.eltype(vnv::VarNameVector) = eltype(vnv.vals) -Base.length(vnv::VarNameVector) = - if isempty(vnv.num_inactive) - length(vnv.vals) - else - sum(length, vnv.ranges) - end -Base.size(vnv::VarNameVector) = (length(vnv),) -Base.isempty(vnv::VarNameVector) = isempty(vnv.varnames) - -# TODO: We should probably remove this -Base.IndexStyle(::Type{<:VarNameVector}) = IndexLinear() - -# Dictionary interface. -Base.keys(vnv::VarNameVector) = vnv.varnames -Base.values(vnv::VarNameVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) -Base.pairs(vnv::VarNameVector) = (vn => vnv[vn] for vn in keys(vnv)) - -Base.haskey(vnv::VarNameVector, vn::VarName) = haskey(vnv.varname_to_index, vn) - -# `getindex` & `setindex!` -Base.getindex(vnv::VarNameVector, i::Int) = getindex_raw(vnv, i) -function Base.getindex(vnv::VarNameVector, vn::VarName) - x = getindex_raw(vnv, vn) - f = gettransform(vnv, vn) - return f(x) -end - -function find_range_from_sorted(ranges::AbstractVector{<:AbstractRange}, x) - # TODO: Assume `ranges` to be sorted and contiguous, and use `searchsortedfirst` - # for a more efficient approach. - range_idx = findfirst(Base.Fix1(∈, x), ranges) - - # If we're out of bounds, we raise an error. - if range_idx === nothing - throw(ArgumentError("Value $x is not in any of the ranges.")) - end - - return range_idx -end - -function adjusted_ranges(vnv::VarNameVector) - # Every range following inactive entries needs to be shifted. - offset = 0 - ranges_adj = similar(vnv.ranges) - for (idx, r) in enumerate(vnv.ranges) - # Remove the `offset` in `r` due to inactive entries. - ranges_adj[idx] = r .- offset - # Update `offset`. - offset += get(vnv.num_inactive, idx, 0) - end - - return ranges_adj -end - -function index_to_raw_index(vnv::VarNameVector, i::Int) - # If we don't have any inactive entries, there's nothing to do. - has_inactive(vnv) || return i - - # Get the adjusted ranges. - ranges_adj = adjusted_ranges(vnv) - # Determine the adjusted range that the index corresponds to. - r_idx = find_range_from_sorted(ranges_adj, i) - r = vnv.ranges[r_idx] - # Determine how much of the index `i` is used to get to this range. - i_used = r_idx == 1 ? 0 : sum(length, ranges_adj[1:(r_idx - 1)]) - # Use remainder to index into `r`. - i_remainder = i - i_used - return r[i_remainder] -end - -getindex_raw(vnv::VarNameVector, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] -getindex_raw(vnv::VarNameVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] - -# `getindex` for `Colon` -function Base.getindex(vnv::VarNameVector, ::Colon) - return if has_inactive(vnv) - mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) - else - vnv.vals - end -end - -function getindex_raw(vnv::VarNameVector, ::Colon) - return if has_inactive(vnv) - mapreduce(Base.Fix1(getindex_raw, vnv.vals), vcat, vnv.ranges) - else - vnv.vals - end -end - -# HACK: remove this as soon as possible. -Base.getindex(vnv::VarNameVector, spl::AbstractSampler) = vnv[:] - -Base.setindex!(vnv::VarNameVector, val, i::Int) = setindex_raw!(vnv, val, i) -function Base.setindex!(vnv::VarNameVector, val, vn::VarName) - f = inverse(gettransform(vnv, vn)) - return setindex_raw!(vnv, f(val), vn) -end - -setindex_raw!(vnv::VarNameVector, val, i::Int) = vnv.vals[index_to_raw_index(vnv, i)] = val -function setindex_raw!(vnv::VarNameVector, val::AbstractVector, vn::VarName) - return vnv.vals[getrange(vnv, vn)] = val -end - -# `empty!(!)` -function Base.empty!(vnv::VarNameVector) - # TODO: Or should the semantics be different, e.g. keeping `varnames`? - empty!(vnv.varname_to_index) - empty!(vnv.varnames) - empty!(vnv.ranges) - empty!(vnv.vals) - empty!(vnv.transforms) - empty!(vnv.num_inactive) - return nothing -end -BangBang.empty!!(vnv::VarNameVector) = (empty!(vnv); return vnv) - -function Base.merge(left_vnv::VarNameVector, right_vnv::VarNameVector) - # Return early if possible. - isempty(left_vnv) && return deepcopy(right_vnv) - isempty(right_vnv) && return deepcopy(left_vnv) - - # Determine varnames. - vns_left = left_vnv.varnames - vns_right = right_vnv.varnames - vns_both = union(vns_left, vns_right) - - # Determine `eltype` of `vals`. - T_left = eltype(left_vnv.vals) - T_right = eltype(right_vnv.vals) - T = promote_type(T_left, T_right) - # TODO: Is this necessary? - if !(T <: Real) - T = Real - end - - # Determine `eltype` of `varnames`. - V_left = eltype(left_vnv.varnames) - V_right = eltype(right_vnv.varnames) - V = promote_type(V_left, V_right) - if !(V <: VarName) - V = VarName - end - - # Determine `eltype` of `transforms`. - F_left = eltype(left_vnv.transforms) - F_right = eltype(right_vnv.transforms) - F = promote_type(F_left, F_right) - - # Allocate. - varnames_to_index = OrderedDict{V,Int}() - ranges = UnitRange{Int}[] - vals = T[] - transforms = F[] - is_transformed = BitVector(undef, length(vns_both)) - - # Range offset. - offset = 0 - - for (idx, vn) in enumerate(vns_both) - # Extract the necessary information from `left` or `right`. - if vn in vns_left && !(vn in vns_right) - # `vn` is only in `left`. - varnames_to_index[vn] = idx - val = getindex_raw(left_vnv, vn) - n = length(val) - r = (offset + 1):(offset + n) - f = gettransform(left_vnv, vn) - is_transformed[idx] = istrans(left_vnv, vn) - else - # `vn` is either in both or just `right`. - varnames_to_index[vn] = idx - val = getindex_raw(right_vnv, vn) - n = length(val) - r = (offset + 1):(offset + n) - f = gettransform(right_vnv, vn) - is_transformed[idx] = istrans(right_vnv, vn) - end - # Update. - append!(vals, val) - push!(ranges, r) - push!(transforms, f) - # Increment `offset`. - offset += n - end - - return VarNameVector(varnames_to_index, vns_both, ranges, vals, transforms) -end - -function subset(vnv::VarNameVector, vns::AbstractVector{<:VarName}) - # NOTE: This does not specialize types when possible. - vnv_new = similar(vnv) - # Return early if possible. - isempty(vnv) && return vnv_new - - for vn in vns - push!(vnv_new, vn, getindex_internal(vnv, vn), gettransform(vnv, vn)) - end - - return vnv_new -end - -# `similar` -similar_metadata(::Nothing) = nothing -similar_metadata(x::Union{AbstractArray,AbstractDict}) = similar(x) -function Base.similar(vnv::VarNameVector) - # NOTE: Whether or not we should empty the underlying containers or note - # is somewhat ambiguous. For example, `similar(vnv.varname_to_index)` will - # result in an empty `AbstractDict`, while the vectors, e.g. `vnv.ranges`, - # will result in non-empty vectors but with entries as `undef`. But it's - # much easier to write the rest of the code assuming that `undef` is not - # present, and so for now we empty the underlying containers, thus differing - # from the behavior of `similar` for `AbstractArray`s. - return VarNameVector( - similar(vnv.varname_to_index), - similar(vnv.varnames, 0), - similar(vnv.ranges, 0), - similar(vnv.vals, 0), - similar(vnv.transforms, 0), - BitVector(), - similar(vnv.num_inactive), - similar_metadata(vnv.metadata), - ) -end - -""" - is_contiguous(vnv::VarNameVector) - -Returns `true` if the underlying data of `vnv` is stored in a contiguous array. - -This is equivalent to negating [`has_inactive(vnv)`](@ref). -""" -is_contiguous(vnv::VarNameVector) = !has_inactive(vnv) - -function nextrange(vnv::VarNameVector, x) - # If `vnv` is empty, return immediately. - isempty(vnv) && return 1:length(x) - - # The offset will be the last range's end + its number of inactive entries. - vn_last = vnv.varnames[end] - idx = getidx(vnv, vn_last) - offset = last(getrange(vnv, idx)) + num_inactive(vnv, idx) - - return (offset + 1):(offset + length(x)) -end - -# `push!` and `push!!`: add a variable to the varname vector. -function Base.push!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) - # Error if we already have the variable. - haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) - # NOTE: We need to compute the `nextrange` BEFORE we start mutating - # the underlying; otherwise we might get some strange behaviors. - val_vec = tovec(val) - r_new = nextrange(vnv, val_vec) - vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 - push!(vnv.varnames, vn) - push!(vnv.ranges, r_new) - append!(vnv.vals, val_vec) - push!(vnv.transforms, transform) - push!(vnv.is_transformed, false) - return nothing -end - -""" - shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) - -Shifts the elements of `x` starting from index `start` by `n` to the right. -""" -function shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) - x[(start + n):end] = x[start:(end - n)] - return x -end - -""" - shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) - -Shifts the ranges of variables in `vnv` starting from index `idx` by `n`. -""" -function shift_subsequent_ranges_by!(vnv::VarNameVector, idx::Int, n) - for i in (idx + 1):length(vnv.ranges) - vnv.ranges[i] = vnv.ranges[i] .+ n - end - return nothing -end - -# `update!` and `update!!`: update a variable in the varname vector. -""" - update!(vnv::VarNameVector, vn::VarName, val[, transform]) - -Either add a new entry or update existing entry for `vn` in `vnv` with the value `val`. - -If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). -""" -function update!(vnv::VarNameVector, vn::VarName, val, transform=from_vec_transform(val)) - if !haskey(vnv, vn) - # Here we just add a new entry. - return push!(vnv, vn, val, transform) - end - - # Here we update an existing entry. - val_vec = tovec(val) - idx = getidx(vnv, vn) - # Extract the old range. - r_old = getrange(vnv, idx) - start_old, end_old = first(r_old), last(r_old) - n_old = length(r_old) - # Compute the new range. - n_new = length(val_vec) - start_new = start_old - end_new = start_old + n_new - 1 - r_new = start_new:end_new - - #= - Suppose we currently have the following: - - | x | x | o | o | o | y | y | y | <- Current entries - - where 'O' denotes an inactive entry, and we're going to - update the variable `x` to be of size `k` instead of 2. - - We then have a few different scenarios: - 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. - E.g. if `k = 7`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | x | x | x | x | y | y | y | <- New entries - - 2. `k = 5`: All inactive entries become active. - Then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | x | x | y | y | y | <- New entries - - 3. `k < 5`: Some inactive entries become active, some remain inactive. - E.g. if `k = 3`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | o | o | y | y | y | <- New entries - - 4. `k = 2`: No inactive entries become active. - Then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | o | o | o | y | y | y | <- New entries - - 5. `k < 2`: More entries become inactive. - E.g. if `k = 1`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | o | o | o | o | y | y | y | <- New entries - =# - - # Compute the allocated space for `vn`. - had_inactive = haskey(vnv.num_inactive, idx) - n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old - - if n_new > n_allocated - # Then we need to grow the underlying vector. - n_extra = n_new - n_allocated - # Allocate. - resize!(vnv.vals, length(vnv.vals) + n_extra) - # Shift current values. - shift_right!(vnv.vals, end_old + 1, n_extra) - # No more inactive entries. - had_inactive && delete!(vnv.num_inactive, idx) - # Update the ranges for all variables after this one. - shift_subsequent_ranges_by!(vnv, idx, n_extra) - elseif n_new == n_allocated - # => No more inactive entries. - had_inactive && delete!(vnv.num_inactive, idx) - else - # `n_new < n_allocated` - # => Need to update the number of inactive entries. - vnv.num_inactive[idx] = n_allocated - n_new - end - - # Update the range for this variable. - vnv.ranges[idx] = r_new - # Update the value. - vnv.vals[r_new] = val_vec - # Update the transform. - vnv.transforms[idx] = transform - - # TODO: Should we maybe sweep over inactive ranges and re-contiguify - # if we the total number of inactive elements is "large" in some sense? - - return nothing -end - -function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) - offset = 0 - for i in 1:length(ranges) - r_old = ranges[i] - ranges[i] = (offset + 1):(offset + length(r_old)) - offset += length(r_old) - end - - return ranges -end - -""" - contiguify!(vnv::VarNameVector) - -Re-contiguify the underlying vector and shrink if possible. -""" -function contiguify!(vnv::VarNameVector) - # Extract the re-contiguified values. - # NOTE: We need to do this before we update the ranges. - vals = vnv[:] - # And then we re-contiguify the ranges. - recontiguify_ranges!(vnv.ranges) - # Clear the inactive ranges. - empty!(vnv.num_inactive) - # Now we update the values. - for (i, r) in enumerate(vnv.ranges) - vnv.vals[r] = vals[r] - end - # And (potentially) shrink the underlying vector. - resize!(vnv.vals, vnv.ranges[end][end]) - # The rest should be left as is. - return vnv -end - -""" - group_by_symbol(vnv::VarNameVector) - -Return a dictionary mapping symbols to `VarNameVector`s with -varnames containing that symbol. -""" -function group_by_symbol(vnv::VarNameVector) - # Group varnames in `vnv` by the symbol. - d = OrderedDict{Symbol,Vector{VarName}}() - for vn in vnv.varnames - push!(get!(d, getsym(vn), Vector{VarName}()), vn) - end - - # Create a `NamedTuple` from the grouped varnames. - nt_vals = map(values(d)) do varnames - # TODO: Do we need to specialize the inputs here? - VarNameVector( - map(identity, varnames), - map(Base.Fix1(getindex, vnv), varnames), - map(Base.Fix1(gettransform, vnv), varnames), - ) - end - - return OrderedDict(zip(keys(d), nt_vals)) -end - -""" - shift_index_left!(vnv::VarNameVector, idx::Int) - -Shift the index `idx` to the left by one and update the relevant fields. - -!!! warning - This does not check if index we're shifting to is already occupied. -""" -function shift_index_left!(vnv::VarNameVector, idx::Int) - # Shift the index in the lookup table. - vn = vnv.varnames[idx] - vnv.varname_to_index[vn] = idx - 1 - # Shift the index in the inactive ranges. - if haskey(vnv.num_inactive, idx) - # Done in increasing order => don't need to worry about - # potentially shifting the same index twice. - vnv.num_inactive[idx - 1] = pop!(vnv.num_inactive, idx) - end -end - -""" - shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) - -Shift the indices for all variables after `idx` to the left by one and update -the relevant fields. - -This just -""" -function shift_subsequent_indices_left!(vnv::VarNameVector, idx::Int) - # Shift the indices for all variables after `idx`. - for idx_to_shift in (idx + 1):length(vnv.varnames) - shift_index_left!(vnv, idx_to_shift) - end -end - -function Base.delete!(vnv::VarNameVector, vn::VarName) - # Error if we don't have the variable. - !haskey(vnv, vn) && throw(ArgumentError("variable name $vn does not exist")) - - # Get the index of the variable. - idx = getidx(vnv, vn) - - # Delete the values. - r_start = first(getrange(vnv, idx)) - n_allocated = num_allocated(vnv, idx) - # NOTE: `deleteat!` also results in a `resize!` so we don't need to do that. - deleteat!(vnv.vals, r_start:(r_start + n_allocated - 1)) - - # Delete `vn` from the lookup table. - delete!(vnv.varname_to_index, vn) - - # Delete any inactive ranges corresponding to `vn`. - haskey(vnv.num_inactive, idx) && delete!(vnv.num_inactive, idx) - - # Re-adjust the indices for varnames occuring after `vn` so - # that they point to the correct indices after the deletions below. - shift_subsequent_indices_left!(vnv, idx) - - # Re-adjust the ranges for varnames occuring after `vn`. - shift_subsequent_ranges_by!(vnv, idx, -n_allocated) - - # Delete references from vector fields, thus shifting the indices of - # varnames occuring after `vn` by one to the left, as we adjusted for above. - deleteat!(vnv.varnames, idx) - deleteat!(vnv.ranges, idx) - deleteat!(vnv.transforms, idx) - - return vnv -end - -""" - values_as(vnv::VarNameVector[, T]) - -Return the values/realizations in `vnv` as type `T`, if implemented. - -If no type `T` is provided, return values as stored in `vnv`. - -# Examples - -```jldoctest -julia> using DynamicPPL: VarNameVector - -julia> vnv = VarNameVector(@varname(x) => 1, @varname(y) => [2.0]); - -julia> values_as(vnv) == [1.0, 2.0] -true - -julia> values_as(vnv, Vector{Float32}) == Vector{Float32}([1.0, 2.0]) -true - -julia> values_as(vnv, OrderedDict) == OrderedDict(@varname(x) => 1.0, @varname(y) => [2.0]) -true - -julia> values_as(vnv, NamedTuple) == (x = 1.0, y = [2.0]) -true -``` -""" -values_as(vnv::VarNameVector) = values_as(vnv, Vector) -values_as(vnv::VarNameVector, ::Type{Vector}) = vnv[:] -function values_as(vnv::VarNameVector, ::Type{Vector{T}}) where {T} - return convert(Vector{T}, values_as(vnv, Vector)) -end -function values_as(vnv::VarNameVector, ::Type{NamedTuple}) - return NamedTuple(zip(map(Symbol, keys(vnv)), values(vnv))) -end -function values_as(vnv::VarNameVector, ::Type{D}) where {D<:AbstractDict} - return ConstructionBase.constructorof(D)(pairs(vnv)) -end diff --git a/test/compiler.jl b/test/compiler.jl index f1f06eabe..f2d7e5852 100644 --- a/test/compiler.jl +++ b/test/compiler.jl @@ -309,11 +309,11 @@ module Issue537 end vi2 = VarInfo(f2()) vi3 = VarInfo(f3()) @test haskey(vi1.metadata, :y) - @test vi1.metadata.y.vns[1] == @varname(y) + @test first(Base.keys(vi1.metadata.y)) == @varname(y) @test haskey(vi2.metadata, :y) - @test vi2.metadata.y.vns[1] == @varname(y[2][:, 1]) + @test first(Base.keys(vi2.metadata.y)) == @varname(y[2][:, 1]) @test haskey(vi3.metadata, :y) - @test vi3.metadata.y.vns[1] == @varname(y[1]) + @test first(Base.keys(vi3.metadata.y)) == @varname(y[1]) # Conditioning f1_c = f1() | (y=1,) diff --git a/test/model.jl b/test/model.jl index 60a8d2461..eaad848f1 100644 --- a/test/model.jl +++ b/test/model.jl @@ -122,7 +122,6 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true @test logjoints[i] ≈ DynamicPPL.TestUtils.logjoint_true(model, samples[:s], samples[:m]) end - println("\n model $(model) passed !!! \n") end end @@ -200,10 +199,10 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true @test !any(map(x -> x isa DynamicPPL.AbstractVarInfo, call_retval)) end - @testset "Dynamic constraints" begin + @testset "Dynamic constraints, Metadata" begin model = DynamicPPL.TestUtils.demo_dynamic_constraint() - vi = VarInfo(model) spl = SampleFromPrior() + vi = VarInfo(model, spl, DefaultContext(), DynamicPPL.Metadata) link!!(vi, spl, model) for i in 1:10 @@ -216,6 +215,18 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true end end + @testset "Dynamic constraints, VectorVarInfo" begin + model = DynamicPPL.TestUtils.demo_dynamic_constraint() + vi = VarInfo(model) + vi = link!!(vi, model) + + for i in 1:10 + # Sample with large variations. + vi[@varname(m)] = randn() * 10 + model(vi) + end + end + @testset "rand" begin model = gdemo_default @@ -324,7 +335,6 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true chain = MCMCChains.Chains( permutedims(stack(vals)), syms; info=(varname_to_symbol=vns_to_syms,) ) - display(chain) # Test! results = generated_quantities(model, chain) @@ -345,7 +355,6 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true vcat(syms, [:y]); info=(varname_to_symbol=vns_to_syms_with_extra,), ) - display(chain_with_extra) # Test! results = generated_quantities(model, chain_with_extra) for (x_true, result) in zip(xs, results) @@ -358,6 +367,7 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true models_to_test = [ DynamicPPL.TestUtils.DEMO_MODELS..., DynamicPPL.TestUtils.demo_lkjchol(2) ] + context = DefaultContext() @testset "$(model.f)" for model in models_to_test vns = DynamicPPL.TestUtils.varnames(model) example_values = DynamicPPL.TestUtils.rand_prior_true(model) @@ -366,18 +376,16 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true DynamicPPL.TestUtils.setup_varinfos(model, example_values, vns), ) @testset "$(short_varinfo_name(varinfo))" for varinfo in varinfos - @test ( - @inferred(DynamicPPL.evaluate!!(model, varinfo, DefaultContext())); + @test begin + @inferred(DynamicPPL.evaluate!!(model, varinfo, context)) true - ) + end varinfo_linked = DynamicPPL.link(varinfo, model) - @test ( - @inferred( - DynamicPPL.evaluate!!(model, varinfo_linked, DefaultContext()) - ); + @test begin + @inferred(DynamicPPL.evaluate!!(model, varinfo_linked, context)) true - ) + end end end end diff --git a/test/runtests.jl b/test/runtests.jl index 806ab8223..9596067eb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -40,7 +40,7 @@ include("test_util.jl") @testset "interface" begin include("utils.jl") include("compiler.jl") - include("varnamevector.jl") + include("varnamedvector.jl") include("varinfo.jl") include("simple_varinfo.jl") include("model.jl") diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 74dcdd842..2684774bf 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -56,13 +56,41 @@ @test !haskey(svi, @varname(m.a[2])) @test !haskey(svi, @varname(m.a.b)) end + + @testset "VarNamedVector" begin + svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m), 1.0)) + @test getlogp(svi) == 0.0 + @test haskey(svi, @varname(m)) + @test !haskey(svi, @varname(m[1])) + + svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m), [1.0])) + @test getlogp(svi) == 0.0 + @test haskey(svi, @varname(m)) + @test haskey(svi, @varname(m[1])) + @test !haskey(svi, @varname(m[2])) + @test svi[@varname(m)][1] == svi[@varname(m[1])] + + svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a), [1.0])) + @test haskey(svi, @varname(m)) + @test haskey(svi, @varname(m.a)) + @test haskey(svi, @varname(m.a[1])) + @test !haskey(svi, @varname(m.a[2])) + @test !haskey(svi, @varname(m.a.b)) + # The implementation of haskey and getvalue fo VarNamedVector is incomplete, the + # next test is here to remind of us that. + svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a.b), [1.0])) + @test_broken (svi[@varname(m.a.b.c.d)]; true) + end end @testset "link!! & invlink!! on $(nameof(model))" for model in DynamicPPL.TestUtils.DEMO_MODELS values_constrained = DynamicPPL.TestUtils.rand_prior_true(model) @testset "$(typeof(vi))" for vi in ( - SimpleVarInfo(Dict()), SimpleVarInfo(values_constrained), VarInfo(model) + SimpleVarInfo(Dict()), + SimpleVarInfo(values_constrained), + SimpleVarInfo(VarNamedVector()), + VarInfo(model), ) for vn in DynamicPPL.TestUtils.varnames(model) vi = DynamicPPL.setindex!!(vi, get(values_constrained, vn), vn) @@ -115,12 +143,19 @@ # to see whether this is the case. svi_nt = SimpleVarInfo(DynamicPPL.TestUtils.rand_prior_true(model)) svi_dict = SimpleVarInfo(VarInfo(model), Dict) + vnv = VarNamedVector() + for (k, v) in pairs(DynamicPPL.TestUtils.rand_prior_true(model)) + vnv = push!!(vnv, VarName{k}(), v) + end + svi_vnv = SimpleVarInfo(vnv) @testset "$(nameof(typeof(DynamicPPL.values_as(svi))))" for svi in ( svi_nt, svi_dict, + svi_vnv, DynamicPPL.settrans!!(svi_nt, true), DynamicPPL.settrans!!(svi_dict, true), + DynamicPPL.settrans!!(svi_vnv, true), ) # RandOM seed is set in each `@testset`, so we need to sample # a new realization for `m` here. @@ -195,36 +230,39 @@ model = DynamicPPL.TestUtils.demo_dynamic_constraint() # Initialize. - svi = DynamicPPL.settrans!!(SimpleVarInfo(), true) - svi = last(DynamicPPL.evaluate!!(model, svi, SamplingContext())) - - # Sample with large variations in unconstrained space. - for i in 1:10 - for vn in keys(svi) - svi = DynamicPPL.setindex!!(svi, 10 * randn(), vn) - end - retval, svi = DynamicPPL.evaluate!!(model, svi, DefaultContext()) - @test retval.m == svi[@varname(m)] # `m` is unconstrained - @test retval.x ≠ svi[@varname(x)] # `x` is constrained depending on `m` + svi_nt = DynamicPPL.settrans!!(SimpleVarInfo(), true) + svi_nt = last(DynamicPPL.evaluate!!(model, svi_nt, SamplingContext())) + svi_vnv = DynamicPPL.settrans!!(SimpleVarInfo(VarNamedVector()), true) + svi_vnv = last(DynamicPPL.evaluate!!(model, svi_vnv, SamplingContext())) + + for svi in (svi_nt, svi_vnv) + # Sample with large variations in unconstrained space. + for i in 1:10 + for vn in keys(svi) + svi = DynamicPPL.setindex!!(svi, 10 * randn(), vn) + end + retval, svi = DynamicPPL.evaluate!!(model, svi, DefaultContext()) + @test retval.m == svi[@varname(m)] # `m` is unconstrained + @test retval.x ≠ svi[@varname(x)] # `x` is constrained depending on `m` + + retval_unconstrained, lp_true = DynamicPPL.TestUtils.logjoint_true_with_logabsdet_jacobian( + model, retval.m, retval.x + ) - retval_unconstrained, lp_true = DynamicPPL.TestUtils.logjoint_true_with_logabsdet_jacobian( - model, retval.m, retval.x - ) + # Realizations from model should all be equal to the unconstrained realization. + for vn in DynamicPPL.TestUtils.varnames(model) + @test get(retval_unconstrained, vn) ≈ svi[vn] rtol = 1e-6 + end - # Realizations from model should all be equal to the unconstrained realization. - for vn in DynamicPPL.TestUtils.varnames(model) - @test get(retval_unconstrained, vn) ≈ svi[vn] rtol = 1e-6 + # `getlogp` should be equal to the logjoint with log-absdet-jac correction. + lp = getlogp(svi) + @test lp ≈ lp_true end - - # `getlogp` should be equal to the logjoint with log-absdet-jac correction. - lp = getlogp(svi) - @test lp ≈ lp_true end end @testset "Static transformation" begin model = DynamicPPL.TestUtils.demo_static_transformation() - priors = extract_priors(model) varinfos = DynamicPPL.TestUtils.setup_varinfos( model, DynamicPPL.TestUtils.rand_prior_true(model), [@varname(s), @varname(m)] diff --git a/test/test_util.jl b/test/test_util.jl index 021da2598..0c7949e48 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -85,12 +85,14 @@ Return string representing a short description of `vi`. short_varinfo_name(vi::DynamicPPL.ThreadSafeVarInfo) = "threadsafe($(short_varinfo_name(vi.varinfo)))" function short_varinfo_name(vi::TypedVarInfo) - DynamicPPL.has_varnamevector(vi) && return "TypedVarInfo with VarNameVector" + DynamicPPL.has_varnamedvector(vi) && return "TypedVarInfo with VarNamedVector" return "TypedVarInfo" end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" +short_varinfo_name(::VectorVarInfo) = "VectorVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" +short_varinfo_name(::SimpleVarInfo{<:VarNamedVector}) = "SimpleVarInfo{<:VarNamedVector}" # convenient functions for testing model.jl # function to modify the representation of values based on their length diff --git a/test/varinfo.jl b/test/varinfo.jl index 6750db416..a00935279 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -19,7 +19,7 @@ struct MySAlg end DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) @testset "varinfo.jl" begin - @testset "TypedVarInfo" begin + @testset "TypedVarInfo with Metadata" begin @model gdemo(x, y) = begin s ~ InverseGamma(2, 3) m ~ truncated(Normal(0.0, sqrt(s)), 0.0, 2.0) @@ -28,7 +28,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) end model = gdemo(1.0, 2.0) - vi = VarInfo() + vi = VarInfo(DynamicPPL.Metadata()) model(vi, SampleFromUniform()) tvi = TypedVarInfo(vi) @@ -51,6 +51,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) end end end + @testset "Base" begin # Test Base functions: # string, Symbol, ==, hash, in, keys, haskey, isempty, push!!, empty!!, @@ -120,6 +121,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) test_base!!(TypedVarInfo(vi)) test_base!!(SimpleVarInfo()) test_base!!(SimpleVarInfo(Dict())) + test_base!!(SimpleVarInfo(VarNamedVector())) end @testset "flags" begin # Test flag setting: @@ -141,12 +143,12 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) unset_flag!(vi, vn_x, "del") @test !is_flagged(vi, vn_x, "del") end - vi = VarInfo() + vi = VarInfo(DynamicPPL.Metadata()) test_varinfo!(vi) test_varinfo!(empty!!(TypedVarInfo(vi))) end @testset "setgid!" begin - vi = VarInfo() + vi = VarInfo(DynamicPPL.Metadata()) meta = vi.metadata vn = @varname x dist = Normal(0, 1) @@ -196,8 +198,12 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) m_vns = model == model_uv ? [@varname(m[i]) for i in 1:5] : @varname(m) s_vns = @varname(s) - vi_typed = VarInfo(model) - vi_untyped = VarInfo() + # TODO(mhauru) Should add similar tests for VarNamedVector. These ones only apply + # to Metadata. + vi_typed = VarInfo( + model, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata + ) + vi_untyped = VarInfo(DynamicPPL.Metadata()) model(vi_untyped, SampleFromPrior()) for vi in [vi_untyped, vi_typed] @@ -338,6 +344,14 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) x = f(DynamicPPL.getindex_internal(vi, vn)) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) + + ## `SimpleVarInfo{<:VarNamedVector}` + vi = DynamicPPL.settrans!!(SimpleVarInfo(VarNamedVector()), true) + # Sample in unconstrained space. + vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) + f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) + x = f(DynamicPPL.getindex_internal(vi, vn)) + @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) end @testset "values_as" begin @@ -409,8 +423,8 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end - if DynamicPPL.has_varnamevector(varinfo) && mutating - # NOTE: Can't handle mutating `link!` and `invlink!` `VarNameVector`. + if DynamicPPL.has_varnamedvector(varinfo) && mutating + # NOTE: Can't handle mutating `link!` and `invlink!` `VarNamedVector`. @test_broken false continue end @@ -642,6 +656,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) varinfo_left = VarInfo(model_left) varinfo_right = VarInfo(model_right) + varinfo_right = DynamicPPL.settrans!!(varinfo_right, true, @varname(x)) varinfo_merged = merge(varinfo_left, varinfo_right) vns = [@varname(x), @varname(y), @varname(z)] @@ -649,13 +664,18 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) # Right has precedence. @test varinfo_merged[@varname(x)] == varinfo_right[@varname(x)] - @test DynamicPPL.getdist(varinfo_merged, @varname(x)) isa Normal + @test DynamicPPL.istrans(varinfo_merged, @varname(x)) end end @testset "VarInfo with selectors" begin @testset "$(model.f)" for model in DynamicPPL.TestUtils.DEMO_MODELS - varinfo = VarInfo(model) + varinfo = VarInfo( + model, + DynamicPPL.SampleFromPrior(), + DynamicPPL.DefaultContext(), + DynamicPPL.Metadata, + ) selector = DynamicPPL.Selector() spl = Sampler(MySAlg(), model, selector) diff --git a/test/varnamevector.jl b/test/varnamedvector.jl similarity index 89% rename from test/varnamevector.jl rename to test/varnamedvector.jl index 6b7acfbeb..604adc521 100644 --- a/test/varnamevector.jl +++ b/test/varnamedvector.jl @@ -7,7 +7,7 @@ decrease_size_for_test(x::Real) = x decrease_size_for_test(x::AbstractVector) = first(x) decrease_size_for_test(x::AbstractArray) = first(eachslice(x; dims=1)) -function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) +function need_varnames_relaxation(vnv::VarNamedVector, vn::VarName, val) if isconcretetype(eltype(vnv.varnames)) # If the container is concrete, we need to make sure that the varname types match. # E.g. if `vnv.varnames` has `eltype` `VarName{:x, IndexLens{Tuple{Int64}}}` then @@ -20,22 +20,22 @@ function need_varnames_relaxation(vnv::VarNameVector, vn::VarName, val) return false end -function need_varnames_relaxation(vnv::VarNameVector, vns, vals) +function need_varnames_relaxation(vnv::VarNamedVector, vns, vals) return any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) end -function need_values_relaxation(vnv::VarNameVector, vn::VarName, val) +function need_values_relaxation(vnv::VarNamedVector, vn::VarName, val) if isconcretetype(eltype(vnv.vals)) return promote_type(eltype(vnv.vals), eltype(val)) != eltype(vnv.vals) end return false end -function need_values_relaxation(vnv::VarNameVector, vns, vals) +function need_values_relaxation(vnv::VarNamedVector, vns, vals) return any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) end -function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) +function need_transforms_relaxation(vnv::VarNamedVector, vn::VarName, val) return if isconcretetype(eltype(vnv.transforms)) # If the container is concrete, we need to make sure that the sizes match. # => If the sizes don't match, we need to relax the container type. @@ -50,13 +50,13 @@ function need_transforms_relaxation(vnv::VarNameVector, vn::VarName, val) false end end -function need_transforms_relaxation(vnv::VarNameVector, vns, vals) +function need_transforms_relaxation(vnv::VarNamedVector, vns, vals) return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) end """ - relax_container_types(vnv::VarNameVector, vn::VarName, val) - relax_container_types(vnv::VarNameVector, vns, val) + relax_container_types(vnv::VarNamedVector, vn::VarName, val) + relax_container_types(vnv::VarNamedVector, vns, val) Relax the container types of `vnv` if necessary to accommodate `vn` and `val`. @@ -74,10 +74,10 @@ Similarly: transformations type in `vnv`, then the underlying transformation type will be changed to `Any`. """ -function relax_container_types(vnv::VarNameVector, vn::VarName, val) +function relax_container_types(vnv::VarNamedVector, vn::VarName, val) return relax_container_types(vnv, [vn], [val]) end -function relax_container_types(vnv::VarNameVector, vns, vals) +function relax_container_types(vnv::VarNamedVector, vns, vals) if need_varnames_relaxation(vnv, vns, vals) varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) varnames_new = convert(Vector{VarName}, vnv.varnames) @@ -98,31 +98,30 @@ function relax_container_types(vnv::VarNameVector, vns, vals) vnv.vals end - return VarNameVector( + return VarNamedVector( varname_to_index_new, varnames_new, vnv.ranges, vals_new, transforms_new, - vnv.is_transformed, + vnv.is_unconstrained, vnv.num_inactive, - vnv.metadata, ) end -@testset "VarNameVector" begin - # Need to test element-related operations: +@testset "VarNamedVector" begin + # Test element-related operations: # - `getindex` # - `setindex!` # - `push!` # - `update!` # - # And these should all be tested for different types of values: + # And these are all be tested for different types of values: # - scalar # - vector # - matrix - # Need to test operations on `VarNameVector`: + # Test operations on `VarNamedVector`: # - `empty!` # - `iterate` # - `convert` to @@ -143,12 +142,12 @@ end @testset "constructor: no args" begin # Empty. - vnv = VarNameVector() + vnv = VarNamedVector() @test isempty(vnv) @test eltype(vnv) == Real # Empty with types. - vnv = VarNameVector{VarName,Float64}() + vnv = VarNamedVector{VarName,Float64}() @test isempty(vnv) @test eltype(vnv) == Float64 end @@ -157,17 +156,17 @@ end @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter val_left = test_pairs[vn_left] val_right = test_pairs[vn_right] - vnv_base = VarNameVector([vn_left, vn_right], [val_left, val_right]) + vnv_base = VarNamedVector([vn_left, vn_right], [val_left, val_right]) # We'll need the transformations later. - # TODO: Should we test other transformations than just `FromVec`? + # TODO: Should we test other transformations than just `ReshapeTransform`? from_vec_left = DynamicPPL.from_vec_transform(val_left) from_vec_right = DynamicPPL.from_vec_transform(val_right) to_vec_left = inverse(from_vec_left) to_vec_right = inverse(from_vec_right) # Compare to alternative constructors. - vnv_from_dict = VarNameVector( + vnv_from_dict = VarNamedVector( OrderedDict(vn_left => val_left, vn_right => val_right) ) @test vnv_base == vnv_from_dict @@ -302,6 +301,7 @@ end end end end + @testset "update!" begin vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn" for vn in test_vns @@ -375,7 +375,7 @@ end @testset "deterministic" begin n = 5 vn = @varname(x) - vnv = VarNameVector(OrderedDict(vn => [true])) + vnv = VarNamedVector(OrderedDict(vn => [true])) @test !DynamicPPL.has_inactive(vnv) # Growing should not create inactive ranges. for i in 1:n @@ -401,7 +401,7 @@ end @testset "random" begin n = 5 vn = @varname(x) - vnv = VarNameVector(OrderedDict(vn => [true])) + vnv = VarNamedVector(OrderedDict(vn => [true])) @test !DynamicPPL.has_inactive(vnv) # Insert a bunch of random-length vectors. @@ -421,9 +421,23 @@ end end end end + + @testset "subset" begin + vnv = VarNamedVector(test_pairs) + @test subset(vnv, test_vns) == vnv + @test subset(vnv, VarName[]) == VarNamedVector() + @test merge(subset(vnv, test_vns[1:3]), subset(vnv, test_vns[4:end])) == vnv + + # Test that subset preseres transformations and unconstrainedness. + vn = @varname(t[1]) + vns = vcat(test_vns, [vn]) + push!(vnv, vn, 2.0, x -> x^2) + vnv.is_unconstrained[vnv.varname_to_index[vn]] = true + @test subset(vnv, vns) == vnv + end end -@testset "VarInfo + VarNameVector" begin +@testset "VarInfo + VarNamedVector" begin models = DynamicPPL.TestUtils.DEMO_MODELS @testset "$(model.f)" for model in models # NOTE: Need to set random seed explicitly to avoid using the same seed @@ -435,8 +449,8 @@ end varinfos = DynamicPPL.TestUtils.setup_varinfos( model, value_true, varnames; include_threadsafe=false ) - # Filter out those which are not based on `VarNameVector`. - varinfos = filter(DynamicPPL.has_varnamevector, varinfos) + # Filter out those which are not based on `VarNamedVector`. + varinfos = filter(DynamicPPL.has_varnamedvector, varinfos) # Get the true log joint. logp_true = DynamicPPL.TestUtils.logjoint_true(model, value_true...) From 1e4efe6fa37490988a96ee55320eb96a1ee484eb Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 3 Sep 2024 10:08:32 +0100 Subject: [PATCH 160/209] Bump Bijectors dependecy --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c3f20180f..b7d1589f0 100644 --- a/Project.toml +++ b/Project.toml @@ -46,7 +46,7 @@ AbstractMCMC = "5" AbstractPPL = "0.8.4" Accessors = "0.1" BangBang = "0.4.1" -Bijectors = "0.13.9" +Bijectors = "0.13.18" ChainRulesCore = "1" Compat = "4" ConstructionBase = "1.5.4" From 3ee9832c05e2c62927106f1af16b04e988842b31 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 3 Sep 2024 10:08:47 +0100 Subject: [PATCH 161/209] Remove dead TODO note --- src/varinfo.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index a6a5c0400..59894c681 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -686,8 +686,6 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) return metadata end -# TODO(mhauru) Isn't this infinite recursion? Shouldn't rather change the `transforms` -# field? function settrans!!(vnv::VarNamedVector, trans::Bool, vn::VarName) settrans!(vnv, trans, vn) return vnv From 26753e9da8bea925514079c2adb2ebaff29f630d Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 3 Sep 2024 11:27:45 +0100 Subject: [PATCH 162/209] Remove old TODOs, improve VNV invlinking --- src/varinfo.jl | 63 +++++++++++--------------------------------------- 1 file changed, 13 insertions(+), 50 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 59894c681..a4f87c146 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -592,8 +592,6 @@ getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, # TODO(torfjelde): An alternative is to implement `view` directly instead. getindex_internal(md::Metadata, vn::VarName) = getindex(md.vals, getrange(md, vn)) # HACK: We shouldn't need this -# TODO(mhauru) This seems to return an array always for VarNamedVector, but a scalar for -# Metadata. What's the right thing to do here? getindex_internal(vnv::VarNamedVector, vn::VarName) = getindex(vnv.vals, getrange(vnv, vn)) function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) @@ -1404,7 +1402,6 @@ function _link_metadata!( end # Otherwise, we derive the transformation from the distribution. - # TODO(mhauru) Could move the mutation outside of the map, just for style. is_unconstrained[getidx(metadata, vn)] = true internal_to_linked_internal_transform(varinfo, vn, dists[vn]) end @@ -1464,7 +1461,7 @@ end function _invlink(model::Model, varinfo::VarInfo, spl::AbstractSampler) varinfo = deepcopy(varinfo) return VarInfo( - _invlink_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), + _invlink_metadata!!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo)), ) @@ -1488,7 +1485,7 @@ end vals = Expr(:tuple) for f in names if inspace(f, space) || length(space) == 0 - push!(vals.args, :(_invlink_metadata!(model, varinfo, metadata.$f, vns.$f))) + push!(vals.args, :(_invlink_metadata!!(model, varinfo, metadata.$f, vns.$f))) else push!(vals.args, :(metadata.$f)) end @@ -1496,7 +1493,7 @@ end return :(NamedTuple{$names}($vals)) end -function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, target_vns) +function _invlink_metadata!!(::Model, varinfo::VarInfo, metadata::Metadata, target_vns) vns = metadata.vns # Construct the new transformed values, and keep track of their lengths. @@ -1545,53 +1542,20 @@ function _invlink_metadata!(::Model, varinfo::VarInfo, metadata::Metadata, targe ) end -function _invlink_metadata!( +function _invlink_metadata!!( model::Model, varinfo::VarInfo, metadata::VarNamedVector, target_vns ) - # HACK: We ignore `target_vns` here. - # TODO: Make use of `update!` to aovid copying values. - # => Only need to allocate for transformations. - - vns = keys(metadata) - is_unconstrained = copy(metadata.is_unconstrained) - - # Compute the transformed values. - xs = map(vns) do vn - f = gettransform(metadata, vn) - y = getindex_internal(metadata, vn) - # No need to use `with_reconstruct` as `f` will include this. - x, logjac = with_logabsdet_jacobian(f, y) - # Accumulate the log-abs-det jacobian correction. + vns = target_vns === nothing ? keys(metadata) : target_vns + for vn in vns + transform = gettransform(metadata, vn) + old_val = getindex_raw(metadata, vn) + new_val, logjac = with_logabsdet_jacobian(transform, old_val) + # TODO(mhauru) We are calling a !! function but ignoring the return value. acclogp!!(varinfo, -logjac) - # Mark as no longer transformed. - is_unconstrained[getidx(metadata, vn)] = false - # Return the transformed value. - return x - end - # Compose the transformations to form a full transformation from - # unconstrained vector representation to constrained space. - transforms = map(from_vec_transform, xs) - # Convert to vector representation. - xvecs = map(tovec, xs) - - # Determine new ranges. - ranges_new = similar(metadata.ranges) - offset = 0 - for (i, v) in enumerate(xvecs) - r_start, r_end = offset + 1, length(v) + offset - offset = r_end - ranges_new[i] = r_start:r_end + metadata = update!!(metadata, vn, new_val) + settrans!(metadata, false, vn) end - - # Now we just create a new metadata with the new `vals` and `ranges`. - return VarNamedVector( - metadata.varname_to_index, - metadata.varnames, - ranges_new, - reduce(vcat, xvecs), - transforms, - is_unconstrained, - ) + return metadata end """ @@ -1698,7 +1662,6 @@ function getindex(vi::VarInfo, vns::Vector{<:VarName}) end function getindex(vi::VarInfo, vns::Vector{<:VarName}, dist::Distribution) - # TODO(mhauru) Does this ever get called? @assert haskey(vi, vns[1]) "[DynamicPPL] attempted to replay unexisting variables in VarInfo" vals_linked = mapreduce(vcat, vns) do vn getindex(vi, vn, dist) From ea18e1f8660f69b6ace385f9a3433dbe3771f995 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 4 Sep 2024 16:08:16 +0100 Subject: [PATCH 163/209] Fix from_vec_transform for 0-dim arrays --- src/utils.jl | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 75bcef327..7d95eef82 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -225,6 +225,22 @@ invlink_transform(dist) = inverse(link_transform(dist)) # Helper functions for vectorize/reconstruct values # ##################################################### +""" + UnwrapSingletonTransform + +A transformation that unwraps a singleton vector into a scalar. +""" +struct UnwrapSingletonTransform <: Bijectors.Bijector end + +(f::UnwrapSingletonTransform)(x) = only(x) + +Bijectors.with_logabsdet_jacobian(f::UnwrapSingletonTransform, x) = (f(x), 0) +function Bijectors.with_logabsdet_jacobian( + ::Bijectors.Inverse{<:UnwrapSingletonTransform}, x +) + return (tovec(x), 0) +end + """ ReshapeTransform(size::Size) @@ -240,9 +256,7 @@ end ReshapeTransform(x::Union{Real,AbstractArray}) = ReshapeTransform(size(x)) # TODO: Should we materialize the `reshape`? -(f::ReshapeTransform)(x::AbstractVector) = reshape(x, f.size) -(f::ReshapeTransform{Tuple{}})(x::AbstractVector) = only(x) -# TODO: Specialize for `Tuple{<:Any}` since this correspond to a `Vector`. +(f::ReshapeTransform)(x) = reshape(x, f.size) Bijectors.with_logabsdet_jacobian(f::ReshapeTransform, x) = (f(x), 0) # We want to use the inverse of `ReshapeTransform` so it preserves the size information. @@ -262,8 +276,9 @@ Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = Return the transformation from the vector representation of `x` to original representation. """ -from_vec_transform(x::Union{Real,AbstractArray}) = from_vec_transform_for_size(size(x)) +from_vec_transform(x::AbstractArray) = from_vec_transform_for_size(size(x)) from_vec_transform(C::Cholesky) = ToChol(C.uplo) ∘ ReshapeTransform(size(C.UL)) +from_vec_transform(::Real) = UnwrapSingletonTransform() """ from_vec_transform_for_size(sz::Tuple) @@ -272,7 +287,7 @@ Return the transformation from the vector representation of a realization of siz original representation. """ from_vec_transform_for_size(sz::Tuple) = ReshapeTransform(sz) -from_vec_transform_for_size(::Tuple{()}) = ReshapeTransform(()) +# TODO(mhauru) Is the below used? If not, this function can be removed. from_vec_transform_for_size(::Tuple{<:Any}) = identity """ @@ -281,6 +296,7 @@ from_vec_transform_for_size(::Tuple{<:Any}) = identity Return the transformation from the vector representation of a realization from distribution `dist` to the original representation compatible with `dist`. """ +from_vec_transform(::UnivariateDistribution) = UnwrapSingletonTransform() from_vec_transform(dist::Distribution) = from_vec_transform_for_size(size(dist)) from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ ReshapeTransform(size(dist)) @@ -310,6 +326,17 @@ function from_linked_vec_transform(dist::Distribution) return f_invlink ∘ f_vec end +# UnivariateDistributions need to be handled as a special case, because size(dist) is (), +# which makes the usual machinery think we are dealing with a 0-dim array, whereas in +# actuality we are dealing with a scalar. +# TODO(mhauru) Hopefully all this can go once the old Gibbs sampler is removed and +# VarNamedVector takes over from Metadata. +function from_linked_vec_transform(dist::UnivariateDistribution) + f_invlink = invlink_transform(dist) + f_vec = from_vec_transform(inverse(f_invlink), size(dist)) + return UnwrapSingletonTransform() ∘ f_invlink ∘ f_vec +end + # Specializations that circumvent the `from_vec_transform` machinery. function from_linked_vec_transform(dist::LKJCholesky) return inverse(Bijectors.VecCholeskyBijector(dist.uplo)) From ffbf2adafad180f9ff42cbf422a7f28d5b1e6256 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 4 Sep 2024 16:19:24 +0100 Subject: [PATCH 164/209] Fix unflatten for VarInfo --- src/varinfo.jl | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index a4f87c146..fc4fa239a 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -200,7 +200,42 @@ VarInfo(model::Model, args...) = VarInfo(Random.default_rng(), model, args...) unflatten(vi::VarInfo, x::AbstractVector) = unflatten(vi, SampleFromPrior(), x) # TODO: deprecate. -unflatten(vi::VarInfo, spl::AbstractSampler, x::AbstractVector) = VarInfo(vi, spl, x) +function unflatten(vi::VarInfo, spl::AbstractSampler, x::AbstractVector) + md = unflatten(vi.metadata, spl, x) + return VarInfo(md, Base.RefValue{eltype(x)}(getlogp(vi)), Ref(get_num_produce(vi))) +end + +# The Val(getspace(spl)) is used to dispatch into the below generated function. +function unflatten(metadata::NamedTuple, spl::AbstractSampler, x::AbstractVector) + return unflatten(metadata, Val(getspace(spl)), x) +end + +@generated function unflatten( + metadata::NamedTuple{names}, ::Val{space}, x +) where {names,space} + exprs = [] + offset = :(0) + for f in names + mdf = :(metadata.$f) + if inspace(f, space) || length(space) == 0 + len = :(sum(length, $mdf.ranges)) + push!(exprs, :($f = unflatten($mdf, x[($offset + 1):($offset + $len)]))) + offset = :($offset + $len) + else + push!(exprs, :($f = $mdf)) + end + end + length(exprs) == 0 && return :(NamedTuple()) + return :($(exprs...),) +end + +# For Metadata unflatten and replace_values are the same. For VarNamedVector they are not. +function unflatten(md::Metadata, x::AbstractVector) + return replace_values(md, x) +end +function unflatten(md::Metadata, spl::AbstractSampler, x::AbstractVector) + return replace_values(md, spl, x) +end # without AbstractSampler function VarInfo(rng::Random.AbstractRNG, model::Model, context::AbstractContext) From f077f4ae6c6c43f04b7b270d01443605dedf5d01 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 4 Sep 2024 16:20:03 +0100 Subject: [PATCH 165/209] Fix some VarInfo index getters --- src/varinfo.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index fc4fa239a..002e869aa 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -627,7 +627,7 @@ getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, # TODO(torfjelde): An alternative is to implement `view` directly instead. getindex_internal(md::Metadata, vn::VarName) = getindex(md.vals, getrange(md, vn)) # HACK: We shouldn't need this -getindex_internal(vnv::VarNamedVector, vn::VarName) = getindex(vnv.vals, getrange(vnv, vn)) +getindex_internal(vnv::VarNamedVector, vn::VarName) = getindex_raw(vnv, vn) function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) return mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) @@ -664,7 +664,7 @@ function getall(md::Metadata) Base.Fix1(getindex_internal, md), vcat, md.vns; init=similar(md.vals, 0) ) end -getall(vnv::VarNamedVector) = vnv.vals +getall(vnv::VarNamedVector) = getindex_raw(vnv, Colon()) """ setall!(vi::VarInfo, val) From e27af809b2b3783f9c8bc81d9de93f42823176df Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 4 Sep 2024 16:27:09 +0100 Subject: [PATCH 166/209] Change how VNV handles transformations, and other VNV stuff --- src/varinfo.jl | 75 ++++++++++-------------------------- src/varnamedvector.jl | 87 ++++++++++++++++++++++++++++++------------ test/varnamedvector.jl | 8 ++-- 3 files changed, 87 insertions(+), 83 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 002e869aa..58a539c42 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1339,7 +1339,7 @@ function _link( ) varinfo = deepcopy(varinfo) return VarInfo( - _link_metadata!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), + _link_metadata!!(model, varinfo, varinfo.metadata, _getvns_link(varinfo, spl)), Base.Ref(getlogp(varinfo)), Ref(get_num_produce(varinfo)), ) @@ -1363,7 +1363,7 @@ end vals = Expr(:tuple) for f in names if inspace(f, space) || length(space) == 0 - push!(vals.args, :(_link_metadata!(model, varinfo, metadata.$f, vns.$f))) + push!(vals.args, :(_link_metadata!!(model, varinfo, metadata.$f, vns.$f))) else push!(vals.args, :(metadata.$f)) end @@ -1371,7 +1371,7 @@ end return :(NamedTuple{$names}($vals)) end -function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, target_vns) +function _link_metadata!!(model::Model, varinfo::VarInfo, metadata::Metadata, target_vns) vns = metadata.vns # Construct the new transformed values, and keep track of their lengths. @@ -1419,62 +1419,27 @@ function _link_metadata!(model::Model, varinfo::VarInfo, metadata::Metadata, tar ) end -function _link_metadata!( +function _link_metadata!!( model::Model, varinfo::VarInfo, metadata::VarNamedVector, target_vns ) - # HACK: We ignore `target_vns` here. - vns = keys(metadata) - # Need to extract the priors from the model. + vns = target_vns === nothing ? keys(metadata) : target_vns dists = extract_priors(model, varinfo) - - is_unconstrained = copy(metadata.is_unconstrained) - - # Construct the linking transformations. - link_transforms = map(vns) do vn - # If `vn` is not part of `target_vns`, the `identity` transformation is used. - if (target_vns !== nothing && vn ∉ target_vns) - return identity - end - - # Otherwise, we derive the transformation from the distribution. - is_unconstrained[getidx(metadata, vn)] = true - internal_to_linked_internal_transform(varinfo, vn, dists[vn]) - end - # Compute the transformed values. - ys = map(vns, link_transforms) do vn, f - x = getindex_internal(metadata, vn) - y, logjac = with_logabsdet_jacobian(f, x) - # Accumulate the log-abs-det jacobian correction. - acclogp!!(varinfo, -logjac) - # Return the transformed value. - return y - end - # Extract the from-vec transformations. - fromvec_transforms = map(from_vec_transform, ys) - # Compose the transformations to form a full transformation from - # unconstrained vector representation to constrained space. - transforms = map(∘, map(inverse, link_transforms), fromvec_transforms) - # Convert to vector representation. - yvecs = map(tovec, ys) - - # Determine new ranges. - ranges_new = similar(metadata.ranges) - offset = 0 - for (i, v) in enumerate(yvecs) - r_start, r_end = offset + 1, length(v) + offset - offset = r_end - ranges_new[i] = r_start:r_end + for vn in vns + # First transform from however the variable is stored in vnv to the model + # representation. + transform_to_orig = gettransform(metadata, vn) + val_old = getindex_raw(metadata, vn) + val_orig, logjac1 = with_logabsdet_jacobian(transform_to_orig, val_old) + # Then transform from the model representation to the linked representation. + transform_from_linked = from_linked_vec_transform(dists[vn]) + transform_to_linked = inverse(transform_from_linked) + val_new, logjac2 = with_logabsdet_jacobian(transform_to_linked, val_orig) + # TODO(mhauru) We are calling a !! function but ignoring the return value. + acclogp!!(varinfo, -logjac1 - logjac2) + metadata = update!!(metadata, vn, val_new, transform_from_linked) + settrans!(metadata, true, vn) end - - # Now we just create a new metadata with the new `vals` and `ranges`. - return VarNamedVector( - metadata.varname_to_index, - metadata.varnames, - ranges_new, - reduce(vcat, yvecs), - transforms, - is_unconstrained, - ) + return metadata end function invlink( diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index e9a7126ae..509bf5480 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -93,8 +93,9 @@ struct VarNamedVector{ num_vals = mapreduce(length, (+), ranges; init=0) + sum(values(num_inactive)) if num_vals != length(vals) msg = """ - The total number of elements in `vals` does not match the sum of the \ - lengths of the ranges and the number of inactive entries.""" + The total number of elements in `vals` ($(length(vals))) does not match the \ + sum of the lengths of the ranges and the number of inactive entries \ + ($(num_vals)).""" throw(ArgumentError(msg)) end @@ -176,11 +177,15 @@ function VarNamedVector(varnames, vals) return VarNamedVector(collect_maybe(varnames), collect_maybe(vals)) end function VarNamedVector( - varnames::AbstractVector, vals::AbstractVector, transforms=map(from_vec_transform, vals) + varnames::AbstractVector, + vals::AbstractVector, + transforms=fill(identity, length(varnames)), ) # Convert `vals` into a vector of vectors. vals_vecs = map(tovec, vals) - + transforms = map( + (t, val) -> _compose_no_identity(t, from_vec_transform(val)), transforms, vals + ) # TODO: Is this really the way to do this? if !(eltype(varnames) <: VarName) varnames = convert(Vector{VarName}, varnames) @@ -515,6 +520,15 @@ function unflatten(vnv::VarNamedVector, vals::AbstractVector) ) end +# TODO(mhauru) To be removed once the old Gibbs sampler is removed. +function unflatten(vnv::VarNamedVector, spl::AbstractSampler, vals::AbstractVector) + if length(getspace(spl)) > 0 + msg = "Selecting values in a VarNamedVector with a space is not supported." + throw(ArgumentError(msg)) + end + return unflatten(vnv, vals) +end + function Base.merge(left_vnv::VarNamedVector, right_vnv::VarNamedVector) # Return early if possible. isempty(left_vnv) && return deepcopy(right_vnv) @@ -677,22 +691,38 @@ function nextrange(vnv::VarNamedVector, x) return (offset + 1):(offset + length(x)) end +""" + _compose_no_identity(f, g) + +Like `f ∘ g`, but if `f` or `g` is `identity` it is omitted. + +This helps avoid trivial cases of `ComposedFunction` that would cause unnecessary type +conflicts. +""" +_compose_no_identity(f, g) = f ∘ g +_compose_no_identity(::typeof(identity), g) = g +_compose_no_identity(f, ::typeof(identity)) = f +_compose_no_identity(::typeof(identity), ::typeof(identity)) = identity + """ push!(vnv::VarNamedVector, vn::VarName, val[, transform]) Add a variable with given value to `vnv`. -By default `transform` is the one that converts the value to a vector, which is how it is -stored in `vnv`. +`transform` should be a function that converts `val` to the original representation, by +default it's `identity`. """ -function Base.push!( - vnv::VarNamedVector, vn::VarName, val, transform=from_vec_transform(val) -) +function Base.push!(vnv::VarNamedVector, vn::VarName, val, transform=identity) # Error if we already have the variable. haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) # NOTE: We need to compute the `nextrange` BEFORE we start mutating the underlying # storage. - val_vec = tovec(val) + if !(val isa AbstractVector) + val_vec = tovec(val) + transform = _compose_no_identity(transform, from_vec_transform(val)) + else + val_vec = val + end r_new = nextrange(vnv, val_vec) vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 push!(vnv.varnames, vn) @@ -707,7 +737,7 @@ end # Remove this method as soon as possible. function Base.push!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) f = from_vec_transform(dist) - return push!(vnv, vn, val, f) + return push!(vnv, vn, tovec(val), f) end """ @@ -780,19 +810,19 @@ then `tighten_types(vnv)` will have element type `Float64`. function tighten_types(vnv::VarNamedVector) return VarNamedVector( OrderedDict(vnv.varname_to_index...), - [vnv.varnames...], + map(identity, vnv.varnames), copy(vnv.ranges), - [vnv.vals...], - [vnv.transforms...], + map(identity, vnv.vals), + map(identity, vnv.transforms), copy(vnv.is_unconstrained), copy(vnv.num_inactive), ) end -function BangBang.push!!( - vnv::VarNamedVector, vn::VarName, val, transform=from_vec_transform(val) -) - vnv = loosen_types!!(vnv, typeof(vn), typeof(transform)) +function BangBang.push!!(vnv::VarNamedVector, vn::VarName, val, transform=identity) + vnv = loosen_types!!( + vnv, typeof(vn), typeof(_compose_no_identity(transform, from_vec_transform(val))) + ) push!(vnv, vn, val, transform) return vnv end @@ -801,7 +831,7 @@ end # Remove this method as soon as possible. function BangBang.push!!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) f = from_vec_transform(dist) - return push!!(vnv, vn, val, f) + return push!!(vnv, vn, tovec(val), f) end """ @@ -833,17 +863,22 @@ Either add a new entry or update existing entry for `vn` in `vnv` with the value If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). -By default `transform` is the one that converts the value to a vector, which is how it is -stored in `vnv`. +`transform` should be a function that converts `val` to the original representation, by +default it's `identity`. """ -function update!(vnv::VarNamedVector, vn::VarName, val, transform=from_vec_transform(val)) +function update!(vnv::VarNamedVector, vn::VarName, val, transform=identity) if !haskey(vnv, vn) # Here we just add a new entry. return push!(vnv, vn, val, transform) end # Here we update an existing entry. - val_vec = tovec(val) + if !(val isa AbstractVector) + val_vec = tovec(val) + transform = _compose_no_identity(transform, from_vec_transform(val)) + else + val_vec = val + end idx = getidx(vnv, vn) # Extract the old range. r_old = getrange(vnv, idx) @@ -932,8 +967,10 @@ function update!(vnv::VarNamedVector, vn::VarName, val, transform=from_vec_trans return nothing end -function update!!(vnv::VarNamedVector, vn::VarName, val, transform=from_vec_transform(val)) - vnv = loosen_types!!(vnv, typeof(vn), typeof(transform)) +function update!!(vnv::VarNamedVector, vn::VarName, val, transform=identity) + vnv = loosen_types!!( + vnv, typeof(vn), typeof(_compose_no_identity(transform, from_vec_transform(val))) + ) update!(vnv, vn, val, transform) return vnv end diff --git a/test/varnamedvector.jl b/test/varnamedvector.jl index 604adc521..425d809dd 100644 --- a/test/varnamedvector.jl +++ b/test/varnamedvector.jl @@ -428,11 +428,13 @@ end @test subset(vnv, VarName[]) == VarNamedVector() @test merge(subset(vnv, test_vns[1:3]), subset(vnv, test_vns[4:end])) == vnv - # Test that subset preseres transformations and unconstrainedness. + # Test that subset preserves transformations and unconstrainedness. vn = @varname(t[1]) vns = vcat(test_vns, [vn]) - push!(vnv, vn, 2.0, x -> x^2) - vnv.is_unconstrained[vnv.varname_to_index[vn]] = true + vnv = push!!(vnv, vn, 2.0, x -> x^2) + DynamicPPL.settrans!(vnv, true, @varname(t[1])) + @test vnv[@varname(t[1])] == 4.0 + @test istrans(vnv, @varname(t[1])) @test subset(vnv, vns) == vnv end end From b5677b4a3a3b0f2ccb70133e90b2014ad69a4bb9 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 4 Sep 2024 16:50:03 +0100 Subject: [PATCH 167/209] Small docs fixes --- docs/src/internals/varinfo.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index d1d2e7116..725249352 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -47,9 +47,7 @@ We also want some additional methods that are *not* part of the `Dict` or `Vecto In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: - - `getindex_raw` and `setindex_raw!` for extracting and mutating the, possibly unconstrained / transformed, realization for a particular `VarName`. - - - `getindex_internal` and `setindex_internal!` for extracting and mutating the internal representaton of a particular `VarName`. + - `getindex_internal` and `setindex_internal!` for extracting and mutating the internal, possibly unconstrained, representaton of a particular `VarName`. Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: @@ -97,7 +95,7 @@ varinfo_typed = DynamicPPL.typed_varinfo(demo()) typeof(varinfo_typed.metadata) ``` -But they both work as expected but one results in concrete typing and the other does not: +They both work as expected but one results in concrete typing and the other does not: ```@example varinfo-design varinfo_untyped[@varname(x)], varinfo_untyped[@varname(y)] @@ -120,7 +118,7 @@ Notice that the untyped `VarInfo` uses `Vector{Real}` to store the boolean entri In this case we'll end up with a `NamedTuple((:x,), Tuple{Vx})` where `Vx` is a container with `eltype` `Union{Bool, Float64}` or something worse. This is *not* type-stable but will still be functional. - In practice, rarely observe such mixing of types, therefore in DynamicPPL, and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. + In practice, we rarely observe such mixing of types, therefore in DynamicPPL, and more widely in Turing.jl, we use a `NamedTuple` approach for type-stability with great success. !!! warning @@ -138,11 +136,11 @@ Efficient storage and iteration we achieve through implementation of the `metada DynamicPPL.VarNamedVector ``` -In a [`VarNamedVector{<:VarName,Vector{T}}`](@ref), we achieve the desirata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. +In a [`VarNamedVector{<:VarName,T}`](@ref), we achieve the desiderata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: - - `varnames::Vector{VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. + - `varnames::Vector{<:VarName}`: the `VarName`s in the order they appear in the `Vector{T}`. - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - `transforms::Vector`: the transforms associated with each `VarName`. From b7780821abf844a732d1e9d09f7cd730a73187ae Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 11:18:20 +0100 Subject: [PATCH 168/209] Small fixes all over for VNV --- src/varinfo.jl | 40 ++++++++++++++++++++++++++-------------- src/varnamedvector.jl | 6 ++++++ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 58a539c42..f9fb12157 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -127,13 +127,22 @@ function metadata_to_varnamedvector(md::Metadata) vns = copy(md.vns) ranges = copy(md.ranges) vals = copy(md.vals) - transforms = map(md.dists) do dist - # TODO: Handle linked distributions. - from_vec_transform(dist) + is_unconstrained = map(Base.Fix1(istrans, md), md.vns) + transforms = map(md.dists, is_unconstrained) do dist, trans + if trans + return from_linked_vec_transform(dist) + else + return from_vec_transform(dist) + end end return VarNamedVector( - OrderedDict{eltype(keys(idcs)),Int}(idcs), vns, ranges, vals, transforms + OrderedDict{eltype(keys(idcs)),Int}(idcs), + vns, + ranges, + vals, + transforms, + is_unconstrained, ) end @@ -191,6 +200,8 @@ function VarInfo( model::Model, sampler::AbstractSampler=SampleFromPrior(), context::AbstractContext=DefaultContext(), + # TODO(mhauru) Revisit the default. We probably don't want it to be VarNamedVector just + # yet. metadata_type::Type=VarNamedVector, ) return typed_varinfo(rng, model, sampler, context, metadata_type) @@ -339,6 +350,7 @@ end function subset(varinfo::VectorVarInfo, vns::AbstractVector{<:VarName}) metadata = subset(varinfo.metadata, vns) + # TODO(mhauru) Should we make deepcopies new RefValues for the logp and num_produce? return VarInfo(metadata, varinfo.logp, varinfo.num_produce) end @@ -615,7 +627,7 @@ Return the distribution from which `vn` was sampled in `vi`. """ getdist(vi::VarInfo, vn::VarName) = getdist(getmetadata(vi, vn), vn) getdist(md::Metadata, vn::VarName) = md.dists[getidx(md, vn)] -# HACK: we shouldn't need this +# TODO(mhauru) Remove this once the old Gibbs sampler stuff is gone. function getdist(::VarNamedVector, ::VarName) throw(ErrorException("getdist does not exist for VarNamedVector")) end @@ -626,7 +638,8 @@ getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, # what a bijector would result in, even if the input is a view (`SubArray`). # TODO(torfjelde): An alternative is to implement `view` directly instead. getindex_internal(md::Metadata, vn::VarName) = getindex(md.vals, getrange(md, vn)) -# HACK: We shouldn't need this +# TODO(mhauru) Maybe rename getindex_raw to getindex_internal and obviate the need for this +# method. getindex_internal(vnv::VarNamedVector, vn::VarName) = getindex_raw(vnv, vn) function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) @@ -681,7 +694,7 @@ function _setall!(metadata::Metadata, val) end end function _setall!(vnv::VarNamedVector, val) - # TODO: Do something more efficient here. + # TODO(mhauru) Do something more efficient here. for i in 1:length(vnv) vnv[i] = val[i] end @@ -719,11 +732,6 @@ function settrans!!(metadata::Metadata, trans::Bool, vn::VarName) return metadata end -function settrans!!(vnv::VarNamedVector, trans::Bool, vn::VarName) - settrans!(vnv, trans, vn) - return vnv -end - function settrans!!(vi::VarInfo, trans::Bool) for vn in keys(vi) settrans!!(vi, trans, vn) @@ -895,6 +903,8 @@ end return results end +# TODO(mhauru) These set_flag! methods return the VarInfo. They should probably be called +# set_flag!!. """ set_flag!(vi::VarInfo, vn::VarName, flag::String) @@ -923,6 +933,8 @@ end # VarInfo +# TODO(mhauru) Revisit the default for meta. We probably should keep it as Metadata as long +# as the old Gibbs sampler is in use. VarInfo(meta=VarNamedVector()) = VarInfo(meta, Ref{Float64}(0.0), Ref(0)) function TypedVarInfo(vi::VectorVarInfo) @@ -1899,8 +1911,8 @@ end Set `vn`'s value for `flag` to `false` in `vi`. -If `ignorable` is `false`, as it is by default, then this will error if setting the flag is -not possible. +Setting some flags for some `VarInfo` types is not possible, and by default attempting to do +so will error. If `ignorable` is set to `true` then this will silently be ignored instead. """ function unset_flag!(vi::VarInfo, vn::VarName, flag::String, ignorable::Bool=false) unset_flag!(getmetadata(vi, vn), vn, flag, ignorable) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 509bf5480..99fee16b4 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -245,6 +245,11 @@ function settrans!(vnv::VarNamedVector, val::Bool, vn::VarName) return vnv.is_unconstrained[vnv.varname_to_index[vn]] = val end +function settrans!!(vnv::VarNamedVector, val::Bool, vn::VarName) + settrans!(vnv, val, vn) + return vnv +end + """ has_inactive(vnv::VarNamedVector) @@ -406,6 +411,7 @@ end Base.setindex!(vnv::VarNamedVector, val, i::Int) = setindex_raw!(vnv, val, i) function Base.setindex!(vnv::VarNamedVector, val, vn::VarName) + # Since setindex! does not change the transform, we need to apply it to `val`. f = inverse(gettransform(vnv, vn)) return setindex_raw!(vnv, f(val), vn) end From 9750e60a51e9a2c35d5b55499653b08a1ce66747 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 11:26:33 +0100 Subject: [PATCH 169/209] Add comments --- src/varinfo.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index f9fb12157..089ac8eda 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2150,9 +2150,9 @@ end Set the values in `vi` to the provided values and those which are not present in `x` or `chains` to *be* resampled. -Note that this does *not* resample the values not provided! It will call `setflag!(vi, vn, "del")` -for variables `vn` for which no values are provided, which means that the next time we call `model(vi)` these -variables will be resampled. +Note that this does *not* resample the values not provided! It will call +`setflag!(vi, vn, "del")` for variables `vn` for which no values are provided, which means +that the next time we call `model(vi)` these variables will be resampled. ## Note - This suffers from the same limitations as [`setval!`](@ref). See `setval!` for more info. @@ -2172,6 +2172,8 @@ julia> rng = StableRNG(42); julia> m = demo([missing]); +# Checking the setting of "del" flags only makes sense for VarInfo{<:Metadata}. For +# VarInfo{<:VarNamedVector} the flag is effectively always set. julia> var_info = DynamicPPL.VarInfo(rng, m, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata); julia> var_info[@varname(m)] From 9ecc5062cbff9b0025ce945565ca1a507cfa9b6c Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 11:35:43 +0100 Subject: [PATCH 170/209] Fix some tests --- src/varinfo.jl | 4 +--- test/model.jl | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 089ac8eda..b644465f0 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -2172,9 +2172,7 @@ julia> rng = StableRNG(42); julia> m = demo([missing]); -# Checking the setting of "del" flags only makes sense for VarInfo{<:Metadata}. For -# VarInfo{<:VarNamedVector} the flag is effectively always set. -julia> var_info = DynamicPPL.VarInfo(rng, m, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata); +julia> var_info = DynamicPPL.VarInfo(rng, m, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata); # Checking the setting of "del" flags only makes sense for VarInfo{<:Metadata}. For VarInfo{<:VarNamedVector} the flag is effectively always set. julia> var_info[@varname(m)] -0.6702516921145671 diff --git a/test/model.jl b/test/model.jl index eaad848f1..dab019c2f 100644 --- a/test/model.jl +++ b/test/model.jl @@ -217,13 +217,9 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true @testset "Dynamic constraints, VectorVarInfo" begin model = DynamicPPL.TestUtils.demo_dynamic_constraint() - vi = VarInfo(model) - vi = link!!(vi, model) - for i in 1:10 - # Sample with large variations. - vi[@varname(m)] = randn() * 10 - model(vi) + vi = VarInfo(model) + @test vi[@varname(x)] >= vi[@varname(m)] end end From 3f1b9a2f8566be2afd8ebc6bcd88fff07f60178d Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 11:37:39 +0100 Subject: [PATCH 171/209] Change long string formatting to support Julia 1.6 --- src/varnamedvector.jl | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 99fee16b4..4126e479f 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -80,22 +80,13 @@ struct VarNamedVector{ length(varnames) != length(transforms) || length(varnames) != length(is_unconstrained) || length(varnames) != length(varname_to_index) - msg = """ - Inputs to VarNamedVector have inconsistent lengths. Got lengths \ - varnames: $(length(varnames)), \ - ranges: $(length(ranges)), \ - transforms: $(length(transforms)), \ - is_unconstrained: $(length(is_unconstrained)), \ - varname_to_index: $(length(varname_to_index)).""" + msg = "Inputs to VarNamedVector have inconsistent lengths. Got lengths varnames: $(length(varnames)), ranges: $(length(ranges)), transforms: $(length(transforms)), is_unconstrained: $(length(is_unconstrained)), varname_to_index: $(length(varname_to_index))." throw(ArgumentError(msg)) end num_vals = mapreduce(length, (+), ranges; init=0) + sum(values(num_inactive)) if num_vals != length(vals) - msg = """ - The total number of elements in `vals` ($(length(vals))) does not match the \ - sum of the lengths of the ranges and the number of inactive entries \ - ($(num_vals)).""" + msg = "The total number of elements in `vals` ($(length(vals))) does not match the sum of the lengths of the ranges and the number of inactive entries ($(num_vals))." throw(ArgumentError(msg)) end @@ -116,9 +107,7 @@ struct VarNamedVector{ for vn2 in keys(varname_to_index) vn1 === vn2 && continue if subsumes(vn1, vn2) - msg = """ - Variables in a VarNamedVector should not subsume each other, \ - but $vn1 subsumes $vn2""" + msg = "Variables in a VarNamedVector should not subsume each other, but $vn1 subsumes $vn2." throw(ArgumentError(msg)) end end From 9145965fea272f35f7ca2f46bcdcf5860c278f36 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 15:38:56 +0100 Subject: [PATCH 172/209] Small changes to ReshapeTransformation --- src/utils.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/utils.jl b/src/utils.jl index 7d95eef82..9db1009b8 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -228,7 +228,9 @@ invlink_transform(dist) = inverse(link_transform(dist)) """ UnwrapSingletonTransform -A transformation that unwraps a singleton vector into a scalar. +A transformation that unwraps a singleton array into a scalar. + +This transformation can be inverted by calling `tovec`. """ struct UnwrapSingletonTransform <: Bijectors.Bijector end @@ -244,8 +246,7 @@ end """ ReshapeTransform(size::Size) -A `Bijector` that transforms an `AbstractVector` to a realization of size `size`. As a -special case, if `size` is an empty tuple it transforms a singleton vector into a scalar. +A `Bijector` that transforms an `AbstractVector` to a realization of size `size`. This transformation can be inverted by calling `tovec`. """ @@ -253,7 +254,7 @@ struct ReshapeTransform{Size} <: Bijectors.Bijector size::Size end -ReshapeTransform(x::Union{Real,AbstractArray}) = ReshapeTransform(size(x)) +ReshapeTransform(x::AbstractArray) = ReshapeTransform(size(x)) # TODO: Should we materialize the `reshape`? (f::ReshapeTransform)(x) = reshape(x, f.size) From 937956d805d1e93ef87abe7d18aa993f110e58ce Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 15:39:17 +0100 Subject: [PATCH 173/209] Revert unrelated changes to ReverseDiff extension --- ext/DynamicPPLReverseDiffExt.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/DynamicPPLReverseDiffExt.jl b/ext/DynamicPPLReverseDiffExt.jl index b2b378d45..3fd174ed1 100644 --- a/ext/DynamicPPLReverseDiffExt.jl +++ b/ext/DynamicPPLReverseDiffExt.jl @@ -9,12 +9,12 @@ else end function LogDensityProblemsAD.ADgradient( - ad::ADTypes.AutoReverseDiff, ℓ::DynamicPPL.LogDensityFunction -) + ad::ADTypes.AutoReverseDiff{Tcompile}, ℓ::DynamicPPL.LogDensityFunction +) where {Tcompile} return LogDensityProblemsAD.ADgradient( Val(:ReverseDiff), ℓ; - compile=Val(ad.compile), + compile=Val(Tcompile), # `getparams` can return `Vector{Real}`, in which case, `ReverseDiff` will initialize the gradients to Integer 0 # because at https://github.com/JuliaDiff/ReverseDiff.jl/blob/c982cde5494fc166965a9d04691f390d9e3073fd/src/tracked.jl#L473 # `zero(D)` will return 0 when D is Real. From 4fbe5d25d9c7a903a16fffb6951ac8205a4d72e0 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 16:28:37 +0100 Subject: [PATCH 174/209] Improve VarNamedVector VarInfo testing --- test/varinfo.jl | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/test/varinfo.jl b/test/varinfo.jl index a00935279..382eb7e58 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -198,22 +198,36 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) m_vns = model == model_uv ? [@varname(m[i]) for i in 1:5] : @varname(m) s_vns = @varname(s) - # TODO(mhauru) Should add similar tests for VarNamedVector. These ones only apply - # to Metadata. vi_typed = VarInfo( model, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata ) vi_untyped = VarInfo(DynamicPPL.Metadata()) + vi_vnv = VarInfo(VarNamedVector()) + vi_vnv_typed = VarInfo( + model, SampleFromPrior(), DefaultContext(), DynamicPPL.VarNamedVector + ) model(vi_untyped, SampleFromPrior()) + model(vi_vnv, SampleFromPrior()) - for vi in [vi_untyped, vi_typed] + model_name = model == model_uv ? "univariate" : "multivariate" + @testset "$(model_name), $(short_varinfo_name(vi))" for vi in [ + vi_untyped, vi_typed, vi_vnv, vi_vnv_typed + ] + Random.seed!(23) vicopy = deepcopy(vi) ### `setval` ### - DynamicPPL.setval!(vicopy, (m=zeros(5),)) + # TODO(mhauru) The interface here seems inconsistent between Metadata and + # VarNamedVector. I'm lazy to fix it though, because I think we need to + # rework it soon anyway. + if vi in [vi_vnv, vi_vnv_typed] + DynamicPPL.setval!(vicopy, zeros(5), m_vns) + else + DynamicPPL.setval!(vicopy, (m=zeros(5),)) + end # Setting `m` fails for univariate due to limitations of `setval!` # and `setval_and_resample!`. See docstring of `setval!` for more info. - if model == model_uv + if model == model_uv && vi in [vi_untyped, vi_typed] @test_broken vicopy[m_vns] == zeros(5) else @test vicopy[m_vns] == zeros(5) @@ -246,6 +260,13 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) continue end + if vi in [vi_vnv, vi_vnv_typed] + # `setval_and_resample!` works differently for `VarNamedVector`: All + # values will be resampled when model(vicopy) is called. Hence the below + # tests are not applicable. + continue + end + vicopy = deepcopy(vi) DynamicPPL.setval_and_resample!(vicopy, (m=zeros(5),)) model(vicopy) From 9f11e7b214a52d2ae47f2291a6b43ff6249423c7 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 16:40:48 +0100 Subject: [PATCH 175/209] Fix some depwarns --- src/varnamedvector.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 4126e479f..56708c6f9 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -657,13 +657,13 @@ function Base.similar(vnv::VarNamedVector) # present, and so for now we empty the underlying containers, thus differing # from the behavior of `similar` for `AbstractArray`s. return VarNamedVector( - similar(vnv.varname_to_index), + empty(vnv.varname_to_index), similar(vnv.varnames, 0), similar(vnv.ranges, 0), similar(vnv.vals, 0), similar(vnv.transforms, 0), BitVector(), - similar(vnv.num_inactive), + empty(vnv.num_inactive), ) end @@ -1069,7 +1069,7 @@ true """ function group_by_symbol(vnv::VarNamedVector) symbols = unique(map(getsym, vnv.varnames)) - nt_vals = map(s -> tighten_types(subset(vnv, [VarName(s)])), symbols) + nt_vals = map(s -> tighten_types(subset(vnv, [VarName{s}()])), symbols) return OrderedDict(zip(symbols, nt_vals)) end From 86d97ae79f24827eeace6327743198659c77f8dc Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 16:41:13 +0100 Subject: [PATCH 176/209] Improvements to test/simple_varinfo.jl --- test/simple_varinfo.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 2684774bf..f5b97dbbc 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -79,7 +79,7 @@ # The implementation of haskey and getvalue fo VarNamedVector is incomplete, the # next test is here to remind of us that. svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a.b), [1.0])) - @test_broken (svi[@varname(m.a.b.c.d)]; true) + @test_broken !haskey(svi, @varname(m.a.b.c.d)) end end @@ -153,9 +153,9 @@ svi_nt, svi_dict, svi_vnv, - DynamicPPL.settrans!!(svi_nt, true), - DynamicPPL.settrans!!(svi_dict, true), - DynamicPPL.settrans!!(svi_vnv, true), + DynamicPPL.settrans!!(deepcopy(svi_nt), true), + DynamicPPL.settrans!!(deepcopy(svi_dict), true), + DynamicPPL.settrans!!(deepcopy(svi_vnv), true), ) # RandOM seed is set in each `@testset`, so we need to sample # a new realization for `m` here. From 2535517b626ad20b57c118ecccd76429d4c12742 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 17:06:47 +0100 Subject: [PATCH 177/209] Fix for unset_flag!, better docstring --- src/threadsafe.jl | 6 ++++-- src/varnamedvector.jl | 17 +++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/threadsafe.jl b/src/threadsafe.jl index 196f243bf..ec890a674 100644 --- a/src/threadsafe.jl +++ b/src/threadsafe.jl @@ -190,8 +190,10 @@ end values_as(vi::ThreadSafeVarInfo) = values_as(vi.varinfo) values_as(vi::ThreadSafeVarInfo, ::Type{T}) where {T} = values_as(vi.varinfo, T) -function unset_flag!(vi::ThreadSafeVarInfo, vn::VarName, flag::String) - return unset_flag!(vi.varinfo, vn, flag) +function unset_flag!( + vi::ThreadSafeVarInfo, vn::VarName, flag::String, ignoreable::Bool=false +) + return unset_flag!(vi.varinfo, vn, flag, ignoreable) end function is_flagged(vi::ThreadSafeVarInfo, vn::VarName, flag::String) return is_flagged(vi.varinfo, vn, flag) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 56708c6f9..da61c3b49 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -46,7 +46,7 @@ struct VarNamedVector{ """ vector of transformations, so that `transforms[varname_to_index[vn]]` is a callable - that transformes the value of `vn` back to its original space, undoing any linking and + that transforms the value of `vn` back to its original space, undoing any linking and vectorisation """ transforms::TTrans @@ -62,8 +62,8 @@ struct VarNamedVector{ """ mapping from a variable index to the number of inactive entries for that variable. Inactive entries are elements in `vals` that are not part of the value of any variable. - They arise when transformations change the dimension of the value stored. In active - entries always come after the last active entry for the given variable. + They arise when a variable is set to a new value with a different dimension, in-place. + Inactive entries always come after the last active entry for the given variable. """ num_inactive::OrderedDict{Int,Int} @@ -149,7 +149,7 @@ function VarNamedVector( ) end -# TODO(mhauru) Are we sure we want the last one to be of type Any[]? Might this call +# TODO(mhauru) Are we sure we want the last one to be of type Any[]? Might this cause # unnecessary type instability? function VarNamedVector{K,V}() where {K,V} return VarNamedVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) @@ -205,7 +205,6 @@ function ==(vnv_left::VarNamedVector, vnv_right::VarNamedVector) vnv_left.num_inactive == vnv_right.num_inactive end -# Some `VarNamedVector` specific functions. getidx(vnv::VarNamedVector, vn::VarName) = vnv.varname_to_index[vn] getrange(vnv::VarNamedVector, idx::Int) = vnv.ranges[idx] @@ -219,8 +218,8 @@ gettransform(vnv::VarNamedVector, vn::VarName) = gettransform(vnv, getidx(vnv, v """ istrans(vnv::VarNamedVector, vn::VarName) -Return a boolean for whether `vn` is guaranteed to have been transformed so that all of -Euclidean space is its domain. +Return a boolean for whether `vn` is guaranteed to have been transformed so that its domain +is all of Euclidean space. """ istrans(vnv::VarNamedVector, vn::VarName) = vnv.is_unconstrained[getidx(vnv, vn)] @@ -289,7 +288,6 @@ Base.length(vnv::VarNamedVector) = Base.size(vnv::VarNamedVector) = (length(vnv),) Base.isempty(vnv::VarNamedVector) = isempty(vnv.varnames) -# TODO: We should probably remove this Base.IndexStyle(::Type{<:VarNamedVector}) = IndexLinear() # Dictionary interface. @@ -686,6 +684,9 @@ function nextrange(vnv::VarNamedVector, x) return (offset + 1):(offset + length(x)) end +# TODO(mhauru) Might add another specialisation to _compose_no_identity, where if +# ReshapeTransforms are composed with each other or with a an UnwrapSingeltonTransform, only +# the latter one would be kept. """ _compose_no_identity(f, g) From 93ef3ee7e0d047c83665517d259468280c840672 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 5 Sep 2024 17:13:17 +0100 Subject: [PATCH 178/209] Add a comment about hasvalue/getvalue --- src/varnamedvector.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index da61c3b49..9d9c9812d 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -1187,6 +1187,9 @@ function values_as(vnv::VarNamedVector, ::Type{D}) where {D<:AbstractDict} return ConstructionBase.constructorof(D)(pairs(vnv)) end +# See the docstring of `getvalue` for the semantics of `hasvalue` and `getvalue`, and how +# they differ from `haskey` and `getindex`. They can be found in src/utils.jl. + # TODO(mhauru) This is tricky to implement in the general case, and the below implementation # only covers some simple cases. It's probably sufficient in most situations though. function hasvalue(vnv::VarNamedVector, vn::VarName) From f35eca668ffe40baa09a7b7eac1d9063fbf739e2 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Mon, 9 Sep 2024 17:10:32 +0100 Subject: [PATCH 179/209] Add @non_differentiable calls to work around Zygote limitations --- ext/DynamicPPLChainRulesCoreExt.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ext/DynamicPPLChainRulesCoreExt.jl b/ext/DynamicPPLChainRulesCoreExt.jl index 1c6e188fb..890dd4307 100644 --- a/ext/DynamicPPLChainRulesCoreExt.jl +++ b/ext/DynamicPPLChainRulesCoreExt.jl @@ -24,4 +24,10 @@ ChainRulesCore.@non_differentiable DynamicPPL.updategid!( # No need + causes issues for some AD backends, e.g. Zygote. ChainRulesCore.@non_differentiable DynamicPPL.infer_nested_eltype(x) +ChainRulesCore.@non_differentiable DynamicPPL.recontiguify_ranges!(ranges) +# TODO(mhauru) A workaround for https://github.com/FluxML/Zygote.jl/issues/1523, remove when +# fixed. Note that this is some serious type piracy. Needed because VarNamedVector uses +# BitVectors. +ChainRulesCore.@non_differentiable BitVector(a, b) + end # module From d55fc002d086abf0b2d339a30e3aca8fbcbd2c7f Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Mon, 9 Sep 2024 17:25:18 +0100 Subject: [PATCH 180/209] Fix docs, workaround Zygote issue --- docs/src/internals/varinfo.md | 4 +++- src/utils.jl | 2 +- src/varinfo.jl | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 725249352..d2af36c32 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -85,7 +85,9 @@ For example, with the model above we have ```@example varinfo-design # Type-unstable `VarInfo` -varinfo_untyped = DynamicPPL.untyped_varinfo(demo()) +varinfo_untyped = DynamicPPL.untyped_varinfo( + demo(), SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata +) typeof(varinfo_untyped.metadata) ``` diff --git a/src/utils.jl b/src/utils.jl index 9db1009b8..fed303021 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -297,8 +297,8 @@ from_vec_transform_for_size(::Tuple{<:Any}) = identity Return the transformation from the vector representation of a realization from distribution `dist` to the original representation compatible with `dist`. """ -from_vec_transform(::UnivariateDistribution) = UnwrapSingletonTransform() from_vec_transform(dist::Distribution) = from_vec_transform_for_size(size(dist)) +from_vec_transform(::UnivariateDistribution) = UnwrapSingletonTransform() from_vec_transform(dist::LKJCholesky) = ToChol(dist.uplo) ∘ ReshapeTransform(size(dist)) """ diff --git a/src/varinfo.jl b/src/varinfo.jl index b644465f0..6c996d0f9 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -184,7 +184,7 @@ function untyped_varinfo( varinfo = VarInfo(metadata_type()) return last(evaluate!!(model, varinfo, SamplingContext(rng, sampler, context))) end -function untyped_varinfo(model::Model, args::Union{AbstractSampler,AbstractContext}...) +function untyped_varinfo(model::Model, args::Union{AbstractSampler,AbstractContext,Type}...) return untyped_varinfo(Random.default_rng(), model, args...) end From 5bbba913e72b25b5c934ebe8f959b3c3017ad4bc Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 17 Sep 2024 13:10:27 +0100 Subject: [PATCH 181/209] Remove outdated workaround --- ext/DynamicPPLChainRulesCoreExt.jl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ext/DynamicPPLChainRulesCoreExt.jl b/ext/DynamicPPLChainRulesCoreExt.jl index 890dd4307..1559467f8 100644 --- a/ext/DynamicPPLChainRulesCoreExt.jl +++ b/ext/DynamicPPLChainRulesCoreExt.jl @@ -25,9 +25,5 @@ ChainRulesCore.@non_differentiable DynamicPPL.updategid!( ChainRulesCore.@non_differentiable DynamicPPL.infer_nested_eltype(x) ChainRulesCore.@non_differentiable DynamicPPL.recontiguify_ranges!(ranges) -# TODO(mhauru) A workaround for https://github.com/FluxML/Zygote.jl/issues/1523, remove when -# fixed. Note that this is some serious type piracy. Needed because VarNamedVector uses -# BitVectors. -ChainRulesCore.@non_differentiable BitVector(a, b) end # module From 851630f8f195eb4eade03bd32674259bd8625fcd Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 17 Sep 2024 13:14:38 +0100 Subject: [PATCH 182/209] Move has_varnamedvector(varinfo::VarInfo) to abstract_varinfo.jl --- src/abstract_varinfo.jl | 11 +++++++++-- src/varinfo.jl | 6 ------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/abstract_varinfo.jl b/src/abstract_varinfo.jl index 551bf87d3..3f513d71d 100644 --- a/src/abstract_varinfo.jl +++ b/src/abstract_varinfo.jl @@ -363,6 +363,13 @@ function Base.eltype(vi::AbstractVarInfo, spl::Union{AbstractSampler,SampleFromP return eltype(T) end +""" + has_varnamedvector(varinfo::VarInfo) + +Returns `true` if `varinfo` uses `VarNamedVector` as metadata. +""" +has_varnamedvector(vi::AbstractVarInfo) = false + # TODO: Should relax constraints on `vns` to be `AbstractVector{<:Any}` and just try to convert # the `eltype` to `VarName`? This might be useful when someone does `[@varname(x[1]), @varname(m)]` which # might result in a `Vector{Any}`. @@ -554,7 +561,7 @@ end link([t::AbstractTransformation, ]vi::AbstractVarInfo, model::Model) link([t::AbstractTransformation, ]vi::AbstractVarInfo, spl::AbstractSampler, model::Model) -Transform the variables in `vi` to their linked space without mutating `vi`, using the transformation `t`. +Transform the variables in `vi` to their linked space without mutating `vi`, using the transformation `t`. If `t` is not provided, `default_transformation(model, vi)` will be used. @@ -573,7 +580,7 @@ end invlink!!([t::AbstractTransformation, ]vi::AbstractVarInfo, model::Model) invlink!!([t::AbstractTransformation, ]vi::AbstractVarInfo, spl::AbstractSampler, model::Model) -Transform the variables in `vi` to their constrained space, using the (inverse of) +Transform the variables in `vi` to their constrained space, using the (inverse of) transformation `t`, mutating `vi` if possible. If `t` is not provided, `default_transformation(model, vi)` will be used. diff --git a/src/varinfo.jl b/src/varinfo.jl index 6c996d0f9..a1f53a402 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -158,12 +158,6 @@ function VectorVarInfo(vi::TypedVarInfo) return VarInfo(md, Base.RefValue{eltype(lp)}(lp), Ref(get_num_produce(vi))) end -""" - has_varnamedvector(varinfo::VarInfo) - -Returns `true` if `varinfo` uses `VarNamedVector` as metadata. -""" -has_varnamedvector(vi::AbstractVarInfo) = false function has_varnamedvector(vi::VarInfo) return vi.metadata isa VarNamedVector || (vi isa TypedVarInfo && any(Base.Fix2(isa, VarNamedVector), values(vi.metadata))) From 45c89c4bc0867e071a43a37f50ba47016540d749 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 17 Sep 2024 13:17:48 +0100 Subject: [PATCH 183/209] Make copies of logp and num_produce in subset --- src/varinfo.jl | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index a1f53a402..c2a535d3f 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -339,19 +339,22 @@ end function subset(varinfo::UntypedVarInfo, vns::AbstractVector{<:VarName}) metadata = subset(varinfo.metadata, vns) - return VarInfo(metadata, varinfo.logp, varinfo.num_produce) + return VarInfo(metadata, deepcopy(varinfo.logp), deepcopy(varinfo.num_produce)) end function subset(varinfo::VectorVarInfo, vns::AbstractVector{<:VarName}) metadata = subset(varinfo.metadata, vns) - # TODO(mhauru) Should we make deepcopies new RefValues for the logp and num_produce? - return VarInfo(metadata, varinfo.logp, varinfo.num_produce) + return VarInfo(metadata, deepcopy(varinfo.logp), deepcopy(varinfo.num_produce)) end function subset(varinfo::TypedVarInfo, vns::AbstractVector{<:VarName{sym}}) where {sym} # If all the variables are using the same symbol, then we can just extract that field from the metadata. metadata = subset(getfield(varinfo.metadata, sym), vns) - return VarInfo(NamedTuple{(sym,)}(tuple(metadata)), varinfo.logp, varinfo.num_produce) + return VarInfo( + NamedTuple{(sym,)}(tuple(metadata)), + deepcopy(varinfo.logp), + deepcopy(varinfo.num_produce), + ) end function subset(varinfo::TypedVarInfo, vns::AbstractVector{<:VarName}) @@ -360,7 +363,9 @@ function subset(varinfo::TypedVarInfo, vns::AbstractVector{<:VarName}) subset(getfield(varinfo.metadata, sym), filter(==(sym) ∘ getsym, vns)) end - return VarInfo(NamedTuple{syms}(metadatas), varinfo.logp, varinfo.num_produce) + return VarInfo( + NamedTuple{syms}(metadatas), deepcopy(varinfo.logp), deepcopy(varinfo.num_produce) + ) end function subset(metadata::Metadata, vns_given::AbstractVector{<:VarName}) From 30252dc72461f3f048860462f7a7c0ba8a7950fa Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 17 Sep 2024 13:22:19 +0100 Subject: [PATCH 184/209] Rename getindex_raw to getindex_internal --- src/varinfo.jl | 9 +++------ src/varnamedvector.jl | 34 +++++++++++++++++----------------- test/varnamedvector.jl | 19 ++++++++++--------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index c2a535d3f..8b548cc14 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -637,9 +637,6 @@ getindex_internal(vi::VarInfo, vn::VarName) = getindex_internal(getmetadata(vi, # what a bijector would result in, even if the input is a view (`SubArray`). # TODO(torfjelde): An alternative is to implement `view` directly instead. getindex_internal(md::Metadata, vn::VarName) = getindex(md.vals, getrange(md, vn)) -# TODO(mhauru) Maybe rename getindex_raw to getindex_internal and obviate the need for this -# method. -getindex_internal(vnv::VarNamedVector, vn::VarName) = getindex_raw(vnv, vn) function getindex_internal(vi::VarInfo, vns::Vector{<:VarName}) return mapreduce(Base.Fix1(getindex_internal, vi), vcat, vns) @@ -676,7 +673,7 @@ function getall(md::Metadata) Base.Fix1(getindex_internal, md), vcat, md.vns; init=similar(md.vals, 0) ) end -getall(vnv::VarNamedVector) = getindex_raw(vnv, Colon()) +getall(vnv::VarNamedVector) = getindex_internal(vnv, Colon()) """ setall!(vi::VarInfo, val) @@ -1439,7 +1436,7 @@ function _link_metadata!!( # First transform from however the variable is stored in vnv to the model # representation. transform_to_orig = gettransform(metadata, vn) - val_old = getindex_raw(metadata, vn) + val_old = getindex_internal(metadata, vn) val_orig, logjac1 = with_logabsdet_jacobian(transform_to_orig, val_old) # Then transform from the model representation to the linked representation. transform_from_linked = from_linked_vec_transform(dists[vn]) @@ -1559,7 +1556,7 @@ function _invlink_metadata!!( vns = target_vns === nothing ? keys(metadata) : target_vns for vn in vns transform = gettransform(metadata, vn) - old_val = getindex_raw(metadata, vn) + old_val = getindex_internal(metadata, vn) new_val, logjac = with_logabsdet_jacobian(transform, old_val) # TODO(mhauru) We are calling a !! function but ignoring the return value. acclogp!!(varinfo, -logjac) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 9d9c9812d..389819ecf 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -298,9 +298,9 @@ Base.pairs(vnv::VarNamedVector) = (vn => vnv[vn] for vn in keys(vnv)) Base.haskey(vnv::VarNamedVector, vn::VarName) = haskey(vnv.varname_to_index, vn) # `getindex` & `setindex!` -Base.getindex(vnv::VarNamedVector, i::Int) = getindex_raw(vnv, i) +Base.getindex(vnv::VarNamedVector, i::Int) = getindex_internal(vnv, i) function Base.getindex(vnv::VarNamedVector, vn::VarName) - x = getindex_raw(vnv, vn) + x = getindex_internal(vnv, vn) f = gettransform(vnv, vn) return f(x) end @@ -369,15 +369,15 @@ function index_to_vals_index(vnv::VarNamedVector, i::Int) end """ - getindex_raw(vnv::VarNamedVector, i::Int) - getindex_raw(vnv::VarNamedVector, vn::VarName) + getindex_internal(vnv::VarNamedVector, i::Int) + getindex_internal(vnv::VarNamedVector, vn::VarName) Like `getindex`, but returns the values as they are stored in `vnv` without transforming. For integer indices this is the same as `getindex`, but for `VarName`s this is different. """ -getindex_raw(vnv::VarNamedVector, i::Int) = vnv.vals[index_to_vals_index(vnv, i)] -getindex_raw(vnv::VarNamedVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] +getindex_internal(vnv::VarNamedVector, i::Int) = vnv.vals[index_to_vals_index(vnv, i)] +getindex_internal(vnv::VarNamedVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] # `getindex` for `Colon` function Base.getindex(vnv::VarNamedVector, ::Colon) @@ -388,7 +388,7 @@ function Base.getindex(vnv::VarNamedVector, ::Colon) end end -getindex_raw(vnv::VarNamedVector, ::Colon) = getindex(vnv, Colon()) +getindex_internal(vnv::VarNamedVector, ::Colon) = getindex(vnv, Colon()) # TODO(mhauru): Remove this as soon as possible. Only needed because of the old Gibbs # sampler. @@ -396,26 +396,26 @@ function Base.getindex(vnv::VarNamedVector, spl::AbstractSampler) throw(ErrorException("Cannot index a VarNamedVector with a sampler.")) end -Base.setindex!(vnv::VarNamedVector, val, i::Int) = setindex_raw!(vnv, val, i) +Base.setindex!(vnv::VarNamedVector, val, i::Int) = setindex_internal!(vnv, val, i) function Base.setindex!(vnv::VarNamedVector, val, vn::VarName) # Since setindex! does not change the transform, we need to apply it to `val`. f = inverse(gettransform(vnv, vn)) - return setindex_raw!(vnv, f(val), vn) + return setindex_internal!(vnv, f(val), vn) end """ - setindex_raw!(vnv::VarNamedVector, val, i::Int) - setindex_raw!(vnv::VarNamedVector, val, vn::VarName) + setindex_internal!(vnv::VarNamedVector, val, i::Int) + setindex_internal!(vnv::VarNamedVector, val, vn::VarName) Like `setindex!`, but sets the values as they are stored in `vnv` without transforming. For integer indices this is the same as `setindex!`, but for `VarName`s this is different. """ -function setindex_raw!(vnv::VarNamedVector, val, i::Int) +function setindex_internal!(vnv::VarNamedVector, val, i::Int) return vnv.vals[index_to_vals_index(vnv, i)] = val end -function setindex_raw!(vnv::VarNamedVector, val::AbstractVector, vn::VarName) +function setindex_internal!(vnv::VarNamedVector, val::AbstractVector, vn::VarName) return vnv.vals[getrange(vnv, vn)] = val end @@ -565,13 +565,13 @@ function Base.merge(left_vnv::VarNamedVector, right_vnv::VarNamedVector) # Extract the necessary information from `left` or `right`. if vn in vns_left && !(vn in vns_right) # `vn` is only in `left`. - val = getindex_raw(left_vnv, vn) + val = getindex_internal(left_vnv, vn) f = gettransform(left_vnv, vn) is_unconstrained[idx] = istrans(left_vnv, vn) else # `vn` is either in both or just `right`. # Note that in a `merge` the right value has precedence. - val = getindex_raw(right_vnv, vn) + val = getindex_internal(right_vnv, vn) f = gettransform(right_vnv, vn) is_unconstrained[idx] = istrans(right_vnv, vn) end @@ -621,7 +621,7 @@ function subset(vnv::VarNamedVector, vns_given::AbstractVector{VN}) where {VN<:V isempty(vnv) && return vnv_new for vn in vns - push!(vnv_new, vn, getindex_raw(vnv, vn), gettransform(vnv, vn)) + push!(vnv_new, vn, getindex_internal(vnv, vn), gettransform(vnv, vn)) settrans!(vnv_new, istrans(vnv, vn), vn) end @@ -977,7 +977,7 @@ end set!!(vnv::VarNamedVector, vn::VarName, val) = update!!(vnv, vn, val) function setval!(vnv::VarNamedVector, val, vn::VarName) - return setindex_raw!(vnv, tovec(val), vn) + return setindex_internal!(vnv, tovec(val), vn) end function recontiguify_ranges!(ranges::AbstractVector{<:AbstractRange}) diff --git a/test/varnamedvector.jl b/test/varnamedvector.jl index 425d809dd..ba365d24a 100644 --- a/test/varnamedvector.jl +++ b/test/varnamedvector.jl @@ -228,25 +228,26 @@ end @test vnv[vn_right] == val_right .+ 100 end - # `getindex_raw` - @testset "getindex_raw" begin + # `getindex_internal` + @testset "getindex_internal" begin # With `VarName` index. - @test DynamicPPL.getindex_raw(vnv_base, vn_left) == to_vec_left(val_left) - @test DynamicPPL.getindex_raw(vnv_base, vn_right) == to_vec_right(val_right) + @test DynamicPPL.getindex_internal(vnv_base, vn_left) == to_vec_left(val_left) + @test DynamicPPL.getindex_internal(vnv_base, vn_right) == + to_vec_right(val_right) # With `Int` index. val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) @test all( - DynamicPPL.getindex_raw(vnv_base, i) == val_vec[i] for + DynamicPPL.getindex_internal(vnv_base, i) == val_vec[i] for i in 1:length(val_vec) ) end - # `setindex_raw!` - @testset "setindex_raw!" begin + # `setindex_internal!` + @testset "setindex_internal!" begin vnv = deepcopy(vnv_base) - DynamicPPL.setindex_raw!(vnv, to_vec_left(val_left .+ 100), vn_left) + DynamicPPL.setindex_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) @test vnv[vn_left] == val_left .+ 100 - DynamicPPL.setindex_raw!(vnv, to_vec_right(val_right .+ 100), vn_right) + DynamicPPL.setindex_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) @test vnv[vn_right] == val_right .+ 100 end From be77c36eb63048d4d3c8faa04d08c96bc8c24bfc Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 17 Sep 2024 14:40:53 +0100 Subject: [PATCH 185/209] Add push!(::VarNamedVector, ::Pair) --- docs/src/internals/varinfo.md | 3 ++- src/varnamedvector.jl | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index d2af36c32..c1219444f 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -25,6 +25,7 @@ To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo` - `haskey(::Dict)`: check if a particular `VarName` is present in `metadata`. - `getindex(::Dict, ::VarName)`: return the realization corresponding to a particular `VarName`. - `setindex!(::Dict, val, ::VarName)`: set the realization corresponding to a particular `VarName`. + - `push!(::Dict, ::Pair)`: add a new key-value pair to the container. - `delete!(::Dict, ::VarName)`: delete the realization corresponding to a particular `VarName`. - `empty!(::Dict)`: delete all realizations in `metadata`. - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. @@ -41,7 +42,7 @@ To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo` We also want some additional methods that are *not* part of the `Dict` or `Vector` interface: - - `push!(container, ::VarName, value[, transform])`: add a new element to the container, _but_ for this we also need the `VarName` to associate to the new `value`, so the semantics are different from `push!` for a `Vector`. + - `push!(container, varname::VarName, value[, transform])`: add a new element to the container, but with an optional transformation that has been applied to `value`, and should be reverted when returning `container[varname]`. One can also provide a `Pair` instead of a `VarName` and a `value`. - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 389819ecf..abd15ed34 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -702,6 +702,7 @@ _compose_no_identity(::typeof(identity), ::typeof(identity)) = identity """ push!(vnv::VarNamedVector, vn::VarName, val[, transform]) + push!(vnv::VarNamedVector, vn => val[, transform]) Add a variable with given value to `vnv`. @@ -729,6 +730,11 @@ function Base.push!(vnv::VarNamedVector, vn::VarName, val, transform=identity) return nothing end +function Base.push!(vnv::VarNamedVector, pair, transform=identity) + vn, val = pair + return push!(vnv, vn, val, transform) +end + # TODO(mhauru) The gidset and num_produce arguments are used by the old Gibbs sampler. # Remove this method as soon as possible. function Base.push!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) From 8b5bd47da42ed9e0631034e5e55302fce9fde2a7 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 24 Sep 2024 17:17:30 +0100 Subject: [PATCH 186/209] Improve VarNamedVector docs --- src/varinfo.jl | 19 ++++++ src/varnamedvector.jl | 152 ++++++++++++++++++++++++++++++++++++----- test/varinfo.jl | 7 ++ test/varnamedvector.jl | 5 +- 4 files changed, 164 insertions(+), 19 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 8b548cc14..7ee785b1d 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1835,6 +1835,20 @@ function BangBang.push!!( return vi end +function Base.push!(vi::VectorVarInfo, vn::VarName, val, args...) + push!(getmetadata(vi, vn), vn, val, args...) + return vi +end + +function Base.push!(vi::VectorVarInfo, pair::Pair, args...) + vn, val = pair + return push!(vi, vn, val, args...) +end + +# TODO(mhauru) push! can't be implemented in-place for TypedVarInfo if the symbol doesn't +# exist in the TypedVarInfo already. We could implement it in the cases where it it does +# exist, but that feels a bit pointless. I think we should rather rely on `push!!`. + function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) val = tovec(r) meta.idcs[vn] = length(meta.idcs) + 1 @@ -1852,6 +1866,11 @@ function Base.push!(meta::Metadata, vn, r, dist, gidset, num_produce) return meta end +function Base.delete!(vi::VarInfo, vn::VarName) + delete!(getmetadata(vi, vn), vn) + return vi +end + """ setorder!(vi::VarInfo, vn::VarName, index::Int) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index abd15ed34..7661d48eb 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -3,20 +3,107 @@ A container that stores values in a vectorised form, but indexable by variable names. -When indexed by integers or `Colon`s, e.g. `vnv[2]` or `vnv[:]`, `VarNamedVector` behaves +When indexed with integers or `Colon`s, e.g. `vnv[2]` or `vnv[:]`, `VarNamedVector` behaves like a `Vector`, and returns the values as they are stored. The stored form is always vectorised, for instance matrix variables have been flattened, and may be further transformed to achieve linking. -When indexed by `VarName`s, e.g. `vnv[@varname(x)]`, `VarNamedVector` returns the values +When indexed with `VarName`s, e.g. `vnv[@varname(x)]`, `VarNamedVector` returns the values in the original space. For instance, a linked matrix variable is first inverse linked and then reshaped to its original form before returning it to the caller. `VarNamedVector` also stores a boolean for whether a variable has been transformed to unconstrained Euclidean space or not. +Internally, `VarNamedVector` stores the values of all variables in a single contiguous +vector. + # Fields + $(FIELDS) + +# Extended help + +The values for different variables are internally all stored in a single vector. For +instance, +```jldoctest varnamedvector-struct +julia> using DynamicPPL: VarNamedVector, @varname, push!, update! + +julia> vnv = VarNamedVector(); + +julia> push!(vnv, @varname(x) => [0.0, 1.0]); + +julia> push!(vnv, @varname(y) => fill(3, (3,3))); + +julia> vnv.vals +11-element Vector{Real}: + 0.0 + 1.0 + 3 + 3 + 3 + 3 + 3 + 3 + 3 + 3 + 3 +``` + +The `varnames`, `ranges`, and `varname_to_index` fields keep track of which value belongs to +which variable. The `transforms` field stores the transformations that needed to transform +the vectorised internal storage back to its original form: + +```jldoctest varnamedvector-struct +julia> vnv.transforms[vnv.varname_to_index[@varname(y)]] +DynamicPPL.ReshapeTransform{Tuple{Int64, Int64}}((3, 3)) +``` + +If a variable is updated with a new value that is of a smaller dimension than the old +value, rather than resizing `vnv.vals`, some elements in `vnv.vals` are marked as inactive. + +```jldoctest varnamedvector-struct +julia> update!(vnv, @varname(y), fill(2, (2, 2))) + +julia> vnv.vals +11-element Vector{Real}: + 0.0 + 1.0 + 2 + 2 + 2 + 2 + 3 + 3 + 3 + 3 + 3 + +julia> vnv.num_inactive +OrderedDict{Int64, Int64} with 1 entry: + 2 => 5 +``` + +This helps avoid unnecessary memory allocations for values that repeatedly change dimension. +The user does not have to worry about the inactive entries as long as they use functions +like `update!` and `getindex!` rather than directly accessing `vnv.vals`. + +```jldoctest varnamedvector-struct +julia> vnv[@varname(y)] +2×2 Matrix{Real}: + 2 2 + 2 2 + + +julia> vnv[:] +6-element Vector{Real}: + 0.0 + 1.0 + 2 + 2 + 2 + 2 +``` """ struct VarNamedVector{ K<:VarName,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector @@ -64,6 +151,7 @@ struct VarNamedVector{ Inactive entries are elements in `vals` that are not part of the value of any variable. They arise when a variable is set to a new value with a different dimension, in-place. Inactive entries always come after the last active entry for the given variable. + See the extended help with `??VarNamedVector` for more details. """ num_inactive::OrderedDict{Int,Int} @@ -80,23 +168,40 @@ struct VarNamedVector{ length(varnames) != length(transforms) || length(varnames) != length(is_unconstrained) || length(varnames) != length(varname_to_index) - msg = "Inputs to VarNamedVector have inconsistent lengths. Got lengths varnames: $(length(varnames)), ranges: $(length(ranges)), transforms: $(length(transforms)), is_unconstrained: $(length(is_unconstrained)), varname_to_index: $(length(varname_to_index))." + msg = ( + "Inputs to VarNamedVector have inconsistent lengths. Got lengths varnames: " * + "$(length(varnames)), ranges: " * + "$(length(ranges)), " * + "transforms: $(length(transforms)), " * + "is_unconstrained: $(length(is_unconstrained)), " * + "varname_to_index: $(length(varname_to_index))." + ) throw(ArgumentError(msg)) end num_vals = mapreduce(length, (+), ranges; init=0) + sum(values(num_inactive)) if num_vals != length(vals) - msg = "The total number of elements in `vals` ($(length(vals))) does not match the sum of the lengths of the ranges and the number of inactive entries ($(num_vals))." + msg = ( + "The total number of elements in `vals` ($(length(vals))) does not match " * + "the sum of the lengths of the ranges and the number of inactive entries " * + "($(num_vals))." + ) throw(ArgumentError(msg)) end - if Set(values(varname_to_index)) != Set(1:length(varnames)) - msg = "The values of `varname_to_index` are not valid indices." + if Set(values(varname_to_index)) != Set(axes(varnames, 1)) + msg = ( + "The set of values of `varname_to_index` is not the set of valid indices " * + "for `varnames`." + ) throw(ArgumentError(msg)) end if !issubset(Set(keys(num_inactive)), Set(values(varname_to_index))) - msg = "The keys of `num_inactive` are not valid indices." + msg = ( + "The keys of `num_inactive` are not a subset of the values of " * + "`varname_to_index`." + ) throw(ArgumentError(msg)) end @@ -107,7 +212,10 @@ struct VarNamedVector{ for vn2 in keys(varname_to_index) vn1 === vn2 && continue if subsumes(vn1, vn2) - msg = "Variables in a VarNamedVector should not subsume each other, but $vn1 subsumes $vn2." + msg = ( + "Variables in a VarNamedVector should not subsume each other, " * + "but $vn1 subsumes $vn2, i.e. $vn2 describes a subrange of $vn1." + ) throw(ArgumentError(msg)) end end @@ -241,7 +349,9 @@ end """ has_inactive(vnv::VarNamedVector) -Returns `true` if `vnv` has inactive ranges. +Returns `true` if `vnv` has inactive entries. + +See also: [`num_inactive`](@ref) """ has_inactive(vnv::VarNamedVector) = !isempty(vnv.num_inactive) @@ -249,6 +359,8 @@ has_inactive(vnv::VarNamedVector) = !isempty(vnv.num_inactive) num_inactive(vnv::VarNamedVector) Return the number of inactive entries in `vnv`. + +See also: [`has_inactive`](@ref), [`num_allocated`](@ref) """ num_inactive(vnv::VarNamedVector) = sum(values(vnv.num_inactive)) @@ -262,16 +374,19 @@ num_inactive(vnv::VarNamedVector, idx::Int) = get(vnv.num_inactive, idx, 0) """ num_allocated(vnv::VarNamedVector) + num_allocated(vnv::VarNamedVector[, vn::VarName]) + num_allocated(vnv::VarNamedVector[, idx::Int]) -Returns the number of allocated entries in `vnv`, both active and inactive. -""" -num_allocated(vnv::VarNamedVector) = length(vnv.vals) +Return the number of allocated entries in `vnv`, both active and inactive. -""" - num_allocated(vnv::VarNamedVector, vn::VarName) +If either a `VarName` or an `Int` index is specified, only count entries allocated for that +variable. -Returns the number of allocated entries for `vn` in `vnv`, both active and inactive. +Allocated entries take up memory in `vnv.vals`, but, if inactive, may not currently hold any +meaningful data. One can remove them with [`contiguify!`](@ref), but doing so may cause more +memory allocations in the future if variables change dimension. """ +num_allocated(vnv::VarNamedVector) = length(vnv.vals) num_allocated(vnv::VarNamedVector, vn::VarName) = num_allocated(vnv, getidx(vnv, vn)) function num_allocated(vnv::VarNamedVector, idx::Int) return length(getrange(vnv, idx)) + num_inactive(vnv, idx) @@ -279,12 +394,13 @@ end # Basic array interface. Base.eltype(vnv::VarNamedVector) = eltype(vnv.vals) -Base.length(vnv::VarNamedVector) = +function Base.length(vnv::VarNamedVector) if !has_inactive(vnv) - length(vnv.vals) + return length(vnv.vals) else - sum(length, vnv.ranges) + return sum(length, vnv.ranges) end +end Base.size(vnv::VarNamedVector) = (length(vnv),) Base.isempty(vnv::VarNamedVector) = isempty(vnv.varnames) diff --git a/test/varinfo.jl b/test/varinfo.jl index 382eb7e58..9f41e125e 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -111,6 +111,13 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) @test vi[vn] == 3 * r @test vi[SampleFromPrior()][1] == 3 * r + # TODO(mhauru) Implement these functions for SimpleVarInfo too. + if vi isa VarInfo + delete!(vi, vn) + @test isempty(vi) + vi = push!!(vi, vn, r, dist, gid) + end + vi = empty!!(vi) @test isempty(vi) return push!!(vi, vn, r, dist, gid) diff --git a/test/varnamedvector.jl b/test/varnamedvector.jl index ba365d24a..b3699e54b 100644 --- a/test/varnamedvector.jl +++ b/test/varnamedvector.jl @@ -297,7 +297,10 @@ end # Should not be possible to `push!` existing varname. @test_throws ArgumentError push!(vnv, vn, val) else - push!(vnv, vn, val) + vnv_copy = deepcopy(vnv) + push!(vnv_copy, vn, val) + @test vnv_copy[vn] == val + push!(vnv, (vn => val)) @test vnv[vn] == val end end From 466acb2829eaad970a08f3442ff3c84837d457dd Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Tue, 24 Sep 2024 17:32:51 +0100 Subject: [PATCH 187/209] Simplify VarNamedVector constructors --- src/varnamedvector.jl | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 7661d48eb..6afa2cc1f 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -161,8 +161,8 @@ struct VarNamedVector{ ranges, vals::TVal, transforms::TTrans, - is_unconstrained, - num_inactive, + is_unconstrained=fill!(BitVector(undef, length(varnames)), 0), + num_inactive=OrderedDict{Int,Int}(), ) where {K,V,TVN<:AbstractVector{K},TVal<:AbstractVector{V},TTrans<:AbstractVector} if length(varnames) != length(ranges) || length(varnames) != length(transforms) || @@ -237,28 +237,7 @@ struct VarNamedVector{ end end -# Default values for is_unconstrained (all false) and num_inactive (empty). -function VarNamedVector( - varname_to_index, - varnames, - ranges, - vals, - transforms, - is_unconstrained=fill!(BitVector(undef, length(varnames)), 0), -) - return VarNamedVector( - varname_to_index, - varnames, - ranges, - vals, - transforms, - is_unconstrained, - OrderedDict{Int,Int}(), - ) -end - -# TODO(mhauru) Are we sure we want the last one to be of type Any[]? Might this cause -# unnecessary type instability? +# TODO(mhauru) Are we sure we want the last one to be of type Any[]? function VarNamedVector{K,V}() where {K,V} return VarNamedVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) end From 40909dd26569dc45002d955e597f6e96f4a525ce Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 2 Oct 2024 12:14:52 +0100 Subject: [PATCH 188/209] Change how VNV setindex! et al work --- src/DynamicPPL.jl | 13 + src/context_implementations.jl | 6 +- src/simple_varinfo.jl | 2 +- src/utils.jl | 79 ++- src/varinfo.jl | 13 +- src/varnamedvector.jl | 1197 +++++++++++++++++++------------- test/simple_varinfo.jl | 10 +- test/varnamedvector.jl | 108 ++- 8 files changed, 841 insertions(+), 587 deletions(-) diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index 969d69936..3bd689ffa 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -48,6 +48,19 @@ export AbstractVarInfo, VectorVarInfo, SimpleVarInfo, VarNamedVector, + length_internal, + getindex_internal, + update!, + insert!, + setindex_internal!, + update_internal!, + insert_internal!, + update!!, + insert!!, + setindex!!, + setindex_internal!!, + update_internal!!, + insert_internal!!, push!!, empty!!, subset, diff --git a/src/context_implementations.jl b/src/context_implementations.jl index 1961965ca..f3c5171b0 100644 --- a/src/context_implementations.jl +++ b/src/context_implementations.jl @@ -241,11 +241,13 @@ function assume( # Always overwrite the parameters with new ones for `SampleFromUniform`. if sampler isa SampleFromUniform || is_flagged(vi, vn, "del") # TODO(mhauru) Is it important to unset the flag here? The `true` allows us - # to ignore the fact that for VarNamedVector this does nothing, but I'm unsure if - # that's okay. + # to ignore the fact that for VarNamedVector this does nothing, but I'm unsure + # if that's okay. unset_flag!(vi, vn, "del", true) r = init(rng, dist, sampler) f = to_maybe_linked_internal_transform(vi, vn, dist) + # TODO(mhauru) This should probably be call a function called setindex_internal! + # Also, if we use !! we shouldn't ignore the return value. BangBang.setindex!!(vi, f(r), vn) setorder!(vi, vn, get_num_produce(vi)) else diff --git a/src/simple_varinfo.jl b/src/simple_varinfo.jl index 06a151f82..88f892a72 100644 --- a/src/simple_varinfo.jl +++ b/src/simple_varinfo.jl @@ -420,7 +420,7 @@ function BangBang.push!!( # SimpleVarInfo, push!! allows the key to exist already, for VarNamedVector it does not. # Hence we need to call update!! here, which has the same semantics as push!! does for # SimpleVarInfo. - return Accessors.@set vi.values = update!!(vi.values, vn, value) + return Accessors.@set vi.values = setindex!!(vi.values, value, vn) end const SimpleOrThreadSafeSimple{T,V,C} = Union{ diff --git a/src/utils.jl b/src/utils.jl index fed303021..64918e496 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -226,43 +226,80 @@ invlink_transform(dist) = inverse(link_transform(dist)) ##################################################### """ - UnwrapSingletonTransform + UnwrapSingletonTransform(input_size::InSize) -A transformation that unwraps a singleton array into a scalar. +A transformation that unwraps a singleton array, returning a scalar. -This transformation can be inverted by calling `tovec`. +The `input_size` field is the expected size of the input. In practice this only determines +the number of indices, since all dimensions must be 1 for a singleton. `input_size` is used +to check the validity of the input, but also to determine the correct inverse operation. + +By default `input_size` is `(1,)`, in which case `tovec` is the inverse. """ -struct UnwrapSingletonTransform <: Bijectors.Bijector end +struct UnwrapSingletonTransform{InSize} <: Bijectors.Bijector + input_size::InSize +end -(f::UnwrapSingletonTransform)(x) = only(x) +UnwrapSingletonTransform() = UnwrapSingletonTransform((1,)) + +function (f::UnwrapSingletonTransform)(x) + if size(x) != f.input_size + throw(DimensionMismatch("Expected input of size $(f.input_size), got $(size(x))")) + end + return only(x) +end Bijectors.with_logabsdet_jacobian(f::UnwrapSingletonTransform, x) = (f(x), 0) function Bijectors.with_logabsdet_jacobian( - ::Bijectors.Inverse{<:UnwrapSingletonTransform}, x + inv_f::Bijectors.Inverse{<:UnwrapSingletonTransform}, x ) - return (tovec(x), 0) + f = inv_f.orig + return (reshape([x], f.input_size), 0) end """ - ReshapeTransform(size::Size) + ReshapeTransform(input_size::InSize, output_size::OutSize) + +A `Bijector` that transforms arrays of size `input_size` to arrays of size `output_size`. -A `Bijector` that transforms an `AbstractVector` to a realization of size `size`. +`input_size` is not needed for the implementation of the transformation. It is only used to +check that the input is of the expected size, and to determine the correct inverse +operation. -This transformation can be inverted by calling `tovec`. +By default `input_size` is the vectorized version of `output_size`. In this case this +transformation is the inverse of `tovec` called on an array. """ -struct ReshapeTransform{Size} <: Bijectors.Bijector - size::Size +struct ReshapeTransform{InSize,OutSize} <: Bijectors.Bijector + input_size::InSize + output_size::OutSize +end + +function ReshapeTransform(output_size::Tuple) + input_size = (prod(output_size),) + return ReshapeTransform(input_size, output_size) end ReshapeTransform(x::AbstractArray) = ReshapeTransform(size(x)) # TODO: Should we materialize the `reshape`? -(f::ReshapeTransform)(x) = reshape(x, f.size) +function (f::ReshapeTransform)(x) + if size(x) != f.input_size + throw(DimensionMismatch("Expected input of size $(f.input_size), got $(size(x))")) + end + # The call to `tovec` is only needed in case `x` is a scalar. + return reshape(tovec(x), f.output_size) +end + +function (inv_f::Bijectors.Inverse{<:ReshapeTransform})(x) + f = inv_f.orig + inverse = ReshapeTransform(f.output_size, f.input_size) + return inverse(x) +end Bijectors.with_logabsdet_jacobian(f::ReshapeTransform, x) = (f(x), 0) -# We want to use the inverse of `ReshapeTransform` so it preserves the size information. -function Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ReshapeTransform}, x) - return (tovec(x), 0) + +function Bijectors.with_logabsdet_jacobian(inv_f::Bijectors.Inverse{<:ReshapeTransform}, x) + return (inv_f(x), 0) end struct ToChol <: Bijectors.Bijector @@ -271,6 +308,12 @@ end Bijectors.with_logabsdet_jacobian(f::ToChol, x) = (Cholesky(Matrix(x), f.uplo, 0), 0) Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y::Cholesky) = (y.UL, 0) +function Bijectors.with_logabsdet_jacobian(::Bijectors.Inverse{<:ToChol}, y) + return error( + "Inverse{ToChol} is only defined for Cholesky factorizations. " * + "Got a $(typeof(y)) instead.", + ) +end """ from_vec_transform(x) @@ -335,7 +378,9 @@ end function from_linked_vec_transform(dist::UnivariateDistribution) f_invlink = invlink_transform(dist) f_vec = from_vec_transform(inverse(f_invlink), size(dist)) - return UnwrapSingletonTransform() ∘ f_invlink ∘ f_vec + f_combined = f_invlink ∘ f_vec + sz = Bijectors.output_size(f_combined, size(dist)) + return UnwrapSingletonTransform(sz) ∘ f_combined end # Specializations that circumvent the `from_vec_transform` machinery. diff --git a/src/varinfo.jl b/src/varinfo.jl index 7ee785b1d..12bd8b377 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -691,8 +691,8 @@ function _setall!(metadata::Metadata, val) end function _setall!(vnv::VarNamedVector, val) # TODO(mhauru) Do something more efficient here. - for i in 1:length(vnv) - vnv[i] = val[i] + for i in 1:length_internal(vnv) + setindex_internal!(vnv, val[i], i) end end @generated function _setall!(metadata::NamedTuple{names}, val) where {names} @@ -1444,7 +1444,7 @@ function _link_metadata!!( val_new, logjac2 = with_logabsdet_jacobian(transform_to_linked, val_orig) # TODO(mhauru) We are calling a !! function but ignoring the return value. acclogp!!(varinfo, -logjac1 - logjac2) - metadata = update!!(metadata, vn, val_new, transform_from_linked) + metadata = setindex_internal!!(metadata, val_new, vn, transform_from_linked) settrans!(metadata, true, vn) end return metadata @@ -1560,7 +1560,8 @@ function _invlink_metadata!!( new_val, logjac = with_logabsdet_jacobian(transform, old_val) # TODO(mhauru) We are calling a !! function but ignoring the return value. acclogp!!(varinfo, -logjac) - metadata = update!!(metadata, vn, new_val) + new_transform = from_vec_transform(new_val) + metadata = setindex_internal!!(metadata, tovec(new_val), vn, new_transform) settrans!(metadata, false, vn) end return metadata @@ -1587,7 +1588,7 @@ end @generated function _islinked(vi, vns::NamedTuple{names}) where {names} out = [] for f in names - push!(out, :(length(vns.$f) == 0 ? false : istrans(vi, vns.$f[1]))) + push!(out, :(isempty(vns.$f) ? false : istrans(vi, vns.$f[1]))) end return Expr(:||, false, out...) end @@ -1700,6 +1701,8 @@ end return expr end +# TODO(mhauru) I think the below implementation of setindex! is a mistake. It should be +# called setindex_internal! since it directly writes to the `vals` field of the metadata. """ setindex!(vi::VarInfo, val, vn::VarName) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 6afa2cc1f..e1d8b5314 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -1,22 +1,46 @@ +# TODO(mhauru) The docstring is an unfinished mess. I got stuck when I started to think +# about whether functions other than `setindex_internal!` need to take transformations. """ VarNamedVector A container that stores values in a vectorised form, but indexable by variable names. -When indexed with integers or `Colon`s, e.g. `vnv[2]` or `vnv[:]`, `VarNamedVector` behaves -like a `Vector`, and returns the values as they are stored. The stored form is always -vectorised, for instance matrix variables have been flattened, and may be further -transformed to achieve linking. - -When indexed with `VarName`s, e.g. `vnv[@varname(x)]`, `VarNamedVector` returns the values -in the original space. For instance, a linked matrix variable is first inverse linked and -then reshaped to its original form before returning it to the caller. - -`VarNamedVector` also stores a boolean for whether a variable has been transformed to -unconstrained Euclidean space or not. +A `VarNamedVector` can be thought of as an ordered mapping from `VarName`s to pairs of +`(internal_value, transform)`. Here `internal_value` is a vectorised value for the variable +and `transform` is a function such that `transform(internal_value)` is the "original" value +of the variable, the one that the user sees. For instance, if the variable has a matrix +value, `internal_value` could bea flattened `Vector` of its elements, and `transform` would +be a `reshape` call. + +`transform` may implement simply vectorisation, but it may do more. Most importantly, it may +implement linking, where the internal storage of a random variable is in a form where all +values in Euclidean space are valid. This is useful for sampling, because the sampler can +make changes to `internal_value` without worrying about constraints on the space of +the random variable. + +The way to access this storage format directly is through the functions `getindex_internal` +and `setindex_internal`. The `transform` argument for `setindex_internal` is optional, by +default it is either the identity, or the existing transform if a value already exists for +this `VarName`. + +`VarNamedVector` also provides a `Dict`-like interface that hides away the internal +vectorisation. This can be accessed with `getindex` and `setindex!`. `setindex!` only takes +the value, the transform is automatically set to be a simple vectorisation. The only notable +deviation from the behavior of a `Dict` is that `setindex!` will throw an error if one tries +to set a new value for a variable that lives in a different "space" than the old one (e.g. +is of a different type or size). This is because `setindex!` does not change the transform +of a variable, e.g. preserve linking, and thus the new value must be compatible with the old +transform. + +For now, a third value is in fact stored for each `VarName`: a boolean indicating whether +the variable has been transformed to unconstrained Euclidean space or not. This is only in +place temporarily due to the needs of our old Gibbs sampler. Internally, `VarNamedVector` stores the values of all variables in a single contiguous -vector. +vector. This makes some operations more efficient, and means that one can access the entire +contents of the internal storage quickly with `getindex_internal(vnv, :)`. The other fields +of `VarNamedVector` are mostly used to keep track of which part of the internal storage +belongs to which `VarName`. # Fields @@ -27,27 +51,26 @@ $(FIELDS) The values for different variables are internally all stored in a single vector. For instance, ```jldoctest varnamedvector-struct -julia> using DynamicPPL: VarNamedVector, @varname, push!, update! +julia> using DynamicPPL: VarNamedVector, @varname, setindex!, update! julia> vnv = VarNamedVector(); -julia> push!(vnv, @varname(x) => [0.0, 1.0]); +julia> setindex!(vnv, [0.0, 0.0, 0.0, 0.0], @varname(x)); -julia> push!(vnv, @varname(y) => fill(3, (3,3))); +julia> setindex!(vnv, reshape(1:6, (2,3)), @varname(y)); julia> vnv.vals -11-element Vector{Real}: +10-element Vector{Real}: 0.0 - 1.0 - 3 - 3 - 3 - 3 - 3 - 3 - 3 - 3 + 0.0 + 0.0 + 0.0 + 1 + 2 3 + 4 + 5 + 6 ``` The `varnames`, `ranges`, and `varname_to_index` fields keep track of which value belongs to @@ -56,53 +79,53 @@ the vectorised internal storage back to its original form: ```jldoctest varnamedvector-struct julia> vnv.transforms[vnv.varname_to_index[@varname(y)]] -DynamicPPL.ReshapeTransform{Tuple{Int64, Int64}}((3, 3)) +DynamicPPL.ReshapeTransform{Tuple{Int64}, Tuple{Int64, Int64}}((6,), (2, 3)) ``` If a variable is updated with a new value that is of a smaller dimension than the old value, rather than resizing `vnv.vals`, some elements in `vnv.vals` are marked as inactive. ```jldoctest varnamedvector-struct -julia> update!(vnv, @varname(y), fill(2, (2, 2))) +julia> update!(vnv, [46.0, 48.0], @varname(x)) julia> vnv.vals -11-element Vector{Real}: - 0.0 - 1.0 - 2 - 2 - 2 - 2 - 3 - 3 - 3 - 3 - 3 +10-element Vector{Real}: + 46.0 + 48.0 + 0.0 + 0.0 + 1 + 2 + 3 + 4 + 5 + 6 julia> vnv.num_inactive OrderedDict{Int64, Int64} with 1 entry: - 2 => 5 + 1 => 2 ``` This helps avoid unnecessary memory allocations for values that repeatedly change dimension. The user does not have to worry about the inactive entries as long as they use functions -like `update!` and `getindex!` rather than directly accessing `vnv.vals`. +like `setindex!` and `getindex!` rather than directly accessing `vnv.vals`. ```jldoctest varnamedvector-struct -julia> vnv[@varname(y)] -2×2 Matrix{Real}: - 2 2 - 2 2 - - -julia> vnv[:] -6-element Vector{Real}: - 0.0 - 1.0 - 2 - 2 - 2 - 2 +julia> vnv[@varname(x)] +2-element Vector{Float64}: + 46.0 + 48.0 + +julia> getindex_internal(vnv, :) +8-element Vector{Real}: + 46.0 + 48.0 + 1 + 2 + 3 + 4 + 5 + 6 ``` """ struct VarNamedVector{ @@ -254,13 +277,13 @@ function VarNamedVector(varnames, vals) end function VarNamedVector( varnames::AbstractVector, - vals::AbstractVector, + orig_vals::AbstractVector, transforms=fill(identity, length(varnames)), ) # Convert `vals` into a vector of vectors. - vals_vecs = map(tovec, vals) + vals_vecs = map(tovec, orig_vals) transforms = map( - (t, val) -> _compose_no_identity(t, from_vec_transform(val)), transforms, vals + (t, val) -> _compose_no_identity(t, from_vec_transform(val)), transforms, orig_vals ) # TODO: Is this really the way to do this? if !(eltype(varnames) <: VarName) @@ -371,29 +394,33 @@ function num_allocated(vnv::VarNamedVector, idx::Int) return length(getrange(vnv, idx)) + num_inactive(vnv, idx) end -# Basic array interface. +# Basic Dictionary interface. Base.eltype(vnv::VarNamedVector) = eltype(vnv.vals) -function Base.length(vnv::VarNamedVector) - if !has_inactive(vnv) - return length(vnv.vals) - else - return sum(length, vnv.ranges) - end -end -Base.size(vnv::VarNamedVector) = (length(vnv),) Base.isempty(vnv::VarNamedVector) = isempty(vnv.varnames) - Base.IndexStyle(::Type{<:VarNamedVector}) = IndexLinear() # Dictionary interface. +Base.length(vnv::VarNamedVector) = length(vnv.varnames) Base.keys(vnv::VarNamedVector) = vnv.varnames Base.values(vnv::VarNamedVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) Base.pairs(vnv::VarNamedVector) = (vn => vnv[vn] for vn in keys(vnv)) - Base.haskey(vnv::VarNamedVector, vn::VarName) = haskey(vnv.varname_to_index, vn) -# `getindex` & `setindex!` -Base.getindex(vnv::VarNamedVector, i::Int) = getindex_internal(vnv, i) +""" + length_internal(vnv::VarNamedVector) + +Return the length of the internal storage vector of `vnv`, ignoring inactive entries. +""" +function length_internal(vnv::VarNamedVector) + if !has_inactive(vnv) + return length(vnv.vals) + else + return sum(length, vnv.ranges) + end +end + +# Getting and setting values + function Base.getindex(vnv::VarNamedVector, vn::VarName) x = getindex_internal(vnv, vn) f = gettransform(vnv, vn) @@ -464,18 +491,13 @@ function index_to_vals_index(vnv::VarNamedVector, i::Int) end """ - getindex_internal(vnv::VarNamedVector, i::Int) getindex_internal(vnv::VarNamedVector, vn::VarName) -Like `getindex`, but returns the values as they are stored in `vnv` without transforming. - -For integer indices this is the same as `getindex`, but for `VarName`s this is different. +Like `getindex`, but returns the values as they are stored in `vnv`, without transforming. """ -getindex_internal(vnv::VarNamedVector, i::Int) = vnv.vals[index_to_vals_index(vnv, i)] getindex_internal(vnv::VarNamedVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] -# `getindex` for `Colon` -function Base.getindex(vnv::VarNamedVector, ::Colon) +function getindex_internal(vnv::VarNamedVector, ::Colon) return if has_inactive(vnv) mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) else @@ -483,220 +505,654 @@ function Base.getindex(vnv::VarNamedVector, ::Colon) end end -getindex_internal(vnv::VarNamedVector, ::Colon) = getindex(vnv, Colon()) - # TODO(mhauru): Remove this as soon as possible. Only needed because of the old Gibbs # sampler. function Base.getindex(vnv::VarNamedVector, spl::AbstractSampler) throw(ErrorException("Cannot index a VarNamedVector with a sampler.")) end -Base.setindex!(vnv::VarNamedVector, val, i::Int) = setindex_internal!(vnv, val, i) function Base.setindex!(vnv::VarNamedVector, val, vn::VarName) - # Since setindex! does not change the transform, we need to apply it to `val`. - f = inverse(gettransform(vnv, vn)) - return setindex_internal!(vnv, f(val), vn) + if haskey(vnv, vn) + return update!(vnv, val, vn) + else + return insert!(vnv, val, vn) + end end """ - setindex_internal!(vnv::VarNamedVector, val, i::Int) - setindex_internal!(vnv::VarNamedVector, val, vn::VarName) - -Like `setindex!`, but sets the values as they are stored in `vnv` without transforming. - -For integer indices this is the same as `setindex!`, but for `VarName`s this is different. -""" -function setindex_internal!(vnv::VarNamedVector, val, i::Int) - return vnv.vals[index_to_vals_index(vnv, i)] = val -end - -function setindex_internal!(vnv::VarNamedVector, val::AbstractVector, vn::VarName) - return vnv.vals[getrange(vnv, vn)] = val -end + reset!(vnv::VarNamedVector, val, vn::VarName) -function Base.empty!(vnv::VarNamedVector) - # TODO: Or should the semantics be different, e.g. keeping `varnames`? - empty!(vnv.varname_to_index) - empty!(vnv.varnames) - empty!(vnv.ranges) - empty!(vnv.vals) - empty!(vnv.transforms) - empty!(vnv.is_unconstrained) - empty!(vnv.num_inactive) - return nothing -end -BangBang.empty!!(vnv::VarNamedVector) = (empty!(vnv); return vnv) +Reset the value of `vn` in `vnv` to `val`. -""" - replace_values(vnv::VarNamedVector, vals::AbstractVector) +This differs from `setindex!` in that it will always change the transform of the variable +to be the default vectorisation transform. This undoes any possible linking. -Replace the values in `vnv` with `vals`, as they are stored internally. +# Examples -This is useful when we want to update the entire underlying vector of values in one go or if -we want to change the how the values are stored, e.g. alter the `eltype`. +```jldoctest varnamedvector-reset +julia> using DynamicPPL: VarNamedVector, @varname, reset! -!!! warning - This replaces the raw underlying values, and so care should be taken when using this - function. For example, if `vnv` has any inactive entries, then the provided `vals` - should also contain the inactive entries to avoid unexpected behavior. +julia> vnv = VarNamedVector(); -# Examples +julia> vnv[@varname(x)] = reshape(1:9, (3, 3)); -```jldoctest varnamedvector-replace-values -julia> using DynamicPPL: VarNamedVector, replace_values +julia> setindex!(vnv, 2.0, @varname(x)) +ERROR: An error occurred while assigning the value 2.0 to variable x. If you are changing the type or size of a variable you'll need to call reset! +[...] -julia> vnv = VarNamedVector(@varname(x) => [1.0]); +julia> reset!(vnv, 2.0, @varname(x)); -julia> replace_values(vnv, [2.0])[@varname(x)] == [2.0] -true +julia> vnv[@varname(x)] +2.0 ``` +""" +function reset!(vnv::VarNamedVector, val, vn::VarName) + f = from_vec_transform(val) + retval = setindex_internal!(vnv, tovec(val), vn, f) + settrans!(vnv, false, vn) + return retval +end -This is also useful when we want to differentiate wrt. the values using automatic -differentiation, e.g. ForwardDiff.jl. - -```jldoctest varnamedvector-replace-values -julia> using ForwardDiff: ForwardDiff +""" + update!(vnv::VarNamedVector, val, vn::VarName) -julia> f(x) = sum(abs2, replace_values(vnv, x)[@varname(x)]) -f (generic function with 1 method) +Update the value of `vn` in `vnv` to `val`. -julia> ForwardDiff.gradient(f, [1.0]) -1-element Vector{Float64}: - 2.0 -``` +Like `setindex!`, but errors if the key `vn` doesn't exist. """ -replace_values(vnv::VarNamedVector, vals) = Accessors.@set vnv.vals = vals - -# TODO(mhauru) The space argument is used by the old Gibbs sampler. To be removed. -function replace_values(vnv::VarNamedVector, ::Val{space}, vals) where {space} - if length(space) > 0 - msg = "Selecting values in a VarNamedVector with a space is not supported." - throw(ArgumentError(msg)) +function update!(vnv::VarNamedVector, val, vn::VarName) + if !haskey(vnv, vn) + throw(KeyError(vn)) + end + f = inverse(gettransform(vnv, vn)) + internal_val = try + f(val) + catch + error( + "An error occurred while assigning the value $val to variable $vn. " * + "If you are changing the type or size of a variable you'll need to call " * + "reset!", + ) end - return replace_values(vnv, vals) + return setindex_internal!(vnv, internal_val, vn) end """ - unflatten(vnv::VarNamedVector, vals::AbstractVector) - -Return a new instance of `vnv` with the values of `vals` assigned to the variables. + insert!(vnv::VarNamedVector, val, vn::VarName) -This assumes that `vals` have been transformed by the same transformations that that the -values in `vnv` have been transformed by. However, unlike [`replace_values`](@ref), -`unflatten` does account for inactive entries in `vnv`, so that the user does not have to -care about them. +Add a variable with given value to `vnv`. -This is in a sense the reverse operation of `vnv[:]`. +Like `setindex!`, but errors if the key `vn` already exists. +""" +function insert!(vnv::VarNamedVector, val, vn::VarName) + if haskey(vnv, vn) + throw("Variable $vn already exists in VarNamedVector.") + end + return reset!(vnv, val, vn) +end -Unflatten recontiguifies the internal storage, getting rid of any inactive entries. +""" + push!(vnv::VarNamedVector, pair::Pair) -# Examples +Add a variable with given value to `vnv`. Pair should be a `VarName` and a value. +""" +function Base.push!(vnv::VarNamedVector, pair::Pair) + vn, val = pair + # TODO(mhauru) Or should this rather call `reset!`? It would be more inline with what + # Dict does, but could also cause confusion. + return setindex!(vnv, val, vn) +end -```jldoctest varnamedvector-unflatten -julia> using DynamicPPL: VarNamedVector, unflatten +""" + setindex_internal!(vnv::VarNamedVector, val, i::Int) + setindex_internal!(vnv::VarNamedVector, val, vn::VarName[, transform]) -julia> vnv = VarNamedVector(@varname(x) => [1.0, 2.0], @varname(y) => [3.0]); +Like `setindex!`, but sets the values as they are stored internally in `vnv`. -julia> unflatten(vnv, vnv[:]) == vnv -true +Optionally can set the transformation, such that `transform(val)` is the original value of +the variable. By default, the transform is the identity if creating a new entry in `vnv`, or +the existing transform if updating an existing entry. """ -function unflatten(vnv::VarNamedVector, vals::AbstractVector) - new_ranges = deepcopy(vnv.ranges) - recontiguify_ranges!(new_ranges) - return VarNamedVector( - vnv.varname_to_index, vnv.varnames, new_ranges, vals, vnv.transforms - ) +function setindex_internal!(vnv::VarNamedVector, val, i::Int) + return vnv.vals[index_to_vals_index(vnv, i)] = val end -# TODO(mhauru) To be removed once the old Gibbs sampler is removed. -function unflatten(vnv::VarNamedVector, spl::AbstractSampler, vals::AbstractVector) - if length(getspace(spl)) > 0 - msg = "Selecting values in a VarNamedVector with a space is not supported." - throw(ArgumentError(msg)) +function setindex_internal!( + vnv::VarNamedVector, val::AbstractVector, vn::VarName, transform=nothing +) + if haskey(vnv, vn) + return update_internal!(vnv, val, vn, transform) + else + return insert_internal!(vnv, val, vn, transform) end - return unflatten(vnv, vals) end -function Base.merge(left_vnv::VarNamedVector, right_vnv::VarNamedVector) - # Return early if possible. - isempty(left_vnv) && return deepcopy(right_vnv) - isempty(right_vnv) && return deepcopy(left_vnv) +""" + insert_internal!(vnv::VarNamedVector, val::AbstractVector, vn::VarName[, transform]) - # Determine varnames. - vns_left = left_vnv.varnames - vns_right = right_vnv.varnames - vns_both = union(vns_left, vns_right) +Add a variable with given value to `vnv`. - # Determine `eltype` of `vals`. - T_left = eltype(left_vnv.vals) - T_right = eltype(right_vnv.vals) - T = promote_type(T_left, T_right) +Like `setindex_internal!`, but errors if the key `vn` already exists. - # Determine `eltype` of `varnames`. - V_left = eltype(left_vnv.varnames) - V_right = eltype(right_vnv.varnames) - V = promote_type(V_left, V_right) - if !(V <: VarName) - V = VarName +`transform` should be a function that converts `val` to the original representation. By +default it's `identity`. +""" +function insert_internal!( + vnv::VarNamedVector, val::AbstractVector, vn::VarName, transform=nothing +) + if transform === nothing + transform = identity end + haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) + # NOTE: We need to compute the `nextrange` BEFORE we start mutating the underlying + # storage. + r_new = nextrange(vnv, val) + vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 + push!(vnv.varnames, vn) + push!(vnv.ranges, r_new) + append!(vnv.vals, val) + push!(vnv.transforms, transform) + push!(vnv.is_unconstrained, false) + return nothing +end - # Determine `eltype` of `transforms`. - F_left = eltype(left_vnv.transforms) - F_right = eltype(right_vnv.transforms) - F = promote_type(F_left, F_right) +""" + update_internal!(vnv::VarNamedVector, vn::VarName, val::AbstractVector[, transform]) - # Allocate. - varname_to_index = OrderedDict{V,Int}() - ranges = UnitRange{Int}[] - vals = T[] - transforms = F[] - is_unconstrained = BitVector(undef, length(vns_both)) +Update an existing entry for `vn` in `vnv` with the value `val`. - # Range offset. - offset = 0 +Like `setindex_internal!`, but errors if the key `vn` doesn't exist. - for (idx, vn) in enumerate(vns_both) - varname_to_index[vn] = idx - # Extract the necessary information from `left` or `right`. - if vn in vns_left && !(vn in vns_right) - # `vn` is only in `left`. - val = getindex_internal(left_vnv, vn) - f = gettransform(left_vnv, vn) - is_unconstrained[idx] = istrans(left_vnv, vn) - else - # `vn` is either in both or just `right`. - # Note that in a `merge` the right value has precedence. - val = getindex_internal(right_vnv, vn) - f = gettransform(right_vnv, vn) - is_unconstrained[idx] = istrans(right_vnv, vn) - end - n = length(val) - r = (offset + 1):(offset + n) - # Update. - append!(vals, val) - push!(ranges, r) - push!(transforms, f) - # Increment `offset`. - offset += n +`transform` should be a function that converts `val` to the original representation. By +default it's the same as the old transform for `vn`. +""" +function update_internal!( + vnv::VarNamedVector, val::AbstractVector, vn::VarName, transform=nothing +) + # Here we update an existing entry. + if !haskey(vnv, vn) + throw(KeyError(vn)) end + idx = getidx(vnv, vn) + # Extract the old range. + r_old = getrange(vnv, idx) + start_old, end_old = first(r_old), last(r_old) + n_old = length(r_old) + # Compute the new range. + n_new = length(val) + start_new = start_old + end_new = start_old + n_new - 1 + r_new = start_new:end_new - return VarNamedVector( - varname_to_index, vns_both, ranges, vals, transforms, is_unconstrained - ) -end + #= + Suppose we currently have the following: -""" - subset(vnv::VarNamedVector, vns::AbstractVector{<:VarName}) + | x | x | o | o | o | y | y | y | <- Current entries -Return a new `VarNamedVector` containing the values from `vnv` for variables in `vns`. + where 'O' denotes an inactive entry, and we're going to + update the variable `x` to be of size `k` instead of 2. -Which variables to include is determined by the `VarName`'s `subsumes` relation, meaning -that e.g. `subset(vnv, [@varname(x)])` will include variables like `@varname(x.a[1])`. + We then have a few different scenarios: + 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. + E.g. if `k = 7`, then -# Examples + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | x | x | y | y | y | <- New entries -```jldoctest varnamedvector-subset -julia> using DynamicPPL: VarNamedVector, @varname, subset + 2. `k = 5`: All inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | x | x | y | y | y | <- New entries + + 3. `k < 5`: Some inactive entries become active, some remain inactive. + E.g. if `k = 3`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | x | o | o | y | y | y | <- New entries + + 4. `k = 2`: No inactive entries become active. + Then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | x | o | o | o | y | y | y | <- New entries + + 5. `k < 2`: More entries become inactive. + E.g. if `k = 1`, then + + | x | x | o | o | o | y | y | y | <- Current entries + | x | o | o | o | o | y | y | y | <- New entries + =# + + # Compute the allocated space for `vn`. + had_inactive = haskey(vnv.num_inactive, idx) + n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old + + if n_new > n_allocated + # Then we need to grow the underlying vector. + n_extra = n_new - n_allocated + # Allocate. + resize!(vnv.vals, length(vnv.vals) + n_extra) + # Shift current values. + shift_right!(vnv.vals, end_old + 1, n_extra) + # No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) + # Update the ranges for all variables after this one. + shift_subsequent_ranges_by!(vnv, idx, n_extra) + elseif n_new == n_allocated + # => No more inactive entries. + had_inactive && delete!(vnv.num_inactive, idx) + else + # `n_new < n_allocated` + # => Need to update the number of inactive entries. + vnv.num_inactive[idx] = n_allocated - n_new + end + + # Update the range for this variable. + vnv.ranges[idx] = r_new + # Update the value. + vnv.vals[r_new] = val + if transform !== nothing + # Update the transform. + vnv.transforms[idx] = transform + end + + # TODO: Should we maybe sweep over inactive ranges and re-contiguify + # if the total number of inactive elements is "large" in some sense? + + return nothing +end + +# TODO(mhauru) The gidset and num_produce arguments are used by the old Gibbs sampler. +# Remove this method as soon as possible. +function BangBang.push!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) + f = from_vec_transform(dist) + return setindex_internal!(vnv, tovec(val), vn, f) +end + +# BangBang versions of the above functions. +# The only difference is that update_internal!! and insert_internal!! check whether the +# container types of the VarNamedVector vector need to be expanded to accommodate the new +# values. If so, they create a new instance, otherwise they mutate in place. All the others +# functions, e.g. setindex!!, setindex_internal!!, etc., are carbon copies of the ! versions +# with every ! call replaced with a !! call. + +""" + loosen_types!!(vnv::VarNamedVector{K,V,TVN,TVal,TTrans}, ::Type{KNew}, ::Type{TransNew}) + +Loosen the types of `vnv` to allow varname type `KNew` and transformation type `TransNew`. + +If `KNew` is a subtype of `K` and `TransNew` is a subtype of the element type of the +`TTrans` then this is a no-op and `vnv` is returned as is. Otherwise a new `VarNamedVector` +is returned with the same data but more abstract types, so that variables of type `KNew` and +transformations of type `TransNew` can be pushed to it. Some of the underlying storage is +shared between `vnv` and the return value, and thus mutating one may affect the other. + +# See also +[`tighten_types`](@ref) + +# Examples + +```jldoctest varnamedvector-loosen-types +julia> using DynamicPPL: VarNamedVector, @varname, loosen_types!!, setindex_internal! + +julia> vnv = VarNamedVector(@varname(x) => [1.0]); + +julia> y_trans(x) = reshape(x, (2, 2)); + +julia> setindex_internal!(vnv, collect(1:4), @varname(y), y_trans) +ERROR: MethodError: Cannot `convert` an object of type + VarName{y,typeof(identity)} to an object of type + VarName{x,typeof(identity)} +[...] + +julia> vnv_loose = DynamicPPL.loosen_types!!(vnv, typeof(@varname(y)), typeof(y_trans)); + +julia> setindex_internal!(vnv_loose, collect(1:4), @varname(y), y_trans) + +julia> vnv_loose[@varname(y)] +2×2 Matrix{Float64}: + 1.0 3.0 + 2.0 4.0 +``` +""" +function loosen_types!!( + vnv::VarNamedVector, ::Type{KNew}, ::Type{TransNew} +) where {KNew,TransNew} + K = eltype(vnv.varnames) + Trans = eltype(vnv.transforms) + if KNew <: K && TransNew <: Trans + return vnv + else + vn_type = promote_type(K, KNew) + transform_type = promote_type(Trans, TransNew) + return VarNamedVector( + OrderedDict{vn_type,Int}(vnv.varname_to_index), + Vector{vn_type}(vnv.varnames), + vnv.ranges, + vnv.vals, + Vector{transform_type}(vnv.transforms), + vnv.is_unconstrained, + vnv.num_inactive, + ) + end +end + +""" + tighten_types(vnv::VarNamedVector) + +Return a copy of `vnv` with the most concrete types possible. + +For instance, if `vnv` has its vector of transforms have eltype `Any`, but all the +transforms are actually identity transformations, this function will return a new +`VarNamedVector` with the transforms vector having eltype `typeof(identity)`. + +This is a lot like the reverse of [`loosen_types!!`](@ref), but with two notable +differences: Unlike `loosen_types!!`, this function does not mutate `vnv`; it also changes +not only the key and transform eltypes, but also the values eltype. + +# See also +[`loosen_types!!`](@ref) + +# Examples + +```jldoctest varnamedvector-tighten-types +julia> using DynamicPPL: VarNamedVector, @varname, loosen_types!!, setindex_internal! + +julia> vnv = VarNamedVector(); + +julia> setindex!(vnv, [23], @varname(x)) + +julia> eltype(vnv) +Real + +julia> vnv.transforms +1-element Vector{Any}: + identity (generic function with 1 method) + +julia> vnv_tight = DynamicPPL.tighten_types(vnv) +VarNamedVector{VarName{:y, typeof(identity)}, Int64, Vector{VarName{:y, typeof(identity)}}, Vector{Int64}, Vector{typeof(identity)}}(OrderedDict(y => 1), [y], UnitRange{Int64}[1:1], [23], [identity], Bool[0], OrderedDict{Int64, Int64}()) + +julia> eltype(vnv_tight) +Int64 + +julia> vnv_tight.transforms +1-element Vector{typeof(identity)}: + identity (generic function with 1 method) +``` +""" +function tighten_types(vnv::VarNamedVector) + return VarNamedVector( + OrderedDict(vnv.varname_to_index...), + map(identity, vnv.varnames), + copy(vnv.ranges), + map(identity, vnv.vals), + map(identity, vnv.transforms), + copy(vnv.is_unconstrained), + copy(vnv.num_inactive), + ) +end + +function BangBang.setindex!!(vnv::VarNamedVector, val, vn::VarName) + if haskey(vnv, vn) + return update!!(vnv, val, vn) + else + return insert!!(vnv, val, vn) + end +end + +function reset!!(vnv::VarNamedVector, val, vn::VarName) + f = from_vec_transform(val) + vnv = setindex_internal!!(vnv, tovec(val), vn, f) + vnv = settrans!!(vnv, false, vn) + return vnv +end + +function update!!(vnv::VarNamedVector, val, vn::VarName) + if !haskey(vnv, vn) + throw(KeyError(vn)) + end + f = inverse(gettransform(vnv, vn)) + internal_val = try + f(val) + catch + error( + "An error occurred while assigning the value $val to variable $vn. " * + "If you are changing the type or size of a variable you'll need to either " * + "`delete!` it first or use `setindex_internal!`", + ) + end + return setindex_internal!!(vnv, internal_val, vn) +end + +function insert!!(vnv::VarNamedVector, val, vn::VarName) + if haskey(vnv, vn) + throw("Variable $vn already exists in VarNamedVector.") + end + return reset!!(vnv, val, vn) +end + +function setindex_internal!!( + vnv::VarNamedVector, val::AbstractVector, vn::VarName, transform=nothing +) + if haskey(vnv, vn) + return update_internal!!(vnv, val, vn, transform) + else + return insert_internal!!(vnv, val, vn, transform) + end +end + +function insert_internal!!(vnv::VarNamedVector, val, vn::VarName, transform=nothing) + if transform === nothing + transform = identity + end + vnv = loosen_types!!(vnv, typeof(vn), typeof(transform)) + insert_internal!(vnv, val, vn, transform) + return vnv +end + +function update_internal!!(vnv::VarNamedVector, val, vn::VarName, transform=nothing) + transform_resolved = transform === nothing ? gettransform(vnv, vn) : transform + vnv = loosen_types!!(vnv, typeof(vn), typeof(transform_resolved)) + update_internal!(vnv, val, vn, transform) + return vnv +end + +function BangBang.push!!(vnv::VarNamedVector, pair::Pair) + vn, val = pair + return setindex!!(vnv, val, vn) +end + +# TODO(mhauru) The gidset and num_produce arguments are used by the old Gibbs sampler. +# Remove this method as soon as possible. +function BangBang.push!!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) + f = from_vec_transform(dist) + return setindex_internal!!(vnv, tovec(val), vn, f) +end + +function Base.empty!(vnv::VarNamedVector) + # TODO: Or should the semantics be different, e.g. keeping `varnames`? + empty!(vnv.varname_to_index) + empty!(vnv.varnames) + empty!(vnv.ranges) + empty!(vnv.vals) + empty!(vnv.transforms) + empty!(vnv.is_unconstrained) + empty!(vnv.num_inactive) + return nothing +end +BangBang.empty!!(vnv::VarNamedVector) = (empty!(vnv); return vnv) + +""" + replace_raw_storage(vnv::VarNamedVector, vals::AbstractVector) + +Replace the values in `vnv` with `vals`, as they are stored internally. + +This is useful when we want to update the entire underlying vector of values in one go or if +we want to change the how the values are stored, e.g. alter the `eltype`. + +!!! warning + This replaces the raw underlying values, and so care should be taken when using this + function. For example, if `vnv` has any inactive entries, then the provided `vals` + should also contain the inactive entries to avoid unexpected behavior. + +# Examples + +```jldoctest varnamedvector-replace-raw-storage +julia> using DynamicPPL: VarNamedVector, replace_raw_storage + +julia> vnv = VarNamedVector(@varname(x) => [1.0]); + +julia> replace_raw_storage(vnv, [2.0])[@varname(x)] == [2.0] +true +``` + +This is also useful when we want to differentiate wrt. the values using automatic +differentiation, e.g. ForwardDiff.jl. + +```jldoctest varnamedvector-replace-raw-storage +julia> using ForwardDiff: ForwardDiff + +julia> f(x) = sum(abs2, replace_raw_storage(vnv, x)[@varname(x)]) +f (generic function with 1 method) + +julia> ForwardDiff.gradient(f, [1.0]) +1-element Vector{Float64}: + 2.0 +``` +""" +replace_raw_storage(vnv::VarNamedVector, vals) = Accessors.@set vnv.vals = vals + +# TODO(mhauru) The space argument is used by the old Gibbs sampler. To be removed. +function replace_raw_storage(vnv::VarNamedVector, ::Val{space}, vals) where {space} + if length(space) > 0 + msg = "Selecting values in a VarNamedVector with a space is not supported." + throw(ArgumentError(msg)) + end + return replace_raw_storage(vnv, vals) +end + +""" + unflatten(vnv::VarNamedVector, vals::AbstractVector) + +Return a new instance of `vnv` with the values of `vals` assigned to the variables. + +This assumes that `vals` have been transformed by the same transformations that that the +values in `vnv` have been transformed by. However, unlike [`replace_raw_storage`](@ref), +`unflatten` does account for inactive entries in `vnv`, so that the user does not have to +care about them. + +This is in a sense the reverse operation of `vnv[:]`. + +Unflatten recontiguifies the internal storage, getting rid of any inactive entries. + +# Examples + +```jldoctest varnamedvector-unflatten +julia> using DynamicPPL: VarNamedVector, unflatten + +julia> vnv = VarNamedVector(@varname(x) => [1.0, 2.0], @varname(y) => [3.0]); + +julia> unflatten(vnv, vnv[:]) == vnv +true +""" +function unflatten(vnv::VarNamedVector, vals::AbstractVector) + new_ranges = deepcopy(vnv.ranges) + recontiguify_ranges!(new_ranges) + return VarNamedVector( + vnv.varname_to_index, vnv.varnames, new_ranges, vals, vnv.transforms + ) +end + +# TODO(mhauru) To be removed once the old Gibbs sampler is removed. +function unflatten(vnv::VarNamedVector, spl::AbstractSampler, vals::AbstractVector) + if length(getspace(spl)) > 0 + msg = "Selecting values in a VarNamedVector with a space is not supported." + throw(ArgumentError(msg)) + end + return unflatten(vnv, vals) +end + +function Base.merge(left_vnv::VarNamedVector, right_vnv::VarNamedVector) + # Return early if possible. + isempty(left_vnv) && return deepcopy(right_vnv) + isempty(right_vnv) && return deepcopy(left_vnv) + + # Determine varnames. + vns_left = left_vnv.varnames + vns_right = right_vnv.varnames + vns_both = union(vns_left, vns_right) + + # Determine `eltype` of `vals`. + T_left = eltype(left_vnv.vals) + T_right = eltype(right_vnv.vals) + T = promote_type(T_left, T_right) + + # Determine `eltype` of `varnames`. + V_left = eltype(left_vnv.varnames) + V_right = eltype(right_vnv.varnames) + V = promote_type(V_left, V_right) + if !(V <: VarName) + V = VarName + end + + # Determine `eltype` of `transforms`. + F_left = eltype(left_vnv.transforms) + F_right = eltype(right_vnv.transforms) + F = promote_type(F_left, F_right) + + # Allocate. + varname_to_index = OrderedDict{V,Int}() + ranges = UnitRange{Int}[] + vals = T[] + transforms = F[] + is_unconstrained = BitVector(undef, length(vns_both)) + + # Range offset. + offset = 0 + + for (idx, vn) in enumerate(vns_both) + varname_to_index[vn] = idx + # Extract the necessary information from `left` or `right`. + if vn in vns_left && !(vn in vns_right) + # `vn` is only in `left`. + val = getindex_internal(left_vnv, vn) + f = gettransform(left_vnv, vn) + is_unconstrained[idx] = istrans(left_vnv, vn) + else + # `vn` is either in both or just `right`. + # Note that in a `merge` the right value has precedence. + val = getindex_internal(right_vnv, vn) + f = gettransform(right_vnv, vn) + is_unconstrained[idx] = istrans(right_vnv, vn) + end + n = length(val) + r = (offset + 1):(offset + n) + # Update. + append!(vals, val) + push!(ranges, r) + push!(transforms, f) + # Increment `offset`. + offset += n + end + + return VarNamedVector( + varname_to_index, vns_both, ranges, vals, transforms, is_unconstrained + ) +end + +""" + subset(vnv::VarNamedVector, vns::AbstractVector{<:VarName}) + +Return a new `VarNamedVector` containing the values from `vnv` for variables in `vns`. + +Which variables to include is determined by the `VarName`'s `subsumes` relation, meaning +that e.g. `subset(vnv, [@varname(x)])` will include variables like `@varname(x.a[1])`. + +# Examples + +```jldoctest varnamedvector-subset +julia> using DynamicPPL: VarNamedVector, @varname, subset julia> vnv = VarNamedVector(@varname(x) => [1.0, 2.0], @varname(y) => [3.0]); @@ -716,7 +1172,7 @@ function subset(vnv::VarNamedVector, vns_given::AbstractVector{VN}) where {VN<:V isempty(vnv) && return vnv_new for vn in vns - push!(vnv_new, vn, getindex_internal(vnv, vn), gettransform(vnv, vn)) + insert_internal!(vnv_new, getindex_internal(vnv, vn), vn, gettransform(vnv, vn)) settrans!(vnv_new, istrans(vnv, vn), vn) end @@ -795,142 +1251,6 @@ _compose_no_identity(::typeof(identity), g) = g _compose_no_identity(f, ::typeof(identity)) = f _compose_no_identity(::typeof(identity), ::typeof(identity)) = identity -""" - push!(vnv::VarNamedVector, vn::VarName, val[, transform]) - push!(vnv::VarNamedVector, vn => val[, transform]) - -Add a variable with given value to `vnv`. - -`transform` should be a function that converts `val` to the original representation, by -default it's `identity`. -""" -function Base.push!(vnv::VarNamedVector, vn::VarName, val, transform=identity) - # Error if we already have the variable. - haskey(vnv, vn) && throw(ArgumentError("variable name $vn already exists")) - # NOTE: We need to compute the `nextrange` BEFORE we start mutating the underlying - # storage. - if !(val isa AbstractVector) - val_vec = tovec(val) - transform = _compose_no_identity(transform, from_vec_transform(val)) - else - val_vec = val - end - r_new = nextrange(vnv, val_vec) - vnv.varname_to_index[vn] = length(vnv.varname_to_index) + 1 - push!(vnv.varnames, vn) - push!(vnv.ranges, r_new) - append!(vnv.vals, val_vec) - push!(vnv.transforms, transform) - push!(vnv.is_unconstrained, false) - return nothing -end - -function Base.push!(vnv::VarNamedVector, pair, transform=identity) - vn, val = pair - return push!(vnv, vn, val, transform) -end - -# TODO(mhauru) The gidset and num_produce arguments are used by the old Gibbs sampler. -# Remove this method as soon as possible. -function Base.push!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) - f = from_vec_transform(dist) - return push!(vnv, vn, tovec(val), f) -end - -""" - loosen_types!!(vnv::VarNamedVector{K,V,TVN,TVal,TTrans}, ::Type{KNew}, ::Type{TransNew}) - -Loosen the types of `vnv` to allow varname type `KNew` and transformation type `TransNew`. - -If `KNew` is a subtype of `K` and `TransNew` is a subtype of the element type of the -`TTrans` then this is a no-op and `vnv` is returned as is. Otherwise a new `VarNamedVector` -is returned with the same data but more abstract types, so that variables of type `KNew` and -transformations of type `TransNew` can be pushed to it. Some of the underlying storage is -shared between `vnv` and the return value, and thus mutating one may affect the other. - -# See also -[`tighten_types`](@ref) - -# Examples - -```jldoctest varnamedvector-loosen-types -julia> using DynamicPPL: VarNamedVector, @varname, loosen_types!! - -julia> vnv = VarNamedVector(@varname(x) => [1.0]); - -julia> vnv_new = loosen_types!!(vnv, VarName{:x}, Real); - -julia> push!(vnv, @varname(y), Float32[2.0]) -ERROR: MethodError: Cannot `convert` an object of type - VarName{y,typeof(identity)} to an object of type - VarName{x,typeof(identity)} -[...] - -julia> vnv_loose = DynamicPPL.loosen_types!!(vnv, typeof(@varname(y)), Float32); - -julia> push!(vnv_loose, @varname(y), Float32[2.0]); vnv_loose # Passes without issues. -VarNamedVector{VarName{sym, typeof(identity)} where sym, Float64, Vector{VarName{sym, typeof(identity)} where sym}, Vector{Float64}, Vector{Any}}(OrderedDict{VarName{sym, typeof(identity)} where sym, Int64}(x => 1, y => 2), VarName{sym, typeof(identity)} where sym[x, y], UnitRange{Int64}[1:1, 2:2], [1.0, 2.0], Any[identity, identity], Bool[0, 0], OrderedDict{Int64, Int64}()) -""" -function loosen_types!!( - vnv::VarNamedVector, ::Type{KNew}, ::Type{TransNew} -) where {KNew,TransNew} - K = eltype(vnv.varnames) - Trans = eltype(vnv.transforms) - if KNew <: K && TransNew <: Trans - return vnv - else - vn_type = promote_type(K, KNew) - transform_type = promote_type(Trans, TransNew) - return VarNamedVector( - OrderedDict{vn_type,Int}(vnv.varname_to_index), - Vector{vn_type}(vnv.varnames), - vnv.ranges, - vnv.vals, - Vector{transform_type}(vnv.transforms), - vnv.is_unconstrained, - vnv.num_inactive, - ) - end -end - -""" - tighten_types(vnv::VarNamedVector) - -Return a copy of `vnv` with the most concrete types possible. - -For instance, if `vnv` has element type `Real`, but all the values are actually `Float64`s, -then `tighten_types(vnv)` will have element type `Float64`. - -# See also -[`loosen_types!!`](@ref) -""" -function tighten_types(vnv::VarNamedVector) - return VarNamedVector( - OrderedDict(vnv.varname_to_index...), - map(identity, vnv.varnames), - copy(vnv.ranges), - map(identity, vnv.vals), - map(identity, vnv.transforms), - copy(vnv.is_unconstrained), - copy(vnv.num_inactive), - ) -end - -function BangBang.push!!(vnv::VarNamedVector, vn::VarName, val, transform=identity) - vnv = loosen_types!!( - vnv, typeof(vn), typeof(_compose_no_identity(transform, from_vec_transform(val))) - ) - push!(vnv, vn, val, transform) - return vnv -end - -# TODO(mhauru) The gidset and num_produce arguments are used by the old Gibbs sampler. -# Remove this method as soon as possible. -function BangBang.push!!(vnv::VarNamedVector, vn, val, dist, gidset, num_produce) - f = from_vec_transform(dist) - return push!!(vnv, vn, tovec(val), f) -end - """ shift_right!(x::AbstractVector{<:Real}, start::Int, n::Int) @@ -953,129 +1273,14 @@ function shift_subsequent_ranges_by!(vnv::VarNamedVector, idx::Int, n) return nothing end -""" - update!(vnv::VarNamedVector, vn::VarName, val[, transform]) - -Either add a new entry or update existing entry for `vn` in `vnv` with the value `val`. - -If `vn` does not exist in `vnv`, this is equivalent to [`push!`](@ref). - -`transform` should be a function that converts `val` to the original representation, by -default it's `identity`. -""" -function update!(vnv::VarNamedVector, vn::VarName, val, transform=identity) - if !haskey(vnv, vn) - # Here we just add a new entry. - return push!(vnv, vn, val, transform) - end - - # Here we update an existing entry. - if !(val isa AbstractVector) - val_vec = tovec(val) - transform = _compose_no_identity(transform, from_vec_transform(val)) - else - val_vec = val - end - idx = getidx(vnv, vn) - # Extract the old range. - r_old = getrange(vnv, idx) - start_old, end_old = first(r_old), last(r_old) - n_old = length(r_old) - # Compute the new range. - n_new = length(val_vec) - start_new = start_old - end_new = start_old + n_new - 1 - r_new = start_new:end_new - - #= - Suppose we currently have the following: - - | x | x | o | o | o | y | y | y | <- Current entries - - where 'O' denotes an inactive entry, and we're going to - update the variable `x` to be of size `k` instead of 2. - - We then have a few different scenarios: - 1. `k > 5`: All inactive entries become active + need to shift `y` to the right. - E.g. if `k = 7`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | x | x | x | x | y | y | y | <- New entries - - 2. `k = 5`: All inactive entries become active. - Then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | x | x | y | y | y | <- New entries - - 3. `k < 5`: Some inactive entries become active, some remain inactive. - E.g. if `k = 3`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | x | o | o | y | y | y | <- New entries - - 4. `k = 2`: No inactive entries become active. - Then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | x | o | o | o | y | y | y | <- New entries - - 5. `k < 2`: More entries become inactive. - E.g. if `k = 1`, then - - | x | x | o | o | o | y | y | y | <- Current entries - | x | o | o | o | o | y | y | y | <- New entries - =# - - # Compute the allocated space for `vn`. - had_inactive = haskey(vnv.num_inactive, idx) - n_allocated = had_inactive ? n_old + vnv.num_inactive[idx] : n_old - - if n_new > n_allocated - # Then we need to grow the underlying vector. - n_extra = n_new - n_allocated - # Allocate. - resize!(vnv.vals, length(vnv.vals) + n_extra) - # Shift current values. - shift_right!(vnv.vals, end_old + 1, n_extra) - # No more inactive entries. - had_inactive && delete!(vnv.num_inactive, idx) - # Update the ranges for all variables after this one. - shift_subsequent_ranges_by!(vnv, idx, n_extra) - elseif n_new == n_allocated - # => No more inactive entries. - had_inactive && delete!(vnv.num_inactive, idx) - else - # `n_new < n_allocated` - # => Need to update the number of inactive entries. - vnv.num_inactive[idx] = n_allocated - n_new - end - - # Update the range for this variable. - vnv.ranges[idx] = r_new - # Update the value. - vnv.vals[r_new] = val_vec - # Update the transform. - vnv.transforms[idx] = transform - - # TODO: Should we maybe sweep over inactive ranges and re-contiguify - # if the total number of inactive elements is "large" in some sense? - - return nothing -end - -function update!!(vnv::VarNamedVector, vn::VarName, val, transform=identity) - vnv = loosen_types!!( - vnv, typeof(vn), typeof(_compose_no_identity(transform, from_vec_transform(val))) - ) - update!(vnv, vn, val, transform) - return vnv -end - # set!! is the function defined in utils.jl that tries to do fancy stuff with optics when # setting the value of a generic container using a VarName. We can bypass all that because -# VarNamedVector handles VarNames natively. -set!!(vnv::VarNamedVector, vn::VarName, val) = update!!(vnv, vn, val) +# VarNamedVector handles VarNames natively. However, it's semantics are slightly different +# from setindex!'s: It allows resetting variables that already have a value with values of +# a different type/size. +function set!!(vnv::VarNamedVector, vn::VarName, val) + return reset!!(vnv, val, vn) +end function setval!(vnv::VarNamedVector, val, vn::VarName) return setindex_internal!(vnv, tovec(val), vn) @@ -1104,7 +1309,7 @@ julia> using DynamicPPL: VarNamedVector, @varname, contiguify!, update!, has_ina julia> vnv = VarNamedVector(@varname(x) => [1.0, 2.0, 3.0], @varname(y) => [3.0]); -julia> update!(vnv, @varname(x), [23.0, 24.0]); +julia> update!(vnv, [23.0, 24.0], @varname(x)); julia> has_inactive(vnv) true @@ -1277,7 +1482,7 @@ true ``` """ values_as(vnv::VarNamedVector) = values_as(vnv, Vector) -values_as(vnv::VarNamedVector, ::Type{Vector}) = vnv[:] +values_as(vnv::VarNamedVector, ::Type{Vector}) = getindex_internal(vnv, :) function values_as(vnv::VarNamedVector, ::Type{Vector{T}}) where {T} return convert(Vector{T}, values_as(vnv, Vector)) end diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index f5b97dbbc..6f6d560f5 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -58,19 +58,19 @@ end @testset "VarNamedVector" begin - svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m), 1.0)) + svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m) => 1.0)) @test getlogp(svi) == 0.0 @test haskey(svi, @varname(m)) @test !haskey(svi, @varname(m[1])) - svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m), [1.0])) + svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m) => [1.0])) @test getlogp(svi) == 0.0 @test haskey(svi, @varname(m)) @test haskey(svi, @varname(m[1])) @test !haskey(svi, @varname(m[2])) @test svi[@varname(m)][1] == svi[@varname(m[1])] - svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a), [1.0])) + svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a) => [1.0])) @test haskey(svi, @varname(m)) @test haskey(svi, @varname(m.a)) @test haskey(svi, @varname(m.a[1])) @@ -78,7 +78,7 @@ @test !haskey(svi, @varname(m.a.b)) # The implementation of haskey and getvalue fo VarNamedVector is incomplete, the # next test is here to remind of us that. - svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a.b), [1.0])) + svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a.b) => [1.0])) @test_broken !haskey(svi, @varname(m.a.b.c.d)) end end @@ -145,7 +145,7 @@ svi_dict = SimpleVarInfo(VarInfo(model), Dict) vnv = VarNamedVector() for (k, v) in pairs(DynamicPPL.TestUtils.rand_prior_true(model)) - vnv = push!!(vnv, VarName{k}(), v) + vnv = push!!(vnv, VarName{k}() => v) end svi_vnv = SimpleVarInfo(vnv) diff --git a/test/varnamedvector.jl b/test/varnamedvector.jl index b3699e54b..655fd7e2e 100644 --- a/test/varnamedvector.jl +++ b/test/varnamedvector.jl @@ -186,49 +186,43 @@ end @test !should_have_restricted_transform_type || has_restricted_transform_type end - # `eltype` @test eltype(vnv_base) == promote_type(eltype(val_left), eltype(val_right)) - # `length` - @test length(vnv_base) == length(val_left) + length(val_right) + @test length_internal(vnv_base) == length(val_left) + length(val_right) - # `isempty` @test !isempty(vnv_base) - # `empty!` @testset "empty!" begin vnv = deepcopy(vnv_base) empty!(vnv) @test isempty(vnv) end - # `similar` @testset "similar" begin vnv = similar(vnv_base) @test isempty(vnv) @test typeof(vnv) == typeof(vnv_base) end - # `getindex` @testset "getindex" begin # With `VarName` index. @test vnv_base[vn_left] == val_left @test vnv_base[vn_right] == val_right - # With `Int` index. + # getindex_internal with `Int` index. val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) - @test all(vnv_base[i] == val_vec[i] for i in 1:length(val_vec)) + @test all( + getindex_internal(vnv_base, i) == val_vec[i] for i in eachindex(val_vec) + ) end - # `setindex!` - @testset "setindex!" begin + @testset "update!" begin vnv = deepcopy(vnv_base) - vnv[vn_left] = val_left .+ 100 + update!(vnv, val_left .+ 100, vn_left) @test vnv[vn_left] == val_left .+ 100 - vnv[vn_right] = val_right .+ 100 + update!(vnv, val_right .+ 100, vn_right) @test vnv[vn_right] == val_right .+ 100 end - # `getindex_internal` @testset "getindex_internal" begin # With `VarName` index. @test DynamicPPL.getindex_internal(vnv_base, vn_left) == to_vec_left(val_left) @@ -238,20 +232,18 @@ end val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) @test all( DynamicPPL.getindex_internal(vnv_base, i) == val_vec[i] for - i in 1:length(val_vec) + i in eachindex(val_vec) ) end - # `setindex_internal!` - @testset "setindex_internal!" begin + @testset "update_internal!" begin vnv = deepcopy(vnv_base) - DynamicPPL.setindex_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) + DynamicPPL.update_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) @test vnv[vn_left] == val_left .+ 100 - DynamicPPL.setindex_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) + DynamicPPL.update_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) @test vnv[vn_right] == val_right .+ 100 end - # `delete!` @testset "delete!" begin vnv = deepcopy(vnv_base) delete!(vnv, vn_left) @@ -261,7 +253,6 @@ end @test !haskey(vnv, vn_right) end - # `merge` @testset "merge" begin # When there are no inactive entries, `merge` on itself result in the same. @test merge(vnv_base, vnv_base) == vnv_base @@ -288,46 +279,38 @@ end @test collect(keys(vnv_merged)) == [vn_right, vn_left] end - # `push!` & `update!` @testset "push!" begin vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn" for vn in test_vns val = test_pairs[vn] - if vn == vn_left || vn == vn_right - # Should not be possible to `push!` existing varname. - @test_throws ArgumentError push!(vnv, vn, val) - else - vnv_copy = deepcopy(vnv) - push!(vnv_copy, vn, val) - @test vnv_copy[vn] == val - push!(vnv, (vn => val)) - @test vnv[vn] == val - end + vnv_copy = deepcopy(vnv) + push!(vnv, (vn => val)) + @test vnv[vn] == val end end - @testset "update!" begin + @testset "setindex!" begin vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn" for vn in test_vns val = test_pairs[vn] expected_length = if haskey(vnv, vn) # If it's already present, the resulting length will be unchanged. - length(vnv) + length_internal(vnv) else - length(vnv) + length(val) + length_internal(vnv) + length(val) end - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] + vnv[vn] = val .+ 1 + x = getindex_internal(vnv, :) @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) + @test length_internal(vnv) == expected_length + @test length(x) == length_internal(vnv) # There should be no redundant values in the underlying vector. @test !DynamicPPL.has_inactive(vnv) # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) + @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -337,19 +320,20 @@ end vn_already_present = haskey(vnv, vn) expected_length = if vn_already_present # If it's already present, the resulting length will be altered. - length(vnv) + length(val) - length(val_original) + length_internal(vnv) + length(val) - length(val_original) else - length(vnv) + length(val) + length_internal(vnv) + length(val) end - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] + haskey(vnv, vn) && delete!(vnv, vn) + vnv[vn] = val .+ 1 + x = getindex_internal(vnv, :) @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) + @test length_internal(vnv) == expected_length + @test length(x) == length_internal(vnv) # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) + @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -359,18 +343,20 @@ end vn_already_present = haskey(vnv, vn) expected_length = if vn_already_present # If it's already present, the resulting length will be altered. - length(vnv) + length(val) - length(val_original) + length_internal(vnv) + length(val) - length(val_original) else - length(vnv) + length(val) + length_internal(vnv) + length(val) end - DynamicPPL.update!(vnv, vn, val .+ 1) - x = vnv[:] + + haskey(vnv, vn) && delete!(vnv, vn) + vnv[vn] = val .+ 1 + x = getindex_internal(vnv, :) @test vnv[vn] == val .+ 1 - @test length(vnv) == expected_length - @test length(x) == length(vnv) + @test length_internal(vnv) == expected_length + @test length(x) == length_internal(vnv) # `getindex` with `Int` index. - @test all(vnv[i] == x[i] for i in 1:length(x)) + @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) end end end @@ -384,19 +370,19 @@ end # Growing should not create inactive ranges. for i in 1:n x = fill(true, i) - DynamicPPL.update!(vnv, vn, x) + DynamicPPL.update_internal!(vnv, x, vn, identity) @test !DynamicPPL.has_inactive(vnv) end # Same size should not create inactive ranges. x = fill(true, n) - DynamicPPL.update!(vnv, vn, x) + DynamicPPL.update_internal!(vnv, x, vn, identity) @test !DynamicPPL.has_inactive(vnv) # Shrinking should create inactive ranges. for i in (n - 1):-1:1 x = fill(true, i) - DynamicPPL.update!(vnv, vn, x) + DynamicPPL.update_internal!(vnv, x, vn, identity) @test DynamicPPL.has_inactive(vnv) @test DynamicPPL.num_inactive(vnv, vn) == n - i end @@ -411,7 +397,7 @@ end # Insert a bunch of random-length vectors. for i in 1:100 x = fill(true, rand(1:n)) - DynamicPPL.update!(vnv, vn, x) + DynamicPPL.update!(vnv, x, vn) end # Should never be allocating more than `n` elements. @test DynamicPPL.num_allocated(vnv, vn) ≤ n @@ -419,7 +405,7 @@ end # If we compaticfy, then it should always be the same size as just inserted. for i in 1:10 x = fill(true, rand(1:n)) - DynamicPPL.update!(vnv, vn, x) + DynamicPPL.update!(vnv, x, vn) DynamicPPL.contiguify!(vnv) @test DynamicPPL.num_allocated(vnv, vn) == length(x) end @@ -435,9 +421,9 @@ end # Test that subset preserves transformations and unconstrainedness. vn = @varname(t[1]) vns = vcat(test_vns, [vn]) - vnv = push!!(vnv, vn, 2.0, x -> x^2) + vnv = setindex_internal!!(vnv, [2.0], vn, x -> x .^ 2) DynamicPPL.settrans!(vnv, true, @varname(t[1])) - @test vnv[@varname(t[1])] == 4.0 + @test vnv[@varname(t[1])] == [4.0] @test istrans(vnv, @varname(t[1])) @test subset(vnv, vns) == vnv end From a9a6ce2cbd8b7a365dcd86c4d4a48b77f81a8db2 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 2 Oct 2024 12:56:12 +0100 Subject: [PATCH 189/209] More improvements to VNV setters and their tests --- src/DynamicPPL.jl | 5 +- src/varnamedvector.jl | 2 +- test/varnamedvector.jl | 178 ++++++++++++++++++++++++++++++++--------- 3 files changed, 146 insertions(+), 39 deletions(-) diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index 3bd689ffa..d41b2285c 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -51,18 +51,21 @@ export AbstractVarInfo, length_internal, getindex_internal, update!, - insert!, + reset!, setindex_internal!, update_internal!, insert_internal!, update!!, insert!!, + reset!!, setindex!!, setindex_internal!!, update_internal!!, insert_internal!!, push!!, empty!!, + loosen_types!!, + tighten_types, subset, getlogp, setlogp!!, diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index e1d8b5314..86b08cea8 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -584,7 +584,7 @@ Add a variable with given value to `vnv`. Like `setindex!`, but errors if the key `vn` already exists. """ -function insert!(vnv::VarNamedVector, val, vn::VarName) +function Base.insert!(vnv::VarNamedVector, val, vn::VarName) if haskey(vnv, vn) throw("Variable $vn already exists in VarNamedVector.") end diff --git a/test/varnamedvector.jl b/test/varnamedvector.jl index 655fd7e2e..75eeaac96 100644 --- a/test/varnamedvector.jl +++ b/test/varnamedvector.jl @@ -115,6 +115,10 @@ end # - `setindex!` # - `push!` # - `update!` + # - `insert!` + # - `reset!` + # - `_internal!` versions of the above + # - !! versions of the above # # And these are all be tested for different types of values: # - scalar @@ -188,6 +192,7 @@ end @test eltype(vnv_base) == promote_type(eltype(val_left), eltype(val_right)) @test length_internal(vnv_base) == length(val_left) + length(val_right) + @test length(vnv_base) == 2 @test !isempty(vnv_base) @@ -207,12 +212,12 @@ end # With `VarName` index. @test vnv_base[vn_left] == val_left @test vnv_base[vn_right] == val_right + end - # getindex_internal with `Int` index. - val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) - @test all( - getindex_internal(vnv_base, i) == val_vec[i] for i in eachindex(val_vec) - ) + @testset "getindex_internal" begin + @test DynamicPPL.getindex_internal(vnv_base, vn_left) == to_vec_left(val_left) + @test DynamicPPL.getindex_internal(vnv_base, vn_right) == + to_vec_right(val_right) end @testset "update!" begin @@ -223,24 +228,27 @@ end @test vnv[vn_right] == val_right .+ 100 end - @testset "getindex_internal" begin - # With `VarName` index. - @test DynamicPPL.getindex_internal(vnv_base, vn_left) == to_vec_left(val_left) - @test DynamicPPL.getindex_internal(vnv_base, vn_right) == - to_vec_right(val_right) - # With `Int` index. - val_vec = vcat(to_vec_left(val_left), to_vec_right(val_right)) - @test all( - DynamicPPL.getindex_internal(vnv_base, i) == val_vec[i] for - i in eachindex(val_vec) - ) + @testset "update!!" begin + vnv = deepcopy(vnv_base) + vnv = update!!(vnv, val_left .+ 100, vn_left) + @test vnv[vn_left] == val_left .+ 100 + vnv = update!!(vnv, val_right .+ 100, vn_right) + @test vnv[vn_right] == val_right .+ 100 end @testset "update_internal!" begin vnv = deepcopy(vnv_base) - DynamicPPL.update_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) + update_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) + @test vnv[vn_left] == val_left .+ 100 + update_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) + @test vnv[vn_right] == val_right .+ 100 + end + + @testset "update_internal!!" begin + vnv = deepcopy(vnv_base) + vnv = update_internal!!(vnv, to_vec_left(val_left .+ 100), vn_left) @test vnv[vn_left] == val_left .+ 100 - DynamicPPL.update_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) + vnv = update_internal!!(vnv, to_vec_right(val_right .+ 100), vn_right) @test vnv[vn_right] == val_right .+ 100 end @@ -253,6 +261,50 @@ end @test !haskey(vnv, vn_right) end + @testset "insert!" begin + vnv = deepcopy(vnv_base) + delete!(vnv, vn_left) + delete!(vnv, vn_right) + insert!(vnv, val_left .+ 100, vn_left) + @test vnv[vn_left] == val_left .+ 100 + insert!(vnv, val_right .+ 100, vn_right) + @test vnv[vn_right] == val_right .+ 100 + end + + @testset "insert!!" begin + vnv = deepcopy(vnv_base) + delete!(vnv, vn_left) + delete!(vnv, vn_right) + vnv = insert!!(vnv, val_left .+ 100, vn_left) + @test vnv[vn_left] == val_left .+ 100 + vnv = insert!!(vnv, val_right .+ 100, vn_right) + @test vnv[vn_right] == val_right .+ 100 + end + + @testset "insert_internal!" begin + vnv = deepcopy(vnv_base) + delete!(vnv, vn_left) + delete!(vnv, vn_right) + insert_internal!(vnv, to_vec_left(val_left .+ 100), vn_left, from_vec_left) + @test vnv[vn_left] == val_left .+ 100 + insert_internal!(vnv, to_vec_right(val_right .+ 100), vn_right, from_vec_right) + @test vnv[vn_right] == val_right .+ 100 + end + + @testset "insert_internal!!" begin + vnv = deepcopy(vnv_base) + delete!(vnv, vn_left) + delete!(vnv, vn_right) + vnv = insert_internal!!( + vnv, to_vec_left(val_left .+ 100), vn_left, from_vec_left + ) + @test vnv[vn_left] == val_left .+ 100 + vnv = insert_internal!!( + vnv, to_vec_right(val_right .+ 100), vn_right, from_vec_right + ) + @test vnv[vn_right] == val_right .+ 100 + end + @testset "merge" begin # When there are no inactive entries, `merge` on itself result in the same. @test merge(vnv_base, vnv_base) == vnv_base @@ -289,7 +341,68 @@ end end end - @testset "setindex!" begin + @testset "setindex_internal!" begin + # Not setting the transformation. + vnv = deepcopy(vnv_base) + setindex_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) + @test vnv[vn_left] == val_left .+ 100 + setindex_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) + @test vnv[vn_right] == val_right .+ 100 + + # Explicitly setting the transformation. + increment(x) = x .+ 10 + vnv = deepcopy(vnv_base) + vnv = loosen_types!!(vnv, typeof(vn_left), typeof(increment)) + setindex_internal!(vnv, to_vec_left(val_left .+ 100), vn_left, increment) + @test vnv[vn_left] == to_vec_left(val_left .+ 110) + + vnv = loosen_types!!(vnv, typeof(vn_right), typeof(increment)) + setindex_internal!(vnv, to_vec_right(val_right .+ 100), vn_right, increment) + @test vnv[vn_right] == to_vec_right(val_right .+ 110) + + # Adding new values. + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) + @testset "$vn" for vn in test_vns + val = test_pairs[vn] + from_vec_vn = DynamicPPL.from_vec_transform(val) + to_vec_vn = inverse(from_vec_vn) + setindex_internal!(vnv, to_vec_vn(val), vn, from_vec_vn) + @test vnv[vn] == val + end + end + + @testset "setindex_internal!!" begin + # Not setting the transformation. + vnv = deepcopy(vnv_base) + vnv = setindex_internal!!(vnv, to_vec_left(val_left .+ 100), vn_left) + @test vnv[vn_left] == val_left .+ 100 + vnv = setindex_internal!!(vnv, to_vec_right(val_right .+ 100), vn_right) + @test vnv[vn_right] == val_right .+ 100 + + # Explicitly setting the transformation. + # Note that unlike with setindex_internal!, we don't need loosen_types!! here. + increment(x) = x .+ 10 + vnv = deepcopy(vnv_base) + vnv = setindex_internal!!(vnv, to_vec_left(val_left .+ 100), vn_left, increment) + @test vnv[vn_left] == to_vec_left(val_left .+ 110) + + vnv = setindex_internal!!( + vnv, to_vec_right(val_right .+ 100), vn_right, increment + ) + @test vnv[vn_right] == to_vec_right(val_right .+ 110) + + # Adding new values. + vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) + @testset "$vn" for vn in test_vns + val = test_pairs[vn] + from_vec_vn = DynamicPPL.from_vec_transform(val) + to_vec_vn = inverse(from_vec_vn) + vnv = setindex_internal!!(vnv, to_vec_vn(val), vn, from_vec_vn) + @test vnv[vn] == val + end + end + + @testset "setindex! and reset!" begin vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @testset "$vn" for vn in test_vns val = test_pairs[vn] @@ -308,9 +421,6 @@ end # There should be no redundant values in the underlying vector. @test !DynamicPPL.has_inactive(vnv) - - # `getindex` with `Int` index. - @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -325,15 +435,12 @@ end length_internal(vnv) + length(val) end - haskey(vnv, vn) && delete!(vnv, vn) - vnv[vn] = val .+ 1 + # Have to use reset!, because setindex! doesn't support decreasing size. + reset!(vnv, val .+ 1, vn) x = getindex_internal(vnv, :) @test vnv[vn] == val .+ 1 @test length_internal(vnv) == expected_length @test length(x) == length_internal(vnv) - - # `getindex` with `Int` index. - @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -348,15 +455,12 @@ end length_internal(vnv) + length(val) end - haskey(vnv, vn) && delete!(vnv, vn) - vnv[vn] = val .+ 1 + # Have to use reset!, because setindex! doesn't support decreasing size. + reset!(vnv, val .+ 1, vn) x = getindex_internal(vnv, :) @test vnv[vn] == val .+ 1 @test length_internal(vnv) == expected_length @test length(x) == length_internal(vnv) - - # `getindex` with `Int` index. - @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) end end end @@ -370,19 +474,19 @@ end # Growing should not create inactive ranges. for i in 1:n x = fill(true, i) - DynamicPPL.update_internal!(vnv, x, vn, identity) + update_internal!(vnv, x, vn, identity) @test !DynamicPPL.has_inactive(vnv) end # Same size should not create inactive ranges. x = fill(true, n) - DynamicPPL.update_internal!(vnv, x, vn, identity) + update_internal!(vnv, x, vn, identity) @test !DynamicPPL.has_inactive(vnv) # Shrinking should create inactive ranges. for i in (n - 1):-1:1 x = fill(true, i) - DynamicPPL.update_internal!(vnv, x, vn, identity) + update_internal!(vnv, x, vn, identity) @test DynamicPPL.has_inactive(vnv) @test DynamicPPL.num_inactive(vnv, vn) == n - i end @@ -397,7 +501,7 @@ end # Insert a bunch of random-length vectors. for i in 1:100 x = fill(true, rand(1:n)) - DynamicPPL.update!(vnv, x, vn) + update!(vnv, x, vn) end # Should never be allocating more than `n` elements. @test DynamicPPL.num_allocated(vnv, vn) ≤ n @@ -405,7 +509,7 @@ end # If we compaticfy, then it should always be the same size as just inserted. for i in 1:10 x = fill(true, rand(1:n)) - DynamicPPL.update!(vnv, x, vn) + update!(vnv, x, vn) DynamicPPL.contiguify!(vnv) @test DynamicPPL.num_allocated(vnv, vn) == length(x) end From 3cdd1d1b745f45febc4c3ef20f99d465b04b7ccc Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 2 Oct 2024 13:03:35 +0100 Subject: [PATCH 190/209] Fix style issues in VNV --- src/varnamedvector.jl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 86b08cea8..c86522609 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -1,5 +1,3 @@ -# TODO(mhauru) The docstring is an unfinished mess. I got stuck when I started to think -# about whether functions other than `setindex_internal!` need to take transformations. """ VarNamedVector @@ -1278,9 +1276,7 @@ end # VarNamedVector handles VarNames natively. However, it's semantics are slightly different # from setindex!'s: It allows resetting variables that already have a value with values of # a different type/size. -function set!!(vnv::VarNamedVector, vn::VarName, val) - return reset!!(vnv, val, vn) -end +set!!(vnv::VarNamedVector, vn::VarName, val) = reset!!(vnv, val, vn) function setval!(vnv::VarNamedVector, val, vn::VarName) return setindex_internal!(vnv, tovec(val), vn) From ad0630077fc3f370dc45db9e6d2fc71d4b804bad Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 2 Oct 2024 13:40:13 +0100 Subject: [PATCH 191/209] Update VNV docs. Add haskey to VarInfo --- docs/src/internals/varinfo.md | 30 ++++++++++-------------------- src/varinfo.jl | 3 +++ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index c1219444f..a678be04e 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -30,25 +30,16 @@ To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo` - `empty!(::Dict)`: delete all realizations in `metadata`. - `merge(::Dict, ::Dict)`: merge two `metadata` structures according to similar rules as `Dict`. -*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. Therefore we also want `varinfo` to be able to replicate the following functionality of `Vector{<:Real}`: +*But* for general-purpose samplers, we often want to work with a simple flattened structure, typically a `Vector{<:Real}`. One can access a vectorised version of a variable's value with the following vector-like functions: - - `getindex(::Vector{<:Real}, ::Int)`: return the i-th value in the flat representation of `metadata`. - - + For example, if `metadata` contains a realization of `x ~ MvNormal(zeros(3), I)`, then `getindex(varinfo, 1)` should return the realization of `x[1]`, `getindex(varinfo, 2)` should return the realization of `x[2]`, etc. - - - `setindex!(::Vector{<:Real}, val, ::Int)`: set the i-th value in the flat representation of `metadata`. - - `length(::Vector{<:Real})`: return the length of the flat representation of `metadata`. - - `similar(::Vector{<:Real})`: return a new instance with the same `eltype` as the input. - -We also want some additional methods that are *not* part of the `Dict` or `Vector` interface: - - - `push!(container, varname::VarName, value[, transform])`: add a new element to the container, but with an optional transformation that has been applied to `value`, and should be reverted when returning `container[varname]`. One can also provide a `Pair` instead of a `VarName` and a `value`. - - - `update!(container, ::VarName, value[, transform])`: similar to `push!` but if the `VarName` is already present in the container, then we update the corresponding value instead of adding a new element. + - `getindex_internal(::VarInfo, ::VarName)`: get the flattened value of a single variable. + - `getindex_internal(::VarInfo, ::Colon)`: get the flattened values of all variables. + - `setindex_internal!(::VarInfo, ::AbstractVector, ::VarName)`: set the flattened value of a variable. + - `length_internal(::VarInfo)`: return the length of the flat representation of `metadata`. -In addition, we want to be able to access the transformed / "unconstrained" realization for a particular `VarName` and so we also need corresponding methods for this: +The functions have `_internal` in their name because internally `VarInfo` always stores values as vectorised. - - `getindex_internal` and `setindex_internal!` for extracting and mutating the internal, possibly unconstrained, representaton of a particular `VarName`. +Moreover, a link transformation can be applied to a `VarInfo` with `link!!` (and reversed with `invlink!!`), which applies a reversible transformation to the internal storage format of a variable that makes the range of the random variable cover all of Euclidean space. `getindex_internal` and `setindex_internal!` give direct access to the vectorised value after such a transformation, which is what samplers often need to be able sample in unconstrained space. One can also manually set a transformation by giving `setindex_internal!` a fourth, optional argument, that is a function that maps internally stored value to the actual value of the variable. Finally, we want want the underlying representation used in `metadata` to have a few performance-related properties: @@ -147,7 +138,7 @@ This does require a bit of book-keeping, in particular when it comes to insertio - `ranges::Vector{UnitRange{Int}}`: the ranges of indices in the `Vector{T}` that correspond to each `VarName`. - `transforms::Vector`: the transforms associated with each `VarName`. -Mutating functions, e.g. `setindex!(vnv::VarNamedVector, val, vn::VarName)`, are then treated according to the following rules: +Mutating functions, e.g. `setindex_internal!(vnv::VarNamedVector, val, vn::VarName)`, are then treated according to the following rules: 1. If `vn` is not already present: add it to the end of `vnv.varnames`, add the `val` to the underlying `vnv.vals`, etc. @@ -293,8 +284,7 @@ In the end, we have the following "rough" performance characteristics for `VarNa | Method | Is blazingly fast? | |:----------------------------------------:|:--------------------------------------------------------------------------------------------:| | `getindex` | ${\color{green} \checkmark}$ | -| `setindex!` | ${\color{green} \checkmark}$ | -| `push!` | ${\color{green} \checkmark}$ | +| `setindex!` on a new `VarName` | ${\color{green} \checkmark}$ | | `delete!` | ${\color{red} \times}$ | | `update!` on existing `VarName` | ${\color{green} \checkmark}$ if smaller or same size / ${\color{red} \times}$ if larger size | | `values_as(::VarNamedVector, Vector{T})` | ${\color{green} \checkmark}$ if contiguous / ${\color{orange} \div}$ otherwise | @@ -302,7 +292,7 @@ In the end, we have the following "rough" performance characteristics for `VarNa ## Other methods ```@docs -DynamicPPL.replace_values(::VarNamedVector, vals::AbstractVector) +DynamicPPL.replace_raw_storage(::VarNamedVector, vals::AbstractVector) ``` ```@docs; canonical=false diff --git a/src/varinfo.jl b/src/varinfo.jl index 12bd8b377..3424bad02 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1046,6 +1046,9 @@ function Base.keys(vi::TypedVarInfo, spl::AbstractSampler) return mapreduce(values, vcat, _getvns(vi, spl)) end +Base.haskey(vi::VectorVarInfo, vn::VarName) = haskey(vi.metadata, vn) +Base.haskey(vi::VarInfo, vn::VarName) = vn in keys(vi) + """ setgid!(vi::VarInfo, gid::Selector, vn::VarName) From a203ab7ebdac5cc32c316da7b1f2d70a0f7d1a26 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 2 Oct 2024 13:58:45 +0100 Subject: [PATCH 192/209] Fix VarInfo docs --- docs/src/api.md | 10 +++++++++- docs/src/internals/varinfo.md | 8 ++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index 156b51e03..a16c928b9 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -294,10 +294,18 @@ resetlogp!! ```@docs keys getindex -DynamicPPL.getindex_internal +reset! +update! push!! empty!! isempty +getindex_internal +setindex_internal! +update_internal! +insert_internal! +length_internal +loosen_types!! +tighten_types ``` ```@docs diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index a678be04e..7d7301356 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -192,7 +192,7 @@ println("Before insertion: number of allocated entries $(DynamicPPL.num_allocat for i in 1:5 x = fill(true, rand(1:100)) - DynamicPPL.update!(vnv, @varname(x), x) + DynamicPPL.update!(vnv, x, @varname(x)) println( "After insertion #$(i) of length $(length(x)): number of allocated entries $(DynamicPPL.num_allocated(vnv))", ) @@ -207,7 +207,7 @@ println("Before insertion: number of allocated entries $(DynamicPPL.num_allocat for i in 1:5 x = fill(true, rand(1:100)) - DynamicPPL.update!(vnv, @varname(x), x) + DynamicPPL.update!(vnv, x, @varname(x)) if DynamicPPL.num_allocated(vnv) > 10 DynamicPPL.contiguify!(vnv) end @@ -260,7 +260,7 @@ haskey(varinfo_untyped_vnv, @varname(x)) Or insert a differently-sized value for `@varname(x)` ```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 1)) +DynamicPPL.insert!(varinfo_untyped_vnv.metadata, fill(true, 1), @varname(x)) varinfo_untyped_vnv[@varname(x)] ``` @@ -269,7 +269,7 @@ DynamicPPL.num_allocated(varinfo_untyped_vnv.metadata, @varname(x)) ``` ```@example varinfo-design -DynamicPPL.update!(varinfo_untyped_vnv.metadata, @varname(x), fill(true, 4)) +DynamicPPL.update!(varinfo_untyped_vnv.metadata, fill(true, 4), @varname(x)) varinfo_untyped_vnv[@varname(x)] ``` From 775284d2af67def149b0207de49a1c8572c1d850 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 2 Oct 2024 13:59:52 +0100 Subject: [PATCH 193/209] Disable a test that only works for VectorVarInfo --- test/varinfo.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/varinfo.jl b/test/varinfo.jl index 9f41e125e..0caa5c3ee 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -111,8 +111,8 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) @test vi[vn] == 3 * r @test vi[SampleFromPrior()][1] == 3 * r - # TODO(mhauru) Implement these functions for SimpleVarInfo too. - if vi isa VarInfo + # TODO(mhauru) Implement these functions for other VarInfo types too. + if vi isa VectorVarInfo delete!(vi, vn) @test isempty(vi) vi = push!!(vi, vn, r, dist, gid) From 79c3a81213133e391a67b3d28f556d8c3064425b Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Wed, 2 Oct 2024 14:06:54 +0100 Subject: [PATCH 194/209] Fix bug in isempty(::TypedVarInfo) --- src/varinfo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 3424bad02..8d45ea72f 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1113,7 +1113,7 @@ isempty(vi::VarInfo) = _isempty(vi.metadata) _isempty(metadata::Metadata) = isempty(metadata.idcs) _isempty(vnv::VarNamedVector) = isempty(vnv) @generated function _isempty(metadata::NamedTuple{names}) where {names} - return Expr(:&&, (:(isempty(metadata.$f)) for f in names)...) + return Expr(:&&, (:(_isempty(metadata.$f)) for f in names)...) end # X -> R for all variables associated with given sampler From 1f45e762d8a89b20acbada7f6e9b9f686597a8be Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 3 Oct 2024 11:47:22 +0100 Subject: [PATCH 195/209] Make some doctests platform independent --- src/varnamedvector.jl | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index c86522609..cc3c4ef51 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -49,7 +49,7 @@ $(FIELDS) The values for different variables are internally all stored in a single vector. For instance, ```jldoctest varnamedvector-struct -julia> using DynamicPPL: VarNamedVector, @varname, setindex!, update! +julia> using DynamicPPL: ReshapeTransform, VarNamedVector, @varname, setindex!, update! julia> vnv = VarNamedVector(); @@ -76,8 +76,8 @@ which variable. The `transforms` field stores the transformations that needed to the vectorised internal storage back to its original form: ```jldoctest varnamedvector-struct -julia> vnv.transforms[vnv.varname_to_index[@varname(y)]] -DynamicPPL.ReshapeTransform{Tuple{Int64}, Tuple{Int64, Int64}}((6,), (2, 3)) +julia> vnv.transforms[vnv.varname_to_index[@varname(y)]] == DynamicPPL.ReshapeTransform((6,), (2,3)) +true ``` If a variable is updated with a new value that is of a smaller dimension than the old @@ -99,9 +99,8 @@ julia> vnv.vals 5 6 -julia> vnv.num_inactive -OrderedDict{Int64, Int64} with 1 entry: - 1 => 2 +julia> println(vnv.num_inactive); +OrderedDict(1 => 2) ``` This helps avoid unnecessary memory allocations for values that repeatedly change dimension. @@ -868,11 +867,10 @@ julia> vnv.transforms 1-element Vector{Any}: identity (generic function with 1 method) -julia> vnv_tight = DynamicPPL.tighten_types(vnv) -VarNamedVector{VarName{:y, typeof(identity)}, Int64, Vector{VarName{:y, typeof(identity)}}, Vector{Int64}, Vector{typeof(identity)}}(OrderedDict(y => 1), [y], UnitRange{Int64}[1:1], [23], [identity], Bool[0], OrderedDict{Int64, Int64}()) +julia> vnv_tight = DynamicPPL.tighten_types(vnv); -julia> eltype(vnv_tight) -Int64 +julia> eltype(vnv_tight) == Int +true julia> vnv_tight.transforms 1-element Vector{typeof(identity)}: From 7fcc7df2223166a55e2b8cb1ebc18ee6df5aa05a Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 3 Oct 2024 14:20:09 +0100 Subject: [PATCH 196/209] Better implementation of haskey(::VarInfo, ::VarName) Co-authored-by: Tor Erlend Fjelde --- src/varinfo.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 8d45ea72f..a95109670 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1046,8 +1046,11 @@ function Base.keys(vi::TypedVarInfo, spl::AbstractSampler) return mapreduce(values, vcat, _getvns(vi, spl)) end -Base.haskey(vi::VectorVarInfo, vn::VarName) = haskey(vi.metadata, vn) -Base.haskey(vi::VarInfo, vn::VarName) = vn in keys(vi) +function Base.haskey(vi::VarInfo, vn::VarName) = _haskey(vi.metadata, vn) +function _haskey(metadata::NamedTuple, vn::VarName{sym}) where {sym} + sym in keys(metadata) || return false + return haskey(metadata[sym], vn) +end """ setgid!(vi::VarInfo, gid::Selector, vn::VarName) From 567c4bc6e47d945b69b1d4c43bc7239656d6cb0e Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 3 Oct 2024 14:24:15 +0100 Subject: [PATCH 197/209] Improve haskey for VarInfo --- src/varinfo.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index a95109670..c4896b30a 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1046,11 +1046,15 @@ function Base.keys(vi::TypedVarInfo, spl::AbstractSampler) return mapreduce(values, vcat, _getvns(vi, spl)) end -function Base.haskey(vi::VarInfo, vn::VarName) = _haskey(vi.metadata, vn) +Base.haskey(vi::VarInfo, vn::VarName) = _haskey(vi.metadata, vn) +# _haskey is only needed to avoid type piracy of haskey(::NamedTuple, ::VarName). For +# everything other than NamedTuple it's the same has haskey. function _haskey(metadata::NamedTuple, vn::VarName{sym}) where {sym} sym in keys(metadata) || return false return haskey(metadata[sym], vn) end +_haskey(any, vn) = haskey(any, vn) +Base.haskey(md::Metadata, vn::VarName) = haskey(md.vns, vn) """ setgid!(vi::VarInfo, gid::Selector, vn::VarName) From 172d128d8f4b33c0bf49260ec6facee3134dfce4 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 3 Oct 2024 14:26:21 +0100 Subject: [PATCH 198/209] Make a VNV doctest more robust --- src/varnamedvector.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index cc3c4ef51..330151034 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -799,8 +799,6 @@ julia> y_trans(x) = reshape(x, (2, 2)); julia> setindex_internal!(vnv, collect(1:4), @varname(y), y_trans) ERROR: MethodError: Cannot `convert` an object of type - VarName{y,typeof(identity)} to an object of type - VarName{x,typeof(identity)} [...] julia> vnv_loose = DynamicPPL.loosen_types!!(vnv, typeof(@varname(y)), typeof(y_trans)); From 186a8461ba1302686fecacd3795cfd854b9f8673 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 3 Oct 2024 14:30:59 +0100 Subject: [PATCH 199/209] Remote IndexStyle for VNV --- src/varnamedvector.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 330151034..5ecbaf683 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -391,18 +391,17 @@ function num_allocated(vnv::VarNamedVector, idx::Int) return length(getrange(vnv, idx)) + num_inactive(vnv, idx) end -# Basic Dictionary interface. -Base.eltype(vnv::VarNamedVector) = eltype(vnv.vals) -Base.isempty(vnv::VarNamedVector) = isempty(vnv.varnames) -Base.IndexStyle(::Type{<:VarNamedVector}) = IndexLinear() - # Dictionary interface. +Base.isempty(vnv::VarNamedVector) = isempty(vnv.varnames) Base.length(vnv::VarNamedVector) = length(vnv.varnames) Base.keys(vnv::VarNamedVector) = vnv.varnames Base.values(vnv::VarNamedVector) = Iterators.map(Base.Fix1(getindex, vnv), vnv.varnames) Base.pairs(vnv::VarNamedVector) = (vn => vnv[vn] for vn in keys(vnv)) Base.haskey(vnv::VarNamedVector, vn::VarName) = haskey(vnv.varname_to_index, vn) +# Vector-like interface. +Base.eltype(vnv::VarNamedVector) = eltype(vnv.vals) + """ length_internal(vnv::VarNamedVector) From 5508370dea01a4cd88e31185aba19a7a14b4a46e Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 3 Oct 2024 14:37:41 +0100 Subject: [PATCH 200/209] Clean up an old comment --- src/varnamedvector.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 5ecbaf683..4071b9ad7 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -257,15 +257,15 @@ struct VarNamedVector{ end end -# TODO(mhauru) Are we sure we want the last one to be of type Any[]? function VarNamedVector{K,V}() where {K,V} return VarNamedVector(OrderedDict{K,Int}(), K[], UnitRange{Int}[], V[], Any[]) end -# TODO(mhauru) I would like for this to be VarNamedVector(Union{}, Union{}). This would -# allow expanding the VarName and element types only as necessary, which would help keep -# them concrete. However, making that change here opens some other cans of worms related to -# how VarInfo uses BangBang, that I don't want to deal with right now. +# TODO(mhauru) I would like for this to be VarNamedVector(Union{}, Union{}). Simlarly the +# transform vector type above could then be Union{}[]. This would allow expanding the +# VarName and element types only as necessary, which would help keep them concrete. However, +# making that change here opens some other cans of worms related to how VarInfo uses +# BangBang, that I don't want to deal with right now. VarNamedVector() = VarNamedVector{VarName,Real}() VarNamedVector(xs::Pair...) = VarNamedVector(OrderedDict(xs...)) VarNamedVector(x::AbstractDict) = VarNamedVector(keys(x), values(x)) From 25b85b4c2fa964ad87044113f36d7a84d64c73e8 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Thu, 3 Oct 2024 17:03:01 +0100 Subject: [PATCH 201/209] Fix haskey(::VarInfo, ::VarName) --- src/varinfo.jl | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index c4896b30a..b340be437 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1046,16 +1046,6 @@ function Base.keys(vi::TypedVarInfo, spl::AbstractSampler) return mapreduce(values, vcat, _getvns(vi, spl)) end -Base.haskey(vi::VarInfo, vn::VarName) = _haskey(vi.metadata, vn) -# _haskey is only needed to avoid type piracy of haskey(::NamedTuple, ::VarName). For -# everything other than NamedTuple it's the same has haskey. -function _haskey(metadata::NamedTuple, vn::VarName{sym}) where {sym} - sym in keys(metadata) || return false - return haskey(metadata[sym], vn) -end -_haskey(any, vn) = haskey(any, vn) -Base.haskey(md::Metadata, vn::VarName) = haskey(md.vns, vn) - """ setgid!(vi::VarInfo, gid::Selector, vn::VarName) @@ -1770,15 +1760,15 @@ end return map(vn -> vi[vn], f_vns) end -haskey(metadata::Metadata, vn::VarName) = haskey(metadata.idcs, vn) +Base.haskey(metadata::Metadata, vn::VarName) = haskey(metadata.idcs, vn) """ haskey(vi::VarInfo, vn::VarName) Check whether `vn` has been sampled in `vi`. """ -haskey(vi::VarInfo, vn::VarName) = haskey(getmetadata(vi, vn), vn) -function haskey(vi::TypedVarInfo, vn::VarName) +Base.haskey(vi::VarInfo, vn::VarName) = haskey(getmetadata(vi, vn), vn) +function Base.haskey(vi::TypedVarInfo, vn::VarName) md_haskey = map(vi.metadata) do metadata haskey(metadata, vn) end From eb5577bec9aae6092869d2fc90b362ca339c77f6 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Fri, 4 Oct 2024 14:48:18 +0100 Subject: [PATCH 202/209] Clarify a TODO note in varinfo.jl --- src/varinfo.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/varinfo.jl b/src/varinfo.jl index b340be437..23afcf023 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1443,6 +1443,7 @@ function _link_metadata!!( transform_to_linked = inverse(transform_from_linked) val_new, logjac2 = with_logabsdet_jacobian(transform_to_linked, val_orig) # TODO(mhauru) We are calling a !! function but ignoring the return value. + # Fix this when attending to issue #653. acclogp!!(varinfo, -logjac1 - logjac2) metadata = setindex_internal!!(metadata, val_new, vn, transform_from_linked) settrans!(metadata, true, vn) From b3f92c22cafc28a2255ff3dc8c5a02623c3675c7 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Fri, 4 Oct 2024 15:04:33 +0100 Subject: [PATCH 203/209] Reintroduce Int indexing to VNV --- docs/src/internals/varinfo.md | 2 ++ src/varnamedvector.jl | 19 +++++++++++++++---- test/varnamedvector.jl | 23 +++++++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 7d7301356..188e50483 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -34,7 +34,9 @@ To ensure that `VarInfo` is simple and intuitive to work with, we want `VarInfo` - `getindex_internal(::VarInfo, ::VarName)`: get the flattened value of a single variable. - `getindex_internal(::VarInfo, ::Colon)`: get the flattened values of all variables. + - `getindex_internal(::VarInfo, i::Int)`: get `i`th value of the flattened vector of all values - `setindex_internal!(::VarInfo, ::AbstractVector, ::VarName)`: set the flattened value of a variable. + - `setindex_internal!(::VarInfo, val, i::Int)`: set the `i`th value of the flattened vector of all values - `length_internal(::VarInfo)`: return the length of the flat representation of `metadata`. The functions have `_internal` in their name because internally `VarInfo` always stores values as vectorised. diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 4071b9ad7..41c6e738e 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -493,6 +493,13 @@ Like `getindex`, but returns the values as they are stored in `vnv`, without tra """ getindex_internal(vnv::VarNamedVector, vn::VarName) = vnv.vals[getrange(vnv, vn)] +""" + getindex_internal(vnv::VarNamedVector, i::Int) + +Gets the `i`th element of the internal storage vector, ignoring inactive entries. +""" +getindex_internal(vnv::VarNamedVector, i::Int) = vnv.vals[index_to_vals_index(vnv, i)] + function getindex_internal(vnv::VarNamedVector, ::Colon) return if has_inactive(vnv) mapreduce(Base.Fix1(getindex, vnv.vals), vcat, vnv.ranges) @@ -601,6 +608,14 @@ end """ setindex_internal!(vnv::VarNamedVector, val, i::Int) + +Sets the `i`th element of the internal storage vector, ignoring inactive entries. +""" +function setindex_internal!(vnv::VarNamedVector, val, i::Int) + return vnv.vals[index_to_vals_index(vnv, i)] = val +end + +""" setindex_internal!(vnv::VarNamedVector, val, vn::VarName[, transform]) Like `setindex!`, but sets the values as they are stored internally in `vnv`. @@ -609,10 +624,6 @@ Optionally can set the transformation, such that `transform(val)` is the origina the variable. By default, the transform is the identity if creating a new entry in `vnv`, or the existing transform if updating an existing entry. """ -function setindex_internal!(vnv::VarNamedVector, val, i::Int) - return vnv.vals[index_to_vals_index(vnv, i)] = val -end - function setindex_internal!( vnv::VarNamedVector, val::AbstractVector, vn::VarName, transform=nothing ) diff --git a/test/varnamedvector.jl b/test/varnamedvector.jl index 75eeaac96..062a77b31 100644 --- a/test/varnamedvector.jl +++ b/test/varnamedvector.jl @@ -220,6 +220,16 @@ end to_vec_right(val_right) end + @testset "getindex_internal with Ints" begin + for (i, val) in enumerate(to_vec_left(val_left)) + @test DynamicPPL.getindex_internal(vnv_base, i) == val + end + offset = length(to_vec_left(val_left)) + for (i, val) in enumerate(to_vec_right(val_right)) + @test DynamicPPL.getindex_internal(vnv_base, offset + i) == val + end + end + @testset "update!" begin vnv = deepcopy(vnv_base) update!(vnv, val_left .+ 100, vn_left) @@ -371,6 +381,16 @@ end end end + @testset "setindex_internal! with Ints" begin + vnv = deepcopy(vnv_base) + for i in 1:length_internal(vnv_base) + setindex_internal!(vnv, i, i) + end + for i in 1:length_internal(vnv_base) + @test getindex_internal(vnv, i) == i + end + end + @testset "setindex_internal!!" begin # Not setting the transformation. vnv = deepcopy(vnv_base) @@ -418,6 +438,7 @@ end @test vnv[vn] == val .+ 1 @test length_internal(vnv) == expected_length @test length(x) == length_internal(vnv) + @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) # There should be no redundant values in the underlying vector. @test !DynamicPPL.has_inactive(vnv) @@ -441,6 +462,7 @@ end @test vnv[vn] == val .+ 1 @test length_internal(vnv) == expected_length @test length(x) == length_internal(vnv) + @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -461,6 +483,7 @@ end @test vnv[vn] == val .+ 1 @test length_internal(vnv) == expected_length @test length(x) == length_internal(vnv) + @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) end end end From 65c94cacdaeb72c124784f5b79c33c43c1bc976c Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Fri, 4 Oct 2024 15:27:15 +0100 Subject: [PATCH 204/209] Stop exporting any VNV stuff --- src/DynamicPPL.jl | 18 ---- src/test_utils.jl | 4 +- src/varnamedvector.jl | 2 +- test/simple_varinfo.jl | 16 ++-- test/test_util.jl | 6 +- test/varinfo.jl | 8 +- test/varnamedvector.jl | 181 +++++++++++++++++++++++------------------ 7 files changed, 121 insertions(+), 114 deletions(-) diff --git a/src/DynamicPPL.jl b/src/DynamicPPL.jl index 6bd34dac0..a5d178125 100644 --- a/src/DynamicPPL.jl +++ b/src/DynamicPPL.jl @@ -45,27 +45,9 @@ export AbstractVarInfo, VarInfo, UntypedVarInfo, TypedVarInfo, - VectorVarInfo, SimpleVarInfo, - VarNamedVector, - length_internal, - getindex_internal, - update!, - reset!, - setindex_internal!, - update_internal!, - insert_internal!, - update!!, - insert!!, - reset!!, - setindex!!, - setindex_internal!!, - update_internal!!, - insert_internal!!, push!!, empty!!, - loosen_types!!, - tighten_types, subset, getlogp, setlogp!!, diff --git a/src/test_utils.jl b/src/test_utils.jl index 500e40496..6199138aa 100644 --- a/src/test_utils.jl +++ b/src/test_utils.jl @@ -47,12 +47,12 @@ function setup_varinfos( # SimpleVarInfo svi_typed = SimpleVarInfo(example_values) svi_untyped = SimpleVarInfo(OrderedDict()) - svi_vnv = SimpleVarInfo(VarNamedVector()) + svi_vnv = SimpleVarInfo(DynamicPPL.VarNamedVector()) # SimpleVarInfo{<:Any,<:Ref} svi_typed_ref = SimpleVarInfo(example_values, Ref(getlogp(svi_typed))) svi_untyped_ref = SimpleVarInfo(OrderedDict(), Ref(getlogp(svi_untyped))) - svi_vnv_ref = SimpleVarInfo(VarNamedVector(), Ref(getlogp(svi_vnv))) + svi_vnv_ref = SimpleVarInfo(DynamicPPL.VarNamedVector(), Ref(getlogp(svi_vnv))) lp = getlogp(vi_typed_metadata) varinfos = map(( diff --git a/src/varnamedvector.jl b/src/varnamedvector.jl index 41c6e738e..a5097602d 100644 --- a/src/varnamedvector.jl +++ b/src/varnamedvector.jl @@ -49,7 +49,7 @@ $(FIELDS) The values for different variables are internally all stored in a single vector. For instance, ```jldoctest varnamedvector-struct -julia> using DynamicPPL: ReshapeTransform, VarNamedVector, @varname, setindex!, update! +julia> using DynamicPPL: ReshapeTransform, VarNamedVector, @varname, setindex!, update!, getindex_internal julia> vnv = VarNamedVector(); diff --git a/test/simple_varinfo.jl b/test/simple_varinfo.jl index 6f6d560f5..4343563eb 100644 --- a/test/simple_varinfo.jl +++ b/test/simple_varinfo.jl @@ -58,19 +58,19 @@ end @testset "VarNamedVector" begin - svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m) => 1.0)) + svi = SimpleVarInfo(push!!(DynamicPPL.VarNamedVector(), @varname(m) => 1.0)) @test getlogp(svi) == 0.0 @test haskey(svi, @varname(m)) @test !haskey(svi, @varname(m[1])) - svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m) => [1.0])) + svi = SimpleVarInfo(push!!(DynamicPPL.VarNamedVector(), @varname(m) => [1.0])) @test getlogp(svi) == 0.0 @test haskey(svi, @varname(m)) @test haskey(svi, @varname(m[1])) @test !haskey(svi, @varname(m[2])) @test svi[@varname(m)][1] == svi[@varname(m[1])] - svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a) => [1.0])) + svi = SimpleVarInfo(push!!(DynamicPPL.VarNamedVector(), @varname(m.a) => [1.0])) @test haskey(svi, @varname(m)) @test haskey(svi, @varname(m.a)) @test haskey(svi, @varname(m.a[1])) @@ -78,7 +78,9 @@ @test !haskey(svi, @varname(m.a.b)) # The implementation of haskey and getvalue fo VarNamedVector is incomplete, the # next test is here to remind of us that. - svi = SimpleVarInfo(push!!(VarNamedVector(), @varname(m.a.b) => [1.0])) + svi = SimpleVarInfo( + push!!(DynamicPPL.VarNamedVector(), @varname(m.a.b) => [1.0]) + ) @test_broken !haskey(svi, @varname(m.a.b.c.d)) end end @@ -89,7 +91,7 @@ @testset "$(typeof(vi))" for vi in ( SimpleVarInfo(Dict()), SimpleVarInfo(values_constrained), - SimpleVarInfo(VarNamedVector()), + SimpleVarInfo(DynamicPPL.VarNamedVector()), VarInfo(model), ) for vn in DynamicPPL.TestUtils.varnames(model) @@ -143,7 +145,7 @@ # to see whether this is the case. svi_nt = SimpleVarInfo(DynamicPPL.TestUtils.rand_prior_true(model)) svi_dict = SimpleVarInfo(VarInfo(model), Dict) - vnv = VarNamedVector() + vnv = DynamicPPL.VarNamedVector() for (k, v) in pairs(DynamicPPL.TestUtils.rand_prior_true(model)) vnv = push!!(vnv, VarName{k}() => v) end @@ -232,7 +234,7 @@ # Initialize. svi_nt = DynamicPPL.settrans!!(SimpleVarInfo(), true) svi_nt = last(DynamicPPL.evaluate!!(model, svi_nt, SamplingContext())) - svi_vnv = DynamicPPL.settrans!!(SimpleVarInfo(VarNamedVector()), true) + svi_vnv = DynamicPPL.settrans!!(SimpleVarInfo(DynamicPPL.VarNamedVector()), true) svi_vnv = last(DynamicPPL.evaluate!!(model, svi_vnv, SamplingContext())) for svi in (svi_nt, svi_vnv) diff --git a/test/test_util.jl b/test/test_util.jl index 0c7949e48..f1325b729 100644 --- a/test/test_util.jl +++ b/test/test_util.jl @@ -89,10 +89,12 @@ function short_varinfo_name(vi::TypedVarInfo) return "TypedVarInfo" end short_varinfo_name(::UntypedVarInfo) = "UntypedVarInfo" -short_varinfo_name(::VectorVarInfo) = "VectorVarInfo" +short_varinfo_name(::DynamicPPL.VectorVarInfo) = "VectorVarInfo" short_varinfo_name(::SimpleVarInfo{<:NamedTuple}) = "SimpleVarInfo{<:NamedTuple}" short_varinfo_name(::SimpleVarInfo{<:OrderedDict}) = "SimpleVarInfo{<:OrderedDict}" -short_varinfo_name(::SimpleVarInfo{<:VarNamedVector}) = "SimpleVarInfo{<:VarNamedVector}" +function short_varinfo_name(::SimpleVarInfo{<:DynamicPPL.VarNamedVector}) + return "SimpleVarInfo{<:VarNamedVector}" +end # convenient functions for testing model.jl # function to modify the representation of values based on their length diff --git a/test/varinfo.jl b/test/varinfo.jl index 0caa5c3ee..be20aec7d 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -112,7 +112,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) @test vi[SampleFromPrior()][1] == 3 * r # TODO(mhauru) Implement these functions for other VarInfo types too. - if vi isa VectorVarInfo + if vi isa DynamicPPL.VectorVarInfo delete!(vi, vn) @test isempty(vi) vi = push!!(vi, vn, r, dist, gid) @@ -128,7 +128,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) test_base!!(TypedVarInfo(vi)) test_base!!(SimpleVarInfo()) test_base!!(SimpleVarInfo(Dict())) - test_base!!(SimpleVarInfo(VarNamedVector())) + test_base!!(SimpleVarInfo(DynamicPPL.VarNamedVector())) end @testset "flags" begin # Test flag setting: @@ -209,7 +209,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) model, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata ) vi_untyped = VarInfo(DynamicPPL.Metadata()) - vi_vnv = VarInfo(VarNamedVector()) + vi_vnv = VarInfo(DynamicPPL.VarNamedVector()) vi_vnv_typed = VarInfo( model, SampleFromPrior(), DefaultContext(), DynamicPPL.VarNamedVector ) @@ -374,7 +374,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) @test getlogp(vi) ≈ Bijectors.logpdf_with_trans(dist, x, true) ## `SimpleVarInfo{<:VarNamedVector}` - vi = DynamicPPL.settrans!!(SimpleVarInfo(VarNamedVector()), true) + vi = DynamicPPL.settrans!!(SimpleVarInfo(DynamicPPL.VarNamedVector()), true) # Sample in unconstrained space. vi = last(DynamicPPL.evaluate!!(model, vi, SamplingContext())) f = DynamicPPL.from_linked_internal_transform(vi, vn, dist) diff --git a/test/varnamedvector.jl b/test/varnamedvector.jl index 062a77b31..bd3f5553f 100644 --- a/test/varnamedvector.jl +++ b/test/varnamedvector.jl @@ -7,7 +7,7 @@ decrease_size_for_test(x::Real) = x decrease_size_for_test(x::AbstractVector) = first(x) decrease_size_for_test(x::AbstractArray) = first(eachslice(x; dims=1)) -function need_varnames_relaxation(vnv::VarNamedVector, vn::VarName, val) +function need_varnames_relaxation(vnv::DynamicPPL.VarNamedVector, vn::VarName, val) if isconcretetype(eltype(vnv.varnames)) # If the container is concrete, we need to make sure that the varname types match. # E.g. if `vnv.varnames` has `eltype` `VarName{:x, IndexLens{Tuple{Int64}}}` then @@ -20,22 +20,22 @@ function need_varnames_relaxation(vnv::VarNamedVector, vn::VarName, val) return false end -function need_varnames_relaxation(vnv::VarNamedVector, vns, vals) +function need_varnames_relaxation(vnv::DynamicPPL.VarNamedVector, vns, vals) return any(need_varnames_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) end -function need_values_relaxation(vnv::VarNamedVector, vn::VarName, val) +function need_values_relaxation(vnv::DynamicPPL.VarNamedVector, vn::VarName, val) if isconcretetype(eltype(vnv.vals)) return promote_type(eltype(vnv.vals), eltype(val)) != eltype(vnv.vals) end return false end -function need_values_relaxation(vnv::VarNamedVector, vns, vals) +function need_values_relaxation(vnv::DynamicPPL.VarNamedVector, vns, vals) return any(need_values_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) end -function need_transforms_relaxation(vnv::VarNamedVector, vn::VarName, val) +function need_transforms_relaxation(vnv::DynamicPPL.VarNamedVector, vn::VarName, val) return if isconcretetype(eltype(vnv.transforms)) # If the container is concrete, we need to make sure that the sizes match. # => If the sizes don't match, we need to relax the container type. @@ -50,7 +50,7 @@ function need_transforms_relaxation(vnv::VarNamedVector, vn::VarName, val) false end end -function need_transforms_relaxation(vnv::VarNamedVector, vns, vals) +function need_transforms_relaxation(vnv::DynamicPPL.VarNamedVector, vns, vals) return any(need_transforms_relaxation(vnv, vn, val) for (vn, val) in zip(vns, vals)) end @@ -74,10 +74,10 @@ Similarly: transformations type in `vnv`, then the underlying transformation type will be changed to `Any`. """ -function relax_container_types(vnv::VarNamedVector, vn::VarName, val) +function relax_container_types(vnv::DynamicPPL.VarNamedVector, vn::VarName, val) return relax_container_types(vnv, [vn], [val]) end -function relax_container_types(vnv::VarNamedVector, vns, vals) +function relax_container_types(vnv::DynamicPPL.VarNamedVector, vns, vals) if need_varnames_relaxation(vnv, vns, vals) varname_to_index_new = convert(OrderedDict{VarName,Int}, vnv.varname_to_index) varnames_new = convert(Vector{VarName}, vnv.varnames) @@ -98,7 +98,7 @@ function relax_container_types(vnv::VarNamedVector, vns, vals) vnv.vals end - return VarNamedVector( + return DynamicPPL.VarNamedVector( varname_to_index_new, varnames_new, vnv.ranges, @@ -146,12 +146,12 @@ end @testset "constructor: no args" begin # Empty. - vnv = VarNamedVector() + vnv = DynamicPPL.VarNamedVector() @test isempty(vnv) @test eltype(vnv) == Real # Empty with types. - vnv = VarNamedVector{VarName,Float64}() + vnv = DynamicPPL.VarNamedVector{VarName,Float64}() @test isempty(vnv) @test eltype(vnv) == Float64 end @@ -160,7 +160,7 @@ end @testset "$(vn_left) and $(vn_right)" for (vn_left, vn_right) in test_varnames_iter val_left = test_pairs[vn_left] val_right = test_pairs[vn_right] - vnv_base = VarNamedVector([vn_left, vn_right], [val_left, val_right]) + vnv_base = DynamicPPL.VarNamedVector([vn_left, vn_right], [val_left, val_right]) # We'll need the transformations later. # TODO: Should we test other transformations than just `ReshapeTransform`? @@ -170,7 +170,7 @@ end to_vec_right = inverse(from_vec_right) # Compare to alternative constructors. - vnv_from_dict = VarNamedVector( + vnv_from_dict = DynamicPPL.VarNamedVector( OrderedDict(vn_left => val_left, vn_right => val_right) ) @test vnv_base == vnv_from_dict @@ -191,7 +191,7 @@ end end @test eltype(vnv_base) == promote_type(eltype(val_left), eltype(val_right)) - @test length_internal(vnv_base) == length(val_left) + length(val_right) + @test DynamicPPL.length_internal(vnv_base) == length(val_left) + length(val_right) @test length(vnv_base) == 2 @test !isempty(vnv_base) @@ -215,50 +215,53 @@ end end @testset "getindex_internal" begin - @test DynamicPPL.getindex_internal(vnv_base, vn_left) == to_vec_left(val_left) - @test DynamicPPL.getindex_internal(vnv_base, vn_right) == + @test DynamicPPL.DynamicPPL.getindex_internal(vnv_base, vn_left) == + to_vec_left(val_left) + @test DynamicPPL.DynamicPPL.getindex_internal(vnv_base, vn_right) == to_vec_right(val_right) end @testset "getindex_internal with Ints" begin for (i, val) in enumerate(to_vec_left(val_left)) - @test DynamicPPL.getindex_internal(vnv_base, i) == val + @test DynamicPPL.DynamicPPL.getindex_internal(vnv_base, i) == val end offset = length(to_vec_left(val_left)) for (i, val) in enumerate(to_vec_right(val_right)) - @test DynamicPPL.getindex_internal(vnv_base, offset + i) == val + @test DynamicPPL.DynamicPPL.getindex_internal(vnv_base, offset + i) == val end end @testset "update!" begin vnv = deepcopy(vnv_base) - update!(vnv, val_left .+ 100, vn_left) + DynamicPPL.update!(vnv, val_left .+ 100, vn_left) @test vnv[vn_left] == val_left .+ 100 - update!(vnv, val_right .+ 100, vn_right) + DynamicPPL.update!(vnv, val_right .+ 100, vn_right) @test vnv[vn_right] == val_right .+ 100 end @testset "update!!" begin vnv = deepcopy(vnv_base) - vnv = update!!(vnv, val_left .+ 100, vn_left) + vnv = DynamicPPL.update!!(vnv, val_left .+ 100, vn_left) @test vnv[vn_left] == val_left .+ 100 - vnv = update!!(vnv, val_right .+ 100, vn_right) + vnv = DynamicPPL.update!!(vnv, val_right .+ 100, vn_right) @test vnv[vn_right] == val_right .+ 100 end @testset "update_internal!" begin vnv = deepcopy(vnv_base) - update_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) + DynamicPPL.update_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) @test vnv[vn_left] == val_left .+ 100 - update_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) + DynamicPPL.update_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) @test vnv[vn_right] == val_right .+ 100 end @testset "update_internal!!" begin vnv = deepcopy(vnv_base) - vnv = update_internal!!(vnv, to_vec_left(val_left .+ 100), vn_left) + vnv = DynamicPPL.update_internal!!(vnv, to_vec_left(val_left .+ 100), vn_left) @test vnv[vn_left] == val_left .+ 100 - vnv = update_internal!!(vnv, to_vec_right(val_right .+ 100), vn_right) + vnv = DynamicPPL.update_internal!!( + vnv, to_vec_right(val_right .+ 100), vn_right + ) @test vnv[vn_right] == val_right .+ 100 end @@ -275,9 +278,9 @@ end vnv = deepcopy(vnv_base) delete!(vnv, vn_left) delete!(vnv, vn_right) - insert!(vnv, val_left .+ 100, vn_left) + DynamicPPL.insert!(vnv, val_left .+ 100, vn_left) @test vnv[vn_left] == val_left .+ 100 - insert!(vnv, val_right .+ 100, vn_right) + DynamicPPL.insert!(vnv, val_right .+ 100, vn_right) @test vnv[vn_right] == val_right .+ 100 end @@ -285,9 +288,9 @@ end vnv = deepcopy(vnv_base) delete!(vnv, vn_left) delete!(vnv, vn_right) - vnv = insert!!(vnv, val_left .+ 100, vn_left) + vnv = DynamicPPL.insert!!(vnv, val_left .+ 100, vn_left) @test vnv[vn_left] == val_left .+ 100 - vnv = insert!!(vnv, val_right .+ 100, vn_right) + vnv = DynamicPPL.insert!!(vnv, val_right .+ 100, vn_right) @test vnv[vn_right] == val_right .+ 100 end @@ -295,9 +298,13 @@ end vnv = deepcopy(vnv_base) delete!(vnv, vn_left) delete!(vnv, vn_right) - insert_internal!(vnv, to_vec_left(val_left .+ 100), vn_left, from_vec_left) + DynamicPPL.insert_internal!( + vnv, to_vec_left(val_left .+ 100), vn_left, from_vec_left + ) @test vnv[vn_left] == val_left .+ 100 - insert_internal!(vnv, to_vec_right(val_right .+ 100), vn_right, from_vec_right) + DynamicPPL.insert_internal!( + vnv, to_vec_right(val_right .+ 100), vn_right, from_vec_right + ) @test vnv[vn_right] == val_right .+ 100 end @@ -305,11 +312,11 @@ end vnv = deepcopy(vnv_base) delete!(vnv, vn_left) delete!(vnv, vn_right) - vnv = insert_internal!!( + vnv = DynamicPPL.insert_internal!!( vnv, to_vec_left(val_left .+ 100), vn_left, from_vec_left ) @test vnv[vn_left] == val_left .+ 100 - vnv = insert_internal!!( + vnv = DynamicPPL.insert_internal!!( vnv, to_vec_right(val_right .+ 100), vn_right, from_vec_right ) @test vnv[vn_right] == val_right .+ 100 @@ -354,20 +361,24 @@ end @testset "setindex_internal!" begin # Not setting the transformation. vnv = deepcopy(vnv_base) - setindex_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) + DynamicPPL.setindex_internal!(vnv, to_vec_left(val_left .+ 100), vn_left) @test vnv[vn_left] == val_left .+ 100 - setindex_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) + DynamicPPL.setindex_internal!(vnv, to_vec_right(val_right .+ 100), vn_right) @test vnv[vn_right] == val_right .+ 100 # Explicitly setting the transformation. increment(x) = x .+ 10 vnv = deepcopy(vnv_base) - vnv = loosen_types!!(vnv, typeof(vn_left), typeof(increment)) - setindex_internal!(vnv, to_vec_left(val_left .+ 100), vn_left, increment) + vnv = DynamicPPL.loosen_types!!(vnv, typeof(vn_left), typeof(increment)) + DynamicPPL.setindex_internal!( + vnv, to_vec_left(val_left .+ 100), vn_left, increment + ) @test vnv[vn_left] == to_vec_left(val_left .+ 110) - vnv = loosen_types!!(vnv, typeof(vn_right), typeof(increment)) - setindex_internal!(vnv, to_vec_right(val_right .+ 100), vn_right, increment) + vnv = DynamicPPL.loosen_types!!(vnv, typeof(vn_right), typeof(increment)) + DynamicPPL.setindex_internal!( + vnv, to_vec_right(val_right .+ 100), vn_right, increment + ) @test vnv[vn_right] == to_vec_right(val_right .+ 110) # Adding new values. @@ -376,37 +387,41 @@ end val = test_pairs[vn] from_vec_vn = DynamicPPL.from_vec_transform(val) to_vec_vn = inverse(from_vec_vn) - setindex_internal!(vnv, to_vec_vn(val), vn, from_vec_vn) + DynamicPPL.setindex_internal!(vnv, to_vec_vn(val), vn, from_vec_vn) @test vnv[vn] == val end end @testset "setindex_internal! with Ints" begin vnv = deepcopy(vnv_base) - for i in 1:length_internal(vnv_base) - setindex_internal!(vnv, i, i) + for i in 1:DynamicPPL.length_internal(vnv_base) + DynamicPPL.setindex_internal!(vnv, i, i) end - for i in 1:length_internal(vnv_base) - @test getindex_internal(vnv, i) == i + for i in 1:DynamicPPL.length_internal(vnv_base) + @test DynamicPPL.getindex_internal(vnv, i) == i end end @testset "setindex_internal!!" begin # Not setting the transformation. vnv = deepcopy(vnv_base) - vnv = setindex_internal!!(vnv, to_vec_left(val_left .+ 100), vn_left) + vnv = DynamicPPL.setindex_internal!!(vnv, to_vec_left(val_left .+ 100), vn_left) @test vnv[vn_left] == val_left .+ 100 - vnv = setindex_internal!!(vnv, to_vec_right(val_right .+ 100), vn_right) + vnv = DynamicPPL.setindex_internal!!( + vnv, to_vec_right(val_right .+ 100), vn_right + ) @test vnv[vn_right] == val_right .+ 100 # Explicitly setting the transformation. # Note that unlike with setindex_internal!, we don't need loosen_types!! here. increment(x) = x .+ 10 vnv = deepcopy(vnv_base) - vnv = setindex_internal!!(vnv, to_vec_left(val_left .+ 100), vn_left, increment) + vnv = DynamicPPL.setindex_internal!!( + vnv, to_vec_left(val_left .+ 100), vn_left, increment + ) @test vnv[vn_left] == to_vec_left(val_left .+ 110) - vnv = setindex_internal!!( + vnv = DynamicPPL.setindex_internal!!( vnv, to_vec_right(val_right .+ 100), vn_right, increment ) @test vnv[vn_right] == to_vec_right(val_right .+ 110) @@ -417,7 +432,7 @@ end val = test_pairs[vn] from_vec_vn = DynamicPPL.from_vec_transform(val) to_vec_vn = inverse(from_vec_vn) - vnv = setindex_internal!!(vnv, to_vec_vn(val), vn, from_vec_vn) + vnv = DynamicPPL.setindex_internal!!(vnv, to_vec_vn(val), vn, from_vec_vn) @test vnv[vn] == val end end @@ -428,17 +443,19 @@ end val = test_pairs[vn] expected_length = if haskey(vnv, vn) # If it's already present, the resulting length will be unchanged. - length_internal(vnv) + DynamicPPL.length_internal(vnv) else - length_internal(vnv) + length(val) + DynamicPPL.length_internal(vnv) + length(val) end vnv[vn] = val .+ 1 - x = getindex_internal(vnv, :) + x = DynamicPPL.getindex_internal(vnv, :) @test vnv[vn] == val .+ 1 - @test length_internal(vnv) == expected_length - @test length(x) == length_internal(vnv) - @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) + @test DynamicPPL.length_internal(vnv) == expected_length + @test length(x) == DynamicPPL.length_internal(vnv) + @test all( + DynamicPPL.getindex_internal(vnv, i) == x[i] for i in eachindex(x) + ) # There should be no redundant values in the underlying vector. @test !DynamicPPL.has_inactive(vnv) @@ -451,18 +468,20 @@ end vn_already_present = haskey(vnv, vn) expected_length = if vn_already_present # If it's already present, the resulting length will be altered. - length_internal(vnv) + length(val) - length(val_original) + DynamicPPL.length_internal(vnv) + length(val) - length(val_original) else - length_internal(vnv) + length(val) + DynamicPPL.length_internal(vnv) + length(val) end # Have to use reset!, because setindex! doesn't support decreasing size. - reset!(vnv, val .+ 1, vn) - x = getindex_internal(vnv, :) + DynamicPPL.reset!(vnv, val .+ 1, vn) + x = DynamicPPL.getindex_internal(vnv, :) @test vnv[vn] == val .+ 1 - @test length_internal(vnv) == expected_length - @test length(x) == length_internal(vnv) - @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) + @test DynamicPPL.length_internal(vnv) == expected_length + @test length(x) == DynamicPPL.length_internal(vnv) + @test all( + DynamicPPL.getindex_internal(vnv, i) == x[i] for i in eachindex(x) + ) end vnv = relax_container_types(deepcopy(vnv_base), test_vns, test_vals) @@ -472,18 +491,20 @@ end vn_already_present = haskey(vnv, vn) expected_length = if vn_already_present # If it's already present, the resulting length will be altered. - length_internal(vnv) + length(val) - length(val_original) + DynamicPPL.length_internal(vnv) + length(val) - length(val_original) else - length_internal(vnv) + length(val) + DynamicPPL.length_internal(vnv) + length(val) end # Have to use reset!, because setindex! doesn't support decreasing size. - reset!(vnv, val .+ 1, vn) - x = getindex_internal(vnv, :) + DynamicPPL.reset!(vnv, val .+ 1, vn) + x = DynamicPPL.getindex_internal(vnv, :) @test vnv[vn] == val .+ 1 - @test length_internal(vnv) == expected_length - @test length(x) == length_internal(vnv) - @test all(getindex_internal(vnv, i) == x[i] for i in eachindex(x)) + @test DynamicPPL.length_internal(vnv) == expected_length + @test length(x) == DynamicPPL.length_internal(vnv) + @test all( + DynamicPPL.getindex_internal(vnv, i) == x[i] for i in eachindex(x) + ) end end end @@ -492,24 +513,24 @@ end @testset "deterministic" begin n = 5 vn = @varname(x) - vnv = VarNamedVector(OrderedDict(vn => [true])) + vnv = DynamicPPL.VarNamedVector(OrderedDict(vn => [true])) @test !DynamicPPL.has_inactive(vnv) # Growing should not create inactive ranges. for i in 1:n x = fill(true, i) - update_internal!(vnv, x, vn, identity) + DynamicPPL.update_internal!(vnv, x, vn, identity) @test !DynamicPPL.has_inactive(vnv) end # Same size should not create inactive ranges. x = fill(true, n) - update_internal!(vnv, x, vn, identity) + DynamicPPL.update_internal!(vnv, x, vn, identity) @test !DynamicPPL.has_inactive(vnv) # Shrinking should create inactive ranges. for i in (n - 1):-1:1 x = fill(true, i) - update_internal!(vnv, x, vn, identity) + DynamicPPL.update_internal!(vnv, x, vn, identity) @test DynamicPPL.has_inactive(vnv) @test DynamicPPL.num_inactive(vnv, vn) == n - i end @@ -518,13 +539,13 @@ end @testset "random" begin n = 5 vn = @varname(x) - vnv = VarNamedVector(OrderedDict(vn => [true])) + vnv = DynamicPPL.VarNamedVector(OrderedDict(vn => [true])) @test !DynamicPPL.has_inactive(vnv) # Insert a bunch of random-length vectors. for i in 1:100 x = fill(true, rand(1:n)) - update!(vnv, x, vn) + DynamicPPL.update!(vnv, x, vn) end # Should never be allocating more than `n` elements. @test DynamicPPL.num_allocated(vnv, vn) ≤ n @@ -532,7 +553,7 @@ end # If we compaticfy, then it should always be the same size as just inserted. for i in 1:10 x = fill(true, rand(1:n)) - update!(vnv, x, vn) + DynamicPPL.update!(vnv, x, vn) DynamicPPL.contiguify!(vnv) @test DynamicPPL.num_allocated(vnv, vn) == length(x) end @@ -540,15 +561,15 @@ end end @testset "subset" begin - vnv = VarNamedVector(test_pairs) + vnv = DynamicPPL.VarNamedVector(test_pairs) @test subset(vnv, test_vns) == vnv - @test subset(vnv, VarName[]) == VarNamedVector() + @test subset(vnv, VarName[]) == DynamicPPL.VarNamedVector() @test merge(subset(vnv, test_vns[1:3]), subset(vnv, test_vns[4:end])) == vnv # Test that subset preserves transformations and unconstrainedness. vn = @varname(t[1]) vns = vcat(test_vns, [vn]) - vnv = setindex_internal!!(vnv, [2.0], vn, x -> x .^ 2) + vnv = DynamicPPL.setindex_internal!!(vnv, [2.0], vn, x -> x .^ 2) DynamicPPL.settrans!(vnv, true, @varname(t[1])) @test vnv[@varname(t[1])] == [4.0] @test istrans(vnv, @varname(t[1])) From 47428c7a23e7877c8a0d002ea6c14ac5221c7518 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Fri, 4 Oct 2024 16:02:43 +0100 Subject: [PATCH 205/209] Fix docs --- docs/src/api.md | 19 ++++++++++--------- docs/src/internals/varinfo.md | 8 ++++---- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/docs/src/api.md b/docs/src/api.md index a16c928b9..638f6f3ee 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -294,18 +294,19 @@ resetlogp!! ```@docs keys getindex -reset! -update! push!! empty!! isempty -getindex_internal -setindex_internal! -update_internal! -insert_internal! -length_internal -loosen_types!! -tighten_types +DynamicPPL.getindex_internal +DynamicPPL.setindex_internal! +DynamicPPL.update_internal! +DynamicPPL.insert_internal! +DynamicPPL.length_internal +DynamicPPL.reset! +DynamicPPL.update! +DynamicPPL.insert! +DynamicPPL.loosen_types!! +DynamicPPL.tighten_types ``` ```@docs diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 188e50483..6c2f8b681 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -126,13 +126,13 @@ Hence we obtain a "type-stable when possible"-representation by wrapping it in a ## Efficient storage and iteration -Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`VarNamedVector`](@ref): +Efficient storage and iteration we achieve through implementation of the `metadata`. In particular, we do so with [`DynamicPPL.VarNamedVector`](@ref): ```@docs DynamicPPL.VarNamedVector ``` -In a [`VarNamedVector{<:VarName,T}`](@ref), we achieve the desiderata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. +In a [`DynamicPPL.VarNamedVector{<:VarName,T}`](@ref), we achieve the desiderata by storing the values for different `VarName`s contiguously in a `Vector{T}` and keeping track of which ranges correspond to which `VarName`s. This does require a bit of book-keeping, in particular when it comes to insertions and deletions. Internally, this is handled by assigning each `VarName` a unique `Int` index in the `varname_to_index` field, which is then used to index into the following fields: @@ -294,9 +294,9 @@ In the end, we have the following "rough" performance characteristics for `VarNa ## Other methods ```@docs -DynamicPPL.replace_raw_storage(::VarNamedVector, vals::AbstractVector) +DynamicPPL.replace_raw_storage(::DynamicPPL.VarNamedVector, vals::AbstractVector) ``` ```@docs; canonical=false -DynamicPPL.values_as(::VarNamedVector) +DynamicPPL.values_as(::DynamicPPL.VarNamedVector) ``` From c966984d2cbf12d4134c00e24c92287824d9b0c1 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Mon, 7 Oct 2024 13:27:20 +0100 Subject: [PATCH 206/209] Revert default VarInfo metadata type to Metadata --- src/varinfo.jl | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 23afcf023..a4752a628 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -173,9 +173,9 @@ function untyped_varinfo( model::Model, sampler::AbstractSampler=SampleFromPrior(), context::AbstractContext=DefaultContext(), - metadata_type::Type=VarNamedVector, + metadata::Union{Metadata,VarNamedVector}=Metadata(), ) - varinfo = VarInfo(metadata_type()) + varinfo = VarInfo(metadata) return last(evaluate!!(model, varinfo, SamplingContext(rng, sampler, context))) end function untyped_varinfo(model::Model, args::Union{AbstractSampler,AbstractContext,Type}...) @@ -194,11 +194,9 @@ function VarInfo( model::Model, sampler::AbstractSampler=SampleFromPrior(), context::AbstractContext=DefaultContext(), - # TODO(mhauru) Revisit the default. We probably don't want it to be VarNamedVector just - # yet. - metadata_type::Type=VarNamedVector, + metadata::Union{Metadata,VarNamedVector}=Metadata(), ) - return typed_varinfo(rng, model, sampler, context, metadata_type) + return typed_varinfo(rng, model, sampler, context, metadata) end VarInfo(model::Model, args...) = VarInfo(Random.default_rng(), model, args...) @@ -929,9 +927,7 @@ end # VarInfo -# TODO(mhauru) Revisit the default for meta. We probably should keep it as Metadata as long -# as the old Gibbs sampler is in use. -VarInfo(meta=VarNamedVector()) = VarInfo(meta, Ref{Float64}(0.0), Ref(0)) +VarInfo(meta=Metadata()) = VarInfo(meta, Ref{Float64}(0.0), Ref(0)) function TypedVarInfo(vi::VectorVarInfo) new_metas = group_by_symbol(vi.metadata) @@ -2191,7 +2187,7 @@ julia> rng = StableRNG(42); julia> m = demo([missing]); -julia> var_info = DynamicPPL.VarInfo(rng, m, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata); # Checking the setting of "del" flags only makes sense for VarInfo{<:Metadata}. For VarInfo{<:VarNamedVector} the flag is effectively always set. +julia> var_info = DynamicPPL.VarInfo(rng, m, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata()); # Checking the setting of "del" flags only makes sense for VarInfo{<:Metadata}. For VarInfo{<:VarNamedVector} the flag is effectively always set. julia> var_info[@varname(m)] -0.6702516921145671 From e013926ac9e0c617e84faee83cc0a56f4c7b0476 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Mon, 7 Oct 2024 13:30:38 +0100 Subject: [PATCH 207/209] Fix a few trivial issues with Metadata --- src/varinfo.jl | 2 +- test/model.jl | 2 +- test/varinfo.jl | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index a4752a628..0a1425c2b 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -1020,7 +1020,7 @@ end # `keys` Base.keys(md::Metadata) = md.vns -Base.keys(vi::VarInfo) = keys(vi.metadata) +Base.keys(vi::VarInfo) = Base.keys(vi.metadata) # HACK: Necessary to avoid returning `Any[]` which won't dispatch correctly # on other methods in the codebase which requires `Vector{<:VarName}`. diff --git a/test/model.jl b/test/model.jl index dab019c2f..d163f55f0 100644 --- a/test/model.jl +++ b/test/model.jl @@ -202,7 +202,7 @@ is_typed_varinfo(varinfo::DynamicPPL.SimpleVarInfo{<:NamedTuple}) = true @testset "Dynamic constraints, Metadata" begin model = DynamicPPL.TestUtils.demo_dynamic_constraint() spl = SampleFromPrior() - vi = VarInfo(model, spl, DefaultContext(), DynamicPPL.Metadata) + vi = VarInfo(model, spl, DefaultContext(), DynamicPPL.Metadata()) link!!(vi, spl, model) for i in 1:10 diff --git a/test/varinfo.jl b/test/varinfo.jl index be20aec7d..65f849dda 100644 --- a/test/varinfo.jl +++ b/test/varinfo.jl @@ -206,12 +206,12 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) s_vns = @varname(s) vi_typed = VarInfo( - model, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata + model, SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata() ) vi_untyped = VarInfo(DynamicPPL.Metadata()) vi_vnv = VarInfo(DynamicPPL.VarNamedVector()) vi_vnv_typed = VarInfo( - model, SampleFromPrior(), DefaultContext(), DynamicPPL.VarNamedVector + model, SampleFromPrior(), DefaultContext(), DynamicPPL.VarNamedVector() ) model(vi_untyped, SampleFromPrior()) model(vi_vnv, SampleFromPrior()) @@ -702,7 +702,7 @@ DynamicPPL.getspace(::DynamicPPL.Sampler{MySAlg}) = (:s,) model, DynamicPPL.SampleFromPrior(), DynamicPPL.DefaultContext(), - DynamicPPL.Metadata, + DynamicPPL.Metadata(), ) selector = DynamicPPL.Selector() spl = Sampler(MySAlg(), model, selector) From ed747a4266713c0bee4fcdab945df3ec1b7541e6 Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Mon, 7 Oct 2024 16:27:01 +0100 Subject: [PATCH 208/209] Docs bug fix --- docs/src/internals/varinfo.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/internals/varinfo.md b/docs/src/internals/varinfo.md index 6c2f8b681..4f0480b61 100644 --- a/docs/src/internals/varinfo.md +++ b/docs/src/internals/varinfo.md @@ -80,7 +80,7 @@ For example, with the model above we have ```@example varinfo-design # Type-unstable `VarInfo` varinfo_untyped = DynamicPPL.untyped_varinfo( - demo(), SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata + demo(), SampleFromPrior(), DefaultContext(), DynamicPPL.Metadata() ) typeof(varinfo_untyped.metadata) ``` From b23d4e232672678ea1910fd090190268e2f231bf Mon Sep 17 00:00:00 2001 From: Markus Hauru Date: Mon, 7 Oct 2024 16:35:06 +0100 Subject: [PATCH 209/209] Fix type constraint --- src/varinfo.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/varinfo.jl b/src/varinfo.jl index 0a1425c2b..8727796bc 100644 --- a/src/varinfo.jl +++ b/src/varinfo.jl @@ -178,7 +178,9 @@ function untyped_varinfo( varinfo = VarInfo(metadata) return last(evaluate!!(model, varinfo, SamplingContext(rng, sampler, context))) end -function untyped_varinfo(model::Model, args::Union{AbstractSampler,AbstractContext,Type}...) +function untyped_varinfo( + model::Model, args::Union{AbstractSampler,AbstractContext,Metadata,VarNamedVector}... +) return untyped_varinfo(Random.default_rng(), model, args...) end