Skip to content

Commit

Permalink
Update ess_hat to not error for short chains (#119)
Browse files Browse the repository at this point in the history 
* Set ess to NaN of insufficient draws

* Update ess_rhat tests

* Increment patch number

* Avoid error with FFTAutocovMethod when 0 draws per chain

* Test case with 0 draws per chain

* Increment patch number

* Separate allocation of rhat array from computation

* Separate allocation of ess/rhat from computation

* Remove no-longer-necessary checks

* Revert "Avoid error with FFTAutocovMethod when 0 draws per chain"

This reverts commit 4d415b8.
  • Loading branch information
@sethaxen
sethaxen authored Feb 14, 2024
1 parent 22220fc commit c460b8c
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 33 deletions.
2 changes: 1 addition & 1 deletion Project.toml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "MCMCDiagnosticTools"
uuid = "be115224-59cd-429b-ad48-344e309966f0"
authors = ["David Widmann", "Seth Axen"]
version = "0.3.9"
version = "0.3.10"

[deps]
AbstractFFTs = "621f4979-c628-5d54-868e-fcf4e3e8185c"
Expand Down
81 changes: 51 additions & 30 deletions src/ess_rhat.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,19 +342,26 @@ function rhat(samples::AbstractArray{<:Union{Missing,Real}}; kind::Symbol=:rank,
return throw(ArgumentError("the `kind` `$kind` is not supported by `rhat`"))
end
end
function _rhat(
::Val{:basic}, chains::AbstractArray{<:Union{Missing,Real}}; split_chains::Int=2
)
# compute size of matrices (each chain may be split!)
niter = size(chains, 1) ÷ split_chains
nchains = split_chains * size(chains, 2)
function _rhat(::Val{:basic}, chains::AbstractArray{<:Union{Missing,Real}}; kwargs...)
# define output array
axes_out = _param_axes(chains)
T = promote_type(eltype(chains), typeof(zero(eltype(chains)) / 1))

# define output arrays
rhat = similar(chains, T, axes_out)

T === Missing && return rhat
if T !== Missing
_rhat_basic!(rhat, chains; kwargs...)
end

return _maybescalar(rhat)
end
function _rhat_basic!(
rhat::AbstractArray{T},
chains::AbstractArray{<:Union{Missing,Real}};
split_chains::Int=2,
) where {T<:Union{Missing,Real}}
# compute size of matrices (each chain may be split!)
niter = size(chains, 1) ÷ split_chains
nchains = split_chains * size(chains, 2)

# define caches for mean and variance
chain_mean = Array{T}(undef, 1, nchains)
Expand Down Expand Up @@ -393,8 +400,7 @@ function _rhat(
# estimate rhat
rhat[i] = sqrt(var₊ / W)
end

return _maybescalar(rhat)
return rhat
end
function _rhat(::Val{:bulk}, x::AbstractArray{<:Union{Missing,Real}}; kwargs...)
return _rhat(Val(:basic), _rank_normalize(x); kwargs...)
Expand Down Expand Up @@ -445,33 +451,48 @@ end
function _ess_rhat(
::Val{:basic},
chains::AbstractArray{<:Union{Missing,Real}};
relative::Bool=false,
autocov_method::AbstractAutocovMethod=AutocovMethod(),
split_chains::Int=2,
maxlag::Int=250,
kwargs...,
)
# compute size of matrices (each chain may be split!)
niter = size(chains, 1) ÷ split_chains
nchains = split_chains * size(chains, 2)
ntotal = niter * nchains
# define output arrays
axes_out = _param_axes(chains)
T = promote_type(eltype(chains), typeof(zero(eltype(chains)) / 1))
ess = similar(chains, T, axes_out)
rhat = similar(chains, T, axes_out)

# compute number of iterations (each chain may be split!)
niter = size(chains, 1) ÷ split_chains

# discard the last pair of autocorrelations, which are poorly estimated and only matter
# when chains have mixed poorly anyways.
# leave the last even autocorrelation as a bias term that reduces variance for
# case of antithetical chains, see below
if !(niter > 4)
throw(ArgumentError("number of draws after splitting must >4 but is $niter."))
# discard the last pair of autocorrelations, which are poorly estimated and only matter
# when chains have mixed poorly anyways.
# leave the last even autocorrelation as a bias term that reduces variance for
# case of antithetical chains, see below
@warn "number of draws after splitting must be >4 but is $niter. ESS cannot be computed."
fill!(ess, NaN)
_rhat_basic!(rhat, chains; split_chains)
elseif T !== Missing
maxlag > 0 || throw(DomainError(maxlag, "maxlag must be >0."))
maxlag = min(maxlag, niter - 4)
_ess_rhat_basic!(ess, rhat, chains; split_chains, maxlag, kwargs...)
end
maxlag > 0 || throw(DomainError(maxlag, "maxlag must be >0."))
maxlag = min(maxlag, niter - 4)

# define output arrays
ess = similar(chains, T, axes_out)
rhat = similar(chains, T, axes_out)

T === Missing && return (; ess, rhat)
return (; ess=_maybescalar(ess), rhat=_maybescalar(rhat))
end
function _ess_rhat_basic!(
ess::TA,
rhat::TA,
chains::AbstractArray{<:Union{Missing,Real}};
relative::Bool=false,
autocov_method::AbstractAutocovMethod=AutocovMethod(),
split_chains::Int=2,
maxlag::Int=250,
) where {T<:Union{Missing,Real},TA<:AbstractArray{T}}
# compute size of matrices (each chain may be split!)
niter = size(chains, 1) ÷ split_chains
nchains = split_chains * size(chains, 2)
ntotal = niter * nchains

# define caches for mean and variance
chain_mean = Array{T}(undef, 1, nchains)
Expand Down Expand Up @@ -573,7 +594,7 @@ function _ess_rhat(
ess .*= ntotal
end

return (; ess=_maybescalar(ess), rhat=_maybescalar(rhat))
return (; ess, rhat)
end
function _ess_rhat(::Val{:bulk}, x::AbstractArray{<:Union{Missing,Real}}; kwargs...)
return _ess_rhat(Val(:basic), _rank_normalize(x); kwargs...)
Expand Down
19 changes: 17 additions & 2 deletions test/ess_rhat.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,27 @@ mymean(x) = mean(x)
x = rand(4, 3, 5)
x2 = rand(5, 3, 5)
x3 = rand(100, 3, 5)
x4 = rand(1, 3, 5)
@testset for f in (ess, ess_rhat)
@testset for kind in (:rank, :bulk, :tail, :basic)
f === ess && kind === :rank && continue
@test_throws ArgumentError f(x; split_chains=1, kind=kind)
if f === ess
@test all(isnan, f(x; split_chains=1, kind=kind))
@test all(isnan, f(x4; split_chains=2, kind=kind))
else
@test all(isnan, f(x; split_chains=1, kind=kind).ess)
@test f(x; split_chains=1, kind=kind).rhat ==
rhat(x; split_chains=1, kind=kind)
@test all(isnan, f(x4; split_chains=2, kind=kind).ess)
end
f(x2; split_chains=1, kind=kind)
@test_throws ArgumentError f(x2; split_chains=2, kind=kind)
if f === ess
@test all(isnan, f(x2; split_chains=2, kind=kind))
else
@test all(isnan, f(x2; split_chains=2, kind=kind).ess)
@test f(x2; split_chains=2, kind=kind).rhat ==
rhat(x2; split_chains=2, kind=kind)
end
f(x3; maxlag=1, kind=kind)
@test_throws DomainError f(x3; maxlag=0, kind=kind)
end
Expand Down

2 comments on commit c460b8c

@sethaxen
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JuliaRegistrator register

@JuliaRegistrator
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Registration pull request created: JuliaRegistries/General/100867

Tip: Release Notes

Did you know you can add release notes too? Just add markdown formatted text underneath the comment after the text
"Release notes:" and it will be added to the registry PR, and if TagBot is installed it will also be added to the
release that TagBot creates. i.e.

@JuliaRegistrator register

Release notes:

## Breaking changes

- blah

To add them here just re-invoke and the PR will be updated.

Tagging

After the above pull request is merged, it is recommended that a tag is created on this repository for the registered package version.

This will be done automatically if the Julia TagBot GitHub Action is installed, or can be done manually through the github interface, or via:

git tag -a v0.3.10 -m "<description of version>" c460b8c49009434ec4d7c1b8151b25da13c7c4b3
git push origin v0.3.10

Please sign in to comment.