diff --git a/.github/workflows/Tests.yml b/.github/workflows/Tests.yml index 99f553b42..077abf1bd 100644 --- a/.github/workflows/Tests.yml +++ b/.github/workflows/Tests.yml @@ -53,8 +53,11 @@ jobs: uses: julia-actions/julia-runtest@v1 env: TEST_GROUP: "compilation" - - name: Profile compiler passes - run: julia --color=yes --project=. -e "import Pkg; Pkg.instantiate(); Pkg.test(;coverage=true);" >> $GITHUB_STEP_SUMMARY + - name: Run profile (Only on ubuntu-latest and Julia version 1) + if: matrix.os == 'ubuntu-latest' && matrix.version == '1' + run: | + julia --color=yes --project=. -e 'import Pkg; Pkg.instantiate(); Pkg.test(;coverage=true);' + shell: bash env: TEST_GROUP: "profile" - name: Running `gibbs` tests diff --git a/src/BUGSExamples/Volume_1/14_LSAT.jl b/src/BUGSExamples/Volume_1/14_LSAT.jl index 5a8aea740..6769b8557 100644 --- a/src/BUGSExamples/Volume_1/14_LSAT.jl +++ b/src/BUGSExamples/Volume_1/14_LSAT.jl @@ -1,7 +1,7 @@ name = "LSAT: item response" model_def = @bugs begin - # Calculate individual (binary) responses to each test from multinomial data + # Calculate individual (binary) responses to each test from Multinomial data for j in 1:culm[1] for k in 1:T r[j, k] = response[1, k] @@ -30,7 +30,7 @@ model_def = @bugs begin alpha[k] ~ dnorm(0, 0.0001) a[k] = alpha[k] - mean(alpha[:]) end - beta ~ truncated(dflat(), 0, nothing) + beta ~ dunif(0, 1000) end original = """ @@ -63,7 +63,7 @@ model { alpha[k] ~ dnorm(0, 0.0001) a[k] <- alpha[k] - mean(alpha[]) } - beta ~ dflat()T(0, ) + beta ~ dunif(0, 1000) } """ diff --git a/src/BUGSExamples/Volume_1/14_LSAT.md b/src/BUGSExamples/Volume_1/14_LSAT.md new file mode 100644 index 000000000..39b413aa3 --- /dev/null +++ b/src/BUGSExamples/Volume_1/14_LSAT.md @@ -0,0 +1,29 @@ +# Section 6 of the Law School Aptitude Test (LSAT) + +Section 6 of the Law School Aptitude Test (LSAT) is a 5-item multiple choice test; students score 1 on each item for the correct answer and 0 otherwise, giving R = 32 possible response patterns. Boch and Lieberman (1970) present data on LSAT for N = 1000 students, part of which is shown below. + +| Response Pattern | Item 1 | Item 2 | Item 3 | Item 4 | Item 5 | Frequency | +|------------------|--------|--------|--------|--------|--------|-----------| +| 1 | 0 | 0 | 0 | 0 | 0 | 3 | +| 2 | 0 | 0 | 0 | 0 | 1 | 6 | +| 3 | 0 | 0 | 0 | 1 | 0 | 2 | +| . | . | . | . | . | . | . | +| . | . | . | . | . | . | . | +| . | . | . | . | . | . | . | +| 30 | 1 | 1 | 1 | 0 | 1 | 61 | +| 31 | 1 | 1 | 1 | 1 | 0 | 28 | +| 32 | 1 | 1 | 1 | 1 | 1 | 298 | + +The above data may be analysed using the one-parameter Rasch model (see Andersen (1980), pp.253-254; Boch and Aitkin (1981)). The probability $p_{jk}$ that student $j$ responds correctly to item $k$ is assumed to follow a logistic function parameterized by an 'item difficulty' or threshold parameter $a_k$ and a latent variable $q_j$ representing the student's underlying ability. The ability parameters are assumed to have a Normal distribution in the population of students. That is: + +$$logit(p_{jk}) = q_j - a_k, j = 1,...,1000; k = 1,...,5$$ + +$$q_j \sim Normal(0, \tau)$$ + +The above model is equivalent to the following random effects logistic regression: + +$$logit(p_{jk}) = \beta q_j - a_k, j = 1,...,1000; k = 1,...,5$$ + +$$q_j \sim Normal(0, 1)$$ + +where $\beta$ corresponds to the scale parameter ($\beta^2 = \tau$) of the latent ability distribution. We assume a half-normal distribution with small precision for $\beta$; this represents vague prior information but constrains $\beta$ to be positive. Standard vague normal priors are assumed for the $a_k$'s. Note that the location of the $a_k$'s depend upon the mean of the prior distribution for $q_j$ which we have arbitrarily fixed to be zero. Alternatively, Boch and Aitkin ensure identifiability by imposing a sum-to-zero constraint on the $a_k$'s. Hence we calculate $a_k = a_k - \bar{a}$ to enable comparision of the BUGS posterior parameter estimates with the Boch and Aitkin marginal maximum likelihood estimates. diff --git a/src/compiler_pass.jl b/src/compiler_pass.jl index 8bfac68dc..a4f8a59d2 100644 --- a/src/compiler_pass.jl +++ b/src/compiler_pass.jl @@ -769,7 +769,7 @@ function analyze_statement(pass::AddEdges, expr::Expr, loop_vars::NamedTuple) vertex_code = filter( !iszero, vertex_code isa AbstractArray ? vertex_code : [vertex_code] ) - vertex_labels = map(x -> label_for(pass.g, x), vertex_code) + vertex_labels = [label_for(pass.g, code) for code in vertex_code] for r in vertex_labels if r != lhs_vn add_edge!(pass.g, r, lhs_vn) diff --git a/test/profile.jl b/test/profile.jl deleted file mode 100644 index 0aee38ec8..000000000 --- a/test/profile.jl +++ /dev/null @@ -1,154 +0,0 @@ -using BenchmarkTools - -using JuliaBUGS -using JuliaBUGS: BUGSExamples - -suite = BenchmarkGroup() - -for name in keys(BUGSExamples.VOLUME_1) - @info "Adding benchmark for $name" - model_def = BUGSExamples.VOLUME_1[name].model_def - data = BUGSExamples.VOLUME_1[name].data - - non_data_scalars, non_data_array_sizes = JuliaBUGS.determine_array_sizes( - model_def, data - ) - - eval_env = JuliaBUGS.compute_data_transformation( - non_data_scalars, non_data_array_sizes, model_def, data - ) - - _suite = BenchmarkGroup() - - _suite["CollectVariables"] = @benchmarkable JuliaBUGS.determine_array_sizes( - $model_def, $data - ) - - _suite["DataTransformation"] = @benchmarkable JuliaBUGS.compute_data_transformation( - $non_data_scalars, $non_data_array_sizes, $model_def, $data - ) - - model_def = JuliaBUGS.concretize_colon_indexing(model_def, eval_env) - _suite["GraphCreation"] = @benchmarkable JuliaBUGS.create_graph($model_def, $eval_env) - - tune!(_suite) - suite[string(name)] = _suite -end - -results = run(suite; verbose=true) - -function create_result_dict(results) - result_dict = Dict{String,Dict{String,Dict{String,String}}}() - for (name, example_suite) in results - _d = Dict{String,Dict{String,String}}() - for k in ("CollectVariables", "DataTransformation", "GraphCreation") - __d = Dict{String,String}() - med = median(example_suite[k]) - min = minimum(example_suite[k]) - max = maximum(example_suite[k]) - for (str, val) in zip(["median", "minimum", "maximum"], [med, min, max]) - __d[str] = BenchmarkTools.prettytime(val.time) - end - __d["memory"] = BenchmarkTools.prettymemory(memory(example_suite[k])) - _d[k] = __d - end - result_dict[name] = _d - end - return result_dict -end - -function print_pure_text_table(result_dict) - # Define the table header - println( - rpad("Example Name", 25), - "|", - lpad("Category", 20), - "|", - lpad("Median Time", 15), - "|", - lpad("Minimum Time", 15), - "|", - lpad("Maximum Time", 15), - "|", - lpad("Memory Usage", 15), - ) - println("-"^105) # Adjust the number based on the total length of the header - - # Iterate through each example and its benchmarks to populate the table rows - for (name, benchmarks) in result_dict - first_category = true - for (category, results) in benchmarks - if first_category - println( - rpad(name, 25), - "|", - lpad(category, 20), - "|", - lpad(results["median"], 15), - "|", - lpad(results["minimum"], 15), - "|", - lpad(results["maximum"], 15), - "|", - lpad(results["memory"], 15), - ) - first_category = false - else - println( - rpad("", 25), - "|", - lpad(category, 20), - "|", - lpad(results["median"], 15), - "|", - lpad(results["minimum"], 15), - "|", - lpad(results["maximum"], 15), - "|", - lpad(results["memory"], 15), - ) - end - end - println("-"^105) # Adjust the number based on the total length of the header - end -end - -function print_markdown_table_to_file(result_dict, filename=nothing) - output_target = filename !== nothing ? open(filename, "w") : stdout - - try - println( - output_target, - "| Example Name | Category | Median Time | Minimum Time | Maximum Time | Memory Usage |", - ) - println( - output_target, - "|--------------|----------|-------------|--------------|--------------|--------------|", - ) - - for (name, benchmarks) in result_dict - first_category = true - for (category, results) in benchmarks - if first_category - println( - output_target, - "| $(name) | $(category) | $(results["median"]) | $(results["minimum"]) | $(results["maximum"]) | $(results["memory"]) |", - ) - first_category = false - else - println( - output_target, - "| | $(category) | $(results["median"]) | $(results["minimum"]) | $(results["maximum"]) | $(results["memory"]) |", - ) - end - end - end - finally - if filename !== nothing - close(output_target) - end - end -end - -result_dict = create_result_dict(results) -print_pure_text_table(result_dict) diff --git a/test/profiles/prof_compile_pass.jl b/test/profiles/prof_compile_pass.jl new file mode 100644 index 000000000..757c3615d --- /dev/null +++ b/test/profiles/prof_compile_pass.jl @@ -0,0 +1,40 @@ +using BenchmarkTools + +using JuliaBUGS +using JuliaBUGS: BUGSExamples + +suite = BenchmarkGroup() + +for name in keys(BUGSExamples.VOLUME_1) + @info "Adding benchmark for $name" + model_def = BUGSExamples.VOLUME_1[name].model_def + data = BUGSExamples.VOLUME_1[name].data + + non_data_scalars, non_data_array_sizes = JuliaBUGS.determine_array_sizes( + model_def, data + ) + + eval_env = JuliaBUGS.compute_data_transformation( + non_data_scalars, non_data_array_sizes, model_def, data + ) + + _suite = BenchmarkGroup() + + _suite["CollectVariables"] = @benchmarkable JuliaBUGS.determine_array_sizes( + $model_def, $data + ) + + _suite["DataTransformation"] = @benchmarkable JuliaBUGS.compute_data_transformation( + $non_data_scalars, $non_data_array_sizes, $model_def, $data + ) + + model_def = JuliaBUGS.concretize_colon_indexing(model_def, eval_env) + _suite["GraphCreation"] = @benchmarkable JuliaBUGS.create_graph($model_def, $eval_env) + + tune!(_suite) + suite[string(name)] = _suite +end + +results = run(suite; verbose=false) +result_dict = create_result_dict(results) +print_markdown_table(result_dict) diff --git a/test/profiles/prof_logdensity.jl b/test/profiles/prof_logdensity.jl new file mode 100644 index 000000000..bf179d300 --- /dev/null +++ b/test/profiles/prof_logdensity.jl @@ -0,0 +1,36 @@ +using JuliaBUGS +using LogDensityProblems, LogDensityProblemsAD +using AdvancedHMC, AbstractMCMC, MCMCChains, ReverseDiff +using BenchmarkTools + +suite = BenchmarkGroup() + +for m in keys(JuliaBUGS.BUGSExamples.VOLUME_1) + exempl = JuliaBUGS.BUGSExamples.VOLUME_1[m] + (; model_def, data, inits) = exempl + model = compile(model_def, data, inits) + ad_model = ADgradient(:ReverseDiff, model; compile=Val(false)) + ad_model_compiled = ADgradient(:ReverseDiff, model; compile=Val(true)) + θ = rand(LogDensityProblems.dimension(model)) + + sub_suite = BenchmarkGroup() + + sub_suite["logdensity"] = @benchmarkable LogDensityProblems.logdensity($model, $θ) + sub_suite["AD logdensity"] = @benchmarkable LogDensityProblems.logdensity($ad_model, $θ) + sub_suite["AD logdensity_and_gradient"] = @benchmarkable LogDensityProblems.logdensity_and_gradient( + $ad_model, $θ + ) + sub_suite["AD compiled logdensity"] = @benchmarkable LogDensityProblems.logdensity( + $ad_model_compiled, $θ + ) + sub_suite["AD compiled logdensity_and_gradient"] = @benchmarkable LogDensityProblems.logdensity_and_gradient( + $ad_model_compiled, $θ + ) + + suite[m] = sub_suite +end + +tune!(suite) +results = run(suite; verbose=false) +result_dict = create_result_dict(results) +print_markdown_table(result_dict) diff --git a/test/profiles/utils.jl b/test/profiles/utils.jl new file mode 100644 index 000000000..c6e31008b --- /dev/null +++ b/test/profiles/utils.jl @@ -0,0 +1,94 @@ +function create_result_dict(results) + result_dict = Dict{String,Dict{String,Dict{String,String}}}() + for (name, example_suite) in results + _d = Dict{String,Dict{String,String}}() + for k in keys(example_suite) + __d = Dict{String,String}() + med = median(example_suite[k]) + min = minimum(example_suite[k]) + max = maximum(example_suite[k]) + for (str, val) in zip(["median", "minimum", "maximum"], [med, min, max]) + __d[str] = BenchmarkTools.prettytime(val.time) + end + __d["memory"] = BenchmarkTools.prettymemory(memory(example_suite[k])) + _d[k] = __d + end + result_dict[name] = _d + end + return result_dict +end + +function print_markdown_table(result_dict) + print("\n\n") + category_padding = maximum(length.(keys(result_dict))) + 5 + println( + "| ", + rpad("Example Name", 30), + " | ", + lpad("Category", category_padding), + " | ", + lpad("Median Time", 20), + " | ", + lpad("Minimum Time", 20), + " | ", + lpad("Maximum Time", 20), + " | ", + lpad("Memory Usage", 20), + " |", + ) + println( + "|", + "-"^32, + "|", + "-"^(category_padding + 2), + "|", + "-"^22, + "|", + "-"^22, + "|", + "-"^22, + "|", + "-"^22, + "|", + ) + + for (name, benchmarks) in result_dict + first_category = true + for (category, results) in benchmarks + if first_category + println( + "| ", + rpad(name, 30), + " | ", + lpad(category, category_padding), + " | ", + lpad(results["median"], 20), + " | ", + lpad(results["minimum"], 20), + " | ", + lpad(results["maximum"], 20), + " | ", + lpad(results["memory"], 20), + " |", + ) + first_category = false + else + println( + "| ", + rpad("", 30), + " | ", + lpad(category, category_padding), + " | ", + lpad(results["median"], 20), + " | ", + lpad(results["minimum"], 20), + " | ", + lpad(results["maximum"], 20), + " | ", + lpad(results["memory"], 20), + " |", + ) + end + end + end +end diff --git a/test/runtests.jl b/test/runtests.jl index aaf760217..0d12f58ab 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -57,7 +57,9 @@ if test_group == "compilation" || test_group == "all" end if test_group == "profile" || test_group == "all" - include("profile.jl") + include("profiles/utils.jl") + include("profiles/prof_compile_pass.jl") + include("profiles/prof_logdensity.jl") end if test_group == "gibbs" || test_group == "all"