From 58fa32334e90b0854075103543611d1c92c0d44e Mon Sep 17 00:00:00 2001 From: Douglas Bates Date: Sun, 10 Mar 2024 11:26:10 -0500 Subject: [PATCH 1/3] Add bench/runbenchmarks.jl --- Project.toml | 4 ++- bench/runbenchmarks.jl | 64 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 bench/runbenchmarks.jl diff --git a/Project.toml b/Project.toml index 30112b4d9..b216f2800 100644 --- a/Project.toml +++ b/Project.toml @@ -33,6 +33,7 @@ TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" Aqua = "0.8" Arrow = "1, 2" BSplineKit = "0.14, 0.15, 0.16, 0.17" +Chairmarks = "1" DataAPI = "1" DataFrames = "1" Distributions = "0.21, 0.22, 0.23, 0.24, 0.25" @@ -65,6 +66,7 @@ julia = "1.8" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -73,4 +75,4 @@ Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "DataFrames", "ExplicitImports", "InteractiveUtils", "StableRNGs", "Suppressor", "Test"] +test = ["Aqua", "Chairmarks", "DataFrames", "ExplicitImports", "InteractiveUtils", "StableRNGs", "Suppressor", "Test"] diff --git a/bench/runbenchmarks.jl b/bench/runbenchmarks.jl new file mode 100644 index 000000000..2502842e2 --- /dev/null +++ b/bench/runbenchmarks.jl @@ -0,0 +1,64 @@ +using Chairmarks, MixedModels, StatsModels, TypedTables +using MixedModels: dataset + +@isdefined(contrasts) || const contrasts = Dict{Symbol, Any}() + +contrasts[:F] = HelmertCoding() # mrk17_exp1 +contrasts[:P] = HelmertCoding() # mrk17_exp1 +contrasts[:Q] = HelmertCoding() # mrk17_exp1 +contrasts[:lQ] = HelmertCoding() # mrk17_exp1 +contrasts[:lT] = HelmertCoding() # mrk17_exp1 +contrasts[:ch] = HelmertCoding() # contra +contrasts[:load] = HelmertCoding() # kb07 +contrasts[:prec] = HelmertCoding() # kb07 +contrasts[:service] = HelmertCoding() # insteval +contrasts[:spkr] = HelmertCoding() # kb07 + +tbl = Table( + dsnm = [ + :dyestuff2, :dyestuff, :machines, :pastes, :pastes, :penicillin, + :sleepstudy, :sleepstudy, :sleepstudy, :sleepstudy, :kb07, :kb07, + :mrk17_exp1, :insteval, :insteval, :kb07, :mrk17_exp1, :d3, :ml1m, + ], + secs = append!( + fill(0.1f0, 12), + [1.0f0], + fill(5.0f0, 3), + fill(25.0f0, 4) + ), + frm = StatsModels.FormulaTerm[ + @formula(yield ~ 1 + (1|batch)), + @formula(yield ~ 1 + (1|batch)), + @formula(score ~ 1 + (1 | Worker) + (1 | Machine)), + @formula(strength ~ 1 + (1 | batch & cask)), + @formula(strength ~ 1 + (1 | batch / cask)), + @formula(diameter ~ 1 + (1 | plate) + (1 | sample)), + @formula(reaction ~ 1 + days + (1 | subj)), + @formula(reaction ~ 1 + days + zerocorr(1 + days | subj)), + @formula(reaction ~ 1 + days + (1 | subj) + (0 + days | subj)), + @formula(reaction ~ 1 + days + (1 + days | subj)), + @formula(log(rt_trunc) ~ 1 + spkr + prec + load + (1 | subj) + (1 | item)), + @formula(log(rt_trunc) ~ 1 + spkr * prec * load + (1 | subj) + (1 + prec | item)), + @formula(1000 / rt ~ 1 + F * P * Q * lQ * lT + (1 | item) + (1 | subj)), + @formula(y ~ 1 + service * dept + (1 | s) + (1 | d)), + @formula(y ~ 1 + service + (1 | s) + (1 | d) + (1 | dept)), + @formula( + log(rt_trunc) ~ + 1 + spkr * prec * load + (1 + spkr + prec + load | subj) + + (1 + spkr + prec + load | item) + ), + @formula( + 1000 / rt ~ + 1 + F * P * Q * lQ * lT + (1 + P + Q + lQ + lT | item) + + (1 + F + P + Q + lQ + lT | subj) + ), + @formula(y ~ 1 + u + (1 + u | g) + (1 + u | h) + (1 + u | i)), + @formula(y ~ 1 + (1 | g) + (1 | h)), + ] +) + +linmark(f, d, t) = @b fit(MixedModel, f, dataset(d); contrasts, progress=false) seconds=t + +function runbmrk(tbl) + Table([(; bmk=linmark(f, d, t), dsnm = d, frm=f) for (d, t, f) in tbl]) +end From a864f3043871b10270175f2c98ea0d71bfa72515 Mon Sep 17 00:00:00 2001 From: Douglas Bates Date: Mon, 11 Mar 2024 11:27:34 -0500 Subject: [PATCH 2/3] Add benchmarks for GLMM fits --- bench/runbenchmarks.jl | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/bench/runbenchmarks.jl b/bench/runbenchmarks.jl index 2502842e2..dc5dc8d73 100644 --- a/bench/runbenchmarks.jl +++ b/bench/runbenchmarks.jl @@ -1,4 +1,4 @@ -using Chairmarks, MixedModels, StatsModels, TypedTables +using Chairmarks, MixedModels, StandardizedPredictors, StatsModels, TypedTables using MixedModels: dataset @isdefined(contrasts) || const contrasts = Dict{Symbol, Any}() @@ -13,25 +13,30 @@ contrasts[:load] = HelmertCoding() # kb07 contrasts[:prec] = HelmertCoding() # kb07 contrasts[:service] = HelmertCoding() # insteval contrasts[:spkr] = HelmertCoding() # kb07 +contrasts[:height] = Center() # grouseticks +contrasts[:gender] = HelmertCoding() # verbagg +contrasts[:btype] = EffectsCoding() # verbagg +contrasts[:situ] = HelmertCoding() # verbagg +contrasts[:mode] = HelmertCoding() # verbagg tbl = Table( dsnm = [ - :dyestuff2, :dyestuff, :machines, :pastes, :pastes, :penicillin, + :dyestuff2, :dyestuff, :pastes, :pastes, :machines, :penicillin, :sleepstudy, :sleepstudy, :sleepstudy, :sleepstudy, :kb07, :kb07, - :mrk17_exp1, :insteval, :insteval, :kb07, :mrk17_exp1, :d3, :ml1m, + :mrk17_exp1, :kb07, :insteval, :insteval, :mrk17_exp1, :d3, :ml1m, ], secs = append!( fill(0.1f0, 12), - [1.0f0], - fill(5.0f0, 3), + fill(1.0f0, 2), + fill(5.0f0, 2), fill(25.0f0, 4) ), frm = StatsModels.FormulaTerm[ @formula(yield ~ 1 + (1|batch)), @formula(yield ~ 1 + (1|batch)), - @formula(score ~ 1 + (1 | Worker) + (1 | Machine)), @formula(strength ~ 1 + (1 | batch & cask)), @formula(strength ~ 1 + (1 | batch / cask)), + @formula(score ~ 1 + (1 | Worker) + (1 | Machine)), @formula(diameter ~ 1 + (1 | plate) + (1 | sample)), @formula(reaction ~ 1 + days + (1 | subj)), @formula(reaction ~ 1 + days + zerocorr(1 + days | subj)), @@ -40,13 +45,13 @@ tbl = Table( @formula(log(rt_trunc) ~ 1 + spkr + prec + load + (1 | subj) + (1 | item)), @formula(log(rt_trunc) ~ 1 + spkr * prec * load + (1 | subj) + (1 + prec | item)), @formula(1000 / rt ~ 1 + F * P * Q * lQ * lT + (1 | item) + (1 | subj)), - @formula(y ~ 1 + service * dept + (1 | s) + (1 | d)), - @formula(y ~ 1 + service + (1 | s) + (1 | d) + (1 | dept)), @formula( log(rt_trunc) ~ 1 + spkr * prec * load + (1 + spkr + prec + load | subj) + (1 + spkr + prec + load | item) ), + @formula(y ~ 1 + service * dept + (1 | s) + (1 | d)), + @formula(y ~ 1 + service + (1 | s) + (1 | d) + (1 | dept)), @formula( 1000 / rt ~ 1 + F * P * Q * lQ * lT + (1 + P + Q + lQ + lT | item) + @@ -60,5 +65,23 @@ tbl = Table( linmark(f, d, t) = @b fit(MixedModel, f, dataset(d); contrasts, progress=false) seconds=t function runbmrk(tbl) - Table([(; bmk=linmark(f, d, t), dsnm = d, frm=f) for (d, t, f) in tbl]) + return Table([(; bmk=linmark(f, d, t), dsnm = d, frm=f) for (d, t, f) in tbl]) +end + +gltbl = Table( + dsnm = [:contra, :contra, :verbagg, :grouseticks], + secs = [2.0, 2.0, 15.0, 15.0], + dist = [Bernoulli(), Bernoulli(), Bernoulli(), Poisson()], + frm = StatsModels.FormulaTerm[ + @formula(use ~ 1+age+abs2(age)+urban+livch+(1|urban&dist)), + @formula(use ~ 1+age+abs2(age)+urban+(≠("0"))(livch)+(1+urban|dist)), + @formula(r2 ~ 1+anger+gender+btype+situ+(1|subj)+(1|item)), + @formula(ticks ~ 1+year+height+(1|index)+(1|brood)+(1|location)), + ] +) + +glmark(f, d, r, t; init_from_lmm=()) = @b fit(MixedModel, f, dataset(d), r; init_from_lmm, contrasts, progress=false) seconds=t + +function runglbmk(tbl; init_from_lmm=()) + return Table((; bmk=glmark(f, d, r, t; init_from_lmm), dsnm=d, dist=r, frm=f) for (d, t, r, f) in tbl) end From 63d15a11f5cda9b261d9406974c2358161bbe932 Mon Sep 17 00:00:00 2001 From: Douglas Bates Date: Tue, 25 Jun 2024 13:34:27 -0500 Subject: [PATCH 3/3] Add calls to @track --- Project.toml | 13 +++++++++- bench/runbenchmarks.jl | 56 ++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/Project.toml b/Project.toml index 38077ce7f..9b1468a6b 100644 --- a/Project.toml +++ b/Project.toml @@ -51,6 +51,7 @@ ProgressMeter = "1.7" Random = "1" SparseArrays = "1" StableRNGs = "0.1, 1" +StandardizedPredictors = "1" StaticArrays = "0.11, 0.12, 1" Statistics = "1" StatsAPI = "1.5" @@ -71,8 +72,18 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" +StandardizedPredictors = "5064a6a7-f8c2-40e2-8bdc-797ec6f1ae18" Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Chairmarks", "DataFrames", "ExplicitImports", "InteractiveUtils", "StableRNGs", "Suppressor", "Test"] +test = [ + "Aqua", + "Chairmarks", + "DataFrames", + "ExplicitImports", + "InteractiveUtils", + "StableRNGs", + "StandardizedPredictors", + "Suppressor", + "Test"] diff --git a/bench/runbenchmarks.jl b/bench/runbenchmarks.jl index dc5dc8d73..bb4b4b4ab 100644 --- a/bench/runbenchmarks.jl +++ b/bench/runbenchmarks.jl @@ -1,5 +1,5 @@ -using Chairmarks, MixedModels, StandardizedPredictors, StatsModels, TypedTables -using MixedModels: dataset +using Chairmarks, MixedModels, StandardizedPredictors +using MixedModels: dataset, FormulaTerm, Table @isdefined(contrasts) || const contrasts = Dict{Symbol, Any}() @@ -16,8 +16,8 @@ contrasts[:spkr] = HelmertCoding() # kb07 contrasts[:height] = Center() # grouseticks contrasts[:gender] = HelmertCoding() # verbagg contrasts[:btype] = EffectsCoding() # verbagg -contrasts[:situ] = HelmertCoding() # verbagg -contrasts[:mode] = HelmertCoding() # verbagg +contrasts[:situ] = HelmertCoding() # verbagg +contrasts[:mode] = HelmertCoding() # verbagg tbl = Table( dsnm = [ @@ -31,7 +31,7 @@ tbl = Table( fill(5.0f0, 2), fill(25.0f0, 4) ), - frm = StatsModels.FormulaTerm[ + frm = FormulaTerm[ @formula(yield ~ 1 + (1|batch)), @formula(yield ~ 1 + (1|batch)), @formula(strength ~ 1 + (1 | batch & cask)), @@ -62,26 +62,34 @@ tbl = Table( ] ) -linmark(f, d, t) = @b fit(MixedModel, f, dataset(d); contrasts, progress=false) seconds=t +# linmark(f, d, t) = @b fit(MixedModel, f, dataset(d); contrasts, progress=false) seconds=t -function runbmrk(tbl) - return Table([(; bmk=linmark(f, d, t), dsnm = d, frm=f) for (d, t, f) in tbl]) -end +@track (@b (first(tbl.frm), dataset(first(tbl.dsnm))) fit(MixedModel, first(_), last(_); progress=false)).time +@track (@b (tbl.frm[2], dataset(tbl.dsnm[2])) fit(MixedModel, first(_), last(_); progress=false)).time +@track (@b (tbl.frm[3], dataset(tbl.dsnm[3])) fit(MixedModel, first(_), last(_); progress=false)).time +@track (@b (tbl.frm[4], dataset(tbl.dsnm[4])) fit(MixedModel, first(_), last(_); progress=false)).time +@track (@b (tbl.frm[5], dataset(tbl.dsnm[5])) fit(MixedModel, first(_), last(_); progress=false)).time +@track (@b (tbl.frm[6], dataset(tbl.dsnm[6])) fit(MixedModel, first(_), last(_); progress=false)).time +@track (@b (tbl.frm[7], dataset(tbl.dsnm[7])) fit(MixedModel, first(_), last(_); progress=false)).time -gltbl = Table( - dsnm = [:contra, :contra, :verbagg, :grouseticks], - secs = [2.0, 2.0, 15.0, 15.0], - dist = [Bernoulli(), Bernoulli(), Bernoulli(), Poisson()], - frm = StatsModels.FormulaTerm[ - @formula(use ~ 1+age+abs2(age)+urban+livch+(1|urban&dist)), - @formula(use ~ 1+age+abs2(age)+urban+(≠("0"))(livch)+(1+urban|dist)), - @formula(r2 ~ 1+anger+gender+btype+situ+(1|subj)+(1|item)), - @formula(ticks ~ 1+year+height+(1|index)+(1|brood)+(1|location)), - ] -) +# function runbmrk(tbl) +# return Table([(; bmk=linmark(f, d, t), dsnm = d, frm=f) for (d, t, f) in tbl]) +# end + +# gltbl = Table( +# dsnm = [:contra, :contra, :verbagg, :grouseticks], +# secs = [2.0, 2.0, 15.0, 15.0], +# dist = [Bernoulli(), Bernoulli(), Bernoulli(), Poisson()], +# frm = FormulaTerm[ +# @formula(use ~ 1+age+abs2(age)+urban+livch+(1|urban&dist)), +# @formula(use ~ 1+age+abs2(age)+urban+(≠("0"))(livch)+(1+urban|dist)), +# @formula(r2 ~ 1+anger+gender+btype+situ+(1|subj)+(1|item)), +# @formula(ticks ~ 1+year+height+(1|index)+(1|brood)+(1|location)), +# ] +# ) -glmark(f, d, r, t; init_from_lmm=()) = @b fit(MixedModel, f, dataset(d), r; init_from_lmm, contrasts, progress=false) seconds=t +# glmark(f, d, r, t; init_from_lmm=()) = @b fit(MixedModel, f, dataset(d), r; init_from_lmm, contrasts, progress=false) seconds=t -function runglbmk(tbl; init_from_lmm=()) - return Table((; bmk=glmark(f, d, r, t; init_from_lmm), dsnm=d, dist=r, frm=f) for (d, t, r, f) in tbl) -end +# function runglbmk(tbl; init_from_lmm=()) +# return Table((; bmk=glmark(f, d, r, t; init_from_lmm), dsnm=d, dist=r, frm=f) for (d, t, r, f) in tbl) +# end