Merge pull request #17 from shravanngoswamii/tor/benchmark-update

Tor/benchmark update
shravanngoswamii · Dec 19, 2024 · e0f9e3a · e0f9e3a
2 parents d0cfaaf + 0460b64
commit e0f9e3a
Show file tree

Hide file tree

Showing 14 changed files with 121,807 additions and 33 deletions.
diff --git a/.github/workflows/Benchmarking.yml b/.github/workflows/Benchmarking.yml
@@ -0,0 +1,32 @@
+name: Benchmarking
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Julia
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+
+      - name: Install Dependencies
+        run: julia --project=benchmarks/ -e 'using Pkg; Pkg.instantiate()'
+
+      - name: Run Benchmarks and Generate Reports
+        run: julia --project=benchmarks/ -e 'using DynamicPPLBenchmarks; weave_benchmarks()'
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./benchmarks/results
+          publish_branch: gh-pages
diff --git a/benchmarks/Project.toml b/benchmarks/Project.toml
@@ -3,11 +3,18 @@ uuid = "d94a1522-c11e-44a7-981a-42bf5dc1a001"
 version = "0.1.0"
 
 [deps]
+AbstractPPL = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
 DiffUtils = "8294860b-85a6-42f8-8c35-d911f667b5f6"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+DrWatson = "634d3b9d-ee7a-5ddf-bec9-22491ea816e1"
 DynamicPPL = "366bfd00-2699-11ea-058f-f148b4cae6d8"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9"
diff --git a/benchmarks/benchmark_body.jmd b/benchmarks/benchmark_body.jmd
@@ -1,14 +1,14 @@
 ```julia
-@time model_def(data)();
+@time model_def(data...)();
 ```
 
 ```julia
-m = time_model_def(model_def, data);
+m = time_model_def(model_def, data...);
 ```
 
 ```julia
 suite = make_suite(m);
-results = run(suite);
+results = run(suite; seconds=WEAVE_ARGS[:seconds]);
 ```
 
 ```julia
@@ -19,13 +19,37 @@ results["evaluation_untyped"]
 results["evaluation_typed"]
 ```
 
+```julia
+let k = "evaluation_simple_varinfo_nt"
+    haskey(results, k) && results[k]
+end
+```
+
+```julia
+let k = "evaluation_simple_varinfo_componentarray"
+    haskey(results, k) && results[k]
+end
+```
+
+```julia
+let k = "evaluation_simple_varinfo_dict"
+    haskey(results, k) && results[k]
+end
+```
+
+```julia
+let k = "evaluation_simple_varinfo_dict_from_nt"
+    haskey(results, k) && results[k]
+end
+```
+
 ```julia; echo=false; results="hidden";
 BenchmarkTools.save(
     joinpath("results", WEAVE_ARGS[:name], "$(nameof(m))_benchmarks.json"), results
 )
 ```
 
-```julia; wrap=false
+```julia; wrap=false; echo=false
 if WEAVE_ARGS[:include_typed_code]
     typed = typed_code(m)
 end
@@ -34,16 +58,15 @@ end
 ```julia; echo=false; results="hidden"
 if WEAVE_ARGS[:include_typed_code]
     # Serialize the output of `typed_code` so we can compare later.
-    haskey(WEAVE_ARGS, :name) &&
-        serialize(joinpath("results", WEAVE_ARGS[:name], "$(nameof(m)).jls"), string(typed))
+    haskey(WEAVE_ARGS, :name) && serialize(joinpath("results", WEAVE_ARGS[:name],"$(nameof(m)).jls"), string(typed));
 end
 ```
 
 ```julia; wrap=false; echo=false;
-if haskey(WEAVE_ARGS, :name_old)
+if WEAVE_ARGS[:include_typed_code] && haskey(WEAVE_ARGS, :name_old)
     # We want to compare the generated code to the previous version.
-    using DiffUtils: DiffUtils
-    typed_old = deserialize(joinpath("results", WEAVE_ARGS[:name_old], "$(nameof(m)).jls"))
-    DiffUtils.diff(typed_old, string(typed); width=130)
+    import DiffUtils
+    typed_old = deserialize(joinpath("results", WEAVE_ARGS[:name_old], "$(nameof(m)).jls"));
+    DiffUtils.diff(typed_old, string(typed), width=130)
 end
-```
+```
diff --git a/benchmarks/benchmarks.jmd b/benchmarks/benchmarks.jmd
@@ -1,36 +1,43 @@
-# Benchmarks
+`j display("text/markdown", "## $(WEAVE_ARGS[:name]) ##")`
 
-## Setup
+### Setup
 
 ```julia
 using BenchmarkTools, DynamicPPL, Distributions, Serialization
 ```
 
 ```julia
-import DynamicPPLBenchmarks: time_model_def, make_suite, typed_code, weave_child
+using DynamicPPLBenchmarks
+using DynamicPPLBenchmarks: time_model_def, make_suite, typed_code, weave_child
 ```
 
-## Models
+### Environment
 
-### `demo1`
+```julia; echo=false; skip="notebook"
+DynamicPPLBenchmarks.display_environment()
+```
+
+### Models
+
+#### `demo1`
 
 ```julia
 @model function demo1(x)
     m ~ Normal()
     x ~ Normal(m, 1)
 
-    return (m=m, x=x)
+    return (m = m, x = x)
 end
 
 model_def = demo1;
-data = 1.0;
+data = (1.0,);
 ```
 
 ```julia; results="markup"; echo=false
 weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
 ```
 
-### `demo2`
+#### `demo2`
 
 ```julia
 @model function demo2(y)
@@ -43,17 +50,19 @@ weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
         # Heads or tails of a coin are drawn from a Bernoulli distribution.
         y[n] ~ Bernoulli(p)
     end
+
+    return (; p)
 end
 
 model_def = demo2;
-data = rand(0:1, 10);
+data = (rand(0:1, 10),);
 ```
 
 ```julia; results="markup"; echo=false
 weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
 ```
 
-### `demo3`
+#### `demo3`
 
 ```julia
 @model function demo3(x)
@@ -76,7 +85,8 @@ weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
         k[i] ~ Categorical(w)
         x[:, i] ~ MvNormal([μ[k[i]], μ[k[i]]], 1.0)
     end
-    return k
+
+    return (; μ1, μ2, k)
 end
 
 model_def = demo3
@@ -88,43 +98,99 @@ N = 30
 μs = [-3.5, 0.0]
 
 # Construct the data points.
-data = mapreduce(c -> rand(MvNormal([μs[c], μs[c]], 1.0), N), hcat, 1:2);
+data = (mapreduce(c -> rand(MvNormal([μs[c], μs[c]], 1.0), N), hcat, 1:2),);
 ```
 
 ```julia; echo=false
 weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
 ```
 
-### `demo4`: loads of indexing
+#### `demo4`: lots of variables
+
+```julia
+@model function demo4_1k(::Type{TV}=Vector{Float64}) where {TV}
+    m ~ Normal()
+    x = TV(undef, 1_000)
+    for i in eachindex(x)
+        x[i] ~ Normal(m, 1.0)
+    end
+
+    return (; m, x)
+end
+
+model_def = demo4_1k
+data = ();
+```
+
+```julia; echo=false
+weave_child(WEAVE_ARGS[:benchmarkbody], mod = @__MODULE__, args = WEAVE_ARGS)
+```
 
 ```julia
-@model function demo4(n, ::Type{TV}=Vector{Float64}) where {TV}
+@model function demo4_10k(::Type{TV}=Vector{Float64}) where {TV}
     m ~ Normal()
-    x = TV(undef, n)
+    x = TV(undef, 10_000)
     for i in eachindex(x)
         x[i] ~ Normal(m, 1.0)
     end
+
+    return (; m, x)
 end
 
-model_def = demo4
-data = (100_000,);
+model_def = demo4_10k
+data = ();
 ```
 
 ```julia; echo=false
 weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
 ```
 
 ```julia
-@model function demo4_dotted(n, ::Type{TV}=Vector{Float64}) where {TV}
+@model function demo4_100k(::Type{TV}=Vector{Float64}) where {TV}
     m ~ Normal()
-    x = TV(undef, n)
-    return x .~ Normal(m, 1.0)
+    x = TV(undef, 100_000)
+    for i in eachindex(x)
+        x[i] ~ Normal(m, 1.0)
+    end
+
+    return (; m, x)
 end
 
-model_def = demo4_dotted
-data = (100_000,);
+model_def = demo4_100k
+data = ();
 ```
 
 ```julia; echo=false
 weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
 ```
+
+#### `demo4_dotted`: `.~` for large number of variables
+
+```julia
+@model function demo4_100k_dotted(::Type{TV}=Vector{Float64}) where {TV}
+    m ~ Normal()
+    x = TV(undef, 100_000)
+    x .~ Normal(m, 1.0)
+
+    return (; m, x)
+end
+
+model_def = demo4_100k_dotted
+data = ();
+```
+
+```julia; echo=false
+weave_child(WEAVE_ARGS[:benchmarkbody]; mod=@__MODULE__, args=WEAVE_ARGS)
+```
+
+```julia; echo=false
+if haskey(WEAVE_ARGS, :name_old)
+    display(MIME"text/markdown"(), "## Comparison with $(WEAVE_ARGS[:name_old]) ##")
+end
+```
+
+```julia; echo=false
+if haskey(WEAVE_ARGS, :name_old)
+    DynamicPPLBenchmarks.judgementtable(WEAVE_ARGS[:name], WEAVE_ARGS[:name_old])
+end
+```