From 70d2ac5cf04306cb695a30db89a28f41c8784aad Mon Sep 17 00:00:00 2001
From: Andrei Zhabinski <andrei.zhabinski@gmail.com>
Date: Tue, 23 Mar 2021 01:28:07 +0300
Subject: [PATCH] Add simple benchmark

---
 .github/workflows/benchmark.yml | 20 ++++++++
 .gitignore                      |  1 +
 benchmark/Project.toml          |  5 ++
 benchmark/benchmarks.jl         | 12 +++++
 benchmark/runbenchmarks.jl      | 81 +++++++++++++++++++++++++++++++++
 5 files changed, 119 insertions(+)
 create mode 100644 .github/workflows/benchmark.yml
 create mode 100644 benchmark/Project.toml
 create mode 100644 benchmark/benchmarks.jl
 create mode 100644 benchmark/runbenchmarks.jl

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 000000000..0c07c6bb4
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,20 @@
+name: Benchmark
+on:
+  - pull_request
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: '1.5'
+      - run: git fetch origin '+refs/heads/master:refs/remotes/origin/master'
+      - run: git branch master origin/master
+      - run: |
+          julia --project=benchmark -e '
+            using Pkg
+            Pkg.develop(PackageSpec(path=pwd()))
+            Pkg.instantiate()'
+      - run: julia --project=benchmark benchmark/runbenchmarks.jl
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 6c213d4c2..64988c94d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ deps/usr
 deps.jl
 *.log
 ./Manifest.toml
+benchmark/*.json
\ No newline at end of file
diff --git a/benchmark/Project.toml b/benchmark/Project.toml
new file mode 100644
index 000000000..d9a016129
--- /dev/null
+++ b/benchmark/Project.toml
@@ -0,0 +1,5 @@
+[deps]
+ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+NNlib = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
new file mode 100644
index 000000000..dfb253454
--- /dev/null
+++ b/benchmark/benchmarks.jl
@@ -0,0 +1,12 @@
+using BenchmarkTools
+using NNlib
+
+const SUITE = BenchmarkGroup()
+
+SUITE["activations"] = BenchmarkGroup()
+
+x = rand(64, 64)
+
+for f in NNlib.ACTIVATIONS
+    SUITE["activations"][string(f)] = @benchmarkable $f.($x)
+end
\ No newline at end of file
diff --git a/benchmark/runbenchmarks.jl b/benchmark/runbenchmarks.jl
new file mode 100644
index 000000000..2afa9b5f7
--- /dev/null
+++ b/benchmark/runbenchmarks.jl
@@ -0,0 +1,81 @@
+# Copied from
+# https://github.com/kul-forbes/ProximalOperators.jl/tree/master/benchmark
+using ArgParse
+using PkgBenchmark
+using Markdown
+
+function displayresult(result)
+    md = sprint(export_markdown, result)
+    md = replace(md, ":x:" => "❌")
+    md = replace(md, ":white_check_mark:" => "✅")
+    display(Markdown.parse(md))
+end
+
+function printnewsection(name)
+    println()
+    println()
+    println()
+    printstyled("▃" ^ displaysize(stdout)[2]; color=:blue)
+    println()
+    printstyled(name; bold=true)
+    println()
+    println()
+end
+
+function parse_commandline()
+    s = ArgParseSettings()
+
+    @add_arg_table! s begin
+        "--target"
+            help = "the branch/commit/tag to use as target"
+            default = "HEAD"
+        "--baseline"
+            help = "the branch/commit/tag to use as baseline"
+            default = "master"
+        "--retune"
+            help = "force re-tuning (ignore existing tuning data)"
+            action = :store_true
+    end
+
+    return parse_args(s)
+end
+
+function main()
+    parsed_args = parse_commandline()
+
+    mkconfig(; kwargs...) =
+        BenchmarkConfig(
+            env = Dict(
+                "JULIA_NUM_THREADS" => "1",
+            );
+            kwargs...
+        )
+
+    target = parsed_args["target"]
+    group_target = benchmarkpkg(
+        dirname(@__DIR__),
+        mkconfig(id = target),
+        resultfile = joinpath(@__DIR__, "result-$(target).json"),
+        retune = parsed_args["retune"],
+    )
+
+    baseline = parsed_args["baseline"]
+    group_baseline = benchmarkpkg(
+        dirname(@__DIR__),
+        mkconfig(id = baseline),
+        resultfile = joinpath(@__DIR__, "result-$(baseline).json"),
+    )
+
+    printnewsection("Target result")
+    displayresult(group_target)
+
+    printnewsection("Baseline result")
+    displayresult(group_baseline)
+
+    judgement = judge(group_target, group_baseline)
+
+    printnewsection("Judgement result")
+    displayresult(judgement)
+end
+
+main()
\ No newline at end of file