Skip to content

Commit 3ca0d2e

Browse files
vouillonvouillon
authored
More benchmarks (#1899)
* Improve timing debug information * Benchmarks: dockerfile clean-up * Benchmark: fiat-crypto * Benchmark: ocamlc * Benchmark: Bonsai's partial render table * Benchmark: Camlboy * Benchmarks: combined view of both JavaSCript and Wasm
1 parent 62e8cdd commit 3ca0d2e

File tree

18 files changed

+270
-29
lines changed

18 files changed

+270
-29
lines changed

‎.dockerignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
bench.Dockerfile
2+
_build
3+
.git
4+
compiler/tests-*
5+
manual
6+
examples

‎bench.Dockerfile‎

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,59 @@
1-
FROM ocaml/opam:debian-ocaml-5.3
1+
FROM ocaml/opam:debian-ocaml-5.2
22
WORKDIR /bench-dir
3-
RUN sudo apt-get update && \
4-
sudo apt-get install -qq -yy --no-install-recommends pkg-config libgmp-dev \
5-
wget
3+
4+
RUN sudo apt-get update \
5+
&& sudo apt-get install -qq -yy --no-install-recommends \
6+
pkg-config libgmp-dev jq time
7+
68
RUN sudo ln -sf /usr/bin/opam-2.3 /usr/bin/opam
7-
RUN opam remote add origin https://github.com/ocaml/opam-repository.git && \
8-
opam update
9-
RUN wget -q https://nodejs.org/download/v8-canary/v24.0.0-v8-canary2025030537242e55ac/node-v24.0.0-v8-canary2025030537242e55ac-linux-x64.tar.xz && \
10-
tar xf node-v24.0.0-v8-canary2025030537242e55ac-linux-x64.tar.xz
11-
ENV PATH="/bench-dir/node-v24.0.0-v8-canary2025030537242e55ac-linux-x64/bin:$PATH"
12-
RUN wget -q https://github.com/WebAssembly/binaryen/releases/download/version_122/binaryen-version_122-x86_64-linux.tar.gz && \
13-
tar xf binaryen-version_122-x86_64-linux.tar.gz
14-
ENV PATH="/bench-dir/binaryen-version_122/bin:$PATH"
9+
RUN opam remote add origin https://github.com/ocaml/opam-repository.git \
10+
&& opam update
11+
12+
# Install node
13+
ENV NODE_VERSION=v24.0.0-v8-canary2025030537242e55ac
14+
ENV NODE=node-$NODE_VERSION-linux-x64
15+
RUN curl -q https://nodejs.org/download/v8-canary/$NODE_VERSION/$NODE.tar.xz \
16+
| tar xJf -
17+
ENV PATH="/bench-dir/$NODE/bin:$PATH"
18+
19+
# Install binaryen
20+
ENV BINARYEN_VERSION=version_122
21+
ENV BINARYEN=binaryen-$BINARYEN_VERSION
22+
RUN curl -Lq https://github.com/WebAssembly/binaryen/releases/download/$BINARYEN_VERSION/$BINARYEN-x86_64-linux.tar.gz \
23+
| tar zxf -
24+
ENV PATH="/bench-dir/$BINARYEN/bin:$PATH"
1525
RUN opam install --fake binaryen-bin
26+
27+
# Jane Street opam packages
28+
RUN mkdir janestreet \
29+
&& cd janestreet \
30+
&& git clone --depth 20 https://github.com/janestreet/opam-repository \
31+
&& cd opam-repository \
32+
&& git checkout 41c89c7824533f6b63cc5b6d75e6ddb1441d1520 \
33+
&& opam remote add js .
34+
35+
# Install dependencies
1636
COPY --chown=opam:opam ./*.opam ./
1737
RUN opam pin -yn --with-version=dev .
1838
RUN opam install -y --deps-only js_of_ocaml-compiler
39+
40+
# Install js_of_ocaml / wasm_of_ocaml
1941
COPY --chown=opam:opam . ./
2042
RUN opam install -y wasm_of_ocaml-compiler
43+
44+
# Compile partial render table benchmark
45+
RUN opam install opam-format stringext uucp cstruct
46+
RUN opam exec -- dune exec tools/ci_setup.exe janestreet .
47+
RUN opam install ppxlib.0.35.0 # temporary workaround
48+
RUN cd janestreet/lib/bonsai_web_components && git config pull.rebase true && git pull
49+
RUN eval $(opam env) \
50+
&& dune build --root janestreet --profile release lib/bonsai_web_components/partial_render_table/bench/bin/main.bc.wasm.js lib/bonsai_web_components/partial_render_table/bench/bin/main.bc.js
51+
RUN cp -r janestreet/_build/default/lib/bonsai_web_components/partial_render_table/bench/bin/main.bc.* ./benchmarks/benchmark-partial-render-table
52+
53+
# CAMLBOY
54+
RUN opam install brr \
55+
&& git clone --depth 1 https://github.com/ocaml-wasm/CAMLBOY -b node \
56+
&& cd CAMLBOY \
57+
&& opam exec -- dune build --root . --profile release ./bin/web
58+
2159
WORKDIR ./benchmarks

‎benchmarks/Makefile‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,21 @@ all: _perf graphsnopr graphspr graphseff _noperf
1212

1313
# This target is the one run by `current-bench`,
1414
# see https://github.com/ocurrent/current-bench
15-
.PHONY: bench
15+
.PHONY: bench microbenchmarks
1616
bench:
17+
$(MAKE) microbenchmarks
18+
$(MAKE) -C benchmark-fiat-crypto bench
19+
$(MAKE) -C benchmark-ocamlc bench
20+
$(MAKE) -C benchmark-partial-render-table bench
21+
$(MAKE) -C benchmark-camlboy bench
22+
23+
microbenchmarks:
1724
make _noprecomp
1825
$(RUN) ./report-wasm-cb.config ./report-jsoo-cb.config
1926
$(REPORT) -format current-bench -config report-wasm-cb.config \
20-
-ylabel "Wasm_of_ocaml"
27+
-ylabel "Wasm_of_ocaml" | sh utils/aggregate.sh wasm
2128
$(REPORT) -format current-bench -config report-jsoo-cb.config \
22-
-ylabel "Js_of_ocaml"
23-
29+
-ylabel "Js_of_ocaml" | sh utils/aggregate.sh js
2430

2531
graphsnopr: _noprecomp $(GRAPHSNOPR)
2632

‎benchmarks/benchmark-camlboy/Makefile‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
.PHONY: bench perform
2+
3+
export NAME=Camlboy
4+
5+
SHELL=/bin/bash -o pipefail
6+
7+
SCRIPT=../../CAMLBOY/_build/default/bin/web/bench_node.bc
8+
ROM=../../CAMLBOY/resource/games/tobu.gb
9+
10+
bench:
11+
$(MAKE) perform COMPILER=Js_of_ocaml SUFFIX=.js KIND=js
12+
$(MAKE) perform COMPILER=Wasm_of_ocaml SUFFIX=.wasm.js KIND=wasm
13+
14+
perform:
15+
node $(SCRIPT)$(SUFFIX) $(ROM) | \
16+
tee /dev/stderr | \
17+
ocaml output_results.ml $(COMPILER) | \
18+
sh ../utils/aggregate.sh $(KIND)

‎benchmarks/benchmark-camlboy/output_results.ml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
let () =
2+
Format.printf
3+
{|{ "name": "%s",
4+
"results":
5+
[ { "name": "Camlboy",
6+
"metrics":
7+
[ { "name": "Frames per second",
8+
"units": "Hz",
9+
"value": %s } ] } ] }@.|}
10+
Sys.argv.(1)
11+
(read_line ())

‎benchmarks/benchmark-fiat-crypto/Makefile‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
.PHONY: bench perform
2+
3+
export NAME=Fiat-Crypto
4+
5+
SHELL=/bin/bash -o pipefail
6+
7+
bench:
8+
$(MAKE) perform COMPILER=js_of_ocaml EXTRA_ARGS="--enable effects" KIND=js
9+
$(MAKE) perform COMPILER=wasm_of_ocaml EXTRA_ARGS="" KIND=wasm
10+
11+
perform: bedrock2_fiat_crypto.byte
12+
/usr/bin/time -f "%E %R" $(COMPILER) --debug times --source-map $(EXTRA_ARGS) $< 2>&1 | \
13+
ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) $(NAME) | \
14+
sh ../utils/aggregate.sh $(KIND)
15+
16+
bedrock2_fiat_crypto.byte: bedrock2_fiat_crypto.ml
17+
ocamlfind ocamlc -w -20 -g -linkpkg -package js_of_ocaml -g $< -o $@
18+
19+
bedrock2_fiat_crypto.ml: bedrock2_fiat_crypto.ml.xz
20+
unxz --keep $<

‎benchmarks/benchmark-fiat-crypto/bedrock2_fiat_crypto.ml.xz‎

691 KB
Binary file not shown.

‎benchmarks/benchmark-ocamlc/Makefile‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
.PHONY: bench perform
2+
3+
export NAME=Ocamlc
4+
5+
SHELL=/bin/bash -o pipefail
6+
7+
bench:
8+
cp -r ../sources/ml .
9+
$(MAKE) perform COMPILER=js_of_ocaml SCRIPT=ocamlc.js KIND=js
10+
$(MAKE) perform COMPILER=wasm_of_ocaml SCRIPT=ocamlc.wasm.js KIND=wasm
11+
12+
ARGS=ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml ml/*.ml
13+
14+
perform:
15+
/usr/bin/time -f "%E %R" $(COMPILER) --debug times --opt 2 --pretty `which ocamlc.byte` -o $(SCRIPT) 2>&1 | \
16+
ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) $(NAME) | \
17+
sh ../utils/aggregate.sh $(KIND)
18+
/usr/bin/time -f '{"compiler": "$(COMPILER)", "time":"%E"}' node $(SCRIPT) -c $(ARGS) 2>&1 | \
19+
sh ../utils/format_metrics.sh exec | \
20+
sh ../utils/aggregate.sh $(KIND)
21+

‎benchmarks/benchmark-partial-render-table/Makefile‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
.PHONY: bench perform
2+
3+
export NAME=Partial Render Table
4+
5+
SHELL=/bin/bash -o pipefail
6+
7+
bench:
8+
$(MAKE) perform COMPILER=js_of_ocaml SCRIPT=main.bc.js KIND=js
9+
$(MAKE) perform COMPILER=wasm_of_ocaml SCRIPT=main.bc.wasm.js KIND=wasm
10+
11+
BYTE_FILE=../../janestreet/_build/default/lib/bonsai_web_components/partial_render_table/bench/bin/main.bc-for-jsoo
12+
13+
perform:
14+
/usr/bin/time -f "%E %R" $(COMPILER) --debug times --opt 2 --pretty $(BYTE_FILE) -o out.js 2>&1 | \
15+
tee /dev/stderr | \
16+
ocaml -I +str str.cma ../utils/compilation_metrics.ml $(COMPILER) "$(NAME)" | \
17+
sh ../utils/aggregate.sh $(KIND)
18+
node $(SCRIPT) | \
19+
tee /dev/stderr | \
20+
ocaml -I +str str.cma summarize_results.ml $(COMPILER) | \
21+
sh ../utils/aggregate.sh $(KIND)

‎benchmarks/benchmark-partial-render-table/summarize_results.ml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
let delim = Str.regexp_string "│"
2+
3+
let geometric_mean l =
4+
exp (List.fold_left ( +. ) 0. (List.map log l) /. float (List.length l))
5+
6+
let parse s =
7+
Scanf.sscanf (String.trim s) "%f%s"
8+
@@ fun f u ->
9+
match u with
10+
| "ns" -> f *. 1e-6
11+
| "us" -> f *. 1e-3
12+
| "ms" -> f
13+
| "s" -> f *. 1e3
14+
| _ -> assert false
15+
16+
let () =
17+
let measures = ref [] in
18+
(try
19+
while true do
20+
let l = read_line () in
21+
if String.starts_with ~prefix:"├" l
22+
then
23+
let l = read_line () |> Str.split delim |> List.tl |> List.map parse in
24+
measures := l @ !measures
25+
done
26+
with End_of_file -> ());
27+
Format.printf
28+
{|{ "name": "%s",
29+
"results":
30+
[ { "name": "Partial Render Table",
31+
"metrics":
32+
[ { "name": "Execution time (geometric mean)",
33+
"units": "ms",
34+
"value": %f } ] } ] }@.|}
35+
(String.capitalize_ascii Sys.argv.(1))
36+
(geometric_mean !measures)

0 commit comments

Comments
 (0)