diff --git a/.github/actions/setup-rust/cleanup.sh b/.github/actions/setup-rust/cleanup.sh index fd51bc92b..280f5e98d 100755 --- a/.github/actions/setup-rust/cleanup.sh +++ b/.github/actions/setup-rust/cleanup.sh @@ -3,8 +3,11 @@ set -e echo Cleanup workspace build artifacts and extra target output -# clean just the direct members of the current workspace, with cargo metadata should generalize to all rust projects -cargo clean -p `cargo metadata --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'` +# clean just the direct members of the current workspace, use cargo metadata to generalize to all rust projects +cargo clean -p `cargo metadata --no-deps --offline --format-version 1 | jq -r '[.workspace_members[]|split(" ")|.[0]]|join(" ")'` # remove directories in /target/ that are not named `debug` or `release` +before=`du -s target | awk '{print $1}'` find ./target -maxdepth 1 -type d ! -name debug ! -name release ! -name target -exec rm -r {} \; +after=`du -s target | awk '{print $1}'` +echo Deleted $(($before - $after)) bytes from target directory diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b87d1432d..9fa463a40 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -142,7 +142,8 @@ jobs: cache-context: report - run: | cargo install cargo2junit --force - cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'` + # cargo install iai-callgrind-runner --force --version `cargo metadata --format-version 1 | jq -r '.resolve.nodes[].id|split(" ")|select(.[0]=="iai-callgrind")|.[1]'` + cargo install iai-callgrind-runner --force --git https://github.com/iai-callgrind/iai-callgrind --rev c77bc3c83d7f4e976cc42d4597236a8db259e772 sudo apt install valgrind -y - name: Test and report @@ -161,21 +162,20 @@ jobs: fail_on: nothing comment_mode: off - - run: cargo build --bench run_iai --release - - run: cargo bench --bench run_iai + - run: cargo build --all-targets --release + - run: cargo test --bench setup --release + - run: cargo bench --bench run_iai -- --save-summary=json - run: cargo bench --bench run_criterion + - run: cargo bench --bench run_criterion -- --profile-time 60 - - name: Publish criterion benchmark results + - name: Publish benchmark results uses: actions/upload-artifact@v3 with: - name: bench-criterion-results - path: target/criterion/**/* - - - name: Publish iai benchmark results - uses: actions/upload-artifact@v3 - with: - name: bench-iai-results - path: target/iai/scryer-prolog/run_iai/bench_group/* + name: benchmark-results + path: | + target/criterion/* + target/iai/* + target/benchmark_inference_counts.json # Publish binaries when building for a tag release: diff --git a/Cargo.lock b/Cargo.lock index ef0cb578c..14652313e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,19 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.0.2" @@ -59,6 +72,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "assert_cmd" version = "1.0.8" @@ -234,6 +253,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +[[package]] +name = "bytemuck" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6" + [[package]] name = "byteorder" version = "1.4.3" @@ -370,6 +395,15 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "cpp_demangle" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" +dependencies = [ + "cfg-if", +] + [[package]] name = "cpu-time" version = "1.0.0" @@ -610,6 +644,15 @@ dependencies = [ "num-order", ] +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + [[package]] name = "derive_deref" version = "1.1.1" @@ -701,6 +744,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.3" @@ -749,6 +798,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "fnv" version = "1.0.7" @@ -965,7 +1026,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1136,9 +1197,8 @@ dependencies = [ [[package]] name = "iai-callgrind" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2679de583a9f5232b45b1f3a31b5e2f739bccfee9b7902488aca8f8c2c5482d1" +version = "0.8.0" +source = "git+https://github.com/iai-callgrind/iai-callgrind.git?rev=c77bc3c83d7f4e976cc42d4597236a8db259e772#c77bc3c83d7f4e976cc42d4597236a8db259e772" dependencies = [ "bincode", "iai-callgrind-macros", @@ -1148,8 +1208,7 @@ dependencies = [ [[package]] name = "iai-callgrind-macros" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af5af66b85e350097b8c0f6329c6347d3323d010443475741c29a1a167f116fb" +source = "git+https://github.com/iai-callgrind/iai-callgrind.git?rev=c77bc3c83d7f4e976cc42d4597236a8db259e772#c77bc3c83d7f4e976cc42d4597236a8db259e772" dependencies = [ "proc-macro-error", "proc-macro2", @@ -1159,9 +1218,8 @@ dependencies = [ [[package]] name = "iai-callgrind-runner" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1093f263249d7c541c53849dac74f4ae756cd31e3cac1a12204efcec68bd" +version = "0.8.0" +source = "git+https://github.com/iai-callgrind/iai-callgrind.git?rev=c77bc3c83d7f4e976cc42d4597236a8db259e772#c77bc3c83d7f4e976cc42d4597236a8db259e772" dependencies = [ "serde", ] @@ -1209,6 +1267,34 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indexmap" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + +[[package]] +name = "inferno" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50453ec3a6555fad17b1cd1a80d16af5bc7cb35094f64e429fd46549018c6a3" +dependencies = [ + "ahash", + "indexmap 2.0.1", + "is-terminal", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml", + "rgb", + "str_stack", +] + [[package]] name = "instant" version = "0.1.12" @@ -1290,7 +1376,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ - "arrayvec", + "arrayvec 0.5.2", "bitflags 1.3.2", "cfg-if", "ryu", @@ -1404,6 +1490,15 @@ version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +[[package]] +name = "memmap2" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a5a03cefb0d953ec0be133036f14e109412fa594edc2f77227249db66cc3ed" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.0" @@ -1572,6 +1667,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec 0.7.4", + "itoa", +] + [[package]] name = "num-modular" version = "0.6.1" @@ -1888,6 +1993,28 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "pprof" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5c97c51bd34c7e742402e216abdeb44d415fbe6ae41d56b114723e953711cb" +dependencies = [ + "backtrace", + "cfg-if", + "criterion", + "findshlibs", + "inferno", + "libc", + "log", + "nix 0.26.4", + "once_cell", + "parking_lot 0.12.1", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1966,6 +2093,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.33" @@ -2148,6 +2284,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "rgb" +version = "0.8.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.16.20" @@ -2331,7 +2476,7 @@ dependencies = [ "git-version", "hostname", "iai-callgrind", - "indexmap", + "indexmap 1.9.3", "js-sys", "lazy_static", "lexical", @@ -2344,6 +2489,7 @@ dependencies = [ "num-order", "ordered-float", "phf 0.9.0", + "pprof", "predicates-core", "proc-macro2", "quote", @@ -2627,6 +2773,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -2639,6 +2791,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0" +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + [[package]] name = "string_cache" version = "0.8.7" @@ -2696,6 +2854,29 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "symbolic-common" +version = "12.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "405af7bd5edd866cef462e22ef73f11cf9bf506c9d62824fef8364eb69d4d4ad" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bcd041ccfb77d9c70639efcd5b804b508ac7a273e9224d227379e225625daf9" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -3037,6 +3218,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" + [[package]] name = "vcpkg" version = "0.2.15" @@ -3342,3 +3529,23 @@ name = "xmlparser" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" + +[[package]] +name = "zerocopy" +version = "0.7.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] diff --git a/Cargo.toml b/Cargo.toml index 5a7cc04bb..ae273b0d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -112,12 +112,23 @@ assert_cmd = "1.0.3" predicates-core = "1.0.2" maplit = "1.0.2" serial_test = "2.0.0" -iai-callgrind = "0.7.3" +iai-callgrind = { git = "https://github.com/iai-callgrind/iai-callgrind.git", rev = "c77bc3c83d7f4e976cc42d4597236a8db259e772" } criterion = "0.5.1" +[target.'cfg(not(target_os = "windows"))'.dev-dependencies] +pprof = { version = "0.13.0", features = ["criterion", "flamegraph"] } + [patch.crates-io] modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" } +[profile.bench] +lto = true +opt-level = 3 + +[profile.release] +lto = true +opt-level = 3 + [[bench]] name = "run_criterion" harness = false diff --git a/benches/README.md b/benches/README.md index d2d8f63da..1dbf90aae 100644 --- a/benches/README.md +++ b/benches/README.md @@ -4,16 +4,16 @@ The `benches` directory contains benchmarks that test scryer-prolog performance. Benchmarks are run via two harnesses: +* `criterion` - criterion performs statistical analysis of benchmark runs and is + great for benchmarking locally. * `iai-callgrind` - this runs the benchmark with callgrind, which is able to precisely track the number of instructions executed during the run. This is especially helpful in a public CI runner context where neighboring VMs can - cause a very high wall time variance. This means that it doesn't track wall - time which is what we really care about, but it is a good tradeoff for CI - where tracking runtime is unreliable. -* `criterion` - criterion performs statistical analysis of benchmark runs and is - great for benchmarking locally. + cause a very high wall time variance. While instructions executed is only + correlated with the desired metric (wall time), this is a good tradeoff for CI + where that metric is unreliable. -Run them using the following commands: +Run benchmarks with the following commands: ``` cargo bench --bench run_criterion @@ -21,6 +21,9 @@ cargo bench --bench run_criterion # run a particular criterion benchmark cargo bench --bench run_criterion -- +# run in profiling mode which outputs flamegraphs. Set profile time in seconds: +cargo bench --bench run_criterion -- --profile-time