Validate benchmark results separately

.github/workflows/ci.yml

@@ -158,7 +158,8 @@ jobs:
           fail_on: nothing
           comment_mode: off
 
-      - run: cargo build --bench run_iai --release
+      - run: cargo build --all-targets --release
+      - run: cargo test --bench setup --release
       - run: cargo bench --bench run_iai
       - run: cargo bench --bench run_criterion
       - run: cargo bench --bench run_criterion -- --profile-time 60

Cargo.toml

@@ -121,6 +121,14 @@ pprof = { version = "0.13.0", features = ["criterion", "flamegraph"] }
 [patch.crates-io]
 modular-bitfield = { git = "https://github.com/mthom/modular-bitfield" }
 
+[profile.bench]
+lto = true
+opt-level = 3
+
+[profile.release]
+lto = true
+opt-level = 3
+
 [[bench]]
 name = "run_criterion"
 harness = false

benches/run_criterion.rs

@@ -29,8 +29,8 @@ fn config() -> Criterion {
 }
 
 criterion_group!(
-    name = bench_group;
+    name = benches;
     config = config();
     targets = bench_criterion
 );
-criterion_main!(bench_group);
+criterion_main!(benches);

benches/run_iai.rs

@@ -1,27 +1,18 @@
 use iai_callgrind::{library_benchmark, library_benchmark_group, main};
+use scryer_prolog::machine::parsed_results::QueryResolution;
 
 mod setup;
 
 #[library_benchmark]
-#[bench::normal(setup::prolog_benches()["count_edges_short"].setup())]
-fn bench_edges(mut run: impl FnMut()) {
-    run();
-}
-
-#[library_benchmark]
-#[bench::normal(setup::prolog_benches()["numlist_short"].setup())]
-fn bench_numlist(mut run: impl FnMut()) {
-    run();
-}
-
-#[library_benchmark]
-#[bench::normal(setup::prolog_benches()["csv_codename"].setup())]
-fn bench_csv_codename(mut run: impl FnMut()) {
-    run();
+#[bench::count_edges(setup::prolog_benches()["count_edges"].setup())]
+#[bench::numlist(setup::prolog_benches()["numlist"].setup())]
+#[bench::csv_codename(setup::prolog_benches()["csv_codename"].setup())]
+fn bench(mut run: impl FnMut() -> QueryResolution) -> QueryResolution {
+    run()
 }
 
 library_benchmark_group!(
-    name = bench_group;
-    benchmarks = bench_edges, bench_numlist, bench_csv_codename
+    name = benches;
+    benchmarks = bench
 );
-main!(library_benchmark_groups = bench_group);
+main!(library_benchmark_groups = benches);

benches/setup.rs

@@ -2,28 +2,21 @@ use std::{collections::BTreeMap, fs, path::Path};
 
 use maplit::btreemap;
 use scryer_prolog::machine::{
-    parsed_results::{QueryMatch, QueryResolution, Value},
+    parsed_results::{QueryResolution, Value},
     Machine,
 };
 
 pub fn prolog_benches() -> BTreeMap<&'static str, PrologBenchmark> {
     [
         (
             "count_edges",                       // name of the benchmark
-            "benches/edges.pl",                  // name of the prolog module file to load
-            "independent_set_count(aa, Count).", // query to benchmark in the context of the loaded module
-            Strategy::Reuse,
-            btreemap! { "Count" => Value::try_from("211954906".to_string()).unwrap(), }, // list of expected bindings
-        ),
-        (
-            "count_edges_short",
-            "benches/edges.pl", // use the same file in multiple benchmarks
-            "independent_set_count(ky, Count).", // consider making the query adjustable to tune the run time to ~0.1s
+            "benches/edges.pl", // name of the prolog module file to load. use the same file in multiple benchmarks
+            "independent_set_count(ky, Count).", // query to benchmark in the context of the loaded module. consider making the query adjustable to tune the run time to ~0.1s
             Strategy::Reuse,
             btreemap! { "Count" => Value::try_from("2869176".to_string()).unwrap() },
         ),
         (
-            "numlist_short",
+            "numlist",
             "benches/numlist.pl",
             "run_numlist(1000000, Head).",
             Strategy::Reuse,
@@ -67,27 +60,40 @@ pub struct PrologBenchmark {
 }
 
 impl PrologBenchmark {
-    pub fn setup(&self) -> impl FnMut() {
+    pub fn make_machine(&self) -> Machine {
         let program = fs::read_to_string(self.filename).unwrap();
         let module_name = Path::new(self.filename)
             .file_stem()
             .and_then(|s| s.to_str())
             .unwrap();
-
         let mut machine = Machine::new_lib();
         machine.load_module_string(module_name, program);
+        machine
+    }
 
-        let benchmark_name = self.name;
+    pub fn setup(&self) -> impl FnMut() -> QueryResolution {
+        let mut machine = self.make_machine();
         let query = self.query;
-        let expected = QueryResolution::Matches(vec![QueryMatch::from(self.bindings.clone())]);
-
         move || {
             use criterion::black_box;
-            let result = black_box(machine.run_query(black_box(query.to_string())));
-            match result {
-                Ok(r) => assert_eq!(&r, &expected),
-                Err(e) => panic!("benchmark {} failed with: {}", benchmark_name, e),
-            }
+            black_box(machine.run_query(black_box(query.to_string()))).unwrap()
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn validate_benchmarks() {
+        use super::prolog_benches;
+        use scryer_prolog::machine::parsed_results::QueryResolution;
+
+        use scryer_prolog::machine::parsed_results::QueryMatch;
+        for (_, r) in prolog_benches() {
+            let mut machine = r.make_machine();
+            let result = machine.run_query(r.query.to_string()).unwrap();
+            let expected = QueryResolution::Matches(vec![QueryMatch::from(r.bindings.clone())]);
+            assert_eq!(result, expected, "validating benchmark {}", r.name);
         }
     }
 }