Fix Benchmark GitHub Action to work with Criterion benchmarking (#277)

 - Give workflow a name
 - Added `bench=false` to `lib`s and `bin`s as per the [criterion faq](https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options)

.github/workflows/benchmark.yml

@@ -1,3 +1,5 @@
+name: Benchmark
+
 # Do not run this workflow on pull request since this workflow has permission to modify contents.
 on:
   push:
@@ -32,9 +34,9 @@ jobs:
           # nightly can be very volatile--pin this to a version we know works well...
           toolchain: nightly-2022-08-11
           override: true
-      # Run benchmark with `cargo +nightly -bench` and stores the output to a file
+      # Run benchmark with `cargo bench` and stores the output to a file
       - name: Run benchmark
-        run: cargo +nightly bench | tee output.txt
+        run: cargo bench -- --output-format bencher | tee output.txt
       # gh-pages branch is updated and pushed automatically with extracted benchmark data
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1

.github/workflows/ci_build_test.yml

@@ -78,7 +78,6 @@ jobs:
   conformance-report:
     name: Create conformance report for `push` and `pull_request` events
     runs-on: ubuntu-20.04
-    needs: [build]
     steps:
       # Pull down the cached `partiql-lang-rust` build from the `Build and Test` job. This allows us to reuse without
       # needing to rebuild. If pulling the build fails, the subsequent `cargo test` will rebuild.
@@ -101,7 +100,7 @@ jobs:
       # to format and use the output.
       - name: Cargo Test of the conformance tests (can fail) and save to json file
         continue-on-error: true
-        run: cargo test --verbose --package partiql-conformance-tests --features "conformance_test" -- -Z unstable-options --ensure-time --format json > cargo_test_results.json
+        run: cargo test --verbose --package partiql-conformance-tests --features "conformance_test" --release -- -Z unstable-options --ensure-time --format json > cargo_test_results.json
       # Create a conformance report from the `cargo test` json file
       - run: cargo run --features report_tool --bin generate_cts_report cargo_test_results.json ${GITHUB_SHA} cts_report.json
       # Upload conformance report for comparison with future runs

partiql-ast/Cargo.toml

@@ -15,10 +15,9 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [lib]
 path = "src/lib.rs"
+bench = false
 
 [dependencies]
 rust_decimal = { version = "1.25.0", default-features = false, features = ["std"] }

partiql-ast/partiql-ast-macros/Cargo.toml

@@ -19,6 +19,7 @@ edition.workspace = true
 [lib]
 proc-macro = true
 path = "src/lib.rs"
+bench = false
 
 [dependencies]
 quote = "1.0.*"

partiql-conformance-test-generator/Cargo.toml

@@ -17,9 +17,8 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# TODO: fill in other details
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]
 walkdir = "2.3"

partiql-conformance-tests/Cargo.toml

@@ -20,12 +20,12 @@ edition.workspace = true
 [[bin]]
 name = "generate_comparison_report"
 required-features = ["report_tool"]
+bench = false
 
 [[bin]]
 name = "generate_cts_report"
 required-features = ["report_tool"]
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+bench = false
 
 [build-dependencies]
 miette = { version ="5.*", features = ["fancy"] }

partiql-eval/Cargo.toml

@@ -17,7 +17,8 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]
 partiql-logical = { path = "../partiql-logical", version = "0.2.*" }

partiql-ir/Cargo.toml

@@ -17,6 +17,7 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]

partiql-irgen/Cargo.toml

@@ -17,6 +17,7 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]

partiql-logical-planner/Cargo.toml

@@ -17,6 +17,9 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
+[lib]
+bench = false
+
 [dependencies]
 partiql-value = { path = "../partiql-value", version = "0.2.*" }
 partiql-logical = { path = "../partiql-logical", version = "0.2.*" }

partiql-logical/Cargo.toml

@@ -17,7 +17,8 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]
 partiql-value = { path = "../partiql-value", version = "0.2.*" }

partiql-parser/Cargo.toml

@@ -17,6 +17,8 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
+[lib]
+bench = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [build-dependencies]

partiql-rewriter/Cargo.toml

@@ -17,6 +17,7 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]

partiql-source-map/Cargo.toml

@@ -15,7 +15,8 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]
 smallvec = { version = "1.*" }

partiql-types/Cargo.toml

@@ -17,7 +17,8 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]
 ordered-float = "3.*"

partiql-value/Cargo.toml

@@ -17,7 +17,8 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+bench = false
 
 [dependencies]
 ordered-float = "3.*"

partiql/Cargo.toml

@@ -17,6 +17,9 @@ exclude = [
 version.workspace = true
 edition.workspace = true
 
+[lib]
+bench = false
+
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]