test: add fuzz test for create table (#3441)

* feat: add create table fuzz test

* chore: add ci cfg for fuzz tests

* refactor: remove redundant nightly config

* chore: run fuzz test in debug mode

* chore: use ubuntu-latest

* fix: close connection

* chore: add cache in fuzz test ci

* chore: apply suggestion from CR

* chore: apply suggestion from CR

* chore: refactor the fuzz test action

.env.example

@@ -21,3 +21,6 @@ GT_GCS_CREDENTIAL_PATH = GCS credential path
 GT_GCS_ENDPOINT = GCS end point
 # Settings for kafka wal test
 GT_KAFKA_ENDPOINTS = localhost:9092
+
+# Setting for fuzz tests
+GT_MYSQL_ADDR = localhost:4002

.github/actions/fuzz-test/action.yaml

@@ -0,0 +1,13 @@
+name: Fuzz Test
+description: 'Fuzz test given setup and service'
+inputs:
+  target:
+    description: "The fuzz target to test"
+runs:
+  using: composite
+  steps:
+  - name: Run Fuzz Test
+    shell: bash
+    run: cargo fuzz run ${{ inputs.target }} --fuzz-dir tests-fuzz -D -s none -- -max_total_time=120
+    env:
+      GT_MYSQL_ADDR: 127.0.0.1:4002

.github/workflows/develop.yml

@@ -117,6 +117,46 @@ jobs:
           artifacts-dir: bins
           version: current
 
+  fuzztest: 
+    name: Fuzz Test
+    needs: build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [ "fuzz_create_table" ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: arduino/setup-protoc@v3
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: ${{ env.RUST_TOOLCHAIN }}
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2
+        with:
+          # Shares across multiple jobs
+          shared-key: "fuzz-test-targets"
+      - name: Set Rust Fuzz
+        shell: bash
+        run: |
+          sudo apt update && sudo apt install -y libfuzzer-14-dev
+          cargo install cargo-fuzz
+      - name: Download pre-built binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: bins
+          path: .
+      - name: Unzip binaries
+        run: tar -xvf ./bins.tar.gz
+      - name: Run GreptimeDB
+        run: | 
+          ./bins/greptime standalone start&
+      - name: Fuzz Test
+        uses: ./.github/actions/fuzz-test
+        env:
+          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
+        with:
+          target: ${{ matrix.target }}
+
   sqlness:
     name: Sqlness Test
     needs: build

.gitignore

@@ -46,3 +46,7 @@ benchmarks/data
 *.code-workspace
 
 venv/
+
+# Fuzz tests 
+tests-fuzz/artifacts/
+tests-fuzz/corpus/

tests-fuzz/Cargo.toml

@@ -7,15 +7,22 @@ license.workspace = true
 [lints]
 workspace = true
 
+[package.metadata]
+cargo-fuzz = true
+
 [dependencies]
+arbitrary = { version = "1.3.0", features = ["derive"] }
 async-trait = { workspace = true }
 common-error = { workspace = true }
 common-macro = { workspace = true }
 common-query = { workspace = true }
+common-runtime = { workspace = true }
 common-telemetry = { workspace = true }
 datatypes = { workspace = true }
 derive_builder = { workspace = true }
+dotenv = "0.15"
 lazy_static = { workspace = true }
+libfuzzer-sys = "0.4"
 partition = { workspace = true }
 rand = { workspace = true }
 rand_chacha = "0.3.1"
@@ -24,6 +31,12 @@ serde_json = { workspace = true }
 snafu = { workspace = true }
 sql = { workspace = true }
 sqlparser.workspace = true
+sqlx = { version = "0.6", features = [
+    "runtime-tokio-rustls",
+    "mysql",
+    "postgres",
+    "chrono",
+] }
 
 [dev-dependencies]
 dotenv = "0.15"
@@ -34,3 +47,10 @@ sqlx = { version = "0.6", features = [
     "chrono",
 ] }
 tokio = { workspace = true }
+
+[[bin]]
+name = "fuzz_create_table"
+path = "targets/fuzz_create_table.rs"
+test = false
+bench = false
+doc = false

tests-fuzz/README.md

@@ -0,0 +1,41 @@
+# Fuzz Test for GreptimeDB
+
+## Setup
+1. Install the [fuzz](https://rust-fuzz.github.io/book/cargo-fuzz/setup.html) cli first.
+```bash
+cargo install cargo-fuzz
+```
+
+2. Start GreptimeDB
+3. Copy the `.env.example`, which is at project root, to `.env` and change the values on need.
+
+## Run
+1. List all fuzz targets
+```bash
+cargo fuzz list --fuzz-dir tests-fuzz
+```
+
+2. Run a fuzz target.
+```bash
+cargo fuzz run fuzz_create_table --fuzz-dir tests-fuzz
+```
+
+## Crash Reproduction
+If you want to reproduce a crash, you first need to obtain the Base64 encoded code, which usually appears at the end of a crash report, and store it in a file.
+
+Alternatively, if you already have the crash file, you can skip this step.
+
+```bash
+echo "Base64" > .crash
+```
+Print the `std::fmt::Debug` output for an input.
+
+```bash
+cargo fuzz fmt fuzz_target .crash --fuzz-dir tests-fuzz  
+```
+Rerun the fuzz test with the input.
+
+```bash
+cargo fuzz run fuzz_target .crash --fuzz-dir tests-fuzz
+```
+For more details, visit [cargo fuzz](https://rust-fuzz.github.io/book/cargo-fuzz/tutorial.html) or run the command `cargo fuzz --help`.

tests-fuzz/src/error.rs

@@ -38,4 +38,12 @@ pub enum Error {
 
     #[snafu(display("No droppable columns"))]
     DroppableColumns { location: Location },
+
+    #[snafu(display("Failed to execute query: {}", sql))]
+    ExecuteQuery {
+        sql: String,
+        #[snafu(source)]
+        error: sqlx::error::Error,
+        location: Location,
+    },
 }

tests-fuzz/src/generator.rs

@@ -57,7 +57,15 @@ macro_rules! impl_random {
     ($type: ident, $value:ident, $values: ident) => {
         impl<R: Rng> Random<$type, R> for $value {
             fn choose(&self, rng: &mut R, amount: usize) -> Vec<$type> {
-                $values.choose_multiple(rng, amount).cloned().collect()
+                // Collects the elements in deterministic order first.
+                let mut result = std::collections::BTreeSet::new();
+                while result.len() != amount {
+                    result.insert($values.choose(rng).unwrap().clone());
+                }
+                let mut result = result.into_iter().collect::<Vec<_>>();
+                // Shuffles the result slice.
+                result.shuffle(rng);
+                result
             }
         }
     };

tests-fuzz/src/generator/alter_expr.rs

@@ -155,7 +155,7 @@ mod tests {
             .generate(&mut rng)
             .unwrap();
         let serialized = serde_json::to_string(&expr).unwrap();
-        let expected = r#"{"table_name":"DigNissIMOS","alter_options":{"AddColumn":{"column":{"name":"sit","column_type":{"Boolean":null},"options":["PrimaryKey"]},"location":null}}}"#;
+        let expected = r#"{"table_name":"animI","alter_options":{"AddColumn":{"column":{"name":"velit","column_type":{"Int32":{}},"options":[{"DefaultValue":{"Int32":853246610}}]},"location":null}}}"#;
         assert_eq!(expected, serialized);
 
         let expr = AlterExprRenameGeneratorBuilder::default()
@@ -165,7 +165,8 @@ mod tests {
             .generate(&mut rng)
             .unwrap();
         let serialized = serde_json::to_string(&expr).unwrap();
-        let expected = r#"{"table_name":"DigNissIMOS","alter_options":{"RenameTable":{"new_table_name":"excepturi"}}}"#;
+        let expected =
+            r#"{"table_name":"animI","alter_options":{"RenameTable":{"new_table_name":"iure"}}}"#;
         assert_eq!(expected, serialized);
 
         let expr = AlterExprDropColumnGeneratorBuilder::default()
@@ -175,8 +176,7 @@ mod tests {
             .generate(&mut rng)
             .unwrap();
         let serialized = serde_json::to_string(&expr).unwrap();
-        let expected =
-            r#"{"table_name":"DigNissIMOS","alter_options":{"DropColumn":{"name":"INVentORE"}}}"#;
+        let expected = r#"{"table_name":"animI","alter_options":{"DropColumn":{"name":"toTAm"}}}"#;
         assert_eq!(expected, serialized);
     }
 }

tests-fuzz/src/generator/create_expr.rs

@@ -230,7 +230,7 @@ mod tests {
             .unwrap();
 
         let serialized = serde_json::to_string(&expr).unwrap();
-        let expected = r#"{"table_name":"iN","columns":[{"name":"CUlpa","column_type":{"Int16":{}},"options":["PrimaryKey","NotNull"]},{"name":"dEBiTiS","column_type":{"Timestamp":{"Second":null}},"options":["TimeIndex"]},{"name":"HArum","column_type":{"Int16":{}},"options":["NotNull"]},{"name":"NObIS","column_type":{"Int32":{}},"options":["PrimaryKey"]},{"name":"IMPEDiT","column_type":{"Int16":{}},"options":[{"DefaultValue":{"Int16":-25151}}]},{"name":"bLanDITIis","column_type":{"Boolean":null},"options":[{"DefaultValue":{"Boolean":true}}]},{"name":"Dolores","column_type":{"Float32":{}},"options":["PrimaryKey"]},{"name":"eSt","column_type":{"Float32":{}},"options":[{"DefaultValue":{"Float32":0.9152612}}]},{"name":"INVentORE","column_type":{"Int64":{}},"options":["PrimaryKey"]},{"name":"aDIpiSci","column_type":{"Float64":{}},"options":["Null"]}],"if_not_exists":true,"partition":{"partition_columns":["CUlpa"],"partition_bounds":[{"Value":{"Int16":15966}},{"Value":{"Int16":31925}},"MaxValue"]},"engine":"mito2","options":{},"primary_keys":[6,0,8,3]}"#;
+        let expected = r#"{"table_name":"tEmporIbUS","columns":[{"name":"IMpEdIT","column_type":{"String":null},"options":["PrimaryKey","NotNull"]},{"name":"natuS","column_type":{"Timestamp":{"Nanosecond":null}},"options":["TimeIndex"]},{"name":"ADIPisCI","column_type":{"Int16":{}},"options":[{"DefaultValue":{"Int16":4864}}]},{"name":"EXpEdita","column_type":{"Int64":{}},"options":["PrimaryKey"]},{"name":"cUlpA","column_type":{"Float64":{}},"options":["NotNull"]},{"name":"MOLeStIAs","column_type":{"Boolean":null},"options":["Null"]},{"name":"cUmquE","column_type":{"Float32":{}},"options":[{"DefaultValue":{"Float32":0.21569687}}]},{"name":"toTAm","column_type":{"Float64":{}},"options":["NotNull"]},{"name":"deBitIs","column_type":{"Float32":{}},"options":["Null"]},{"name":"QUi","column_type":{"Int64":{}},"options":["Null"]}],"if_not_exists":true,"partition":{"partition_columns":["IMpEdIT"],"partition_bounds":[{"Value":{"String":"򟘲"}},{"Value":{"String":"򴥫"}},"MaxValue"]},"engine":"mito2","options":{},"primary_keys":[0,3]}"#;
         assert_eq!(expected, serialized);
     }
 }

tests-fuzz/src/lib.rs

@@ -21,6 +21,7 @@ pub mod fake;
 pub mod generator;
 pub mod ir;
 pub mod translator;
+pub mod utils;
 
 #[cfg(test)]
 pub mod test_utils;

tests-fuzz/src/utils.rs

@@ -0,0 +1,42 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+
+use common_telemetry::info;
+use sqlx::mysql::MySqlPoolOptions;
+use sqlx::{MySql, Pool};
+
+pub struct Connections {
+    pub mysql: Option<Pool<MySql>>,
+}
+
+const GT_MYSQL_ADDR: &str = "GT_MYSQL_ADDR";
+
+pub async fn init_greptime_connections() -> Connections {
+    let _ = dotenv::dotenv();
+    let mysql = if let Ok(addr) = env::var(GT_MYSQL_ADDR) {
+        Some(
+            MySqlPoolOptions::new()
+                .connect(&format!("mysql://{addr}/public"))
+                .await
+                .unwrap(),
+        )
+    } else {
+        info!("GT_MYSQL_ADDR is empty, ignores test");
+        None
+    };
+
+    Connections { mysql }
+}