Replace hardcoded percentages with values from a file

CMakeLists.txt

@@ -21,6 +21,7 @@
 set(EXTENSION_SOURCES src/sqlsmith_extension.cpp
         src/statement_generator.cpp
         src/statement_simplifier.cpp
+        src/random_nums_config.cpp
         src/fuzzyduck.cpp ${EXTENSION_OBJECT_FILES})
 
 build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES})

config.json

@@ -0,0 +1,8 @@
+{
+    "select_percentage": "70",
+    "attach_percentage": "20",
+    "attach_use_percentage": "80",
+    "detach_percentage": "15",
+    "set_percentage": "5",
+    "delete_percentage": "25"
+}

config_nested.json

@@ -0,0 +1,30 @@
+{
+    "attach_percentage": "20",
+    "select": {
+        "select_percentage": "1",
+        "select_node": {
+            "select_node_perc": "2",
+            "select_node_is_distinct_perc": "3",
+            "select_node_from_table_perc": "4",
+            "select_node_where_perc": "5",
+            "select_node_having_perc": "6",
+            "select_node_groups": {
+                "select_node_groups_perc": "7",
+                "select_node_group_by_perc": "8"
+            },
+            "select_node_qualify_perc": "9",
+            "select_node_aggregate_perc": "10",
+            "select_node_sample": {
+                "select_node_sample_perc": "11",
+                "select_node_sample_is_perc": "12",
+                "select_node_sample_size": "13"
+            }
+        },
+        "result_modifiers": "14",
+        "limit_percent_modifier": "15",
+        "limit_percent_modifier_limit": "16",
+        "limit_percent_modifier_offset": "17",
+        "limit_modifier_limit": "18", 
+        "limit_modifier_offset": "19"
+    }
+}

scripts/run_fuzzer.py

@@ -34,6 +34,8 @@
         perform_checks = False
     elif param.startswith('--enable_verification'):
         verification = param.replace('--enable_verification=', '').lower() == 'true'
+    elif param.startswith('--randoms_config_filepath'):
+        randoms_config_filepath = param.replace('--randoms_config_filepath=', '')
     elif param.startswith('--shell='):
         shell = param.replace('--shell=', '')
     elif param.startswith('--seed='):
@@ -76,7 +78,8 @@ def run_fuzzer_script(fuzzer):
     if fuzzer == 'sqlsmith':
         return "call sqlsmith(max_queries=${MAX_QUERIES}, seed=${SEED}, verbose_output=1, log='${LAST_LOG_FILE}', complete_log='${COMPLETE_LOG_FILE}');"
     elif fuzzer == 'duckfuzz':
-        return "call fuzzyduck(max_queries=${MAX_QUERIES}, seed=${SEED}, verbose_output=1, log='${LAST_LOG_FILE}', complete_log='${COMPLETE_LOG_FILE}', enable_verification='${ENABLE_VERIFICATION}');"
+        return "call fuzzyduck(max_queries=${MAX_QUERIES}, seed=${SEED}, verbose_output=1, log='${LAST_LOG_FILE}', complete_log='${COMPLETE_LOG_FILE}', \
+        enable_verification='${ENABLE_VERIFICATION}', randoms_config_filepath='${RANDOMS_CONFIG_FILEPATH}');"
     elif fuzzer == 'duckfuzz_functions':
         return "call fuzz_all_functions(seed=${SEED}, verbose_output=1, log='${LAST_LOG_FILE}', complete_log='${COMPLETE_LOG_FILE}');"
     else:
@@ -132,6 +135,7 @@ def run_shell_command(cmd):
     .replace('${COMPLETE_LOG_FILE}', complete_log_file)
     .replace('${SEED}', str(seed))
     .replace('${ENABLE_VERIFICATION}', str(verification))
+    .replace('${RANDOMS_CONFIG_FILEPATH}', randoms_config_filepath)
 )
 
 print(load_script)

src/CMakeLists.txt

@@ -7,7 +7,7 @@ include_directories(third_party/sqlsmith/include)
 add_subdirectory(third_party)
 
 set(SQLSMITH_SOURCES
-    sqlsmith_extension.cpp statement_generator.cpp statement_simplifier.cpp
+    sqlsmith_extension.cpp statement_generator.cpp statement_simplifier.cpp random_nums_config.cpp
     fuzzyduck.cpp ${SQLSMITH_OBJECT_FILES})
 
 build_static_extension(sqlsmith ${SQLSMITH_SOURCES})

src/fuzzyduck.cpp

@@ -22,15 +22,23 @@ void FuzzyDuck::BeginFuzzing() {
 	if (max_queries == 0) {
 		throw BinderException("Provide a max_queries argument greater than 0");
 	}
+	auto &fs = FileSystem::GetFileSystem(context);
 	if (!complete_log.empty()) {
-		auto &fs = FileSystem::GetFileSystem(context);
 		TryRemoveFile(complete_log);
 		complete_log_handle =
 			fs.OpenFile(complete_log, FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW);
 	}
 	if (enable_verification) {
 		RunQuery("PRAGMA enable_verification");
 	}
+	if (!randoms_config_filepath.empty()) {
+		config = GetConfigFromFile(randoms_config_filepath.c_str());
+		if (!config.size()) {
+			config = GetDefaultConfig();
+		}
+	} else {
+		config = GetDefaultConfig();
+	}
 }
 
 void FuzzyDuck::EndFuzzing() {
@@ -78,12 +86,12 @@ string FuzzyDuck::GenerateQuery() {
 		LogTask("Generating Multi-Statement query of " + to_string(number_of_statements) + " statements with seed " +
 			to_string(seed));
 		for (idx_t i = 0; i < number_of_statements; i++) {
-			statement += generator.GenerateStatement()->ToString() + "; ";
+			statement += generator.GenerateStatement(config)->ToString() + "; ";
 		}
 	} else {
 		// normal statement
 		LogTask("Generating Single-Statement query with seed " + to_string(seed));
-		statement = generator.GenerateStatement()->ToString();
+		statement = generator.GenerateStatement(config)->ToString();
 	}
 	return statement;
 }

src/include/fuzzyduck.hpp

@@ -10,8 +10,10 @@
 
 #include "duckdb.hpp"
 #include "duckdb/parser/query_node.hpp"
+#include "random_nums_config.hpp"
 
 namespace duckdb {
+
 struct FileHandle;
 
 class FuzzyDuck {
@@ -27,6 +29,8 @@ class FuzzyDuck {
 	bool verbose_output = false;
 	bool enable_verification = false;
 	idx_t timeout = 30;
+	string randoms_config_filepath;
+	unordered_map<RandomPercentagesEnum, idx_t> config;
 
 public:
 	void Fuzz();
@@ -50,6 +54,7 @@ class FuzzyDuck {
 
 private:
 	unique_ptr<FileHandle> complete_log_handle;
+	unique_ptr<FileHandle> randoms_config_handle;
 };
 
 } // namespace duckdb

src/include/random_nums_config.hpp

@@ -0,0 +1,54 @@
+// ===----------------------------------------------------------------------===//
+//                         DuckDB
+
+// random_nums_config.hpp
+
+
+// ===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include "duckdb.hpp"
+#include "yyjson.hpp"
+
+namespace duckdb {
+
+enum class RandomPercentagesEnum : idx_t {
+	SELECT = 0,
+    ATTACH = 1,
+    ATTACH_USE = 2,
+    DETACH = 3,
+    SET = 4,
+    DELETE = 5,
+
+    // ----------------------------------
+	// Generate Select Percentages Types
+	// ----------------------------------
+    SELECT_NODE = 6,
+    SELECT_NODE_IS_DISTINCT = 7,
+    SELECT_NODE_FROM_TABLE = 8,
+    SELECT_NODE_WHERE = 9,
+    SELECT_NODE_HAVING = 10,
+    SELECT_NODE_GROUPS = 11,
+    SELECT_NODE_GROUP_BY = 12,
+    SELECT_NODE_QUALIFY = 13,
+    SELECT_NODE_AGGREGATE = 14,
+    SELECT_NODE_SAMPLE = 15,
+    SELECT_NODE_SAMPLE_IS_PERC = 16,
+    SELECT_NODE_SAMPLE_SIZE = 17,
+    RESULT_MODIFIERS = 18,
+    LIMIT_PERCENT_MODIFIER = 19,
+    LIMIT_PERCENT_MODIFIER_LIMIT = 20,
+    LIMIT_PERCENT_MODIFIER_OFFSET = 21,
+    LIMIT_MODIFIER_LIMIT = 22,
+    LIMIT_MODIFIER_OFFSET = 23,
+
+    COUNT
+
+};
+
+unordered_map<RandomPercentagesEnum, idx_t> GetDefaultConfig();
+unordered_map<RandomPercentagesEnum, idx_t> GetConfigFromFile(const char *json_string);
+string RandomPercentagesEnumToString(RandomPercentagesEnum type);
+
+} // namespace duckdb

src/include/statement_generator.hpp

@@ -11,6 +11,7 @@
 #include "duckdb.hpp"
 #include "duckdb/parser/parsed_data/detach_info.hpp"
 #include "duckdb/parser/query_node.hpp"
+#include "random_nums_config.hpp"
 
 #define TESTING_DIRECTORY_NAME "duckdb_unittest_tempdir"
 
@@ -47,14 +48,16 @@ class StatementGenerator {
 	~StatementGenerator();
 
 public:
-	unique_ptr<SQLStatement> GenerateStatement();
+	unique_ptr<SQLStatement> GenerateStatement(unordered_map<RandomPercentagesEnum, idx_t> config);
 
 	vector<string> GenerateAllFunctionCalls();
 
 	//! Returns true with a percentage change (0-100)
 	bool RandomPercentage(idx_t percentage);
 	bool verification_enabled = false;
 	idx_t RandomValue(idx_t max);
+
+
 	string GetRandomAttachedDataBase();
 	unique_ptr<SQLStatement> GenerateStatement(StatementType type); // came from private
 

src/random_nums_config.cpp

@@ -0,0 +1,96 @@
+#include "include/random_nums_config.hpp"
+
+#include <algorithm>
+#include <iterator>
+#include <ctype.h>
+
+namespace duckdb {
+
+using namespace duckdb_yyjson;
+
+unordered_map<RandomPercentagesEnum, idx_t> GetDefaultConfig() {
+    unordered_map<RandomPercentagesEnum, idx_t> default_config = {
+        { RandomPercentagesEnum::ATTACH, 40 },
+        { RandomPercentagesEnum::ATTACH_USE, 50 },
+        { RandomPercentagesEnum::DELETE, 40 },
+        { RandomPercentagesEnum::DETACH, 60 },
+        { RandomPercentagesEnum::SELECT, 60 },
+        { RandomPercentagesEnum::SET, 30 }
+    };
+    return default_config;
+}
+
+unordered_map<string, RandomPercentagesEnum> StringToRandomPercentagesEnum = {
+    { "attach_percentage", RandomPercentagesEnum::ATTACH },
+    { "attach_use_percentage", RandomPercentagesEnum::ATTACH_USE },
+    { "delete_percentage", RandomPercentagesEnum::DELETE },
+    { "detach_percentage", RandomPercentagesEnum::DETACH },
+    { "select_percentage", RandomPercentagesEnum::SELECT },
+    { "select_node_perc", RandomPercentagesEnum::SELECT_NODE },
+    { "select_node_is_distinct_perc", RandomPercentagesEnum::SELECT_NODE_IS_DISTINCT },
+    { "select_node_from_table_perc", RandomPercentagesEnum::SELECT_NODE_FROM_TABLE },
+    { "select_node_where_perc", RandomPercentagesEnum::SELECT_NODE_WHERE },
+    { "select_node_having_perc", RandomPercentagesEnum::SELECT_NODE_HAVING },
+    { "select_node_groups_perc", RandomPercentagesEnum::SELECT_NODE_GROUPS },
+    { "select_node_group_by_perc", RandomPercentagesEnum::SELECT_NODE_GROUP_BY },
+    { "select_node_qualify_perc", RandomPercentagesEnum::SELECT_NODE_QUALIFY },
+    { "select_node_aggregate_perc", RandomPercentagesEnum::SELECT_NODE_AGGREGATE },
+    { "select_node_sample_perc", RandomPercentagesEnum::SELECT_NODE_SAMPLE },
+    { "select_node_sample_is_perc", RandomPercentagesEnum::SELECT_NODE_SAMPLE_IS_PERC },
+    { "select_node_sample_size", RandomPercentagesEnum::SELECT_NODE_SAMPLE_SIZE },
+    { "result_modifiers", RandomPercentagesEnum::RESULT_MODIFIERS },
+    { "limit_percent_modifier", RandomPercentagesEnum::LIMIT_PERCENT_MODIFIER },
+    { "limit_percent_modifier_limit", RandomPercentagesEnum::LIMIT_PERCENT_MODIFIER_LIMIT },
+    { "limit_percent_modifier_offset", RandomPercentagesEnum::LIMIT_PERCENT_MODIFIER_OFFSET },
+    { "limit_modifier_limit", RandomPercentagesEnum::LIMIT_MODIFIER_LIMIT },
+    { "limit_modifier_offset", RandomPercentagesEnum::LIMIT_MODIFIER_OFFSET }
+};
+
+void ParseJsonObj(yyjson_val *obj, unordered_map<RandomPercentagesEnum, idx_t> &config_from_file) {
+    yyjson_obj_iter iter;
+    yyjson_obj_iter_init(obj, &iter);
+    size_t idx, max;
+    yyjson_val *key, *val;
+    yyjson_obj_foreach(obj, idx, max, key, val) {
+        const char* root_key = yyjson_get_str(key);
+        auto it = StringToRandomPercentagesEnum.find(root_key);
+        if (it != StringToRandomPercentagesEnum.end()) {
+            RandomPercentagesEnum perc_type = it->second;
+            auto perc_value = yyjson_get_str(val);
+            if (perc_value) {
+                config_from_file[perc_type] = std::stoi(perc_value);
+            }
+        }
+        if (yyjson_is_obj(val)) {
+            ParseJsonObj(val, config_from_file);
+        }
+    }
+}
+
+
+
+unordered_map<RandomPercentagesEnum, idx_t> GetConfigFromFile(const char *json_string) {
+
+    unordered_map<RandomPercentagesEnum, idx_t> config_from_file;
+    auto doc = yyjson_read_file(json_string, YYJSON_READ_NOFLAG, NULL, NULL);
+    if (doc) {
+        yyjson_val *root = yyjson_doc_get_root(doc);
+        if (yyjson_is_obj(root)) {
+            ParseJsonObj(root, config_from_file);
+        }
+        yyjson_doc_free(doc);
+    } else {
+        // Couldn't read JSON with percentages config
+        yyjson_doc_free(doc);
+        return GetDefaultConfig();
+    }
+    // set values of missing statement types to 0
+    for (idx_t i = 0; i < static_cast<int>(RandomPercentagesEnum::COUNT); ++i) {
+        RandomPercentagesEnum statement_type = static_cast<RandomPercentagesEnum>(i);
+        if (config_from_file.find(statement_type) == config_from_file.end()) {
+            config_from_file[statement_type] = 0;
+        }
+    }
+    return config_from_file;
+}
+} // namespace duckdb

src/sqlsmith_extension.cpp

@@ -143,6 +143,8 @@ static duckdb::unique_ptr<FunctionData> FuzzyDuckBind(ClientContext &context, Ta
 			result->fuzzer.verbose_output = BooleanValue::Get(kv.second);
 		} else if (kv.first == "enable_verification") {
 			result->fuzzer.enable_verification = BooleanValue::Get(kv.second);
+		} else if (kv.first == "randoms_config_filepath") {
+			result->fuzzer.randoms_config_filepath = StringValue::Get(kv.second);
 		}
 	}
 	return_types.emplace_back(LogicalType::BOOLEAN);
@@ -191,6 +193,7 @@ void SqlsmithExtension::Load(DuckDB &db) {
 	fuzzy_duck_fun.named_parameters["complete_log"] = LogicalType::VARCHAR;
 	fuzzy_duck_fun.named_parameters["verbose_output"] = LogicalType::BOOLEAN;
 	fuzzy_duck_fun.named_parameters["enable_verification"] = LogicalType::BOOLEAN;
+	fuzzy_duck_fun.named_parameters["randoms_config_filepath"] = LogicalType::VARCHAR;
 	ExtensionUtil::RegisterFunction(db_instance, fuzzy_duck_fun);
 
 	TableFunction fuzz_all_functions("fuzz_all_functions", {}, FuzzAllFunctions, FuzzyDuckBind);

src/statement_generator.cpp

@@ -92,24 +92,24 @@ std::shared_ptr<GeneratorContext> StatementGenerator::GetDatabaseState(ClientCon
 	return result;
 }
 
-unique_ptr<SQLStatement> StatementGenerator::GenerateStatement() {
-	if (RandomPercentage(80)) {
+unique_ptr<SQLStatement> StatementGenerator::GenerateStatement(unordered_map<RandomPercentagesEnum, idx_t> config) {
+	if (RandomPercentage(config[RandomPercentagesEnum::SELECT])) {
 		return GenerateStatement(StatementType::SELECT_STATEMENT);
 	}
-	if (RandomPercentage(40)) {
-		if (RandomPercentage(50)) {
+	if (RandomPercentage(config[RandomPercentagesEnum::ATTACH])) {
+		if (RandomPercentage(config[RandomPercentagesEnum::ATTACH_USE])) {
 			// We call this directly so we have a higher chance to fuzz persistent databases
 			return GenerateAttachUse();
 		}
 		return GenerateStatement(StatementType::ATTACH_STATEMENT);
 	}
-	if (RandomPercentage(60)) {
+	if (RandomPercentage(config[RandomPercentagesEnum::DETACH])) {
 		return GenerateStatement(StatementType::DETACH_STATEMENT);
 	}
-	if (RandomPercentage(30)) {
+	if (RandomPercentage(config[RandomPercentagesEnum::SET])) {
 		return GenerateStatement(StatementType::SET_STATEMENT);
 	}
-	if (RandomPercentage(40)) { //20
+	if (RandomPercentage(config[RandomPercentagesEnum::DELETE])) { //20
 		return GenerateStatement(StatementType::DELETE_STATEMENT);
 	}
 	return GenerateStatement(StatementType::CREATE_STATEMENT);

test/sql/call_fuzzyduck.test

@@ -5,7 +5,4 @@
 require sqlsmith
 
 statement ok
-call fuzzyduck(max_queries=2, verbose_output=1, log='sqlsmith.log', complete_log='sqlsmith.complete.log', enable_verification=True);
-
-statement ok
-call fuzzyduck(max_queries=2, verbose_output=1, log='__TEST_DIR__/logs.txt', complete_log='__TEST_DIR__/clog.txt', enable_verification=false);
+call fuzzyduck(max_queries=2, verbose_output=1, log='sqlsmith.log', complete_log='sqlsmith.complete.log', enable_verification=True, randoms_config_filepath='config_nested.json');