Backend cleanup tool

backend/db/mysql.c

@@ -589,7 +589,7 @@ j_sql_open(gpointer backend_data)
 	return backend_db;
 
 _error:
-	fprintf(stderr, "%s\n", mysql_error(backend_db));
+	g_error("Could not connect: %s\n", mysql_error(backend_db));
 	mysql_close(backend_db);
 
 	return NULL;
@@ -725,12 +725,112 @@ backend_fini(gpointer backend_data)
 	g_slice_free(JMySQLData, bd);
 }
 
+static gboolean
+backend_clean(gpointer backend_data)
+{
+	MYSQL_RES* schemas = NULL;
+	MYSQL* conn = j_sql_open(backend_data);
+	MYSQL_ROW schema = NULL;
+	gboolean res = FALSE;
+	gboolean hit = FALSE;
+
+	// The metadata table structure:
+	// "CREATE TABLE IF NOT EXISTS schema_structure ("
+	// "namespace VARCHAR(255),"
+	// "name VARCHAR(255),"
+	// "varname VARCHAR(255),"
+	// "vartype INTEGER )"
+	gchar* schema_query = "SELECT DISTINCT namespace, name FROM schema_structure";
+	gchar* metadata_clear = "DELETE FROM schema_structure";
+
+	// collect all table names and issue drop table on all at once
+	// relevant restriction is the maximum query length
+	// however, its default is 16MB which should be enough
+	// https://dev.mysql.com/doc/refman/8.0/en/packet-too-large.html
+	g_autoptr(GString) schema_delete = g_string_new("DROP TABLE ");
+
+	if (mysql_query(conn, schema_query) != 0)
+	{
+		// table might not exist on before first startup (nothing to do in this case)
+		g_warning("Could not get table information for DB clean. This could be normal on first usage.");
+		return TRUE;
+
+	}
+
+	if (!(schemas = mysql_use_result(conn)))
+	{
+		g_error("Querying schema info for backend_clean failed.\nError: %s", mysql_error(conn));
+		goto _error;
+	}
+
+	while ((schema = mysql_fetch_row(schemas)))
+	{
+		unsigned long* lengths = mysql_fetch_lengths(schemas);
+		g_autoptr(GString) table = g_string_sized_new(lengths[0] + lengths[1] + 5);
+		hit = TRUE;
+
+		// set up namespace_name as table name
+		// g_string_append_printf would be more convinient but also much slower
+		g_string_append_len(table, "`", 1);
+		g_string_append_len(table, schema[0], lengths[0]);
+		g_string_append_len(table, "_", 1);
+		g_string_append_len(table, schema[1], lengths[1]);
+		g_string_append_len(table, "`", 1);
+		g_string_append_len(table, ", ", 2);
+
+		// avoid too many relocates of the query string
+		g_string_append_len(schema_delete, table->str, table->len);
+	}
+
+	// check whether loop terminated due to no more elements or error
+	if (*mysql_error(conn))
+	{
+		g_error("mysql_fetch_row for backend_clean failed.\nError: %s", mysql_error(conn));
+		goto _error;
+	}
+
+	if (hit)
+	{
+		// remove last colon
+		g_string_set_size(schema_delete, schema_delete->len - 2);
+		printf("%s\n", schema_delete->str);
+
+		if (mysql_query(conn, schema_delete->str) != 0)
+		{
+			g_error("Could not clean MySQL backend. Failed to execute DROP TABLE.\nError: %s", mysql_error(conn));
+			goto _error;
+		}
+
+		if (mysql_query(conn, metadata_clear) != 0)
+		{
+			g_error("Could not clean MySQL backend. Failed to clear metadata.\nError: %s", mysql_error(conn));
+			goto _error;
+		}
+	}
+
+	res = TRUE;
+
+_error:
+	if (schemas)
+	{
+		mysql_free_result(schemas);
+	}
+
+	if (conn)
+	{
+		j_sql_close(conn);
+	}
+
+	return res;
+}
+
 static JBackend mysql_backend = {
 	.type = J_BACKEND_TYPE_DB,
 	.component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER,
 	.db = {
 		.backend_init = backend_init,
 		.backend_fini = backend_fini,
+		.backend_clean = backend_clean,
 		.backend_schema_create = sql_generic_schema_create,
 		.backend_schema_get = sql_generic_schema_get,
 		.backend_schema_delete = sql_generic_schema_delete,

backend/db/null.c

@@ -161,12 +161,21 @@ backend_fini(gpointer backend_data)
 	(void)backend_data;
 }
 
+static gboolean
+backend_clean(gpointer backend_data)
+{
+	(void)backend_data;
+
+	return TRUE;
+}
+
 static JBackend null_backend = {
 	.type = J_BACKEND_TYPE_DB,
 	.component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER,
 	.db = {
 		.backend_init = backend_init,
 		.backend_fini = backend_fini,
+		.backend_clean = backend_clean,
 		.backend_schema_create = backend_schema_create,
 		.backend_schema_get = backend_schema_get,
 		.backend_schema_delete = backend_schema_delete,

backend/db/sqlite.c

@@ -466,15 +466,21 @@ backend_init(gchar const* _path, gpointer* backend_data)
 	J_TRACE_FUNCTION(NULL);
 
 	JSQLiteData* bd;
+	gboolean ret = TRUE;
 
 	bd = g_slice_new(JSQLiteData);
 	bd->path = g_strdup(_path);
 	bd->db = NULL;
 
+	// Hold an extra reference to the shared in-memory database to make sure it is not freed.
 	if (g_strcmp0(bd->path, ":memory:") == 0)
 	{
-		// Hold an extra reference to the shared in-memory database to make sure it is not freed.
-		sqlite3_open_v2("file:julea-db", &(bd->db), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI | SQLITE_OPEN_MEMORY | SQLITE_OPEN_SHAREDCACHE, NULL);
+		int success = sqlite3_open_v2("file:julea-db", &(bd->db), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI | SQLITE_OPEN_MEMORY | SQLITE_OPEN_SHAREDCACHE, NULL);
+		if (success != SQLITE_OK)
+		{
+			g_error("%s", sqlite3_errmsg(bd->db));
+			ret = FALSE;
+		}
 	}
 
 	*backend_data = bd;
@@ -483,7 +489,7 @@ backend_init(gchar const* _path, gpointer* backend_data)
 
 	sql_generic_init(&specifics);
 
-	return TRUE;
+	return ret;
 }
 
 static void
@@ -504,12 +510,36 @@ backend_fini(gpointer backend_data)
 	g_slice_free(JSQLiteData, bd);
 }
 
+static gboolean
+backend_clean(gpointer backend_data)
+{
+	gboolean ret = FALSE;
+	sqlite3* conn = j_sql_open(backend_data);
+
+	// see https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase
+	sqlite3_db_config(conn, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0);
+
+	if (sqlite3_exec(conn, "VACUUM", NULL, NULL, NULL) != SQLITE_OK)
+	{
+		goto _error;
+	}
+
+	ret = TRUE;
+
+_error:
+	sqlite3_db_config(conn, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0);
+	j_sql_close(conn);
+
+	return ret;
+}
+
 static JBackend sqlite_backend = {
 	.type = J_BACKEND_TYPE_DB,
 	.component = J_BACKEND_COMPONENT_SERVER,
 	.db = {
 		.backend_init = backend_init,
 		.backend_fini = backend_fini,
+		.backend_clean = backend_clean,
 		.backend_schema_create = sql_generic_schema_create,
 		.backend_schema_get = sql_generic_schema_get,
 		.backend_schema_delete = sql_generic_schema_delete,

backend/kv/leveldb.c

@@ -36,6 +36,7 @@ typedef struct JLevelDBBatch JLevelDBBatch;
 
 struct JLevelDBData
 {
+	gchar* path;
 	leveldb_t* db;
 
 	leveldb_readoptions_t* read_options;
@@ -269,25 +270,13 @@ backend_iterate(gpointer backend_data, gpointer backend_iterator, gchar const**
 	return FALSE;
 }
 
-static gboolean
-backend_init(gchar const* path, gpointer* backend_data)
+static leveldb_t*
+create_db(gchar const* path)
 {
-	JLevelDBData* bd;
+	leveldb_t* res = NULL;
 	leveldb_options_t* options;
-	g_autofree gchar* dirname = NULL;
 	gint const compressions[] = { leveldb_snappy_compression, leveldb_no_compression };
 
-	g_return_val_if_fail(path != NULL, FALSE);
-
-	dirname = g_path_get_dirname(path);
-	g_mkdir_with_parents(dirname, 0700);
-
-	bd = g_slice_new(JLevelDBData);
-	bd->read_options = leveldb_readoptions_create();
-	bd->write_options = leveldb_writeoptions_create();
-	bd->write_options_sync = leveldb_writeoptions_create();
-	leveldb_writeoptions_set_sync(bd->write_options_sync, 1);
-
 	options = leveldb_options_create();
 	leveldb_options_set_create_if_missing(options, 1);
 
@@ -296,16 +285,39 @@ backend_init(gchar const* path, gpointer* backend_data)
 		g_autofree gchar* error = NULL;
 
 		leveldb_options_set_compression(options, compressions[i]);
-		bd->db = leveldb_open(options, path, &error);
+		res = leveldb_open(options, path, &error);
 
-		if (bd->db != NULL)
+		if (res != NULL)
 		{
 			break;
 		}
 	}
 
 	leveldb_options_destroy(options);
 
+	return res;
+}
+
+static gboolean
+backend_init(gchar const* path, gpointer* backend_data)
+{
+	JLevelDBData* bd;
+	g_autofree gchar* dirname = NULL;
+
+	g_return_val_if_fail(path != NULL, FALSE);
+
+	dirname = g_path_get_dirname(path);
+	g_mkdir_with_parents(dirname, 0700);
+
+	bd = g_slice_new(JLevelDBData);
+	bd->read_options = leveldb_readoptions_create();
+	bd->write_options = leveldb_writeoptions_create();
+	bd->write_options_sync = leveldb_writeoptions_create();
+	leveldb_writeoptions_set_sync(bd->write_options_sync, 1);
+
+	bd->db = create_db(path);
+	bd->path = g_strdup(path);
+
 	*backend_data = bd;
 
 	return (bd->db != NULL);
@@ -325,15 +337,55 @@ backend_fini(gpointer backend_data)
 		leveldb_close(bd->db);
 	}
 
+	g_free((void*)bd->path);
+
 	g_slice_free(JLevelDBData, bd);
 }
 
+static gboolean
+backend_clean(gpointer backend_data)
+{
+	JLevelDBData* bd = backend_data;
+	char* err = NULL;
+	leveldb_options_t* options;
+	gboolean ret = FALSE;
+
+	options = leveldb_options_create();
+
+	if (bd->db != NULL)
+	{
+		leveldb_close(bd->db);
+	}
+
+	leveldb_destroy_db(options, bd->path, &err);
+
+	if (err)
+	{
+		free(err);
+		goto _error;
+	}
+
+	bd->db = create_db(bd->path);
+
+	if (!bd->db)
+	{
+		goto _error;
+	}
+
+	ret = TRUE;
+
+_error:
+	leveldb_options_destroy(options);
+	return ret;
+}
+
 static JBackend leveldb_backend = {
 	.type = J_BACKEND_TYPE_KV,
 	.component = J_BACKEND_COMPONENT_SERVER,
 	.kv = {
 		.backend_init = backend_init,
 		.backend_fini = backend_fini,
+		.backend_clean = backend_clean,
 		.backend_batch_start = backend_batch_start,
 		.backend_batch_execute = backend_batch_execute,
 		.backend_put = backend_put,

backend/kv/lmdb.c

@@ -341,12 +341,40 @@ backend_fini(gpointer backend_data)
 	g_slice_free(JLMDBData, bd);
 }
 
+static gboolean
+backend_clean(gpointer backend_data)
+{
+	MDB_txn* txn;
+	JLMDBData* bd = backend_data;
+	gboolean ret = FALSE;
+
+	if (mdb_txn_begin(bd->env, NULL, 0, &txn) == 0)
+	{
+		// 0 empties the DB which is still open afterwards
+		if (mdb_drop(txn, bd->dbi, 0) != 0)
+		{
+			goto _error;
+		}
+
+		if (mdb_txn_commit(txn) != 0)
+		{
+			goto _error;
+		}
+
+		ret = TRUE;
+	}
+
+_error:
+	return ret;
+}
+
 static JBackend lmdb_backend = {
 	.type = J_BACKEND_TYPE_KV,
 	.component = J_BACKEND_COMPONENT_SERVER,
 	.kv = {
 		.backend_init = backend_init,
 		.backend_fini = backend_fini,
+		.backend_clean = backend_clean,
 		.backend_batch_start = backend_batch_start,
 		.backend_batch_execute = backend_batch_execute,
 		.backend_put = backend_put,