diff --git a/backend/db/mysql.c b/backend/db/mysql.c index cf9b10962..05277fc16 100644 --- a/backend/db/mysql.c +++ b/backend/db/mysql.c @@ -589,7 +589,7 @@ j_sql_open(gpointer backend_data) return backend_db; _error: - fprintf(stderr, "%s\n", mysql_error(backend_db)); + g_error("Could not connect: %s\n", mysql_error(backend_db)); mysql_close(backend_db); return NULL; @@ -725,12 +725,112 @@ backend_fini(gpointer backend_data) g_slice_free(JMySQLData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + MYSQL_RES* schemas = NULL; + MYSQL* conn = j_sql_open(backend_data); + MYSQL_ROW schema = NULL; + gboolean res = FALSE; + gboolean hit = FALSE; + + // The metadata table structure: + // "CREATE TABLE IF NOT EXISTS schema_structure (" + // "namespace VARCHAR(255)," + // "name VARCHAR(255)," + // "varname VARCHAR(255)," + // "vartype INTEGER )" + gchar* schema_query = "SELECT DISTINCT namespace, name FROM schema_structure"; + gchar* metadata_clear = "DELETE FROM schema_structure"; + + // collect all table names and issue drop table on all at once + // relevant restriction is the maximum query length + // however, its default is 16MB which should be enough + // https://dev.mysql.com/doc/refman/8.0/en/packet-too-large.html + g_autoptr(GString) schema_delete = g_string_new("DROP TABLE "); + + if (mysql_query(conn, schema_query) != 0) + { + // table might not exist on before first startup (nothing to do in this case) + g_warning("Could not get table information for DB clean. This could be normal on first usage."); + return TRUE; + + } + + if (!(schemas = mysql_use_result(conn))) + { + g_error("Querying schema info for backend_clean failed.\nError: %s", mysql_error(conn)); + goto _error; + } + + while ((schema = mysql_fetch_row(schemas))) + { + unsigned long* lengths = mysql_fetch_lengths(schemas); + g_autoptr(GString) table = g_string_sized_new(lengths[0] + lengths[1] + 5); + hit = TRUE; + + // set up namespace_name as table name + // g_string_append_printf would be more convinient but also much slower + g_string_append_len(table, "`", 1); + g_string_append_len(table, schema[0], lengths[0]); + g_string_append_len(table, "_", 1); + g_string_append_len(table, schema[1], lengths[1]); + g_string_append_len(table, "`", 1); + g_string_append_len(table, ", ", 2); + + // avoid too many relocates of the query string + g_string_append_len(schema_delete, table->str, table->len); + } + + // check whether loop terminated due to no more elements or error + if (*mysql_error(conn)) + { + g_error("mysql_fetch_row for backend_clean failed.\nError: %s", mysql_error(conn)); + goto _error; + } + + if (hit) + { + // remove last colon + g_string_set_size(schema_delete, schema_delete->len - 2); + printf("%s\n", schema_delete->str); + + if (mysql_query(conn, schema_delete->str) != 0) + { + g_error("Could not clean MySQL backend. Failed to execute DROP TABLE.\nError: %s", mysql_error(conn)); + goto _error; + } + + if (mysql_query(conn, metadata_clear) != 0) + { + g_error("Could not clean MySQL backend. Failed to clear metadata.\nError: %s", mysql_error(conn)); + goto _error; + } + } + + res = TRUE; + +_error: + if (schemas) + { + mysql_free_result(schemas); + } + + if (conn) + { + j_sql_close(conn); + } + + return res; +} + static JBackend mysql_backend = { .type = J_BACKEND_TYPE_DB, .component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER, .db = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_schema_create = sql_generic_schema_create, .backend_schema_get = sql_generic_schema_get, .backend_schema_delete = sql_generic_schema_delete, diff --git a/backend/db/null.c b/backend/db/null.c index 2db3e2359..1221bfe57 100644 --- a/backend/db/null.c +++ b/backend/db/null.c @@ -161,12 +161,21 @@ backend_fini(gpointer backend_data) (void)backend_data; } +static gboolean +backend_clean(gpointer backend_data) +{ + (void)backend_data; + + return TRUE; +} + static JBackend null_backend = { .type = J_BACKEND_TYPE_DB, .component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER, .db = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_schema_create = backend_schema_create, .backend_schema_get = backend_schema_get, .backend_schema_delete = backend_schema_delete, diff --git a/backend/db/sqlite.c b/backend/db/sqlite.c index e07369997..303b1f18b 100644 --- a/backend/db/sqlite.c +++ b/backend/db/sqlite.c @@ -466,15 +466,21 @@ backend_init(gchar const* _path, gpointer* backend_data) J_TRACE_FUNCTION(NULL); JSQLiteData* bd; + gboolean ret = TRUE; bd = g_slice_new(JSQLiteData); bd->path = g_strdup(_path); bd->db = NULL; + // Hold an extra reference to the shared in-memory database to make sure it is not freed. if (g_strcmp0(bd->path, ":memory:") == 0) { - // Hold an extra reference to the shared in-memory database to make sure it is not freed. - sqlite3_open_v2("file:julea-db", &(bd->db), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI | SQLITE_OPEN_MEMORY | SQLITE_OPEN_SHAREDCACHE, NULL); + int success = sqlite3_open_v2("file:julea-db", &(bd->db), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI | SQLITE_OPEN_MEMORY | SQLITE_OPEN_SHAREDCACHE, NULL); + if (success != SQLITE_OK) + { + g_error("%s", sqlite3_errmsg(bd->db)); + ret = FALSE; + } } *backend_data = bd; @@ -483,7 +489,7 @@ backend_init(gchar const* _path, gpointer* backend_data) sql_generic_init(&specifics); - return TRUE; + return ret; } static void @@ -504,12 +510,36 @@ backend_fini(gpointer backend_data) g_slice_free(JSQLiteData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + gboolean ret = FALSE; + sqlite3* conn = j_sql_open(backend_data); + + // see https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase + sqlite3_db_config(conn, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + + if (sqlite3_exec(conn, "VACUUM", NULL, NULL, NULL) != SQLITE_OK) + { + goto _error; + } + + ret = TRUE; + +_error: + sqlite3_db_config(conn, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + j_sql_close(conn); + + return ret; +} + static JBackend sqlite_backend = { .type = J_BACKEND_TYPE_DB, .component = J_BACKEND_COMPONENT_SERVER, .db = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_schema_create = sql_generic_schema_create, .backend_schema_get = sql_generic_schema_get, .backend_schema_delete = sql_generic_schema_delete, diff --git a/backend/kv/leveldb.c b/backend/kv/leveldb.c index 5a4ec6fae..870df3457 100644 --- a/backend/kv/leveldb.c +++ b/backend/kv/leveldb.c @@ -36,6 +36,7 @@ typedef struct JLevelDBBatch JLevelDBBatch; struct JLevelDBData { + gchar* path; leveldb_t* db; leveldb_readoptions_t* read_options; @@ -269,25 +270,13 @@ backend_iterate(gpointer backend_data, gpointer backend_iterator, gchar const** return FALSE; } -static gboolean -backend_init(gchar const* path, gpointer* backend_data) +static leveldb_t* +create_db(gchar const* path) { - JLevelDBData* bd; + leveldb_t* res = NULL; leveldb_options_t* options; - g_autofree gchar* dirname = NULL; gint const compressions[] = { leveldb_snappy_compression, leveldb_no_compression }; - g_return_val_if_fail(path != NULL, FALSE); - - dirname = g_path_get_dirname(path); - g_mkdir_with_parents(dirname, 0700); - - bd = g_slice_new(JLevelDBData); - bd->read_options = leveldb_readoptions_create(); - bd->write_options = leveldb_writeoptions_create(); - bd->write_options_sync = leveldb_writeoptions_create(); - leveldb_writeoptions_set_sync(bd->write_options_sync, 1); - options = leveldb_options_create(); leveldb_options_set_create_if_missing(options, 1); @@ -296,9 +285,9 @@ backend_init(gchar const* path, gpointer* backend_data) g_autofree gchar* error = NULL; leveldb_options_set_compression(options, compressions[i]); - bd->db = leveldb_open(options, path, &error); + res = leveldb_open(options, path, &error); - if (bd->db != NULL) + if (res != NULL) { break; } @@ -306,6 +295,29 @@ backend_init(gchar const* path, gpointer* backend_data) leveldb_options_destroy(options); + return res; +} + +static gboolean +backend_init(gchar const* path, gpointer* backend_data) +{ + JLevelDBData* bd; + g_autofree gchar* dirname = NULL; + + g_return_val_if_fail(path != NULL, FALSE); + + dirname = g_path_get_dirname(path); + g_mkdir_with_parents(dirname, 0700); + + bd = g_slice_new(JLevelDBData); + bd->read_options = leveldb_readoptions_create(); + bd->write_options = leveldb_writeoptions_create(); + bd->write_options_sync = leveldb_writeoptions_create(); + leveldb_writeoptions_set_sync(bd->write_options_sync, 1); + + bd->db = create_db(path); + bd->path = g_strdup(path); + *backend_data = bd; return (bd->db != NULL); @@ -325,15 +337,55 @@ backend_fini(gpointer backend_data) leveldb_close(bd->db); } + g_free((void*)bd->path); + g_slice_free(JLevelDBData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + JLevelDBData* bd = backend_data; + char* err = NULL; + leveldb_options_t* options; + gboolean ret = FALSE; + + options = leveldb_options_create(); + + if (bd->db != NULL) + { + leveldb_close(bd->db); + } + + leveldb_destroy_db(options, bd->path, &err); + + if (err) + { + free(err); + goto _error; + } + + bd->db = create_db(bd->path); + + if (!bd->db) + { + goto _error; + } + + ret = TRUE; + +_error: + leveldb_options_destroy(options); + return ret; +} + static JBackend leveldb_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/lmdb.c b/backend/kv/lmdb.c index a23458f61..f3fd44655 100644 --- a/backend/kv/lmdb.c +++ b/backend/kv/lmdb.c @@ -341,12 +341,40 @@ backend_fini(gpointer backend_data) g_slice_free(JLMDBData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + MDB_txn* txn; + JLMDBData* bd = backend_data; + gboolean ret = FALSE; + + if (mdb_txn_begin(bd->env, NULL, 0, &txn) == 0) + { + // 0 empties the DB which is still open afterwards + if (mdb_drop(txn, bd->dbi, 0) != 0) + { + goto _error; + } + + if (mdb_txn_commit(txn) != 0) + { + goto _error; + } + + ret = TRUE; + } + +_error: + return ret; +} + static JBackend lmdb_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/mongodb.c b/backend/kv/mongodb.c index d3c49580b..abec37dde 100644 --- a/backend/kv/mongodb.c +++ b/backend/kv/mongodb.c @@ -476,12 +476,26 @@ backend_fini(gpointer backend_data) mongoc_cleanup(); } +static gboolean +backend_clean(gpointer backend_data) +{ + JMongoDBData* bd = backend_data; + mongoc_database_t* m_database; + gboolean ret; + + m_database = mongoc_client_get_database(bd->connection, bd->database); + // the database will be recreated on next insert + ret = mongoc_database_drop(m_database, NULL); + return ret; +} + static JBackend mongodb_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_CLIENT, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/null.c b/backend/kv/null.c index 006666122..c1dd5978a 100644 --- a/backend/kv/null.c +++ b/backend/kv/null.c @@ -146,12 +146,21 @@ backend_fini(gpointer backend_data) (void)backend_data; } +static gboolean +backend_clean(gpointer backend_data) +{ + (void)backend_data; + + return TRUE; +} + static JBackend null_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/rocksdb.c b/backend/kv/rocksdb.c index cf3aefdb3..13b681563 100644 --- a/backend/kv/rocksdb.c +++ b/backend/kv/rocksdb.c @@ -36,6 +36,7 @@ typedef struct JRocksDBBatch JRocksDBBatch; struct JRocksDBData { + char* path; rocksdb_t* db; rocksdb_readoptions_t* read_options; @@ -270,26 +271,13 @@ backend_iterate(gpointer backend_data, gpointer backend_iterator, gchar const** return FALSE; } -static gboolean -backend_init(gchar const* path, gpointer* backend_data) +static rocksdb_t* +create_db(gchar const* path) { - JRocksDBData* bd; - rocksdb_options_t* options; - g_autofree gchar* dirname = NULL; + rocksdb_t* res = NULL; gint const compressions[] = { rocksdb_lz4_compression, rocksdb_snappy_compression, rocksdb_no_compression }; + rocksdb_options_t* options = rocksdb_options_create(); - g_return_val_if_fail(path != NULL, FALSE); - - dirname = g_path_get_dirname(path); - g_mkdir_with_parents(dirname, 0700); - - bd = g_slice_new(JRocksDBData); - bd->read_options = rocksdb_readoptions_create(); - bd->write_options = rocksdb_writeoptions_create(); - bd->write_options_sync = rocksdb_writeoptions_create(); - rocksdb_writeoptions_set_sync(bd->write_options_sync, 1); - - options = rocksdb_options_create(); rocksdb_options_set_create_if_missing(options, 1); for (guint i = 0; i < G_N_ELEMENTS(compressions); i++) @@ -297,9 +285,9 @@ backend_init(gchar const* path, gpointer* backend_data) g_autofree gchar* error = NULL; rocksdb_options_set_compression(options, compressions[i]); - bd->db = rocksdb_open(options, path, &error); + res = rocksdb_open(options, path, &error); - if (bd->db != NULL) + if (res != NULL) { break; } @@ -307,6 +295,29 @@ backend_init(gchar const* path, gpointer* backend_data) rocksdb_options_destroy(options); + return res; +} + +static gboolean +backend_init(gchar const* path, gpointer* backend_data) +{ + JRocksDBData* bd; + g_autofree gchar* dirname = NULL; + + g_return_val_if_fail(path != NULL, FALSE); + + dirname = g_path_get_dirname(path); + g_mkdir_with_parents(dirname, 0700); + + bd = g_slice_new(JRocksDBData); + bd->read_options = rocksdb_readoptions_create(); + bd->write_options = rocksdb_writeoptions_create(); + bd->write_options_sync = rocksdb_writeoptions_create(); + rocksdb_writeoptions_set_sync(bd->write_options_sync, 1); + + bd->path = g_strdup(path); + bd->db = create_db(path); + *backend_data = bd; return (bd->db != NULL); @@ -321,6 +332,8 @@ backend_fini(gpointer backend_data) rocksdb_writeoptions_destroy(bd->write_options); rocksdb_writeoptions_destroy(bd->write_options_sync); + g_free(bd->path); + if (bd->db != NULL) { rocksdb_close(bd->db); @@ -329,12 +342,50 @@ backend_fini(gpointer backend_data) g_slice_free(JRocksDBData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + JRocksDBData* bd = backend_data; + char* err = NULL; + rocksdb_options_t* options; + gboolean ret = FALSE; + + options = rocksdb_options_create(); + + if (bd->db != NULL) + { + rocksdb_close(bd->db); + } + + rocksdb_destroy_db(options, bd->path, &err); + + if (err) + { + free(err); + goto _error; + } + + bd->db = create_db(bd->path); + + if (!bd->db) + { + goto _error; + } + + ret = TRUE; + +_error: + rocksdb_options_destroy(options); + return ret; +} + static JBackend rocksdb_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/sqlite.c b/backend/kv/sqlite.c index 8daa6cabb..d2ec137e8 100644 --- a/backend/kv/sqlite.c +++ b/backend/kv/sqlite.c @@ -20,6 +20,7 @@ #include #include +#include #include @@ -35,6 +36,7 @@ typedef struct JSQLiteBatch JSQLiteBatch; struct JSQLiteData { + char* path; sqlite3* db; }; @@ -235,6 +237,32 @@ backend_iterate(gpointer backend_data, gpointer backend_iterator, gchar const** return FALSE; } +static gboolean +setup_db(sqlite3* conn) +{ + gboolean ret = FALSE; + const char* table = "CREATE TABLE IF NOT EXISTS julea (namespace TEXT NOT NULL, key TEXT NOT NULL, value BLOB NOT NULL);"; + const char* index = "CREATE UNIQUE INDEX IF NOT EXISTS julea_namespace_key ON julea (namespace, key);"; + + if (sqlite3_exec(conn, table, NULL, NULL, NULL) != SQLITE_OK) + { + g_error("Could not create KV schema.\nError: %s", sqlite3_errmsg(conn)); + goto error; + } + + if (sqlite3_exec(conn, index, NULL, NULL, NULL) != SQLITE_OK) + { + // "here2 no such table: main.julea" + g_error("Could not create KV index.\nError: %s", sqlite3_errmsg(conn)); + goto error; + } + + ret = TRUE; + +error: + return ret; +} + static gboolean backend_init(gchar const* path, gpointer* backend_data) { @@ -247,18 +275,14 @@ backend_init(gchar const* path, gpointer* backend_data) g_mkdir_with_parents(dirname, 0700); bd = g_slice_new(JSQLiteData); + bd->path = g_strdup(path); if (sqlite3_open(path, &(bd->db)) != SQLITE_OK) { goto error; } - if (sqlite3_exec(bd->db, "CREATE TABLE IF NOT EXISTS julea (namespace TEXT NOT NULL, key TEXT NOT NULL, value BLOB NOT NULL);", NULL, NULL, NULL) != SQLITE_OK) - { - goto error; - } - - if (sqlite3_exec(bd->db, "CREATE UNIQUE INDEX IF NOT EXISTS julea_namespace_key ON julea (namespace, key);", NULL, NULL, NULL) != SQLITE_OK) + if (!setup_db(bd->db)) { goto error; } @@ -269,6 +293,7 @@ backend_init(gchar const* path, gpointer* backend_data) error: sqlite3_close(bd->db); + g_free(bd->path); g_slice_free(JSQLiteData, bd); return FALSE; @@ -284,15 +309,44 @@ backend_fini(gpointer backend_data) sqlite3_close(bd->db); } + g_free(bd->path); + g_slice_free(JSQLiteData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + JSQLiteData* bd = backend_data; + gboolean ret = G_TYPE_RESOURCE_ERROR; + + // https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase + sqlite3_db_config(bd->db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + + if (sqlite3_exec(bd->db, "VACUUM", NULL, NULL, NULL) != SQLITE_OK) + { + g_error("Could not VACUUM KV backend.\nError: %s", sqlite3_errmsg(bd->db)); + + ret = FALSE; + } + + sqlite3_db_config(bd->db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + + if (!setup_db(bd->db)) + { + ret = FALSE; + } + + return ret; +} + static JBackend sqlite_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/object/gio.c b/backend/object/gio.c index 8abea789c..7898fb778 100644 --- a/backend/object/gio.c +++ b/backend/object/gio.c @@ -407,12 +407,73 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +static gboolean +delete_directory_tree(const char* directory) +{ + g_autoptr(GFileInfo) info; + g_autoptr(GFile) file = g_file_new_for_path(directory); + g_autoptr(GFileEnumerator) enumerator = g_file_enumerate_children(file, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, NULL); + + while ((info = g_file_enumerator_next_file(enumerator, NULL, NULL)) != NULL) + { + const char* file_name = g_file_info_get_name(info); + GFileType file_type = g_file_info_get_file_type(info); + g_autofree char* child_path = g_build_filename(directory, file_name, NULL); + + if (file_type == G_FILE_TYPE_DIRECTORY) + { + if (!delete_directory_tree(child_path)) + { + return FALSE; + } + } + else + { + g_autoptr(GFile) child = g_file_new_for_path(child_path); + g_debug("Deleting file: %s\n", child_path); + + if (!g_file_delete(child, NULL, NULL)) + { + g_error("Could not delete file: %s", child_path); + return FALSE; + } + } + } + + if (!g_file_delete(file, NULL, NULL)) + { + g_error("Could not delete directory: %s", directory); + return FALSE; + } + + return TRUE; +} + +static gboolean +backend_clean(gpointer backend_data) +{ + JBackendData* bd = backend_data; + GFile* file; + + if (!delete_directory_tree(bd->path)) + { + return FALSE; + } + + file = g_file_new_for_path(bd->path); + g_file_make_directory_with_parents(file, NULL, NULL); + g_object_unref(file); + + return TRUE; +} + static JBackend gio_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_SERVER, .object = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_create = backend_create, .backend_delete = backend_delete, .backend_open = backend_open, diff --git a/backend/object/null.c b/backend/object/null.c index 21bb3e2ee..c516b9796 100644 --- a/backend/object/null.c +++ b/backend/object/null.c @@ -214,12 +214,21 @@ backend_fini(gpointer backend_data) (void)backend_data; } +static gboolean +backend_clean(gpointer backend_data) +{ + (void)backend_data; + + return TRUE; +} + static JBackend null_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER, .object = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_create = backend_create, .backend_delete = backend_delete, .backend_open = backend_open, diff --git a/backend/object/posix.c b/backend/object/posix.c index 6004a636b..5ab5c3829 100644 --- a/backend/object/posix.c +++ b/backend/object/posix.c @@ -18,6 +18,9 @@ #include +// for nftw usage +#define _XOPEN_SOURCE 500 + #include #include #include @@ -26,6 +29,9 @@ #include #include #include +#include +#include +#include #include @@ -524,12 +530,45 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +static int +remove_entry(const char* path, const struct stat* sb, int typeflag, struct FTW* ftwbuf) +{ + (void)sb; + (void)typeflag; + (void)ftwbuf; + + return remove(path); +} + +static gboolean +backend_clean(gpointer backend_data) +{ + JBackendData* bd = backend_data; + + gboolean ret = TRUE; + + if (nftw(bd->path, remove_entry, 30, FTW_DEPTH | FTW_PHYS)) + { + g_error("Could not clean POSIX backend: %s\n", strerror(errno)); + ret = FALSE; + } + + if (g_mkdir_with_parents(bd->path, 0700)) + { + g_error("Could not create POSIX backend directory: %s\n", strerror(errno)); + ret = FALSE; + } + + return ret; +} + static JBackend posix_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_SERVER, .object = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_create = backend_create, .backend_delete = backend_delete, .backend_open = backend_open, diff --git a/backend/object/rados.c b/backend/object/rados.c index dbef4d554..566e2d3d3 100644 --- a/backend/object/rados.c +++ b/backend/object/rados.c @@ -275,12 +275,23 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +/// \todo implement when rados test setup becomes easier +static gboolean +backend_clean(gpointer backend_data) +{ + (void)backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend rados_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_CLIENT, .object = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_create = backend_create, .backend_delete = backend_delete, .backend_open = backend_open, diff --git a/include/core/jbackend.h b/include/core/jbackend.h index 5019a855b..963ac6401 100644 --- a/include/core/jbackend.h +++ b/include/core/jbackend.h @@ -134,6 +134,7 @@ struct JBackend { gboolean (*backend_init)(gchar const*, gpointer*); void (*backend_fini)(gpointer); + gboolean (*backend_clean)(gpointer); gboolean (*backend_create)(gpointer, gchar const*, gchar const*, gpointer*); gboolean (*backend_open)(gpointer, gchar const*, gchar const*, gpointer*); @@ -156,6 +157,7 @@ struct JBackend { gboolean (*backend_init)(gchar const*, gpointer*); void (*backend_fini)(gpointer); + gboolean (*backend_clean)(gpointer); gboolean (*backend_batch_start)(gpointer, gchar const*, JSemantics*, gpointer*); gboolean (*backend_batch_execute)(gpointer, gpointer); @@ -173,6 +175,7 @@ struct JBackend { gboolean (*backend_init)(gchar const*, gpointer*); void (*backend_fini)(gpointer); + gboolean (*backend_clean)(gpointer); gboolean (*backend_batch_start)(gpointer, gchar const*, JSemantics*, gpointer*, GError**); gboolean (*backend_batch_execute)(gpointer, gpointer, GError**); @@ -390,6 +393,7 @@ gboolean j_backend_load_server(gchar const*, gchar const*, JBackendType, GModule gboolean j_backend_object_init(JBackend*, gchar const*); void j_backend_object_fini(JBackend*); +gboolean j_backend_object_clean(JBackend*); gboolean j_backend_object_create(JBackend*, gchar const*, gchar const*, gpointer*); gboolean j_backend_object_open(JBackend*, gchar const*, gchar const*, gpointer*); @@ -409,6 +413,7 @@ gboolean j_backend_object_iterate(JBackend*, gpointer, gchar const**); gboolean j_backend_kv_init(JBackend*, gchar const*); void j_backend_kv_fini(JBackend*); +gboolean j_backend_kv_clean(JBackend*); gboolean j_backend_kv_batch_start(JBackend*, gchar const*, JSemantics*, gpointer*); gboolean j_backend_kv_batch_execute(JBackend*, gpointer); @@ -423,6 +428,7 @@ gboolean j_backend_kv_iterate(JBackend*, gpointer, gchar const**, gconstpointer* gboolean j_backend_db_init(JBackend*, gchar const*); void j_backend_db_fini(JBackend*); +gboolean j_backend_db_clean(JBackend*); gboolean j_backend_db_batch_start(JBackend*, gchar const*, JSemantics*, gpointer*, GError**); gboolean j_backend_db_batch_execute(JBackend*, gpointer, GError**); diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 51a2b3b1c..d7de3d1f2 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -162,6 +162,7 @@ j_backend_load(gchar const* name, JBackendComponent component, JBackendType type { if (tmp_backend->object.backend_init == NULL || tmp_backend->object.backend_fini == NULL + || tmp_backend->object.backend_clean == NULL || tmp_backend->object.backend_create == NULL || tmp_backend->object.backend_delete == NULL || tmp_backend->object.backend_open == NULL @@ -182,6 +183,7 @@ j_backend_load(gchar const* name, JBackendComponent component, JBackendType type { if (tmp_backend->kv.backend_init == NULL || tmp_backend->kv.backend_fini == NULL + || tmp_backend->kv.backend_clean == NULL || tmp_backend->kv.backend_batch_start == NULL || tmp_backend->kv.backend_batch_execute == NULL || tmp_backend->kv.backend_put == NULL @@ -199,6 +201,7 @@ j_backend_load(gchar const* name, JBackendComponent component, JBackendType type { if (tmp_backend->db.backend_init == NULL || tmp_backend->db.backend_fini == NULL + || tmp_backend->db.backend_clean == NULL || tmp_backend->db.backend_batch_start == NULL || tmp_backend->db.backend_batch_execute == NULL || tmp_backend->db.backend_schema_create == NULL @@ -310,6 +313,24 @@ j_backend_object_fini(JBackend* backend) } } +gboolean +j_backend_object_clean(JBackend* backend) +{ + J_TRACE_FUNCTION(NULL); + + gboolean ret = FALSE; + + g_return_val_if_fail(backend != NULL, FALSE); + g_return_val_if_fail(backend->type == J_BACKEND_TYPE_OBJECT, FALSE); + + { + J_TRACE("backend_clean", NULL); + ret = backend->object.backend_clean(backend->data); + } + + return ret; +} + gboolean j_backend_object_create(JBackend* backend, gchar const* namespace, gchar const* path, gpointer* data) { @@ -565,6 +586,24 @@ j_backend_kv_fini(JBackend* backend) } } +gboolean +j_backend_kv_clean(JBackend* backend) +{ + J_TRACE_FUNCTION(NULL); + + gboolean ret; + + g_return_val_if_fail(backend != NULL, FALSE); + g_return_val_if_fail(backend->type == J_BACKEND_TYPE_KV, FALSE); + + { + J_TRACE("backend_clean", NULL); + ret = backend->kv.backend_clean(backend->data); + } + + return ret; +} + gboolean j_backend_kv_batch_start(JBackend* backend, gchar const* namespace, JSemantics* semantics, gpointer* batch) { @@ -763,6 +802,24 @@ j_backend_db_fini(JBackend* backend) } } +gboolean +j_backend_db_clean(JBackend* backend) +{ + J_TRACE_FUNCTION(NULL); + + gboolean ret; + + g_return_val_if_fail(backend != NULL, FALSE); + g_return_val_if_fail(backend->type == J_BACKEND_TYPE_DB, FALSE); + + { + J_TRACE("backend_clean", NULL); + ret = backend->db.backend_clean(backend->data); + } + + return ret; +} + gboolean j_backend_db_batch_start(JBackend* backend, gchar const* namespace, JSemantics* semantics, gpointer* batch, GError** error) { diff --git a/meson.build b/meson.build index b4c686a4f..0509a49a5 100644 --- a/meson.build +++ b/meson.build @@ -723,6 +723,13 @@ executable('julea-statistics', 'tools/statistics.c', install: true, ) + +executable('julea-backend-cleanup', 'tools/backend_cleanup.c', + dependencies: common_deps + [julea_dep], + include_directories: julea_incs, + install: true, +) + if hdf_dep.found() executable('julea-h5migrate', 'tools/h5migrate.c', dependencies: common_deps + [julea_dep] + [hdf_dep], diff --git a/tools/backend_cleanup.c b/tools/backend_cleanup.c new file mode 100644 index 000000000..923615c61 --- /dev/null +++ b/tools/backend_cleanup.c @@ -0,0 +1,179 @@ +/* + * JULEA - Flexible storage framework + * Copyright (C) 2023-2023 Julian Benda + * Copyright (C) 2023 Timm Leon Erxleben + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include +#include + +#include + +JBackend* object_backend = NULL; +JBackend* kv_backend = NULL; +JBackend* db_backend = NULL; + +static gboolean +setup_backend(JConfiguration* configuration, JBackendType type, gchar const* port_str, GModule** module, JBackend** j_backend) +{ + gchar const* backend; + gchar const* component; + g_autofree gchar* path; + gboolean load_module = FALSE; + + backend = j_configuration_get_backend(configuration, type); + component = j_configuration_get_backend_component(configuration, type); + path = j_helper_str_replace(j_configuration_get_backend_path(configuration, type), "{PORT}", port_str); + + if (strcmp(component, "server") == 0) + { + load_module = j_backend_load_server(backend, component, type, module, j_backend); + } + else + { + load_module = j_backend_load_client(backend, component, type, module, j_backend); + } + + if (load_module) + { + gboolean res = TRUE; + + if (j_backend == NULL) + { + g_warning("Failed to load component: %s.", backend); + return FALSE; + } + + switch (type) + { + case J_BACKEND_TYPE_OBJECT: + res = j_backend_object_init(*j_backend, path); + break; + case J_BACKEND_TYPE_KV: + res = j_backend_kv_init(*j_backend, path); + break; + case J_BACKEND_TYPE_DB: + res = j_backend_db_init(*j_backend, path); + break; + default: + g_warning("unknown backend type: (%d), unable to setup backend!", type); + res = FALSE; + } + + if (!res) + { + g_warning("Failed to initalize backend: %s", backend); + return FALSE; + } + + return TRUE; + } + + return FALSE; +} + +int +main(int argc, char** argv) +{ + JConfiguration* configuration = NULL; + GModule* db_module = NULL; + GModule* kv_module = NULL; + GModule* object_module = NULL; + g_autofree gchar* port_str = NULL; + gint res = 1; + (void)argc; + (void)argv; + + setlocale(LC_ALL, "C.UTF-8"); + + configuration = j_configuration(); + + if (configuration == NULL) + { + g_warning("unable to read config"); + goto end; + } + + port_str = g_strdup_printf("%d", j_configuration_get_port(configuration)); + if (!setup_backend(configuration, J_BACKEND_TYPE_OBJECT, port_str, &object_module, &object_backend)) + { + g_warning("failed to initealize object backend"); + goto end; + } + if (!setup_backend(configuration, J_BACKEND_TYPE_KV, port_str, &kv_module, &kv_backend)) + { + g_warning("failed to initealize kv backend"); + goto end; + } + if (!setup_backend(configuration, J_BACKEND_TYPE_DB, port_str, &db_module, &db_backend)) + { + g_warning("failed to initealize db backend"); + goto end; + } + + if (!j_backend_db_clean(db_backend)) + { + g_warning("Could not clean db backend"); + goto end; + } + + if (!j_backend_kv_clean(kv_backend)) + { + g_warning("Could not clean kv backend"); + goto end; + } + + if (!j_backend_object_clean(object_backend)) + { + g_warning("Could not clean object backend"); + goto end; + } + + res = 0; +end: + if (db_backend != NULL) + { + j_backend_db_fini(db_backend); + } + if (kv_backend != NULL) + { + j_backend_kv_fini(kv_backend); + } + if (object_backend != NULL) + { + j_backend_object_fini(object_backend); + } + if (db_module != NULL) + { + g_module_close(db_module); + } + if (kv_module != NULL) + { + g_module_close(kv_module); + } + if (object_module != NULL) + { + g_module_close(object_module); + } + j_configuration_unref(configuration); + + return res; +}