From d10b98dee80399e68d0aabebca5f136d95d0884b Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Tue, 21 Mar 2023 14:29:57 +0100 Subject: [PATCH 01/12] add data clean option --- server/server.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/server/server.c b/server/server.c index 86c84c3bc..0d6052dbd 100644 --- a/server/server.c +++ b/server/server.c @@ -203,6 +203,7 @@ main(int argc, char** argv) J_TRACE_FUNCTION(NULL); gboolean opt_daemon = FALSE; + gboolean opt_clean = FALSE; g_autofree gchar* opt_host = NULL; gint opt_port = 0; @@ -230,6 +231,7 @@ main(int argc, char** argv) { "daemon", 0, 0, G_OPTION_ARG_NONE, &opt_daemon, "Run as daemon", NULL }, { "host", 0, 0, G_OPTION_ARG_STRING, &opt_host, "Override host name", "hostname" }, { "port", 0, 0, G_OPTION_ARG_INT, &opt_port, "Port to use", "0" }, + { "clean", 0, 0, G_OPTION_ARG_NONE, &opt_clean, "Clean all data on configured backends.", NULL }, { NULL, 0, 0, 0, NULL, NULL, NULL } }; @@ -338,6 +340,11 @@ main(int argc, char** argv) } g_debug("Initialized object backend %s.", object_backend); + + if(opt_clean) + { + printf("Would clean\n"); + } } if (jd_is_server_for_backend(opt_host, opt_port, J_BACKEND_TYPE_KV) @@ -350,6 +357,11 @@ main(int argc, char** argv) } g_debug("Initialized kv backend %s.", kv_backend); + + if(opt_clean) + { + printf("Would clean\n"); + } } if (jd_is_server_for_backend(opt_host, opt_port, J_BACKEND_TYPE_DB) @@ -362,6 +374,11 @@ main(int argc, char** argv) } g_debug("Initialized db backend %s.", db_backend); + + if(opt_clean) + { + printf("Would clean\n"); + } } jd_statistics = j_statistics_new(FALSE); From 9af49d9a95c97c7d310ff4baa929f65ac50cfbe5 Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Tue, 21 Mar 2023 15:02:01 +0100 Subject: [PATCH 02/12] add core lib functionality for backend clean --- include/core/jbackend.h | 6 +++++ lib/core/jbackend.c | 58 +++++++++++++++++++++++++++++++++++++++++ server/server.c | 18 ++++++++++--- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/include/core/jbackend.h b/include/core/jbackend.h index 5019a855b..963ac6401 100644 --- a/include/core/jbackend.h +++ b/include/core/jbackend.h @@ -134,6 +134,7 @@ struct JBackend { gboolean (*backend_init)(gchar const*, gpointer*); void (*backend_fini)(gpointer); + gboolean (*backend_clean)(gpointer); gboolean (*backend_create)(gpointer, gchar const*, gchar const*, gpointer*); gboolean (*backend_open)(gpointer, gchar const*, gchar const*, gpointer*); @@ -156,6 +157,7 @@ struct JBackend { gboolean (*backend_init)(gchar const*, gpointer*); void (*backend_fini)(gpointer); + gboolean (*backend_clean)(gpointer); gboolean (*backend_batch_start)(gpointer, gchar const*, JSemantics*, gpointer*); gboolean (*backend_batch_execute)(gpointer, gpointer); @@ -173,6 +175,7 @@ struct JBackend { gboolean (*backend_init)(gchar const*, gpointer*); void (*backend_fini)(gpointer); + gboolean (*backend_clean)(gpointer); gboolean (*backend_batch_start)(gpointer, gchar const*, JSemantics*, gpointer*, GError**); gboolean (*backend_batch_execute)(gpointer, gpointer, GError**); @@ -390,6 +393,7 @@ gboolean j_backend_load_server(gchar const*, gchar const*, JBackendType, GModule gboolean j_backend_object_init(JBackend*, gchar const*); void j_backend_object_fini(JBackend*); +gboolean j_backend_object_clean(JBackend*); gboolean j_backend_object_create(JBackend*, gchar const*, gchar const*, gpointer*); gboolean j_backend_object_open(JBackend*, gchar const*, gchar const*, gpointer*); @@ -409,6 +413,7 @@ gboolean j_backend_object_iterate(JBackend*, gpointer, gchar const**); gboolean j_backend_kv_init(JBackend*, gchar const*); void j_backend_kv_fini(JBackend*); +gboolean j_backend_kv_clean(JBackend*); gboolean j_backend_kv_batch_start(JBackend*, gchar const*, JSemantics*, gpointer*); gboolean j_backend_kv_batch_execute(JBackend*, gpointer); @@ -423,6 +428,7 @@ gboolean j_backend_kv_iterate(JBackend*, gpointer, gchar const**, gconstpointer* gboolean j_backend_db_init(JBackend*, gchar const*); void j_backend_db_fini(JBackend*); +gboolean j_backend_db_clean(JBackend*); gboolean j_backend_db_batch_start(JBackend*, gchar const*, JSemantics*, gpointer*, GError**); gboolean j_backend_db_batch_execute(JBackend*, gpointer, GError**); diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 51a2b3b1c..2d833260c 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -162,6 +162,7 @@ j_backend_load(gchar const* name, JBackendComponent component, JBackendType type { if (tmp_backend->object.backend_init == NULL || tmp_backend->object.backend_fini == NULL + || tmp_backend->object.backend_clean == NULL || tmp_backend->object.backend_create == NULL || tmp_backend->object.backend_delete == NULL || tmp_backend->object.backend_open == NULL @@ -182,6 +183,7 @@ j_backend_load(gchar const* name, JBackendComponent component, JBackendType type { if (tmp_backend->kv.backend_init == NULL || tmp_backend->kv.backend_fini == NULL + || tmp_backend->kv.backend_clean == NULL || tmp_backend->kv.backend_batch_start == NULL || tmp_backend->kv.backend_batch_execute == NULL || tmp_backend->kv.backend_put == NULL @@ -199,6 +201,7 @@ j_backend_load(gchar const* name, JBackendComponent component, JBackendType type { if (tmp_backend->db.backend_init == NULL || tmp_backend->db.backend_fini == NULL + || tmp_backend->db.backend_clean == NULL || tmp_backend->db.backend_batch_start == NULL || tmp_backend->db.backend_batch_execute == NULL || tmp_backend->db.backend_schema_create == NULL @@ -310,6 +313,25 @@ j_backend_object_fini(JBackend* backend) } } +gboolean +j_backend_object_clean(JBackend* backend) +{ + J_TRACE_FUNCTION(NULL); + + gboolean ret; + + g_return_if_fail(backend != NULL); + g_return_if_fail(backend->type == J_BACKEND_TYPE_OBJECT); + + { + J_TRACE("backend_clean", NULL); + ret = backend->object.backend_clean(backend->data); + } + + return ret; +} + + gboolean j_backend_object_create(JBackend* backend, gchar const* namespace, gchar const* path, gpointer* data) { @@ -565,6 +587,24 @@ j_backend_kv_fini(JBackend* backend) } } +gboolean +j_backend_kv_clean(JBackend* backend) +{ + J_TRACE_FUNCTION(NULL); + + gboolean ret; + + g_return_if_fail(backend != NULL); + g_return_if_fail(backend->type == J_BACKEND_TYPE_KV); + + { + J_TRACE("backend_clean", NULL); + ret = backend->kv.backend_clean(backend->data); + } + + return ret; +} + gboolean j_backend_kv_batch_start(JBackend* backend, gchar const* namespace, JSemantics* semantics, gpointer* batch) { @@ -763,6 +803,24 @@ j_backend_db_fini(JBackend* backend) } } +gboolean +j_backend_db_clean(JBackend* backend) +{ + J_TRACE_FUNCTION(NULL); + + gboolean ret; + + g_return_if_fail(backend != NULL); + g_return_if_fail(backend->type == J_BACKEND_TYPE_DB); + + { + J_TRACE("backend_clean", NULL); + ret = backend->db.backend_clean(backend->data); + } + + return ret; +} + gboolean j_backend_db_batch_start(JBackend* backend, gchar const* namespace, JSemantics* semantics, gpointer* batch, GError** error) { diff --git a/server/server.c b/server/server.c index 0d6052dbd..f2b19f0af 100644 --- a/server/server.c +++ b/server/server.c @@ -343,7 +343,11 @@ main(int argc, char** argv) if(opt_clean) { - printf("Would clean\n"); + if (!j_backend_object_clean(jd_object_backend)) + { + g_warning("Could not clean object backend %s.", object_backend); + return 1; + } } } @@ -360,7 +364,11 @@ main(int argc, char** argv) if(opt_clean) { - printf("Would clean\n"); + if (!j_backend_kv_clean(jd_kv_backend)) + { + g_warning("Could not clean kv backend %s.", kv_backend); + return 1; + } } } @@ -377,7 +385,11 @@ main(int argc, char** argv) if(opt_clean) { - printf("Would clean\n"); + if (!j_backend_db_clean(jd_db_backend)) + { + g_warning("Could not clean db backend %s.", db_backend); + return 1; + } } } From e4d62f7529be98ba451f24d137e8ff3ba476a852 Mon Sep 17 00:00:00 2001 From: Timm Erxleben Date: Tue, 21 Mar 2023 16:29:43 +0100 Subject: [PATCH 03/12] add clean function stub for all backends --- backend/db/mysql.c | 9 +++++++++ backend/db/null.c | 9 +++++++++ backend/db/sqlite.c | 9 +++++++++ backend/kv/leveldb.c | 10 ++++++++++ backend/kv/lmdb.c | 10 ++++++++++ backend/kv/mongodb.c | 10 ++++++++++ backend/kv/null.c | 9 +++++++++ backend/kv/rocksdb.c | 10 ++++++++++ backend/kv/sqlite.c | 10 ++++++++++ backend/object/gio.c | 10 ++++++++++ backend/object/null.c | 9 +++++++++ backend/object/posix.c | 14 ++++++++++++++ backend/object/rados.c | 10 ++++++++++ 13 files changed, 129 insertions(+) diff --git a/backend/db/mysql.c b/backend/db/mysql.c index cf9b10962..aa3f80827 100644 --- a/backend/db/mysql.c +++ b/backend/db/mysql.c @@ -725,6 +725,15 @@ backend_fini(gpointer backend_data) g_slice_free(JMySQLData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend mysql_backend = { .type = J_BACKEND_TYPE_DB, .component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER, diff --git a/backend/db/null.c b/backend/db/null.c index 2db3e2359..981316b46 100644 --- a/backend/db/null.c +++ b/backend/db/null.c @@ -161,12 +161,21 @@ backend_fini(gpointer backend_data) (void)backend_data; } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + + return TRUE; +} + static JBackend null_backend = { .type = J_BACKEND_TYPE_DB, .component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER, .db = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_schema_create = backend_schema_create, .backend_schema_get = backend_schema_get, .backend_schema_delete = backend_schema_delete, diff --git a/backend/db/sqlite.c b/backend/db/sqlite.c index e07369997..ca0bcc6e6 100644 --- a/backend/db/sqlite.c +++ b/backend/db/sqlite.c @@ -504,6 +504,15 @@ backend_fini(gpointer backend_data) g_slice_free(JSQLiteData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend sqlite_backend = { .type = J_BACKEND_TYPE_DB, .component = J_BACKEND_COMPONENT_SERVER, diff --git a/backend/kv/leveldb.c b/backend/kv/leveldb.c index 5a4ec6fae..cc6d38a14 100644 --- a/backend/kv/leveldb.c +++ b/backend/kv/leveldb.c @@ -328,12 +328,22 @@ backend_fini(gpointer backend_data) g_slice_free(JLevelDBData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend leveldb_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/lmdb.c b/backend/kv/lmdb.c index a23458f61..43e2ad5a6 100644 --- a/backend/kv/lmdb.c +++ b/backend/kv/lmdb.c @@ -341,12 +341,22 @@ backend_fini(gpointer backend_data) g_slice_free(JLMDBData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend lmdb_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/mongodb.c b/backend/kv/mongodb.c index d3c49580b..efdc213d7 100644 --- a/backend/kv/mongodb.c +++ b/backend/kv/mongodb.c @@ -476,12 +476,22 @@ backend_fini(gpointer backend_data) mongoc_cleanup(); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend mongodb_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_CLIENT, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/null.c b/backend/kv/null.c index 006666122..8311bb55a 100644 --- a/backend/kv/null.c +++ b/backend/kv/null.c @@ -146,12 +146,21 @@ backend_fini(gpointer backend_data) (void)backend_data; } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + + return TRUE; +} + static JBackend null_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/rocksdb.c b/backend/kv/rocksdb.c index cf3aefdb3..7d519077f 100644 --- a/backend/kv/rocksdb.c +++ b/backend/kv/rocksdb.c @@ -329,12 +329,22 @@ backend_fini(gpointer backend_data) g_slice_free(JRocksDBData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend rocksdb_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/kv/sqlite.c b/backend/kv/sqlite.c index 8daa6cabb..2a7547831 100644 --- a/backend/kv/sqlite.c +++ b/backend/kv/sqlite.c @@ -287,12 +287,22 @@ backend_fini(gpointer backend_data) g_slice_free(JSQLiteData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend sqlite_backend = { .type = J_BACKEND_TYPE_KV, .component = J_BACKEND_COMPONENT_SERVER, .kv = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_batch_start = backend_batch_start, .backend_batch_execute = backend_batch_execute, .backend_put = backend_put, diff --git a/backend/object/gio.c b/backend/object/gio.c index 8abea789c..aed128eea 100644 --- a/backend/object/gio.c +++ b/backend/object/gio.c @@ -407,12 +407,22 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend gio_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_SERVER, .object = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_create = backend_create, .backend_delete = backend_delete, .backend_open = backend_open, diff --git a/backend/object/null.c b/backend/object/null.c index 21bb3e2ee..1def77428 100644 --- a/backend/object/null.c +++ b/backend/object/null.c @@ -214,12 +214,21 @@ backend_fini(gpointer backend_data) (void)backend_data; } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + + return TRUE; +} + static JBackend null_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_CLIENT | J_BACKEND_COMPONENT_SERVER, .object = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_create = backend_create, .backend_delete = backend_delete, .backend_open = backend_open, diff --git a/backend/object/posix.c b/backend/object/posix.c index 6004a636b..799f79f95 100644 --- a/backend/object/posix.c +++ b/backend/object/posix.c @@ -18,6 +18,9 @@ #include +// for nftw usage +#define _XOPEN_SOURCE 500 + #include #include #include @@ -26,6 +29,7 @@ #include #include #include +#include #include @@ -524,12 +528,22 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend posix_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_SERVER, .object = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_create = backend_create, .backend_delete = backend_delete, .backend_open = backend_open, diff --git a/backend/object/rados.c b/backend/object/rados.c index dbef4d554..ad8c73e3a 100644 --- a/backend/object/rados.c +++ b/backend/object/rados.c @@ -275,12 +275,22 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +static gboolean +backend_clean(gpointer backend_data) +{ + (void) backend_data; + g_warning("Backend clean is currently not supported for this backend."); + + return TRUE; +} + static JBackend rados_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_CLIENT, .object = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_create = backend_create, .backend_delete = backend_delete, .backend_open = backend_open, From ebbd937b07737c6a73862fdcedc87a2c54c49f37 Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Mon, 15 May 2023 12:07:18 +0200 Subject: [PATCH 04/12] prototype clean function for all relevant backends --- backend/db/mysql.c | 75 +++++++++++++++++++++++++++++++++++-- backend/db/sqlite.c | 32 +++++++++++++--- backend/kv/leveldb.c | 83 +++++++++++++++++++++++++++++++---------- backend/kv/lmdb.c | 24 ++++++++++-- backend/kv/mongodb.c | 10 +++-- backend/kv/rocksdb.c | 85 +++++++++++++++++++++++++++++++----------- backend/kv/sqlite.c | 19 ++++++++-- backend/object/gio.c | 44 +++++++++++++++++++++- backend/object/posix.c | 32 ++++++++++++++-- lib/core/jbackend.c | 14 +++---- 10 files changed, 346 insertions(+), 72 deletions(-) diff --git a/backend/db/mysql.c b/backend/db/mysql.c index aa3f80827..12824171d 100644 --- a/backend/db/mysql.c +++ b/backend/db/mysql.c @@ -728,10 +728,79 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + MYSQL_RES * schemas = NULL; + MYSQL* conn = j_sql_open(backend_data); + MYSQL_ROW schema = NULL; + gboolean res = FALSE; + + // The metadata table structure: + // "CREATE TABLE IF NOT EXISTS schema_structure (" + // "namespace VARCHAR(255)," + // "name VARCHAR(255)," + // "varname VARCHAR(255)," + // "vartype INTEGER )" + gchar* schema_query = "SELECT namespace, name FROM schema_structure"; + + // collect all table names and issue drop table on all at once + // relevant restriction is the maximum query length + // however, its default is 16MB which should be enough + // https://dev.mysql.com/doc/refman/8.0/en/packet-too-large.html + GString* schema_delete = g_string_new("DROP TABLE "); + + if (!mysql_query(conn, schema_query)) + { + goto _error; + } - return TRUE; + if (!(schemas = mysql_use_result(conn))) + { + g_error("Querying schema info for backend_clean failed./nError: %s", mysql_error(conn)); + goto _error; + } + + while ((schema = mysql_fetch_row(schemas))) + { + unsigned long *lengths; + lengths = mysql_fetch_lengths(schemas); + + // pre-allocate new string size + g_string_set_size (schema_delete, schema_delete->len + lengths[0] + lengths[1] + 5); + + // set up namespace_name as table name + // g_string_append_printf would be more convinient but also much slower + g_string_append_len(schema_delete, "\"", 1); + g_string_append_len(schema_delete, schema[0], lengths[0]); + g_string_append_len(schema_delete, "_" , 1); + g_string_append_len(schema_delete, schema[1], lengths[1]); + g_string_append_len(schema_delete, "\"", 1); + g_string_append_len(schema_delete, ", ", 2); + } + + // check whether loop terminated due to no more elements or error + if(*mysql_error(conn)) + { + g_error("mysql_fetch_row for backend_clean failed./nError: %s", mysql_error(conn)); + goto _error; + } + + // remove last colon + g_string_set_size(schema_delete, schema_delete->len-2); + + if (!mysql_query(conn, schema_delete->str)) + { + g_error("Could not clean MySQL backend. Failed to execute DROP TABLE"); + goto _error; + } + + res = TRUE; + +_error: + if(schemas) + { + mysql_free_result(schemas); + } + + return res; } static JBackend mysql_backend = { diff --git a/backend/db/sqlite.c b/backend/db/sqlite.c index ca0bcc6e6..9284229dd 100644 --- a/backend/db/sqlite.c +++ b/backend/db/sqlite.c @@ -466,15 +466,21 @@ backend_init(gchar const* _path, gpointer* backend_data) J_TRACE_FUNCTION(NULL); JSQLiteData* bd; + gboolean ret = TRUE; bd = g_slice_new(JSQLiteData); bd->path = g_strdup(_path); bd->db = NULL; + // Hold an extra reference to the shared in-memory database to make sure it is not freed. if (g_strcmp0(bd->path, ":memory:") == 0) { - // Hold an extra reference to the shared in-memory database to make sure it is not freed. - sqlite3_open_v2("file:julea-db", &(bd->db), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI | SQLITE_OPEN_MEMORY | SQLITE_OPEN_SHAREDCACHE, NULL); + int success = sqlite3_open_v2("file:julea-db", &(bd->db), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI | SQLITE_OPEN_MEMORY | SQLITE_OPEN_SHAREDCACHE, NULL); + if ( success != SQLITE_OK) + { + g_error("%s", sqlite3_errmsg(bd->db)); + ret = FALSE; + } } *backend_data = bd; @@ -483,7 +489,7 @@ backend_init(gchar const* _path, gpointer* backend_data) sql_generic_init(&specifics); - return TRUE; + return ret; } static void @@ -507,10 +513,24 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + gboolean ret = FALSE; + sqlite3* conn = j_sql_open(backend_data); - return TRUE; + // see https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase + sqlite3_db_config(conn, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + + if (sqlite3_exec(conn, "VACUUM", NULL, NULL, NULL) != SQLITE_OK) + { + goto _error; + } + + ret = TRUE; + +_error: + sqlite3_db_config(conn, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + j_sql_close(conn); + + return ret; } static JBackend sqlite_backend = { diff --git a/backend/kv/leveldb.c b/backend/kv/leveldb.c index cc6d38a14..a74dfaffc 100644 --- a/backend/kv/leveldb.c +++ b/backend/kv/leveldb.c @@ -36,6 +36,7 @@ typedef struct JLevelDBBatch JLevelDBBatch; struct JLevelDBData { + gchar* path; leveldb_t* db; leveldb_readoptions_t* read_options; @@ -269,25 +270,13 @@ backend_iterate(gpointer backend_data, gpointer backend_iterator, gchar const** return FALSE; } -static gboolean -backend_init(gchar const* path, gpointer* backend_data) +static leveldb_t* +create_db(gchar const* path) { - JLevelDBData* bd; + leveldb_t* res = NULL; leveldb_options_t* options; - g_autofree gchar* dirname = NULL; gint const compressions[] = { leveldb_snappy_compression, leveldb_no_compression }; - g_return_val_if_fail(path != NULL, FALSE); - - dirname = g_path_get_dirname(path); - g_mkdir_with_parents(dirname, 0700); - - bd = g_slice_new(JLevelDBData); - bd->read_options = leveldb_readoptions_create(); - bd->write_options = leveldb_writeoptions_create(); - bd->write_options_sync = leveldb_writeoptions_create(); - leveldb_writeoptions_set_sync(bd->write_options_sync, 1); - options = leveldb_options_create(); leveldb_options_set_create_if_missing(options, 1); @@ -296,9 +285,9 @@ backend_init(gchar const* path, gpointer* backend_data) g_autofree gchar* error = NULL; leveldb_options_set_compression(options, compressions[i]); - bd->db = leveldb_open(options, path, &error); + res = leveldb_open(options, path, &error); - if (bd->db != NULL) + if (res != NULL) { break; } @@ -306,6 +295,30 @@ backend_init(gchar const* path, gpointer* backend_data) leveldb_options_destroy(options); + return res; +} + +static gboolean +backend_init(gchar const* path, gpointer* backend_data) +{ + JLevelDBData* bd; + g_autofree gchar* dirname = NULL; + + + g_return_val_if_fail(path != NULL, FALSE); + + dirname = g_path_get_dirname(path); + g_mkdir_with_parents(dirname, 0700); + + bd = g_slice_new(JLevelDBData); + bd->read_options = leveldb_readoptions_create(); + bd->write_options = leveldb_writeoptions_create(); + bd->write_options_sync = leveldb_writeoptions_create(); + leveldb_writeoptions_set_sync(bd->write_options_sync, 1); + + bd->db = create_db(path); + bd->path = g_strdup(path); + *backend_data = bd; return (bd->db != NULL); @@ -325,16 +338,46 @@ backend_fini(gpointer backend_data) leveldb_close(bd->db); } + g_free((void*) bd->path); + g_slice_free(JLevelDBData, bd); } static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + JLevelDBData* bd = backend_data; + char* err = NULL; + leveldb_options_t* options; + gboolean ret = FALSE; - return TRUE; + options = leveldb_options_create(); + + if (bd->db != NULL) + { + leveldb_close(bd->db); + } + + leveldb_destroy_db(options, bd->path, &err); + + if (err) + { + free(err); + goto _error; + } + + bd->db = create_db(bd->path); + + if (!bd->db) + { + goto _error; + } + + ret = TRUE; + +_error: + leveldb_options_destroy(options); + return ret; } static JBackend leveldb_backend = { diff --git a/backend/kv/lmdb.c b/backend/kv/lmdb.c index 43e2ad5a6..70fe92ca8 100644 --- a/backend/kv/lmdb.c +++ b/backend/kv/lmdb.c @@ -344,10 +344,28 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + MDB_txn* txn; + JLMDBData* bd = backend_data; + gboolean ret = FALSE; + + if (mdb_txn_begin(bd->env, NULL, 0, &txn) == 0) + { + // 0 empties the DB which is still open afterwards + if (mdb_drop(txn, bd->dbi, 0) != 0) + { + goto _error; + } - return TRUE; + if (mdb_txn_commit(txn) != 0) + { + goto _error; + } + + ret = TRUE; + } + +_error: + return ret; } static JBackend lmdb_backend = { diff --git a/backend/kv/mongodb.c b/backend/kv/mongodb.c index efdc213d7..abec37dde 100644 --- a/backend/kv/mongodb.c +++ b/backend/kv/mongodb.c @@ -479,10 +479,14 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + JMongoDBData* bd = backend_data; + mongoc_database_t* m_database; + gboolean ret; - return TRUE; + m_database = mongoc_client_get_database(bd->connection, bd->database); + // the database will be recreated on next insert + ret = mongoc_database_drop(m_database, NULL); + return ret; } static JBackend mongodb_backend = { diff --git a/backend/kv/rocksdb.c b/backend/kv/rocksdb.c index 7d519077f..946391ed2 100644 --- a/backend/kv/rocksdb.c +++ b/backend/kv/rocksdb.c @@ -36,6 +36,7 @@ typedef struct JRocksDBBatch JRocksDBBatch; struct JRocksDBData { + char* path; rocksdb_t* db; rocksdb_readoptions_t* read_options; @@ -270,26 +271,13 @@ backend_iterate(gpointer backend_data, gpointer backend_iterator, gchar const** return FALSE; } -static gboolean -backend_init(gchar const* path, gpointer* backend_data) +static rocksdb_t* +create_db(gchar const* path) { - JRocksDBData* bd; - rocksdb_options_t* options; - g_autofree gchar* dirname = NULL; + rocksdb_t* res = NULL; gint const compressions[] = { rocksdb_lz4_compression, rocksdb_snappy_compression, rocksdb_no_compression }; + rocksdb_options_t* options = rocksdb_options_create(); - g_return_val_if_fail(path != NULL, FALSE); - - dirname = g_path_get_dirname(path); - g_mkdir_with_parents(dirname, 0700); - - bd = g_slice_new(JRocksDBData); - bd->read_options = rocksdb_readoptions_create(); - bd->write_options = rocksdb_writeoptions_create(); - bd->write_options_sync = rocksdb_writeoptions_create(); - rocksdb_writeoptions_set_sync(bd->write_options_sync, 1); - - options = rocksdb_options_create(); rocksdb_options_set_create_if_missing(options, 1); for (guint i = 0; i < G_N_ELEMENTS(compressions); i++) @@ -297,9 +285,9 @@ backend_init(gchar const* path, gpointer* backend_data) g_autofree gchar* error = NULL; rocksdb_options_set_compression(options, compressions[i]); - bd->db = rocksdb_open(options, path, &error); + res = rocksdb_open(options, path, &error); - if (bd->db != NULL) + if (res != NULL) { break; } @@ -307,6 +295,29 @@ backend_init(gchar const* path, gpointer* backend_data) rocksdb_options_destroy(options); + return res; +} + +static gboolean +backend_init(gchar const* path, gpointer* backend_data) +{ + JRocksDBData* bd; + g_autofree gchar* dirname = NULL; + + g_return_val_if_fail(path != NULL, FALSE); + + dirname = g_path_get_dirname(path); + g_mkdir_with_parents(dirname, 0700); + + bd = g_slice_new(JRocksDBData); + bd->read_options = rocksdb_readoptions_create(); + bd->write_options = rocksdb_writeoptions_create(); + bd->write_options_sync = rocksdb_writeoptions_create(); + rocksdb_writeoptions_set_sync(bd->write_options_sync, 1); + + bd->path = g_strdup(path); + bd->db = create_db(path); + *backend_data = bd; return (bd->db != NULL); @@ -321,6 +332,8 @@ backend_fini(gpointer backend_data) rocksdb_writeoptions_destroy(bd->write_options); rocksdb_writeoptions_destroy(bd->write_options_sync); + g_free(bd->path); + if (bd->db != NULL) { rocksdb_close(bd->db); @@ -332,10 +345,38 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + JRocksDBData* bd = backend_data; + char* err = NULL; + rocksdb_options_t* options; + gboolean ret = FALSE; - return TRUE; + options = rocksdb_options_create(); + + if (bd->db != NULL) + { + rocksdb_close(bd->db); + } + + rocksdb_destroy_db(options, bd->path, &err); + + if (err) + { + free(err); + goto _error; + } + + bd->db = create_db(bd->path); + + if (!bd->db) + { + goto _error; + } + + ret = TRUE; + +_error: + rocksdb_options_destroy(options); + return ret; } static JBackend rocksdb_backend = { diff --git a/backend/kv/sqlite.c b/backend/kv/sqlite.c index 2a7547831..1523201b6 100644 --- a/backend/kv/sqlite.c +++ b/backend/kv/sqlite.c @@ -290,10 +290,23 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + JSQLiteData* bd = backend_data; + gboolean ret = FALSE; - return TRUE; + // https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase + sqlite3_db_config(bd->db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + + if (sqlite3_exec(bd->db, "VACUUM", NULL, NULL, NULL) != SQLITE_OK) + { + goto _error; + } + + ret = TRUE; + +_error: + sqlite3_db_config(bd->db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + + return ret; } static JBackend sqlite_backend = { diff --git a/backend/object/gio.c b/backend/object/gio.c index aed128eea..28540194d 100644 --- a/backend/object/gio.c +++ b/backend/object/gio.c @@ -407,11 +407,51 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +static gboolean +delete_directory_tree(const char* directory) { + g_autoptr(GFileInfo) info; + g_autoptr(GFile) file = g_file_new_for_path(directory); + g_autoptr(GFileEnumerator) enumerator = g_file_enumerate_children(file, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, NULL); + + while ((info = g_file_enumerator_next_file(enumerator, NULL, NULL)) != NULL) { + const char* file_name = g_file_info_get_name(info); + GFileType file_type = g_file_info_get_file_type(info); + g_autofree char* child_path = g_build_filename(directory, file_name, NULL); + + if (file_type == G_FILE_TYPE_DIRECTORY) { + if (!delete_directory_tree(child_path)) + { + return FALSE; + } + } else { + g_autoptr(GFile) child = g_file_new_for_path(child_path); + g_debug("Deleting file: %s\n", child_path); + + if (!g_file_delete(child, NULL, NULL)) + { + g_error("Could not delete file: %s", child_path); + return FALSE; + } + } + } + + return TRUE; +} + static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + JBackendData* bd = backend_data; + GFile* file; + + if (!delete_directory_tree(bd->path)) + { + return FALSE; + } + + file = g_file_new_for_path(bd->path); + g_file_make_directory_with_parents(file, NULL, NULL); + g_object_unref(file); return TRUE; } diff --git a/backend/object/posix.c b/backend/object/posix.c index 799f79f95..1df312cd0 100644 --- a/backend/object/posix.c +++ b/backend/object/posix.c @@ -30,6 +30,8 @@ #include #include #include +#include +#include #include @@ -528,15 +530,39 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +static int +remove_entry(const char* path, const struct stat *sb, int typeflag, struct FTW *ftwbuf) +{ + (void) sb; + (void) typeflag; + (void) ftwbuf; + + return remove(path); +} + static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; - g_warning("Backend clean is currently not supported for this backend."); + JBackendData* bd = backend_data; - return TRUE; + gboolean ret = TRUE; + + if (nftw(bd->path, remove_entry, 30, FTW_DEPTH | FTW_PHYS)) + { + g_error("Could not clean POSIX backend: %s\n", strerror(errno)); + ret = FALSE; + } + + if (g_mkdir_with_parents(bd->path, 0700)) + { + g_error("Could not create POSIX backend directory: %s\n", strerror(errno)); + ret = FALSE; + } + + return ret; } + static JBackend posix_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_SERVER, diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 2d833260c..144995217 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -318,10 +318,10 @@ j_backend_object_clean(JBackend* backend) { J_TRACE_FUNCTION(NULL); - gboolean ret; + gboolean ret = FALSE; - g_return_if_fail(backend != NULL); - g_return_if_fail(backend->type == J_BACKEND_TYPE_OBJECT); + g_return_val_if_fail(backend != NULL, FALSE); + g_return_val_if_fail(backend->type == J_BACKEND_TYPE_OBJECT, FALSE); { J_TRACE("backend_clean", NULL); @@ -594,8 +594,8 @@ j_backend_kv_clean(JBackend* backend) gboolean ret; - g_return_if_fail(backend != NULL); - g_return_if_fail(backend->type == J_BACKEND_TYPE_KV); + g_return_val_if_fail(backend != NULL, FALSE); + g_return_val_if_fail(backend->type == J_BACKEND_TYPE_KV, FALSE); { J_TRACE("backend_clean", NULL); @@ -810,8 +810,8 @@ j_backend_db_clean(JBackend* backend) gboolean ret; - g_return_if_fail(backend != NULL); - g_return_if_fail(backend->type == J_BACKEND_TYPE_DB); + g_return_val_if_fail(backend != NULL, FALSE); + g_return_val_if_fail(backend->type == J_BACKEND_TYPE_DB, FALSE); { J_TRACE("backend_clean", NULL); From 1a027e83e9953e22407d041b6b7daefc213992d2 Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Mon, 15 May 2023 13:11:00 +0200 Subject: [PATCH 05/12] fixes in mysql clean --- backend/db/mysql.c | 68 ++++++++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/backend/db/mysql.c b/backend/db/mysql.c index 12824171d..48f29b5a2 100644 --- a/backend/db/mysql.c +++ b/backend/db/mysql.c @@ -589,7 +589,7 @@ j_sql_open(gpointer backend_data) return backend_db; _error: - fprintf(stderr, "%s\n", mysql_error(backend_db)); + g_error("Could not connect: %s\n", mysql_error(backend_db)); mysql_close(backend_db); return NULL; @@ -732,6 +732,7 @@ backend_clean(gpointer backend_data) MYSQL* conn = j_sql_open(backend_data); MYSQL_ROW schema = NULL; gboolean res = FALSE; + gboolean hit = FALSE; // The metadata table structure: // "CREATE TABLE IF NOT EXISTS schema_structure (" @@ -739,57 +740,73 @@ backend_clean(gpointer backend_data) // "name VARCHAR(255)," // "varname VARCHAR(255)," // "vartype INTEGER )" - gchar* schema_query = "SELECT namespace, name FROM schema_structure"; + gchar* schema_query = "SELECT DISTINCT namespace, name FROM schema_structure"; + gchar* metadata_clear = "DELETE FROM schema_structure"; // collect all table names and issue drop table on all at once // relevant restriction is the maximum query length // however, its default is 16MB which should be enough // https://dev.mysql.com/doc/refman/8.0/en/packet-too-large.html - GString* schema_delete = g_string_new("DROP TABLE "); + g_autoptr(GString) schema_delete = g_string_new("DROP TABLE "); - if (!mysql_query(conn, schema_query)) + if (mysql_query(conn, schema_query) != 0) { + // table might not exist on first startup (nothing to do in this case) + // check this again when rebased onto sql_generic PR + g_error("Querying schema info for backend_clean failed.\nError: %s", mysql_error(conn)); goto _error; } if (!(schemas = mysql_use_result(conn))) { - g_error("Querying schema info for backend_clean failed./nError: %s", mysql_error(conn)); + g_error("Querying schema info for backend_clean failed.\nError: %s", mysql_error(conn)); goto _error; } while ((schema = mysql_fetch_row(schemas))) { - unsigned long *lengths; - lengths = mysql_fetch_lengths(schemas); - - // pre-allocate new string size - g_string_set_size (schema_delete, schema_delete->len + lengths[0] + lengths[1] + 5); + unsigned long *lengths = mysql_fetch_lengths(schemas); + g_autoptr(GString) table = g_string_sized_new(lengths[0] + lengths[1] + 5); + hit = TRUE; // set up namespace_name as table name // g_string_append_printf would be more convinient but also much slower - g_string_append_len(schema_delete, "\"", 1); - g_string_append_len(schema_delete, schema[0], lengths[0]); - g_string_append_len(schema_delete, "_" , 1); - g_string_append_len(schema_delete, schema[1], lengths[1]); - g_string_append_len(schema_delete, "\"", 1); - g_string_append_len(schema_delete, ", ", 2); + g_string_append_len(table, "`", 1); + g_string_append_len(table, schema[0], lengths[0]); + g_string_append_len(table, "_" , 1); + g_string_append_len(table, schema[1], lengths[1]); + g_string_append_len(table, "`", 1); + g_string_append_len(table, ", ", 2); + + // avoid too many relocates of the query string + g_string_append_len(schema_delete, table->str, table->len); } // check whether loop terminated due to no more elements or error if(*mysql_error(conn)) { - g_error("mysql_fetch_row for backend_clean failed./nError: %s", mysql_error(conn)); + g_error("mysql_fetch_row for backend_clean failed.\nError: %s", mysql_error(conn)); goto _error; } - // remove last colon - g_string_set_size(schema_delete, schema_delete->len-2); - - if (!mysql_query(conn, schema_delete->str)) + if (hit) { - g_error("Could not clean MySQL backend. Failed to execute DROP TABLE"); - goto _error; + // remove last colon + g_string_set_size(schema_delete, schema_delete->len-2); + printf("%s\n", schema_delete->str); + + + if (mysql_query(conn, schema_delete->str) != 0) + { + g_error("Could not clean MySQL backend. Failed to execute DROP TABLE.\nError: %s", mysql_error(conn)); + goto _error; + } + + if (mysql_query(conn, metadata_clear) != 0) + { + g_error("Could not clean MySQL backend. Failed to clear metadata.\nError: %s", mysql_error(conn)); + goto _error; + } } res = TRUE; @@ -800,6 +817,11 @@ backend_clean(gpointer backend_data) mysql_free_result(schemas); } + if(conn) + { + j_sql_close(conn); + } + return res; } From 96824d2ab512e72e45d0439bbcf993b21318be96 Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Mon, 15 May 2023 23:31:27 +0200 Subject: [PATCH 06/12] fixes and comments --- backend/db/mysql.c | 9 ++++--- backend/db/sqlite.c | 2 ++ backend/kv/mongodb.c | 1 + backend/kv/sqlite.c | 53 +++++++++++++++++++++++++++++++++--------- backend/object/gio.c | 6 +++++ backend/object/rados.c | 1 + 6 files changed, 58 insertions(+), 14 deletions(-) diff --git a/backend/db/mysql.c b/backend/db/mysql.c index 48f29b5a2..2f07ad875 100644 --- a/backend/db/mysql.c +++ b/backend/db/mysql.c @@ -752,9 +752,12 @@ backend_clean(gpointer backend_data) if (mysql_query(conn, schema_query) != 0) { // table might not exist on first startup (nothing to do in this case) - // check this again when rebased onto sql_generic PR - g_error("Querying schema info for backend_clean failed.\nError: %s", mysql_error(conn)); - goto _error; + g_warning("Could not get table information for DB clean. This could be normal on first usage."); + return TRUE; + + /// \todo check this again when rebased onto sql_generic PR + //g_error("Querying schema info for backend_clean failed.\nError: %s", mysql_error(conn)); + //goto _error; } if (!(schemas = mysql_use_result(conn))) diff --git a/backend/db/sqlite.c b/backend/db/sqlite.c index 9284229dd..4485593fa 100644 --- a/backend/db/sqlite.c +++ b/backend/db/sqlite.c @@ -524,6 +524,8 @@ backend_clean(gpointer backend_data) goto _error; } + /// \todo check after rebase onto sql generic PR that metadata table does not need to be recreated here + ret = TRUE; _error: diff --git a/backend/kv/mongodb.c b/backend/kv/mongodb.c index abec37dde..3a3771e51 100644 --- a/backend/kv/mongodb.c +++ b/backend/kv/mongodb.c @@ -476,6 +476,7 @@ backend_fini(gpointer backend_data) mongoc_cleanup(); } +/// \todo untested because mongodb is client side only static gboolean backend_clean(gpointer backend_data) { diff --git a/backend/kv/sqlite.c b/backend/kv/sqlite.c index 1523201b6..dbf90458f 100644 --- a/backend/kv/sqlite.c +++ b/backend/kv/sqlite.c @@ -20,6 +20,7 @@ #include #include +#include #include @@ -35,6 +36,7 @@ typedef struct JSQLiteBatch JSQLiteBatch; struct JSQLiteData { + char* path; sqlite3* db; }; @@ -235,6 +237,32 @@ backend_iterate(gpointer backend_data, gpointer backend_iterator, gchar const** return FALSE; } +static gboolean +setup_db(sqlite3* conn) +{ + gboolean ret = FALSE; + const char* table = "CREATE TABLE IF NOT EXISTS julea (namespace TEXT NOT NULL, key TEXT NOT NULL, value BLOB NOT NULL);"; + const char* index = "CREATE UNIQUE INDEX IF NOT EXISTS julea_namespace_key ON julea (namespace, key);"; + + if (sqlite3_exec(conn, table , NULL, NULL, NULL) != SQLITE_OK) + { + g_error("Could not create KV schema.\nError: %s", sqlite3_errmsg(conn)); + goto error; + } + + if (sqlite3_exec(conn, index, NULL, NULL, NULL) != SQLITE_OK) + { + // "here2 no such table: main.julea" + g_error("Could not create KV index.\nError: %s", sqlite3_errmsg(conn)); + goto error; + } + + ret = TRUE; + +error: + return ret; +} + static gboolean backend_init(gchar const* path, gpointer* backend_data) { @@ -247,18 +275,14 @@ backend_init(gchar const* path, gpointer* backend_data) g_mkdir_with_parents(dirname, 0700); bd = g_slice_new(JSQLiteData); + bd->path = g_strdup(path); if (sqlite3_open(path, &(bd->db)) != SQLITE_OK) { goto error; } - if (sqlite3_exec(bd->db, "CREATE TABLE IF NOT EXISTS julea (namespace TEXT NOT NULL, key TEXT NOT NULL, value BLOB NOT NULL);", NULL, NULL, NULL) != SQLITE_OK) - { - goto error; - } - - if (sqlite3_exec(bd->db, "CREATE UNIQUE INDEX IF NOT EXISTS julea_namespace_key ON julea (namespace, key);", NULL, NULL, NULL) != SQLITE_OK) + if (!setup_db(bd->db)) { goto error; } @@ -269,6 +293,7 @@ backend_init(gchar const* path, gpointer* backend_data) error: sqlite3_close(bd->db); + g_free(bd->path); g_slice_free(JSQLiteData, bd); return FALSE; @@ -284,6 +309,8 @@ backend_fini(gpointer backend_data) sqlite3_close(bd->db); } + g_free(bd->path); + g_slice_free(JSQLiteData, bd); } @@ -291,20 +318,24 @@ static gboolean backend_clean(gpointer backend_data) { JSQLiteData* bd = backend_data; - gboolean ret = FALSE; + gboolean ret = G_TYPE_RESOURCE_ERROR; // https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase sqlite3_db_config(bd->db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); if (sqlite3_exec(bd->db, "VACUUM", NULL, NULL, NULL) != SQLITE_OK) { - goto _error; - } + g_error("Could not VACUUM KV backend.\nError: %s", sqlite3_errmsg(bd->db)); - ret = TRUE; + ret = FALSE; + } -_error: sqlite3_db_config(bd->db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + + if (!setup_db(bd->db)) + { + ret = FALSE; + } return ret; } diff --git a/backend/object/gio.c b/backend/object/gio.c index 28540194d..9604e9c4e 100644 --- a/backend/object/gio.c +++ b/backend/object/gio.c @@ -435,6 +435,12 @@ delete_directory_tree(const char* directory) { } } + if (!g_file_delete(file, NULL, NULL)) + { + g_error("Could not delete directory: %s", directory); + return FALSE; + } + return TRUE; } diff --git a/backend/object/rados.c b/backend/object/rados.c index ad8c73e3a..ac3ab7ecd 100644 --- a/backend/object/rados.c +++ b/backend/object/rados.c @@ -275,6 +275,7 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } +/// \todo untested because rados is client side only static gboolean backend_clean(gpointer backend_data) { From e61c2e663546a5abaaaa2fbe2a5622ea1c1cedaa Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Mon, 15 May 2023 23:32:36 +0200 Subject: [PATCH 07/12] format --- backend/db/mysql.c | 15 +++++++-------- backend/db/null.c | 2 +- backend/db/sqlite.c | 2 +- backend/kv/leveldb.c | 3 +-- backend/kv/lmdb.c | 2 +- backend/kv/null.c | 2 +- backend/kv/rocksdb.c | 2 +- backend/kv/sqlite.c | 4 ++-- backend/object/gio.c | 33 +++++++++++++++++++-------------- backend/object/null.c | 2 +- backend/object/posix.c | 15 +++++++-------- backend/object/rados.c | 2 +- lib/core/jbackend.c | 1 - server/server.c | 8 ++++---- 14 files changed, 47 insertions(+), 46 deletions(-) diff --git a/backend/db/mysql.c b/backend/db/mysql.c index 2f07ad875..9f4253da9 100644 --- a/backend/db/mysql.c +++ b/backend/db/mysql.c @@ -728,7 +728,7 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - MYSQL_RES * schemas = NULL; + MYSQL_RES* schemas = NULL; MYSQL* conn = j_sql_open(backend_data); MYSQL_ROW schema = NULL; gboolean res = FALSE; @@ -768,7 +768,7 @@ backend_clean(gpointer backend_data) while ((schema = mysql_fetch_row(schemas))) { - unsigned long *lengths = mysql_fetch_lengths(schemas); + unsigned long* lengths = mysql_fetch_lengths(schemas); g_autoptr(GString) table = g_string_sized_new(lengths[0] + lengths[1] + 5); hit = TRUE; @@ -776,7 +776,7 @@ backend_clean(gpointer backend_data) // g_string_append_printf would be more convinient but also much slower g_string_append_len(table, "`", 1); g_string_append_len(table, schema[0], lengths[0]); - g_string_append_len(table, "_" , 1); + g_string_append_len(table, "_", 1); g_string_append_len(table, schema[1], lengths[1]); g_string_append_len(table, "`", 1); g_string_append_len(table, ", ", 2); @@ -786,7 +786,7 @@ backend_clean(gpointer backend_data) } // check whether loop terminated due to no more elements or error - if(*mysql_error(conn)) + if (*mysql_error(conn)) { g_error("mysql_fetch_row for backend_clean failed.\nError: %s", mysql_error(conn)); goto _error; @@ -795,10 +795,9 @@ backend_clean(gpointer backend_data) if (hit) { // remove last colon - g_string_set_size(schema_delete, schema_delete->len-2); + g_string_set_size(schema_delete, schema_delete->len - 2); printf("%s\n", schema_delete->str); - if (mysql_query(conn, schema_delete->str) != 0) { g_error("Could not clean MySQL backend. Failed to execute DROP TABLE.\nError: %s", mysql_error(conn)); @@ -815,12 +814,12 @@ backend_clean(gpointer backend_data) res = TRUE; _error: - if(schemas) + if (schemas) { mysql_free_result(schemas); } - if(conn) + if (conn) { j_sql_close(conn); } diff --git a/backend/db/null.c b/backend/db/null.c index 981316b46..1221bfe57 100644 --- a/backend/db/null.c +++ b/backend/db/null.c @@ -164,7 +164,7 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; + (void)backend_data; return TRUE; } diff --git a/backend/db/sqlite.c b/backend/db/sqlite.c index 4485593fa..873861d7c 100644 --- a/backend/db/sqlite.c +++ b/backend/db/sqlite.c @@ -476,7 +476,7 @@ backend_init(gchar const* _path, gpointer* backend_data) if (g_strcmp0(bd->path, ":memory:") == 0) { int success = sqlite3_open_v2("file:julea-db", &(bd->db), SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI | SQLITE_OPEN_MEMORY | SQLITE_OPEN_SHAREDCACHE, NULL); - if ( success != SQLITE_OK) + if (success != SQLITE_OK) { g_error("%s", sqlite3_errmsg(bd->db)); ret = FALSE; diff --git a/backend/kv/leveldb.c b/backend/kv/leveldb.c index a74dfaffc..870df3457 100644 --- a/backend/kv/leveldb.c +++ b/backend/kv/leveldb.c @@ -303,7 +303,6 @@ backend_init(gchar const* path, gpointer* backend_data) { JLevelDBData* bd; g_autofree gchar* dirname = NULL; - g_return_val_if_fail(path != NULL, FALSE); @@ -338,7 +337,7 @@ backend_fini(gpointer backend_data) leveldb_close(bd->db); } - g_free((void*) bd->path); + g_free((void*)bd->path); g_slice_free(JLevelDBData, bd); } diff --git a/backend/kv/lmdb.c b/backend/kv/lmdb.c index 70fe92ca8..f3fd44655 100644 --- a/backend/kv/lmdb.c +++ b/backend/kv/lmdb.c @@ -361,7 +361,7 @@ backend_clean(gpointer backend_data) goto _error; } - ret = TRUE; + ret = TRUE; } _error: diff --git a/backend/kv/null.c b/backend/kv/null.c index 8311bb55a..c1dd5978a 100644 --- a/backend/kv/null.c +++ b/backend/kv/null.c @@ -149,7 +149,7 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; + (void)backend_data; return TRUE; } diff --git a/backend/kv/rocksdb.c b/backend/kv/rocksdb.c index 946391ed2..13b681563 100644 --- a/backend/kv/rocksdb.c +++ b/backend/kv/rocksdb.c @@ -314,7 +314,7 @@ backend_init(gchar const* path, gpointer* backend_data) bd->write_options = rocksdb_writeoptions_create(); bd->write_options_sync = rocksdb_writeoptions_create(); rocksdb_writeoptions_set_sync(bd->write_options_sync, 1); - + bd->path = g_strdup(path); bd->db = create_db(path); diff --git a/backend/kv/sqlite.c b/backend/kv/sqlite.c index dbf90458f..d2ec137e8 100644 --- a/backend/kv/sqlite.c +++ b/backend/kv/sqlite.c @@ -244,7 +244,7 @@ setup_db(sqlite3* conn) const char* table = "CREATE TABLE IF NOT EXISTS julea (namespace TEXT NOT NULL, key TEXT NOT NULL, value BLOB NOT NULL);"; const char* index = "CREATE UNIQUE INDEX IF NOT EXISTS julea_namespace_key ON julea (namespace, key);"; - if (sqlite3_exec(conn, table , NULL, NULL, NULL) != SQLITE_OK) + if (sqlite3_exec(conn, table, NULL, NULL, NULL) != SQLITE_OK) { g_error("Could not create KV schema.\nError: %s", sqlite3_errmsg(conn)); goto error; @@ -331,7 +331,7 @@ backend_clean(gpointer backend_data) } sqlite3_db_config(bd->db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); - + if (!setup_db(bd->db)) { ret = FALSE; diff --git a/backend/object/gio.c b/backend/object/gio.c index 9604e9c4e..7898fb778 100644 --- a/backend/object/gio.c +++ b/backend/object/gio.c @@ -408,32 +408,37 @@ backend_fini(gpointer backend_data) } static gboolean -delete_directory_tree(const char* directory) { - g_autoptr(GFileInfo) info; - g_autoptr(GFile) file = g_file_new_for_path(directory); - g_autoptr(GFileEnumerator) enumerator = g_file_enumerate_children(file, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, NULL); - - while ((info = g_file_enumerator_next_file(enumerator, NULL, NULL)) != NULL) { - const char* file_name = g_file_info_get_name(info); - GFileType file_type = g_file_info_get_file_type(info); +delete_directory_tree(const char* directory) +{ + g_autoptr(GFileInfo) info; + g_autoptr(GFile) file = g_file_new_for_path(directory); + g_autoptr(GFileEnumerator) enumerator = g_file_enumerate_children(file, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, NULL); + + while ((info = g_file_enumerator_next_file(enumerator, NULL, NULL)) != NULL) + { + const char* file_name = g_file_info_get_name(info); + GFileType file_type = g_file_info_get_file_type(info); g_autofree char* child_path = g_build_filename(directory, file_name, NULL); - if (file_type == G_FILE_TYPE_DIRECTORY) { - if (!delete_directory_tree(child_path)) + if (file_type == G_FILE_TYPE_DIRECTORY) + { + if (!delete_directory_tree(child_path)) { return FALSE; } - } else { + } + else + { g_autoptr(GFile) child = g_file_new_for_path(child_path); - g_debug("Deleting file: %s\n", child_path); + g_debug("Deleting file: %s\n", child_path); if (!g_file_delete(child, NULL, NULL)) { g_error("Could not delete file: %s", child_path); return FALSE; } - } - } + } + } if (!g_file_delete(file, NULL, NULL)) { diff --git a/backend/object/null.c b/backend/object/null.c index 1def77428..c516b9796 100644 --- a/backend/object/null.c +++ b/backend/object/null.c @@ -217,7 +217,7 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; + (void)backend_data; return TRUE; } diff --git a/backend/object/posix.c b/backend/object/posix.c index 1df312cd0..5ab5c3829 100644 --- a/backend/object/posix.c +++ b/backend/object/posix.c @@ -19,7 +19,7 @@ #include // for nftw usage -#define _XOPEN_SOURCE 500 +#define _XOPEN_SOURCE 500 #include #include @@ -531,11 +531,11 @@ backend_fini(gpointer backend_data) } static int -remove_entry(const char* path, const struct stat *sb, int typeflag, struct FTW *ftwbuf) +remove_entry(const char* path, const struct stat* sb, int typeflag, struct FTW* ftwbuf) { - (void) sb; - (void) typeflag; - (void) ftwbuf; + (void)sb; + (void)typeflag; + (void)ftwbuf; return remove(path); } @@ -552,17 +552,16 @@ backend_clean(gpointer backend_data) g_error("Could not clean POSIX backend: %s\n", strerror(errno)); ret = FALSE; } - + if (g_mkdir_with_parents(bd->path, 0700)) { g_error("Could not create POSIX backend directory: %s\n", strerror(errno)); ret = FALSE; } - + return ret; } - static JBackend posix_backend = { .type = J_BACKEND_TYPE_OBJECT, .component = J_BACKEND_COMPONENT_SERVER, diff --git a/backend/object/rados.c b/backend/object/rados.c index ac3ab7ecd..7acde24fb 100644 --- a/backend/object/rados.c +++ b/backend/object/rados.c @@ -279,7 +279,7 @@ backend_fini(gpointer backend_data) static gboolean backend_clean(gpointer backend_data) { - (void) backend_data; + (void)backend_data; g_warning("Backend clean is currently not supported for this backend."); return TRUE; diff --git a/lib/core/jbackend.c b/lib/core/jbackend.c index 144995217..d7de3d1f2 100644 --- a/lib/core/jbackend.c +++ b/lib/core/jbackend.c @@ -331,7 +331,6 @@ j_backend_object_clean(JBackend* backend) return ret; } - gboolean j_backend_object_create(JBackend* backend, gchar const* namespace, gchar const* path, gpointer* data) { diff --git a/server/server.c b/server/server.c index f2b19f0af..ac2672b7a 100644 --- a/server/server.c +++ b/server/server.c @@ -341,7 +341,7 @@ main(int argc, char** argv) g_debug("Initialized object backend %s.", object_backend); - if(opt_clean) + if (opt_clean) { if (!j_backend_object_clean(jd_object_backend)) { @@ -361,8 +361,8 @@ main(int argc, char** argv) } g_debug("Initialized kv backend %s.", kv_backend); - - if(opt_clean) + + if (opt_clean) { if (!j_backend_kv_clean(jd_kv_backend)) { @@ -383,7 +383,7 @@ main(int argc, char** argv) g_debug("Initialized db backend %s.", db_backend); - if(opt_clean) + if (opt_clean) { if (!j_backend_db_clean(jd_db_backend)) { From b71376944cc8c75907faeb947cec8ea8835fd565 Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Sun, 11 Jun 2023 14:10:05 +0200 Subject: [PATCH 08/12] use new sql generic --- backend/db/mysql.c | 1 + backend/db/sqlite.c | 1 + 2 files changed, 2 insertions(+) diff --git a/backend/db/mysql.c b/backend/db/mysql.c index 9f4253da9..fa83bb087 100644 --- a/backend/db/mysql.c +++ b/backend/db/mysql.c @@ -833,6 +833,7 @@ static JBackend mysql_backend = { .db = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_schema_create = sql_generic_schema_create, .backend_schema_get = sql_generic_schema_get, .backend_schema_delete = sql_generic_schema_delete, diff --git a/backend/db/sqlite.c b/backend/db/sqlite.c index 873861d7c..1ae1f33f9 100644 --- a/backend/db/sqlite.c +++ b/backend/db/sqlite.c @@ -541,6 +541,7 @@ static JBackend sqlite_backend = { .db = { .backend_init = backend_init, .backend_fini = backend_fini, + .backend_clean = backend_clean, .backend_schema_create = sql_generic_schema_create, .backend_schema_get = sql_generic_schema_get, .backend_schema_delete = sql_generic_schema_delete, From 8459b3816567a893dd0a0bc2aec43e7f651ad8b8 Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Sun, 11 Jun 2023 14:10:49 +0200 Subject: [PATCH 09/12] make deletion tool --- meson.build | 7 ++ tools/backend_cleanup.c | 180 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 tools/backend_cleanup.c diff --git a/meson.build b/meson.build index b4c686a4f..0509a49a5 100644 --- a/meson.build +++ b/meson.build @@ -723,6 +723,13 @@ executable('julea-statistics', 'tools/statistics.c', install: true, ) + +executable('julea-backend-cleanup', 'tools/backend_cleanup.c', + dependencies: common_deps + [julea_dep], + include_directories: julea_incs, + install: true, +) + if hdf_dep.found() executable('julea-h5migrate', 'tools/h5migrate.c', dependencies: common_deps + [julea_dep] + [hdf_dep], diff --git a/tools/backend_cleanup.c b/tools/backend_cleanup.c new file mode 100644 index 000000000..30699af41 --- /dev/null +++ b/tools/backend_cleanup.c @@ -0,0 +1,180 @@ +/* + * JULEA - Flexible storage framework + * Copyright (C) 2023-2023 Julian Benda + * Copyright (C) 2023 Timm Leon Erxleben + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include +#include + +#include + +JBackend* object_backend = NULL; +JBackend* kv_backend = NULL; +JBackend* db_backend = NULL; + +static gboolean +setup_backend(JConfiguration* configuration, JBackendType type, gchar const* port_str, GModule** module, JBackend** j_backend) +{ + gchar const* backend; + gchar const* component; + g_autofree gchar* path; + gboolean load_module = FALSE; + + backend = j_configuration_get_backend(configuration, type); + component = j_configuration_get_backend_component(configuration, type); + path = j_helper_str_replace(j_configuration_get_backend_path(configuration, type), "{PORT}", port_str); + + if (strcmp(component, "server") == 0) + { + load_module = j_backend_load_server(backend, component, type, module, j_backend); + } + else + { + load_module = j_backend_load_client(backend, component, type, module, j_backend); + } + + if (load_module) + { + gboolean res = TRUE; + + if (j_backend == NULL) + { + g_warning("Failed to load component: %s.", backend); + return FALSE; + } + + switch (type) + { + case J_BACKEND_TYPE_OBJECT: + res = j_backend_object_init(*j_backend, path); + break; + case J_BACKEND_TYPE_KV: + res = j_backend_kv_init(*j_backend, path); + break; + case J_BACKEND_TYPE_DB: + res = j_backend_db_init(*j_backend, path); + break; + default: + g_warning("unknown backend type: (%d), unable to setup backend!", type); + res = FALSE; + } + + if (!res) + { + g_warning("Failed to initalize backend: %s", backend); + return FALSE; + } + + return TRUE; + } + + return FALSE; +} + +int +main(int argc, char** argv) +{ + JConfiguration* configuration = NULL; + GModule* db_module = NULL; + GModule* kv_module = NULL; + GModule* object_module = NULL; + g_autofree gchar* port_str = NULL; + gint res = 1; + (void) argc; + (void) argv; + + setlocale(LC_ALL, "C.UTF-8"); + + configuration = j_configuration(); + + if (configuration == NULL) + { + g_warning("unable to read config"); + goto end; + } + + port_str = g_strdup_printf("%d", j_configuration_get_port(configuration)); + if (!setup_backend(configuration, J_BACKEND_TYPE_OBJECT, port_str, &object_module, &object_backend)) + { + g_warning("failed to initealize object backend"); + goto end; + } + if (!setup_backend(configuration, J_BACKEND_TYPE_KV, port_str, &kv_module, &kv_backend)) + { + g_warning("failed to initealize kv backend"); + goto end; + } + if (!setup_backend(configuration, J_BACKEND_TYPE_DB, port_str, &db_module, &db_backend)) + { + g_warning("failed to initealize db backend"); + goto end; + } + + if (!j_backend_db_clean(db_backend)) + { + g_warning("Could not clean db backend"); + goto end; + } + + if (!j_backend_kv_clean(kv_backend)) + { + g_warning("Could not clean kv backend"); + goto end; + } + + if (!j_backend_object_clean(object_backend)) + { + g_warning("Could not clean object backend"); + goto end; + } + + + res = 0; +end: + if (db_backend != NULL) + { + j_backend_db_fini(db_backend); + } + if (kv_backend != NULL) + { + j_backend_kv_fini(kv_backend); + } + if (object_backend != NULL) + { + j_backend_object_fini(object_backend); + } + if (db_module != NULL) + { + g_module_close(db_module); + } + if (kv_module != NULL) + { + g_module_close(kv_module); + } + if (object_module != NULL) + { + g_module_close(object_module); + } + j_configuration_unref(configuration); + + return res; +} From f26e63bb284226c2e4164760dec4b9ad4677cae5 Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Sun, 11 Jun 2023 14:33:06 +0200 Subject: [PATCH 10/12] revert changes on server --- server/server.c | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/server/server.c b/server/server.c index ac2672b7a..86c84c3bc 100644 --- a/server/server.c +++ b/server/server.c @@ -203,7 +203,6 @@ main(int argc, char** argv) J_TRACE_FUNCTION(NULL); gboolean opt_daemon = FALSE; - gboolean opt_clean = FALSE; g_autofree gchar* opt_host = NULL; gint opt_port = 0; @@ -231,7 +230,6 @@ main(int argc, char** argv) { "daemon", 0, 0, G_OPTION_ARG_NONE, &opt_daemon, "Run as daemon", NULL }, { "host", 0, 0, G_OPTION_ARG_STRING, &opt_host, "Override host name", "hostname" }, { "port", 0, 0, G_OPTION_ARG_INT, &opt_port, "Port to use", "0" }, - { "clean", 0, 0, G_OPTION_ARG_NONE, &opt_clean, "Clean all data on configured backends.", NULL }, { NULL, 0, 0, 0, NULL, NULL, NULL } }; @@ -340,15 +338,6 @@ main(int argc, char** argv) } g_debug("Initialized object backend %s.", object_backend); - - if (opt_clean) - { - if (!j_backend_object_clean(jd_object_backend)) - { - g_warning("Could not clean object backend %s.", object_backend); - return 1; - } - } } if (jd_is_server_for_backend(opt_host, opt_port, J_BACKEND_TYPE_KV) @@ -361,15 +350,6 @@ main(int argc, char** argv) } g_debug("Initialized kv backend %s.", kv_backend); - - if (opt_clean) - { - if (!j_backend_kv_clean(jd_kv_backend)) - { - g_warning("Could not clean kv backend %s.", kv_backend); - return 1; - } - } } if (jd_is_server_for_backend(opt_host, opt_port, J_BACKEND_TYPE_DB) @@ -382,15 +362,6 @@ main(int argc, char** argv) } g_debug("Initialized db backend %s.", db_backend); - - if (opt_clean) - { - if (!j_backend_db_clean(jd_db_backend)) - { - g_warning("Could not clean db backend %s.", db_backend); - return 1; - } - } } jd_statistics = j_statistics_new(FALSE); From 9b58aff04e68cbc1a65a7fd11079c68a5c293c88 Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Sun, 11 Jun 2023 14:34:51 +0200 Subject: [PATCH 11/12] format --- tools/backend_cleanup.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/backend_cleanup.c b/tools/backend_cleanup.c index 30699af41..923615c61 100644 --- a/tools/backend_cleanup.c +++ b/tools/backend_cleanup.c @@ -46,8 +46,8 @@ setup_backend(JConfiguration* configuration, JBackendType type, gchar const* por if (strcmp(component, "server") == 0) { load_module = j_backend_load_server(backend, component, type, module, j_backend); - } - else + } + else { load_module = j_backend_load_client(backend, component, type, module, j_backend); } @@ -83,7 +83,7 @@ setup_backend(JConfiguration* configuration, JBackendType type, gchar const* por g_warning("Failed to initalize backend: %s", backend); return FALSE; } - + return TRUE; } @@ -99,8 +99,8 @@ main(int argc, char** argv) GModule* object_module = NULL; g_autofree gchar* port_str = NULL; gint res = 1; - (void) argc; - (void) argv; + (void)argc; + (void)argv; setlocale(LC_ALL, "C.UTF-8"); @@ -147,7 +147,6 @@ main(int argc, char** argv) goto end; } - res = 0; end: if (db_backend != NULL) From 5869f9cf01923f57274c5d29904add127eecf842 Mon Sep 17 00:00:00 2001 From: Timm Leon Erxleben Date: Sun, 11 Jun 2023 15:32:53 +0200 Subject: [PATCH 12/12] check remaining todos --- backend/db/mysql.c | 5 +---- backend/db/sqlite.c | 2 -- backend/kv/mongodb.c | 1 - backend/object/rados.c | 2 +- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/backend/db/mysql.c b/backend/db/mysql.c index fa83bb087..05277fc16 100644 --- a/backend/db/mysql.c +++ b/backend/db/mysql.c @@ -751,13 +751,10 @@ backend_clean(gpointer backend_data) if (mysql_query(conn, schema_query) != 0) { - // table might not exist on first startup (nothing to do in this case) + // table might not exist on before first startup (nothing to do in this case) g_warning("Could not get table information for DB clean. This could be normal on first usage."); return TRUE; - /// \todo check this again when rebased onto sql_generic PR - //g_error("Querying schema info for backend_clean failed.\nError: %s", mysql_error(conn)); - //goto _error; } if (!(schemas = mysql_use_result(conn))) diff --git a/backend/db/sqlite.c b/backend/db/sqlite.c index 1ae1f33f9..303b1f18b 100644 --- a/backend/db/sqlite.c +++ b/backend/db/sqlite.c @@ -524,8 +524,6 @@ backend_clean(gpointer backend_data) goto _error; } - /// \todo check after rebase onto sql generic PR that metadata table does not need to be recreated here - ret = TRUE; _error: diff --git a/backend/kv/mongodb.c b/backend/kv/mongodb.c index 3a3771e51..abec37dde 100644 --- a/backend/kv/mongodb.c +++ b/backend/kv/mongodb.c @@ -476,7 +476,6 @@ backend_fini(gpointer backend_data) mongoc_cleanup(); } -/// \todo untested because mongodb is client side only static gboolean backend_clean(gpointer backend_data) { diff --git a/backend/object/rados.c b/backend/object/rados.c index 7acde24fb..566e2d3d3 100644 --- a/backend/object/rados.c +++ b/backend/object/rados.c @@ -275,7 +275,7 @@ backend_fini(gpointer backend_data) g_slice_free(JBackendData, bd); } -/// \todo untested because rados is client side only +/// \todo implement when rados test setup becomes easier static gboolean backend_clean(gpointer backend_data) {