diff --git a/common/config-mgr.c b/common/config-mgr.c index 9bf7951e..4578762a 100644 --- a/common/config-mgr.c +++ b/common/config-mgr.c @@ -2,6 +2,35 @@ #include "config-mgr.h" #include "seaf-db.h" #include "log.h" +#include "utils.h" +#include + +/* These configurations in this table are displayed as */ +/* "group:option:default_value:property" format . */ +const static char *config_table [] = { + "quota:default:-2:0", + "history:keep_days:-1:0", + "fileserver:max_upload_size:-1:0", + "fileserver:max_download_dir_size:100:0", + "fileserver:host:0.0.0.0:1", + "fileserver:port:8082:1", + "fileserver:worker_threads:10:1", + "fileserver:fixed_block_size:8:1", + "fileserver:web_token_expire_time:8:1", + "fileserver:max_indexing_threads:1:1", + "fileserver:max_index_processing_threads:3:1", + "fileserver:cluster_shared_temp_file_mode:0600:1", + "library_trash:expire_days:30:0", + "library_trash:scan_days:1:1", + "web_copy:max_files:0:1", + "web_copy:max_size:0:1", + "scheduler:size_sched_thread_num:1:1", + "zip:windows_encoding:none:1", + "general:enable_syslog:false:1", + "fuse:excluded_users:none:1", + "t_group:t_key:0:0", + NULL +}; int seaf_cfg_manager_init (SeafCfgManager *mgr) @@ -23,6 +52,54 @@ seaf_cfg_manager_init (SeafCfgManager *mgr) return 0; } +static void load_config_option (SeafCfgManager *mgr, char **option_item) +{ + char *group = NULL, *key = NULL, + *property = NULL, *default_value = NULL, + *cache_key = NULL, *cache_value = NULL, + *sql = NULL, *value = NULL; + + group = option_item[0]; + key = option_item[1]; + default_value = option_item[2]; + property = option_item[3]; + + sql = "SELECT value FROM SeafileConf WHERE cfg_group=? AND cfg_key=?"; + value = seaf_db_statement_get_string(mgr->db, sql, + 2, "string", group, "string", key); + if (value) { + value = g_strstrip(value); + } else { + value = seaf_key_file_get_string (mgr->config, group, key, NULL); + if (!value) { + if (strcmp ("zip", group) == 0 && + strcmp ("windows_encoding", key) == 0) { + setlocale (LC_ALL, "en_US.UTF-8"); + } + value = g_strdup (default_value); + } + } + + cache_key = g_strdup_printf ("%s,%s", group, key); + cache_value = g_strdup_printf ("%s,%s", value, property); + g_free (value); + + g_hash_table_insert (mgr->config_cache, cache_key, cache_value); +} + +static void load_config_cache (SeafCfgManager *mgr) +{ + int index = 0; + char **option_item = NULL; + + while (config_table[index]) { + option_item = g_strsplit (config_table[index], ":", -1); + load_config_option (mgr, option_item); + g_strfreev (option_item); + index++; + } +} + SeafCfgManager * seaf_cfg_manager_new (SeafileSession *session) { @@ -32,6 +109,13 @@ seaf_cfg_manager_new (SeafileSession *session) mgr->config = session->config; mgr->db = session->db; + if ((session->create_tables || seaf_db_type(session->db) == SEAF_DB_TYPE_PGSQL) + && seaf_cfg_manager_init (session->cfg_mgr) < 0) { + seaf_warning ("Failed to init config manager.\n"); + return NULL; + } + mgr->config_cache = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + load_config_cache (mgr); return mgr; } @@ -95,6 +179,19 @@ int seaf_cfg_manager_set_config (SeafCfgManager *mgr, const char *group, const char *key, const char *value) { gboolean exists, err = FALSE; + char *cache_key = NULL, *cache_value = NULL, *property = NULL; + char **tokens = NULL; + int ret = 0; + + cache_key = g_strdup_printf ("%s,%s" ,group, key); + cache_value = g_hash_table_lookup (mgr->config_cache, cache_key); + tokens = g_strsplit (cache_value, ",", -1); + property = tokens[1]; + + if (strcmp (property, "0") == 0) { + cache_value = g_strdup_printf ("%s,%s" ,value, property); + g_hash_table_insert (mgr->config_cache, cache_key, cache_value); + } char *sql = "SELECT 1 FROM SeafileConf WHERE cfg_group=? AND cfg_key=?"; exists = seaf_db_statement_exists(mgr->db, sql, &err, @@ -102,7 +199,7 @@ seaf_cfg_manager_set_config (SeafCfgManager *mgr, const char *group, const char "string", key); if (err) { seaf_warning ("[db error]Failed to set config [%s:%s] to db.\n", group, key); - return -1; + ret = -1; } if (exists) sql = "UPDATE SeafileConf SET value=? WHERE cfg_group=? AND cfg_key=?"; @@ -113,10 +210,12 @@ seaf_cfg_manager_set_config (SeafCfgManager *mgr, const char *group, const char "string", value, "string", group, "string", key) < 0) { seaf_warning ("Failed to set config [%s:%s] to db.\n", group, key); - return -1; + ret = -1; } - return 0; + g_strfreev (tokens); + + return ret; } int @@ -127,12 +226,8 @@ seaf_cfg_manager_get_config_int (SeafCfgManager *mgr, const char *group, const c char *value = seaf_cfg_manager_get_config (mgr, group, key); if (!value) { - GError *err = NULL; - ret = g_key_file_get_integer (mgr->config, group, key, &err); - if (err) { - ret = -1; - g_clear_error(&err); - } + seaf_warning ("Config [%s:%s] not set, default is -1.\n", group, key); + ret = -1; } else { ret = strtol (value, &invalid, 10); if (*invalid != '\0') { @@ -153,12 +248,8 @@ seaf_cfg_manager_get_config_int64 (SeafCfgManager *mgr, const char *group, const char *value = seaf_cfg_manager_get_config (mgr, group, key); if (!value) { - GError *err = NULL; - ret = g_key_file_get_int64(mgr->config, group, key, &err); - if (err) { - ret = -1; - g_clear_error(&err); - } + seaf_warning ("Config [%s:%s] not set, default is -1.\n", group, key); + ret = -1; } else { ret = strtoll (value, &invalid, 10); if (*invalid != '\0') { @@ -178,13 +269,8 @@ seaf_cfg_manager_get_config_boolean (SeafCfgManager *mgr, const char *group, con char *value = seaf_cfg_manager_get_config (mgr, group, key); if (!value) { - GError *err = NULL; - ret = g_key_file_get_boolean(mgr->config, group, key, &err); - if (err) { - seaf_warning ("Config [%s:%s] not set, default is false.\n", group, key); - ret = FALSE; - g_clear_error(&err); - } + seaf_warning ("Config [%s:%s] not set, default is false.\n", group, key); + ret = FALSE; } else { if (strcmp ("true", value) == 0) ret = TRUE; @@ -199,28 +285,33 @@ seaf_cfg_manager_get_config_boolean (SeafCfgManager *mgr, const char *group, con char * seaf_cfg_manager_get_config_string (SeafCfgManager *mgr, const char *group, const char *key) { - char *ret = NULL; - char *value = seaf_cfg_manager_get_config (mgr, group, key); - if (!value) { - ret = g_key_file_get_string (mgr->config, group, key, NULL); - if (ret != NULL) - ret = g_strstrip(ret); - } else { - ret = value; - } - return ret; + if (!value) + seaf_warning ("Config [%s:%s] not set, default is NULL.\n", group, key); + + return value; } char * seaf_cfg_manager_get_config (SeafCfgManager *mgr, const char *group, const char *key) { - char *sql = "SELECT value FROM SeafileConf WHERE cfg_group=? AND cfg_key=?"; - char *value = seaf_db_statement_get_string(mgr->db, sql, - 2, "string", group, "string", key); - if (value != NULL) - value = g_strstrip(value); + char *ret = NULL, *cache_key = NULL, + *cache_value = NULL, *option_value = NULL; + char **tokens = NULL; - return value; + cache_key = g_strdup_printf ("%s,%s", group, key); + cache_value = g_hash_table_lookup (mgr->config_cache, cache_key); + tokens = g_strsplit(cache_value, ",", -1); + option_value = tokens[0]; + + if (strcmp (option_value, "none") == 0) + ret = NULL; + else + ret = g_strdup (option_value); + + g_free (cache_key); + g_strfreev (tokens); + + return ret; } diff --git a/common/config-mgr.h b/common/config-mgr.h index 483555e0..71e3bb7e 100644 --- a/common/config-mgr.h +++ b/common/config-mgr.h @@ -7,6 +7,7 @@ typedef struct _SeafCfgManager SeafCfgManager; struct _SeafCfgManager { GKeyFile *config; SeafDB *db; + GHashTable *config_cache; }; typedef struct _SeafileSession SeafileSession; diff --git a/common/fs-mgr.c b/common/fs-mgr.c index 943a02d5..86a1a36f 100644 --- a/common/fs-mgr.c +++ b/common/fs-mgr.c @@ -31,8 +31,11 @@ #endif /* SEAFILE_SERVER */ #include "db.h" +#include "../server/fileserver-config.h" #define SEAF_TMP_EXT "~" +#define DEFAULT_FIXED_BLOCK_SIZE ((gint64)1 << 23) /* 8MB */ +#define DEFAULT_MAX_INDEXING_THREADS 1 struct _SeafFSManagerPriv { /* GHashTable *seafile_cache; */ @@ -612,6 +615,13 @@ chunking_worker (gpointer vdata, gpointer user_data) int fd = -1; ssize_t n; int idx; + gint64 fixed_block_size; + int fixed_block_size_mb = fileserver_config_get_integer (seaf->cfg_mgr, seaf->config, "fixed_block_size"); + + if (fixed_block_size_mb <= 0) + fixed_block_size = DEFAULT_FIXED_BLOCK_SIZE; + else + fixed_block_size = fixed_block_size_mb * ((gint64)1 << 20); chunk->block_buf = g_new0 (char, chunk->len); if (!chunk->block_buf) { @@ -646,7 +656,7 @@ chunking_worker (gpointer vdata, gpointer user_data) if (chunk->result < 0) goto out; - idx = chunk->offset / seaf->http_server->fixed_block_size; + idx = chunk->offset / fixed_block_size; memcpy (data->blk_sha1s + idx * CHECKSUM_LENGTH, chunk->checksum, CHECKSUM_LENGTH); out: @@ -673,8 +683,24 @@ split_file_to_block (const char *repo_id, int n_pending = 0; CDCDescriptor *chunk; int ret = 0; + gint64 fixed_block_size; + int fixed_block_size_mb = fileserver_config_get_integer (seaf->cfg_mgr, + seaf->config, + "fixed_block_size"); + if (fixed_block_size_mb <= 0) + fixed_block_size = DEFAULT_FIXED_BLOCK_SIZE; + else + fixed_block_size = fixed_block_size_mb * ((gint64)1 << 20); + + int max_indexing_threads = fileserver_config_get_integer (seaf->cfg_mgr, + seaf->config, + "max_indexing_threads"); + if (max_indexing_threads <= 0) + max_indexing_threads = DEFAULT_MAX_INDEXING_THREADS; + + + n_blocks = (file_size + fixed_block_size - 1) / fixed_block_size; - n_blocks = (file_size + seaf->http_server->fixed_block_size - 1) / seaf->http_server->fixed_block_size; block_sha1s = g_new0 (uint8_t, n_blocks * CHECKSUM_LENGTH); if (!block_sha1s) { seaf_warning ("Failed to allocate block_sha1s.\n"); @@ -694,7 +720,7 @@ split_file_to_block (const char *repo_id, data.finished_tasks = finished_tasks; tpool = g_thread_pool_new (chunking_worker, &data, - seaf->http_server->max_indexing_threads, FALSE, NULL); + max_indexing_threads, FALSE, NULL); if (!tpool) { seaf_warning ("Failed to allocate thread pool\n"); ret = -1; @@ -705,7 +731,7 @@ split_file_to_block (const char *repo_id, guint64 len; guint64 left = (guint64)file_size; while (left > 0) { - len = ((left >= seaf->http_server->fixed_block_size) ? seaf->http_server->fixed_block_size : left); + len = ((left >= fixed_block_size) ? fixed_block_size : left); chunk = g_new0 (CDCDescriptor, 1); chunk->offset = offset; @@ -725,7 +751,7 @@ split_file_to_block (const char *repo_id, goto out; } if (indexed) - *indexed += seaf->http_server->fixed_block_size; + *indexed += fixed_block_size; if ((--n_pending) <= 0) { if (indexed) diff --git a/common/rpc-service.c b/common/rpc-service.c index 74e8e2a8..67315d35 100644 --- a/common/rpc-service.c +++ b/common/rpc-service.c @@ -2768,6 +2768,7 @@ seafile_copy_file (const char *src_repo_id, int synchronous, GError **error) { + seaf_message ("Mydebug seafile_copy_file\n"); char *norm_src_dir = NULL, *norm_src_filename = NULL; char *norm_dst_dir = NULL, *norm_dst_filename = NULL; char *rsrc_dir = NULL, *rdst_dir = NULL; @@ -2926,6 +2927,7 @@ seafile_move_file (const char *src_repo_id, GObject * seafile_get_copy_task (const char *task_id, GError **error) + { return (GObject *)seaf_copy_manager_get_task (seaf->copy_mgr, task_id); } diff --git a/fuse/Makefile.am b/fuse/Makefile.am index 62fb4559..725b4a34 100644 --- a/fuse/Makefile.am +++ b/fuse/Makefile.am @@ -34,7 +34,8 @@ seaf_fuse_SOURCES = seaf-fuse.c \ ../common/obj-store.c \ ../common/obj-backend-fs.c \ ../common/obj-backend-riak.c \ - ../common/seafile-crypt.c + ../common/seafile-crypt.c \ + ../common/config-mgr.c seaf_fuse_LDADD = @CCNET_LIBS@ \ @GLIB2_LIBS@ @GOBJECT_LIBS@ @SSL_LIBS@ @LIB_RT@ @LIB_UUID@ \ diff --git a/fuse/seafile-session.c b/fuse/seafile-session.c index 65fb5336..0acf15bd 100644 --- a/fuse/seafile-session.c +++ b/fuse/seafile-session.c @@ -74,6 +74,10 @@ seafile_session_new(const char *central_config_dir, goto onerror; } + session->cfg_mgr = seaf_cfg_manager_new (session); + if (!session->cfg_mgr) + goto onerror; + if (read_excluded_users (session) < 0) { seaf_warning ("Failed to load excluded users.\n"); goto onerror; @@ -111,7 +115,7 @@ read_excluded_users (SeafileSession *session) int l, i; char *hash_value; - users = seaf_key_file_get_string (session->config, "fuse", "excluded_users", NULL); + users = seaf_cfg_manager_get_config_string (session->cfg_mgr, "fuse", "excluded_users"); if (!users) return 0; diff --git a/fuse/seafile-session.h b/fuse/seafile-session.h index a5d677be..5b10d779 100644 --- a/fuse/seafile-session.h +++ b/fuse/seafile-session.h @@ -11,6 +11,7 @@ #include "branch-mgr.h" #include "commit-mgr.h" #include "repo-mgr.h" +#include "config-mgr.h" typedef struct _SeafileSession SeafileSession; @@ -27,6 +28,7 @@ struct _SeafileSession { SeafBranchManager *branch_mgr; SeafCommitManager *commit_mgr; SeafRepoManager *repo_mgr; + SeafCfgManager *cfg_mgr; GHashTable *excluded_users; diff --git a/server/copy-mgr.c b/server/copy-mgr.c index 55051046..ae16ee1e 100644 --- a/server/copy-mgr.c +++ b/server/copy-mgr.c @@ -44,13 +44,6 @@ seaf_copy_manager_new (struct _SeafileSession *session) (GDestroyNotify)copy_task_free); pthread_mutex_init (&mgr->priv->lock, NULL); - mgr->max_files = g_key_file_get_int64 (session->config, - "web_copy", "max_files", NULL); - mgr->max_size = g_key_file_get_int64 (session->config, - "web_copy", "max_size", NULL); - /* size is given in MB */ - mgr->max_size <<= 20; - return mgr; } @@ -66,6 +59,7 @@ SeafileCopyTask * seaf_copy_manager_get_task (SeafCopyManager *mgr, const char *task_id) { + seaf_message ("Mydebug seaf_copy_manager_get_task task_id = %s\n", task_id); SeafCopyManagerPriv *priv = mgr->priv; CopyTask *task; SeafileCopyTask *t = NULL; @@ -80,6 +74,7 @@ seaf_copy_manager_get_task (SeafCopyManager *mgr, "canceled", task->canceled, "failed", task->failed, "failed_reason", task->failed_reason, "successful", task->successful, NULL); + seaf_message ("Mydebug task->canceled = %d task->failed = %d task->successful = %d\n", task->canceled, task->failed, task->successful); if (task->canceled || task->failed || task->successful) g_hash_table_remove(priv->copy_tasks, task_id); } @@ -153,6 +148,7 @@ seaf_copy_manager_add_task (SeafCopyManager *mgr, memcpy (task->task_id, task_id, 36); pthread_mutex_lock (&priv->lock); + seaf_message ("Mydebug insert copy task task_id = %s\n", task_id); g_hash_table_insert (priv->copy_tasks, g_strdup(task_id), task); pthread_mutex_unlock (&priv->lock); } diff --git a/server/copy-mgr.h b/server/copy-mgr.h index ec16ba30..8bcb7063 100644 --- a/server/copy-mgr.h +++ b/server/copy-mgr.h @@ -16,9 +16,6 @@ struct _SeafileCopyTask; struct _SeafCopyManager { struct _SeafileSession *session; struct _SeafCopyManagerPriv *priv; - - gint64 max_files; - gint64 max_size; }; typedef struct _SeafCopyManager SeafCopyManager; typedef struct _SeafCopyManagerPriv SeafCopyManagerPriv; diff --git a/server/fileserver-config.c b/server/fileserver-config.c index 984e77ce..090b6d30 100644 --- a/server/fileserver-config.c +++ b/server/fileserver-config.c @@ -2,6 +2,7 @@ #include +#include "seafile-session.h" #include "fileserver-config.h" const char *OLD_GROUP_NAME = "httpserver"; @@ -14,22 +15,22 @@ get_group_name(GKeyFile *config) } int -fileserver_config_get_integer(GKeyFile *config, char *key, GError **error) +fileserver_config_get_integer(SeafCfgManager *mgr, GKeyFile *config, char *key) { const char *group = get_group_name(config); - return g_key_file_get_integer (config, group, key, error); + return seaf_cfg_manager_get_config_int (mgr, group, key); } char * -fileserver_config_get_string(GKeyFile *config, char *key, GError **error) +fileserver_config_get_string(SeafCfgManager *mgr, GKeyFile *config, char *key) { const char *group = get_group_name(config); - return g_key_file_get_string (config, group, key, error); + return seaf_cfg_manager_get_config_string (mgr, group, key); } gboolean -fileserver_config_get_boolean(GKeyFile *config, char *key, GError **error) +fileserver_config_get_boolean(SeafCfgManager *mgr, GKeyFile *config, char *key) { const char *group = get_group_name(config); - return g_key_file_get_boolean (config, group, key, error); + return seaf_cfg_manager_get_config_boolean (mgr, group, key); } diff --git a/server/fileserver-config.h b/server/fileserver-config.h index 9bc44073..ec40c0f0 100644 --- a/server/fileserver-config.h +++ b/server/fileserver-config.h @@ -4,12 +4,12 @@ struct GKeyFile; int -fileserver_config_get_integer(GKeyFile *config, char *key, GError **error); +fileserver_config_get_integer(SeafCfgManager *mgr, GKeyFile *config, char *key); char * -fileserver_config_get_string(GKeyFile *config, char *key, GError **error); +fileserver_config_get_string(SeafCfgManager *mgr, GKeyFile *config, char *key); gboolean -fileserver_config_get_boolean(GKeyFile *config, char *key, GError **error); +fileserver_config_get_boolean(SeafCfgManager *mgr, GKeyFile *config, char *key); #endif // SEAFILE_FILESERVER_CONFIG_H diff --git a/server/http-server.c b/server/http-server.c index a1410f87..03f522bb 100644 --- a/server/http-server.c +++ b/server/http-server.c @@ -29,14 +29,8 @@ #include "http-status-codes.h" -#define DEFAULT_BIND_HOST "0.0.0.0" -#define DEFAULT_BIND_PORT 8082 #define DEFAULT_WORKER_THREADS 10 #define DEFAULT_MAX_DOWNLOAD_DIR_SIZE 100 * ((gint64)1 << 20) /* 100MB */ -#define DEFAULT_MAX_INDEXING_THREADS 1 -#define DEFAULT_MAX_INDEX_PROCESSING_THREADS 3 -#define DEFAULT_FIXED_BLOCK_SIZE ((gint64)1 << 23) /* 8MB */ -#define DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE 0600 #define HOST "host" #define PORT "port" @@ -117,155 +111,6 @@ const char *POST_RECV_FS_REGEX = "^/repo/[\\da-z]{8}-[\\da-z]{4}-[\\da-z]{4}-[\\ const char *POST_PACK_FS_REGEX = "^/repo/[\\da-z]{8}-[\\da-z]{4}-[\\da-z]{4}-[\\da-z]{4}-[\\da-z]{12}/pack-fs"; const char *GET_BLOCK_MAP_REGEX = "^/repo/[\\da-z]{8}-[\\da-z]{4}-[\\da-z]{4}-[\\da-z]{4}-[\\da-z]{12}/block-map/[\\da-z]{40}"; -static void -load_http_config (HttpServerStruct *htp_server, SeafileSession *session) -{ - GError *error = NULL; - char *host = NULL; - int port = 0; - int worker_threads; - int web_token_expire_time; - int fixed_block_size_mb; - char *encoding; - int max_indexing_threads; - int max_index_processing_threads; - char *cluster_shared_temp_file_mode = NULL; - - host = fileserver_config_get_string (session->config, HOST, &error); - if (!error) { - htp_server->bind_addr = host; - } else { - if (error->code != G_KEY_FILE_ERROR_KEY_NOT_FOUND && - error->code != G_KEY_FILE_ERROR_GROUP_NOT_FOUND) { - seaf_warning ("[conf] Error: failed to read the value of 'host'\n"); - exit (1); - } - - htp_server->bind_addr = g_strdup (DEFAULT_BIND_HOST); - g_clear_error (&error); - } - - port = fileserver_config_get_integer (session->config, PORT, &error); - if (!error) { - htp_server->bind_port = port; - } else { - if (error->code != G_KEY_FILE_ERROR_KEY_NOT_FOUND && - error->code != G_KEY_FILE_ERROR_GROUP_NOT_FOUND) { - seaf_warning ("[conf] Error: failed to read the value of 'port'\n"); - exit (1); - } - - htp_server->bind_port = DEFAULT_BIND_PORT; - g_clear_error (&error); - } - - worker_threads = fileserver_config_get_integer (session->config, "worker_threads", - &error); - if (error) { - htp_server->worker_threads = DEFAULT_WORKER_THREADS; - g_clear_error (&error); - } else { - if (worker_threads <= 0) - htp_server->worker_threads = DEFAULT_WORKER_THREADS; - else - htp_server->worker_threads = worker_threads; - } - seaf_message ("fileserver: worker_threads = %d\n", htp_server->worker_threads); - - fixed_block_size_mb = fileserver_config_get_integer (session->config, - "fixed_block_size", - &error); - if (error){ - htp_server->fixed_block_size = DEFAULT_FIXED_BLOCK_SIZE; - g_clear_error(&error); - } else { - if (fixed_block_size_mb <= 0) - htp_server->fixed_block_size = DEFAULT_FIXED_BLOCK_SIZE; - else - htp_server->fixed_block_size = fixed_block_size_mb * ((gint64)1 << 20); - } - seaf_message ("fileserver: fixed_block_size = %"G_GINT64_FORMAT"\n", - htp_server->fixed_block_size); - - web_token_expire_time = fileserver_config_get_integer (session->config, - "web_token_expire_time", - &error); - if (error){ - htp_server->web_token_expire_time = 3600; /* default 3600s */ - g_clear_error(&error); - } else { - if (web_token_expire_time <= 0) - htp_server->web_token_expire_time = 3600; /* default 3600s */ - else - htp_server->web_token_expire_time = web_token_expire_time; - } - seaf_message ("fileserver: web_token_expire_time = %d\n", - htp_server->web_token_expire_time); - - max_indexing_threads = fileserver_config_get_integer (session->config, - "max_indexing_threads", - &error); - if (error) { - htp_server->max_indexing_threads = DEFAULT_MAX_INDEXING_THREADS; - g_clear_error (&error); - } else { - if (max_indexing_threads <= 0) - htp_server->max_indexing_threads = DEFAULT_MAX_INDEXING_THREADS; - else - htp_server->max_indexing_threads = max_indexing_threads; - } - seaf_message ("fileserver: max_indexing_threads = %d\n", - htp_server->max_indexing_threads); - - max_index_processing_threads = fileserver_config_get_integer (session->config, - "max_index_processing_threads", - &error); - if (error) { - htp_server->max_index_processing_threads = DEFAULT_MAX_INDEX_PROCESSING_THREADS; - g_clear_error (&error); - } else { - if (max_index_processing_threads <= 0) - htp_server->max_index_processing_threads = DEFAULT_MAX_INDEX_PROCESSING_THREADS; - else - htp_server->max_index_processing_threads = max_index_processing_threads; - } - seaf_message ("fileserver: max_index_processing_threads= %d\n", - htp_server->max_index_processing_threads); - - cluster_shared_temp_file_mode = fileserver_config_get_string (session->config, - "cluster_shared_temp_file_mode", - &error); - if (error) { - htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE; - g_clear_error (&error); - } else { - if (!cluster_shared_temp_file_mode) { - htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE; - } else { - htp_server->cluster_shared_temp_file_mode = strtol(cluster_shared_temp_file_mode, NULL, 8); - - if (htp_server->cluster_shared_temp_file_mode < 0001 || - htp_server->cluster_shared_temp_file_mode > 0777) - htp_server->cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE; - - g_free (cluster_shared_temp_file_mode); - } - } - seaf_message ("fileserver: cluster_shared_temp_file_mode = %o\n", - htp_server->cluster_shared_temp_file_mode); - - encoding = g_key_file_get_string (session->config, - "zip", "windows_encoding", - &error); - if (encoding) { - htp_server->windows_encoding = encoding; - } else { - g_clear_error (&error); - /* No windows specific encoding is specified. Set the ZIP_UTF8 flag. */ - setlocale (LC_ALL, "en_US.UTF-8"); - } -} - static int validate_token (HttpServer *htp_server, evhtp_request_t *req, const char *repo_id, char **username, @@ -2213,20 +2058,26 @@ http_server_run (void *arg) { HttpServerStruct *server = arg; HttpServer *priv = server->priv; + char *bind_addr = fileserver_config_get_string (seaf->cfg_mgr, seaf->config, HOST); + int bind_port = fileserver_config_get_integer (seaf->cfg_mgr, seaf->config, PORT); + int worker_threads = fileserver_config_get_integer (seaf->cfg_mgr, seaf->config, "worker_threads"); + if (worker_threads <= 0) + worker_threads = DEFAULT_WORKER_THREADS; priv->evbase = event_base_new(); priv->evhtp = evhtp_new(priv->evbase, NULL); if (evhtp_bind_socket(priv->evhtp, - server->bind_addr, - server->bind_port, 128) < 0) { + bind_addr, + bind_port, 128) < 0) { seaf_warning ("Could not bind socket: %s\n", strerror (errno)); exit(-1); } + g_free (bind_addr); http_request_init (server); - evhtp_use_threads (priv->evhtp, NULL, server->worker_threads, NULL); + evhtp_use_threads (priv->evhtp, NULL, worker_threads, NULL); struct timeval tv; tv.tv_sec = CLEANING_INTERVAL_SEC; @@ -2252,8 +2103,6 @@ seaf_http_server_new (struct _SeafileSession *session) priv->evbase = NULL; priv->evhtp = NULL; - load_http_config (server, session); - priv->token_cache = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, token_cache_value_free); pthread_mutex_init (&priv->token_cache_lock, NULL); diff --git a/server/http-server.h b/server/http-server.h index 3d725e4d..7d88aa7e 100644 --- a/server/http-server.h +++ b/server/http-server.h @@ -12,16 +12,7 @@ struct _HttpServerStruct { struct _HttpServer *priv; - char *bind_addr; - int bind_port; char *http_temp_dir; /* temp dir for file upload */ - char *windows_encoding; - gint64 fixed_block_size; - int web_token_expire_time; - int max_indexing_threads; - int worker_threads; - int max_index_processing_threads; - int cluster_shared_temp_file_mode; }; typedef struct _HttpServerStruct HttpServerStruct; diff --git a/server/index-blocks-mgr.c b/server/index-blocks-mgr.c index 6fa48474..bbc95c68 100644 --- a/server/index-blocks-mgr.c +++ b/server/index-blocks-mgr.c @@ -18,10 +18,12 @@ #include "seafile-error.h" #include "seafile-crypt.h" #include "index-blocks-mgr.h" +#include "fileserver-config.h" #define TOKEN_LEN 36 #define PROGRESS_TTL 5 * 3600 // 5 hours #define SCAN_PROGRESS_INTERVAL 24 * 3600 // 1 day +#define DEFAULT_MAX_INDEX_PROCESSING_THREADS 3 static void start_index_task (gpointer data, gpointer user_data); @@ -71,10 +73,16 @@ index_blocks_mgr_new (SeafileSession *session) GError *error = NULL; IndexBlksMgr *mgr = g_new0 (IndexBlksMgr, 1); IndexBlksMgrPriv *priv = g_new0 (IndexBlksMgrPriv, 1); + int max_index_processing_threads = fileserver_config_get_integer (session->cfg_mgr, + session->config, + "max_index_processing_threads"); + + if (max_index_processing_threads <= 0) + max_index_processing_threads = DEFAULT_MAX_INDEX_PROCESSING_THREADS; priv->idx_tpool = g_thread_pool_new (start_index_task, priv, - session->http_server->max_index_processing_threads, + max_index_processing_threads, FALSE, &error); if (!priv->idx_tpool) { if (error) { diff --git a/server/pack-dir.c b/server/pack-dir.c index 7763d187..3daf811f 100644 --- a/server/pack-dir.c +++ b/server/pack-dir.c @@ -91,6 +91,7 @@ add_file_to_archive (PackDirData *data, char *dec_out = NULL; int dec_out_len = -1; int ret = 0; + char *windows_encoding = NULL; pathname = g_build_filename (top_dir_name, parent_dir, dent->name, NULL); @@ -105,13 +106,14 @@ add_file_to_archive (PackDirData *data, entry = archive_entry_new (); /* File name fixup for WinRAR */ - if (is_windows && seaf->http_server->windows_encoding) { + windows_encoding = seaf_cfg_manager_get_config_string (seaf->cfg_mgr, "zip", "windows_encoding"); + if (is_windows && windows_encoding) { char *win_file_name = do_iconv ("UTF-8", - seaf->http_server->windows_encoding, + windows_encoding, pathname); if (!win_file_name) { seaf_warning ("Failed to convert file name to %s\n", - seaf->http_server->windows_encoding); + windows_encoding); ret = -1; goto out; } @@ -281,6 +283,7 @@ archive_dir (PackDirData *data, GList *ptr; char *subpath = NULL; int ret = 0; + char *windows_encoding = NULL; dir = seaf_fs_manager_get_seafdir (seaf->fs_mgr, data->store_id, data->repo_version, @@ -294,13 +297,14 @@ archive_dir (PackDirData *data, struct archive_entry *entry = archive_entry_new (); gboolean is_windows = data->is_windows; - if (is_windows && seaf->http_server->windows_encoding) { + windows_encoding = seaf_cfg_manager_get_config_string (seaf->cfg_mgr, "zip", "windows_encoding"); + if (is_windows && windows_encoding) { char *win_file_name = do_iconv ("UTF-8", - seaf->http_server->windows_encoding, + windows_encoding, pathname); if (!win_file_name) { seaf_warning ("Failed to convert file name to %s\n", - seaf->http_server->windows_encoding); + windows_encoding); ret = -1; goto out; } diff --git a/server/quota-mgr.c b/server/quota-mgr.c index 373de296..e93ee00a 100644 --- a/server/quota-mgr.c +++ b/server/quota-mgr.c @@ -28,7 +28,8 @@ get_default_quota (SeafCfgManager *mgr) gint64 quota; quota_str = seaf_cfg_manager_get_config_string (mgr, "quota", "default"); - if (!quota_str) + seaf_message ("Mydebug quota_str is %s\n", quota_str); + if (strcmp (quota_str, "-2") == 0) return INFINITE_QUOTA; quota_int = strtoll (quota_str, &end, 10); @@ -216,6 +217,7 @@ seaf_quota_manager_get_user_quota (SeafQuotaManager *mgr, if (quota <= 0) quota = get_default_quota (seaf->cfg_mgr); + seaf_message ("Mydebug get user quota %ld\n", quota); return quota; } diff --git a/server/repo-mgr.c b/server/repo-mgr.c index a6a5c204..b3f07384 100644 --- a/server/repo-mgr.c +++ b/server/repo-mgr.c @@ -293,18 +293,11 @@ scan_trash (void *data) } static void -init_scan_trash_timer (SeafRepoManagerPriv *priv, GKeyFile *config) +init_scan_trash_timer (SeafRepoManagerPriv *priv, SeafileSession *seaf) { - int scan_days; - GError *error = NULL; - - scan_days = g_key_file_get_integer (config, - "library_trash", "scan_days", - &error); - if (error) { - scan_days = SCAN_TRASH_DAYS; - g_clear_error (&error); - } + int scan_days = seaf_cfg_manager_get_config_int (seaf->cfg_mgr, + "library_trash", + "scan_days"); priv->scan_trash_timer = ccnet_timer_new (scan_trash, NULL, scan_days * 24 * 3600 * 1000); @@ -325,7 +318,7 @@ seaf_repo_manager_new (SeafileSession *seaf) mgr->priv->reap_token_timer = ccnet_timer_new (reap_token, mgr, REAP_TOKEN_INTERVAL * 1000); - init_scan_trash_timer (mgr->priv, seaf->config); + init_scan_trash_timer (mgr->priv, seaf); return mgr; } diff --git a/server/repo-op.c b/server/repo-op.c index 427155da..d7376494 100644 --- a/server/repo-op.c +++ b/server/repo-op.c @@ -2243,6 +2243,7 @@ cross_repo_copy (const char *src_repo_id, } else { if (task && !task->canceled) { task->failed = TRUE; + seaf_message ("Mydebug failed reason is %s\n", err_str); set_failed_reason (&(task->failed_reason), err_str); } } @@ -2268,9 +2269,13 @@ check_file_count_and_size (SeafRepo *repo, SeafDirent *dent, gint64 total_files, { gint64 total_file_size = 0; gint64 size = -1; + gint64 max_files = seaf_cfg_manager_get_config_int64 (seaf->cfg_mgr, "web_copy", "max_files"); + gint64 max_size = seaf_cfg_manager_get_config_int64 (seaf->cfg_mgr, "web_copy", "max_size"); + /* size is given in MB */ + max_size <<= 20; - if (seaf->copy_mgr->max_files > 0 && - total_files > seaf->copy_mgr->max_files) { + if (max_files > 0 && + total_files > max_files) { *err_str = COPY_ERR_TOO_MANY_FILES; seaf_warning("Failed to copy/move file from repo %.8s: Too many files\n", repo->id); return FALSE; @@ -2303,8 +2308,8 @@ check_file_count_and_size (SeafRepo *repo, SeafDirent *dent, gint64 total_files, total_file_size = *total_size_all; } - if (seaf->copy_mgr->max_size > 0) { - if (total_file_size > seaf->copy_mgr->max_size) { + if (max_size > 0) { + if (total_file_size > max_size) { *err_str = COPY_ERR_SIZE_TOO_LARGE; seaf_warning("Failed to copy/move file from repo %.8s: " "Folder or file size is too large.\n", repo->id); diff --git a/server/seafile-session.c b/server/seafile-session.c index fcf9355b..5ece25dc 100644 --- a/server/seafile-session.c +++ b/server/seafile-session.c @@ -95,6 +95,7 @@ seafile_session_new(const char *central_config_dir, session->cfg_mgr = seaf_cfg_manager_new (session); if (!session->cfg_mgr) goto onerror; + session->fs_mgr = seaf_fs_manager_new (session, abs_seafile_dir); if (!session->fs_mgr) goto onerror; @@ -184,12 +185,6 @@ seafile_session_init (SeafileSession *session) return -1; } - if ((session->create_tables || seaf_db_type(session->db) == SEAF_DB_TYPE_PGSQL) - && seaf_cfg_manager_init (session->cfg_mgr) < 0) { - seaf_warning ("Failed to init config manager.\n"); - return -1; - } - return 0; } diff --git a/server/size-sched.c b/server/size-sched.c index 4058b70e..14054b84 100644 --- a/server/size-sched.c +++ b/server/size-sched.c @@ -52,9 +52,8 @@ size_scheduler_new (SeafileSession *session) sched->seaf = session; - sched_thread_num = g_key_file_get_integer (session->config, "scheduler", "size_sched_thread_num", NULL); - - if (sched_thread_num == 0) + sched_thread_num = seaf_cfg_manager_get_config_int (session->cfg_mgr, "scheduler", "size_sched_thread_num"); + if (sched_thread_num <= 0) sched_thread_num = DEFAULT_SCHEDULE_THREAD_NUMBER; sched->priv->compute_repo_size_thread_pool = g_thread_pool_new (compute_task, NULL, diff --git a/server/upload-file.c b/server/upload-file.c index 21f955a0..48d2b362 100755 --- a/server/upload-file.c +++ b/server/upload-file.c @@ -28,6 +28,7 @@ #include "http-server.h" #include "seafile-error.h" +#include "fileserver-config.h" enum RecvState { RECV_INIT, @@ -85,6 +86,7 @@ typedef struct RecvFSM { } RecvFSM; #define MAX_CONTENT_LINE 10240 +#define DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE 0600 static GHashTable *upload_progress; static pthread_mutex_t pg_lock; @@ -959,8 +961,14 @@ write_block_data_to_tmp_file (RecvFSM *fsm, const char *parent_dir, GError *error = NULL; int tmp_fd = -1; int ret = 0; - HttpServerStruct *htp_server = seaf->http_server; - int cluster_shared_temp_file_mode = htp_server->cluster_shared_temp_file_mode; + char *cluster_shared_temp_file_mode_str = fileserver_config_get_string (seaf->cfg_mgr, + seaf->config, + "fixed_block_size"); + int cluster_shared_temp_file_mode = strtol(cluster_shared_temp_file_mode_str, NULL, 8); + + if (cluster_shared_temp_file_mode < 0001 || + cluster_shared_temp_file_mode > 0777) + cluster_shared_temp_file_mode = DEFAULT_CLUSTER_SHARED_TEMP_FILE_MODE; abs_path = g_build_path ("/", parent_dir, file_name, NULL); @@ -2662,6 +2670,7 @@ upload_file_init (evhtp_t *htp, const char *http_temp_dir) seaf_warning ("Failed to create cluster shared dir %s.\n", cluster_shared_dir); g_free (cluster_shared_dir); + return -1; } g_free (cluster_shared_dir); diff --git a/server/web-accesstoken-mgr.c b/server/web-accesstoken-mgr.c index 08b27c05..c00d1f0d 100644 --- a/server/web-accesstoken-mgr.c +++ b/server/web-accesstoken-mgr.c @@ -13,6 +13,7 @@ #include "utils.h" #include "log.h" +#include "fileserver-config.h" #define CLEANING_INTERVAL_MSEC 1000*300 /* 5 minutes */ #define TOKEN_EXPIRE_TIME 3600 /* 1 hour */ @@ -136,6 +137,10 @@ seaf_web_at_manager_get_access_token (SeafWebAccessTokenManager *mgr, long expire; char *t; SeafileWebAccess *webaccess; + int web_token_expire_time = fileserver_config_get_integer (seaf->cfg_mgr, seaf->config, "web_token_expire_time"); + + if (web_token_expire_time <= 0) + web_token_expire_time = 3600; if (strcmp(op, "view") != 0 && strcmp(op, "download") != 0 && @@ -160,7 +165,7 @@ seaf_web_at_manager_get_access_token (SeafWebAccessTokenManager *mgr, pthread_mutex_lock (&mgr->priv->lock); t = gen_new_token (mgr->priv->access_token_hash); - expire = now + seaf->http_server->web_token_expire_time; + expire = now + web_token_expire_time; info = g_new0 (AccessInfo, 1); info->repo_id = g_strdup (repo_id); diff --git a/tests/test_file_operation/test_file_operation.py b/tests/test_file_operation/test_file_operation.py index b5539076..1f6fd469 100644 --- a/tests/test_file_operation/test_file_operation.py +++ b/tests/test_file_operation/test_file_operation.py @@ -48,7 +48,7 @@ def test_file_operation(): assert t_copy_file_result2 assert t_copy_file_result2.background while True: - time.sleep(0.1) + time.sleep(2) t_copy_task = api.get_copy_task(t_copy_file_result2.task_id) assert t_copy_task.failed assert t_copy_task.failed_reason == 'Quota is full' @@ -60,8 +60,9 @@ def test_file_operation(): assert t_copy_file_result2 assert t_copy_file_result2.task_id assert t_copy_file_result2.background + print ('Mydebug task_id = ', t_copy_file_result2.task_id) while True: - time.sleep(0.1) + time.sleep(2) t_copy_task = api.get_copy_task(t_copy_file_result2.task_id) if t_copy_task.successful: break; @@ -91,7 +92,7 @@ def test_file_operation(): assert t_move_file_result2.task_id assert t_move_file_result2.background while True: - time.sleep(0.1) + time.sleep(2) t_move_task = api.get_copy_task(t_move_file_result2.task_id) assert t_move_task.failed assert t_move_task.failed_reason == 'Quota is full' @@ -104,7 +105,7 @@ def test_file_operation(): assert t_move_file_result2.task_id assert t_move_file_result2.background while True: - time.sleep(0.1) + time.sleep(2) t_move_task = api.get_copy_task(t_move_file_result2.task_id) if t_move_task.successful: break