Skip to content

Commit

Permalink
Optimize the configure options.
Browse files Browse the repository at this point in the history
  • Loading branch information
ly1217 committed Oct 4, 2019
1 parent a33525c commit daa129f
Show file tree
Hide file tree
Showing 23 changed files with 255 additions and 273 deletions.
167 changes: 129 additions & 38 deletions common/config-mgr.c
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,35 @@
#include "config-mgr.h"
#include "seaf-db.h"
#include "log.h"
#include "utils.h"
#include <locale.h>

/* These configurations in this table are displayed as */
/* "group:option:default_value:property" format . */
const static char *config_table [] = {
"quota:default:-2:0",
"history:keep_days:-1:0",
"fileserver:max_upload_size:-1:0",
"fileserver:max_download_dir_size:100:0",
"fileserver:host:0.0.0.0:1",
"fileserver:port:8082:1",
"fileserver:worker_threads:10:1",
"fileserver:fixed_block_size:8:1",
"fileserver:web_token_expire_time:8:1",
"fileserver:max_indexing_threads:1:1",
"fileserver:max_index_processing_threads:3:1",
"fileserver:cluster_shared_temp_file_mode:0600:1",
"library_trash:expire_days:30:0",
"library_trash:scan_days:1:1",
"web_copy:max_files:0:1",
"web_copy:max_size:0:1",
"scheduler:size_sched_thread_num:1:1",
"zip:windows_encoding:none:1",
"general:enable_syslog:false:1",
"fuse:excluded_users:none:1",
"t_group:t_key:0:0",
NULL
};

int
seaf_cfg_manager_init (SeafCfgManager *mgr)
Expand All @@ -23,6 +52,54 @@ seaf_cfg_manager_init (SeafCfgManager *mgr)
return 0;
}

static void load_config_option (SeafCfgManager *mgr, char **option_item)
{
char *group = NULL, *key = NULL,
*property = NULL, *default_value = NULL,
*cache_key = NULL, *cache_value = NULL,
*sql = NULL, *value = NULL;

group = option_item[0];
key = option_item[1];
default_value = option_item[2];
property = option_item[3];

sql = "SELECT value FROM SeafileConf WHERE cfg_group=? AND cfg_key=?";
value = seaf_db_statement_get_string(mgr->db, sql,
2, "string", group, "string", key);
if (value) {
value = g_strstrip(value);
} else {
value = seaf_key_file_get_string (mgr->config, group, key, NULL);
if (!value) {
if (strcmp ("zip", group) == 0 &&
strcmp ("windows_encoding", key) == 0) {
setlocale (LC_ALL, "en_US.UTF-8");
}
value = g_strdup (default_value);
}
}

cache_key = g_strdup_printf ("%s,%s", group, key);
cache_value = g_strdup_printf ("%s,%s", value, property);
g_free (value);

g_hash_table_insert (mgr->config_cache, cache_key, cache_value);
}

static void load_config_cache (SeafCfgManager *mgr)
{
int index = 0;
char **option_item = NULL;

while (config_table[index]) {
option_item = g_strsplit (config_table[index], ":", -1);
load_config_option (mgr, option_item);
g_strfreev (option_item);
index++;
}
}

SeafCfgManager *
seaf_cfg_manager_new (SeafileSession *session)
{
Expand All @@ -32,6 +109,13 @@ seaf_cfg_manager_new (SeafileSession *session)

mgr->config = session->config;
mgr->db = session->db;
if ((session->create_tables || seaf_db_type(session->db) == SEAF_DB_TYPE_PGSQL)
&& seaf_cfg_manager_init (session->cfg_mgr) < 0) {
seaf_warning ("Failed to init config manager.\n");
return NULL;
}
mgr->config_cache = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
load_config_cache (mgr);

return mgr;
}
Expand Down Expand Up @@ -95,14 +179,27 @@ int
seaf_cfg_manager_set_config (SeafCfgManager *mgr, const char *group, const char *key, const char *value)
{
gboolean exists, err = FALSE;
char *cache_key = NULL, *cache_value = NULL, *property = NULL;
char **tokens = NULL;
int ret = 0;

cache_key = g_strdup_printf ("%s,%s" ,group, key);
cache_value = g_hash_table_lookup (mgr->config_cache, cache_key);
tokens = g_strsplit (cache_value, ",", -1);
property = tokens[1];

if (strcmp (property, "0") == 0) {
cache_value = g_strdup_printf ("%s,%s" ,value, property);
g_hash_table_insert (mgr->config_cache, cache_key, cache_value);
}

char *sql = "SELECT 1 FROM SeafileConf WHERE cfg_group=? AND cfg_key=?";
exists = seaf_db_statement_exists(mgr->db, sql, &err,
2, "string", group,
"string", key);
if (err) {
seaf_warning ("[db error]Failed to set config [%s:%s] to db.\n", group, key);
return -1;
ret = -1;
}
if (exists)
sql = "UPDATE SeafileConf SET value=? WHERE cfg_group=? AND cfg_key=?";
Expand All @@ -113,10 +210,12 @@ seaf_cfg_manager_set_config (SeafCfgManager *mgr, const char *group, const char
"string", value, "string",
group, "string", key) < 0) {
seaf_warning ("Failed to set config [%s:%s] to db.\n", group, key);
return -1;
ret = -1;
}

return 0;
g_strfreev (tokens);

return ret;
}

int
Expand All @@ -127,12 +226,8 @@ seaf_cfg_manager_get_config_int (SeafCfgManager *mgr, const char *group, const c

char *value = seaf_cfg_manager_get_config (mgr, group, key);
if (!value) {
GError *err = NULL;
ret = g_key_file_get_integer (mgr->config, group, key, &err);
if (err) {
ret = -1;
g_clear_error(&err);
}
seaf_warning ("Config [%s:%s] not set, default is -1.\n", group, key);
ret = -1;
} else {
ret = strtol (value, &invalid, 10);
if (*invalid != '\0') {
Expand All @@ -153,12 +248,8 @@ seaf_cfg_manager_get_config_int64 (SeafCfgManager *mgr, const char *group, const

char *value = seaf_cfg_manager_get_config (mgr, group, key);
if (!value) {
GError *err = NULL;
ret = g_key_file_get_int64(mgr->config, group, key, &err);
if (err) {
ret = -1;
g_clear_error(&err);
}
seaf_warning ("Config [%s:%s] not set, default is -1.\n", group, key);
ret = -1;
} else {
ret = strtoll (value, &invalid, 10);
if (*invalid != '\0') {
Expand All @@ -178,13 +269,8 @@ seaf_cfg_manager_get_config_boolean (SeafCfgManager *mgr, const char *group, con

char *value = seaf_cfg_manager_get_config (mgr, group, key);
if (!value) {
GError *err = NULL;
ret = g_key_file_get_boolean(mgr->config, group, key, &err);
if (err) {
seaf_warning ("Config [%s:%s] not set, default is false.\n", group, key);
ret = FALSE;
g_clear_error(&err);
}
seaf_warning ("Config [%s:%s] not set, default is false.\n", group, key);
ret = FALSE;
} else {
if (strcmp ("true", value) == 0)
ret = TRUE;
Expand All @@ -199,28 +285,33 @@ seaf_cfg_manager_get_config_boolean (SeafCfgManager *mgr, const char *group, con
char *
seaf_cfg_manager_get_config_string (SeafCfgManager *mgr, const char *group, const char *key)
{
char *ret = NULL;

char *value = seaf_cfg_manager_get_config (mgr, group, key);
if (!value) {
ret = g_key_file_get_string (mgr->config, group, key, NULL);
if (ret != NULL)
ret = g_strstrip(ret);
} else {
ret = value;
}

return ret;
if (!value)
seaf_warning ("Config [%s:%s] not set, default is NULL.\n", group, key);

return value;
}

char *
seaf_cfg_manager_get_config (SeafCfgManager *mgr, const char *group, const char *key)
{
char *sql = "SELECT value FROM SeafileConf WHERE cfg_group=? AND cfg_key=?";
char *value = seaf_db_statement_get_string(mgr->db, sql,
2, "string", group, "string", key);
if (value != NULL)
value = g_strstrip(value);
char *ret = NULL, *cache_key = NULL,
*cache_value = NULL, *option_value = NULL;
char **tokens = NULL;

return value;
cache_key = g_strdup_printf ("%s,%s", group, key);
cache_value = g_hash_table_lookup (mgr->config_cache, cache_key);
tokens = g_strsplit(cache_value, ",", -1);
option_value = tokens[0];

if (strcmp (option_value, "none") == 0)
ret = NULL;
else
ret = g_strdup (option_value);

g_free (cache_key);
g_strfreev (tokens);

return ret;
}
1 change: 1 addition & 0 deletions common/config-mgr.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ typedef struct _SeafCfgManager SeafCfgManager;
struct _SeafCfgManager {
GKeyFile *config;
SeafDB *db;
GHashTable *config_cache;
};

typedef struct _SeafileSession SeafileSession;
Expand Down
36 changes: 31 additions & 5 deletions common/fs-mgr.c
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,11 @@
#endif /* SEAFILE_SERVER */

#include "db.h"
#include "../server/fileserver-config.h"

#define SEAF_TMP_EXT "~"
#define DEFAULT_FIXED_BLOCK_SIZE ((gint64)1 << 23) /* 8MB */
#define DEFAULT_MAX_INDEXING_THREADS 1

struct _SeafFSManagerPriv {
/* GHashTable *seafile_cache; */
Expand Down Expand Up @@ -612,6 +615,13 @@ chunking_worker (gpointer vdata, gpointer user_data)
int fd = -1;
ssize_t n;
int idx;
gint64 fixed_block_size;
int fixed_block_size_mb = fileserver_config_get_integer (seaf->cfg_mgr, seaf->config, "fixed_block_size");

if (fixed_block_size_mb <= 0)
fixed_block_size = DEFAULT_FIXED_BLOCK_SIZE;
else
fixed_block_size = fixed_block_size_mb * ((gint64)1 << 20);

chunk->block_buf = g_new0 (char, chunk->len);
if (!chunk->block_buf) {
Expand Down Expand Up @@ -646,7 +656,7 @@ chunking_worker (gpointer vdata, gpointer user_data)
if (chunk->result < 0)
goto out;

idx = chunk->offset / seaf->http_server->fixed_block_size;
idx = chunk->offset / fixed_block_size;
memcpy (data->blk_sha1s + idx * CHECKSUM_LENGTH, chunk->checksum, CHECKSUM_LENGTH);

out:
Expand All @@ -673,8 +683,24 @@ split_file_to_block (const char *repo_id,
int n_pending = 0;
CDCDescriptor *chunk;
int ret = 0;
gint64 fixed_block_size;
int fixed_block_size_mb = fileserver_config_get_integer (seaf->cfg_mgr,
seaf->config,
"fixed_block_size");
if (fixed_block_size_mb <= 0)
fixed_block_size = DEFAULT_FIXED_BLOCK_SIZE;
else
fixed_block_size = fixed_block_size_mb * ((gint64)1 << 20);

int max_indexing_threads = fileserver_config_get_integer (seaf->cfg_mgr,
seaf->config,
"max_indexing_threads");
if (max_indexing_threads <= 0)
max_indexing_threads = DEFAULT_MAX_INDEXING_THREADS;


n_blocks = (file_size + fixed_block_size - 1) / fixed_block_size;

n_blocks = (file_size + seaf->http_server->fixed_block_size - 1) / seaf->http_server->fixed_block_size;
block_sha1s = g_new0 (uint8_t, n_blocks * CHECKSUM_LENGTH);
if (!block_sha1s) {
seaf_warning ("Failed to allocate block_sha1s.\n");
Expand All @@ -694,7 +720,7 @@ split_file_to_block (const char *repo_id,
data.finished_tasks = finished_tasks;

tpool = g_thread_pool_new (chunking_worker, &data,
seaf->http_server->max_indexing_threads, FALSE, NULL);
max_indexing_threads, FALSE, NULL);
if (!tpool) {
seaf_warning ("Failed to allocate thread pool\n");
ret = -1;
Expand All @@ -705,7 +731,7 @@ split_file_to_block (const char *repo_id,
guint64 len;
guint64 left = (guint64)file_size;
while (left > 0) {
len = ((left >= seaf->http_server->fixed_block_size) ? seaf->http_server->fixed_block_size : left);
len = ((left >= fixed_block_size) ? fixed_block_size : left);

chunk = g_new0 (CDCDescriptor, 1);
chunk->offset = offset;
Expand All @@ -725,7 +751,7 @@ split_file_to_block (const char *repo_id,
goto out;
}
if (indexed)
*indexed += seaf->http_server->fixed_block_size;
*indexed += fixed_block_size;

if ((--n_pending) <= 0) {
if (indexed)
Expand Down
2 changes: 2 additions & 0 deletions common/rpc-service.c
Original file line number Diff line number Diff line change
Expand Up @@ -2768,6 +2768,7 @@ seafile_copy_file (const char *src_repo_id,
int synchronous,
GError **error)
{
seaf_message ("Mydebug seafile_copy_file\n");
char *norm_src_dir = NULL, *norm_src_filename = NULL;
char *norm_dst_dir = NULL, *norm_dst_filename = NULL;
char *rsrc_dir = NULL, *rdst_dir = NULL;
Expand Down Expand Up @@ -2926,6 +2927,7 @@ seafile_move_file (const char *src_repo_id,

GObject *
seafile_get_copy_task (const char *task_id, GError **error)

{
return (GObject *)seaf_copy_manager_get_task (seaf->copy_mgr, task_id);
}
Expand Down
3 changes: 2 additions & 1 deletion fuse/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ seaf_fuse_SOURCES = seaf-fuse.c \
../common/obj-store.c \
../common/obj-backend-fs.c \
../common/obj-backend-riak.c \
../common/seafile-crypt.c
../common/seafile-crypt.c \
../common/config-mgr.c

seaf_fuse_LDADD = @CCNET_LIBS@ \
@GLIB2_LIBS@ @GOBJECT_LIBS@ @SSL_LIBS@ @LIB_RT@ @LIB_UUID@ \
Expand Down
6 changes: 5 additions & 1 deletion fuse/seafile-session.c
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ seafile_session_new(const char *central_config_dir,
goto onerror;
}

session->cfg_mgr = seaf_cfg_manager_new (session);
if (!session->cfg_mgr)
goto onerror;

if (read_excluded_users (session) < 0) {
seaf_warning ("Failed to load excluded users.\n");
goto onerror;
Expand Down Expand Up @@ -111,7 +115,7 @@ read_excluded_users (SeafileSession *session)
int l, i;
char *hash_value;

users = seaf_key_file_get_string (session->config, "fuse", "excluded_users", NULL);
users = seaf_cfg_manager_get_config_string (session->cfg_mgr, "fuse", "excluded_users");
if (!users)
return 0;

Expand Down
Loading

0 comments on commit daa129f

Please sign in to comment.