From af47c2cf0c0615e13fa36cef16dab20ca3d955c8 Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Sat, 1 Apr 2017 01:02:22 -0700 Subject: [PATCH 1/6] add key dumping and counting logic; enable commands on admin for twemcache --- src/protocol/admin/format.h | 2 + src/protocol/admin/parse.c | 11 ++++ src/protocol/admin/request.h | 6 ++- src/protocol/data/memcache/request.h | 2 +- src/server/twemcache/admin/process.c | 46 +++++++++++++++++ src/storage/slab/item.c | 75 ++++++++++++++++++++++++++++ src/storage/slab/item.h | 3 ++ 7 files changed, 142 insertions(+), 3 deletions(-) diff --git a/src/protocol/admin/format.h b/src/protocol/admin/format.h index 2d93459ba..33b02c698 100644 --- a/src/protocol/admin/format.h +++ b/src/protocol/admin/format.h @@ -8,6 +8,8 @@ #define METRIC_DESCRIBE_LEN 120 /* 34 (name) + 16 (type) + 68 (description) + CRLF */ #define METRIC_END "END\r\n" #define METRIC_END_LEN (sizeof(METRIC_END) - 1) +#define KEYCOUNT_FMT "%zu %zu %zu\r\n" +#define KEYCOUNT_LEN 64 /* 20 * 3 (numbers) + 2 (spaces) + CRLF */ #define VERSION_PRINTED "VERSION " VERSION_STRING "\r\n" diff --git a/src/protocol/admin/parse.c b/src/protocol/admin/parse.c index ecf40befb..0eb946698 100644 --- a/src/protocol/admin/parse.c +++ b/src/protocol/admin/parse.c @@ -26,6 +26,7 @@ _get_req_type(struct request *req, struct bstring *type) { ASSERT(req->type == REQ_UNKNOWN); + /* use loop + bcmp() to simplify this function, perf doesn't matter */ switch (type->len) { case 4: if (str4cmp(type->data, 'q', 'u', 'i', 't')) { @@ -33,6 +34,11 @@ _get_req_type(struct request *req, struct bstring *type) break; } + if (str4cmp(type->data, 'd', 'u', 'm', 'p')) { + req->type = REQ_DUMP; + break; + } + break; case 5: @@ -41,6 +47,11 @@ _get_req_type(struct request *req, struct bstring *type) break; } + if (str5cmp(type->data, 'c', 'o', 'u', 'n', 't')) { + req->type = REQ_COUNT; + break; + } + break; case 7: diff --git a/src/protocol/admin/request.h b/src/protocol/admin/request.h index 866b9bb2d..60f69deba 100644 --- a/src/protocol/admin/request.h +++ b/src/protocol/admin/request.h @@ -25,6 +25,8 @@ ACTION( REQ_UNKNOWN, "" )\ ACTION( REQ_STATS, "stats" )\ ACTION( REQ_VERSION, "version" )\ + ACTION( REQ_COUNT, "count" )\ + ACTION( REQ_DUMP, "dump" )\ ACTION( REQ_QUIT, "quit" ) #define GET_TYPE(_name, _str) _name, @@ -41,8 +43,8 @@ typedef enum request_state { } request_state_t; struct request { - request_state_t state; /* request state */ - request_type_t type; + request_state_t state; /* request state */ + request_type_t type; struct bstring arg; }; diff --git a/src/protocol/data/memcache/request.h b/src/protocol/data/memcache/request.h index f67e8adbe..2bd472cb8 100644 --- a/src/protocol/data/memcache/request.h +++ b/src/protocol/data/memcache/request.h @@ -48,7 +48,7 @@ typedef struct { ACTION( REQ_INCR, "incr " )\ ACTION( REQ_DECR, "decr " )\ ACTION( REQ_FLUSH, "flush_all\r\n" )\ - ACTION( REQ_QUIT, "quit\r\n" )\ + ACTION( REQ_QUIT, "quit\r\n" ) #define GET_TYPE(_name, _str) _name, typedef enum request_type { diff --git a/src/server/twemcache/admin/process.c b/src/server/twemcache/admin/process.c index 7b8bcfa48..54fa9f4e8 100644 --- a/src/server/twemcache/admin/process.c +++ b/src/server/twemcache/admin/process.c @@ -96,6 +96,42 @@ _admin_stats(struct response *rsp, struct request *req) } } +static void +_key_dump(struct response *rsp, struct request *req) +{ + if (item_dump() == true) { + rsp->type = RSP_OK; + } else { + rsp->type = RSP_GENERIC; + rsp->data = str2bstr("ERROR: key dump unsuccesful"); + } + + log_info("dump request %p processed"); +} + +static void +_key_count(struct response *rsp, struct request *req) +{ + size_t nkey, ksize, vsize; + int ret = 0; + + if (req->arg.len > 0) { /* skip initial space */ + req->arg.len--; + req->arg.data++; + } + log_info("count keys with prefix %.*s", req->arg.len, req->arg.data); + + item_count(&nkey, &ksize, &vsize, &req->arg); + rsp->type = RSP_GENERIC; + ret = cc_scnprintf(buf, cap, KEYCOUNT_FMT, nkey, ksize, vsize); + if (ret < 0) { + rsp->data = str2bstr("ERROR: cannot format key count"); + } else { + rsp->data.len = ret; + rsp->data.data = buf; + } +} + void admin_process_request(struct response *rsp, struct request *req) { @@ -105,9 +141,19 @@ admin_process_request(struct response *rsp, struct request *req) case REQ_STATS: _admin_stats(rsp, req); break; + case REQ_VERSION: rsp->data = str2bstr(VERSION_PRINTED); break; + + case REQ_DUMP: + _key_dump(rsp, req); + break; + + case REQ_COUNT: + _key_count(rsp, req); + break; + default: rsp->type = RSP_INVALID; break; diff --git a/src/storage/slab/item.c b/src/storage/slab/item.c index 8a1941b63..7abd4e00d 100644 --- a/src/storage/slab/item.c +++ b/src/storage/slab/item.c @@ -1,7 +1,9 @@ #include "slab.h" #include +#include +#include #include #include @@ -379,3 +381,76 @@ item_flush(void) flush_at = time_proc_sec(); log_info("all keys flushed at %"PRIu32, flush_at); } + +void +item_count(size_t *nkey, size_t *ksize, size_t *vsize, struct bstring *prefix) +{ + uint32_t nbucket = HASHSIZE(hash_table->hash_power); + + log_info("start scanning all %"PRIu32" keys", hash_table->nhash_item); + + *nkey = 0; + *ksize = 0; + *vsize = 0; + for (uint32_t i = 0; i < nbucket; i++) { + struct item_slh *entry = &hash_table->table[i]; + struct item *it; + + SLIST_FOREACH(it, entry, i_sle) { + if (it->klen >= prefix->len && + cc_bcmp(prefix->data, item_key(it), prefix->len) == 0) { + *nkey += 1; + *ksize += it->klen; + *vsize += it->vlen; + } + } + + if (i % 1000000 == 0) { + log_info("... %"PRIu32" out of %"PRIu32" buckets scanned ...", i, + nbucket); + } + } + + log_info("finish scanning all keys"); +} + +bool +item_dump(void) +{ + int fd; + uint32_t nbucket = HASHSIZE(hash_table->hash_power); + + log_info("start scanning all %"PRIu32" keys", hash_table->nhash_item); + + fd = open("key.dump", O_WRONLY | O_TRUNC | O_CREAT, 0644); + if (fd < 0) { + log_stderr("Could not create key dump - cannot open file"); + return false; + } + + for (uint32_t i = 0; i < nbucket; i++) { + struct item_slh *entry = &hash_table->table[i]; + struct item *it; + + SLIST_FOREACH(it, entry, i_sle) { + if (write(fd, item_key(it), it->klen) < it->klen) { + log_error("write error, aborting at hash bucket %"PRIu32, i); + return false; + } + if (write(fd, CRLF, CRLF_LEN) < CRLF_LEN) { + log_error("write error, aborting at hash bucket %"PRIu32, i); + return false; + } + } + + if (i % 1000000 == 0) { + log_info("... %"PRIu32" out of %"PRIu32" buckets scanned ...", i, + nbucket); + } + } + close(fd); + + log_info("finish scanning all keys"); + + return true; +} diff --git a/src/storage/slab/item.h b/src/storage/slab/item.h index 3f1e38832..2a5fb1f18 100644 --- a/src/storage/slab/item.h +++ b/src/storage/slab/item.h @@ -233,3 +233,6 @@ bool item_delete(const struct bstring *key); /* flush the cache */ void item_flush(void); + +void item_count(size_t *nkey, size_t *ksize, size_t *vsize, struct bstring *prefix); +bool item_dump(void); From b55fd0967d0990bade57fc8d36eeeaa00f91294f Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Fri, 26 Oct 2018 20:36:50 -0700 Subject: [PATCH 2/6] let dump command take a prefix as well --- src/server/twemcache/admin/process.c | 8 +++++++- src/storage/slab/item.c | 20 ++++++++++++-------- src/storage/slab/item.h | 4 +++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/server/twemcache/admin/process.c b/src/server/twemcache/admin/process.c index 54fa9f4e8..1e115187b 100644 --- a/src/server/twemcache/admin/process.c +++ b/src/server/twemcache/admin/process.c @@ -99,7 +99,13 @@ _admin_stats(struct response *rsp, struct request *req) static void _key_dump(struct response *rsp, struct request *req) { - if (item_dump() == true) { + if (req->arg.len > 0) { /* skip initial space */ + req->arg.len--; + req->arg.data++; + } + log_info("dump keys with prefix %.*s", req->arg.len, req->arg.data); + + if (item_dump(&req->arg) == true) { rsp->type = RSP_OK; } else { rsp->type = RSP_GENERIC; diff --git a/src/storage/slab/item.c b/src/storage/slab/item.c index 7abd4e00d..908f8f4f5 100644 --- a/src/storage/slab/item.c +++ b/src/storage/slab/item.c @@ -382,6 +382,7 @@ item_flush(void) log_info("all keys flushed at %"PRIu32, flush_at); } +/* this dumps all keys (matching a prefix if given) regardless of expiry status */ void item_count(size_t *nkey, size_t *ksize, size_t *vsize, struct bstring *prefix) { @@ -415,7 +416,7 @@ item_count(size_t *nkey, size_t *ksize, size_t *vsize, struct bstring *prefix) } bool -item_dump(void) +item_dump(struct bstring *prefix) { int fd; uint32_t nbucket = HASHSIZE(hash_table->hash_power); @@ -433,13 +434,16 @@ item_dump(void) struct item *it; SLIST_FOREACH(it, entry, i_sle) { - if (write(fd, item_key(it), it->klen) < it->klen) { - log_error("write error, aborting at hash bucket %"PRIu32, i); - return false; - } - if (write(fd, CRLF, CRLF_LEN) < CRLF_LEN) { - log_error("write error, aborting at hash bucket %"PRIu32, i); - return false; + if (it->klen >= prefix->len && + cc_bcmp(prefix->data, item_key(it), prefix->len) == 0) { + if (write(fd, item_key(it), it->klen) < it->klen) { + log_error("write error, aborting at hash bucket %"PRIu32, i); + return false; + } + if (write(fd, CRLF, CRLF_LEN) < CRLF_LEN) { + log_error("write error, aborting at hash bucket %"PRIu32, i); + return false; + } } } diff --git a/src/storage/slab/item.h b/src/storage/slab/item.h index 2a5fb1f18..39f26e133 100644 --- a/src/storage/slab/item.h +++ b/src/storage/slab/item.h @@ -234,5 +234,7 @@ bool item_delete(const struct bstring *key); /* flush the cache */ void item_flush(void); +/* this surveys all keys (matching a prefix if given) regardless of expiry status */ void item_count(size_t *nkey, size_t *ksize, size_t *vsize, struct bstring *prefix); -bool item_dump(void); +/* this dumps all keys (matching a prefix if given) regardless of expiry status */ +bool item_dump(struct bstring *prefix); From 6386e3cc4e13c92b0dc8b51d3914b3b870d1ab49 Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Fri, 26 Oct 2018 22:07:56 -0700 Subject: [PATCH 3/6] rename admin command count to census, report min/max on key & value sizes in addition to counts & total sizes --- src/protocol/admin/format.h | 10 ++++++++-- src/protocol/admin/parse.c | 4 ++-- src/protocol/admin/request.h | 2 +- src/server/twemcache/admin/process.c | 18 ++++++++++-------- src/storage/slab/item.c | 28 ++++++++++++++++++++++------ src/storage/slab/item.h | 6 +++--- 6 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/protocol/admin/format.h b/src/protocol/admin/format.h index 33b02c698..502e194a9 100644 --- a/src/protocol/admin/format.h +++ b/src/protocol/admin/format.h @@ -8,8 +8,14 @@ #define METRIC_DESCRIBE_LEN 120 /* 34 (name) + 16 (type) + 68 (description) + CRLF */ #define METRIC_END "END\r\n" #define METRIC_END_LEN (sizeof(METRIC_END) - 1) -#define KEYCOUNT_FMT "%zu %zu %zu\r\n" -#define KEYCOUNT_LEN 64 /* 20 * 3 (numbers) + 2 (spaces) + CRLF */ +#define CENSUS_COUNT_FMT "item count: %zu %zu %zu\r\n" +#define CENSUS_COUNT_LEN 34 /* 12 (name string) + 20 + CRLF */ +#define CENSUS_KEY_FMT "key min: %zu, max: %zu, total: %zu\r\n" +#define CENSUS_KEY_LEN 87 /* 9 + 7 + 9 (name strings) + 20 * 3 + CRLF */ +#define CENSUS_VAL_FMT "val min: %zu, max: %zu, total: %zu\r\n" +#define CENSUS_VAL_LEN 87 /* 9 + 7 + 9 (name strings) + 20 * 3 + CRLF */ +#define CENSUS_FMT CENSUS_COUNT_FMT CENSUS_KEY_FMT CENSUS_VAL_FMT +#define CENSUS_LEN CENSUS_COUNT_LEN + CENSUS_KEY_LEN + CENSUS_VAL_LEN #define VERSION_PRINTED "VERSION " VERSION_STRING "\r\n" diff --git a/src/protocol/admin/parse.c b/src/protocol/admin/parse.c index 0eb946698..c3c196a9d 100644 --- a/src/protocol/admin/parse.c +++ b/src/protocol/admin/parse.c @@ -47,8 +47,8 @@ _get_req_type(struct request *req, struct bstring *type) break; } - if (str5cmp(type->data, 'c', 'o', 'u', 'n', 't')) { - req->type = REQ_COUNT; + if (str6cmp(type->data, 'c', 'e', 'n', 's', 'u', 's')) { + req->type = REQ_CENSUS; break; } diff --git a/src/protocol/admin/request.h b/src/protocol/admin/request.h index 60f69deba..cd1638641 100644 --- a/src/protocol/admin/request.h +++ b/src/protocol/admin/request.h @@ -25,7 +25,7 @@ ACTION( REQ_UNKNOWN, "" )\ ACTION( REQ_STATS, "stats" )\ ACTION( REQ_VERSION, "version" )\ - ACTION( REQ_COUNT, "count" )\ + ACTION( REQ_CENSUS, "census" )\ ACTION( REQ_DUMP, "dump" )\ ACTION( REQ_QUIT, "quit" ) diff --git a/src/server/twemcache/admin/process.c b/src/server/twemcache/admin/process.c index 1e115187b..09d93a5b5 100644 --- a/src/server/twemcache/admin/process.c +++ b/src/server/twemcache/admin/process.c @@ -31,6 +31,7 @@ admin_process_setup(void) } nmetric_perslab = METRIC_CARDINALITY(perslab[0]); + /* so far the largest response comes from per-slab metrics */ /* perslab metric size <(32 + 20)B, prefix/suffix 12B, total < 64 */ cap = MAX(nmetric, nmetric_perslab * SLABCLASS_MAX_ID) * METRIC_PRINT_LEN + METRIC_END_LEN; @@ -116,22 +117,23 @@ _key_dump(struct response *rsp, struct request *req) } static void -_key_count(struct response *rsp, struct request *req) +_key_census(struct response *rsp, struct request *req) { - size_t nkey, ksize, vsize; + size_t nkey, ktotal, vtotal, kmin, kmax, vmin, vmax; int ret = 0; if (req->arg.len > 0) { /* skip initial space */ req->arg.len--; req->arg.data++; } - log_info("count keys with prefix %.*s", req->arg.len, req->arg.data); + log_info("census on keys with prefix %.*s", req->arg.len, req->arg.data); - item_count(&nkey, &ksize, &vsize, &req->arg); + item_census(&nkey, &ktotal, &kmin, &kmax, &vtotal, &vmin, &vmax, &req->arg); rsp->type = RSP_GENERIC; - ret = cc_scnprintf(buf, cap, KEYCOUNT_FMT, nkey, ksize, vsize); + ret = cc_scnprintf(buf, cap, CENSUS_FMT, nkey, ktotal, kmin, kmax, vtotal, + vmin, vmax); if (ret < 0) { - rsp->data = str2bstr("ERROR: cannot format key count"); + rsp->data = str2bstr("ERROR: cannot format key census result"); } else { rsp->data.len = ret; rsp->data.data = buf; @@ -156,8 +158,8 @@ admin_process_request(struct response *rsp, struct request *req) _key_dump(rsp, req); break; - case REQ_COUNT: - _key_count(rsp, req); + case REQ_CENSUS: + _key_census(rsp, req); break; default: diff --git a/src/storage/slab/item.c b/src/storage/slab/item.c index 908f8f4f5..a4583df89 100644 --- a/src/storage/slab/item.c +++ b/src/storage/slab/item.c @@ -384,25 +384,37 @@ item_flush(void) /* this dumps all keys (matching a prefix if given) regardless of expiry status */ void -item_count(size_t *nkey, size_t *ksize, size_t *vsize, struct bstring *prefix) +item_census(size_t *nkey, size_t *ktotal, size_t *kmin, size_t *kmax, + size_t *vtotal, size_t *vmin, size_t *vmax, struct bstring *prefix) { uint32_t nbucket = HASHSIZE(hash_table->hash_power); + size_t klen, vlen; log_info("start scanning all %"PRIu32" keys", hash_table->nhash_item); *nkey = 0; - *ksize = 0; - *vsize = 0; + *ktotal = 0; + *vtotal = 0; + *kmin = SIZE_MAX; + *kmax = 0; + *vmin = SIZE_MAX; + *vmax = 0; for (uint32_t i = 0; i < nbucket; i++) { struct item_slh *entry = &hash_table->table[i]; struct item *it; SLIST_FOREACH(it, entry, i_sle) { - if (it->klen >= prefix->len && + klen = it->klen; + vlen = it->vlen; + if (klen >= prefix->len && cc_bcmp(prefix->data, item_key(it), prefix->len) == 0) { *nkey += 1; - *ksize += it->klen; - *vsize += it->vlen; + *ktotal += klen; + *vtotal += vlen; + *kmin = MIN(*kmin, klen); + *kmax = MAX(*kmax, klen); + *vmin = MIN(*vmin, vlen); + *vmax = MAX(*vmax, vlen); } } @@ -412,6 +424,10 @@ item_count(size_t *nkey, size_t *ksize, size_t *vsize, struct bstring *prefix) } } + if (*nkey == 0) { /* report 0 on all fields if no match has been found */ + *kmin = *vmin = 0; + } + log_info("finish scanning all keys"); } diff --git a/src/storage/slab/item.h b/src/storage/slab/item.h index 39f26e133..443f1da5b 100644 --- a/src/storage/slab/item.h +++ b/src/storage/slab/item.h @@ -39,10 +39,10 @@ * | | \ * \ | item_key() * item \ - * item->end, (if enabled) item_get_cas(), metadata + * item->end, (if used) optional metadata * * item->end is followed by: - * - 8-byte cas, if ITEM_CAS flag is set + * - optional metadata * - key * - data */ @@ -235,6 +235,6 @@ bool item_delete(const struct bstring *key); void item_flush(void); /* this surveys all keys (matching a prefix if given) regardless of expiry status */ -void item_count(size_t *nkey, size_t *ksize, size_t *vsize, struct bstring *prefix); +void item_census(size_t *nkey, size_t *ktotal, size_t *kmin, size_t *kmax, size_t *vtotal, size_t *vmin, size_t *vmax, struct bstring *prefix); /* this dumps all keys (matching a prefix if given) regardless of expiry status */ bool item_dump(struct bstring *prefix); From 116bcf715d9367b03be0706a7d30fb409372c982 Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Fri, 2 Nov 2018 17:42:39 -0700 Subject: [PATCH 4/6] seperating debugging commands so we can ban them on the admin thread --- src/protocol/admin/parse.c | 25 +++++++++++++++++++++++-- src/protocol/admin/parse.h | 2 ++ src/protocol/admin/request.h | 10 ++++++++-- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/protocol/admin/parse.c b/src/protocol/admin/parse.c index c3c196a9d..23d7b1641 100644 --- a/src/protocol/admin/parse.c +++ b/src/protocol/admin/parse.c @@ -71,8 +71,8 @@ _get_req_type(struct request *req, struct bstring *type) return PARSE_OK; } -parse_rstatus_e -admin_parse_req(struct request *req, struct buf *buf) +static parse_rstatus_e +_parse_req(struct request *req, struct buf *buf) { char *p, *q; struct bstring type; @@ -103,3 +103,24 @@ admin_parse_req(struct request *req, struct buf *buf) buf->rpos = p + CRLF_LEN; return _get_req_type(req, &type); } + +parse_rstatus_e +admin_parse_req(struct request *req, struct buf *buf) +{ + parse_rstatus_e status; + + status = _parse_req(req, buf); + + if (status == PARSE_OK && + (req->type == REQ_CENSUS || req->type == REQ_DUMP)) { + return PARSE_EINVALID; + } else { + return status; + } +} + +parse_rstatus_e +debug_parse_req(struct request *req, struct buf *buf) +{ + return _parse_req(req, buf); +} diff --git a/src/protocol/admin/parse.h b/src/protocol/admin/parse.h index c929fca22..9d04b6c49 100644 --- a/src/protocol/admin/parse.h +++ b/src/protocol/admin/parse.h @@ -10,4 +10,6 @@ typedef enum parse_rstatus { struct buf; struct request; +/* admin parser blocks "debug" commands which can block for a long time */ parse_rstatus_e admin_parse_req(struct request *req, struct buf *buf); +parse_rstatus_e debug_parse_req(struct request *req, struct buf *buf); diff --git a/src/protocol/admin/request.h b/src/protocol/admin/request.h index cd1638641..6f45c948e 100644 --- a/src/protocol/admin/request.h +++ b/src/protocol/admin/request.h @@ -21,13 +21,19 @@ * whole blob. */ -#define REQ_TYPE_MSG(ACTION) \ +#define ADMIN_TYPE_MSG(ACTION) \ ACTION( REQ_UNKNOWN, "" )\ ACTION( REQ_STATS, "stats" )\ ACTION( REQ_VERSION, "version" )\ + ACTION( REQ_QUIT, "quit" ) + +#define DEBUG_TYPE_MSG(ACTION) \ ACTION( REQ_CENSUS, "census" )\ ACTION( REQ_DUMP, "dump" )\ - ACTION( REQ_QUIT, "quit" ) + +#define REQ_TYPE_MSG(ACTION) \ + ADMIN_TYPE_MSG(ACTION) \ + DEBUG_TYPE_MSG(ACTION) #define GET_TYPE(_name, _str) _name, typedef enum request_type { From 9fdc42bce83948fadafed1b98e938d7e92ca0304 Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Mon, 19 Nov 2018 17:29:22 -0800 Subject: [PATCH 5/6] add debug thread --- src/core/admin/debug.c | 335 +++++++++++++++++++++++++++++++++++++++++ src/core/admin/debug.h | 32 ++++ 2 files changed, 367 insertions(+) create mode 100644 src/core/admin/debug.c create mode 100644 src/core/admin/debug.h diff --git a/src/core/admin/debug.c b/src/core/admin/debug.c new file mode 100644 index 000000000..c0638658d --- /dev/null +++ b/src/core/admin/debug.c @@ -0,0 +1,335 @@ +#include "debug.h" + +#include "core/context.h" + +#include "protocol/debug/debug_include.h" +#include "util/util.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define DEBUG_MODULE_NAME "core::debug" + +static struct context context; +static struct context *ctx = &context; + +static channel_handler_st handlers; +static channel_handler_st *hdl = &handlers; + +static struct addrinfo *debug_ai; +static struct buf_sock *debug_sock; + +static struct request req; +static struct response rsp; + +static inline void +_debug_close(struct buf_sock *s) +{ + event_del(ctx->evb, hdl->rid(s->ch)); + hdl->term(s->ch); + buf_sock_destroy(&s); +} + +static inline void +_tcp_accept(struct buf_sock *ss) +{ + struct buf_sock *s; + struct tcp_conn *sc = ss->ch; + + s = buf_sock_create(); /* debug thread: always directly create not borrow */ + if (s == NULL) { + log_error("establish connection failed: cannot allocate buf_sock, " + "reject connection request"); + ss->hdl->reject(sc); /* server rejects connection by closing it */ + return; + } + + if (!ss->hdl->accept(sc, s->ch)) { + return; + } + + s->owner = ctx; + s->hdl = hdl; + + event_add_read(ctx->evb, hdl->rid(s->ch), s); +} + +static inline rstatus_i +_debug_write(struct buf_sock *s) +{ + rstatus_i status; + + ASSERT(s != NULL); + ASSERT(s->wbuf != NULL && s->rbuf != NULL); + + status = buf_tcp_write(s); + + return status; +} + +static inline void +_debug_post_write(struct buf_sock *s) +{ + buf_lshift(s->rbuf); + buf_lshift(s->wbuf); + + dbuf_shrink(&(s->rbuf)); + dbuf_shrink(&(s->wbuf)); +} + +static inline void +_debug_event_write(struct buf_sock *s) +{ + rstatus_i status; + struct tcp_conn *c = s->ch; + + status = _debug_write(s); + if (status == CC_ERETRY || status == CC_EAGAIN) { + event_add_write(ctx->evb, hdl->wid(c), s); + } else if (status == CC_ERROR) { + c->state = CHANNEL_TERM; + } + _debug_post_write(s); +} + +static inline void +_debug_read(struct buf_sock *s) +{ + ASSERT(s != NULL); + ASSERT(s->wbuf != NULL && s->rbuf != NULL); + + dbuf_tcp_read(s); +} + +static void +_debug_post_read(struct buf_sock *s) +{ + parse_rstatus_e status; + + debug_request_reset(&req); + + while (buf_rsize(s->rbuf) > 0) { + int n; + + status = debug_parse_req(&req, s->rbuf); + if (status == PARSE_EUNFIN) { + goto done; + } + if (status != PARSE_OK) { + log_info("illegal request received on debug port status %d", + status); + goto error; + } + + /* processing */ + if (req.type == REQ_QUIT) { + log_info("peer called quit"); + s->ch->state = CHANNEL_TERM; + goto done; + } + + debug_response_reset(&rsp); + + debug_process_request(&rsp, &req); + + n = debug_compose_rsp(&s->wbuf, &rsp); + if (n < 0) { + log_error("compose response error"); + goto error; + } + } + +done: + if (buf_rsize(s->wbuf) > 0) { + _debug_event_write(s); + } + return; + +error: + s->ch->state = CHANNEL_TERM; +} + +static void +_debug_event_read(struct buf_sock *s) +{ + struct tcp_conn *c = s->ch; + + if (c->level == CHANNEL_META) { + _tcp_accept(s); + } else if (c->level == CHANNEL_BASE) { + _debug_read(s); + _debug_post_read(s); + } else { + NOT_REACHED(); + } +} + +static void +_debug_event(void *arg, uint32_t events) +{ + struct buf_sock *s = arg; + + if (events & EVENT_READ) { + _debug_event_read(s); + } else if (events & EVENT_WRITE) { + _debug_event_write(s); + } else if (events & EVENT_ERR) { + s->ch->state = CHANNEL_TERM; + } else { + NOT_REACHED(); + } + + if (s->ch->state == CHANNEL_TERM || s->ch->state == CHANNEL_ERROR) { + _debug_close(s); + } +} + +void +core_debug_setup(debug_options_st *options) +{ + struct tcp_conn *c; + struct timeout tick; + char *host = DEBUG_HOST; + char *port = DEBUG_PORT; + int timeout = DEBUG_TIMEOUT; + int nevent = DEBUG_NEVENT; + uint64_t tick_ms = DEBUG_TW_TICK; + size_t cap = DEBUG_TW_CAP; + size_t ntick = DEBUG_TW_NTICK; + + log_info("set up the %s module", DEBUG_MODULE_NAME); + + if (debug_init) { + log_warn("debug has already been setup, re-creating"); + core_debug_teardown(); + } + + if (options != NULL) { + host = option_str(&options->debug_host); + port = option_str(&options->debug_port); + timeout = option_uint(&options->debug_timeout); + nevent = option_uint(&options->debug_nevent); + tick_ms = option_uint(&options->debug_tw_tick); + cap = option_uint(&options->debug_tw_cap); + ntick = option_uint(&options->debug_tw_ntick); + } + + ctx->timeout = timeout; + ctx->evb = event_base_create(nevent, _debug_event); + if (ctx->evb == NULL) { + log_crit("failed to set up debug thread; could not create event " + "base for control plane"); + goto error; + } + + hdl->accept = (channel_accept_fn)tcp_accept; + hdl->reject = (channel_reject_fn)tcp_reject; + hdl->open = (channel_open_fn)tcp_listen; + hdl->term = (channel_term_fn)tcp_close; + hdl->recv = (channel_recv_fn)tcp_recv; + hdl->send = (channel_send_fn)tcp_send; + hdl->rid = (channel_id_fn)tcp_read_id; + hdl->wid = (channel_id_fn)tcp_write_id; + + debug_sock = buf_sock_create(); + if (debug_sock == NULL) { + log_crit("failed to set up debug thread; could not get buf_sock"); + goto error; + } + + debug_sock->hdl = hdl; + + if (CC_OK != getaddr(&debug_ai, host, port)) { + log_crit("failed to resolve address for debug host & port"); + goto error; + } + c = debug_sock->ch; + if (!hdl->open(debug_ai, c)) { + log_crit("debug connection setup failed"); + goto error; + } + c->level = CHANNEL_META; + event_add_read(ctx->evb, hdl->rid(c), debug_sock); + + timeout_set_ms(&tick, tick_ms); + tw = timing_wheel_create(&tick, cap, ntick); + if (tw == NULL) { + log_crit("create timing wheel failed"); + goto error; + } + timing_wheel_start(tw); + + debug_init = true; + + return; + +error: + core_debug_teardown(); + exit(EX_CONFIG); +} + +void +core_debug_teardown(void) +{ + log_info("tear down the %s module", DEBUG_MODULE_NAME); + + if (!debug_init) { + log_warn("%s has never been setup", DEBUG_MODULE_NAME); + } else { + timing_wheel_stop(tw); + timing_wheel_destroy(&tw); + event_base_destroy(&(ctx->evb)); + freeaddrinfo(debug_ai); + buf_sock_destroy(&debug_sock); + } + debug_init = false; +} + +struct timeout_event * +core_debug_register(uint64_t intvl_ms, timeout_cb_fn cb, void *arg) +{ + struct timeout delay; + + ASSERT(debug_init); + + timeout_set_ms(&delay, intvl_ms); + return timing_wheel_insert(tw, &delay, true, cb, arg); +} + +static rstatus_i +_debug_evwait(void) +{ + int n; + + n = event_wait(ctx->evb, ctx->timeout); + if (n < 0) { + return n; + } + + return CC_OK; +} + +void +core_debug_evloop(void) +{ + for(;;) { + if (_debug_evwait() != CC_OK) { + log_crit("debug loop exited due to failure"); + break; + } + + timing_wheel_execute(tw); + } + + exit(1); +} diff --git a/src/core/admin/debug.h b/src/core/admin/debug.h new file mode 100644 index 000000000..f23c4237b --- /dev/null +++ b/src/core/admin/debug.h @@ -0,0 +1,32 @@ +#pragma once + +/** + * The debug thread is for performing potentially expensive investigative tasks. + * User should avoid concurrent access to this port/thread. + */ + +#include +#include + +#include + +#define DEBUG_HOST NULL +#define DEBUG_PORT "9900" +#define DEBUG_TIMEOUT 100 /* in ms */ +#define DEBUG_NEVENT 1 + +/* name type default description */ +#define DEBUG_OPTION(ACTION) \ + ACTION( debug_host, OPTION_TYPE_STR, DEBUG_HOST, "debug interfaces listening on")\ + ACTION( debug_port, OPTION_TYPE_STR, DEBUG_PORT, "debug port" )\ + ACTION( debug_timeout, OPTION_TYPE_UINT, DEBUG_TIMEOUT, "evwait timeout" )\ + ACTION( debug_nevent, OPTION_TYPE_UINT, DEBUG_NEVENT, "evwait max nevent returned" ) + +typedef struct { + DEBUG_OPTION(OPTION_DECLARE) +} debug_options_st; + +void core_debug_setup(debug_options_st *options); +void core_debug_teardown(void); + +void core_debug_evloop(void); From 79f7fb0b681f93e7816125caee734ca24a0609a8 Mon Sep 17 00:00:00 2001 From: Yao Yue Date: Mon, 19 Nov 2018 17:29:27 -0800 Subject: [PATCH 6/6] temp --- src/.gitignore | 3 -- src/core/admin/CMakeLists.txt | 1 + src/core/admin/debug.c | 46 +++++----------------------- src/core/admin/debug.h | 10 +++--- src/core/context.h | 1 + src/core/core.c | 17 +++++++++- src/core/core.h | 1 + src/protocol/admin/format.h | 4 +-- src/protocol/admin/parse.c | 3 ++ src/server/twemcache/admin/process.c | 6 ++-- src/server/twemcache/main.c | 2 ++ src/server/twemcache/setting.c | 1 + src/server/twemcache/setting.h | 1 + 13 files changed, 44 insertions(+), 52 deletions(-) diff --git a/src/.gitignore b/src/.gitignore index 08c5af542..0638d7514 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1,4 +1 @@ .dirstamp - -# binaries -broadbill_slimcache diff --git a/src/core/admin/CMakeLists.txt b/src/core/admin/CMakeLists.txt index 9d0abec62..fc1db8c5e 100644 --- a/src/core/admin/CMakeLists.txt +++ b/src/core/admin/CMakeLists.txt @@ -1,4 +1,5 @@ set(SOURCE ${SOURCE} ${CMAKE_CURRENT_SOURCE_DIR}/admin.c + ${CMAKE_CURRENT_SOURCE_DIR}/debug.c PARENT_SCOPE) diff --git a/src/core/admin/debug.c b/src/core/admin/debug.c index c0638658d..3895ae95c 100644 --- a/src/core/admin/debug.c +++ b/src/core/admin/debug.c @@ -2,7 +2,7 @@ #include "core/context.h" -#include "protocol/debug/debug_include.h" +#include "protocol/admin/admin_include.h" #include "util/util.h" #include @@ -116,7 +116,7 @@ _debug_post_read(struct buf_sock *s) { parse_rstatus_e status; - debug_request_reset(&req); + admin_request_reset(&req); while (buf_rsize(s->rbuf) > 0) { int n; @@ -138,11 +138,11 @@ _debug_post_read(struct buf_sock *s) goto done; } - debug_response_reset(&rsp); + admin_response_reset(&rsp); - debug_process_request(&rsp, &req); + admin_process_request(&rsp, &req); - n = debug_compose_rsp(&s->wbuf, &rsp); + n = admin_compose_rsp(&s->wbuf, &rsp); if (n < 0) { log_error("compose response error"); goto error; @@ -195,17 +195,13 @@ _debug_event(void *arg, uint32_t events) } void -core_debug_setup(debug_options_st *options) +core_debug_setup(core_debug_options_st *options) { struct tcp_conn *c; - struct timeout tick; char *host = DEBUG_HOST; char *port = DEBUG_PORT; int timeout = DEBUG_TIMEOUT; int nevent = DEBUG_NEVENT; - uint64_t tick_ms = DEBUG_TW_TICK; - size_t cap = DEBUG_TW_CAP; - size_t ntick = DEBUG_TW_NTICK; log_info("set up the %s module", DEBUG_MODULE_NAME); @@ -219,9 +215,6 @@ core_debug_setup(debug_options_st *options) port = option_str(&options->debug_port); timeout = option_uint(&options->debug_timeout); nevent = option_uint(&options->debug_nevent); - tick_ms = option_uint(&options->debug_tw_tick); - cap = option_uint(&options->debug_tw_cap); - ntick = option_uint(&options->debug_tw_ntick); } ctx->timeout = timeout; @@ -261,14 +254,6 @@ core_debug_setup(debug_options_st *options) c->level = CHANNEL_META; event_add_read(ctx->evb, hdl->rid(c), debug_sock); - timeout_set_ms(&tick, tick_ms); - tw = timing_wheel_create(&tick, cap, ntick); - if (tw == NULL) { - log_crit("create timing wheel failed"); - goto error; - } - timing_wheel_start(tw); - debug_init = true; return; @@ -286,8 +271,6 @@ core_debug_teardown(void) if (!debug_init) { log_warn("%s has never been setup", DEBUG_MODULE_NAME); } else { - timing_wheel_stop(tw); - timing_wheel_destroy(&tw); event_base_destroy(&(ctx->evb)); freeaddrinfo(debug_ai); buf_sock_destroy(&debug_sock); @@ -295,17 +278,6 @@ core_debug_teardown(void) debug_init = false; } -struct timeout_event * -core_debug_register(uint64_t intvl_ms, timeout_cb_fn cb, void *arg) -{ - struct timeout delay; - - ASSERT(debug_init); - - timeout_set_ms(&delay, intvl_ms); - return timing_wheel_insert(tw, &delay, true, cb, arg); -} - static rstatus_i _debug_evwait(void) { @@ -319,16 +291,14 @@ _debug_evwait(void) return CC_OK; } -void -core_debug_evloop(void) +void * +core_debug_evloop(void *arg) { for(;;) { if (_debug_evwait() != CC_OK) { log_crit("debug loop exited due to failure"); break; } - - timing_wheel_execute(tw); } exit(1); diff --git a/src/core/admin/debug.h b/src/core/admin/debug.h index f23c4237b..a5bd5a0e3 100644 --- a/src/core/admin/debug.h +++ b/src/core/admin/debug.h @@ -16,17 +16,17 @@ #define DEBUG_NEVENT 1 /* name type default description */ -#define DEBUG_OPTION(ACTION) \ +#define CORE_DEBUG_OPTION(ACTION) \ ACTION( debug_host, OPTION_TYPE_STR, DEBUG_HOST, "debug interfaces listening on")\ ACTION( debug_port, OPTION_TYPE_STR, DEBUG_PORT, "debug port" )\ ACTION( debug_timeout, OPTION_TYPE_UINT, DEBUG_TIMEOUT, "evwait timeout" )\ ACTION( debug_nevent, OPTION_TYPE_UINT, DEBUG_NEVENT, "evwait max nevent returned" ) typedef struct { - DEBUG_OPTION(OPTION_DECLARE) -} debug_options_st; + CORE_DEBUG_OPTION(OPTION_DECLARE) +} core_debug_options_st; -void core_debug_setup(debug_options_st *options); +void core_debug_setup(core_debug_options_st *options); void core_debug_teardown(void); -void core_debug_evloop(void); +void *core_debug_evloop(void *arg); diff --git a/src/core/context.h b/src/core/context.h index f3d215eb4..d32001e27 100644 --- a/src/core/context.h +++ b/src/core/context.h @@ -14,3 +14,4 @@ struct context { bool admin_init; bool server_init; bool worker_init; +bool debug_init; diff --git a/src/core/core.c b/src/core/core.c index a4ea7433b..7309e0725 100644 --- a/src/core/core.c +++ b/src/core/core.c @@ -12,7 +12,7 @@ void core_run(void *arg_worker) { - pthread_t worker, server; + pthread_t worker, server, debug; int ret; if (!admin_init || !server_init || !worker_init) { @@ -24,12 +24,27 @@ core_run(void *arg_worker) if (ret != 0) { log_crit("pthread create failed for worker thread: %s", strerror(ret)); goto error; + } else { + log_info("worker thread of ID %d has been created", worker); } + ret = pthread_create(&server, NULL, core_server_evloop, NULL); if (ret != 0) { log_crit("pthread create failed for server thread: %s", strerror(ret)); goto error; + } else { + log_info("server thread of ID %d has been created", server); + } + + if (debug_init) { + ret = pthread_create(&debug, NULL, core_debug_evloop, NULL); + if (ret != 0) { + log_crit("pthread create failed for debug thread: %s", strerror(ret)); + goto error; + } else { + log_info("debug thread of ID %d has been created", debug); + } } core_admin_evloop(); diff --git a/src/core/core.h b/src/core/core.h index c05542c13..f91690149 100644 --- a/src/core/core.h +++ b/src/core/core.h @@ -5,6 +5,7 @@ */ #include "admin/admin.h" +#include "admin/debug.h" #include "data/shared.h" #include "data/server.h" #include "data/worker.h" diff --git a/src/protocol/admin/format.h b/src/protocol/admin/format.h index 502e194a9..f68d0ed62 100644 --- a/src/protocol/admin/format.h +++ b/src/protocol/admin/format.h @@ -8,8 +8,8 @@ #define METRIC_DESCRIBE_LEN 120 /* 34 (name) + 16 (type) + 68 (description) + CRLF */ #define METRIC_END "END\r\n" #define METRIC_END_LEN (sizeof(METRIC_END) - 1) -#define CENSUS_COUNT_FMT "item count: %zu %zu %zu\r\n" -#define CENSUS_COUNT_LEN 34 /* 12 (name string) + 20 + CRLF */ +#define CENSUS_COUNT_FMT "item: %zu, total: %zu\r\n" +#define CENSUS_COUNT_LEN 56 /* 14 (name string) + 20 * 2 + CRLF */ #define CENSUS_KEY_FMT "key min: %zu, max: %zu, total: %zu\r\n" #define CENSUS_KEY_LEN 87 /* 9 + 7 + 9 (name strings) + 20 * 3 + CRLF */ #define CENSUS_VAL_FMT "val min: %zu, max: %zu, total: %zu\r\n" diff --git a/src/protocol/admin/parse.c b/src/protocol/admin/parse.c index 23d7b1641..f58c7f3ff 100644 --- a/src/protocol/admin/parse.c +++ b/src/protocol/admin/parse.c @@ -47,6 +47,9 @@ _get_req_type(struct request *req, struct bstring *type) break; } + break; + + case 6: if (str6cmp(type->data, 'c', 'e', 'n', 's', 'u', 's')) { req->type = REQ_CENSUS; break; diff --git a/src/server/twemcache/admin/process.c b/src/server/twemcache/admin/process.c index 09d93a5b5..ce78c9e35 100644 --- a/src/server/twemcache/admin/process.c +++ b/src/server/twemcache/admin/process.c @@ -126,12 +126,12 @@ _key_census(struct response *rsp, struct request *req) req->arg.len--; req->arg.data++; } - log_info("census on keys with prefix %.*s", req->arg.len, req->arg.data); + log_info("census on keys with prefix '%.*s'", req->arg.len, req->arg.data); item_census(&nkey, &ktotal, &kmin, &kmax, &vtotal, &vmin, &vmax, &req->arg); rsp->type = RSP_GENERIC; - ret = cc_scnprintf(buf, cap, CENSUS_FMT, nkey, ktotal, kmin, kmax, vtotal, - vmin, vmax); + ret = cc_scnprintf(buf, cap, CENSUS_FMT, nkey, ktotal + vtotal, kmin, kmax, + ktotal, vmin, vmax, vtotal); if (ret < 0) { rsp->data = str2bstr("ERROR: cannot format key census result"); } else { diff --git a/src/server/twemcache/main.c b/src/server/twemcache/main.c index a0dc85a89..73f08cce7 100644 --- a/src/server/twemcache/main.c +++ b/src/server/twemcache/main.c @@ -52,6 +52,7 @@ teardown(void) { core_worker_teardown(); core_server_teardown(); + core_debug_teardown(); core_admin_teardown(); admin_process_teardown(); process_teardown(); @@ -127,6 +128,7 @@ setup(void) process_setup(&setting.process, &stats.process); admin_process_setup(); core_admin_setup(&setting.admin); + core_debug_setup(&setting.core_debug); core_server_setup(&setting.server, &stats.server); core_worker_setup(&setting.worker, &stats.worker); diff --git a/src/server/twemcache/setting.c b/src/server/twemcache/setting.c index e6e820545..e483b34a9 100644 --- a/src/server/twemcache/setting.c +++ b/src/server/twemcache/setting.c @@ -3,6 +3,7 @@ struct setting setting = { { TWEMCACHE_OPTION(OPTION_INIT) }, { ADMIN_OPTION(OPTION_INIT) }, + { CORE_DEBUG_OPTION(OPTION_INIT)}, { SERVER_OPTION(OPTION_INIT) }, { WORKER_OPTION(OPTION_INIT) }, { PROCESS_OPTION(OPTION_INIT) }, diff --git a/src/server/twemcache/setting.h b/src/server/twemcache/setting.h index 68e0aadef..cf3b2fa7d 100644 --- a/src/server/twemcache/setting.h +++ b/src/server/twemcache/setting.h @@ -36,6 +36,7 @@ struct setting { twemcache_options_st twemcache; /* application modules */ admin_options_st admin; + core_debug_options_st core_debug; server_options_st server; worker_options_st worker; process_options_st process;