Skip to content

Commit

Permalink
dedup-tool: add options for operating usability
Browse files Browse the repository at this point in the history
Daemonization is used for background execution of crawler.
Iterative execution makes crawler threads wake up at each period.

[usage]
  ceph_dedup_tool --op sample-dedup --pool POOL --chunk-pool POOL \
    --iterative --daemonize
  • Loading branch information
jyha200 committed Feb 25, 2022
1 parent aa171cb commit ba87902
Showing 1 changed file with 47 additions and 3 deletions.
50 changes: 47 additions & 3 deletions src/tools/ceph_dedup_tool.cc
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ void usage()
" [--op dump-chunk-refs --chunk-pool POOL --object OID] \n"
" [--op chunk-dedup --pool POOL --object OID --chunk-pool POOL --fingerprint-algorithm FP --source-off OFFSET --source-length LENGTH] \n"
" [--op object-dedup --pool POOL --object OID --chunk-pool POOL --fingerprint-algorithm FP --dedup-cdc-chunk-size CHUNK_SIZE] \n"
" [--op sample-dedup --pool POOL --chunk-pool POOL --fingerprint-algorithm FP]\n"
" [--op sample-dedup --pool POOL --chunk-pool POOL --fingerprint-algorithm FP --deamon --iterative]\n"
<< std::endl;
cout << "optional arguments: " << std::endl;
cout << " --object <object_name> " << std::endl;
Expand All @@ -160,11 +160,15 @@ void usage()
cout << " --chunk-dedup-threshold <number>" << std::endl;
cout << " --sampling-ratio <percentile>" << std::endl;
cout << " --shallow-crawling" << std::endl;
cout << " --daemon" << std::endl;
cout << " --iterative" << std::endl;
cout << "explanations: " << std::endl;
cout << " chunk-dedup performs deduplication using a chunk generated by given source" << std::endl;
cout << " offset and length. object-dedup deduplicates the entire object, not a chunk" << std::endl;
cout << " sample-dedup makes crawling threads which crawl objects in base pool and" << std::endl;
cout << "deduplicate them based on their deduplcation efficiency" << std::endl;
cout << " deduplicate them based on their deduplcation efficiency. For deaminizing" << std::endl;
cout << " and iterative execution, add --deamon and --iterative as arguemnts," << std::endl;
cout << " respectively." << std::endl;
exit(1);
}

Expand Down Expand Up @@ -1757,6 +1761,28 @@ int make_dedup_object(const std::map < std::string, std::string > &opts,

int make_crawling_daemon(const map<string, string> &opts,
vector<const char*> &nargs) {

map<string, string>::const_iterator i = opts.find("daemon");
if (i != opts.end()) {
pid_t pid = fork();
if (pid < 0) {
cerr << "daemon process creation failed\n";
return -EINVAL;
}

if (pid != 0) {
return 0;
}
signal(SIGHUP, SIG_IGN);
close(STDIN_FILENO);
close(STDOUT_FILENO);
}

i = opts.find("iterative");
bool iterative = false;
if (i != opts.end()) {
iterative = true;
}
string base_pool_name;
auto i = opts.find("pool");
if (i != opts.end()) {
Expand Down Expand Up @@ -1839,6 +1865,13 @@ int make_crawling_daemon(const map<string, string> &opts,
cerr << "couldn't connect to cluster: " << cpp_strerror(ret) << std::endl;
return -EINVAL;
}
uint32_t wakeup_period = 100;
i = opts.find("wakeup-period");
if (i != opts.end()) {
if (rados_sistrtoll(i, &wakeup_period)) {
return -EINVAL;
}
}

std::string fp_algo;
i = opts.find("fingerprint-algorithm");
Expand Down Expand Up @@ -1949,7 +1982,12 @@ int make_crawling_daemon(const map<string, string> &opts,
for (auto &p : estimate_threads) {
p->join();
}
break;

if (iterative) {
sleep(wakeup_period);
} else {
break;
}
}

return 0;
Expand Down Expand Up @@ -2027,6 +2065,12 @@ int main(int argc, const char **argv)
opts["object-dedup-threshold"] = val;
} else if (ceph_argparse_witharg(args, i, &val, "--chunk-dedup-threshold", (char*)NULL)) {
opts["chunk-dedup-threshold"] = val;
} else if (ceph_argparse_witharg(args, i, &val, "--wakeup-period", (char*)NULL)) {
opts["wakeup-period"] = val;
} else if (ceph_argparse_flag(args, i, "--daemon", (char*)NULL)) {
opts["daemon"] = "true";
} else if (ceph_argparse_flag(args, i, "--iterative", (char*)NULL)) {
opts["iterative"] = "true";
} else if (ceph_argparse_flag(args, i, "--shallow-crawling", (char*)NULL)) {
opts["shallow-crawling"] = true;
} else if (ceph_argparse_flag(args, i, "--debug", (char*)NULL)) {
Expand Down

0 comments on commit ba87902

Please sign in to comment.