From 3f598e8c65731c3451c1aecf8ec0925e34b0bec8 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 14:38:33 -0700 Subject: [PATCH 01/44] AdminSocket users: use generic formatting All call() routines get a format parameter; all places where JSONFormatter was created get a new_formatter() instead. 'plain' formatting is unsupported, and help is forced to be 'json-pretty' as it was. Signed-off-by: Dan Mick --- src/client/Client.cc | 14 ++- src/client/Client.h | 3 +- src/common/admin_socket.cc | 41 +++++-- src/common/admin_socket.h | 3 +- src/common/ceph_context.cc | 35 +++--- src/common/ceph_context.h | 3 +- src/common/perf_counters.cc | 100 ++++++++--------- src/common/perf_counters.h | 4 +- src/mon/Monitor.cc | 11 +- src/mon/Monitor.h | 3 +- src/osd/OSD.cc | 78 ++++++------- src/osd/OSD.h | 2 +- src/osd/OpRequest.cc | 28 +++-- src/osd/OpRequest.h | 4 +- src/osdc/Objecter.cc | 164 ++++++++++++++-------------- src/osdc/Objecter.h | 17 +-- src/test/bench/small_io_bench_fs.cc | 4 +- 17 files changed, 267 insertions(+), 247 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index ba036ad998036..5a9c5fdafcca2 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -102,22 +102,24 @@ Client::CommandHook::CommandHook(Client *client) : { } -bool Client::CommandHook::call(std::string command, std::string args, bufferlist& out) +bool Client::CommandHook::call(std::string command, std::string args, + std::string format, bufferlist& out) { stringstream ss; - JSONFormatter formatter(true); + Formatter *f = new_formatter(format); m_client->client_lock.Lock(); if (command == "mds_requests") - m_client->dump_mds_requests(&formatter); + m_client->dump_mds_requests(f); else if (command == "mds_sessions") - m_client->dump_mds_sessions(&formatter); + m_client->dump_mds_sessions(f); else if (command == "dump_cache") - m_client->dump_cache(&formatter); + m_client->dump_cache(f); else assert(0 == "bad command registered"); m_client->client_lock.Unlock(); - formatter.flush(ss); + f->flush(ss); out.append(ss); + delete f; return true; } diff --git a/src/client/Client.h b/src/client/Client.h index ade0b8f29c84d..bc1fbc0401b5c 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -196,7 +196,8 @@ class Client : public Dispatcher { Client *m_client; public: CommandHook(Client *client); - bool call(std::string command, std::string args, bufferlist& out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out); }; CommandHook m_command_hook; diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index f7ab3501dfff9..5ebb3e040db8d 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -310,6 +310,25 @@ bool AdminSocket::do_accept() bool rval = false; + map cmdmap; + string format; + vector cmdvec; + stringstream errss; + cmdvec.push_back(cmd); + if (!cmdmap_from_json(cmdvec, &cmdmap, errss)) { + ldout(m_cct, 0) << "AdminSocket: " << errss << dendl; + return false; + } + cmd_getval(m_cct, cmdmap, "format", format); + cmd_getval(m_cct, cmdmap, "prefix", c); + + // we don't do plain here + if (format != "json" && + format != "json-pretty" && + format != "xml" && + format != "xml-pretty") + format = "json"; + string firstword; if (c.find(" ") == string::npos) firstword = c; @@ -341,7 +360,7 @@ bool AdminSocket::do_accept() string args; if (match != c) args = c.substr(match.length() + 1); - bool success = p->second->call(match, args, out); + bool success = p->second->call(match, args, format, out); if (!success) { ldout(m_cct, 0) << "AdminSocket: request '" << match << "' args '" << args << "' to " << p->second << " failed" << dendl; @@ -406,7 +425,8 @@ int AdminSocket::unregister_command(std::string command) class VersionHook : public AdminSocketHook { public: - virtual bool call(std::string command, std::string args, bufferlist& out) { + virtual bool call(std::string command, std::string args, std::string format, + bufferlist& out) { if (command == "0") { out.append(CEPH_ADMIN_SOCK_VERSION); } else { @@ -429,18 +449,21 @@ class HelpHook : public AdminSocketHook { AdminSocket *m_as; public: HelpHook(AdminSocket *as) : m_as(as) {} - bool call(string command, string args, bufferlist& out) { - JSONFormatter jf(true); - jf.open_object_section("help"); + bool call(string command, string args, string format, bufferlist& out) { + // override format here because help should always be pretty and + // predictable + Formatter *f = new_formatter("json-pretty"); + f->open_object_section("help"); for (map::iterator p = m_as->m_help.begin(); p != m_as->m_help.end(); ++p) { - jf.dump_string(p->first.c_str(), p->second); + f->dump_string(p->first.c_str(), p->second); } - jf.close_section(); + f->close_section(); ostringstream ss; - jf.flush(ss); + f->flush(ss); out.append(ss.str()); + delete f; return true; } }; @@ -449,7 +472,7 @@ class GetdescsHook : public AdminSocketHook { AdminSocket *m_as; public: GetdescsHook(AdminSocket *as) : m_as(as) {} - bool call(string command, string args, bufferlist& out) { + bool call(string command, string args, string format, bufferlist& out) { int cmdnum = 0; JSONFormatter jf(false); jf.open_object_section("command_descriptions"); diff --git a/src/common/admin_socket.h b/src/common/admin_socket.h index c390bca0382c3..30c5eb96ab8d6 100644 --- a/src/common/admin_socket.h +++ b/src/common/admin_socket.h @@ -29,7 +29,8 @@ class CephContext; class AdminSocketHook { public: - virtual bool call(std::string command, std::string args, bufferlist& out) = 0; + virtual bool call(std::string command, std::string args, std::string format, + bufferlist& out) = 0; virtual ~AdminSocketHook() {}; }; diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index cad980bb2a6f2..e0fcf530222f4 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -156,44 +156,46 @@ class CephContextHook : public AdminSocketHook { public: CephContextHook(CephContext *cct) : m_cct(cct) {} - bool call(std::string command, std::string args, bufferlist& out) { - m_cct->do_command(command, args, &out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { + m_cct->do_command(command, args, format, &out); return true; } }; -void CephContext::do_command(std::string command, std::string args, bufferlist *out) +void CephContext::do_command(std::string command, std::string args, + std::string format, bufferlist *out) { + Formatter *f = new_formatter(format); lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "'" << dendl; if (command == "perfcounters_dump" || command == "1" || command == "perf dump") { - _perf_counters_collection->write_json_to_buf(*out, false); + _perf_counters_collection->dump_formatted(f, *out, false); } else if (command == "perfcounters_schema" || command == "2" || command == "perf schema") { - _perf_counters_collection->write_json_to_buf(*out, true); + _perf_counters_collection->dump_formatted(f, *out, true); } else { - JSONFormatter jf(true); - jf.open_object_section(command.c_str()); + f->open_object_section(command.c_str()); if (command == "config show") { - _conf->show_config(&jf); + _conf->show_config(f); } else if (command == "config set") { std::string var = args; size_t pos = var.find(' '); if (pos == string::npos) { - jf.dump_string("error", "syntax error: 'config set '"); + f->dump_string("error", "syntax error: 'config set '"); } else { std::string val = var.substr(pos+1); var.resize(pos); int r = _conf->set_val(var.c_str(), val.c_str()); if (r < 0) { - jf.dump_stream("error") << "error setting '" << var << "' to '" << val << "': " << cpp_strerror(r); + f->dump_stream("error") << "error setting '" << var << "' to '" << val << "': " << cpp_strerror(r); } else { ostringstream ss; _conf->apply_changes(&ss); - jf.dump_string("success", ss.str()); + f->dump_string("success", ss.str()); } } } else if (command == "config get") { @@ -202,9 +204,9 @@ void CephContext::do_command(std::string command, std::string args, bufferlist * char *tmp = buf; int r = _conf->get_val(args.c_str(), &tmp, sizeof(buf)); if (r < 0) { - jf.dump_stream("error") << "error getting '" << args << "': " << cpp_strerror(r); + f->dump_stream("error") << "error getting '" << args << "': " << cpp_strerror(r); } else { - jf.dump_string(args.c_str(), buf); + f->dump_string(args.c_str(), buf); } } else if (command == "log flush") { _log->flush(); @@ -218,11 +220,10 @@ void CephContext::do_command(std::string command, std::string args, bufferlist * else { assert(0 == "registered under wrong command?"); } - ostringstream ss; - jf.close_section(); - jf.flush(ss); - out->append(ss.str()); + f->close_section(); + f->flush(*out); } + delete f; lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "' result is " << out->length() << " bytes" << dendl; }; diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h index 1678680fa6db5..85618e3521934 100644 --- a/src/common/ceph_context.h +++ b/src/common/ceph_context.h @@ -97,7 +97,8 @@ class CephContext { /** * process an admin socket command */ - void do_command(std::string command, std::string args, bufferlist *out); + void do_command(std::string command, std::string args, std::string foramt, + bufferlist *out); /** * get a crypto handler diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index 67a777497b38a..46f55fae51bfc 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -15,6 +15,7 @@ #include "common/perf_counters.h" #include "common/dout.h" #include "common/errno.h" +#include "common/Formatter.h" #include #include @@ -72,21 +73,22 @@ void PerfCountersCollection::clear() } } -void PerfCountersCollection::write_json_to_buf(bufferlist& bl, bool schema) +void PerfCountersCollection::dump_formatted(Formatter *f, bufferlist &bl, + bool schema) { Mutex::Locker lck(m_lock); - bl.append('{'); + f->open_object_section("perfcounter_collection"); perf_counters_set_t::iterator l = m_loggers.begin(); perf_counters_set_t::iterator l_end = m_loggers.end(); if (l != l_end) { while (true) { - (*l)->write_json_to_buf(bl, schema); + (*l)->dump_formatted(f, schema); if (++l == l_end) break; - bl.append(','); } } - bl.append('}'); + f->close_section(); + f->flush(bl); } // --------------------------- @@ -203,34 +205,54 @@ utime_t PerfCounters::tget(int idx) const return utime_t(data.u64 / 1000000000ull, data.u64 % 1000000000ull); } -void PerfCounters::write_json_to_buf(bufferlist& bl, bool schema) +void PerfCounters::dump_formatted(Formatter *f, bool schema) { - char buf[512]; Mutex::Locker lck(m_lock); - snprintf(buf, sizeof(buf), "\"%s\":{", m_name.c_str()); - bl.append(buf); - + f->open_object_section(m_name.c_str()); perf_counter_data_vec_t::const_iterator d = m_data.begin(); perf_counter_data_vec_t::const_iterator d_end = m_data.end(); if (d == d_end) { - bl.append('}'); + f->close_section(); return; } while (true) { - const perf_counter_data_any_d &data(*d); - buf[0] = '\0'; - if (schema) - data.write_schema_json(buf, sizeof(buf)); - else - data.write_json(buf, sizeof(buf)); - - bl.append(buf); + if (schema) { + f->open_object_section(d->name); + f->dump_int("type", d->type); + f->close_section(); + } else { + if (d->type & PERFCOUNTER_LONGRUNAVG) { + f->open_object_section(d->name); + if (d->type & PERFCOUNTER_U64) { + f->dump_format("avgcount", "%"PRId64, d->avgcount); + f->dump_format("sum", "%"PRId64, d->u64); + } else if (d->type & PERFCOUNTER_TIME) { + f->dump_format("avgcount", "%"PRId64, d->avgcount); + f->dump_format("sum", "%"PRId64"%09"PRId64, + d->u64 / 1000000000ull, + d->u64 % 1000000000ull); + } else { + assert(0); + } + f->close_section(); + } else { + if (d->type & PERFCOUNTER_U64) { + f->dump_format(d->name, "%"PRId64, d->u64); + } else if (d->type & PERFCOUNTER_TIME) { + f->dump_format(d->name, "%"PRId64"%09"PRId64, + d->u64 / 1000000000ull, + d->u64 % 1000000000ull); + } else { + assert(0); + } + } + } + if (++d == d_end) break; - bl.append(','); } - bl.append('}'); + f->close_section(); } const std::string &PerfCounters::get_name() const @@ -258,42 +280,6 @@ PerfCounters::perf_counter_data_any_d::perf_counter_data_any_d() { } -void PerfCounters::perf_counter_data_any_d::write_schema_json(char *buf, size_t buf_sz) const -{ - snprintf(buf, buf_sz, "\"%s\":{\"type\":%d}", name, type); -} - -void PerfCounters::perf_counter_data_any_d::write_json(char *buf, size_t buf_sz) const -{ - if (type & PERFCOUNTER_LONGRUNAVG) { - if (type & PERFCOUNTER_U64) { - snprintf(buf, buf_sz, "\"%s\":{\"avgcount\":%" PRId64 "," - "\"sum\":%" PRId64 "}", - name, avgcount, u64); - } - else if (type & PERFCOUNTER_TIME) { - snprintf(buf, buf_sz, "\"%s\":{\"avgcount\":%" PRId64 "," - "\"sum\":%llu.%09llu}", - name, avgcount, u64 / 1000000000ull, u64 % 1000000000ull); - } - else { - assert(0); - } - } - else { - if (type & PERFCOUNTER_U64) { - snprintf(buf, buf_sz, "\"%s\":%" PRId64, - name, u64); - } - else if (type & PERFCOUNTER_TIME) { - snprintf(buf, buf_sz, "\"%s\":%llu.%09llu", name, u64 / 1000000000ull, u64 % 1000000000ull); - } - else { - assert(0); - } - } -} - PerfCountersBuilder::PerfCountersBuilder(CephContext *cct, const std::string &name, int first, int last) : m_perf_counters(new PerfCounters(cct, name, first, last)) diff --git a/src/common/perf_counters.h b/src/common/perf_counters.h index 269a32f2c46f8..125d84c04e34a 100644 --- a/src/common/perf_counters.h +++ b/src/common/perf_counters.h @@ -76,7 +76,7 @@ class PerfCounters void tinc(int idx, utime_t v); utime_t tget(int idx) const; - void write_json_to_buf(ceph::bufferlist& bl, bool schema); + void dump_formatted(ceph::Formatter *f, bool schema); const std::string& get_name() const; void set_name(std::string s) { @@ -136,7 +136,7 @@ class PerfCountersCollection void add(class PerfCounters *l); void remove(class PerfCounters *l); void clear(); - void write_json_to_buf(ceph::bufferlist& bl, bool schema); + void dump_formatted(ceph::Formatter *f, bufferlist &bl, bool schema); private: CephContext *m_cct; diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index bf500dff218fc..119ef740aa834 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -225,21 +225,20 @@ class AdminHook : public AdminSocketHook { Monitor *mon; public: AdminHook(Monitor *m) : mon(m) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; - mon->do_admin_command(command, args, ss); + mon->do_admin_command(command, args, format, ss); out.append(ss); return true; } }; -void Monitor::do_admin_command(string command, string args, ostream& ss) +void Monitor::do_admin_command(string command, string args, string format, + ostream& ss) { Mutex::Locker l(lock); - map cmdmap; - string format; - cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain")); boost::scoped_ptr f(new_formatter(format)); if (command == "mon_status") diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 82b08816702b0..bed48ecee3436 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -745,7 +745,8 @@ class Monitor : public Dispatcher { int write_fsid(); int write_fsid(MonitorDBStore::Transaction &t); - void do_admin_command(std::string command, std::string args, ostream& ss); + void do_admin_command(std::string command, std::string args, + std::string format, ostream& ss); private: // don't allow copying diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index bc3aa604fecf6..dd9a5b7293f33 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -997,44 +997,46 @@ class OSDSocketHook : public AdminSocketHook { OSD *osd; public: OSDSocketHook(OSD *o) : osd(o) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; - bool r = osd->asok_command(command, args, ss); + bool r = osd->asok_command(command, args, format, ss); out.append(ss); return r; } }; -bool OSD::asok_command(string command, string args, ostream& ss) +bool OSD::asok_command(string command, string args, string format, ostream& ss) { + if (format == "") + format = "json-pretty"; + Formatter *f = new_formatter(format); if (command == "dump_ops_in_flight") { - op_tracker.dump_ops_in_flight(ss); + op_tracker.dump_ops_in_flight(f, ss); } else if (command == "dump_historic_ops") { - op_tracker.dump_historic_ops(ss); + op_tracker.dump_historic_ops(f, ss); } else if (command == "dump_op_pq_state") { - JSONFormatter f(true); - f.open_object_section("pq"); - op_wq.dump(&f); - f.close_section(); - f.flush(ss); + f->open_object_section("pq"); + op_wq.dump(f); + f->close_section(); + f->flush(ss); } else if (command == "dump_blacklist") { list > bl; OSDMapRef curmap = service.get_osdmap(); - JSONFormatter f(true); - f.open_array_section("blacklist"); + f->open_array_section("blacklist"); curmap->get_blacklist(&bl); for (list >::iterator it = bl.begin(); it != bl.end(); ++it) { - f.open_array_section("entry"); - f.open_object_section("entity_addr_t"); - it->first.dump(&f); - f.close_section(); //entity_addr_t - it->second.localtime(f.dump_stream("expire_time")); - f.close_section(); //entry - } - f.close_section(); //blacklist - f.flush(ss); + f->open_array_section("entry"); + f->open_object_section("entity_addr_t"); + it->first.dump(f); + f->close_section(); //entity_addr_t + it->second.localtime(f->dump_stream("expire_time")); + f->close_section(); //entry + } + f->close_section(); //blacklist + f->flush(ss); } else if (command == "dump_watchers") { list watchers; osd_lock.Lock(); @@ -1052,32 +1054,31 @@ bool OSD::asok_command(string command, string args, ostream& ss) } osd_lock.Unlock(); - JSONFormatter f(true); - f.open_array_section("watchers"); + f->open_array_section("watchers"); for (list::iterator it = watchers.begin(); it != watchers.end(); ++it) { - f.open_array_section("watch"); + f->open_array_section("watch"); - f.dump_string("namespace", it->obj.nspace); - f.dump_string("object", it->obj.oid.name); + f->dump_string("namespace", it->obj.nspace); + f->dump_string("object", it->obj.oid.name); - f.open_object_section("entity_name"); - it->wi.name.dump(&f); - f.close_section(); //entity_name_t + f->open_object_section("entity_name"); + it->wi.name.dump(f); + f->close_section(); //entity_name_t - f.dump_int("cookie", it->wi.cookie); - f.dump_int("timeout", it->wi.timeout_seconds); + f->dump_int("cookie", it->wi.cookie); + f->dump_int("timeout", it->wi.timeout_seconds); - f.open_object_section("entity_addr_t"); - it->wi.addr.dump(&f); - f.close_section(); //entity_addr_t + f->open_object_section("entity_addr_t"); + it->wi.addr.dump(f); + f->close_section(); //entity_addr_t - f.close_section(); //watch + f->close_section(); //watch } - f.close_section(); //watches - f.flush(ss); + f->close_section(); //watches + f->flush(ss); } else { assert(0 == "broken asok registration"); } @@ -1089,7 +1090,8 @@ class TestOpsSocketHook : public AdminSocketHook { ObjectStore *store; public: TestOpsSocketHook(OSDService *s, ObjectStore *st) : service(s), store(st) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; test_ops(service, store, command, args, ss); out.append(ss); diff --git a/src/osd/OSD.h b/src/osd/OSD.h index f9ceaf81bf391..5bcff7442d7a6 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -622,7 +622,7 @@ class OSD : public Dispatcher, // asok friend class OSDSocketHook; class OSDSocketHook *asok_hook; - bool asok_command(string command, string args, ostream& ss); + bool asok_command(string command, string args, string format, ostream& ss); public: ClassHandler *class_handler; diff --git a/src/osd/OpRequest.cc b/src/osd/OpRequest.cc index 3b8a8714d9221..c0d167a5f0ad1 100644 --- a/src/osd/OpRequest.cc +++ b/src/osd/OpRequest.cc @@ -76,31 +76,29 @@ void OpHistory::dump_ops(utime_t now, Formatter *f) f->close_section(); } -void OpTracker::dump_historic_ops(ostream &ss) +void OpTracker::dump_historic_ops(Formatter *f, ostream &ss) { - JSONFormatter jf(true); Mutex::Locker locker(ops_in_flight_lock); utime_t now = ceph_clock_now(g_ceph_context); - history.dump_ops(now, &jf); - jf.flush(ss); + history.dump_ops(now, f); + f->flush(ss); } -void OpTracker::dump_ops_in_flight(ostream &ss) +void OpTracker::dump_ops_in_flight(Formatter *f, ostream &ss) { - JSONFormatter jf(true); Mutex::Locker locker(ops_in_flight_lock); - jf.open_object_section("ops_in_flight"); // overall dump - jf.dump_int("num_ops", ops_in_flight.size()); - jf.open_array_section("ops"); // list of OpRequests + f->open_object_section("ops_in_flight"); // overall dump + f->dump_int("num_ops", ops_in_flight.size()); + f->open_array_section("ops"); // list of OpRequests utime_t now = ceph_clock_now(g_ceph_context); for (xlist::iterator p = ops_in_flight.begin(); !p.end(); ++p) { - jf.open_object_section("op"); - (*p)->dump(now, &jf); - jf.close_section(); // this OpRequest + f->open_object_section("op"); + (*p)->dump(now, f); + f->close_section(); // this OpRequest } - jf.close_section(); // list of OpRequests - jf.close_section(); // overall dump - jf.flush(ss); + f->close_section(); // list of OpRequests + f->close_section(); // overall dump + f->flush(ss); } void OpTracker::register_inflight_op(xlist::item *i) diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h index 47b050b853815..67ee26b02ec14 100644 --- a/src/osd/OpRequest.h +++ b/src/osd/OpRequest.h @@ -59,8 +59,8 @@ class OpTracker { public: OpTracker() : seq(0), ops_in_flight_lock("OpTracker mutex") {} - void dump_ops_in_flight(std::ostream& ss); - void dump_historic_ops(std::ostream& ss); + void dump_ops_in_flight(Formatter *f, std::ostream& ss); + void dump_historic_ops(Formatter *f, std::ostream& ss); void register_inflight_op(xlist::item *i); void unregister_inflight_op(OpRequest *i); diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc index a5a023cb33e66..e07d0626d21e4 100644 --- a/src/osdc/Objecter.cc +++ b/src/osdc/Objecter.cc @@ -2180,154 +2180,154 @@ void Objecter::dump_active() } } -void Objecter::dump_requests(Formatter& fmt) const +void Objecter::dump_requests(Formatter *fmt) const { assert(client_lock.is_locked()); - fmt.open_object_section("requests"); + fmt->open_object_section("requests"); dump_ops(fmt); dump_linger_ops(fmt); dump_pool_ops(fmt); dump_pool_stat_ops(fmt); dump_statfs_ops(fmt); dump_command_ops(fmt); - fmt.close_section(); // requests object + fmt->close_section(); // requests object } -void Objecter::dump_ops(Formatter& fmt) const +void Objecter::dump_ops(Formatter *fmt) const { - fmt.open_array_section("ops"); + fmt->open_array_section("ops"); for (map::const_iterator p = ops.begin(); p != ops.end(); ++p) { Op *op = p->second; - fmt.open_object_section("op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("pg") << op->pgid; - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.dump_stream("last_sent") << op->stamp; - fmt.dump_int("attempts", op->attempts); - fmt.dump_stream("object_id") << op->oid; - fmt.dump_stream("object_locator") << op->oloc; - fmt.dump_stream("snapid") << op->snapid; - fmt.dump_stream("snap_context") << op->snapc; - fmt.dump_stream("mtime") << op->mtime; - - fmt.open_array_section("osd_ops"); + fmt->open_object_section("op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("pg") << op->pgid; + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->dump_stream("last_sent") << op->stamp; + fmt->dump_int("attempts", op->attempts); + fmt->dump_stream("object_id") << op->oid; + fmt->dump_stream("object_locator") << op->oloc; + fmt->dump_stream("snapid") << op->snapid; + fmt->dump_stream("snap_context") << op->snapc; + fmt->dump_stream("mtime") << op->mtime; + + fmt->open_array_section("osd_ops"); for (vector::const_iterator it = op->ops.begin(); it != op->ops.end(); ++it) { - fmt.dump_stream("osd_op") << *it; + fmt->dump_stream("osd_op") << *it; } - fmt.close_section(); // osd_ops array + fmt->close_section(); // osd_ops array - fmt.close_section(); // op object + fmt->close_section(); // op object } - fmt.close_section(); // ops array + fmt->close_section(); // ops array } -void Objecter::dump_linger_ops(Formatter& fmt) const +void Objecter::dump_linger_ops(Formatter *fmt) const { - fmt.open_array_section("linger_ops"); + fmt->open_array_section("linger_ops"); for (map::const_iterator p = linger_ops.begin(); p != linger_ops.end(); ++p) { LingerOp *op = p->second; - fmt.open_object_section("linger_op"); - fmt.dump_unsigned("linger_id", op->linger_id); - fmt.dump_stream("pg") << op->pgid; - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.dump_stream("object_id") << op->oid; - fmt.dump_stream("object_locator") << op->oloc; - fmt.dump_stream("snapid") << op->snap; - fmt.dump_stream("registering") << op->snap; - fmt.dump_stream("registered") << op->snap; - fmt.close_section(); // linger_op object + fmt->open_object_section("linger_op"); + fmt->dump_unsigned("linger_id", op->linger_id); + fmt->dump_stream("pg") << op->pgid; + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->dump_stream("object_id") << op->oid; + fmt->dump_stream("object_locator") << op->oloc; + fmt->dump_stream("snapid") << op->snap; + fmt->dump_stream("registering") << op->snap; + fmt->dump_stream("registered") << op->snap; + fmt->close_section(); // linger_op object } - fmt.close_section(); // linger_ops array + fmt->close_section(); // linger_ops array } -void Objecter::dump_command_ops(Formatter& fmt) const +void Objecter::dump_command_ops(Formatter *fmt) const { - fmt.open_array_section("command_ops"); + fmt->open_array_section("command_ops"); for (map::const_iterator p = command_ops.begin(); p != command_ops.end(); ++p) { CommandOp *op = p->second; - fmt.open_object_section("command_op"); - fmt.dump_unsigned("command_id", op->tid); - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.open_array_section("command"); + fmt->open_object_section("command_op"); + fmt->dump_unsigned("command_id", op->tid); + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->open_array_section("command"); for (vector::const_iterator q = op->cmd.begin(); q != op->cmd.end(); ++q) - fmt.dump_string("word", *q); - fmt.close_section(); + fmt->dump_string("word", *q); + fmt->close_section(); if (op->target_osd >= 0) - fmt.dump_int("target_osd", op->target_osd); + fmt->dump_int("target_osd", op->target_osd); else - fmt.dump_stream("target_pg") << op->target_pg; - fmt.close_section(); // command_op object + fmt->dump_stream("target_pg") << op->target_pg; + fmt->close_section(); // command_op object } - fmt.close_section(); // command_ops array + fmt->close_section(); // command_ops array } -void Objecter::dump_pool_ops(Formatter& fmt) const +void Objecter::dump_pool_ops(Formatter *fmt) const { - fmt.open_array_section("pool_ops"); + fmt->open_array_section("pool_ops"); for (map::const_iterator p = pool_ops.begin(); p != pool_ops.end(); ++p) { PoolOp *op = p->second; - fmt.open_object_section("pool_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_int("pool", op->pool); - fmt.dump_string("name", op->name); - fmt.dump_int("operation_type", op->pool_op); - fmt.dump_unsigned("auid", op->auid); - fmt.dump_unsigned("crush_rule", op->crush_rule); - fmt.dump_stream("snapid") << op->snapid; - fmt.dump_stream("last_sent") << op->last_submit; - fmt.close_section(); // pool_op object + fmt->open_object_section("pool_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_int("pool", op->pool); + fmt->dump_string("name", op->name); + fmt->dump_int("operation_type", op->pool_op); + fmt->dump_unsigned("auid", op->auid); + fmt->dump_unsigned("crush_rule", op->crush_rule); + fmt->dump_stream("snapid") << op->snapid; + fmt->dump_stream("last_sent") << op->last_submit; + fmt->close_section(); // pool_op object } - fmt.close_section(); // pool_ops array + fmt->close_section(); // pool_ops array } -void Objecter::dump_pool_stat_ops(Formatter& fmt) const +void Objecter::dump_pool_stat_ops(Formatter *fmt) const { - fmt.open_array_section("pool_stat_ops"); + fmt->open_array_section("pool_stat_ops"); for (map::const_iterator p = poolstat_ops.begin(); p != poolstat_ops.end(); ++p) { PoolStatOp *op = p->second; - fmt.open_object_section("pool_stat_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("last_sent") << op->last_submit; + fmt->open_object_section("pool_stat_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("last_sent") << op->last_submit; - fmt.open_array_section("pools"); + fmt->open_array_section("pools"); for (list::const_iterator it = op->pools.begin(); it != op->pools.end(); ++it) { - fmt.dump_string("pool", *it); + fmt->dump_string("pool", *it); } - fmt.close_section(); // pool_op object + fmt->close_section(); // pool_op object - fmt.close_section(); // pool_stat_op object + fmt->close_section(); // pool_stat_op object } - fmt.close_section(); // pool_stat_ops array + fmt->close_section(); // pool_stat_ops array } -void Objecter::dump_statfs_ops(Formatter& fmt) const +void Objecter::dump_statfs_ops(Formatter *fmt) const { - fmt.open_array_section("statfs_ops"); + fmt->open_array_section("statfs_ops"); for (map::const_iterator p = statfs_ops.begin(); p != statfs_ops.end(); ++p) { StatfsOp *op = p->second; - fmt.open_object_section("statfs_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("last_sent") << op->last_submit; - fmt.close_section(); // pool_stat_op object + fmt->open_object_section("statfs_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("last_sent") << op->last_submit; + fmt->close_section(); // pool_stat_op object } - fmt.close_section(); // pool_stat_ops array + fmt->close_section(); // pool_stat_ops array } Objecter::RequestStateHook::RequestStateHook(Objecter *objecter) : @@ -2335,14 +2335,16 @@ Objecter::RequestStateHook::RequestStateHook(Objecter *objecter) : { } -bool Objecter::RequestStateHook::call(std::string command, std::string args, bufferlist& out) +bool Objecter::RequestStateHook::call(std::string command, std::string args, + std::string format, bufferlist& out) { stringstream ss; - JSONFormatter formatter(true); + Formatter *f = new_formatter(format); m_objecter->client_lock.Lock(); - m_objecter->dump_requests(formatter); + m_objecter->dump_requests(f); m_objecter->client_lock.Unlock(); - formatter.flush(ss); + f->flush(ss); + delete f; out.append(ss); return true; } diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h index c1cac88b60e4f..aa4a20d8b0bec 100644 --- a/src/osdc/Objecter.h +++ b/src/osdc/Objecter.h @@ -726,7 +726,8 @@ class Objecter { Objecter *m_objecter; public: RequestStateHook(Objecter *objecter); - bool call(std::string command, std::string args, bufferlist& out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out); }; RequestStateHook *m_request_state_hook; @@ -1236,13 +1237,13 @@ class Objecter { * Output in-flight requests */ void dump_active(); - void dump_requests(Formatter& fmt) const; - void dump_ops(Formatter& fmt) const; - void dump_linger_ops(Formatter& fmt) const; - void dump_command_ops(Formatter& fmt) const; - void dump_pool_ops(Formatter& fmt) const; - void dump_pool_stat_ops(Formatter& fmt) const; - void dump_statfs_ops(Formatter& fmt) const; + void dump_requests(Formatter *fmt) const; + void dump_ops(Formatter *fmt) const; + void dump_linger_ops(Formatter *fmt) const; + void dump_command_ops(Formatter *fmt) const; + void dump_pool_ops(Formatter *fmt) const; + void dump_pool_stat_ops(Formatter *fmt) const; + void dump_statfs_ops(Formatter *fmt) const; int get_client_incarnation() const { return client_inc; } void set_client_incarnation(int inc) { client_inc = inc; } diff --git a/src/test/bench/small_io_bench_fs.cc b/src/test/bench/small_io_bench_fs.cc index 61fbacc5570ca..138757f7304c1 100644 --- a/src/test/bench/small_io_bench_fs.cc +++ b/src/test/bench/small_io_bench_fs.cc @@ -32,7 +32,9 @@ struct MorePrinting : public DetailedStatCollector::AdditionalPrinting { MorePrinting(CephContext *cct) : cct(cct) {} void operator()(std::ostream *out) { bufferlist bl; - cct->get_perfcounters_collection()->write_json_to_buf(bl, 0); + Formatter *f = new_formatter("json-pretty"); + cct->get_perfcounters_collection()->dump_formatted(f, bl, 0); + delete f; bl.append('\0'); *out << bl.c_str() << std::endl; } From c7c4c23e5faea2182b8f9d18d12a31608e428f28 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 23 Jul 2013 17:23:50 -0700 Subject: [PATCH 02/44] Formatter, admin_socket: make default formatter be json-pretty If not given, default to json-pretty; if given but not equal to one of the formatter choices, return NULL as before. Remove defaulting code in admin_socket.cc in favor of this. Signed-off-by: Dan Mick --- src/common/Formatter.cc | 14 +++++++++----- src/common/Formatter.h | 2 +- src/common/admin_socket.cc | 11 +---------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index 357b287fe3222..7362684c070eb 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -62,15 +62,19 @@ Formatter::~Formatter() } Formatter * -new_formatter(const std::string &type) +new_formatter(const std::string type) { - if (type == "json") + std::string mytype = type; + if (mytype == "") + mytype = "json-pretty"; + + if (mytype == "json") return new JSONFormatter(false); - else if (type == "json-pretty") + else if (mytype == "json-pretty") return new JSONFormatter(true); - else if (type == "xml") + else if (mytype == "xml") return new XMLFormatter(false); - else if (type == "xml-pretty") + else if (mytype == "xml-pretty") return new XMLFormatter(true); else return (Formatter *)NULL; diff --git a/src/common/Formatter.h b/src/common/Formatter.h index 8775c0cf9df5c..c62a8303ce113 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -60,7 +60,7 @@ class Formatter { } }; -Formatter *new_formatter(const std::string &type); +Formatter *new_formatter(const std::string type); class JSONFormatter : public Formatter { public: diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index 5ebb3e040db8d..4afd685b72acd 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -322,13 +322,6 @@ bool AdminSocket::do_accept() cmd_getval(m_cct, cmdmap, "format", format); cmd_getval(m_cct, cmdmap, "prefix", c); - // we don't do plain here - if (format != "json" && - format != "json-pretty" && - format != "xml" && - format != "xml-pretty") - format = "json"; - string firstword; if (c.find(" ") == string::npos) firstword = c; @@ -450,9 +443,7 @@ class HelpHook : public AdminSocketHook { public: HelpHook(AdminSocket *as) : m_as(as) {} bool call(string command, string args, string format, bufferlist& out) { - // override format here because help should always be pretty and - // predictable - Formatter *f = new_formatter("json-pretty"); + Formatter *f = new_formatter(format); f->open_object_section("help"); for (map::iterator p = m_as->m_help.begin(); p != m_as->m_help.end(); From ba6ca5829a6ec20c4cdbc990f1e27a0709ec5a41 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 23 Jul 2013 17:24:52 -0700 Subject: [PATCH 03/44] In general, flush in caller of dump worker rather than worker This allows easier refactoring of workers (no dual flushes when code changes). Signed-off-by: Dan Mick --- src/common/ceph_context.cc | 6 +++--- src/common/perf_counters.cc | 4 +--- src/common/perf_counters.h | 2 +- src/osd/OSD.cc | 9 ++++----- src/osd/OpRequest.cc | 6 ++---- src/osd/OpRequest.h | 4 ++-- src/test/bench/small_io_bench_fs.cc | 3 ++- 7 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index e0fcf530222f4..6b227d8689e31 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -170,11 +170,11 @@ void CephContext::do_command(std::string command, std::string args, lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "'" << dendl; if (command == "perfcounters_dump" || command == "1" || command == "perf dump") { - _perf_counters_collection->dump_formatted(f, *out, false); + _perf_counters_collection->dump_formatted(f, false); } else if (command == "perfcounters_schema" || command == "2" || command == "perf schema") { - _perf_counters_collection->dump_formatted(f, *out, true); + _perf_counters_collection->dump_formatted(f, true); } else { f->open_object_section(command.c_str()); @@ -221,8 +221,8 @@ void CephContext::do_command(std::string command, std::string args, assert(0 == "registered under wrong command?"); } f->close_section(); - f->flush(*out); } + f->flush(*out); delete f; lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "' result is " << out->length() << " bytes" << dendl; }; diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index 46f55fae51bfc..86fb531f737c8 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -73,8 +73,7 @@ void PerfCountersCollection::clear() } } -void PerfCountersCollection::dump_formatted(Formatter *f, bufferlist &bl, - bool schema) +void PerfCountersCollection::dump_formatted(Formatter *f, bool schema) { Mutex::Locker lck(m_lock); f->open_object_section("perfcounter_collection"); @@ -88,7 +87,6 @@ void PerfCountersCollection::dump_formatted(Formatter *f, bufferlist &bl, } } f->close_section(); - f->flush(bl); } // --------------------------- diff --git a/src/common/perf_counters.h b/src/common/perf_counters.h index 125d84c04e34a..ec10f9a928286 100644 --- a/src/common/perf_counters.h +++ b/src/common/perf_counters.h @@ -136,7 +136,7 @@ class PerfCountersCollection void add(class PerfCounters *l); void remove(class PerfCounters *l); void clear(); - void dump_formatted(ceph::Formatter *f, bufferlist &bl, bool schema); + void dump_formatted(ceph::Formatter *f, bool schema); private: CephContext *m_cct; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index dd9a5b7293f33..c767a2c0fe5f6 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -1012,14 +1012,13 @@ bool OSD::asok_command(string command, string args, string format, ostream& ss) format = "json-pretty"; Formatter *f = new_formatter(format); if (command == "dump_ops_in_flight") { - op_tracker.dump_ops_in_flight(f, ss); + op_tracker.dump_ops_in_flight(f); } else if (command == "dump_historic_ops") { - op_tracker.dump_historic_ops(f, ss); + op_tracker.dump_historic_ops(f); } else if (command == "dump_op_pq_state") { f->open_object_section("pq"); op_wq.dump(f); f->close_section(); - f->flush(ss); } else if (command == "dump_blacklist") { list > bl; OSDMapRef curmap = service.get_osdmap(); @@ -1036,7 +1035,6 @@ bool OSD::asok_command(string command, string args, string format, ostream& ss) f->close_section(); //entry } f->close_section(); //blacklist - f->flush(ss); } else if (command == "dump_watchers") { list watchers; osd_lock.Lock(); @@ -1078,10 +1076,11 @@ bool OSD::asok_command(string command, string args, string format, ostream& ss) } f->close_section(); //watches - f->flush(ss); } else { assert(0 == "broken asok registration"); } + f->flush(ss); + delete f; return true; } diff --git a/src/osd/OpRequest.cc b/src/osd/OpRequest.cc index c0d167a5f0ad1..a6cdc9ecffb5e 100644 --- a/src/osd/OpRequest.cc +++ b/src/osd/OpRequest.cc @@ -76,15 +76,14 @@ void OpHistory::dump_ops(utime_t now, Formatter *f) f->close_section(); } -void OpTracker::dump_historic_ops(Formatter *f, ostream &ss) +void OpTracker::dump_historic_ops(Formatter *f) { Mutex::Locker locker(ops_in_flight_lock); utime_t now = ceph_clock_now(g_ceph_context); history.dump_ops(now, f); - f->flush(ss); } -void OpTracker::dump_ops_in_flight(Formatter *f, ostream &ss) +void OpTracker::dump_ops_in_flight(Formatter *f) { Mutex::Locker locker(ops_in_flight_lock); f->open_object_section("ops_in_flight"); // overall dump @@ -98,7 +97,6 @@ void OpTracker::dump_ops_in_flight(Formatter *f, ostream &ss) } f->close_section(); // list of OpRequests f->close_section(); // overall dump - f->flush(ss); } void OpTracker::register_inflight_op(xlist::item *i) diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h index 67ee26b02ec14..a2014472432da 100644 --- a/src/osd/OpRequest.h +++ b/src/osd/OpRequest.h @@ -59,8 +59,8 @@ class OpTracker { public: OpTracker() : seq(0), ops_in_flight_lock("OpTracker mutex") {} - void dump_ops_in_flight(Formatter *f, std::ostream& ss); - void dump_historic_ops(Formatter *f, std::ostream& ss); + void dump_ops_in_flight(Formatter *f); + void dump_historic_ops(Formatter *f); void register_inflight_op(xlist::item *i); void unregister_inflight_op(OpRequest *i); diff --git a/src/test/bench/small_io_bench_fs.cc b/src/test/bench/small_io_bench_fs.cc index 138757f7304c1..a37a7e71153a6 100644 --- a/src/test/bench/small_io_bench_fs.cc +++ b/src/test/bench/small_io_bench_fs.cc @@ -33,7 +33,8 @@ struct MorePrinting : public DetailedStatCollector::AdditionalPrinting { void operator()(std::ostream *out) { bufferlist bl; Formatter *f = new_formatter("json-pretty"); - cct->get_perfcounters_collection()->dump_formatted(f, bl, 0); + cct->get_perfcounters_collection()->dump_formatted(f, 0); + f->flush(bl); delete f; bl.append('\0'); *out << bl.c_str() << std::endl; From 67eb7de42f40eb080438a2f22303265c987a4b27 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 14:39:51 -0700 Subject: [PATCH 04/44] ceph_argparse.py, ceph.in: validate_command: stop handling format We were passing the entire parsed_args Namespace just to look at and tack on 'format', and that's kinda silly; do it in the callers instead. Signed-off-by: Dan Mick --- src/ceph.in | 9 ++++++--- src/pybind/ceph_argparse.py | 6 +----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ceph.in b/src/ceph.in index 0d361e1c76ce6..90795058127ca 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -344,10 +344,11 @@ def new_style_command(parsed_args, cmdargs, target, sigdict, inbuf, verbose): if not got_command: if cmdargs: # Validate input args against list of sigs - valid_dict = validate_command(parsed_args, sigdict, cmdargs, - verbose) + valid_dict = validate_command(sigdict, cmdargs, verbose) if valid_dict: got_command = True + if parsed_args.output_format: + valid_dict['format'] = parsed_args.output_format else: return -errno.EINVAL, '', 'invalid command' else: @@ -360,8 +361,10 @@ def new_style_command(parsed_args, cmdargs, target, sigdict, inbuf, verbose): return 0, '', '' cmdargs = parse_cmdargs(interactive_input.split())[2] target = find_cmd_target(cmdargs) - valid_dict = validate_command(parsed_args, sigdict, cmdargs) + valid_dict = validate_command(sigdict, cmdargs, verbose) if valid_dict: + if parsed_args.output_format: + valid_dict['format'] = parsed_args.output_format if verbose: print >> sys.stderr, "Submitting command ", valid_dict ret, outbuf, outs = json_command(cluster_handle, diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index b82cc833ba66d..b014d7d626c48 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -820,11 +820,10 @@ def validate(args, signature, partial=False): raise ArgumentError("unused arguments: " + str(myargs)) return d -def validate_command(parsed_args, sigdict, args, verbose=False): +def validate_command(sigdict, args, verbose=False): """ turn args into a valid dictionary ready to be sent off as JSON, validated against sigdict. - parsed_args is the namespace back from argparse """ found = [] valid_dict = {} @@ -882,9 +881,6 @@ def validate_command(parsed_args, sigdict, args, verbose=False): print >> sys.stderr, concise_sig(cmd['sig']) return None - if parsed_args.output_format: - valid_dict['format'] = parsed_args.output_format - return valid_dict def find_cmd_target(childargs): From cb38762bca942e8e8c1a89bca6d3c6362182385f Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 14:43:37 -0700 Subject: [PATCH 05/44] ceph.in: admin_socket() now validates command and passes format Signed-off-by: Dan Mick --- src/ceph.in | 70 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/src/ceph.in b/src/ceph.in index 90795058127ca..a72761216bd52 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -285,26 +285,57 @@ def format_help(cmddict, partial=None): return fullusage -def admin_socket(asok_path, cmd): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +def admin_socket(asok_path, cmd, format=''): + """ + Send a daemon (--admin-daemon) command 'cmd'. asok_path is the + path to the admin socket; cmd is a list of strings; format may be + set to one of the formatted forms to get output in that form + (daemon commands don't support 'plain' output). + """ + + def do_sockio(path, cmd): + """ helper: do all the actual low-level stream I/O """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + try: + sock.sendall(cmd + '\0') + len_str = sock.recv(4) + if len(len_str) < 4: + raise RuntimeError("no data returned from admin socket") + l, = struct.unpack(">I", len_str) + ret = '' + + got = 0 + while got < l: + bit = sock.recv(l - got) + ret += bit + got += len(bit) + + except Exception as e: + raise RuntimeError('exception: ' + str(e)) + return ret + try: - sock.connect(asok_path) - sock.sendall(' '.join(cmd) + '\0') + cmd_json = do_sockio(asok_path, + json.dumps({"prefix":"get_command_descriptions"})) + except Exception as e: + raise RuntimeError('exception getting command descriptions: ' + str(e)) + + if cmd[0] == 'get_command_descriptions': + return cmd_json - len_str = sock.recv(4) - if len(len_str) < 4: - raise RuntimeError("no data returned from admin socket") - l, = struct.unpack(">I", len_str) - ret = '' + sigdict = parse_json_funcsigs(cmd_json, 'cli') + valid_dict = validate_command(sigdict, cmd) + if not valid_dict: + return -errno.EINVAL - got = 0 - while got < l: - bit = sock.recv(l - got) - ret += bit - got += len(bit) + if format: + valid_dict['format'] = format + try: + ret = do_sockio(asok_path, json.dumps(valid_dict)) except Exception as e: - raise RuntimeError('exception: {0}'.format(e)) + raise RuntimeError('exception: ' + str(e)) return ret @@ -473,9 +504,12 @@ def main(): conffile = parsed_args.cephconf # For now, --admin-daemon is handled as usual. Try it # first in case we can't connect() to the cluster + + format = parsed_args.output_format + if parsed_args.admin_socket: try: - print admin_socket(parsed_args.admin_socket, childargs) + print admin_socket(parsed_args.admin_socket, childargs, format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 @@ -484,7 +518,7 @@ def main(): if len(childargs) > 2: if childargs[1].find('/') >= 0: try: - print admin_socket(childargs[1], childargs[2:]) + print admin_socket(childargs[1], childargs[2:], format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 @@ -492,7 +526,7 @@ def main(): # try resolve daemon name path = ceph_conf('admin_socket', childargs[1]) try: - print admin_socket(path, childargs[2:]) + print admin_socket(path, childargs[2:], format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 From cd16d73fda4fc25fc44d1677b788ad7aae208d30 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 14:44:17 -0700 Subject: [PATCH 06/44] ceph.in: clean up help, fix partial matching on all help Remove --help-all; too much effort for low benefit Signed-off-by: Dan Mick --- src/ceph.in | 57 +++++++++++++++++------------------------------------ 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/src/ceph.in b/src/ceph.in index a72761216bd52..63c41343f9a52 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -100,8 +100,6 @@ def parse_cmdargs(args=None, target=''): parser.add_argument('-h', '--help', help='request mon help', action='store_true') - parser.add_argument('--help-all', help='request help for all daemons', - action='store_true') parser.add_argument('-c', '--conf', dest='cephconf', help='ceph configuration file') @@ -150,14 +148,16 @@ def parse_cmdargs(args=None, target=''): return parser, parsed_args, extras -def do_help(parser, args, help_all = False): +def do_help(parser, args): """ Print basic parser help - If the cluster is available: - get and print monitor help; - if help_all, print help for daemon commands as well + If the cluster is available, get and print monitor help """ + def help_for_sigs(sigs, partial=None): + sys.stdout.write(format_help(parse_json_funcsigs(sigs, 'cli'), + partial=partial)) + def help_for_target(target, partial=None): ret, outbuf, outs = json_command(cluster_handle, target=target, prefix='get_command_descriptions', @@ -167,40 +167,19 @@ def do_help(parser, args, help_all = False): "couldn't get command descriptions for {0}: {1}".\ format(target, outs) else: - sys.stdout.write(format_help(parse_json_funcsigs(outbuf, 'cli'), - partial)) - - parser.print_help() - print '\n' - if (cluster_handle): - help_for_target(target=('mon', ''), partial=' '.join(args)) + help_for_sigs(outbuf, partial) - if help_all and cluster_handle: - # try/except in case there are no daemons of that type - try: - firstosd = osdids()[0] - print '\nOSD.{0} tell commands and pg pgid commands:\n\n'.\ - format(firstosd) - help_for_target(target=('osd', osdids()[0])) - print '\nOSD daemon commands:\n\n' - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'osd.' + firstosd), ['get_command_descriptions']), 'cli'))) - except: - pass + def hdr(s): + print '\n', s, '\n', '=' * len(s) - try: - firstmon = monids()[0] - print '\nmon.{0} daemon commands:\n\n'.format(firstmon) - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'mon.' + firstmon), ['get_command_descriptions']), 'cli'))) - except: - pass + hdr('Monitor commands:') + partial = ' '.join(args) + parser.print_help() + print '\n' - try: - firstmds = mdsids()[0] - print '\nmds.{0} daemon commands:\n\n'.format(firstmds) - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'mds.' + firstmds), ['get_command_descriptions']), 'cli'))) - except: - pass + if (cluster_handle): + help_for_target(target=('mon', ''), partial=partial) return 0 @@ -321,7 +300,7 @@ def admin_socket(asok_path, cmd, format=''): except Exception as e: raise RuntimeError('exception getting command descriptions: ' + str(e)) - if cmd[0] == 'get_command_descriptions': + if cmd == 'get_command_descriptions': return cmd_json sigdict = parse_json_funcsigs(cmd_json, 'cli') @@ -581,8 +560,8 @@ def main(): format(e.__class__.__name__) return 1 - if parsed_args.help or parsed_args.help_all: - return do_help(parser, childargs, parsed_args.help_all) + if parsed_args.help: + return do_help(parser, childargs) # implement -w/--watch_* # This is ugly, but Namespace() isn't quite rich enough. From c9fcda88a7831ccbf879cf8baf0a91163e437aa3 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 16:34:23 -0700 Subject: [PATCH 07/44] OSD: provide newer command descs with module/perm/avail Signed-off-by: Dan Mick --- src/osd/OSD.cc | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index c767a2c0fe5f6..a65772ada3d03 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3813,10 +3813,13 @@ void OSD::handle_command(MCommand *m) struct OSDCommand { string cmdstring; string helpstring; + string module; + string perm; + string availability; } osd_commands[] = { -#define COMMAND(parsesig, helptext) \ - {parsesig, helptext}, +#define COMMAND(parsesig, helptext, module, perm, availability) \ + {parsesig, helptext, module, perm, availability}, // yes, these are really pg commands, but there's a limit to how // much work it's worth. The OSD returns all of them. @@ -3824,40 +3827,45 @@ struct OSDCommand { COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=query", \ - "show details of a specific pg") + "show details of a specific pg", "osd", "r", "cli,rest") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=mark_unfound_lost " \ "name=mulcmd,type=CephChoices,strings=revert", \ - "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available") + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", + "osd", "rw", "cli,rest") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=list_missing " \ "name=offset,type=CephString,req=false", - "list missing objects on this pg, perhaps starting at an offset given in JSON") + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "rw", "cli,rest") -COMMAND("version", "report version of OSD") +COMMAND("version", "report version of OSD", "osd", "r", "cli,rest") COMMAND("injectargs " \ "name=injected_args,type=CephString,n=N", - "inject configuration arguments into running OSD") + "inject configuration arguments into running OSD", + "osd", "rw", "cli,rest") COMMAND("bench " \ "name=count,type=CephInt,req=false " \ "name=size,type=CephInt,req=false ", \ "OSD benchmark: write -byte objects, " \ - "(default 1G size 4MB). Results in log.") -COMMAND("flush_pg_stats", "flush pg stats") -COMMAND("debug dump_missing " \ + "(default 1G size 4MB). Results in log.", + "osd", "rw", "cli,rest") +COMMAND("flush_pg_stats", "flush pg stats", "osd", "rw", "cli,rest") +COMMAND("debug_dump_missing " \ "name=filename,type=CephFilepath", - "dump missing objects to a named file") + "dump missing objects to a named file", "osd", "r", "cli,rest") COMMAND("debug kick_recovery_wq " \ "name=delay,type=CephInt,range=0", - "set osd_recovery_delay_start to ") + "set osd_recovery_delay_start to ", "osd", "rw", "cli,rest") COMMAND("cpu_profiler " \ "name=arg,type=CephChoices,strings=status|flush", - "run cpu profiling on daemon") -COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics") -COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics") - + "run cpu profiling on daemon", "osd", "rw", "cli,rest") +COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics", + "osd", "r", "cli,rest") +COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics", + "osd", "rw", "cli,rest") }; void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist& data) @@ -3893,8 +3901,8 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist ostringstream secname; secname << "cmd" << setfill('0') << std::setw(3) << cmdnum; - dump_cmd_and_help_to_json(f, secname.str(), - cp->cmdstring, cp->helpstring); + dump_cmddesc_to_json(f, secname.str(), cp->cmdstring, cp->helpstring, + cp->module, cp->perm, cp->availability); cmdnum++; } f->close_section(); // command_descriptions From 6e6ceffa1b36c4d1d6b6fc7885322c8000f7dd77 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 19 Jul 2013 18:03:24 -0700 Subject: [PATCH 08/44] OSD: "tell " version, bench, dump_pg_recovery_stats: formatted Signed-off-by: Dan Mick --- src/osd/OSD.cc | 40 +++++++++++++++++++++++++++++++++------- src/osd/PG.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index a65772ada3d03..76b11672b8137 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3879,6 +3879,8 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist map cmdmap; string prefix; + string format; + boost::scoped_ptr f; if (cmd.empty()) { ss << "no command given"; @@ -3913,8 +3915,18 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist goto out; } + cmd_getval(g_ceph_context, cmdmap, "format", format); + f.reset(new_formatter(format)); + if (prefix == "version") { - ds << pretty_version_to_str(); + if (f) { + f->open_object_section("version"); + f->dump_string("version", pretty_version_to_str()); + f->close_section(); + f->flush(ds); + } else { + ds << pretty_version_to_str(); + } goto out; } else if (prefix == "injectargs") { @@ -3952,7 +3964,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist ss << "i don't have pgid " << pgid; r = -ENOENT; } else { - r = pg->do_command(cmd, ss, data, odata); + r = pg->do_command(cmdmap, ss, data, odata); pg->unlock(); } } @@ -3991,9 +4003,18 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist store->queue_transaction(NULL, cleanupt); uint64_t rate = (double)count / (end - start); - ss << "bench: wrote " << prettybyte_t(count) - << " in blocks of " << prettybyte_t(bsize) << " in " - << (end-start) << " sec at " << prettybyte_t(rate) << "/sec"; + if (f) { + f->open_object_section("osd_bench_results"); + f->dump_int("bytes_written", count); + f->dump_int("blocksize", bsize); + f->dump_float("bytes_per_sec", rate); + f->close_section(); + f->flush(ss); + } else { + ss << "bench: wrote " << prettybyte_t(count) + << " in blocks of " << prettybyte_t(bsize) << " in " + << (end-start) << " sec at " << prettybyte_t(rate) << "/sec"; + } } else if (prefix == "flush_pg_stats") { @@ -4090,8 +4111,13 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist else if (prefix == "dump_pg_recovery_stats") { stringstream s; - pg_recovery_stats.dump(s); - ds << "dump pg recovery stats: " << s.str(); + if (f) { + pg_recovery_stats.dump_formatted(f.get()); + f->flush(ds); + } else { + pg_recovery_stats.dump(s); + ds << "dump pg recovery stats: " << s.str(); + } } else if (prefix == "reset_pg_recovery_stats") { diff --git a/src/osd/PG.h b/src/osd/PG.h index 819c9c62f62e3..5525c7f46dd26 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -45,6 +45,7 @@ #include "messages/MOSDPGLog.h" #include "common/tracked_int_ptr.hpp" #include "common/WorkQueue.h" +#include "include/str_list.h" #include #include @@ -112,6 +113,33 @@ struct PGRecoveryStats { } } + void dump_formatted(Formatter *f) { + Mutex::Locker l(lock); + f->open_array_section("pg_recovery_stats"); + for (map::iterator p = info.begin(); + p != info.end(); ++p) { + per_state_info& i = p->second; + f->open_object_section("recovery_state"); + f->dump_int("enter", i.enter); + f->dump_int("exit", i.exit); + f->dump_int("events", i.events); + f->dump_stream("event_time") << i.event_time; + f->dump_stream("total_time") << i.total_time; + f->dump_stream("min_time") << i.min_time; + f->dump_stream("max_time") << i.max_time; + vector states; + get_str_vec(p->first, "/", states); + f->open_array_section("nested_states"); + for (vector::iterator st = states.begin(); + st != states.end(); ++st) { + f->dump_string("state", *st); + } + f->close_section(); + f->close_section(); + } + f->close_section(); + } + void log_enter(const char *s) { Mutex::Locker l(lock); info[s].enter++; From bcbb807c018f89d473a252d87e8d48b5220b3a61 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 19 Jul 2013 18:04:29 -0700 Subject: [PATCH 09/44] PG: add formatted output to pg query, list_missing Signed-off-by: Dan Mick --- src/osd/PG.h | 4 +- src/osd/ReplicatedPG.cc | 107 ++++++++++++++++++---------------------- src/osd/ReplicatedPG.h | 4 +- 3 files changed, 53 insertions(+), 62 deletions(-) diff --git a/src/osd/PG.h b/src/osd/PG.h index 5525c7f46dd26..10e9a2544a9b0 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -43,6 +43,7 @@ #include "msg/Messenger.h" #include "messages/MOSDRepScrub.h" #include "messages/MOSDPGLog.h" +#include "common/cmdparse.h" #include "common/tracked_int_ptr.hpp" #include "common/WorkQueue.h" #include "include/str_list.h" @@ -109,7 +110,6 @@ struct PGRecoveryStats { << i.total_time << "\t" << i.min_time << "\t" << i.max_time << "\t" << p->first << "\n"; - } } @@ -1814,7 +1814,7 @@ class PG { virtual void do_push_reply(OpRequestRef op) = 0; virtual void snap_trimmer() = 0; - virtual int do_command(vector& cmd, ostream& ss, + virtual int do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, bufferlist& odata) = 0; virtual bool same_for_read_since(epoch_t e) = 0; diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 298d38d6ace64..4a59a23cdb794 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -268,23 +268,12 @@ int ReplicatedPG::get_pgls_filter(bufferlist::iterator& iter, PGLSFilter **pfilt // ========================================================== -int ReplicatedPG::do_command(vector& cmd, ostream& ss, +int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, bufferlist& odata) { const pg_missing_t &missing = pg_log.get_missing(); - map cmdmap; string prefix; - - if (cmd.empty()) { - ss << "no command given"; - return -EINVAL; - } - - stringstream ss2; - if (!cmdmap_from_json(cmd, &cmdmap, ss2)) { - ss << ss2.str(); - return -EINVAL; - } + string format; cmd_getval(g_ceph_context, cmdmap, "prefix", prefix); if (prefix != "pg") { @@ -292,33 +281,36 @@ int ReplicatedPG::do_command(vector& cmd, ostream& ss, return -EINVAL; } + cmd_getval(g_ceph_context, cmdmap, "format", format); + boost::scoped_ptr f(new_formatter(format)); + // demand that we have a formatter + if (!f) + f.reset(new_formatter("json")); + string command; cmd_getval(g_ceph_context, cmdmap, "cmd", command); if (command == "query") { - JSONFormatter jsf(true); - jsf.open_object_section("pg"); - jsf.dump_string("state", pg_state_string(get_state())); - jsf.dump_unsigned("epoch", get_osdmap()->get_epoch()); - jsf.open_array_section("up"); + f->open_object_section("pg"); + f->dump_string("state", pg_state_string(get_state())); + f->dump_unsigned("epoch", get_osdmap()->get_epoch()); + f->open_array_section("up"); for (vector::iterator p = up.begin(); p != up.end(); ++p) - jsf.dump_unsigned("osd", *p); - jsf.close_section(); - jsf.open_array_section("acting"); + f->dump_unsigned("osd", *p); + f->close_section(); + f->open_array_section("acting"); for (vector::iterator p = acting.begin(); p != acting.end(); ++p) - jsf.dump_unsigned("osd", *p); - jsf.close_section(); - jsf.open_object_section("info"); - info.dump(&jsf); - jsf.close_section(); - - jsf.open_array_section("recovery_state"); - handle_query_state(&jsf); - jsf.close_section(); - - jsf.close_section(); - stringstream dss; - jsf.flush(dss); - odata.append(dss); + f->dump_unsigned("osd", *p); + f->close_section(); + f->open_object_section("info"); + info.dump(f.get()); + f->close_section(); + + f->open_array_section("recovery_state"); + handle_query_state(f.get()); + f->close_section(); + + f->close_section(); + f->flush(odata); return 0; } else if (command == "mark_unfound_lost") { @@ -352,7 +344,6 @@ int ReplicatedPG::do_command(vector& cmd, ostream& ss, return 0; } else if (command == "list_missing") { - JSONFormatter jf(true); hobject_t offset; string offset_json; if (cmd_getval(g_ceph_context, cmdmap, "offset", offset_json)) { @@ -366,50 +357,48 @@ int ReplicatedPG::do_command(vector& cmd, ostream& ss, return -EINVAL; } } - jf.open_object_section("missing"); + f->open_object_section("missing"); { - jf.open_object_section("offset"); - offset.dump(&jf); - jf.close_section(); + f->open_object_section("offset"); + offset.dump(f.get()); + f->close_section(); } - jf.dump_int("num_missing", missing.num_missing()); - jf.dump_int("num_unfound", get_num_unfound()); + f->dump_int("num_missing", missing.num_missing()); + f->dump_int("num_unfound", get_num_unfound()); map::const_iterator p = missing.missing.upper_bound(offset); { - jf.open_array_section("objects"); + f->open_array_section("objects"); int32_t num = 0; bufferlist bl; while (p != missing.missing.end() && num < g_conf->osd_command_max_records) { - jf.open_object_section("object"); + f->open_object_section("object"); { - jf.open_object_section("oid"); - p->first.dump(&jf); - jf.close_section(); + f->open_object_section("oid"); + p->first.dump(f.get()); + f->close_section(); } - p->second.dump(&jf); // have, need keys + p->second.dump(f.get()); // have, need keys { - jf.open_array_section("locations"); + f->open_array_section("locations"); map >::iterator q = missing_loc.find(p->first); if (q != missing_loc.end()) for (set::iterator r = q->second.begin(); r != q->second.end(); ++r) - jf.dump_int("osd", *r); - jf.close_section(); + f->dump_int("osd", *r); + f->close_section(); } - jf.close_section(); + f->close_section(); ++p; num++; } - jf.close_section(); + f->close_section(); } - jf.dump_int("more", p != missing.missing.end()); - jf.close_section(); - stringstream jss; - jf.flush(jss); - odata.append(jss); + f->dump_int("more", p != missing.missing.end()); + f->close_section(); + f->flush(odata); return 0; }; - ss << "unknown command " << cmd; + ss << "unknown pg command " << prefix; return -EINVAL; } diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 9dafe23faa1fb..7b70b4381ea4f 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -17,6 +17,7 @@ #include #include "include/assert.h" +#include "common/cmdparse.h" #include "PG.h" #include "OSD.h" @@ -930,7 +931,8 @@ class ReplicatedPG : public PG { const hobject_t& ioid); ~ReplicatedPG() {} - int do_command(vector& cmd, ostream& ss, bufferlist& idata, bufferlist& odata); + int do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, + bufferlist& odata); void do_op(OpRequestRef op); bool pg_op_must_wait(MOSDOp *op); From 6ac8aed040049358aaf8aee5cfb16791c4bb54a2 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 15:25:12 -0700 Subject: [PATCH 10/44] mon/PGMonitor: reset in-core PGMap if on-disk format changes We might have a sequence like: - start mon, load pgmap 100 - sync - including a format upgrade at say v 150 - refresh - see format_version==1, and try read pgmap:101 as new format This simply clears our in-memory state if we see that the format has changed. That will make update_from_paxos reload the latest and prevent it from walking through the old and useless inc updates. Note: this does not affect the auth monitor because we unconditionally load the latest map in update_from_paxos on upgrade. Also, the upgrade there wasn't a format change--just a translation of cap strings from the old to new style. Fixes: #5764 Signed-off-by: Sage Weil Reviewed-by: Greg Farnum --- src/mon/PGMonitor.cc | 6 ++++++ src/mon/PGMonitor.h | 1 + src/mon/PaxosService.cc | 8 +++++++- src/mon/PaxosService.h | 5 +++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc index 648a8fe238466..d86cbe70c19fd 100644 --- a/src/mon/PGMonitor.cc +++ b/src/mon/PGMonitor.cc @@ -255,6 +255,12 @@ void PGMonitor::update_from_paxos(bool *need_bootstrap) update_logger(); } +void PGMonitor::on_upgrade() +{ + dout(1) << __func__ << " discarding in-core PGMap" << dendl; + pg_map = PGMap(); +} + void PGMonitor::upgrade_format() { unsigned current = 1; diff --git a/src/mon/PGMonitor.h b/src/mon/PGMonitor.h index e8e1b4210aaaa..44015395e943e 100644 --- a/src/mon/PGMonitor.h +++ b/src/mon/PGMonitor.h @@ -60,6 +60,7 @@ class PGMonitor : public PaxosService { void create_initial(); void update_from_paxos(bool *need_bootstrap); void upgrade_format(); + void on_upgrade(); void post_paxos_update(); void handle_osd_timeouts(); void create_pending(); // prepare a new pending diff --git a/src/mon/PaxosService.cc b/src/mon/PaxosService.cc index d6e67a1c4b4c0..1b21689863bc6 100644 --- a/src/mon/PaxosService.cc +++ b/src/mon/PaxosService.cc @@ -114,7 +114,13 @@ void PaxosService::refresh(bool *need_bootstrap) // update cached versions cached_first_committed = mon->store->get(get_service_name(), first_committed_name); cached_last_committed = mon->store->get(get_service_name(), last_committed_name); - format_version = get_value("format_version"); + + version_t new_format = get_value("format_version"); + if (new_format != format_version) { + dout(1) << __func__ << " upgraded, format " << format_version << " -> " << new_format << dendl; + on_upgrade(); + } + format_version = new_format; dout(10) << __func__ << dendl; diff --git a/src/mon/PaxosService.h b/src/mon/PaxosService.h index 74d5a90494c87..5321bebcacefc 100644 --- a/src/mon/PaxosService.h +++ b/src/mon/PaxosService.h @@ -458,6 +458,11 @@ class PaxosService { */ virtual void upgrade_format() { } + /** + * this is called when we detect the store has just upgraded underneath us + */ + virtual void on_upgrade() {} + /** * Called when the Paxos system enters a Leader election. * From 7b42deef3810e207f0e044769e7c70175993954a Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 23 Jul 2013 00:50:15 -0700 Subject: [PATCH 11/44] ceph_rest_api.py: obtain and handle tell commands Contact an OSD that's up to get a list of the commands, and use them to add to the URL map. Special treatment throughout for these commands: * hack the help signature dump * keep a 'flavor' per command to allow special handler() processing * strip off 'tell/' when constructing command * allow multiple dicts with the same url (the parameters and get/put methods can change) * because of above, method must be validated in handler() * validate the given OSD * calculate target for command (mon, osd, pg) Unrelated: make method_dict into global METHOD_DICT Signed-off-by: Dan Mick --- src/osd/OSD.cc | 23 ++- src/pybind/ceph_argparse.py | 3 + src/pybind/ceph_rest_api.py | 296 +++++++++++++++++++++++++----------- 3 files changed, 229 insertions(+), 93 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 76b11672b8137..d611afdd08fb8 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3822,24 +3822,41 @@ struct OSDCommand { {parsesig, helptext, module, perm, availability}, // yes, these are really pg commands, but there's a limit to how -// much work it's worth. The OSD returns all of them. +// much work it's worth. The OSD returns all of them. Make this +// form (pg ) valid only for the cli. +// Rest uses "tell " COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=query", \ - "show details of a specific pg", "osd", "r", "cli,rest") + "show details of a specific pg", "osd", "r", "cli") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=mark_unfound_lost " \ "name=mulcmd,type=CephChoices,strings=revert", \ "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", - "osd", "rw", "cli,rest") + "osd", "rw", "cli") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=list_missing " \ "name=offset,type=CephString,req=false", "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "r", "cli") + +// new form: tell for both cli and rest + +COMMAND("query", + "show details of a specific pg", "osd", "r", "cli,rest") +COMMAND("mark_unfound_lost " \ + "name=mulcmd,type=CephChoices,strings=revert", \ + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", "osd", "rw", "cli,rest") +COMMAND("list_missing " \ + "name=offset,type=CephString,req=false", + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "r", "cli,rest") + +// tell commands. Validation of osd.n must be special-cased in client COMMAND("version", "report version of OSD", "osd", "r", "cli,rest") COMMAND("injectargs " \ diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index b014d7d626c48..855e21c25084e 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -263,6 +263,8 @@ def valid(self, s, partial=False): if p is not None and long(p) > 65535: raise ArgumentValid("{0} not a valid port number".format(p)) self.val = s + self.addr = a + self.port = p def __str__(self): return '' @@ -274,6 +276,7 @@ class CephEntityAddr(CephIPAddr): def valid(self, s, partial=False): ip, nonce = s.split('/') super(self.__class__, self).valid(ip) + self.nonce = nonce self.val = s def __str__(self): diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index 28a0419c33c21..f8d9b92129e2d 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -1,13 +1,15 @@ #!/usr/bin/python # vim: ts=4 sw=4 smarttab expandtab -import os import collections import ConfigParser +import errno import json import logging import logging.handlers +import os import rados +import socket import textwrap import xml.etree.ElementTree import xml.sax.saxutils @@ -99,6 +101,26 @@ def get_conf(cfg, clientname, key): pass return None +METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} + +def find_up_osd(): + ''' + Find an up OSD. Return the last one that's up. + Returns id as an int. + ''' + ret, outbuf, outs = json_command(glob.cluster, prefix="osd dump", + argdict=dict(format='json')) + if ret: + raise EnvironmentError(ret, 'Can\'t get osd dump output') + try: + osddump = json.loads(outbuf) + except: + raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump') + osds = [osd['osd'] for osd in osddump['osds'] if osd['up']] + if not osds: + raise EnvironmentError(errno.ENOENT, 'No up OSDs found') + return int(osds[-1]) + # XXX this is done globally, and cluster connection kept open; there # are facilities to pass around global info to requests and to # tear down connections between requests if it becomes important @@ -109,6 +131,22 @@ def api_setup(): signatures, module, perms, and help; stuff them away in the glob.urls dict. """ + def get_command_descriptions(target=('mon','')): + ret, outbuf, outs = json_command(glob.cluster, target, + prefix='get_command_descriptions', + timeout=30) + if ret: + err = "Can't get command descriptions: {0}".format(outs) + app.logger.error(err) + raise EnvironmentError(ret, err) + + try: + sigdict = parse_json_funcsigs(outbuf, 'rest') + except Exception as e: + err = "Can't parse command descriptions: {}".format(e) + app.logger.error(err) + raise EnvironmentError(err) + return sigdict conffile = os.environ.get('CEPH_CONF', '') clustername = os.environ.get('CEPH_CLUSTER_NAME', 'ceph') @@ -148,19 +186,22 @@ def api_setup(): h.setFormatter(logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s')) - ret, outbuf, outs = json_command(glob.cluster, - prefix='get_command_descriptions') - if ret: - err = "Can't contact cluster for command descriptions: {0}".format(outs) - app.logger.error(err) - raise EnvironmentError(ret, err) + glob.sigdict = get_command_descriptions() - try: - glob.sigdict = parse_json_funcsigs(outbuf, 'rest') - except Exception as e: - err = "Can't parse command descriptions: {}".format(e) - app.logger.error(err) - raise EnvironmentError(err) + osdid = find_up_osd() + if osdid: + osd_sigdict = get_command_descriptions(target=('osd', int(osdid))) + + # shift osd_sigdict keys up to fit at the end of the mon's glob.sigdict + maxkey = sorted(glob.sigdict.keys())[-1] + maxkey = int(maxkey.replace('cmd', '')) + osdkey = maxkey + 1 + for k, v in osd_sigdict.iteritems(): + newv = v + newv['flavor'] = 'tell' + globk = 'cmd' + str(osdkey) + glob.sigdict[globk] = newv + osdkey += 1 # glob.sigdict maps "cmdNNN" to a dict containing: # 'sig', an array of argdescs @@ -173,27 +214,37 @@ def api_setup(): glob.urls = {} for cmdnum, cmddict in glob.sigdict.iteritems(): cmdsig = cmddict['sig'] - url, params = generate_url_and_params(cmdsig) - if url in glob.urls: - continue + flavor = cmddict.get('flavor', 'mon') + url, params = generate_url_and_params(cmdsig, flavor) + perm = cmddict['perm'] + for k in METHOD_DICT.iterkeys(): + if k in perm: + methods = METHOD_DICT[k] + urldict = {'paramsig':params, + 'help':cmddict['help'], + 'module':cmddict['module'], + 'perm':perm, + 'flavor':flavor, + 'methods':methods, + } + + # glob.urls contains a list of urldicts (usually only one long) + if url not in glob.urls: + glob.urls[url] = [urldict] else: - perm = cmddict['perm'] - urldict = {'paramsig':params, - 'help':cmddict['help'], - 'module':cmddict['module'], - 'perm':perm, - } - method_dict = {'r':['GET'], - 'w':['PUT', 'DELETE']} - for k in method_dict.iterkeys(): - if k in perm: - methods = method_dict[k] - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict - - url += '.' - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict + # If more than one, need to make union of methods of all. + # Method must be checked in handler + methodset = set(methods) + for old_urldict in glob.urls[url]: + methodset |= set(old_urldict['methods']) + methods = list(methodset) + glob.urls[url].append(urldict) + + # add, or re-add, rule with all methods and urldicts + app.add_url_rule(url, url, handler, methods=methods) + url += '.' + app.add_url_rule(url, url, handler, methods=methods) + app.logger.debug("urls added: %d", len(glob.urls)) app.add_url_rule('/', '/', @@ -201,63 +252,84 @@ def api_setup(): return addr, port -def generate_url_and_params(sig): +def generate_url_and_params(sig, flavor): """ Digest command signature from cluster; generate an absolute (including glob.baseurl) endpoint from all the prefix words, - and a dictionary of non-prefix parameters + and a list of non-prefix param descs """ url = '' params = [] + # the OSD command descriptors don't include the 'tell ', so + # tack it onto the front of sig + if flavor == 'tell': + tellsig = parse_funcsig(['tell', + {'name':'target', 'type':'CephOsdName'}]) + sig = tellsig + sig + for desc in sig: + # prefixes go in the URL path if desc.t == CephPrefix: url += '/' + desc.instance.prefix + # CephChoices with 1 required string (not --) do too, unless + # we've already started collecting params, in which case they + # too are params elif desc.t == CephChoices and \ len(desc.instance.strings) == 1 and \ desc.req and \ - not str(desc.instance).startswith('--'): + not str(desc.instance).startswith('--') and \ + not params: url += '/' + str(desc.instance) else: - params.append(desc) - return glob.baseurl + url, params + # tell/ is a weird case; the URL includes what + # would everywhere else be a parameter + if flavor == 'tell' and \ + (desc.t, desc.name) == (CephOsdName, 'target'): + url += '/' + else: + params.append(desc) + return glob.baseurl + url, params -def concise_sig_for_uri(sig): +def concise_sig_for_uri(sig, flavor): """ Return a generic description of how one would send a REST request for sig """ prefix = [] args = [] + ret = '' + if flavor == 'tell': + ret = 'tell//' for d in sig: if d.t == CephPrefix: prefix.append(d.instance.prefix) else: args.append(d.name + '=' + str(d)) - sig = '/'.join(prefix) + ret += '/'.join(prefix) if args: - sig += '?' + '&'.join(args) - return sig + ret += '?' + '&'.join(args) + return ret def show_human_help(prefix): """ Dump table showing commands matching prefix """ - # XXX this really needs to be a template - #s = '' - #s += '' - #s += '' - # XXX the above mucking with css doesn't cause sensible columns. + # XXX There ought to be a better discovery mechanism than an HTML table s = '
Possible commands:
' - possible = [] permmap = {'r':'GET', 'rw':'PUT'} line = '' for cmdsig in sorted(glob.sigdict.itervalues(), cmp=descsort): concise = concise_sig(cmdsig['sig']) + flavor = cmdsig.get('flavor', 'mon') + if flavor == 'tell': + concise = 'tell//' + concise if concise.startswith(prefix): line = ['
Possible commands:MethodDescription
'] - wrapped_sig = textwrap.wrap(concise_sig_for_uri(cmdsig['sig']), 40) + wrapped_sig = textwrap.wrap( + concise_sig_for_uri(cmdsig['sig'], flavor), 40 + ) for sigline in wrapped_sig: line.append(flask.escape(sigline) + '\n') line.append('') @@ -328,19 +400,23 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response(response, errorcode) -def handler(catchall_path=None, fmt=None): +def handler(catchall_path=None, fmt=None, target=None): """ Main endpoint handler; generic for every endpoint """ - if (catchall_path): - ep = catchall_path.replace('.', '') - else: - ep = flask.request.endpoint.replace('.', '') + ep = catchall_path or flask.request.endpoint + ep = ep.replace('.', '') if ep[0] != '/': ep = '/' + ep + # demand that endpoint begin with glob.baseurl + if not ep.startswith(glob.baseurl): + return make_response(fmt, '', 'Page not found', 404) + + rel_ep = ep[len(glob.baseurl)+1:] + # Extensions override Accept: headers override defaults if not fmt: if 'application/json' in flask.request.accept_mimetypes.values(): @@ -348,12 +424,36 @@ def handler(catchall_path=None, fmt=None): elif 'application/xml' in flask.request.accept_mimetypes.values(): fmt = 'xml' - # demand that endpoint begin with glob.baseurl - if not ep.startswith(glob.baseurl): - return make_response(fmt, '', 'Page not found', 404) + valid = True + prefix = '' + pgid = None + cmdtarget = 'mon', '' + + if target: + # got tell/; validate osdid or pgid + name = CephOsdName() + pgidobj = CephPgid() + try: + name.valid(target) + except ArgumentError: + # try pgid + try: + pgidobj.valid(target) + except ArgumentError: + return flask.make_response("invalid osdid or pgid", 400) + else: + # it's a pgid + pgid = pgidobj.val + cmdtarget = 'pg', pgid + else: + # it's an osd + cmdtarget = name.nametype, name.nameid - relative_endpoint = ep[len(glob.baseurl)+1:] - prefix = ' '.join(relative_endpoint.split('/')).strip() + # prefix does not include tell// + prefix = ' '.join(rel_ep.split('/')[2:]).strip() + else: + # non-target command: prefix is entire path + prefix = ' '.join(rel_ep.split('/')).strip() # show "match as much as you gave me" help for unknown endpoints if not ep in glob.urls: @@ -365,43 +465,59 @@ def handler(catchall_path=None, fmt=None): else: return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) - urldict = glob.urls[ep] - paramsig = urldict['paramsig'] - - # allow '?help' for any specifically-known endpoint - if 'help' in flask.request.args: - response = flask.make_response('{0}: {1}'.\ - format(prefix + concise_sig(paramsig), urldict['help'])) - response.headers['Content-Type'] = 'text/plain' - return response - - # if there are parameters for this endpoint, process them - if paramsig: - args = {} - for k, l in flask.request.args.iterlists(): - if len(l) == 1: - args[k] = l[0] - else: - args[k] = l - - # is this a valid set of params? - try: - argdict = validate(args, paramsig) - except Exception as e: - return make_response(fmt, '', str(e) + '\n', 400) - else: - # no parameters for this endpoint; complain if args are supplied - if flask.request.args: - return make_response(fmt, '', ep + 'takes no params', 400) - argdict = {} + found = None + exc = '' + for urldict in glob.urls[ep]: + if flask.request.method not in urldict['methods']: + continue + paramsig = urldict['paramsig'] + + # allow '?help' for any specifically-known endpoint + if 'help' in flask.request.args: + response = flask.make_response('{0}: {1}'.\ + format(prefix + concise_sig(paramsig), urldict['help'])) + response.headers['Content-Type'] = 'text/plain' + return response + + # if there are parameters for this endpoint, process them + if paramsig: + args = {} + for k, l in flask.request.args.iterlists(): + if len(l) == 1: + args[k] = l[0] + else: + args[k] = l + + # is this a valid set of params? + try: + argdict = validate(args, paramsig) + found = urldict + break + except Exception as e: + exc += str(e) + continue + else: + if flask.request.args: + continue + found = urldict + argdict = {} + break + if not found: + return make_response(fmt, '', exc + '\n', 400) argdict['format'] = fmt or 'plain' - argdict['module'] = urldict['module'] - argdict['perm'] = urldict['perm'] + argdict['module'] = found['module'] + argdict['perm'] = found['perm'] + if pgid: + argdict['pgid'] = pgid + + if not cmdtarget: + cmdtarget = ('mon', '') app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) ret, outbuf, outs = json_command(glob.cluster, prefix=prefix, + target=cmdtarget, inbuf=flask.request.data, argdict=argdict) if ret: return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) From 8985e1c9e80d1e0a646e4ae7d0e0d89c6b622068 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 23 Jul 2013 22:13:14 -0700 Subject: [PATCH 12/44] ceph_argparse, mon: make "tell " work (duplicating "pg ") It's a wad of special cases, but it implements "tell " such that it has the same effect as "pg ". Signed-off-by: Dan Mick --- src/osd/OSD.cc | 31 +++++++++++++++++++++++++++---- src/osd/ReplicatedPG.cc | 6 ------ src/pybind/ceph_argparse.py | 9 ++++++++- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index d611afdd08fb8..ea13b25b38a9b 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3858,6 +3858,7 @@ COMMAND("list_missing " \ // tell commands. Validation of osd.n must be special-cased in client +// tell commands. Validation of osd.n must be special-cased in client COMMAND("version", "report version of OSD", "osd", "r", "cli,rest") COMMAND("injectargs " \ "name=injected_args,type=CephString,n=N", @@ -3883,6 +3884,19 @@ COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics", "osd", "r", "cli,rest") COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics", "osd", "rw", "cli,rest") + +// experiment: restate pg commands as "tell ". Validation of +// pgid must be special-cased in client. +COMMAND("query", + "show details of a specific pg", "osd", "r", "cli,rest") +COMMAND("mark_unfound_lost revert " \ + "name=mulcmd,type=CephChoices,strings=revert", \ + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", + "osd", "rw", "cli,rest") +COMMAND("list_missing " \ + "name=offset,type=CephString,req=false", + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "rw", "cli,rest") }; void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist& data) @@ -3897,6 +3911,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist map cmdmap; string prefix; string format; + string pgidstr; boost::scoped_ptr f; if (cmd.empty()) { @@ -3963,9 +3978,16 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist osd_lock.Lock(); } - else if (prefix == "pg") { + // either 'pg ' or + // 'tell ' (which comes in without any of that prefix)? + + else if (prefix == "pg" || + (cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr) && + (prefix == "query" || + prefix == "mark_unfound_lost" || + prefix == "list_missing") + )) { pg_t pgid; - string pgidstr; if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) { ss << "no pgid specified"; @@ -3974,13 +3996,14 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist ss << "couldn't parse pgid '" << pgidstr << "'"; r = -EINVAL; } else { - vector args; - cmd_getval(g_ceph_context, cmdmap, "args", args); PG *pg = _lookup_lock_pg(pgid); if (!pg) { ss << "i don't have pgid " << pgid; r = -ENOENT; } else { + // simulate pg cmd= for pg->do-command + if (prefix != "pg") + cmd_putval(g_ceph_context, cmdmap, "cmd", prefix); r = pg->do_command(cmdmap, ss, data, odata); pg->unlock(); } diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 4a59a23cdb794..658ea7cb74633 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -275,12 +275,6 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, string prefix; string format; - cmd_getval(g_ceph_context, cmdmap, "prefix", prefix); - if (prefix != "pg") { - ss << "ReplicatedPG::do_command: not pg command"; - return -EINVAL; - } - cmd_getval(g_ceph_context, cmdmap, "format", format); boost::scoped_ptr f(new_formatter(format)); // demand that we have a formatter diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index 855e21c25084e..354459a2cd57a 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -938,8 +938,15 @@ def send_command(cluster, target=('mon', ''), cmd=[], inbuf='', timeout=0, cluster.osd_command(osdid, cmd, inbuf, timeout) elif target[0] == 'pg': - # leave it in cmddict for the OSD to use too pgid = target[1] + # pgid will already be in the command for the pg + # form, but for tell , we need to put it in + if cmd: + cmddict = json.loads(cmd[0]) + cmddict['pgid'] = pgid + else: + cmddict = dict(pgid=pgid) + cmd = [json.dumps(cmddict)] if verbose: print >> sys.stderr, 'submit {0} for pgid {1}'.\ format(cmd, pgid) From d75b6ea1a47f8b240b5dfead28bd9a7d9257400e Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Wed, 24 Jul 2013 21:56:15 -0700 Subject: [PATCH 13/44] ceph_argparse.py: make find_cmd_target handle tell Signed-off-by: Dan Mick --- src/pybind/ceph_argparse.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index 354459a2cd57a..ee71b76d6a1f3 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -897,20 +897,35 @@ def find_cmd_target(childargs): sig = parse_funcsig(['tell', {'name':'target','type':'CephName'}]) try: valid_dict = validate(childargs, sig, partial=True); + except ArgumentError: + pass + else: if len(valid_dict) == 2: + # revalidate to isolate type and id name = CephName() + # if this fails, something is horribly wrong, as it just + # validated successfully above name.valid(valid_dict['target']) return name.nametype, name.nameid + + sig = parse_funcsig(['tell', {'name':'pgid','type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True); except ArgumentError: pass + else: + if len(valid_dict) == 2: + # pg doesn't need revalidation; the string is fine + return 'pg', valid_dict['pgid'] sig = parse_funcsig(['pg', {'name':'pgid','type':'CephPgid'}]) try: valid_dict = validate(childargs, sig, partial=True); - if len(valid_dict) == 2: - return 'pg', valid_dict['pgid'] except ArgumentError: pass + else: + if len(valid_dict) == 2: + return 'pg', valid_dict['pgid'] return 'mon', '' From aa00ace1d806526e02dbd65fddaccba2efa94163 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 26 Jul 2013 15:23:57 -0700 Subject: [PATCH 14/44] ceph_rest_api.py: cleanup, more docstrings, unused vars Signed-off-by: Dan Mick --- src/pybind/ceph_rest_api.py | 80 ++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index f8d9b92129e2d..830fb2249ea14 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -3,13 +3,13 @@ import collections import ConfigParser +import contextlib import errno import json import logging import logging.handlers import os import rados -import socket import textwrap import xml.etree.ElementTree import xml.sax.saxutils @@ -21,13 +21,13 @@ # Globals # -APPNAME = '__main__' DEFAULT_BASEURL = '/api/v0.1' DEFAULT_ADDR = '0.0.0.0:5000' DEFAULT_LOG_LEVEL = 'warning' DEFAULT_CLIENTNAME = 'client.restapi' DEFAULT_LOG_FILE = '/var/log/ceph/' + DEFAULT_CLIENTNAME + '.log' +APPNAME = '__main__' app = flask.Flask(APPNAME) LOGLEVELS = { @@ -38,7 +38,9 @@ 'debug':logging.DEBUG, } -# my globals, in a named tuple for usage clarity +# my globals, in a named tuple for usage clarity. I promise +# these are never written once initialized, and are global +# to every thread. glob = collections.namedtuple('gvars', 'cluster urls sigdict baseurl') glob.cluster = None @@ -47,8 +49,15 @@ glob.baseurl = '' def load_conf(clustername='ceph', conffile=None): - import contextlib + ''' + Load the ceph conf file using ConfigParser. Use the standard + fallback order: + 1) the passed in arg (from CEPH_CONF) + 2) /etc/ceph/{cluster}.conf + 3) ~/.ceph/{cluster}.conf + 4) {cluster}.conf + ''' class _TrimIndentFile(object): def __init__(self, fp): @@ -93,6 +102,10 @@ def load(path): raise EnvironmentError('No conf file found for "{0}"'.format(clustername)) def get_conf(cfg, clientname, key): + ''' + Get config entry from conf file, first in [clientname], then [client], + then [global]. + ''' fullkey = 'restapi_' + key for sectionname in clientname, 'client', 'global': try: @@ -101,8 +114,6 @@ def get_conf(cfg, clientname, key): pass return None -METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} - def find_up_osd(): ''' Find an up OSD. Return the last one that's up. @@ -121,16 +132,19 @@ def find_up_osd(): raise EnvironmentError(errno.ENOENT, 'No up OSDs found') return int(osds[-1]) -# XXX this is done globally, and cluster connection kept open; there -# are facilities to pass around global info to requests and to -# tear down connections between requests if it becomes important + +METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} def api_setup(): - """ + ''' + This is done globally, and cluster connection kept open for + the lifetime of the daemon. librados should assure that even + if the cluster goes away and comes back, our connection remains. + Initialize the running instance. Open the cluster, get the command signatures, module, perms, and help; stuff them away in the glob.urls - dict. - """ + dict. Also save glob.sigdict for help() handling. + ''' def get_command_descriptions(target=('mon','')): ret, outbuf, outs = json_command(glob.cluster, target, prefix='get_command_descriptions', @@ -253,11 +267,11 @@ def get_command_descriptions(target=('mon','')): def generate_url_and_params(sig, flavor): - """ + ''' Digest command signature from cluster; generate an absolute (including glob.baseurl) endpoint from all the prefix words, and a list of non-prefix param descs - """ + ''' url = '' params = [] @@ -292,10 +306,15 @@ def generate_url_and_params(sig, flavor): return glob.baseurl + url, params + +# +# end setup (import-time) functions, begin request-time functions +# + def concise_sig_for_uri(sig, flavor): - """ + ''' Return a generic description of how one would send a REST request for sig - """ + ''' prefix = [] args = [] ret = '' @@ -312,9 +331,9 @@ def concise_sig_for_uri(sig, flavor): return ret def show_human_help(prefix): - """ + ''' Dump table showing commands matching prefix - """ + ''' # XXX There ought to be a better discovery mechanism than an HTML table s = '' @@ -347,23 +366,22 @@ def show_human_help(prefix): @app.before_request def log_request(): - """ + ''' For every request, log it. XXX Probably overkill for production - """ + ''' app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) - @app.route('/') def root_redir(): return flask.redirect(glob.baseurl) def make_response(fmt, output, statusmsg, errorcode): - """ + ''' If formatted output, cobble up a response object that contains the output and status wrapped in enclosing objects; if nonformatted, just - use output. Return HTTP status errorcode in any event. - """ + use output+status. Return HTTP status errorcode in any event. + ''' response = output if fmt: if 'json' in fmt: @@ -375,6 +393,7 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response("Error decoding JSON from " + output, 500) elif 'xml' in fmt: + # XXX # one is tempted to do this with xml.etree, but figuring out how # to 'un-XML' the XML-dumped output so it can be reassembled into # a piece of the tree here is beyond me right now. @@ -401,9 +420,12 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response(response, errorcode) def handler(catchall_path=None, fmt=None, target=None): - """ - Main endpoint handler; generic for every endpoint - """ + ''' + Main endpoint handler; generic for every endpoint, including catchall. + Handles the catchall, anything with <.fmt>, anything with embedded + . Partial match or ?help cause the HTML-table + "show_human_help" output. + ''' ep = catchall_path or flask.request.endpoint ep = ep.replace('.', '') @@ -424,7 +446,6 @@ def handler(catchall_path=None, fmt=None, target=None): elif 'application/xml' in flask.request.accept_mimetypes.values(): fmt = 'xml' - valid = True prefix = '' pgid = None cmdtarget = 'mon', '' @@ -530,4 +551,7 @@ def handler(catchall_path=None, fmt=None, target=None): response.headers['Content-Type'] = contenttype return response +# +# Last module-level (import-time) ask: set up the cluster connection +# addr, port = api_setup() From 6faf8b680d1474b0f47fc87cde8e406ce985d60f Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 16:21:09 -0700 Subject: [PATCH 15/44] PendingReleaseNotes: note on 'ceph tell ...' Signed-off-by: Sage Weil --- PendingReleaseNotes | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/PendingReleaseNotes b/PendingReleaseNotes index bed4cc4c91c53..b1c141492f60d 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -31,3 +31,10 @@ v0.67 in order to accomodate the new write back throttle system. upstart now sets the fd limit to 32k. sysvinit will set it to 32k by default (still overrideable via max_open_files). + +* The 'ceph pg ...' commands (like 'ceph pg query') are + deprecated in favor of 'ceph tell ...'. This makes the + distinction between 'ceph pg ' and 'ceph pg + ' less awkward by making it clearer that the 'tell' + commands are talking to the OSD serving the placement group, not the + monitor. From 14a3e2ddce1487e47498249c3f63d7df57b55ec5 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 16:42:41 -0700 Subject: [PATCH 16/44] remove unused fiemap code Signed-off-by: Sage Weil --- COPYING | 5 --- debian/copyright | 5 --- src/Makefile.am | 3 +- src/common/fiemap.cc | 96 -------------------------------------------- src/include/fiemap.h | 27 ------------- src/os/FileStore.cc | 2 +- src/rbd.cc | 2 - 7 files changed, 2 insertions(+), 138 deletions(-) delete mode 100644 src/common/fiemap.cc delete mode 100644 src/include/fiemap.h diff --git a/COPYING b/COPYING index b374bdc180176..fe6ffaac0f80f 100644 --- a/COPYING +++ b/COPYING @@ -15,11 +15,6 @@ Copyright: Copyright (C) 2004-2006 Sage Weil License: GPL2 -Files: src/common/fiemap.cc -Copyright: - Copyright (C) 2010 Canonical -License: GPL2 - Files: src/mount/canonicalize.c Copyright: Copyright (C) 1993 Rick Sladkey License: LGPL2 or later diff --git a/debian/copyright b/debian/copyright index 4295a1564f9a3..e94a11b996267 100644 --- a/debian/copyright +++ b/debian/copyright @@ -16,11 +16,6 @@ Copyright: Copyright (C) 2004-2006 Sage Weil License: GPL2 -Files: src/common/fiemap.cc -Copyright: - Copyright (C) 2010 Canonical -License: GPL2 - Files: src/mount/canonicalize.c Copyright: Copyright (C) 1993 Rick Sladkey License: LGPL2 or later diff --git a/src/Makefile.am b/src/Makefile.am index 307abf2f9653a..a9bbde3268601 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -504,7 +504,7 @@ ceph_radosacl_SOURCES = radosacl.cc ceph_radosacl_LDADD = librados.la $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) bin_DEBUGPROGRAMS += ceph_scratchtool ceph_scratchtoolpp ceph_radosacl -rbd_SOURCES = rbd.cc common/fiemap.cc common/secret.c common/TextTable.cc common/util.cc +rbd_SOURCES = rbd.cc common/secret.c common/TextTable.cc common/util.cc rbd_CXXFLAGS = ${AM_CXXFLAGS} rbd_LDADD = librbd.la librados.la $(LIBGLOBAL_LDA) -lkeyutils if LINUX @@ -1895,7 +1895,6 @@ noinst_HEADERS = \ include/encoding.h\ include/err.h\ include/error.h\ - include/fiemap.h\ include/filepath.h\ include/frag.h\ include/hash.h\ diff --git a/src/common/fiemap.cc b/src/common/fiemap.cc deleted file mode 100644 index a1d5fbe9396ad..0000000000000 --- a/src/common/fiemap.cc +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2010 Canonical - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ - -/* - * Author Colin Ian King, colin.king@canonical.com - */ - -#include -#include -#include -#include -#include - -#include -#include -#include - -#if defined(__linux__) -#include -#endif -#include "include/inttypes.h" -#include "include/fiemap.h" - -struct fiemap *read_fiemap(int fd) -{ - struct fiemap *fiemap; - struct fiemap *_realloc_fiemap = NULL; - int extents_size; - int r; - - if ((fiemap = (struct fiemap*)malloc(sizeof(struct fiemap))) == NULL) { - fprintf(stderr, "Out of memory allocating fiemap\n"); - return NULL; - } - memset(fiemap, 0, sizeof(struct fiemap)); - - fiemap->fm_start = 0; - fiemap->fm_length = ~0; /* Lazy */ - fiemap->fm_flags = 0; - fiemap->fm_extent_count = 0; - fiemap->fm_mapped_extents = 0; - - /* Find out how many extents there are */ - r = ioctl(fd, FS_IOC_FIEMAP, fiemap); - if (r < 0) { - goto done_err; - } - - if (!fiemap->fm_mapped_extents) { - goto done_err; - } - - /* Read in the extents */ - extents_size = sizeof(struct fiemap_extent) * (fiemap->fm_mapped_extents); - - /* Resize fiemap to allow us to read in the extents */ - - if ((_realloc_fiemap = (struct fiemap*)realloc(fiemap,sizeof(struct fiemap) + - extents_size)) == NULL) { - fprintf(stderr, "Out of memory allocating fiemap\n"); - goto done_err; - } else { - fiemap = _realloc_fiemap; - } - - memset(fiemap->fm_extents, 0, extents_size); - fiemap->fm_extent_count = fiemap->fm_mapped_extents; - fiemap->fm_mapped_extents = 0; - - if (ioctl(fd, FS_IOC_FIEMAP, fiemap) < 0) { - fprintf(stderr, "fiemap ioctl() failed\n"); - goto done_err; - } - - return fiemap; -done_err: - free(fiemap); - return NULL; -} - diff --git a/src/include/fiemap.h b/src/include/fiemap.h deleted file mode 100644 index 846adb155ff6a..0000000000000 --- a/src/include/fiemap.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef __CEPH_FIEMAP_H -#define __CEPH_FIEMAP_H - -#include "acconfig.h" - -/* - * the header is missing on most systems. for the time being at - * least, include our own copy in the repo. - */ -#ifdef HAVE_FIEMAP_H -# include -#else -# include "linux_fiemap.h" -#endif - -#if defined(__linux__) -#include -#elif defined(__FreeBSD__) -#include -#endif -#ifndef FS_IOC_FIEMAP -# define FS_IOC_FIEMAP _IOWR('f', 11, struct fiemap) -#endif - -extern "C" struct fiemap *read_fiemap(int fd); - -#endif diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index 28f81b7547fd0..108a857ab9fc9 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -36,7 +36,7 @@ #endif #include "include/compat.h" -#include "include/fiemap.h" +#include "include/linux_fiemap.h" #include "common/xattr.h" #include "chain_xattr.h" diff --git a/src/rbd.cc b/src/rbd.cc index c9b2f0a272ca3..7f90c1f118eaf 100644 --- a/src/rbd.cc +++ b/src/rbd.cc @@ -60,8 +60,6 @@ #include #endif -#include "include/fiemap.h" - #define MAX_SECRET_LEN 1000 #define MAX_POOL_NAME_SIZE 128 From 323bdaa25709182476d655e62aa4b99f2a53e49b Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 15:46:51 -0700 Subject: [PATCH 17/44] mon/MonCap: mds needs to subscribe to the osdmap Usually it can get it from the OSD, but it assumes the mon will also share and doesn't behave when it does not. Fixes: #5767 Signed-off-by: Sage Weil --- src/mon/MonCap.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mon/MonCap.cc b/src/mon/MonCap.cc index 7ac8d142d87af..b03873ad7ddc6 100644 --- a/src/mon/MonCap.cc +++ b/src/mon/MonCap.cc @@ -133,6 +133,7 @@ void MonCapGrant::expand_profile(entity_name_t name) const if (profile == "mds") { profile_grants.push_back(MonCapGrant("mds", MON_CAP_ALL)); profile_grants.push_back(MonCapGrant("mon", MON_CAP_R)); + profile_grants.push_back(MonCapGrant("osd", MON_CAP_R)); profile_grants.push_back(MonCapGrant("log", MON_CAP_W)); } if (profile == "osd" || profile == "mds" || profile == "mon") { From 4b739005a548b9cbf04856646aa993e2b71793c7 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 17:14:44 -0700 Subject: [PATCH 18/44] osd: humor coverity CID 1058392: Out-of-bounds access (OVERRUN_DYNAMIC) [select issue] CID 1058391 (#1 of 1): Out-of-bounds access (OVERRUN) 32. alloc_strlen: Allocating insufficient memory for the terminating null of the string. CID 1058390 (#1 of 1): Unchecked return value from library (CHECKED_RETURN) 13. check_return: Calling function "this->class_handler->open_all_classes()" without checking return value. It wraps a library function that may fail and return an error code. [show details] 14. unchecked_value: No check of the return value of "this->class_handler->open_all_classes()". Signed-off-by: Sage Weil --- src/osd/ClassHandler.cc | 2 +- src/osd/OSD.cc | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/osd/ClassHandler.cc b/src/osd/ClassHandler.cc index 3cc319efabeef..a9a920ba078e9 100644 --- a/src/osd/ClassHandler.cc +++ b/src/osd/ClassHandler.cc @@ -52,7 +52,7 @@ int ClassHandler::open_all_classes() if (strlen(pde->d_name) > sizeof(CLS_PREFIX) - 1 + sizeof(CLS_SUFFIX) - 1 && strncmp(pde->d_name, CLS_PREFIX, sizeof(CLS_PREFIX) - 1) == 0 && strcmp(pde->d_name + strlen(pde->d_name) - (sizeof(CLS_SUFFIX) - 1), CLS_SUFFIX) == 0) { - char cname[strlen(pde->d_name)]; + char cname[PATH_MAX + 1]; strcpy(cname, pde->d_name + sizeof(CLS_PREFIX) - 1); cname[strlen(cname) - (sizeof(CLS_SUFFIX) - 1)] = '\0'; dout(10) << __func__ << " found " << cname << dendl; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index a2cc0ed229e6b..e3a7c227e1553 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -1167,8 +1167,11 @@ int OSD::init() class_handler = new ClassHandler(); cls_initialize(class_handler); - if (g_conf->osd_open_classes_on_start) - class_handler->open_all_classes(); + if (g_conf->osd_open_classes_on_start) { + int r = class_handler->open_all_classes(); + if (r) + dout(1) << "warning: got an error loading one or more classes: " << cpp_strerror(r) << dendl; + } // load up "current" osdmap assert_warn(!osdmap); From 3f93691bf35f7ac1bb1ae68d5ef2dc68d2928ad7 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 26 Jul 2013 16:25:26 -0700 Subject: [PATCH 19/44] ceph.in: make osdids() (and mon, mds) work on old mons Signed-off-by: Dan Mick Reviewed-by: Sage Weil --- src/ceph.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/ceph.in b/src/ceph.in index 63c41343f9a52..6caef354a1025 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -64,6 +64,9 @@ cluster_handle = None def osdids(): ret, outbuf, outs = json_command(cluster_handle, prefix='osd ls') + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, cmd=['osd', 'ls']) if ret: raise RuntimeError('Can\'t contact mon for osd list') return [i for i in outbuf.split('\n') if i != ''] @@ -71,6 +74,10 @@ def osdids(): def monids(): ret, outbuf, outs = json_command(cluster_handle, prefix='mon dump', argdict={'format':'json'}) + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, + cmd=['mon', 'dump', '--format=json']) if ret: raise RuntimeError('Can\'t contact mon for mon list') d = json.loads(outbuf) @@ -79,6 +86,10 @@ def monids(): def mdsids(): ret, outbuf, outs = json_command(cluster_handle, prefix='mds dump', argdict={'format':'json'}) + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, + cmd=['mds', 'dump', '--format=json']) if ret: raise RuntimeError('Can\'t contact mon for mds list') d = json.loads(outbuf) From a9ca6234c8ab3c77c7bf2d435c5ac5def8632d7c Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 17:26:48 -0700 Subject: [PATCH 20/44] librados: EINVAL on a negative osd id Signed-off-by: Sage Weil --- src/librados/RadosClient.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc index f68125fb8c071..e8dd019af3a28 100644 --- a/src/librados/RadosClient.cc +++ b/src/librados/RadosClient.cc @@ -642,6 +642,10 @@ int librados::RadosClient::osd_command(int osd, vector& cmd, bool done; int ret; tid_t tid; + + if (osd < 0) + return -EINVAL; + lock.Lock(); // XXX do anything with tid? int r = objecter->osd_command(osd, cmd, inbl, &tid, poutbl, prs, From a419354120cb650e2a25faba5441deb15b98f4f1 Mon Sep 17 00:00:00 2001 From: David Zafman Date: Fri, 26 Jul 2013 14:00:06 -0700 Subject: [PATCH 21/44] message: Fix asserts that dont' trigger Signed-off-by: David Zafman Reviewed-by: Sage Weil --- src/messages/MMonScrub.h | 2 +- src/messages/MMonSync.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/messages/MMonScrub.h b/src/messages/MMonScrub.h index ab4588f4a76e2..b16728bcdd555 100644 --- a/src/messages/MMonScrub.h +++ b/src/messages/MMonScrub.h @@ -31,7 +31,7 @@ class MMonScrub : public Message switch (op) { case OP_SCRUB: return "scrub"; case OP_RESULT: return "result"; - default: assert("unknown op type"); return NULL; + default: assert(0 == "unknown op type"); return NULL; } } diff --git a/src/messages/MMonSync.h b/src/messages/MMonSync.h index a5415a8f45157..48229d15bcc0b 100644 --- a/src/messages/MMonSync.h +++ b/src/messages/MMonSync.h @@ -49,7 +49,7 @@ class MMonSync : public Message case OP_CHUNK: return "chunk"; case OP_LAST_CHUNK: return "last_chunk"; case OP_NO_COOKIE: return "no_cookie"; - default: assert("unknown op type"); return NULL; + default: assert(0 == "unknown op type"); return NULL; } } From 803a1fdd08ced49572fd0aadf80d2b75732bef88 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 21:06:06 -0700 Subject: [PATCH 22/44] ceph_test_admin_socket: fix unit test Signed-off-by: Sage Weil --- src/test/admin_socket.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc index 0dbcb6d2f756a..7f595f596186f 100644 --- a/src/test/admin_socket.cc +++ b/src/test/admin_socket.cc @@ -70,7 +70,7 @@ TEST(AdminSocket, SendNoOp) { } class MyTest : public AdminSocketHook { - bool call(std::string command, std::string args, bufferlist& result) { + bool call(std::string command, std::string args, std::string format, bufferlist& result) { result.append(command); result.append("|"); result.append(args); @@ -93,7 +93,7 @@ TEST(AdminSocket, RegisterCommand) { } class MyTest2 : public AdminSocketHook { - bool call(std::string command, std::string args, bufferlist& result) { + bool call(std::string command, std::string args, std::string format, bufferlist& result) { result.append(command); result.append("|"); result.append(args); From 629326aa92f82a0bc66df30e572223743a55ccf4 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 21:06:14 -0700 Subject: [PATCH 23/44] qa/fs/.gitignore Signed-off-by: Sage Weil --- qa/fs/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 qa/fs/.gitignore diff --git a/qa/fs/.gitignore b/qa/fs/.gitignore new file mode 100644 index 0000000000000..a2d280e079454 --- /dev/null +++ b/qa/fs/.gitignore @@ -0,0 +1 @@ +/test_o_trunc From 8af47755af26826f205402db9fef9632e0f1462e Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 26 Jul 2013 17:47:32 -0700 Subject: [PATCH 24/44] ceph-rest-api: clean up options/environment ceph-rest-api: * create app from wrapper by calling generate_app() * pass args to generate_app() (early parsed in wrapper) * parse -i/--id here as well * set addr:port on returned app object * handle only EnvironmentError exceptions; let others spew traceback * turn off debug when running singlethreaded server ceph_rest_api.py: * put glob.* on app.ceph_* instead; pass around app in init code * drop conf parsing (let librados do its job) Documentation updated to match. Signed-off-by: Dan Mick Reviewed-by: Sage Weil --- doc/man/8/ceph-rest-api.rst | 75 +++++++----- man/ceph-rest-api.8 | 76 +++++++----- src/ceph-rest-api | 31 ++--- src/pybind/ceph_rest_api.py | 226 +++++++++++++----------------------- 4 files changed, 188 insertions(+), 220 deletions(-) diff --git a/doc/man/8/ceph-rest-api.rst b/doc/man/8/ceph-rest-api.rst index 8a87f97ce19bf..a000d09c67603 100644 --- a/doc/man/8/ceph-rest-api.rst +++ b/doc/man/8/ceph-rest-api.rst @@ -7,7 +7,7 @@ Synopsis ======== -| **ceph-rest-api** [ -c *conffile* ] [ -n *name* ... ] +| **ceph-rest-api** [ -c *conffile* ] [--cluster *clustername* ] [ -n *name* ] [-i *id* ] Description @@ -21,7 +21,7 @@ command-line tool through an HTTP-accessible interface. Options ======= -.. option:: -c/--conf *conffile* +.. option:: -c/--conf conffile names the ceph.conf file to use for configuration. If -c is not specified, the default depends on the state of the --cluster option @@ -35,21 +35,26 @@ Options so you can also pass this option in the environment as CEPH_CONF. -.. option:: --cluster *clustername* +.. option:: --cluster clustername set *clustername* for use in the $cluster metavariable, for locating the ceph.conf file. The default is 'ceph'. - You can also pass this option in the environment as - CEPH_CLUSTER_NAME. -.. option:: -n/--name *name* +.. option:: -n/--name name specifies the client 'name', which is used to find the client-specific configuration options in the config file, and also is the name used for authentication when connecting to the cluster (the entity name appearing in ceph auth list output, - for example). The default is 'client.restapi'. You can also - pass this option in the environment as CEPH_NAME. + for example). The default is 'client.restapi'. + +.. option:: -i/--id id + + specifies the client 'id', which will form the clientname + as 'client.' if clientname is not set. If -n/-name is + set, that takes precedence. + + Also, global Ceph options are supported. Configuration parameters @@ -57,16 +62,22 @@ Configuration parameters Supported configuration parameters include: -* **restapi client name** the 'clientname' used for auth and ceph.conf -* **restapi keyring** the keyring file holding the key for 'clientname' -* **restapi public addr** ip:port to listen on (default 0.0.0.0:5000) +* **keyring** the keyring file holding the key for 'clientname' +* **public addr** ip:port to listen on (default 0.0.0.0:5000) +* **log file** (usual Ceph default) * **restapi base url** the base URL to answer requests on (default /api/v0.1) -* **restapi log level** critical, error, warning, info, debug -* **restapi log file** (default /var/local/ceph/.log) +* **restapi log level** critical, error, warning, info, debug (default warning) + +Configuration parameters are searched in the standard order: +first in the section named '', then 'client', then 'global'. -A server will run on **restapi public addr** if the ceph-rest-api -executed directly; otherwise, configuration is specified by the -enclosing WSGI web server. + is either supplied by -n/--name, "client." where + is supplied by -i/--id, or 'client.restapi' if neither option +is present. + +A single-threaded server will run on **public addr** if the ceph-rest-api +executed directly; otherwise, configuration is specified by the enclosing +WSGI web server. Commands ======== @@ -92,7 +103,9 @@ with a small description of each command, is provided when the requested path is incomplete/partially matching. Requesting / will redirect to the value of **restapi base url**, and that path will give a full list of all known commands. The command set is very similar to the commands -supported by the **ceph** tool. +supported by the **ceph** tool. One notable exception is that the +``ceph pg `` style of commands is supported here +as ``tell//command?args``. Deployment as WSGI application ============================== @@ -101,18 +114,22 @@ When deploying as WSGI application (say, with Apache/mod_wsgi, or nginx/uwsgi, or gunicorn, etc.), use the ``ceph_rest_api.py`` module (``ceph-rest-api`` is a thin layer around this module). The standalone web server is of course not used, so address/port configuration is done in -the WSGI server. Also, configuration switches are not passed; rather, -environment variables are used: - -* CEPH_CONF holds -c/--conf -* CEPH_CLUSTER_NAME holds --cluster -* CEPH_NAME holds -n/--name - -Any errors reading configuration or connecting to the cluster cause -ImportError to be raised with a descriptive message on import; see -your WSGI server documentation for how to see those messages in case -of problem. - +the WSGI server. Use a python .wsgi module or the equivalent to call +``app = generate_app(conf, cluster, clientname, clientid, args)`` where: + +* conf is as -c/--conf above +* cluster is as --cluster above +* clientname, -n/--name +* clientid, -i/--id, and +* args are any other generic Ceph arguments + +When app is returned, it will have attributes 'ceph_addr' and 'ceph_port' +set to what the address and port are in the Ceph configuration; +those may be used for the server, or ignored. + +Any errors reading configuration or connecting to the cluster cause an +exception to be raised; see your WSGI server documentation for how to +see those messages in case of problem. Availability ============ diff --git a/man/ceph-rest-api.8 b/man/ceph-rest-api.8 index 33425fecc00b6..5170b7f37cfa7 100644 --- a/man/ceph-rest-api.8 +++ b/man/ceph-rest-api.8 @@ -1,4 +1,4 @@ -.TH "CEPH-REST-API" "8" "July 12, 2013" "dev" "Ceph" +.TH "CEPH-REST-API" "8" "July 26, 2013" "dev" "Ceph" .SH NAME ceph-rest-api \- ceph RESTlike administration server . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] . .SH SYNOPSIS .nf -\fBceph\-rest\-api\fP [ \-c \fIconffile\fP ] [ \-n \fIname\fP ... ] +\fBceph\-rest\-api\fP [ \-c \fIconffile\fP ] [\-\-cluster \fIclustername\fP ] [ \-n \fIname\fP ] [\-i \fIid\fP ] .fi .sp .SH DESCRIPTION @@ -44,7 +44,7 @@ command\-line tool through an HTTP\-accessible interface. .SH OPTIONS .INDENT 0.0 .TP -.B \-c/\-\-conf *conffile* +.B \-c/\-\-conf conffile names the ceph.conf file to use for configuration. If \-c is not specified, the default depends on the state of the \-\-cluster option (default \(aqceph\(aq; see below). The configuration file is searched @@ -64,43 +64,54 @@ so you can also pass this option in the environment as CEPH_CONF. .UNINDENT .INDENT 0.0 .TP -.B \-\-cluster *clustername* +.B \-\-cluster clustername set \fIclustername\fP for use in the $cluster metavariable, for locating the ceph.conf file. The default is \(aqceph\(aq. -You can also pass this option in the environment as -CEPH_CLUSTER_NAME. .UNINDENT .INDENT 0.0 .TP -.B \-n/\-\-name *name* +.B \-n/\-\-name name specifies the client \(aqname\(aq, which is used to find the client\-specific configuration options in the config file, and also is the name used for authentication when connecting to the cluster (the entity name appearing in ceph auth list output, -for example). The default is \(aqclient.restapi\(aq. You can also -pass this option in the environment as CEPH_NAME. +for example). The default is \(aqclient.restapi\(aq. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i/\-\-id id +specifies the client \(aqid\(aq, which will form the clientname +as \(aqclient.\(aq if clientname is not set. If \-n/\-name is +set, that takes precedence. +.sp +Also, global Ceph options are supported. .UNINDENT .SH CONFIGURATION PARAMETERS .sp Supported configuration parameters include: .INDENT 0.0 .IP \(bu 2 -\fBrestapi client name\fP the \(aqclientname\(aq used for auth and ceph.conf +\fBkeyring\fP the keyring file holding the key for \(aqclientname\(aq .IP \(bu 2 -\fBrestapi keyring\fP the keyring file holding the key for \(aqclientname\(aq +\fBpublic addr\fP ip:port to listen on (default 0.0.0.0:5000) .IP \(bu 2 -\fBrestapi public addr\fP ip:port to listen on (default 0.0.0.0:5000) +\fBlog file\fP (usual Ceph default) .IP \(bu 2 \fBrestapi base url\fP the base URL to answer requests on (default /api/v0.1) .IP \(bu 2 -\fBrestapi log level\fP critical, error, warning, info, debug -.IP \(bu 2 -\fBrestapi log file\fP (default /var/local/ceph/.log) +\fBrestapi log level\fP critical, error, warning, info, debug (default warning) .UNINDENT .sp -A server will run on \fBrestapi public addr\fP if the ceph\-rest\-api -executed directly; otherwise, configuration is specified by the -enclosing WSGI web server. +Configuration parameters are searched in the standard order: +first in the section named \(aq\(aq, then \(aqclient\(aq, then \(aqglobal\(aq. +.sp + is either supplied by \-n/\-\-name, "client." where + is supplied by \-i/\-\-id, or \(aqclient.restapi\(aq if neither option +is present. +.sp +A single\-threaded server will run on \fBpublic addr\fP if the ceph\-rest\-api +executed directly; otherwise, configuration is specified by the enclosing +WSGI web server. .SH COMMANDS .sp Commands are submitted with HTTP GET requests (for commands that @@ -122,28 +133,37 @@ with a small description of each command, is provided when the requested path is incomplete/partially matching. Requesting / will redirect to the value of \fBrestapi base url\fP, and that path will give a full list of all known commands. The command set is very similar to the commands -supported by the \fBceph\fP tool. +supported by the \fBceph\fP tool. One notable exception is that the +\fBceph pg \fP style of commands is supported here +as \fBtell//command?args\fP. .SH DEPLOYMENT AS WSGI APPLICATION .sp When deploying as WSGI application (say, with Apache/mod_wsgi, or nginx/uwsgi, or gunicorn, etc.), use the \fBceph_rest_api.py\fP module (\fBceph\-rest\-api\fP is a thin layer around this module). The standalone web server is of course not used, so address/port configuration is done in -the WSGI server. Also, configuration switches are not passed; rather, -environment variables are used: +the WSGI server. Use a python .wsgi module or the equivalent to call +\fBapp = generate_app(conf, cluster, clientname, clientid, args)\fP where: .INDENT 0.0 .IP \(bu 2 -CEPH_CONF holds \-c/\-\-conf +conf is as \-c/\-\-conf above +.IP \(bu 2 +cluster is as \-\-cluster above .IP \(bu 2 -CEPH_CLUSTER_NAME holds \-\-cluster +clientname, \-n/\-\-name .IP \(bu 2 -CEPH_NAME holds \-n/\-\-name +clientid, \-i/\-\-id, and +.IP \(bu 2 +args are any other generic Ceph arguments .UNINDENT .sp -Any errors reading configuration or connecting to the cluster cause -ImportError to be raised with a descriptive message on import; see -your WSGI server documentation for how to see those messages in case -of problem. +When app is returned, it will have attributes \(aqceph_addr\(aq and \(aqceph_port\(aq +set to what the address and port are in the Ceph configuration; +those may be used for the server, or ignored. +.sp +Any errors reading configuration or connecting to the cluster cause an +exception to be raised; see your WSGI server documentation for how to +see those messages in case of problem. .SH AVAILABILITY .sp \fBceph\-rest\-api\fP is part of the Ceph distributed file system. Please refer to the Ceph documentation at diff --git a/src/ceph-rest-api b/src/ceph-rest-api index a44919acd6fc1..ae5245b4f76f2 100755 --- a/src/ceph-rest-api +++ b/src/ceph-rest-api @@ -33,34 +33,35 @@ def parse_args(): parser.add_argument('-c', '--conf', help='Ceph configuration file') parser.add_argument('--cluster', help='Ceph cluster name') parser.add_argument('-n', '--name', help='Ceph client name') + parser.add_argument('-i', '--id', help='Ceph client id') - return parser.parse_args() - + return parser.parse_known_args() # main -parsed_args = parse_args() -if parsed_args.conf: - os.environ['CEPH_CONF'] = parsed_args.conf -if parsed_args.cluster: - os.environ['CEPH_CLUSTER_NAME'] = parsed_args.cluster -if parsed_args.name: - os.environ['CEPH_NAME'] = parsed_args.name +parsed_args, rest = parse_args() # import now that env vars are available to imported module try: import ceph_rest_api -except Exception as e: +except EnvironmentError as e: print >> sys.stderr, "Error importing ceph_rest_api: ", str(e) sys.exit(1) -# importing ceph_rest_api has set module globals 'app', 'addr', and 'port' +# let other exceptions generate traceback + +app = ceph_rest_api.generate_app( + parsed_args.conf, + parsed_args.cluster, + parsed_args.name, + parsed_args.id, + rest, +) files = [os.path.split(fr[1])[-1] for fr in inspect.stack()] if 'pdb.py' in files: - ceph_rest_api.app.run(host=ceph_rest_api.addr, port=ceph_rest_api.port, - debug=True, use_reloader=False, use_debugger=False) + app.run(host=app.ceph_addr, port=app.ceph_port, + debug=True, use_reloader=False, use_debugger=False) else: - ceph_rest_api.app.run(host=ceph_rest_api.addr, port=ceph_rest_api.port, - debug=True) + app.run(host=app.ceph_addr, port=app.ceph_port) diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index 830fb2249ea14..9e53eba05fd17 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -2,7 +2,6 @@ # vim: ts=4 sw=4 smarttab expandtab import collections -import ConfigParser import contextlib import errno import json @@ -18,15 +17,18 @@ from ceph_argparse import * # -# Globals +# Globals and defaults # +DEFAULT_ADDR = '0.0.0.0' +DEFAULT_PORT = '5000' +DEFAULT_ID = 'restapi' + DEFAULT_BASEURL = '/api/v0.1' -DEFAULT_ADDR = '0.0.0.0:5000' DEFAULT_LOG_LEVEL = 'warning' -DEFAULT_CLIENTNAME = 'client.restapi' -DEFAULT_LOG_FILE = '/var/log/ceph/' + DEFAULT_CLIENTNAME + '.log' +# default client name will be 'client.' +# 'app' must be global for decorators, etc. APPNAME = '__main__' app = flask.Flask(APPNAME) @@ -38,88 +40,12 @@ 'debug':logging.DEBUG, } -# my globals, in a named tuple for usage clarity. I promise -# these are never written once initialized, and are global -# to every thread. - -glob = collections.namedtuple('gvars', 'cluster urls sigdict baseurl') -glob.cluster = None -glob.urls = {} -glob.sigdict = {} -glob.baseurl = '' - -def load_conf(clustername='ceph', conffile=None): - ''' - Load the ceph conf file using ConfigParser. Use the standard - fallback order: - - 1) the passed in arg (from CEPH_CONF) - 2) /etc/ceph/{cluster}.conf - 3) ~/.ceph/{cluster}.conf - 4) {cluster}.conf - ''' - - class _TrimIndentFile(object): - def __init__(self, fp): - self.fp = fp - - def readline(self): - line = self.fp.readline() - return line.lstrip(' \t') - - - def _optionxform(s): - s = s.replace('_', ' ') - s = '_'.join(s.split()) - return s - - - def parse(fp): - cfg = ConfigParser.RawConfigParser() - cfg.optionxform = _optionxform - ifp = _TrimIndentFile(fp) - cfg.readfp(ifp) - return cfg - - - def load(path): - f = file(path) - with contextlib.closing(f): - return parse(f) - - if conffile: - # from CEPH_CONF - return load(conffile) - else: - for path in [ - '/etc/ceph/{0}.conf'.format(clustername), - os.path.expanduser('~/.ceph/{0}.conf'.format(clustername)), - '{0}.conf'.format(clustername), - ]: - if os.path.exists(path): - return load(path) - - raise EnvironmentError('No conf file found for "{0}"'.format(clustername)) - -def get_conf(cfg, clientname, key): - ''' - Get config entry from conf file, first in [clientname], then [client], - then [global]. - ''' - fullkey = 'restapi_' + key - for sectionname in clientname, 'client', 'global': - try: - return cfg.get(sectionname, fullkey) - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - pass - return None - -def find_up_osd(): +def find_up_osd(app): ''' Find an up OSD. Return the last one that's up. Returns id as an int. ''' - ret, outbuf, outs = json_command(glob.cluster, prefix="osd dump", + ret, outbuf, outs = json_command(app.ceph_cluster, prefix="osd dump", argdict=dict(format='json')) if ret: raise EnvironmentError(ret, 'Can\'t get osd dump output') @@ -135,18 +61,18 @@ def find_up_osd(): METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} -def api_setup(): +def api_setup(app, conf, cluster, clientname, clientid, args): ''' This is done globally, and cluster connection kept open for the lifetime of the daemon. librados should assure that even if the cluster goes away and comes back, our connection remains. Initialize the running instance. Open the cluster, get the command - signatures, module, perms, and help; stuff them away in the glob.urls - dict. Also save glob.sigdict for help() handling. + signatures, module, perms, and help; stuff them away in the app.ceph_urls + dict. Also save app.ceph_sigdict for help() handling. ''' - def get_command_descriptions(target=('mon','')): - ret, outbuf, outs = json_command(glob.cluster, target, + def get_command_descriptions(cluster, target=('mon','')): + ret, outbuf, outs = json_command(cluster, target, prefix='get_command_descriptions', timeout=30) if ret: @@ -162,62 +88,61 @@ def get_command_descriptions(target=('mon','')): raise EnvironmentError(err) return sigdict - conffile = os.environ.get('CEPH_CONF', '') - clustername = os.environ.get('CEPH_CLUSTER_NAME', 'ceph') - clientname = os.environ.get('CEPH_NAME', DEFAULT_CLIENTNAME) - try: - err = '' - cfg = load_conf(clustername, conffile) - except Exception as e: - err = "Can't load Ceph conf file: " + str(e) - app.logger.critical(err) - app.logger.critical("CEPH_CONF: %s", conffile) - app.logger.critical("CEPH_CLUSTER_NAME: %s", clustername) - raise EnvironmentError(err) - - client_logfile = '/var/log/ceph' + clientname + '.log' - - glob.cluster = rados.Rados(name=clientname, conffile=conffile) - glob.cluster.connect() - - glob.baseurl = get_conf(cfg, clientname, 'base_url') or DEFAULT_BASEURL - if glob.baseurl.endswith('/'): - glob.baseurl = glob.baseurl[:-1] - addr = get_conf(cfg, clientname, 'public_addr') or DEFAULT_ADDR - addrport = addr.rsplit(':', 1) - addr = addrport[0] - if len(addrport) > 1: - port = addrport[1] - else: - port = DEFAULT_ADDR.rsplit(':', 1) + app.ceph_cluster = cluster or 'ceph' + app.ceph_urls = {} + app.ceph_sigdict = {} + app.ceph_baseurl = '' + + conf = conf or '' + cluster = cluster or 'ceph' + clientid = clientid or DEFAULT_ID + clientname = clientname or 'client.' + clientid + + app.ceph_cluster = rados.Rados(name=clientname, conffile=conf) + app.ceph_cluster.conf_parse_argv(args) + app.ceph_cluster.connect() + + app.ceph_baseurl = app.ceph_cluster.conf_get('restapi_base_url') \ + or DEFAULT_BASEURL + if app.ceph_baseurl.endswith('/'): + app.ceph_baseurl = app.ceph_baseurl[:-1] + addr = app.ceph_cluster.conf_get('public_addr') or DEFAULT_ADDR + + # remove any nonce from the conf value + addr = addr.split('/')[0] + addr, port = addr.rsplit(':', 1) + addr = addr or DEFAULT_ADDR + port = port or DEFAULT_PORT port = int(port) - loglevel = get_conf(cfg, clientname, 'log_level') or DEFAULT_LOG_LEVEL - logfile = get_conf(cfg, clientname, 'log_file') or client_logfile + loglevel = app.ceph_cluster.conf_get('restapi_log_level') \ + or DEFAULT_LOG_LEVEL + logfile = app.ceph_cluster.conf_get('log_file') app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) app.logger.setLevel(LOGLEVELS[loglevel.lower()]) for h in app.logger.handlers: h.setFormatter(logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s')) - glob.sigdict = get_command_descriptions() + app.ceph_sigdict = get_command_descriptions(app.ceph_cluster) - osdid = find_up_osd() + osdid = find_up_osd(app) if osdid: - osd_sigdict = get_command_descriptions(target=('osd', int(osdid))) + osd_sigdict = get_command_descriptions(app.ceph_cluster, + target=('osd', int(osdid))) - # shift osd_sigdict keys up to fit at the end of the mon's glob.sigdict - maxkey = sorted(glob.sigdict.keys())[-1] + # shift osd_sigdict keys up to fit at the end of the mon's app.ceph_sigdict + maxkey = sorted(app.ceph_sigdict.keys())[-1] maxkey = int(maxkey.replace('cmd', '')) osdkey = maxkey + 1 for k, v in osd_sigdict.iteritems(): newv = v newv['flavor'] = 'tell' globk = 'cmd' + str(osdkey) - glob.sigdict[globk] = newv + app.ceph_sigdict[globk] = newv osdkey += 1 - # glob.sigdict maps "cmdNNN" to a dict containing: + # app.ceph_sigdict maps "cmdNNN" to a dict containing: # 'sig', an array of argdescs # 'help', the helptext # 'module', the Ceph module this command relates to @@ -225,11 +150,11 @@ def get_command_descriptions(target=('mon','')): # a hint as to whether this is a GET or POST/PUT operation # 'avail', a comma-separated list of strings of consumers that should # display this command (filtered by parse_json_funcsigs() above) - glob.urls = {} - for cmdnum, cmddict in glob.sigdict.iteritems(): + app.ceph_urls = {} + for cmdnum, cmddict in app.ceph_sigdict.iteritems(): cmdsig = cmddict['sig'] flavor = cmddict.get('flavor', 'mon') - url, params = generate_url_and_params(cmdsig, flavor) + url, params = generate_url_and_params(app, cmdsig, flavor) perm = cmddict['perm'] for k in METHOD_DICT.iterkeys(): if k in perm: @@ -242,34 +167,34 @@ def get_command_descriptions(target=('mon','')): 'methods':methods, } - # glob.urls contains a list of urldicts (usually only one long) - if url not in glob.urls: - glob.urls[url] = [urldict] + # app.ceph_urls contains a list of urldicts (usually only one long) + if url not in app.ceph_urls: + app.ceph_urls[url] = [urldict] else: # If more than one, need to make union of methods of all. # Method must be checked in handler methodset = set(methods) - for old_urldict in glob.urls[url]: + for old_urldict in app.ceph_urls[url]: methodset |= set(old_urldict['methods']) methods = list(methodset) - glob.urls[url].append(urldict) + app.ceph_urls[url].append(urldict) # add, or re-add, rule with all methods and urldicts app.add_url_rule(url, url, handler, methods=methods) url += '.' app.add_url_rule(url, url, handler, methods=methods) - app.logger.debug("urls added: %d", len(glob.urls)) + app.logger.debug("urls added: %d", len(app.ceph_urls)) app.add_url_rule('/', '/', handler, methods=['GET', 'PUT']) return addr, port -def generate_url_and_params(sig, flavor): +def generate_url_and_params(app, sig, flavor): ''' Digest command signature from cluster; generate an absolute - (including glob.baseurl) endpoint from all the prefix words, + (including app.ceph_baseurl) endpoint from all the prefix words, and a list of non-prefix param descs ''' @@ -304,7 +229,7 @@ def generate_url_and_params(sig, flavor): else: params.append(desc) - return glob.baseurl + url, params + return app.ceph_baseurl + url, params # @@ -339,7 +264,7 @@ def show_human_help(prefix): permmap = {'r':'GET', 'rw':'PUT'} line = '' - for cmdsig in sorted(glob.sigdict.itervalues(), cmp=descsort): + for cmdsig in sorted(app.ceph_sigdict.itervalues(), cmp=descsort): concise = concise_sig(cmdsig['sig']) flavor = cmdsig.get('flavor', 'mon') if flavor == 'tell': @@ -374,7 +299,7 @@ def log_request(): @app.route('/') def root_redir(): - return flask.redirect(glob.baseurl) + return flask.redirect(app.ceph_baseurl) def make_response(fmt, output, statusmsg, errorcode): ''' @@ -433,11 +358,11 @@ def handler(catchall_path=None, fmt=None, target=None): if ep[0] != '/': ep = '/' + ep - # demand that endpoint begin with glob.baseurl - if not ep.startswith(glob.baseurl): + # demand that endpoint begin with app.ceph_baseurl + if not ep.startswith(app.ceph_baseurl): return make_response(fmt, '', 'Page not found', 404) - rel_ep = ep[len(glob.baseurl)+1:] + rel_ep = ep[len(app.ceph_baseurl)+1:] # Extensions override Accept: headers override defaults if not fmt: @@ -477,7 +402,7 @@ def handler(catchall_path=None, fmt=None, target=None): prefix = ' '.join(rel_ep.split('/')).strip() # show "match as much as you gave me" help for unknown endpoints - if not ep in glob.urls: + if not ep in app.ceph_urls: helptext = show_human_help(prefix) if helptext: resp = flask.make_response(helptext, 400) @@ -488,7 +413,7 @@ def handler(catchall_path=None, fmt=None, target=None): found = None exc = '' - for urldict in glob.urls[ep]: + for urldict in app.ceph_urls[ep]: if flask.request.method not in urldict['methods']: continue paramsig = urldict['paramsig'] @@ -537,7 +462,7 @@ def handler(catchall_path=None, fmt=None, target=None): cmdtarget = ('mon', '') app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) - ret, outbuf, outs = json_command(glob.cluster, prefix=prefix, + ret, outbuf, outs = json_command(app.ceph_cluster, prefix=prefix, target=cmdtarget, inbuf=flask.request.data, argdict=argdict) if ret: @@ -552,6 +477,11 @@ def handler(catchall_path=None, fmt=None, target=None): return response # -# Last module-level (import-time) ask: set up the cluster connection -# -addr, port = api_setup() +# Main entry point from wrapper/WSGI server: call with cmdline args, +# get back the WSGI app entry point +# +def generate_app(conf, cluster, clientname, clientid, args): + addr, port = api_setup(app, conf, cluster, clientname, clientid, args) + app.ceph_addr = addr + app.ceph_port = port + return app From 0041e9f8ffeb4354740f2ab34b674f9acac86273 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 26 Jul 2013 17:47:49 -0700 Subject: [PATCH 25/44] rados.py: missing parameter to make_ex() Signed-off-by: Dan Mick Reviewed-by: Sage Weil --- src/pybind/rados.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/rados.py b/src/pybind/rados.py index 13ce006f8ced9..040009271cbaa 100644 --- a/src/pybind/rados.py +++ b/src/pybind/rados.py @@ -278,7 +278,7 @@ def conf_parse_argv(self, args): ret = run_in_thread(self.librados.rados_conf_parse_argv_remainder, (self.cluster, len(args), cargs, cretargs)) if ret: - raise make_ex("error calling conf_parse_argv_remainder") + raise make_ex(ret, "error calling conf_parse_argv_remainder") # cretargs was allocated with fixed length; collapse return # list to eliminate any missing args From a8c1a2a28be426f030a50822d8b335d8a3bbb3a0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 21:41:27 -0700 Subject: [PATCH 26/44] common/Formatter: add dump_format_unquoted() Sometimes a savvy caller wants to format the output but doesn't want it quoted. Signed-off-by: Sage Weil --- src/common/Formatter.cc | 27 +++++++++++++++++++++++++++ src/common/Formatter.h | 3 +++ 2 files changed, 30 insertions(+) diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index 7362684c070eb..c08ea5b9a2051 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -254,6 +254,18 @@ void JSONFormatter::dump_format(const char *name, const char *fmt, ...) print_quoted_string(buf); } +void JSONFormatter::dump_format_unquoted(const char *name, const char *fmt, ...) +{ + char buf[LARGE_SIZE]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, LARGE_SIZE, fmt, ap); + va_end(ap); + + print_name(name); + m_ss << buf; +} + int JSONFormatter::get_len() const { return m_ss.str().size(); @@ -404,6 +416,21 @@ void XMLFormatter::dump_format(const char *name, const char *fmt, ...) m_ss << "\n"; } +void XMLFormatter::dump_format_unquoted(const char *name, const char *fmt, ...) +{ + char buf[LARGE_SIZE]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, LARGE_SIZE, fmt, ap); + va_end(ap); + + std::string e(name); + print_spaces(); + m_ss << "<" << e << ">" << buf << ""; + if (m_pretty) + m_ss << "\n"; +} + int XMLFormatter::get_len() const { return m_ss.str().size(); diff --git a/src/common/Formatter.h b/src/common/Formatter.h index c62a8303ce113..da730103f41c2 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -45,6 +45,7 @@ class Formatter { virtual void dump_string(const char *name, std::string s) = 0; virtual std::ostream& dump_stream(const char *name) = 0; virtual void dump_format(const char *name, const char *fmt, ...) = 0; + virtual void dump_format_unquoted(const char *name, const char *fmt, ...) = 0; virtual int get_len() const = 0; virtual void write_raw_data(const char *data) = 0; @@ -79,6 +80,7 @@ class JSONFormatter : public Formatter { void dump_string(const char *name, std::string s); std::ostream& dump_stream(const char *name); void dump_format(const char *name, const char *fmt, ...); + void dump_format_unquoted(const char *name, const char *fmt, ...); int get_len() const; void write_raw_data(const char *data); @@ -119,6 +121,7 @@ class XMLFormatter : public Formatter { void dump_string(const char *name, std::string s); std::ostream& dump_stream(const char *name); void dump_format(const char *name, const char *fmt, ...); + void dump_format_unquoted(const char *name, const char *fmt, ...); int get_len() const; void write_raw_data(const char *data); From 2aa9afa5c542914a5ab679070ef6500e74a42770 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 21:42:25 -0700 Subject: [PATCH 27/44] common/perf_counters: fix missing decimal in time, quoting We shouldn't quote integer or float values. Also easier to use dump_unsigned. Signed-off-by: Sage Weil --- src/common/perf_counters.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index 86fb531f737c8..1dd4cdabd9d85 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -223,24 +223,24 @@ void PerfCounters::dump_formatted(Formatter *f, bool schema) if (d->type & PERFCOUNTER_LONGRUNAVG) { f->open_object_section(d->name); if (d->type & PERFCOUNTER_U64) { - f->dump_format("avgcount", "%"PRId64, d->avgcount); - f->dump_format("sum", "%"PRId64, d->u64); + f->dump_unsigned("avgcount", d->avgcount); + f->dump_unsigned("sum", d->u64); } else if (d->type & PERFCOUNTER_TIME) { - f->dump_format("avgcount", "%"PRId64, d->avgcount); - f->dump_format("sum", "%"PRId64"%09"PRId64, - d->u64 / 1000000000ull, - d->u64 % 1000000000ull); + f->dump_unsigned("avgcount", d->avgcount); + f->dump_format_unquoted("sum", "%"PRId64".%09"PRId64, + d->u64 / 1000000000ull, + d->u64 % 1000000000ull); } else { assert(0); } f->close_section(); } else { if (d->type & PERFCOUNTER_U64) { - f->dump_format(d->name, "%"PRId64, d->u64); + f->dump_unsigned(d->name, d->u64); } else if (d->type & PERFCOUNTER_TIME) { - f->dump_format(d->name, "%"PRId64"%09"PRId64, - d->u64 / 1000000000ull, - d->u64 % 1000000000ull); + f->dump_format_unquoted(d->name, "%"PRId64".%09"PRId64, + d->u64 / 1000000000ull, + d->u64 % 1000000000ull); } else { assert(0); } From b4bde3cbc044b615a60b6a5f3ffc03cda7e696f0 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 21:42:41 -0700 Subject: [PATCH 28/44] common/perf_counters: fix unit tests The commands are now in json. Signed-off-by: Sage Weil --- src/test/perf_counters.cc | 46 +++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/test/perf_counters.cc b/src/test/perf_counters.cc index d8f04ca7d10b5..d0b05f9f04945 100644 --- a/src/test/perf_counters.cc +++ b/src/test/perf_counters.cc @@ -60,7 +60,7 @@ int main(int argc, char **argv) { TEST(PerfCounters, SimpleTest) { AdminSocketClient client(get_rand_socket_path()); std::string message; - ASSERT_EQ("", client.do_request("perfcounters_dump", &message)); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\" }", &message)); ASSERT_EQ("{}", message); } @@ -100,20 +100,20 @@ TEST(PerfCounters, SinglePerfCounters) { coll->add(fake_pf); AdminSocketClient client(get_rand_socket_path()); std::string msg; - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':0," - "'element2':0.000000000,'element3':{'avgcount':0,'sum':0.000000000}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":0," + "\"element2\":0.000000000,\"element3\":{\"avgcount\":0,\"sum\":0.000000000}}}"), msg); fake_pf->inc(TEST_PERFCOUNTERS1_ELEMENT_1); fake_pf->tset(TEST_PERFCOUNTERS1_ELEMENT_2, utime_t(0, 500000000)); fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(100, 0)); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':1," - "'element2':0.500000000,'element3':{'avgcount':1,'sum':100.000000000}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1," + "\"element2\":0.500000000,\"element3\":{\"avgcount\":1,\"sum\":100.000000000}}}"), msg); fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t()); fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(25,0)); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':1,'element2':0.500000000," - "'element3':{'avgcount':3,'sum':125.000000000}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1,\"element2\":0.500000000," + "\"element3\":{\"avgcount\":3,\"sum\":125.000000000}}}"), msg); } enum { @@ -142,24 +142,24 @@ TEST(PerfCounters, MultiplePerfCounters) { AdminSocketClient client(get_rand_socket_path()); std::string msg; - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':0,'element2':0.000000000,'element3':" - "{'avgcount':0,'sum':0.000000000}},'test_perfcounter_2':{'foo':0,'bar':0.000000000}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":0,\"element2\":0.000000000,\"element3\":" + "{\"avgcount\":0,\"sum\":0.000000000}},\"test_perfcounter_2\":{\"foo\":0,\"bar\":0.000000000}}"), msg); fake_pf1->inc(TEST_PERFCOUNTERS1_ELEMENT_1); fake_pf1->inc(TEST_PERFCOUNTERS1_ELEMENT_1, 5); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':6,'element2':0.000000000,'element3':" - "{'avgcount':0,'sum':0.000000000}},'test_perfcounter_2':{'foo':0,'bar':0.000000000}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":6,\"element2\":0.000000000,\"element3\":" + "{\"avgcount\":0,\"sum\":0.000000000}},\"test_perfcounter_2\":{\"foo\":0,\"bar\":0.000000000}}"), msg); coll->remove(fake_pf2); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':6,'element2':0.000000000," - "'element3':{'avgcount':0,'sum':0.000000000}}}"), msg); - ASSERT_EQ("", client.do_request("perfcounters_schema", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':{'type':2}," - "'element2':{'type':1},'element3':{'type':5}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":6,\"element2\":0.000000000," + "\"element3\":{\"avgcount\":0,\"sum\":0.000000000}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perf schema\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":{\"type\":2}," + "\"element2\":{\"type\":1},\"element3\":{\"type\":5}}}"), msg); coll->clear(); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); ASSERT_EQ("{}", msg); } From 535d8701b95e55926cc87422587d30b271944596 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 22:08:26 -0700 Subject: [PATCH 29/44] common/admin_socket: do not populate empty help strings Hidden commands have no help string. Make this consistent: the m_help entry is always there, but may be empty. Signed-off-by: Sage Weil --- src/common/admin_socket.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index 4afd685b72acd..e73f3ce0a0c2a 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -390,8 +390,7 @@ int AdminSocket::register_command(std::string command, std::string cmddesc, Admi ldout(m_cct, 5) << "register_command " << command << " hook " << hook << dendl; m_hooks[command] = hook; m_descs[command] = cmddesc; - if (help.length()) - m_help[command] = help; + m_help[command] = help; ret = 0; } m_lock.Unlock(); @@ -448,7 +447,8 @@ class HelpHook : public AdminSocketHook { for (map::iterator p = m_as->m_help.begin(); p != m_as->m_help.end(); ++p) { - f->dump_string(p->first.c_str(), p->second); + if (p->second.length()) + f->dump_string(p->first.c_str(), p->second); } f->close_section(); ostringstream ss; From 94865368ca76368af72f39688eb688a4bee136b6 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 22:14:47 -0700 Subject: [PATCH 30/44] rgw: fix RGWFormatter_Plain Needs dump_format_unqouted. Signed-off-by: Sage Weil --- src/rgw/rgw_formats.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/rgw/rgw_formats.h b/src/rgw/rgw_formats.h index 0ae917fe7d1a5..3653ef4f48079 100644 --- a/src/rgw/rgw_formats.h +++ b/src/rgw/rgw_formats.h @@ -36,6 +36,9 @@ class RGWFormatter_Plain : public Formatter { virtual void dump_string(const char *name, std::string s); virtual std::ostream& dump_stream(const char *name); virtual void dump_format(const char *name, const char *fmt, ...); + virtual void dump_format_unquoted(const char *name, const char *fmt, ...) { + assert(0 == "not implemented"); + } virtual int get_len() const; virtual void write_raw_data(const char *data); From bd0e35f2e198b7a0683e185e4969a26488be8386 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 26 Jul 2013 22:28:40 -0700 Subject: [PATCH 31/44] test/admin_socket: fix admin_socket unit tests Command is now JSON! Signed-off-by: Sage Weil --- src/test/admin_socket.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc index 7f595f596186f..8f67918e644e4 100644 --- a/src/test/admin_socket.cc +++ b/src/test/admin_socket.cc @@ -64,7 +64,7 @@ TEST(AdminSocket, SendNoOp) { ASSERT_EQ(true, asoct.init(get_rand_socket_path())); AdminSocketClient client(get_rand_socket_path()); string version; - ASSERT_EQ("", client.do_request("0", &version)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"0\"}", &version)); ASSERT_EQ(CEPH_ADMIN_SOCK_VERSION, version); ASSERT_EQ(true, asoct.shutdown()); } @@ -87,7 +87,7 @@ TEST(AdminSocket, RegisterCommand) { AdminSocketClient client(get_rand_socket_path()); ASSERT_EQ(0, asoct.m_asokc->register_command("test", "test", new MyTest(), "")); string result; - ASSERT_EQ("", client.do_request("test", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test\"}", &result)); ASSERT_EQ("test|", result); ASSERT_EQ(true, asoct.shutdown()); } @@ -111,20 +111,20 @@ TEST(AdminSocket, RegisterCommandPrefixes) { ASSERT_EQ(0, asoct.m_asokc->register_command("test", "test", new MyTest(), "")); ASSERT_EQ(0, asoct.m_asokc->register_command("test command", "test command", new MyTest2(), "")); string result; - ASSERT_EQ("", client.do_request("test", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test\"}", &result)); ASSERT_EQ("test|", result); - ASSERT_EQ("", client.do_request("test command", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test command\"}", &result)); ASSERT_EQ("test command|", result); - ASSERT_EQ("", client.do_request("test command post", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test command post\"}", &result)); ASSERT_EQ("test command|post", result); - ASSERT_EQ("", client.do_request("test command post", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test command post\"}", &result)); ASSERT_EQ("test command| post", result); - ASSERT_EQ("", client.do_request("test this thing", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test this thing\"}", &result)); ASSERT_EQ("test|this thing", result); - ASSERT_EQ("", client.do_request("test command post", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test command post\"}", &result)); ASSERT_EQ("test| command post", result); - ASSERT_EQ("", client.do_request("test this thing", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test this thing\"}", &result)); ASSERT_EQ("test| this thing", result); ASSERT_EQ(true, asoct.shutdown()); } From a48b391fce126964b3990e54a1c2a68bcb2622a9 Mon Sep 17 00:00:00 2001 From: Danny Al-Gaaf Date: Sat, 27 Jul 2013 11:32:34 +0200 Subject: [PATCH 32/44] ceph_rest_api.py: fix bad indentation Signed-off-by: Danny Al-Gaaf --- src/pybind/ceph_rest_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index 9e53eba05fd17..59e3f60a3a731 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -219,7 +219,7 @@ def generate_url_and_params(app, sig, flavor): desc.req and \ not str(desc.instance).startswith('--') and \ not params: - url += '/' + str(desc.instance) + url += '/' + str(desc.instance) else: # tell/ is a weird case; the URL includes what # would everywhere else be a parameter From da4d23ec08cef328b400ac8f7b4c6593b2f96c74 Mon Sep 17 00:00:00 2001 From: Danny Al-Gaaf Date: Sat, 27 Jul 2013 11:35:38 +0200 Subject: [PATCH 33/44] ceph_argparse.py: remove unnecessary semicolons Signed-off-by: Danny Al-Gaaf --- src/pybind/ceph_argparse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index ee71b76d6a1f3..c4081d8d752bd 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -896,7 +896,7 @@ def find_cmd_target(childargs): """ sig = parse_funcsig(['tell', {'name':'target','type':'CephName'}]) try: - valid_dict = validate(childargs, sig, partial=True); + valid_dict = validate(childargs, sig, partial=True) except ArgumentError: pass else: @@ -910,7 +910,7 @@ def find_cmd_target(childargs): sig = parse_funcsig(['tell', {'name':'pgid','type':'CephPgid'}]) try: - valid_dict = validate(childargs, sig, partial=True); + valid_dict = validate(childargs, sig, partial=True) except ArgumentError: pass else: @@ -920,7 +920,7 @@ def find_cmd_target(childargs): sig = parse_funcsig(['pg', {'name':'pgid','type':'CephPgid'}]) try: - valid_dict = validate(childargs, sig, partial=True); + valid_dict = validate(childargs, sig, partial=True) except ArgumentError: pass else: From 6eadb870b432d68f2cfc5d7501039d0ee23ec91d Mon Sep 17 00:00:00 2001 From: Danny Al-Gaaf Date: Sat, 27 Jul 2013 11:36:35 +0200 Subject: [PATCH 34/44] ceph_argparse.py: add missing spaces after comma Signed-off-by: Danny Al-Gaaf --- src/pybind/ceph_argparse.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index c4081d8d752bd..55069d47be274 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -894,7 +894,7 @@ def find_cmd_target(childargs): right daemon. Returns ('osd', osdid), ('pg', pgid), or ('mon', '') """ - sig = parse_funcsig(['tell', {'name':'target','type':'CephName'}]) + sig = parse_funcsig(['tell', {'name':'target', 'type':'CephName'}]) try: valid_dict = validate(childargs, sig, partial=True) except ArgumentError: @@ -908,7 +908,7 @@ def find_cmd_target(childargs): name.valid(valid_dict['target']) return name.nametype, name.nameid - sig = parse_funcsig(['tell', {'name':'pgid','type':'CephPgid'}]) + sig = parse_funcsig(['tell', {'name':'pgid', 'type':'CephPgid'}]) try: valid_dict = validate(childargs, sig, partial=True) except ArgumentError: @@ -918,7 +918,7 @@ def find_cmd_target(childargs): # pg doesn't need revalidation; the string is fine return 'pg', valid_dict['pgid'] - sig = parse_funcsig(['pg', {'name':'pgid','type':'CephPgid'}]) + sig = parse_funcsig(['pg', {'name':'pgid', 'type':'CephPgid'}]) try: valid_dict = validate(childargs, sig, partial=True) except ArgumentError: From 865d5e92553d607ae9ff2c8f8eee36bbb448d72a Mon Sep 17 00:00:00 2001 From: Danny Al-Gaaf Date: Sat, 27 Jul 2013 11:37:26 +0200 Subject: [PATCH 35/44] rados.py: fix bad indentation Signed-off-by: Danny Al-Gaaf --- src/pybind/rados.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/rados.py b/src/pybind/rados.py index 040009271cbaa..e543ff7930505 100644 --- a/src/pybind/rados.py +++ b/src/pybind/rados.py @@ -278,7 +278,7 @@ def conf_parse_argv(self, args): ret = run_in_thread(self.librados.rados_conf_parse_argv_remainder, (self.cluster, len(args), cargs, cretargs)) if ret: - raise make_ex(ret, "error calling conf_parse_argv_remainder") + raise make_ex(ret, "error calling conf_parse_argv_remainder") # cretargs was allocated with fixed length; collapse return # list to eliminate any missing args From 246011807b5f00f5948b1444f00bf5f87f8e7e09 Mon Sep 17 00:00:00 2001 From: Danny Al-Gaaf Date: Sat, 27 Jul 2013 10:56:49 +0200 Subject: [PATCH 36/44] cephfs.py: remove unused imports Removed ctypes, c_uint64, c_ubyte, pointer, CFUNCTYPE since they are not used in the code. Signed-off-by: Danny Al-Gaaf --- src/pybind/cephfs.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pybind/cephfs.py b/src/pybind/cephfs.py index f89f53fb194fc..80b7e4b773ff4 100644 --- a/src/pybind/cephfs.py +++ b/src/pybind/cephfs.py @@ -2,9 +2,7 @@ This module is a thin wrapper around libcephfs. """ from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_int, c_long, c_uint, c_ulong, \ - create_string_buffer, byref, Structure, c_uint64, c_ubyte, pointer, \ - CFUNCTYPE -import ctypes + create_string_buffer, byref, Structure import errno class Error(Exception): From 09ee092ffd09de85e3b813c7595e1f818c49c385 Mon Sep 17 00:00:00 2001 From: Danny Al-Gaaf Date: Sat, 27 Jul 2013 10:59:18 +0200 Subject: [PATCH 37/44] pybind/rbd.py: remove unused import of 'pointer' Signed-off-by: Danny Al-Gaaf --- src/pybind/rbd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pybind/rbd.py b/src/pybind/rbd.py index 9d71738e7284f..6e9ca8a22525d 100644 --- a/src/pybind/rbd.py +++ b/src/pybind/rbd.py @@ -17,7 +17,7 @@ # Copyright 2011 Josh Durgin from ctypes import CDLL, c_char, c_char_p, c_size_t, c_void_p, c_int, \ create_string_buffer, byref, Structure, c_uint64, c_int64, c_uint8, \ - CFUNCTYPE, pointer + CFUNCTYPE import ctypes import errno From 6881ab3b39259adcd7aed7026ccae1cba1db7aa1 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sat, 27 Jul 2013 17:32:49 -0700 Subject: [PATCH 38/44] debian, rpm: make python-ceph depend on python-requests For ceph-rest-api. Signed-off-by: Sage Weil --- ceph.spec.in | 1 + debian/control | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ceph.spec.in b/ceph.spec.in index fb8bcca4b7140..696d9ad3332e2 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -189,6 +189,7 @@ Requires: librados2 = %{version}-%{release} Requires: librbd1 = %{version}-%{release} Requires: libcephfs1 = %{version}-%{release} Requires: python-flask +Requires: python-requests %if 0%{defined suse_version} %py_requires %endif diff --git a/debian/control b/debian/control index 4a12374bc61dd..a03259007a075 100644 --- a/debian/control +++ b/debian/control @@ -389,7 +389,7 @@ Description: Ceph test and benchmarking tools. Package: python-ceph Architecture: linux-any Section: python -Depends: librados2, librbd1, python-flask, ${misc:Depends}, ${python:Depends} +Depends: librados2, librbd1, python-flask, ${misc:Depends}, ${python:Depends}, python-requests X-Python-Version: >= 2.6 Description: Python libraries for the Ceph distributed filesystem Ceph is a distributed storage and network file system designed to provide From 4b6c569a1143d997724fbce57fdf8ae9099ebc1a Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sun, 28 Jul 2013 08:19:12 -0700 Subject: [PATCH 39/44] mon/DataHealthService: do not name xml key after mon The name might be something like '0', which is illegal. This should be the *type*, not name. Signed-off-by: Sage Weil --- src/mon/DataHealthService.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mon/DataHealthService.cc b/src/mon/DataHealthService.cc index a55e8c392e22e..6e8aa313a3616 100644 --- a/src/mon/DataHealthService.cc +++ b/src/mon/DataHealthService.cc @@ -93,7 +93,7 @@ health_status_t DataHealthService::get_health( } if (f) { - f->open_object_section(mon_name.c_str()); + f->open_object_section("mon"); f->dump_string("name", mon_name.c_str()); f->dump_int("kb_total", stats.kb_total); f->dump_int("kb_used", stats.kb_used); From 86b26328312f1d17327a77800ae0dfc10d56adf6 Mon Sep 17 00:00:00 2001 From: Danny Al-Gaaf Date: Sun, 28 Jul 2013 23:25:58 +0200 Subject: [PATCH 40/44] ceph_authtool.cc: update help/usage text Added implemented but not listed commands to the help/usage text: * -g shortcut for --gen-key * -a shortcut for --add-key * -u/--set-uid to set auid * --gen-print-key * --import-keyring Signed-off-by: Danny Al-Gaaf --- src/ceph_authtool.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/ceph_authtool.cc b/src/ceph_authtool.cc index 3075d9c69a780..f66a3c66eee9a 100644 --- a/src/ceph_authtool.cc +++ b/src/ceph_authtool.cc @@ -36,9 +36,15 @@ void usage() << " 'mount -o secret=..' argument\n" << " -C, --create-keyring will create a new keyring, overwriting any\n" << " existing keyringfile\n" - << " --gen-key will generate a new secret key for the\n" + << " -g, --gen-key will generate a new secret key for the\n" << " specified entityname\n" - << " --add-key will add an encoded key to the keyring\n" + << " --gen-print-key will generate a new secret key without set it\n" + << " to the keyringfile, prints the secret to stdout\n" + << " --import-keyring will import the content of a given keyring\n" + << " into the keyringfile\n" + << " -u, --set-uid sets the auid (authenticated user id) for the\n" + << " specified entityname\n" + << " -a, --add-key will add an encoded key to the keyring\n" << " --cap subsystem capability will set the capability for given subsystem\n" << " --caps capsfile will set all of capabilities associated with a\n" << " given key, for all subsystems" From 347b5a207846fe058d2deb19c976df810bc0c922 Mon Sep 17 00:00:00 2001 From: Danny Al-Gaaf Date: Sun, 28 Jul 2013 23:39:09 +0200 Subject: [PATCH 41/44] ceph-authtool.8: add missing commands to man page Signed-off-by: Danny Al-Gaaf --- man/ceph-authtool.8 | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/man/ceph-authtool.8 b/man/ceph-authtool.8 index 47888af1f2245..e64cac95f0a59 100644 --- a/man/ceph-authtool.8 +++ b/man/ceph-authtool.8 @@ -69,16 +69,33 @@ will create a new keyring, overwriting any existing keyringfile .UNINDENT .INDENT 0.0 .TP -.B \-\-gen\-key +.B \-g, \-\-gen\-key will generate a new secret key for the specified entityname .UNINDENT .INDENT 0.0 .TP -.B \-\-add\-key +.B \-a, \-\-add\-key will add an encoded key to the keyring .UNINDENT .INDENT 0.0 .TP +.B \-u, \-\-set\-uid +sets the auid (authenticated user id) for the specified entityname +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-gen\-print\-key +will generate and print a new secret key without adding it to the keyringfile + +NOTE: will work without a given keyringfile +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-import\-keyring +will import the content of a given keyring into the keyringfile +.UNINDENT +.INDENT 0.0 +.TP .B \-\-cap subsystem capability will set the capability for given subsystem .UNINDENT From 7b683f7f6a4b1fbb526c4f5f5660cc36f6a3abbd Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sun, 28 Jul 2013 09:04:05 -0700 Subject: [PATCH 42/44] test/system/*: parse CEPH_ARGS environment Signed-off-by: Sage Weil --- src/test/system/rados_list_parallel.cc | 2 ++ src/test/system/rados_open_pools_parallel.cc | 1 + src/test/system/st_rados_create_pool.cc | 1 + src/test/system/st_rados_delete_objs.cc | 1 + src/test/system/st_rados_delete_pool.cc | 1 + src/test/system/st_rados_list_objects.cc | 1 + src/test/system/st_rados_notify.cc | 2 ++ src/test/system/st_rados_watch.cc | 2 ++ 8 files changed, 11 insertions(+) diff --git a/src/test/system/rados_list_parallel.cc b/src/test/system/rados_list_parallel.cc index a1c6e270265dc..d530c83441c6b 100644 --- a/src/test/system/rados_list_parallel.cc +++ b/src/test/system/rados_list_parallel.cc @@ -62,6 +62,7 @@ class RadosDeleteObjectsR : public SysTestRunnable RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); std::string log_name = SysTestSettings::inst().get_log_name(get_id_str()); if (!log_name.empty()) rados_conf_set(cl, "log_file", log_name.c_str()); @@ -142,6 +143,7 @@ class RadosAddObjectsR : public SysTestRunnable RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); std::string log_name = SysTestSettings::inst().get_log_name(get_id_str()); if (!log_name.empty()) rados_conf_set(cl, "log_file", log_name.c_str()); diff --git a/src/test/system/rados_open_pools_parallel.cc b/src/test/system/rados_open_pools_parallel.cc index 445f2ebc485d9..82c712077f272 100644 --- a/src/test/system/rados_open_pools_parallel.cc +++ b/src/test/system/rados_open_pools_parallel.cc @@ -68,6 +68,7 @@ class StRadosOpenPool : public SysTestRunnable if (!log_name.empty()) rados_conf_set(cl, "log_file", log_name.c_str()); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); RETURN1_IF_NONZERO(rados_connect(cl)); if (m_pool_setup_sem) m_pool_setup_sem->wait(); diff --git a/src/test/system/st_rados_create_pool.cc b/src/test/system/st_rados_create_pool.cc index dcae15375afa7..29b8d14b7e7ea 100644 --- a/src/test/system/st_rados_create_pool.cc +++ b/src/test/system/st_rados_create_pool.cc @@ -72,6 +72,7 @@ run() std::string log_name = SysTestSettings::inst().get_log_name(get_id_str()); if (!log_name.empty()) rados_conf_set(cl, "log_file", log_name.c_str()); + rados_conf_parse_env(cl, NULL); if (m_setup_sem) { m_setup_sem->wait(); diff --git a/src/test/system/st_rados_delete_objs.cc b/src/test/system/st_rados_delete_objs.cc index 38dc47a35579b..b43ffdda6aeb5 100644 --- a/src/test/system/st_rados_delete_objs.cc +++ b/src/test/system/st_rados_delete_objs.cc @@ -45,6 +45,7 @@ int StRadosDeleteObjs::run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); RETURN1_IF_NONZERO(rados_connect(cl)); m_setup_sem->wait(); m_setup_sem->post(); diff --git a/src/test/system/st_rados_delete_pool.cc b/src/test/system/st_rados_delete_pool.cc index d954bf46c231a..de553e98e13a3 100644 --- a/src/test/system/st_rados_delete_pool.cc +++ b/src/test/system/st_rados_delete_pool.cc @@ -41,6 +41,7 @@ int StRadosDeletePool::run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); RETURN1_IF_NONZERO(rados_connect(cl)); m_pool_setup_sem->wait(); m_pool_setup_sem->post(); diff --git a/src/test/system/st_rados_list_objects.cc b/src/test/system/st_rados_list_objects.cc index bb153affeb836..be6ead6498745 100644 --- a/src/test/system/st_rados_list_objects.cc +++ b/src/test/system/st_rados_list_objects.cc @@ -56,6 +56,7 @@ run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); RETURN1_IF_NONZERO(rados_connect(cl)); m_pool_setup_sem->wait(); m_pool_setup_sem->post(); diff --git a/src/test/system/st_rados_notify.cc b/src/test/system/st_rados_notify.cc index 10e3e4bfbc39e..e74711cb6410c 100644 --- a/src/test/system/st_rados_notify.cc +++ b/src/test/system/st_rados_notify.cc @@ -44,6 +44,8 @@ int StRadosNotify::run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); + if (m_setup_sem) { m_setup_sem->wait(); m_setup_sem->post(); diff --git a/src/test/system/st_rados_watch.cc b/src/test/system/st_rados_watch.cc index 991dde8ea77e9..696b0867e8893 100644 --- a/src/test/system/st_rados_watch.cc +++ b/src/test/system/st_rados_watch.cc @@ -52,6 +52,8 @@ run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); + if (m_setup_sem) { m_setup_sem->wait(); m_setup_sem->post(); From 74c1bec2aa01caf0237b4750cf05404b6d09aa18 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sun, 28 Jul 2013 15:29:49 -0700 Subject: [PATCH 43/44] ceph-authtool: fix cli tests Signed-off-by: Sage Weil --- src/test/cli/ceph-authtool/help.t | 10 ++++++++-- src/test/cli/ceph-authtool/manpage.t | 10 ++++++++-- src/test/cli/ceph-authtool/simple.t | 10 ++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/test/cli/ceph-authtool/help.t b/src/test/cli/ceph-authtool/help.t index 7434ec4a7c00d..062c967a1548a 100644 --- a/src/test/cli/ceph-authtool/help.t +++ b/src/test/cli/ceph-authtool/help.t @@ -10,9 +10,15 @@ 'mount -o secret=..' argument -C, --create-keyring will create a new keyring, overwriting any existing keyringfile - --gen-key will generate a new secret key for the + -g, --gen-key will generate a new secret key for the specified entityname - --add-key will add an encoded key to the keyring + --gen-print-key will generate a new secret key without set it + to the keyringfile, prints the secret to stdout + --import-keyring will import the content of a given keyring + into the keyringfile + -u, --set-uid sets the auid (authenticated user id) for the + specified entityname + -a, --add-key will add an encoded key to the keyring --cap subsystem capability will set the capability for given subsystem --caps capsfile will set all of capabilities associated with a given key, for all subsystems diff --git a/src/test/cli/ceph-authtool/manpage.t b/src/test/cli/ceph-authtool/manpage.t index 69e15fa51c357..a9e1408d7163e 100644 --- a/src/test/cli/ceph-authtool/manpage.t +++ b/src/test/cli/ceph-authtool/manpage.t @@ -9,9 +9,15 @@ 'mount -o secret=..' argument -C, --create-keyring will create a new keyring, overwriting any existing keyringfile - --gen-key will generate a new secret key for the + -g, --gen-key will generate a new secret key for the specified entityname - --add-key will add an encoded key to the keyring + --gen-print-key will generate a new secret key without set it + to the keyringfile, prints the secret to stdout + --import-keyring will import the content of a given keyring + into the keyringfile + -u, --set-uid sets the auid (authenticated user id) for the + specified entityname + -a, --add-key will add an encoded key to the keyring --cap subsystem capability will set the capability for given subsystem --caps capsfile will set all of capabilities associated with a given key, for all subsystems diff --git a/src/test/cli/ceph-authtool/simple.t b/src/test/cli/ceph-authtool/simple.t index 69d027c13421d..b86476a5ad0f5 100644 --- a/src/test/cli/ceph-authtool/simple.t +++ b/src/test/cli/ceph-authtool/simple.t @@ -9,9 +9,15 @@ 'mount -o secret=..' argument -C, --create-keyring will create a new keyring, overwriting any existing keyringfile - --gen-key will generate a new secret key for the + -g, --gen-key will generate a new secret key for the specified entityname - --add-key will add an encoded key to the keyring + --gen-print-key will generate a new secret key without set it + to the keyringfile, prints the secret to stdout + --import-keyring will import the content of a given keyring + into the keyringfile + -u, --set-uid sets the auid (authenticated user id) for the + specified entityname + -a, --add-key will add an encoded key to the keyring --cap subsystem capability will set the capability for given subsystem --caps capsfile will set all of capabilities associated with a given key, for all subsystems From 12c1f1157c7b9513a3d9f716a8ec62fce00d28f5 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Sun, 28 Jul 2013 15:42:08 -0700 Subject: [PATCH 44/44] ceph_test_rados: print version banner on startup It is helpful when looking at qa run logs to see what version of the tester is running. Signed-off-by: Sage Weil --- src/test/osd/TestRados.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/osd/TestRados.cc b/src/test/osd/TestRados.cc index 43530f008284c..6ac661c06294a 100644 --- a/src/test/osd/TestRados.cc +++ b/src/test/osd/TestRados.cc @@ -2,6 +2,7 @@ #include "common/Mutex.h" #include "common/Cond.h" #include "common/errno.h" +#include "common/version.h" #include #include @@ -14,6 +15,7 @@ #include "test/osd/RadosModel.h" + using namespace std; class WeightedTestGenerator : public TestOpGenerator @@ -250,6 +252,7 @@ int main(int argc, char **argv) if (max_stride_size < 0) max_stride_size = size / 5; + cout << pretty_version_to_str() << std::endl; cout << "Configuration:" << std::endl << "\tNumber of operations: " << ops << std::endl << "\tNumber of objects: " << objects << std::endl
Possible commands:MethodDescription