From 3f598e8c65731c3451c1aecf8ec0925e34b0bec8 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 14:38:33 -0700 Subject: [PATCH 01/13] AdminSocket users: use generic formatting All call() routines get a format parameter; all places where JSONFormatter was created get a new_formatter() instead. 'plain' formatting is unsupported, and help is forced to be 'json-pretty' as it was. Signed-off-by: Dan Mick --- src/client/Client.cc | 14 ++- src/client/Client.h | 3 +- src/common/admin_socket.cc | 41 +++++-- src/common/admin_socket.h | 3 +- src/common/ceph_context.cc | 35 +++--- src/common/ceph_context.h | 3 +- src/common/perf_counters.cc | 100 ++++++++--------- src/common/perf_counters.h | 4 +- src/mon/Monitor.cc | 11 +- src/mon/Monitor.h | 3 +- src/osd/OSD.cc | 78 ++++++------- src/osd/OSD.h | 2 +- src/osd/OpRequest.cc | 28 +++-- src/osd/OpRequest.h | 4 +- src/osdc/Objecter.cc | 164 ++++++++++++++-------------- src/osdc/Objecter.h | 17 +-- src/test/bench/small_io_bench_fs.cc | 4 +- 17 files changed, 267 insertions(+), 247 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index ba036ad998036..5a9c5fdafcca2 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -102,22 +102,24 @@ Client::CommandHook::CommandHook(Client *client) : { } -bool Client::CommandHook::call(std::string command, std::string args, bufferlist& out) +bool Client::CommandHook::call(std::string command, std::string args, + std::string format, bufferlist& out) { stringstream ss; - JSONFormatter formatter(true); + Formatter *f = new_formatter(format); m_client->client_lock.Lock(); if (command == "mds_requests") - m_client->dump_mds_requests(&formatter); + m_client->dump_mds_requests(f); else if (command == "mds_sessions") - m_client->dump_mds_sessions(&formatter); + m_client->dump_mds_sessions(f); else if (command == "dump_cache") - m_client->dump_cache(&formatter); + m_client->dump_cache(f); else assert(0 == "bad command registered"); m_client->client_lock.Unlock(); - formatter.flush(ss); + f->flush(ss); out.append(ss); + delete f; return true; } diff --git a/src/client/Client.h b/src/client/Client.h index ade0b8f29c84d..bc1fbc0401b5c 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -196,7 +196,8 @@ class Client : public Dispatcher { Client *m_client; public: CommandHook(Client *client); - bool call(std::string command, std::string args, bufferlist& out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out); }; CommandHook m_command_hook; diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index f7ab3501dfff9..5ebb3e040db8d 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -310,6 +310,25 @@ bool AdminSocket::do_accept() bool rval = false; + map cmdmap; + string format; + vector cmdvec; + stringstream errss; + cmdvec.push_back(cmd); + if (!cmdmap_from_json(cmdvec, &cmdmap, errss)) { + ldout(m_cct, 0) << "AdminSocket: " << errss << dendl; + return false; + } + cmd_getval(m_cct, cmdmap, "format", format); + cmd_getval(m_cct, cmdmap, "prefix", c); + + // we don't do plain here + if (format != "json" && + format != "json-pretty" && + format != "xml" && + format != "xml-pretty") + format = "json"; + string firstword; if (c.find(" ") == string::npos) firstword = c; @@ -341,7 +360,7 @@ bool AdminSocket::do_accept() string args; if (match != c) args = c.substr(match.length() + 1); - bool success = p->second->call(match, args, out); + bool success = p->second->call(match, args, format, out); if (!success) { ldout(m_cct, 0) << "AdminSocket: request '" << match << "' args '" << args << "' to " << p->second << " failed" << dendl; @@ -406,7 +425,8 @@ int AdminSocket::unregister_command(std::string command) class VersionHook : public AdminSocketHook { public: - virtual bool call(std::string command, std::string args, bufferlist& out) { + virtual bool call(std::string command, std::string args, std::string format, + bufferlist& out) { if (command == "0") { out.append(CEPH_ADMIN_SOCK_VERSION); } else { @@ -429,18 +449,21 @@ class HelpHook : public AdminSocketHook { AdminSocket *m_as; public: HelpHook(AdminSocket *as) : m_as(as) {} - bool call(string command, string args, bufferlist& out) { - JSONFormatter jf(true); - jf.open_object_section("help"); + bool call(string command, string args, string format, bufferlist& out) { + // override format here because help should always be pretty and + // predictable + Formatter *f = new_formatter("json-pretty"); + f->open_object_section("help"); for (map::iterator p = m_as->m_help.begin(); p != m_as->m_help.end(); ++p) { - jf.dump_string(p->first.c_str(), p->second); + f->dump_string(p->first.c_str(), p->second); } - jf.close_section(); + f->close_section(); ostringstream ss; - jf.flush(ss); + f->flush(ss); out.append(ss.str()); + delete f; return true; } }; @@ -449,7 +472,7 @@ class GetdescsHook : public AdminSocketHook { AdminSocket *m_as; public: GetdescsHook(AdminSocket *as) : m_as(as) {} - bool call(string command, string args, bufferlist& out) { + bool call(string command, string args, string format, bufferlist& out) { int cmdnum = 0; JSONFormatter jf(false); jf.open_object_section("command_descriptions"); diff --git a/src/common/admin_socket.h b/src/common/admin_socket.h index c390bca0382c3..30c5eb96ab8d6 100644 --- a/src/common/admin_socket.h +++ b/src/common/admin_socket.h @@ -29,7 +29,8 @@ class CephContext; class AdminSocketHook { public: - virtual bool call(std::string command, std::string args, bufferlist& out) = 0; + virtual bool call(std::string command, std::string args, std::string format, + bufferlist& out) = 0; virtual ~AdminSocketHook() {}; }; diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index cad980bb2a6f2..e0fcf530222f4 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -156,44 +156,46 @@ class CephContextHook : public AdminSocketHook { public: CephContextHook(CephContext *cct) : m_cct(cct) {} - bool call(std::string command, std::string args, bufferlist& out) { - m_cct->do_command(command, args, &out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { + m_cct->do_command(command, args, format, &out); return true; } }; -void CephContext::do_command(std::string command, std::string args, bufferlist *out) +void CephContext::do_command(std::string command, std::string args, + std::string format, bufferlist *out) { + Formatter *f = new_formatter(format); lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "'" << dendl; if (command == "perfcounters_dump" || command == "1" || command == "perf dump") { - _perf_counters_collection->write_json_to_buf(*out, false); + _perf_counters_collection->dump_formatted(f, *out, false); } else if (command == "perfcounters_schema" || command == "2" || command == "perf schema") { - _perf_counters_collection->write_json_to_buf(*out, true); + _perf_counters_collection->dump_formatted(f, *out, true); } else { - JSONFormatter jf(true); - jf.open_object_section(command.c_str()); + f->open_object_section(command.c_str()); if (command == "config show") { - _conf->show_config(&jf); + _conf->show_config(f); } else if (command == "config set") { std::string var = args; size_t pos = var.find(' '); if (pos == string::npos) { - jf.dump_string("error", "syntax error: 'config set '"); + f->dump_string("error", "syntax error: 'config set '"); } else { std::string val = var.substr(pos+1); var.resize(pos); int r = _conf->set_val(var.c_str(), val.c_str()); if (r < 0) { - jf.dump_stream("error") << "error setting '" << var << "' to '" << val << "': " << cpp_strerror(r); + f->dump_stream("error") << "error setting '" << var << "' to '" << val << "': " << cpp_strerror(r); } else { ostringstream ss; _conf->apply_changes(&ss); - jf.dump_string("success", ss.str()); + f->dump_string("success", ss.str()); } } } else if (command == "config get") { @@ -202,9 +204,9 @@ void CephContext::do_command(std::string command, std::string args, bufferlist * char *tmp = buf; int r = _conf->get_val(args.c_str(), &tmp, sizeof(buf)); if (r < 0) { - jf.dump_stream("error") << "error getting '" << args << "': " << cpp_strerror(r); + f->dump_stream("error") << "error getting '" << args << "': " << cpp_strerror(r); } else { - jf.dump_string(args.c_str(), buf); + f->dump_string(args.c_str(), buf); } } else if (command == "log flush") { _log->flush(); @@ -218,11 +220,10 @@ void CephContext::do_command(std::string command, std::string args, bufferlist * else { assert(0 == "registered under wrong command?"); } - ostringstream ss; - jf.close_section(); - jf.flush(ss); - out->append(ss.str()); + f->close_section(); + f->flush(*out); } + delete f; lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "' result is " << out->length() << " bytes" << dendl; }; diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h index 1678680fa6db5..85618e3521934 100644 --- a/src/common/ceph_context.h +++ b/src/common/ceph_context.h @@ -97,7 +97,8 @@ class CephContext { /** * process an admin socket command */ - void do_command(std::string command, std::string args, bufferlist *out); + void do_command(std::string command, std::string args, std::string foramt, + bufferlist *out); /** * get a crypto handler diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index 67a777497b38a..46f55fae51bfc 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -15,6 +15,7 @@ #include "common/perf_counters.h" #include "common/dout.h" #include "common/errno.h" +#include "common/Formatter.h" #include #include @@ -72,21 +73,22 @@ void PerfCountersCollection::clear() } } -void PerfCountersCollection::write_json_to_buf(bufferlist& bl, bool schema) +void PerfCountersCollection::dump_formatted(Formatter *f, bufferlist &bl, + bool schema) { Mutex::Locker lck(m_lock); - bl.append('{'); + f->open_object_section("perfcounter_collection"); perf_counters_set_t::iterator l = m_loggers.begin(); perf_counters_set_t::iterator l_end = m_loggers.end(); if (l != l_end) { while (true) { - (*l)->write_json_to_buf(bl, schema); + (*l)->dump_formatted(f, schema); if (++l == l_end) break; - bl.append(','); } } - bl.append('}'); + f->close_section(); + f->flush(bl); } // --------------------------- @@ -203,34 +205,54 @@ utime_t PerfCounters::tget(int idx) const return utime_t(data.u64 / 1000000000ull, data.u64 % 1000000000ull); } -void PerfCounters::write_json_to_buf(bufferlist& bl, bool schema) +void PerfCounters::dump_formatted(Formatter *f, bool schema) { - char buf[512]; Mutex::Locker lck(m_lock); - snprintf(buf, sizeof(buf), "\"%s\":{", m_name.c_str()); - bl.append(buf); - + f->open_object_section(m_name.c_str()); perf_counter_data_vec_t::const_iterator d = m_data.begin(); perf_counter_data_vec_t::const_iterator d_end = m_data.end(); if (d == d_end) { - bl.append('}'); + f->close_section(); return; } while (true) { - const perf_counter_data_any_d &data(*d); - buf[0] = '\0'; - if (schema) - data.write_schema_json(buf, sizeof(buf)); - else - data.write_json(buf, sizeof(buf)); - - bl.append(buf); + if (schema) { + f->open_object_section(d->name); + f->dump_int("type", d->type); + f->close_section(); + } else { + if (d->type & PERFCOUNTER_LONGRUNAVG) { + f->open_object_section(d->name); + if (d->type & PERFCOUNTER_U64) { + f->dump_format("avgcount", "%"PRId64, d->avgcount); + f->dump_format("sum", "%"PRId64, d->u64); + } else if (d->type & PERFCOUNTER_TIME) { + f->dump_format("avgcount", "%"PRId64, d->avgcount); + f->dump_format("sum", "%"PRId64"%09"PRId64, + d->u64 / 1000000000ull, + d->u64 % 1000000000ull); + } else { + assert(0); + } + f->close_section(); + } else { + if (d->type & PERFCOUNTER_U64) { + f->dump_format(d->name, "%"PRId64, d->u64); + } else if (d->type & PERFCOUNTER_TIME) { + f->dump_format(d->name, "%"PRId64"%09"PRId64, + d->u64 / 1000000000ull, + d->u64 % 1000000000ull); + } else { + assert(0); + } + } + } + if (++d == d_end) break; - bl.append(','); } - bl.append('}'); + f->close_section(); } const std::string &PerfCounters::get_name() const @@ -258,42 +280,6 @@ PerfCounters::perf_counter_data_any_d::perf_counter_data_any_d() { } -void PerfCounters::perf_counter_data_any_d::write_schema_json(char *buf, size_t buf_sz) const -{ - snprintf(buf, buf_sz, "\"%s\":{\"type\":%d}", name, type); -} - -void PerfCounters::perf_counter_data_any_d::write_json(char *buf, size_t buf_sz) const -{ - if (type & PERFCOUNTER_LONGRUNAVG) { - if (type & PERFCOUNTER_U64) { - snprintf(buf, buf_sz, "\"%s\":{\"avgcount\":%" PRId64 "," - "\"sum\":%" PRId64 "}", - name, avgcount, u64); - } - else if (type & PERFCOUNTER_TIME) { - snprintf(buf, buf_sz, "\"%s\":{\"avgcount\":%" PRId64 "," - "\"sum\":%llu.%09llu}", - name, avgcount, u64 / 1000000000ull, u64 % 1000000000ull); - } - else { - assert(0); - } - } - else { - if (type & PERFCOUNTER_U64) { - snprintf(buf, buf_sz, "\"%s\":%" PRId64, - name, u64); - } - else if (type & PERFCOUNTER_TIME) { - snprintf(buf, buf_sz, "\"%s\":%llu.%09llu", name, u64 / 1000000000ull, u64 % 1000000000ull); - } - else { - assert(0); - } - } -} - PerfCountersBuilder::PerfCountersBuilder(CephContext *cct, const std::string &name, int first, int last) : m_perf_counters(new PerfCounters(cct, name, first, last)) diff --git a/src/common/perf_counters.h b/src/common/perf_counters.h index 269a32f2c46f8..125d84c04e34a 100644 --- a/src/common/perf_counters.h +++ b/src/common/perf_counters.h @@ -76,7 +76,7 @@ class PerfCounters void tinc(int idx, utime_t v); utime_t tget(int idx) const; - void write_json_to_buf(ceph::bufferlist& bl, bool schema); + void dump_formatted(ceph::Formatter *f, bool schema); const std::string& get_name() const; void set_name(std::string s) { @@ -136,7 +136,7 @@ class PerfCountersCollection void add(class PerfCounters *l); void remove(class PerfCounters *l); void clear(); - void write_json_to_buf(ceph::bufferlist& bl, bool schema); + void dump_formatted(ceph::Formatter *f, bufferlist &bl, bool schema); private: CephContext *m_cct; diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index bf500dff218fc..119ef740aa834 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -225,21 +225,20 @@ class AdminHook : public AdminSocketHook { Monitor *mon; public: AdminHook(Monitor *m) : mon(m) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; - mon->do_admin_command(command, args, ss); + mon->do_admin_command(command, args, format, ss); out.append(ss); return true; } }; -void Monitor::do_admin_command(string command, string args, ostream& ss) +void Monitor::do_admin_command(string command, string args, string format, + ostream& ss) { Mutex::Locker l(lock); - map cmdmap; - string format; - cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain")); boost::scoped_ptr f(new_formatter(format)); if (command == "mon_status") diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 82b08816702b0..bed48ecee3436 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -745,7 +745,8 @@ class Monitor : public Dispatcher { int write_fsid(); int write_fsid(MonitorDBStore::Transaction &t); - void do_admin_command(std::string command, std::string args, ostream& ss); + void do_admin_command(std::string command, std::string args, + std::string format, ostream& ss); private: // don't allow copying diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index bc3aa604fecf6..dd9a5b7293f33 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -997,44 +997,46 @@ class OSDSocketHook : public AdminSocketHook { OSD *osd; public: OSDSocketHook(OSD *o) : osd(o) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; - bool r = osd->asok_command(command, args, ss); + bool r = osd->asok_command(command, args, format, ss); out.append(ss); return r; } }; -bool OSD::asok_command(string command, string args, ostream& ss) +bool OSD::asok_command(string command, string args, string format, ostream& ss) { + if (format == "") + format = "json-pretty"; + Formatter *f = new_formatter(format); if (command == "dump_ops_in_flight") { - op_tracker.dump_ops_in_flight(ss); + op_tracker.dump_ops_in_flight(f, ss); } else if (command == "dump_historic_ops") { - op_tracker.dump_historic_ops(ss); + op_tracker.dump_historic_ops(f, ss); } else if (command == "dump_op_pq_state") { - JSONFormatter f(true); - f.open_object_section("pq"); - op_wq.dump(&f); - f.close_section(); - f.flush(ss); + f->open_object_section("pq"); + op_wq.dump(f); + f->close_section(); + f->flush(ss); } else if (command == "dump_blacklist") { list > bl; OSDMapRef curmap = service.get_osdmap(); - JSONFormatter f(true); - f.open_array_section("blacklist"); + f->open_array_section("blacklist"); curmap->get_blacklist(&bl); for (list >::iterator it = bl.begin(); it != bl.end(); ++it) { - f.open_array_section("entry"); - f.open_object_section("entity_addr_t"); - it->first.dump(&f); - f.close_section(); //entity_addr_t - it->second.localtime(f.dump_stream("expire_time")); - f.close_section(); //entry - } - f.close_section(); //blacklist - f.flush(ss); + f->open_array_section("entry"); + f->open_object_section("entity_addr_t"); + it->first.dump(f); + f->close_section(); //entity_addr_t + it->second.localtime(f->dump_stream("expire_time")); + f->close_section(); //entry + } + f->close_section(); //blacklist + f->flush(ss); } else if (command == "dump_watchers") { list watchers; osd_lock.Lock(); @@ -1052,32 +1054,31 @@ bool OSD::asok_command(string command, string args, ostream& ss) } osd_lock.Unlock(); - JSONFormatter f(true); - f.open_array_section("watchers"); + f->open_array_section("watchers"); for (list::iterator it = watchers.begin(); it != watchers.end(); ++it) { - f.open_array_section("watch"); + f->open_array_section("watch"); - f.dump_string("namespace", it->obj.nspace); - f.dump_string("object", it->obj.oid.name); + f->dump_string("namespace", it->obj.nspace); + f->dump_string("object", it->obj.oid.name); - f.open_object_section("entity_name"); - it->wi.name.dump(&f); - f.close_section(); //entity_name_t + f->open_object_section("entity_name"); + it->wi.name.dump(f); + f->close_section(); //entity_name_t - f.dump_int("cookie", it->wi.cookie); - f.dump_int("timeout", it->wi.timeout_seconds); + f->dump_int("cookie", it->wi.cookie); + f->dump_int("timeout", it->wi.timeout_seconds); - f.open_object_section("entity_addr_t"); - it->wi.addr.dump(&f); - f.close_section(); //entity_addr_t + f->open_object_section("entity_addr_t"); + it->wi.addr.dump(f); + f->close_section(); //entity_addr_t - f.close_section(); //watch + f->close_section(); //watch } - f.close_section(); //watches - f.flush(ss); + f->close_section(); //watches + f->flush(ss); } else { assert(0 == "broken asok registration"); } @@ -1089,7 +1090,8 @@ class TestOpsSocketHook : public AdminSocketHook { ObjectStore *store; public: TestOpsSocketHook(OSDService *s, ObjectStore *st) : service(s), store(st) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; test_ops(service, store, command, args, ss); out.append(ss); diff --git a/src/osd/OSD.h b/src/osd/OSD.h index f9ceaf81bf391..5bcff7442d7a6 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -622,7 +622,7 @@ class OSD : public Dispatcher, // asok friend class OSDSocketHook; class OSDSocketHook *asok_hook; - bool asok_command(string command, string args, ostream& ss); + bool asok_command(string command, string args, string format, ostream& ss); public: ClassHandler *class_handler; diff --git a/src/osd/OpRequest.cc b/src/osd/OpRequest.cc index 3b8a8714d9221..c0d167a5f0ad1 100644 --- a/src/osd/OpRequest.cc +++ b/src/osd/OpRequest.cc @@ -76,31 +76,29 @@ void OpHistory::dump_ops(utime_t now, Formatter *f) f->close_section(); } -void OpTracker::dump_historic_ops(ostream &ss) +void OpTracker::dump_historic_ops(Formatter *f, ostream &ss) { - JSONFormatter jf(true); Mutex::Locker locker(ops_in_flight_lock); utime_t now = ceph_clock_now(g_ceph_context); - history.dump_ops(now, &jf); - jf.flush(ss); + history.dump_ops(now, f); + f->flush(ss); } -void OpTracker::dump_ops_in_flight(ostream &ss) +void OpTracker::dump_ops_in_flight(Formatter *f, ostream &ss) { - JSONFormatter jf(true); Mutex::Locker locker(ops_in_flight_lock); - jf.open_object_section("ops_in_flight"); // overall dump - jf.dump_int("num_ops", ops_in_flight.size()); - jf.open_array_section("ops"); // list of OpRequests + f->open_object_section("ops_in_flight"); // overall dump + f->dump_int("num_ops", ops_in_flight.size()); + f->open_array_section("ops"); // list of OpRequests utime_t now = ceph_clock_now(g_ceph_context); for (xlist::iterator p = ops_in_flight.begin(); !p.end(); ++p) { - jf.open_object_section("op"); - (*p)->dump(now, &jf); - jf.close_section(); // this OpRequest + f->open_object_section("op"); + (*p)->dump(now, f); + f->close_section(); // this OpRequest } - jf.close_section(); // list of OpRequests - jf.close_section(); // overall dump - jf.flush(ss); + f->close_section(); // list of OpRequests + f->close_section(); // overall dump + f->flush(ss); } void OpTracker::register_inflight_op(xlist::item *i) diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h index 47b050b853815..67ee26b02ec14 100644 --- a/src/osd/OpRequest.h +++ b/src/osd/OpRequest.h @@ -59,8 +59,8 @@ class OpTracker { public: OpTracker() : seq(0), ops_in_flight_lock("OpTracker mutex") {} - void dump_ops_in_flight(std::ostream& ss); - void dump_historic_ops(std::ostream& ss); + void dump_ops_in_flight(Formatter *f, std::ostream& ss); + void dump_historic_ops(Formatter *f, std::ostream& ss); void register_inflight_op(xlist::item *i); void unregister_inflight_op(OpRequest *i); diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc index a5a023cb33e66..e07d0626d21e4 100644 --- a/src/osdc/Objecter.cc +++ b/src/osdc/Objecter.cc @@ -2180,154 +2180,154 @@ void Objecter::dump_active() } } -void Objecter::dump_requests(Formatter& fmt) const +void Objecter::dump_requests(Formatter *fmt) const { assert(client_lock.is_locked()); - fmt.open_object_section("requests"); + fmt->open_object_section("requests"); dump_ops(fmt); dump_linger_ops(fmt); dump_pool_ops(fmt); dump_pool_stat_ops(fmt); dump_statfs_ops(fmt); dump_command_ops(fmt); - fmt.close_section(); // requests object + fmt->close_section(); // requests object } -void Objecter::dump_ops(Formatter& fmt) const +void Objecter::dump_ops(Formatter *fmt) const { - fmt.open_array_section("ops"); + fmt->open_array_section("ops"); for (map::const_iterator p = ops.begin(); p != ops.end(); ++p) { Op *op = p->second; - fmt.open_object_section("op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("pg") << op->pgid; - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.dump_stream("last_sent") << op->stamp; - fmt.dump_int("attempts", op->attempts); - fmt.dump_stream("object_id") << op->oid; - fmt.dump_stream("object_locator") << op->oloc; - fmt.dump_stream("snapid") << op->snapid; - fmt.dump_stream("snap_context") << op->snapc; - fmt.dump_stream("mtime") << op->mtime; - - fmt.open_array_section("osd_ops"); + fmt->open_object_section("op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("pg") << op->pgid; + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->dump_stream("last_sent") << op->stamp; + fmt->dump_int("attempts", op->attempts); + fmt->dump_stream("object_id") << op->oid; + fmt->dump_stream("object_locator") << op->oloc; + fmt->dump_stream("snapid") << op->snapid; + fmt->dump_stream("snap_context") << op->snapc; + fmt->dump_stream("mtime") << op->mtime; + + fmt->open_array_section("osd_ops"); for (vector::const_iterator it = op->ops.begin(); it != op->ops.end(); ++it) { - fmt.dump_stream("osd_op") << *it; + fmt->dump_stream("osd_op") << *it; } - fmt.close_section(); // osd_ops array + fmt->close_section(); // osd_ops array - fmt.close_section(); // op object + fmt->close_section(); // op object } - fmt.close_section(); // ops array + fmt->close_section(); // ops array } -void Objecter::dump_linger_ops(Formatter& fmt) const +void Objecter::dump_linger_ops(Formatter *fmt) const { - fmt.open_array_section("linger_ops"); + fmt->open_array_section("linger_ops"); for (map::const_iterator p = linger_ops.begin(); p != linger_ops.end(); ++p) { LingerOp *op = p->second; - fmt.open_object_section("linger_op"); - fmt.dump_unsigned("linger_id", op->linger_id); - fmt.dump_stream("pg") << op->pgid; - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.dump_stream("object_id") << op->oid; - fmt.dump_stream("object_locator") << op->oloc; - fmt.dump_stream("snapid") << op->snap; - fmt.dump_stream("registering") << op->snap; - fmt.dump_stream("registered") << op->snap; - fmt.close_section(); // linger_op object + fmt->open_object_section("linger_op"); + fmt->dump_unsigned("linger_id", op->linger_id); + fmt->dump_stream("pg") << op->pgid; + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->dump_stream("object_id") << op->oid; + fmt->dump_stream("object_locator") << op->oloc; + fmt->dump_stream("snapid") << op->snap; + fmt->dump_stream("registering") << op->snap; + fmt->dump_stream("registered") << op->snap; + fmt->close_section(); // linger_op object } - fmt.close_section(); // linger_ops array + fmt->close_section(); // linger_ops array } -void Objecter::dump_command_ops(Formatter& fmt) const +void Objecter::dump_command_ops(Formatter *fmt) const { - fmt.open_array_section("command_ops"); + fmt->open_array_section("command_ops"); for (map::const_iterator p = command_ops.begin(); p != command_ops.end(); ++p) { CommandOp *op = p->second; - fmt.open_object_section("command_op"); - fmt.dump_unsigned("command_id", op->tid); - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.open_array_section("command"); + fmt->open_object_section("command_op"); + fmt->dump_unsigned("command_id", op->tid); + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->open_array_section("command"); for (vector::const_iterator q = op->cmd.begin(); q != op->cmd.end(); ++q) - fmt.dump_string("word", *q); - fmt.close_section(); + fmt->dump_string("word", *q); + fmt->close_section(); if (op->target_osd >= 0) - fmt.dump_int("target_osd", op->target_osd); + fmt->dump_int("target_osd", op->target_osd); else - fmt.dump_stream("target_pg") << op->target_pg; - fmt.close_section(); // command_op object + fmt->dump_stream("target_pg") << op->target_pg; + fmt->close_section(); // command_op object } - fmt.close_section(); // command_ops array + fmt->close_section(); // command_ops array } -void Objecter::dump_pool_ops(Formatter& fmt) const +void Objecter::dump_pool_ops(Formatter *fmt) const { - fmt.open_array_section("pool_ops"); + fmt->open_array_section("pool_ops"); for (map::const_iterator p = pool_ops.begin(); p != pool_ops.end(); ++p) { PoolOp *op = p->second; - fmt.open_object_section("pool_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_int("pool", op->pool); - fmt.dump_string("name", op->name); - fmt.dump_int("operation_type", op->pool_op); - fmt.dump_unsigned("auid", op->auid); - fmt.dump_unsigned("crush_rule", op->crush_rule); - fmt.dump_stream("snapid") << op->snapid; - fmt.dump_stream("last_sent") << op->last_submit; - fmt.close_section(); // pool_op object + fmt->open_object_section("pool_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_int("pool", op->pool); + fmt->dump_string("name", op->name); + fmt->dump_int("operation_type", op->pool_op); + fmt->dump_unsigned("auid", op->auid); + fmt->dump_unsigned("crush_rule", op->crush_rule); + fmt->dump_stream("snapid") << op->snapid; + fmt->dump_stream("last_sent") << op->last_submit; + fmt->close_section(); // pool_op object } - fmt.close_section(); // pool_ops array + fmt->close_section(); // pool_ops array } -void Objecter::dump_pool_stat_ops(Formatter& fmt) const +void Objecter::dump_pool_stat_ops(Formatter *fmt) const { - fmt.open_array_section("pool_stat_ops"); + fmt->open_array_section("pool_stat_ops"); for (map::const_iterator p = poolstat_ops.begin(); p != poolstat_ops.end(); ++p) { PoolStatOp *op = p->second; - fmt.open_object_section("pool_stat_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("last_sent") << op->last_submit; + fmt->open_object_section("pool_stat_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("last_sent") << op->last_submit; - fmt.open_array_section("pools"); + fmt->open_array_section("pools"); for (list::const_iterator it = op->pools.begin(); it != op->pools.end(); ++it) { - fmt.dump_string("pool", *it); + fmt->dump_string("pool", *it); } - fmt.close_section(); // pool_op object + fmt->close_section(); // pool_op object - fmt.close_section(); // pool_stat_op object + fmt->close_section(); // pool_stat_op object } - fmt.close_section(); // pool_stat_ops array + fmt->close_section(); // pool_stat_ops array } -void Objecter::dump_statfs_ops(Formatter& fmt) const +void Objecter::dump_statfs_ops(Formatter *fmt) const { - fmt.open_array_section("statfs_ops"); + fmt->open_array_section("statfs_ops"); for (map::const_iterator p = statfs_ops.begin(); p != statfs_ops.end(); ++p) { StatfsOp *op = p->second; - fmt.open_object_section("statfs_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("last_sent") << op->last_submit; - fmt.close_section(); // pool_stat_op object + fmt->open_object_section("statfs_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("last_sent") << op->last_submit; + fmt->close_section(); // pool_stat_op object } - fmt.close_section(); // pool_stat_ops array + fmt->close_section(); // pool_stat_ops array } Objecter::RequestStateHook::RequestStateHook(Objecter *objecter) : @@ -2335,14 +2335,16 @@ Objecter::RequestStateHook::RequestStateHook(Objecter *objecter) : { } -bool Objecter::RequestStateHook::call(std::string command, std::string args, bufferlist& out) +bool Objecter::RequestStateHook::call(std::string command, std::string args, + std::string format, bufferlist& out) { stringstream ss; - JSONFormatter formatter(true); + Formatter *f = new_formatter(format); m_objecter->client_lock.Lock(); - m_objecter->dump_requests(formatter); + m_objecter->dump_requests(f); m_objecter->client_lock.Unlock(); - formatter.flush(ss); + f->flush(ss); + delete f; out.append(ss); return true; } diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h index c1cac88b60e4f..aa4a20d8b0bec 100644 --- a/src/osdc/Objecter.h +++ b/src/osdc/Objecter.h @@ -726,7 +726,8 @@ class Objecter { Objecter *m_objecter; public: RequestStateHook(Objecter *objecter); - bool call(std::string command, std::string args, bufferlist& out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out); }; RequestStateHook *m_request_state_hook; @@ -1236,13 +1237,13 @@ class Objecter { * Output in-flight requests */ void dump_active(); - void dump_requests(Formatter& fmt) const; - void dump_ops(Formatter& fmt) const; - void dump_linger_ops(Formatter& fmt) const; - void dump_command_ops(Formatter& fmt) const; - void dump_pool_ops(Formatter& fmt) const; - void dump_pool_stat_ops(Formatter& fmt) const; - void dump_statfs_ops(Formatter& fmt) const; + void dump_requests(Formatter *fmt) const; + void dump_ops(Formatter *fmt) const; + void dump_linger_ops(Formatter *fmt) const; + void dump_command_ops(Formatter *fmt) const; + void dump_pool_ops(Formatter *fmt) const; + void dump_pool_stat_ops(Formatter *fmt) const; + void dump_statfs_ops(Formatter *fmt) const; int get_client_incarnation() const { return client_inc; } void set_client_incarnation(int inc) { client_inc = inc; } diff --git a/src/test/bench/small_io_bench_fs.cc b/src/test/bench/small_io_bench_fs.cc index 61fbacc5570ca..138757f7304c1 100644 --- a/src/test/bench/small_io_bench_fs.cc +++ b/src/test/bench/small_io_bench_fs.cc @@ -32,7 +32,9 @@ struct MorePrinting : public DetailedStatCollector::AdditionalPrinting { MorePrinting(CephContext *cct) : cct(cct) {} void operator()(std::ostream *out) { bufferlist bl; - cct->get_perfcounters_collection()->write_json_to_buf(bl, 0); + Formatter *f = new_formatter("json-pretty"); + cct->get_perfcounters_collection()->dump_formatted(f, bl, 0); + delete f; bl.append('\0'); *out << bl.c_str() << std::endl; } From c7c4c23e5faea2182b8f9d18d12a31608e428f28 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 23 Jul 2013 17:23:50 -0700 Subject: [PATCH 02/13] Formatter, admin_socket: make default formatter be json-pretty If not given, default to json-pretty; if given but not equal to one of the formatter choices, return NULL as before. Remove defaulting code in admin_socket.cc in favor of this. Signed-off-by: Dan Mick --- src/common/Formatter.cc | 14 +++++++++----- src/common/Formatter.h | 2 +- src/common/admin_socket.cc | 11 +---------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index 357b287fe3222..7362684c070eb 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -62,15 +62,19 @@ Formatter::~Formatter() } Formatter * -new_formatter(const std::string &type) +new_formatter(const std::string type) { - if (type == "json") + std::string mytype = type; + if (mytype == "") + mytype = "json-pretty"; + + if (mytype == "json") return new JSONFormatter(false); - else if (type == "json-pretty") + else if (mytype == "json-pretty") return new JSONFormatter(true); - else if (type == "xml") + else if (mytype == "xml") return new XMLFormatter(false); - else if (type == "xml-pretty") + else if (mytype == "xml-pretty") return new XMLFormatter(true); else return (Formatter *)NULL; diff --git a/src/common/Formatter.h b/src/common/Formatter.h index 8775c0cf9df5c..c62a8303ce113 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -60,7 +60,7 @@ class Formatter { } }; -Formatter *new_formatter(const std::string &type); +Formatter *new_formatter(const std::string type); class JSONFormatter : public Formatter { public: diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index 5ebb3e040db8d..4afd685b72acd 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -322,13 +322,6 @@ bool AdminSocket::do_accept() cmd_getval(m_cct, cmdmap, "format", format); cmd_getval(m_cct, cmdmap, "prefix", c); - // we don't do plain here - if (format != "json" && - format != "json-pretty" && - format != "xml" && - format != "xml-pretty") - format = "json"; - string firstword; if (c.find(" ") == string::npos) firstword = c; @@ -450,9 +443,7 @@ class HelpHook : public AdminSocketHook { public: HelpHook(AdminSocket *as) : m_as(as) {} bool call(string command, string args, string format, bufferlist& out) { - // override format here because help should always be pretty and - // predictable - Formatter *f = new_formatter("json-pretty"); + Formatter *f = new_formatter(format); f->open_object_section("help"); for (map::iterator p = m_as->m_help.begin(); p != m_as->m_help.end(); From ba6ca5829a6ec20c4cdbc990f1e27a0709ec5a41 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 23 Jul 2013 17:24:52 -0700 Subject: [PATCH 03/13] In general, flush in caller of dump worker rather than worker This allows easier refactoring of workers (no dual flushes when code changes). Signed-off-by: Dan Mick --- src/common/ceph_context.cc | 6 +++--- src/common/perf_counters.cc | 4 +--- src/common/perf_counters.h | 2 +- src/osd/OSD.cc | 9 ++++----- src/osd/OpRequest.cc | 6 ++---- src/osd/OpRequest.h | 4 ++-- src/test/bench/small_io_bench_fs.cc | 3 ++- 7 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index e0fcf530222f4..6b227d8689e31 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -170,11 +170,11 @@ void CephContext::do_command(std::string command, std::string args, lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "'" << dendl; if (command == "perfcounters_dump" || command == "1" || command == "perf dump") { - _perf_counters_collection->dump_formatted(f, *out, false); + _perf_counters_collection->dump_formatted(f, false); } else if (command == "perfcounters_schema" || command == "2" || command == "perf schema") { - _perf_counters_collection->dump_formatted(f, *out, true); + _perf_counters_collection->dump_formatted(f, true); } else { f->open_object_section(command.c_str()); @@ -221,8 +221,8 @@ void CephContext::do_command(std::string command, std::string args, assert(0 == "registered under wrong command?"); } f->close_section(); - f->flush(*out); } + f->flush(*out); delete f; lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "' result is " << out->length() << " bytes" << dendl; }; diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index 46f55fae51bfc..86fb531f737c8 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -73,8 +73,7 @@ void PerfCountersCollection::clear() } } -void PerfCountersCollection::dump_formatted(Formatter *f, bufferlist &bl, - bool schema) +void PerfCountersCollection::dump_formatted(Formatter *f, bool schema) { Mutex::Locker lck(m_lock); f->open_object_section("perfcounter_collection"); @@ -88,7 +87,6 @@ void PerfCountersCollection::dump_formatted(Formatter *f, bufferlist &bl, } } f->close_section(); - f->flush(bl); } // --------------------------- diff --git a/src/common/perf_counters.h b/src/common/perf_counters.h index 125d84c04e34a..ec10f9a928286 100644 --- a/src/common/perf_counters.h +++ b/src/common/perf_counters.h @@ -136,7 +136,7 @@ class PerfCountersCollection void add(class PerfCounters *l); void remove(class PerfCounters *l); void clear(); - void dump_formatted(ceph::Formatter *f, bufferlist &bl, bool schema); + void dump_formatted(ceph::Formatter *f, bool schema); private: CephContext *m_cct; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index dd9a5b7293f33..c767a2c0fe5f6 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -1012,14 +1012,13 @@ bool OSD::asok_command(string command, string args, string format, ostream& ss) format = "json-pretty"; Formatter *f = new_formatter(format); if (command == "dump_ops_in_flight") { - op_tracker.dump_ops_in_flight(f, ss); + op_tracker.dump_ops_in_flight(f); } else if (command == "dump_historic_ops") { - op_tracker.dump_historic_ops(f, ss); + op_tracker.dump_historic_ops(f); } else if (command == "dump_op_pq_state") { f->open_object_section("pq"); op_wq.dump(f); f->close_section(); - f->flush(ss); } else if (command == "dump_blacklist") { list > bl; OSDMapRef curmap = service.get_osdmap(); @@ -1036,7 +1035,6 @@ bool OSD::asok_command(string command, string args, string format, ostream& ss) f->close_section(); //entry } f->close_section(); //blacklist - f->flush(ss); } else if (command == "dump_watchers") { list watchers; osd_lock.Lock(); @@ -1078,10 +1076,11 @@ bool OSD::asok_command(string command, string args, string format, ostream& ss) } f->close_section(); //watches - f->flush(ss); } else { assert(0 == "broken asok registration"); } + f->flush(ss); + delete f; return true; } diff --git a/src/osd/OpRequest.cc b/src/osd/OpRequest.cc index c0d167a5f0ad1..a6cdc9ecffb5e 100644 --- a/src/osd/OpRequest.cc +++ b/src/osd/OpRequest.cc @@ -76,15 +76,14 @@ void OpHistory::dump_ops(utime_t now, Formatter *f) f->close_section(); } -void OpTracker::dump_historic_ops(Formatter *f, ostream &ss) +void OpTracker::dump_historic_ops(Formatter *f) { Mutex::Locker locker(ops_in_flight_lock); utime_t now = ceph_clock_now(g_ceph_context); history.dump_ops(now, f); - f->flush(ss); } -void OpTracker::dump_ops_in_flight(Formatter *f, ostream &ss) +void OpTracker::dump_ops_in_flight(Formatter *f) { Mutex::Locker locker(ops_in_flight_lock); f->open_object_section("ops_in_flight"); // overall dump @@ -98,7 +97,6 @@ void OpTracker::dump_ops_in_flight(Formatter *f, ostream &ss) } f->close_section(); // list of OpRequests f->close_section(); // overall dump - f->flush(ss); } void OpTracker::register_inflight_op(xlist::item *i) diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h index 67ee26b02ec14..a2014472432da 100644 --- a/src/osd/OpRequest.h +++ b/src/osd/OpRequest.h @@ -59,8 +59,8 @@ class OpTracker { public: OpTracker() : seq(0), ops_in_flight_lock("OpTracker mutex") {} - void dump_ops_in_flight(Formatter *f, std::ostream& ss); - void dump_historic_ops(Formatter *f, std::ostream& ss); + void dump_ops_in_flight(Formatter *f); + void dump_historic_ops(Formatter *f); void register_inflight_op(xlist::item *i); void unregister_inflight_op(OpRequest *i); diff --git a/src/test/bench/small_io_bench_fs.cc b/src/test/bench/small_io_bench_fs.cc index 138757f7304c1..a37a7e71153a6 100644 --- a/src/test/bench/small_io_bench_fs.cc +++ b/src/test/bench/small_io_bench_fs.cc @@ -33,7 +33,8 @@ struct MorePrinting : public DetailedStatCollector::AdditionalPrinting { void operator()(std::ostream *out) { bufferlist bl; Formatter *f = new_formatter("json-pretty"); - cct->get_perfcounters_collection()->dump_formatted(f, bl, 0); + cct->get_perfcounters_collection()->dump_formatted(f, 0); + f->flush(bl); delete f; bl.append('\0'); *out << bl.c_str() << std::endl; From 67eb7de42f40eb080438a2f22303265c987a4b27 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 14:39:51 -0700 Subject: [PATCH 04/13] ceph_argparse.py, ceph.in: validate_command: stop handling format We were passing the entire parsed_args Namespace just to look at and tack on 'format', and that's kinda silly; do it in the callers instead. Signed-off-by: Dan Mick --- src/ceph.in | 9 ++++++--- src/pybind/ceph_argparse.py | 6 +----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/ceph.in b/src/ceph.in index 0d361e1c76ce6..90795058127ca 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -344,10 +344,11 @@ def new_style_command(parsed_args, cmdargs, target, sigdict, inbuf, verbose): if not got_command: if cmdargs: # Validate input args against list of sigs - valid_dict = validate_command(parsed_args, sigdict, cmdargs, - verbose) + valid_dict = validate_command(sigdict, cmdargs, verbose) if valid_dict: got_command = True + if parsed_args.output_format: + valid_dict['format'] = parsed_args.output_format else: return -errno.EINVAL, '', 'invalid command' else: @@ -360,8 +361,10 @@ def new_style_command(parsed_args, cmdargs, target, sigdict, inbuf, verbose): return 0, '', '' cmdargs = parse_cmdargs(interactive_input.split())[2] target = find_cmd_target(cmdargs) - valid_dict = validate_command(parsed_args, sigdict, cmdargs) + valid_dict = validate_command(sigdict, cmdargs, verbose) if valid_dict: + if parsed_args.output_format: + valid_dict['format'] = parsed_args.output_format if verbose: print >> sys.stderr, "Submitting command ", valid_dict ret, outbuf, outs = json_command(cluster_handle, diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index b82cc833ba66d..b014d7d626c48 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -820,11 +820,10 @@ def validate(args, signature, partial=False): raise ArgumentError("unused arguments: " + str(myargs)) return d -def validate_command(parsed_args, sigdict, args, verbose=False): +def validate_command(sigdict, args, verbose=False): """ turn args into a valid dictionary ready to be sent off as JSON, validated against sigdict. - parsed_args is the namespace back from argparse """ found = [] valid_dict = {} @@ -882,9 +881,6 @@ def validate_command(parsed_args, sigdict, args, verbose=False): print >> sys.stderr, concise_sig(cmd['sig']) return None - if parsed_args.output_format: - valid_dict['format'] = parsed_args.output_format - return valid_dict def find_cmd_target(childargs): From cb38762bca942e8e8c1a89bca6d3c6362182385f Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 14:43:37 -0700 Subject: [PATCH 05/13] ceph.in: admin_socket() now validates command and passes format Signed-off-by: Dan Mick --- src/ceph.in | 70 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/src/ceph.in b/src/ceph.in index 90795058127ca..a72761216bd52 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -285,26 +285,57 @@ def format_help(cmddict, partial=None): return fullusage -def admin_socket(asok_path, cmd): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +def admin_socket(asok_path, cmd, format=''): + """ + Send a daemon (--admin-daemon) command 'cmd'. asok_path is the + path to the admin socket; cmd is a list of strings; format may be + set to one of the formatted forms to get output in that form + (daemon commands don't support 'plain' output). + """ + + def do_sockio(path, cmd): + """ helper: do all the actual low-level stream I/O """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + try: + sock.sendall(cmd + '\0') + len_str = sock.recv(4) + if len(len_str) < 4: + raise RuntimeError("no data returned from admin socket") + l, = struct.unpack(">I", len_str) + ret = '' + + got = 0 + while got < l: + bit = sock.recv(l - got) + ret += bit + got += len(bit) + + except Exception as e: + raise RuntimeError('exception: ' + str(e)) + return ret + try: - sock.connect(asok_path) - sock.sendall(' '.join(cmd) + '\0') + cmd_json = do_sockio(asok_path, + json.dumps({"prefix":"get_command_descriptions"})) + except Exception as e: + raise RuntimeError('exception getting command descriptions: ' + str(e)) + + if cmd[0] == 'get_command_descriptions': + return cmd_json - len_str = sock.recv(4) - if len(len_str) < 4: - raise RuntimeError("no data returned from admin socket") - l, = struct.unpack(">I", len_str) - ret = '' + sigdict = parse_json_funcsigs(cmd_json, 'cli') + valid_dict = validate_command(sigdict, cmd) + if not valid_dict: + return -errno.EINVAL - got = 0 - while got < l: - bit = sock.recv(l - got) - ret += bit - got += len(bit) + if format: + valid_dict['format'] = format + try: + ret = do_sockio(asok_path, json.dumps(valid_dict)) except Exception as e: - raise RuntimeError('exception: {0}'.format(e)) + raise RuntimeError('exception: ' + str(e)) return ret @@ -473,9 +504,12 @@ def main(): conffile = parsed_args.cephconf # For now, --admin-daemon is handled as usual. Try it # first in case we can't connect() to the cluster + + format = parsed_args.output_format + if parsed_args.admin_socket: try: - print admin_socket(parsed_args.admin_socket, childargs) + print admin_socket(parsed_args.admin_socket, childargs, format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 @@ -484,7 +518,7 @@ def main(): if len(childargs) > 2: if childargs[1].find('/') >= 0: try: - print admin_socket(childargs[1], childargs[2:]) + print admin_socket(childargs[1], childargs[2:], format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 @@ -492,7 +526,7 @@ def main(): # try resolve daemon name path = ceph_conf('admin_socket', childargs[1]) try: - print admin_socket(path, childargs[2:]) + print admin_socket(path, childargs[2:], format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 From cd16d73fda4fc25fc44d1677b788ad7aae208d30 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 14:44:17 -0700 Subject: [PATCH 06/13] ceph.in: clean up help, fix partial matching on all help Remove --help-all; too much effort for low benefit Signed-off-by: Dan Mick --- src/ceph.in | 57 +++++++++++++++++------------------------------------ 1 file changed, 18 insertions(+), 39 deletions(-) diff --git a/src/ceph.in b/src/ceph.in index a72761216bd52..63c41343f9a52 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -100,8 +100,6 @@ def parse_cmdargs(args=None, target=''): parser.add_argument('-h', '--help', help='request mon help', action='store_true') - parser.add_argument('--help-all', help='request help for all daemons', - action='store_true') parser.add_argument('-c', '--conf', dest='cephconf', help='ceph configuration file') @@ -150,14 +148,16 @@ def parse_cmdargs(args=None, target=''): return parser, parsed_args, extras -def do_help(parser, args, help_all = False): +def do_help(parser, args): """ Print basic parser help - If the cluster is available: - get and print monitor help; - if help_all, print help for daemon commands as well + If the cluster is available, get and print monitor help """ + def help_for_sigs(sigs, partial=None): + sys.stdout.write(format_help(parse_json_funcsigs(sigs, 'cli'), + partial=partial)) + def help_for_target(target, partial=None): ret, outbuf, outs = json_command(cluster_handle, target=target, prefix='get_command_descriptions', @@ -167,40 +167,19 @@ def do_help(parser, args, help_all = False): "couldn't get command descriptions for {0}: {1}".\ format(target, outs) else: - sys.stdout.write(format_help(parse_json_funcsigs(outbuf, 'cli'), - partial)) - - parser.print_help() - print '\n' - if (cluster_handle): - help_for_target(target=('mon', ''), partial=' '.join(args)) + help_for_sigs(outbuf, partial) - if help_all and cluster_handle: - # try/except in case there are no daemons of that type - try: - firstosd = osdids()[0] - print '\nOSD.{0} tell commands and pg pgid commands:\n\n'.\ - format(firstosd) - help_for_target(target=('osd', osdids()[0])) - print '\nOSD daemon commands:\n\n' - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'osd.' + firstosd), ['get_command_descriptions']), 'cli'))) - except: - pass + def hdr(s): + print '\n', s, '\n', '=' * len(s) - try: - firstmon = monids()[0] - print '\nmon.{0} daemon commands:\n\n'.format(firstmon) - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'mon.' + firstmon), ['get_command_descriptions']), 'cli'))) - except: - pass + hdr('Monitor commands:') + partial = ' '.join(args) + parser.print_help() + print '\n' - try: - firstmds = mdsids()[0] - print '\nmds.{0} daemon commands:\n\n'.format(firstmds) - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'mds.' + firstmds), ['get_command_descriptions']), 'cli'))) - except: - pass + if (cluster_handle): + help_for_target(target=('mon', ''), partial=partial) return 0 @@ -321,7 +300,7 @@ def admin_socket(asok_path, cmd, format=''): except Exception as e: raise RuntimeError('exception getting command descriptions: ' + str(e)) - if cmd[0] == 'get_command_descriptions': + if cmd == 'get_command_descriptions': return cmd_json sigdict = parse_json_funcsigs(cmd_json, 'cli') @@ -581,8 +560,8 @@ def main(): format(e.__class__.__name__) return 1 - if parsed_args.help or parsed_args.help_all: - return do_help(parser, childargs, parsed_args.help_all) + if parsed_args.help: + return do_help(parser, childargs) # implement -w/--watch_* # This is ugly, but Namespace() isn't quite rich enough. From c9fcda88a7831ccbf879cf8baf0a91163e437aa3 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Thu, 18 Jul 2013 16:34:23 -0700 Subject: [PATCH 07/13] OSD: provide newer command descs with module/perm/avail Signed-off-by: Dan Mick --- src/osd/OSD.cc | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index c767a2c0fe5f6..a65772ada3d03 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3813,10 +3813,13 @@ void OSD::handle_command(MCommand *m) struct OSDCommand { string cmdstring; string helpstring; + string module; + string perm; + string availability; } osd_commands[] = { -#define COMMAND(parsesig, helptext) \ - {parsesig, helptext}, +#define COMMAND(parsesig, helptext, module, perm, availability) \ + {parsesig, helptext, module, perm, availability}, // yes, these are really pg commands, but there's a limit to how // much work it's worth. The OSD returns all of them. @@ -3824,40 +3827,45 @@ struct OSDCommand { COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=query", \ - "show details of a specific pg") + "show details of a specific pg", "osd", "r", "cli,rest") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=mark_unfound_lost " \ "name=mulcmd,type=CephChoices,strings=revert", \ - "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available") + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", + "osd", "rw", "cli,rest") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=list_missing " \ "name=offset,type=CephString,req=false", - "list missing objects on this pg, perhaps starting at an offset given in JSON") + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "rw", "cli,rest") -COMMAND("version", "report version of OSD") +COMMAND("version", "report version of OSD", "osd", "r", "cli,rest") COMMAND("injectargs " \ "name=injected_args,type=CephString,n=N", - "inject configuration arguments into running OSD") + "inject configuration arguments into running OSD", + "osd", "rw", "cli,rest") COMMAND("bench " \ "name=count,type=CephInt,req=false " \ "name=size,type=CephInt,req=false ", \ "OSD benchmark: write -byte objects, " \ - "(default 1G size 4MB). Results in log.") -COMMAND("flush_pg_stats", "flush pg stats") -COMMAND("debug dump_missing " \ + "(default 1G size 4MB). Results in log.", + "osd", "rw", "cli,rest") +COMMAND("flush_pg_stats", "flush pg stats", "osd", "rw", "cli,rest") +COMMAND("debug_dump_missing " \ "name=filename,type=CephFilepath", - "dump missing objects to a named file") + "dump missing objects to a named file", "osd", "r", "cli,rest") COMMAND("debug kick_recovery_wq " \ "name=delay,type=CephInt,range=0", - "set osd_recovery_delay_start to ") + "set osd_recovery_delay_start to ", "osd", "rw", "cli,rest") COMMAND("cpu_profiler " \ "name=arg,type=CephChoices,strings=status|flush", - "run cpu profiling on daemon") -COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics") -COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics") - + "run cpu profiling on daemon", "osd", "rw", "cli,rest") +COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics", + "osd", "r", "cli,rest") +COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics", + "osd", "rw", "cli,rest") }; void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist& data) @@ -3893,8 +3901,8 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist ostringstream secname; secname << "cmd" << setfill('0') << std::setw(3) << cmdnum; - dump_cmd_and_help_to_json(f, secname.str(), - cp->cmdstring, cp->helpstring); + dump_cmddesc_to_json(f, secname.str(), cp->cmdstring, cp->helpstring, + cp->module, cp->perm, cp->availability); cmdnum++; } f->close_section(); // command_descriptions From 6e6ceffa1b36c4d1d6b6fc7885322c8000f7dd77 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 19 Jul 2013 18:03:24 -0700 Subject: [PATCH 08/13] OSD: "tell " version, bench, dump_pg_recovery_stats: formatted Signed-off-by: Dan Mick --- src/osd/OSD.cc | 40 +++++++++++++++++++++++++++++++++------- src/osd/PG.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index a65772ada3d03..76b11672b8137 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3879,6 +3879,8 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist map cmdmap; string prefix; + string format; + boost::scoped_ptr f; if (cmd.empty()) { ss << "no command given"; @@ -3913,8 +3915,18 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist goto out; } + cmd_getval(g_ceph_context, cmdmap, "format", format); + f.reset(new_formatter(format)); + if (prefix == "version") { - ds << pretty_version_to_str(); + if (f) { + f->open_object_section("version"); + f->dump_string("version", pretty_version_to_str()); + f->close_section(); + f->flush(ds); + } else { + ds << pretty_version_to_str(); + } goto out; } else if (prefix == "injectargs") { @@ -3952,7 +3964,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist ss << "i don't have pgid " << pgid; r = -ENOENT; } else { - r = pg->do_command(cmd, ss, data, odata); + r = pg->do_command(cmdmap, ss, data, odata); pg->unlock(); } } @@ -3991,9 +4003,18 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist store->queue_transaction(NULL, cleanupt); uint64_t rate = (double)count / (end - start); - ss << "bench: wrote " << prettybyte_t(count) - << " in blocks of " << prettybyte_t(bsize) << " in " - << (end-start) << " sec at " << prettybyte_t(rate) << "/sec"; + if (f) { + f->open_object_section("osd_bench_results"); + f->dump_int("bytes_written", count); + f->dump_int("blocksize", bsize); + f->dump_float("bytes_per_sec", rate); + f->close_section(); + f->flush(ss); + } else { + ss << "bench: wrote " << prettybyte_t(count) + << " in blocks of " << prettybyte_t(bsize) << " in " + << (end-start) << " sec at " << prettybyte_t(rate) << "/sec"; + } } else if (prefix == "flush_pg_stats") { @@ -4090,8 +4111,13 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist else if (prefix == "dump_pg_recovery_stats") { stringstream s; - pg_recovery_stats.dump(s); - ds << "dump pg recovery stats: " << s.str(); + if (f) { + pg_recovery_stats.dump_formatted(f.get()); + f->flush(ds); + } else { + pg_recovery_stats.dump(s); + ds << "dump pg recovery stats: " << s.str(); + } } else if (prefix == "reset_pg_recovery_stats") { diff --git a/src/osd/PG.h b/src/osd/PG.h index 819c9c62f62e3..5525c7f46dd26 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -45,6 +45,7 @@ #include "messages/MOSDPGLog.h" #include "common/tracked_int_ptr.hpp" #include "common/WorkQueue.h" +#include "include/str_list.h" #include #include @@ -112,6 +113,33 @@ struct PGRecoveryStats { } } + void dump_formatted(Formatter *f) { + Mutex::Locker l(lock); + f->open_array_section("pg_recovery_stats"); + for (map::iterator p = info.begin(); + p != info.end(); ++p) { + per_state_info& i = p->second; + f->open_object_section("recovery_state"); + f->dump_int("enter", i.enter); + f->dump_int("exit", i.exit); + f->dump_int("events", i.events); + f->dump_stream("event_time") << i.event_time; + f->dump_stream("total_time") << i.total_time; + f->dump_stream("min_time") << i.min_time; + f->dump_stream("max_time") << i.max_time; + vector states; + get_str_vec(p->first, "/", states); + f->open_array_section("nested_states"); + for (vector::iterator st = states.begin(); + st != states.end(); ++st) { + f->dump_string("state", *st); + } + f->close_section(); + f->close_section(); + } + f->close_section(); + } + void log_enter(const char *s) { Mutex::Locker l(lock); info[s].enter++; From bcbb807c018f89d473a252d87e8d48b5220b3a61 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 19 Jul 2013 18:04:29 -0700 Subject: [PATCH 09/13] PG: add formatted output to pg query, list_missing Signed-off-by: Dan Mick --- src/osd/PG.h | 4 +- src/osd/ReplicatedPG.cc | 107 ++++++++++++++++++---------------------- src/osd/ReplicatedPG.h | 4 +- 3 files changed, 53 insertions(+), 62 deletions(-) diff --git a/src/osd/PG.h b/src/osd/PG.h index 5525c7f46dd26..10e9a2544a9b0 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -43,6 +43,7 @@ #include "msg/Messenger.h" #include "messages/MOSDRepScrub.h" #include "messages/MOSDPGLog.h" +#include "common/cmdparse.h" #include "common/tracked_int_ptr.hpp" #include "common/WorkQueue.h" #include "include/str_list.h" @@ -109,7 +110,6 @@ struct PGRecoveryStats { << i.total_time << "\t" << i.min_time << "\t" << i.max_time << "\t" << p->first << "\n"; - } } @@ -1814,7 +1814,7 @@ class PG { virtual void do_push_reply(OpRequestRef op) = 0; virtual void snap_trimmer() = 0; - virtual int do_command(vector& cmd, ostream& ss, + virtual int do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, bufferlist& odata) = 0; virtual bool same_for_read_since(epoch_t e) = 0; diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 298d38d6ace64..4a59a23cdb794 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -268,23 +268,12 @@ int ReplicatedPG::get_pgls_filter(bufferlist::iterator& iter, PGLSFilter **pfilt // ========================================================== -int ReplicatedPG::do_command(vector& cmd, ostream& ss, +int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, bufferlist& odata) { const pg_missing_t &missing = pg_log.get_missing(); - map cmdmap; string prefix; - - if (cmd.empty()) { - ss << "no command given"; - return -EINVAL; - } - - stringstream ss2; - if (!cmdmap_from_json(cmd, &cmdmap, ss2)) { - ss << ss2.str(); - return -EINVAL; - } + string format; cmd_getval(g_ceph_context, cmdmap, "prefix", prefix); if (prefix != "pg") { @@ -292,33 +281,36 @@ int ReplicatedPG::do_command(vector& cmd, ostream& ss, return -EINVAL; } + cmd_getval(g_ceph_context, cmdmap, "format", format); + boost::scoped_ptr f(new_formatter(format)); + // demand that we have a formatter + if (!f) + f.reset(new_formatter("json")); + string command; cmd_getval(g_ceph_context, cmdmap, "cmd", command); if (command == "query") { - JSONFormatter jsf(true); - jsf.open_object_section("pg"); - jsf.dump_string("state", pg_state_string(get_state())); - jsf.dump_unsigned("epoch", get_osdmap()->get_epoch()); - jsf.open_array_section("up"); + f->open_object_section("pg"); + f->dump_string("state", pg_state_string(get_state())); + f->dump_unsigned("epoch", get_osdmap()->get_epoch()); + f->open_array_section("up"); for (vector::iterator p = up.begin(); p != up.end(); ++p) - jsf.dump_unsigned("osd", *p); - jsf.close_section(); - jsf.open_array_section("acting"); + f->dump_unsigned("osd", *p); + f->close_section(); + f->open_array_section("acting"); for (vector::iterator p = acting.begin(); p != acting.end(); ++p) - jsf.dump_unsigned("osd", *p); - jsf.close_section(); - jsf.open_object_section("info"); - info.dump(&jsf); - jsf.close_section(); - - jsf.open_array_section("recovery_state"); - handle_query_state(&jsf); - jsf.close_section(); - - jsf.close_section(); - stringstream dss; - jsf.flush(dss); - odata.append(dss); + f->dump_unsigned("osd", *p); + f->close_section(); + f->open_object_section("info"); + info.dump(f.get()); + f->close_section(); + + f->open_array_section("recovery_state"); + handle_query_state(f.get()); + f->close_section(); + + f->close_section(); + f->flush(odata); return 0; } else if (command == "mark_unfound_lost") { @@ -352,7 +344,6 @@ int ReplicatedPG::do_command(vector& cmd, ostream& ss, return 0; } else if (command == "list_missing") { - JSONFormatter jf(true); hobject_t offset; string offset_json; if (cmd_getval(g_ceph_context, cmdmap, "offset", offset_json)) { @@ -366,50 +357,48 @@ int ReplicatedPG::do_command(vector& cmd, ostream& ss, return -EINVAL; } } - jf.open_object_section("missing"); + f->open_object_section("missing"); { - jf.open_object_section("offset"); - offset.dump(&jf); - jf.close_section(); + f->open_object_section("offset"); + offset.dump(f.get()); + f->close_section(); } - jf.dump_int("num_missing", missing.num_missing()); - jf.dump_int("num_unfound", get_num_unfound()); + f->dump_int("num_missing", missing.num_missing()); + f->dump_int("num_unfound", get_num_unfound()); map::const_iterator p = missing.missing.upper_bound(offset); { - jf.open_array_section("objects"); + f->open_array_section("objects"); int32_t num = 0; bufferlist bl; while (p != missing.missing.end() && num < g_conf->osd_command_max_records) { - jf.open_object_section("object"); + f->open_object_section("object"); { - jf.open_object_section("oid"); - p->first.dump(&jf); - jf.close_section(); + f->open_object_section("oid"); + p->first.dump(f.get()); + f->close_section(); } - p->second.dump(&jf); // have, need keys + p->second.dump(f.get()); // have, need keys { - jf.open_array_section("locations"); + f->open_array_section("locations"); map >::iterator q = missing_loc.find(p->first); if (q != missing_loc.end()) for (set::iterator r = q->second.begin(); r != q->second.end(); ++r) - jf.dump_int("osd", *r); - jf.close_section(); + f->dump_int("osd", *r); + f->close_section(); } - jf.close_section(); + f->close_section(); ++p; num++; } - jf.close_section(); + f->close_section(); } - jf.dump_int("more", p != missing.missing.end()); - jf.close_section(); - stringstream jss; - jf.flush(jss); - odata.append(jss); + f->dump_int("more", p != missing.missing.end()); + f->close_section(); + f->flush(odata); return 0; }; - ss << "unknown command " << cmd; + ss << "unknown pg command " << prefix; return -EINVAL; } diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 9dafe23faa1fb..7b70b4381ea4f 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -17,6 +17,7 @@ #include #include "include/assert.h" +#include "common/cmdparse.h" #include "PG.h" #include "OSD.h" @@ -930,7 +931,8 @@ class ReplicatedPG : public PG { const hobject_t& ioid); ~ReplicatedPG() {} - int do_command(vector& cmd, ostream& ss, bufferlist& idata, bufferlist& odata); + int do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, + bufferlist& odata); void do_op(OpRequestRef op); bool pg_op_must_wait(MOSDOp *op); From 7b42deef3810e207f0e044769e7c70175993954a Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 23 Jul 2013 00:50:15 -0700 Subject: [PATCH 10/13] ceph_rest_api.py: obtain and handle tell commands Contact an OSD that's up to get a list of the commands, and use them to add to the URL map. Special treatment throughout for these commands: * hack the help signature dump * keep a 'flavor' per command to allow special handler() processing * strip off 'tell/' when constructing command * allow multiple dicts with the same url (the parameters and get/put methods can change) * because of above, method must be validated in handler() * validate the given OSD * calculate target for command (mon, osd, pg) Unrelated: make method_dict into global METHOD_DICT Signed-off-by: Dan Mick --- src/osd/OSD.cc | 23 ++- src/pybind/ceph_argparse.py | 3 + src/pybind/ceph_rest_api.py | 296 +++++++++++++++++++++++++----------- 3 files changed, 229 insertions(+), 93 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 76b11672b8137..d611afdd08fb8 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3822,24 +3822,41 @@ struct OSDCommand { {parsesig, helptext, module, perm, availability}, // yes, these are really pg commands, but there's a limit to how -// much work it's worth. The OSD returns all of them. +// much work it's worth. The OSD returns all of them. Make this +// form (pg ) valid only for the cli. +// Rest uses "tell " COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=query", \ - "show details of a specific pg", "osd", "r", "cli,rest") + "show details of a specific pg", "osd", "r", "cli") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=mark_unfound_lost " \ "name=mulcmd,type=CephChoices,strings=revert", \ "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", - "osd", "rw", "cli,rest") + "osd", "rw", "cli") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=list_missing " \ "name=offset,type=CephString,req=false", "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "r", "cli") + +// new form: tell for both cli and rest + +COMMAND("query", + "show details of a specific pg", "osd", "r", "cli,rest") +COMMAND("mark_unfound_lost " \ + "name=mulcmd,type=CephChoices,strings=revert", \ + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", "osd", "rw", "cli,rest") +COMMAND("list_missing " \ + "name=offset,type=CephString,req=false", + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "r", "cli,rest") + +// tell commands. Validation of osd.n must be special-cased in client COMMAND("version", "report version of OSD", "osd", "r", "cli,rest") COMMAND("injectargs " \ diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index b014d7d626c48..855e21c25084e 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -263,6 +263,8 @@ def valid(self, s, partial=False): if p is not None and long(p) > 65535: raise ArgumentValid("{0} not a valid port number".format(p)) self.val = s + self.addr = a + self.port = p def __str__(self): return '' @@ -274,6 +276,7 @@ class CephEntityAddr(CephIPAddr): def valid(self, s, partial=False): ip, nonce = s.split('/') super(self.__class__, self).valid(ip) + self.nonce = nonce self.val = s def __str__(self): diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index 28a0419c33c21..f8d9b92129e2d 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -1,13 +1,15 @@ #!/usr/bin/python # vim: ts=4 sw=4 smarttab expandtab -import os import collections import ConfigParser +import errno import json import logging import logging.handlers +import os import rados +import socket import textwrap import xml.etree.ElementTree import xml.sax.saxutils @@ -99,6 +101,26 @@ def get_conf(cfg, clientname, key): pass return None +METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} + +def find_up_osd(): + ''' + Find an up OSD. Return the last one that's up. + Returns id as an int. + ''' + ret, outbuf, outs = json_command(glob.cluster, prefix="osd dump", + argdict=dict(format='json')) + if ret: + raise EnvironmentError(ret, 'Can\'t get osd dump output') + try: + osddump = json.loads(outbuf) + except: + raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump') + osds = [osd['osd'] for osd in osddump['osds'] if osd['up']] + if not osds: + raise EnvironmentError(errno.ENOENT, 'No up OSDs found') + return int(osds[-1]) + # XXX this is done globally, and cluster connection kept open; there # are facilities to pass around global info to requests and to # tear down connections between requests if it becomes important @@ -109,6 +131,22 @@ def api_setup(): signatures, module, perms, and help; stuff them away in the glob.urls dict. """ + def get_command_descriptions(target=('mon','')): + ret, outbuf, outs = json_command(glob.cluster, target, + prefix='get_command_descriptions', + timeout=30) + if ret: + err = "Can't get command descriptions: {0}".format(outs) + app.logger.error(err) + raise EnvironmentError(ret, err) + + try: + sigdict = parse_json_funcsigs(outbuf, 'rest') + except Exception as e: + err = "Can't parse command descriptions: {}".format(e) + app.logger.error(err) + raise EnvironmentError(err) + return sigdict conffile = os.environ.get('CEPH_CONF', '') clustername = os.environ.get('CEPH_CLUSTER_NAME', 'ceph') @@ -148,19 +186,22 @@ def api_setup(): h.setFormatter(logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s')) - ret, outbuf, outs = json_command(glob.cluster, - prefix='get_command_descriptions') - if ret: - err = "Can't contact cluster for command descriptions: {0}".format(outs) - app.logger.error(err) - raise EnvironmentError(ret, err) + glob.sigdict = get_command_descriptions() - try: - glob.sigdict = parse_json_funcsigs(outbuf, 'rest') - except Exception as e: - err = "Can't parse command descriptions: {}".format(e) - app.logger.error(err) - raise EnvironmentError(err) + osdid = find_up_osd() + if osdid: + osd_sigdict = get_command_descriptions(target=('osd', int(osdid))) + + # shift osd_sigdict keys up to fit at the end of the mon's glob.sigdict + maxkey = sorted(glob.sigdict.keys())[-1] + maxkey = int(maxkey.replace('cmd', '')) + osdkey = maxkey + 1 + for k, v in osd_sigdict.iteritems(): + newv = v + newv['flavor'] = 'tell' + globk = 'cmd' + str(osdkey) + glob.sigdict[globk] = newv + osdkey += 1 # glob.sigdict maps "cmdNNN" to a dict containing: # 'sig', an array of argdescs @@ -173,27 +214,37 @@ def api_setup(): glob.urls = {} for cmdnum, cmddict in glob.sigdict.iteritems(): cmdsig = cmddict['sig'] - url, params = generate_url_and_params(cmdsig) - if url in glob.urls: - continue + flavor = cmddict.get('flavor', 'mon') + url, params = generate_url_and_params(cmdsig, flavor) + perm = cmddict['perm'] + for k in METHOD_DICT.iterkeys(): + if k in perm: + methods = METHOD_DICT[k] + urldict = {'paramsig':params, + 'help':cmddict['help'], + 'module':cmddict['module'], + 'perm':perm, + 'flavor':flavor, + 'methods':methods, + } + + # glob.urls contains a list of urldicts (usually only one long) + if url not in glob.urls: + glob.urls[url] = [urldict] else: - perm = cmddict['perm'] - urldict = {'paramsig':params, - 'help':cmddict['help'], - 'module':cmddict['module'], - 'perm':perm, - } - method_dict = {'r':['GET'], - 'w':['PUT', 'DELETE']} - for k in method_dict.iterkeys(): - if k in perm: - methods = method_dict[k] - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict - - url += '.' - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict + # If more than one, need to make union of methods of all. + # Method must be checked in handler + methodset = set(methods) + for old_urldict in glob.urls[url]: + methodset |= set(old_urldict['methods']) + methods = list(methodset) + glob.urls[url].append(urldict) + + # add, or re-add, rule with all methods and urldicts + app.add_url_rule(url, url, handler, methods=methods) + url += '.' + app.add_url_rule(url, url, handler, methods=methods) + app.logger.debug("urls added: %d", len(glob.urls)) app.add_url_rule('/', '/', @@ -201,63 +252,84 @@ def api_setup(): return addr, port -def generate_url_and_params(sig): +def generate_url_and_params(sig, flavor): """ Digest command signature from cluster; generate an absolute (including glob.baseurl) endpoint from all the prefix words, - and a dictionary of non-prefix parameters + and a list of non-prefix param descs """ url = '' params = [] + # the OSD command descriptors don't include the 'tell ', so + # tack it onto the front of sig + if flavor == 'tell': + tellsig = parse_funcsig(['tell', + {'name':'target', 'type':'CephOsdName'}]) + sig = tellsig + sig + for desc in sig: + # prefixes go in the URL path if desc.t == CephPrefix: url += '/' + desc.instance.prefix + # CephChoices with 1 required string (not --) do too, unless + # we've already started collecting params, in which case they + # too are params elif desc.t == CephChoices and \ len(desc.instance.strings) == 1 and \ desc.req and \ - not str(desc.instance).startswith('--'): + not str(desc.instance).startswith('--') and \ + not params: url += '/' + str(desc.instance) else: - params.append(desc) - return glob.baseurl + url, params + # tell/ is a weird case; the URL includes what + # would everywhere else be a parameter + if flavor == 'tell' and \ + (desc.t, desc.name) == (CephOsdName, 'target'): + url += '/' + else: + params.append(desc) + return glob.baseurl + url, params -def concise_sig_for_uri(sig): +def concise_sig_for_uri(sig, flavor): """ Return a generic description of how one would send a REST request for sig """ prefix = [] args = [] + ret = '' + if flavor == 'tell': + ret = 'tell//' for d in sig: if d.t == CephPrefix: prefix.append(d.instance.prefix) else: args.append(d.name + '=' + str(d)) - sig = '/'.join(prefix) + ret += '/'.join(prefix) if args: - sig += '?' + '&'.join(args) - return sig + ret += '?' + '&'.join(args) + return ret def show_human_help(prefix): """ Dump table showing commands matching prefix """ - # XXX this really needs to be a template - #s = '' - #s += '' - #s += '' - # XXX the above mucking with css doesn't cause sensible columns. + # XXX There ought to be a better discovery mechanism than an HTML table s = '
Possible commands:
' - possible = [] permmap = {'r':'GET', 'rw':'PUT'} line = '' for cmdsig in sorted(glob.sigdict.itervalues(), cmp=descsort): concise = concise_sig(cmdsig['sig']) + flavor = cmdsig.get('flavor', 'mon') + if flavor == 'tell': + concise = 'tell//' + concise if concise.startswith(prefix): line = ['
Possible commands:MethodDescription
'] - wrapped_sig = textwrap.wrap(concise_sig_for_uri(cmdsig['sig']), 40) + wrapped_sig = textwrap.wrap( + concise_sig_for_uri(cmdsig['sig'], flavor), 40 + ) for sigline in wrapped_sig: line.append(flask.escape(sigline) + '\n') line.append('') @@ -328,19 +400,23 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response(response, errorcode) -def handler(catchall_path=None, fmt=None): +def handler(catchall_path=None, fmt=None, target=None): """ Main endpoint handler; generic for every endpoint """ - if (catchall_path): - ep = catchall_path.replace('.', '') - else: - ep = flask.request.endpoint.replace('.', '') + ep = catchall_path or flask.request.endpoint + ep = ep.replace('.', '') if ep[0] != '/': ep = '/' + ep + # demand that endpoint begin with glob.baseurl + if not ep.startswith(glob.baseurl): + return make_response(fmt, '', 'Page not found', 404) + + rel_ep = ep[len(glob.baseurl)+1:] + # Extensions override Accept: headers override defaults if not fmt: if 'application/json' in flask.request.accept_mimetypes.values(): @@ -348,12 +424,36 @@ def handler(catchall_path=None, fmt=None): elif 'application/xml' in flask.request.accept_mimetypes.values(): fmt = 'xml' - # demand that endpoint begin with glob.baseurl - if not ep.startswith(glob.baseurl): - return make_response(fmt, '', 'Page not found', 404) + valid = True + prefix = '' + pgid = None + cmdtarget = 'mon', '' + + if target: + # got tell/; validate osdid or pgid + name = CephOsdName() + pgidobj = CephPgid() + try: + name.valid(target) + except ArgumentError: + # try pgid + try: + pgidobj.valid(target) + except ArgumentError: + return flask.make_response("invalid osdid or pgid", 400) + else: + # it's a pgid + pgid = pgidobj.val + cmdtarget = 'pg', pgid + else: + # it's an osd + cmdtarget = name.nametype, name.nameid - relative_endpoint = ep[len(glob.baseurl)+1:] - prefix = ' '.join(relative_endpoint.split('/')).strip() + # prefix does not include tell// + prefix = ' '.join(rel_ep.split('/')[2:]).strip() + else: + # non-target command: prefix is entire path + prefix = ' '.join(rel_ep.split('/')).strip() # show "match as much as you gave me" help for unknown endpoints if not ep in glob.urls: @@ -365,43 +465,59 @@ def handler(catchall_path=None, fmt=None): else: return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) - urldict = glob.urls[ep] - paramsig = urldict['paramsig'] - - # allow '?help' for any specifically-known endpoint - if 'help' in flask.request.args: - response = flask.make_response('{0}: {1}'.\ - format(prefix + concise_sig(paramsig), urldict['help'])) - response.headers['Content-Type'] = 'text/plain' - return response - - # if there are parameters for this endpoint, process them - if paramsig: - args = {} - for k, l in flask.request.args.iterlists(): - if len(l) == 1: - args[k] = l[0] - else: - args[k] = l - - # is this a valid set of params? - try: - argdict = validate(args, paramsig) - except Exception as e: - return make_response(fmt, '', str(e) + '\n', 400) - else: - # no parameters for this endpoint; complain if args are supplied - if flask.request.args: - return make_response(fmt, '', ep + 'takes no params', 400) - argdict = {} + found = None + exc = '' + for urldict in glob.urls[ep]: + if flask.request.method not in urldict['methods']: + continue + paramsig = urldict['paramsig'] + + # allow '?help' for any specifically-known endpoint + if 'help' in flask.request.args: + response = flask.make_response('{0}: {1}'.\ + format(prefix + concise_sig(paramsig), urldict['help'])) + response.headers['Content-Type'] = 'text/plain' + return response + + # if there are parameters for this endpoint, process them + if paramsig: + args = {} + for k, l in flask.request.args.iterlists(): + if len(l) == 1: + args[k] = l[0] + else: + args[k] = l + + # is this a valid set of params? + try: + argdict = validate(args, paramsig) + found = urldict + break + except Exception as e: + exc += str(e) + continue + else: + if flask.request.args: + continue + found = urldict + argdict = {} + break + if not found: + return make_response(fmt, '', exc + '\n', 400) argdict['format'] = fmt or 'plain' - argdict['module'] = urldict['module'] - argdict['perm'] = urldict['perm'] + argdict['module'] = found['module'] + argdict['perm'] = found['perm'] + if pgid: + argdict['pgid'] = pgid + + if not cmdtarget: + cmdtarget = ('mon', '') app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) ret, outbuf, outs = json_command(glob.cluster, prefix=prefix, + target=cmdtarget, inbuf=flask.request.data, argdict=argdict) if ret: return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) From 8985e1c9e80d1e0a646e4ae7d0e0d89c6b622068 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Tue, 23 Jul 2013 22:13:14 -0700 Subject: [PATCH 11/13] ceph_argparse, mon: make "tell " work (duplicating "pg ") It's a wad of special cases, but it implements "tell " such that it has the same effect as "pg ". Signed-off-by: Dan Mick --- src/osd/OSD.cc | 31 +++++++++++++++++++++++++++---- src/osd/ReplicatedPG.cc | 6 ------ src/pybind/ceph_argparse.py | 9 ++++++++- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index d611afdd08fb8..ea13b25b38a9b 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -3858,6 +3858,7 @@ COMMAND("list_missing " \ // tell commands. Validation of osd.n must be special-cased in client +// tell commands. Validation of osd.n must be special-cased in client COMMAND("version", "report version of OSD", "osd", "r", "cli,rest") COMMAND("injectargs " \ "name=injected_args,type=CephString,n=N", @@ -3883,6 +3884,19 @@ COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics", "osd", "r", "cli,rest") COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics", "osd", "rw", "cli,rest") + +// experiment: restate pg commands as "tell ". Validation of +// pgid must be special-cased in client. +COMMAND("query", + "show details of a specific pg", "osd", "r", "cli,rest") +COMMAND("mark_unfound_lost revert " \ + "name=mulcmd,type=CephChoices,strings=revert", \ + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", + "osd", "rw", "cli,rest") +COMMAND("list_missing " \ + "name=offset,type=CephString,req=false", + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "rw", "cli,rest") }; void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist& data) @@ -3897,6 +3911,7 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist map cmdmap; string prefix; string format; + string pgidstr; boost::scoped_ptr f; if (cmd.empty()) { @@ -3963,9 +3978,16 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist osd_lock.Lock(); } - else if (prefix == "pg") { + // either 'pg ' or + // 'tell ' (which comes in without any of that prefix)? + + else if (prefix == "pg" || + (cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr) && + (prefix == "query" || + prefix == "mark_unfound_lost" || + prefix == "list_missing") + )) { pg_t pgid; - string pgidstr; if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) { ss << "no pgid specified"; @@ -3974,13 +3996,14 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist ss << "couldn't parse pgid '" << pgidstr << "'"; r = -EINVAL; } else { - vector args; - cmd_getval(g_ceph_context, cmdmap, "args", args); PG *pg = _lookup_lock_pg(pgid); if (!pg) { ss << "i don't have pgid " << pgid; r = -ENOENT; } else { + // simulate pg cmd= for pg->do-command + if (prefix != "pg") + cmd_putval(g_ceph_context, cmdmap, "cmd", prefix); r = pg->do_command(cmdmap, ss, data, odata); pg->unlock(); } diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 4a59a23cdb794..658ea7cb74633 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -275,12 +275,6 @@ int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, string prefix; string format; - cmd_getval(g_ceph_context, cmdmap, "prefix", prefix); - if (prefix != "pg") { - ss << "ReplicatedPG::do_command: not pg command"; - return -EINVAL; - } - cmd_getval(g_ceph_context, cmdmap, "format", format); boost::scoped_ptr f(new_formatter(format)); // demand that we have a formatter diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index 855e21c25084e..354459a2cd57a 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -938,8 +938,15 @@ def send_command(cluster, target=('mon', ''), cmd=[], inbuf='', timeout=0, cluster.osd_command(osdid, cmd, inbuf, timeout) elif target[0] == 'pg': - # leave it in cmddict for the OSD to use too pgid = target[1] + # pgid will already be in the command for the pg + # form, but for tell , we need to put it in + if cmd: + cmddict = json.loads(cmd[0]) + cmddict['pgid'] = pgid + else: + cmddict = dict(pgid=pgid) + cmd = [json.dumps(cmddict)] if verbose: print >> sys.stderr, 'submit {0} for pgid {1}'.\ format(cmd, pgid) From d75b6ea1a47f8b240b5dfead28bd9a7d9257400e Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Wed, 24 Jul 2013 21:56:15 -0700 Subject: [PATCH 12/13] ceph_argparse.py: make find_cmd_target handle tell Signed-off-by: Dan Mick --- src/pybind/ceph_argparse.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index 354459a2cd57a..ee71b76d6a1f3 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -897,20 +897,35 @@ def find_cmd_target(childargs): sig = parse_funcsig(['tell', {'name':'target','type':'CephName'}]) try: valid_dict = validate(childargs, sig, partial=True); + except ArgumentError: + pass + else: if len(valid_dict) == 2: + # revalidate to isolate type and id name = CephName() + # if this fails, something is horribly wrong, as it just + # validated successfully above name.valid(valid_dict['target']) return name.nametype, name.nameid + + sig = parse_funcsig(['tell', {'name':'pgid','type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True); except ArgumentError: pass + else: + if len(valid_dict) == 2: + # pg doesn't need revalidation; the string is fine + return 'pg', valid_dict['pgid'] sig = parse_funcsig(['pg', {'name':'pgid','type':'CephPgid'}]) try: valid_dict = validate(childargs, sig, partial=True); - if len(valid_dict) == 2: - return 'pg', valid_dict['pgid'] except ArgumentError: pass + else: + if len(valid_dict) == 2: + return 'pg', valid_dict['pgid'] return 'mon', '' From aa00ace1d806526e02dbd65fddaccba2efa94163 Mon Sep 17 00:00:00 2001 From: Dan Mick Date: Fri, 26 Jul 2013 15:23:57 -0700 Subject: [PATCH 13/13] ceph_rest_api.py: cleanup, more docstrings, unused vars Signed-off-by: Dan Mick --- src/pybind/ceph_rest_api.py | 80 ++++++++++++++++++++++++------------- 1 file changed, 52 insertions(+), 28 deletions(-) diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index f8d9b92129e2d..830fb2249ea14 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -3,13 +3,13 @@ import collections import ConfigParser +import contextlib import errno import json import logging import logging.handlers import os import rados -import socket import textwrap import xml.etree.ElementTree import xml.sax.saxutils @@ -21,13 +21,13 @@ # Globals # -APPNAME = '__main__' DEFAULT_BASEURL = '/api/v0.1' DEFAULT_ADDR = '0.0.0.0:5000' DEFAULT_LOG_LEVEL = 'warning' DEFAULT_CLIENTNAME = 'client.restapi' DEFAULT_LOG_FILE = '/var/log/ceph/' + DEFAULT_CLIENTNAME + '.log' +APPNAME = '__main__' app = flask.Flask(APPNAME) LOGLEVELS = { @@ -38,7 +38,9 @@ 'debug':logging.DEBUG, } -# my globals, in a named tuple for usage clarity +# my globals, in a named tuple for usage clarity. I promise +# these are never written once initialized, and are global +# to every thread. glob = collections.namedtuple('gvars', 'cluster urls sigdict baseurl') glob.cluster = None @@ -47,8 +49,15 @@ glob.baseurl = '' def load_conf(clustername='ceph', conffile=None): - import contextlib + ''' + Load the ceph conf file using ConfigParser. Use the standard + fallback order: + 1) the passed in arg (from CEPH_CONF) + 2) /etc/ceph/{cluster}.conf + 3) ~/.ceph/{cluster}.conf + 4) {cluster}.conf + ''' class _TrimIndentFile(object): def __init__(self, fp): @@ -93,6 +102,10 @@ def load(path): raise EnvironmentError('No conf file found for "{0}"'.format(clustername)) def get_conf(cfg, clientname, key): + ''' + Get config entry from conf file, first in [clientname], then [client], + then [global]. + ''' fullkey = 'restapi_' + key for sectionname in clientname, 'client', 'global': try: @@ -101,8 +114,6 @@ def get_conf(cfg, clientname, key): pass return None -METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} - def find_up_osd(): ''' Find an up OSD. Return the last one that's up. @@ -121,16 +132,19 @@ def find_up_osd(): raise EnvironmentError(errno.ENOENT, 'No up OSDs found') return int(osds[-1]) -# XXX this is done globally, and cluster connection kept open; there -# are facilities to pass around global info to requests and to -# tear down connections between requests if it becomes important + +METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} def api_setup(): - """ + ''' + This is done globally, and cluster connection kept open for + the lifetime of the daemon. librados should assure that even + if the cluster goes away and comes back, our connection remains. + Initialize the running instance. Open the cluster, get the command signatures, module, perms, and help; stuff them away in the glob.urls - dict. - """ + dict. Also save glob.sigdict for help() handling. + ''' def get_command_descriptions(target=('mon','')): ret, outbuf, outs = json_command(glob.cluster, target, prefix='get_command_descriptions', @@ -253,11 +267,11 @@ def get_command_descriptions(target=('mon','')): def generate_url_and_params(sig, flavor): - """ + ''' Digest command signature from cluster; generate an absolute (including glob.baseurl) endpoint from all the prefix words, and a list of non-prefix param descs - """ + ''' url = '' params = [] @@ -292,10 +306,15 @@ def generate_url_and_params(sig, flavor): return glob.baseurl + url, params + +# +# end setup (import-time) functions, begin request-time functions +# + def concise_sig_for_uri(sig, flavor): - """ + ''' Return a generic description of how one would send a REST request for sig - """ + ''' prefix = [] args = [] ret = '' @@ -312,9 +331,9 @@ def concise_sig_for_uri(sig, flavor): return ret def show_human_help(prefix): - """ + ''' Dump table showing commands matching prefix - """ + ''' # XXX There ought to be a better discovery mechanism than an HTML table s = '' @@ -347,23 +366,22 @@ def show_human_help(prefix): @app.before_request def log_request(): - """ + ''' For every request, log it. XXX Probably overkill for production - """ + ''' app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) - @app.route('/') def root_redir(): return flask.redirect(glob.baseurl) def make_response(fmt, output, statusmsg, errorcode): - """ + ''' If formatted output, cobble up a response object that contains the output and status wrapped in enclosing objects; if nonformatted, just - use output. Return HTTP status errorcode in any event. - """ + use output+status. Return HTTP status errorcode in any event. + ''' response = output if fmt: if 'json' in fmt: @@ -375,6 +393,7 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response("Error decoding JSON from " + output, 500) elif 'xml' in fmt: + # XXX # one is tempted to do this with xml.etree, but figuring out how # to 'un-XML' the XML-dumped output so it can be reassembled into # a piece of the tree here is beyond me right now. @@ -401,9 +420,12 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response(response, errorcode) def handler(catchall_path=None, fmt=None, target=None): - """ - Main endpoint handler; generic for every endpoint - """ + ''' + Main endpoint handler; generic for every endpoint, including catchall. + Handles the catchall, anything with <.fmt>, anything with embedded + . Partial match or ?help cause the HTML-table + "show_human_help" output. + ''' ep = catchall_path or flask.request.endpoint ep = ep.replace('.', '') @@ -424,7 +446,6 @@ def handler(catchall_path=None, fmt=None, target=None): elif 'application/xml' in flask.request.accept_mimetypes.values(): fmt = 'xml' - valid = True prefix = '' pgid = None cmdtarget = 'mon', '' @@ -530,4 +551,7 @@ def handler(catchall_path=None, fmt=None, target=None): response.headers['Content-Type'] = contenttype return response +# +# Last module-level (import-time) ask: set up the cluster connection +# addr, port = api_setup()
Possible commands:MethodDescription