diff --git a/COPYING b/COPYING index b374bdc180176..fe6ffaac0f80f 100644 --- a/COPYING +++ b/COPYING @@ -15,11 +15,6 @@ Copyright: Copyright (C) 2004-2006 Sage Weil License: GPL2 -Files: src/common/fiemap.cc -Copyright: - Copyright (C) 2010 Canonical -License: GPL2 - Files: src/mount/canonicalize.c Copyright: Copyright (C) 1993 Rick Sladkey License: LGPL2 or later diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 2c001894745c7..4b6cb80029061 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -6,3 +6,10 @@ v0.67 in order to accomodate the new write back throttle system. upstart now sets the fd limit to 32k. sysvinit will set it to 32k by default (still overrideable via max_open_files). + +* The 'ceph pg ...' commands (like 'ceph pg query') are + deprecated in favor of 'ceph tell ...'. This makes the + distinction between 'ceph pg ' and 'ceph pg + ' less awkward by making it clearer that the 'tell' + commands are talking to the OSD serving the placement group, not the + monitor. diff --git a/ceph.spec.in b/ceph.spec.in index fb8bcca4b7140..696d9ad3332e2 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -189,6 +189,7 @@ Requires: librados2 = %{version}-%{release} Requires: librbd1 = %{version}-%{release} Requires: libcephfs1 = %{version}-%{release} Requires: python-flask +Requires: python-requests %if 0%{defined suse_version} %py_requires %endif diff --git a/debian/control b/debian/control index 3967d2f1c20ca..195cb37fe6289 100644 --- a/debian/control +++ b/debian/control @@ -389,7 +389,7 @@ Description: Ceph test and benchmarking tools. Package: python-ceph Architecture: linux-any Section: python -Depends: librados2, librbd1, python-flask, ${misc:Depends}, ${python:Depends} +Depends: librados2, librbd1, python-flask, ${misc:Depends}, ${python:Depends}, python-requests X-Python-Version: >= 2.6 Description: Python libraries for the Ceph distributed filesystem Ceph is a distributed storage and network file system designed to provide diff --git a/debian/copyright b/debian/copyright index 4295a1564f9a3..e94a11b996267 100644 --- a/debian/copyright +++ b/debian/copyright @@ -16,11 +16,6 @@ Copyright: Copyright (C) 2004-2006 Sage Weil License: GPL2 -Files: src/common/fiemap.cc -Copyright: - Copyright (C) 2010 Canonical -License: GPL2 - Files: src/mount/canonicalize.c Copyright: Copyright (C) 1993 Rick Sladkey License: LGPL2 or later diff --git a/doc/man/8/ceph-rest-api.rst b/doc/man/8/ceph-rest-api.rst index 8a87f97ce19bf..a000d09c67603 100644 --- a/doc/man/8/ceph-rest-api.rst +++ b/doc/man/8/ceph-rest-api.rst @@ -7,7 +7,7 @@ Synopsis ======== -| **ceph-rest-api** [ -c *conffile* ] [ -n *name* ... ] +| **ceph-rest-api** [ -c *conffile* ] [--cluster *clustername* ] [ -n *name* ] [-i *id* ] Description @@ -21,7 +21,7 @@ command-line tool through an HTTP-accessible interface. Options ======= -.. option:: -c/--conf *conffile* +.. option:: -c/--conf conffile names the ceph.conf file to use for configuration. If -c is not specified, the default depends on the state of the --cluster option @@ -35,21 +35,26 @@ Options so you can also pass this option in the environment as CEPH_CONF. -.. option:: --cluster *clustername* +.. option:: --cluster clustername set *clustername* for use in the $cluster metavariable, for locating the ceph.conf file. The default is 'ceph'. - You can also pass this option in the environment as - CEPH_CLUSTER_NAME. -.. option:: -n/--name *name* +.. option:: -n/--name name specifies the client 'name', which is used to find the client-specific configuration options in the config file, and also is the name used for authentication when connecting to the cluster (the entity name appearing in ceph auth list output, - for example). The default is 'client.restapi'. You can also - pass this option in the environment as CEPH_NAME. + for example). The default is 'client.restapi'. + +.. option:: -i/--id id + + specifies the client 'id', which will form the clientname + as 'client.' if clientname is not set. If -n/-name is + set, that takes precedence. + + Also, global Ceph options are supported. Configuration parameters @@ -57,16 +62,22 @@ Configuration parameters Supported configuration parameters include: -* **restapi client name** the 'clientname' used for auth and ceph.conf -* **restapi keyring** the keyring file holding the key for 'clientname' -* **restapi public addr** ip:port to listen on (default 0.0.0.0:5000) +* **keyring** the keyring file holding the key for 'clientname' +* **public addr** ip:port to listen on (default 0.0.0.0:5000) +* **log file** (usual Ceph default) * **restapi base url** the base URL to answer requests on (default /api/v0.1) -* **restapi log level** critical, error, warning, info, debug -* **restapi log file** (default /var/local/ceph/.log) +* **restapi log level** critical, error, warning, info, debug (default warning) + +Configuration parameters are searched in the standard order: +first in the section named '', then 'client', then 'global'. -A server will run on **restapi public addr** if the ceph-rest-api -executed directly; otherwise, configuration is specified by the -enclosing WSGI web server. + is either supplied by -n/--name, "client." where + is supplied by -i/--id, or 'client.restapi' if neither option +is present. + +A single-threaded server will run on **public addr** if the ceph-rest-api +executed directly; otherwise, configuration is specified by the enclosing +WSGI web server. Commands ======== @@ -92,7 +103,9 @@ with a small description of each command, is provided when the requested path is incomplete/partially matching. Requesting / will redirect to the value of **restapi base url**, and that path will give a full list of all known commands. The command set is very similar to the commands -supported by the **ceph** tool. +supported by the **ceph** tool. One notable exception is that the +``ceph pg `` style of commands is supported here +as ``tell//command?args``. Deployment as WSGI application ============================== @@ -101,18 +114,22 @@ When deploying as WSGI application (say, with Apache/mod_wsgi, or nginx/uwsgi, or gunicorn, etc.), use the ``ceph_rest_api.py`` module (``ceph-rest-api`` is a thin layer around this module). The standalone web server is of course not used, so address/port configuration is done in -the WSGI server. Also, configuration switches are not passed; rather, -environment variables are used: - -* CEPH_CONF holds -c/--conf -* CEPH_CLUSTER_NAME holds --cluster -* CEPH_NAME holds -n/--name - -Any errors reading configuration or connecting to the cluster cause -ImportError to be raised with a descriptive message on import; see -your WSGI server documentation for how to see those messages in case -of problem. - +the WSGI server. Use a python .wsgi module or the equivalent to call +``app = generate_app(conf, cluster, clientname, clientid, args)`` where: + +* conf is as -c/--conf above +* cluster is as --cluster above +* clientname, -n/--name +* clientid, -i/--id, and +* args are any other generic Ceph arguments + +When app is returned, it will have attributes 'ceph_addr' and 'ceph_port' +set to what the address and port are in the Ceph configuration; +those may be used for the server, or ignored. + +Any errors reading configuration or connecting to the cluster cause an +exception to be raised; see your WSGI server documentation for how to +see those messages in case of problem. Availability ============ diff --git a/man/ceph-authtool.8 b/man/ceph-authtool.8 index 47888af1f2245..e64cac95f0a59 100644 --- a/man/ceph-authtool.8 +++ b/man/ceph-authtool.8 @@ -69,16 +69,33 @@ will create a new keyring, overwriting any existing keyringfile .UNINDENT .INDENT 0.0 .TP -.B \-\-gen\-key +.B \-g, \-\-gen\-key will generate a new secret key for the specified entityname .UNINDENT .INDENT 0.0 .TP -.B \-\-add\-key +.B \-a, \-\-add\-key will add an encoded key to the keyring .UNINDENT .INDENT 0.0 .TP +.B \-u, \-\-set\-uid +sets the auid (authenticated user id) for the specified entityname +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-gen\-print\-key +will generate and print a new secret key without adding it to the keyringfile + +NOTE: will work without a given keyringfile +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-import\-keyring +will import the content of a given keyring into the keyringfile +.UNINDENT +.INDENT 0.0 +.TP .B \-\-cap subsystem capability will set the capability for given subsystem .UNINDENT diff --git a/man/ceph-rest-api.8 b/man/ceph-rest-api.8 index 33425fecc00b6..5170b7f37cfa7 100644 --- a/man/ceph-rest-api.8 +++ b/man/ceph-rest-api.8 @@ -1,4 +1,4 @@ -.TH "CEPH-REST-API" "8" "July 12, 2013" "dev" "Ceph" +.TH "CEPH-REST-API" "8" "July 26, 2013" "dev" "Ceph" .SH NAME ceph-rest-api \- ceph RESTlike administration server . @@ -32,7 +32,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] . .SH SYNOPSIS .nf -\fBceph\-rest\-api\fP [ \-c \fIconffile\fP ] [ \-n \fIname\fP ... ] +\fBceph\-rest\-api\fP [ \-c \fIconffile\fP ] [\-\-cluster \fIclustername\fP ] [ \-n \fIname\fP ] [\-i \fIid\fP ] .fi .sp .SH DESCRIPTION @@ -44,7 +44,7 @@ command\-line tool through an HTTP\-accessible interface. .SH OPTIONS .INDENT 0.0 .TP -.B \-c/\-\-conf *conffile* +.B \-c/\-\-conf conffile names the ceph.conf file to use for configuration. If \-c is not specified, the default depends on the state of the \-\-cluster option (default \(aqceph\(aq; see below). The configuration file is searched @@ -64,43 +64,54 @@ so you can also pass this option in the environment as CEPH_CONF. .UNINDENT .INDENT 0.0 .TP -.B \-\-cluster *clustername* +.B \-\-cluster clustername set \fIclustername\fP for use in the $cluster metavariable, for locating the ceph.conf file. The default is \(aqceph\(aq. -You can also pass this option in the environment as -CEPH_CLUSTER_NAME. .UNINDENT .INDENT 0.0 .TP -.B \-n/\-\-name *name* +.B \-n/\-\-name name specifies the client \(aqname\(aq, which is used to find the client\-specific configuration options in the config file, and also is the name used for authentication when connecting to the cluster (the entity name appearing in ceph auth list output, -for example). The default is \(aqclient.restapi\(aq. You can also -pass this option in the environment as CEPH_NAME. +for example). The default is \(aqclient.restapi\(aq. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i/\-\-id id +specifies the client \(aqid\(aq, which will form the clientname +as \(aqclient.\(aq if clientname is not set. If \-n/\-name is +set, that takes precedence. +.sp +Also, global Ceph options are supported. .UNINDENT .SH CONFIGURATION PARAMETERS .sp Supported configuration parameters include: .INDENT 0.0 .IP \(bu 2 -\fBrestapi client name\fP the \(aqclientname\(aq used for auth and ceph.conf +\fBkeyring\fP the keyring file holding the key for \(aqclientname\(aq .IP \(bu 2 -\fBrestapi keyring\fP the keyring file holding the key for \(aqclientname\(aq +\fBpublic addr\fP ip:port to listen on (default 0.0.0.0:5000) .IP \(bu 2 -\fBrestapi public addr\fP ip:port to listen on (default 0.0.0.0:5000) +\fBlog file\fP (usual Ceph default) .IP \(bu 2 \fBrestapi base url\fP the base URL to answer requests on (default /api/v0.1) .IP \(bu 2 -\fBrestapi log level\fP critical, error, warning, info, debug -.IP \(bu 2 -\fBrestapi log file\fP (default /var/local/ceph/.log) +\fBrestapi log level\fP critical, error, warning, info, debug (default warning) .UNINDENT .sp -A server will run on \fBrestapi public addr\fP if the ceph\-rest\-api -executed directly; otherwise, configuration is specified by the -enclosing WSGI web server. +Configuration parameters are searched in the standard order: +first in the section named \(aq\(aq, then \(aqclient\(aq, then \(aqglobal\(aq. +.sp + is either supplied by \-n/\-\-name, "client." where + is supplied by \-i/\-\-id, or \(aqclient.restapi\(aq if neither option +is present. +.sp +A single\-threaded server will run on \fBpublic addr\fP if the ceph\-rest\-api +executed directly; otherwise, configuration is specified by the enclosing +WSGI web server. .SH COMMANDS .sp Commands are submitted with HTTP GET requests (for commands that @@ -122,28 +133,37 @@ with a small description of each command, is provided when the requested path is incomplete/partially matching. Requesting / will redirect to the value of \fBrestapi base url\fP, and that path will give a full list of all known commands. The command set is very similar to the commands -supported by the \fBceph\fP tool. +supported by the \fBceph\fP tool. One notable exception is that the +\fBceph pg \fP style of commands is supported here +as \fBtell//command?args\fP. .SH DEPLOYMENT AS WSGI APPLICATION .sp When deploying as WSGI application (say, with Apache/mod_wsgi, or nginx/uwsgi, or gunicorn, etc.), use the \fBceph_rest_api.py\fP module (\fBceph\-rest\-api\fP is a thin layer around this module). The standalone web server is of course not used, so address/port configuration is done in -the WSGI server. Also, configuration switches are not passed; rather, -environment variables are used: +the WSGI server. Use a python .wsgi module or the equivalent to call +\fBapp = generate_app(conf, cluster, clientname, clientid, args)\fP where: .INDENT 0.0 .IP \(bu 2 -CEPH_CONF holds \-c/\-\-conf +conf is as \-c/\-\-conf above +.IP \(bu 2 +cluster is as \-\-cluster above .IP \(bu 2 -CEPH_CLUSTER_NAME holds \-\-cluster +clientname, \-n/\-\-name .IP \(bu 2 -CEPH_NAME holds \-n/\-\-name +clientid, \-i/\-\-id, and +.IP \(bu 2 +args are any other generic Ceph arguments .UNINDENT .sp -Any errors reading configuration or connecting to the cluster cause -ImportError to be raised with a descriptive message on import; see -your WSGI server documentation for how to see those messages in case -of problem. +When app is returned, it will have attributes \(aqceph_addr\(aq and \(aqceph_port\(aq +set to what the address and port are in the Ceph configuration; +those may be used for the server, or ignored. +.sp +Any errors reading configuration or connecting to the cluster cause an +exception to be raised; see your WSGI server documentation for how to +see those messages in case of problem. .SH AVAILABILITY .sp \fBceph\-rest\-api\fP is part of the Ceph distributed file system. Please refer to the Ceph documentation at diff --git a/qa/fs/.gitignore b/qa/fs/.gitignore new file mode 100644 index 0000000000000..a2d280e079454 --- /dev/null +++ b/qa/fs/.gitignore @@ -0,0 +1 @@ +/test_o_trunc diff --git a/src/Makefile.am b/src/Makefile.am index 307abf2f9653a..a9bbde3268601 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -504,7 +504,7 @@ ceph_radosacl_SOURCES = radosacl.cc ceph_radosacl_LDADD = librados.la $(PTHREAD_LIBS) -lm $(CRYPTO_LIBS) $(EXTRALIBS) bin_DEBUGPROGRAMS += ceph_scratchtool ceph_scratchtoolpp ceph_radosacl -rbd_SOURCES = rbd.cc common/fiemap.cc common/secret.c common/TextTable.cc common/util.cc +rbd_SOURCES = rbd.cc common/secret.c common/TextTable.cc common/util.cc rbd_CXXFLAGS = ${AM_CXXFLAGS} rbd_LDADD = librbd.la librados.la $(LIBGLOBAL_LDA) -lkeyutils if LINUX @@ -1895,7 +1895,6 @@ noinst_HEADERS = \ include/encoding.h\ include/err.h\ include/error.h\ - include/fiemap.h\ include/filepath.h\ include/frag.h\ include/hash.h\ diff --git a/src/ceph-rest-api b/src/ceph-rest-api index a44919acd6fc1..ae5245b4f76f2 100755 --- a/src/ceph-rest-api +++ b/src/ceph-rest-api @@ -33,34 +33,35 @@ def parse_args(): parser.add_argument('-c', '--conf', help='Ceph configuration file') parser.add_argument('--cluster', help='Ceph cluster name') parser.add_argument('-n', '--name', help='Ceph client name') + parser.add_argument('-i', '--id', help='Ceph client id') - return parser.parse_args() - + return parser.parse_known_args() # main -parsed_args = parse_args() -if parsed_args.conf: - os.environ['CEPH_CONF'] = parsed_args.conf -if parsed_args.cluster: - os.environ['CEPH_CLUSTER_NAME'] = parsed_args.cluster -if parsed_args.name: - os.environ['CEPH_NAME'] = parsed_args.name +parsed_args, rest = parse_args() # import now that env vars are available to imported module try: import ceph_rest_api -except Exception as e: +except EnvironmentError as e: print >> sys.stderr, "Error importing ceph_rest_api: ", str(e) sys.exit(1) -# importing ceph_rest_api has set module globals 'app', 'addr', and 'port' +# let other exceptions generate traceback + +app = ceph_rest_api.generate_app( + parsed_args.conf, + parsed_args.cluster, + parsed_args.name, + parsed_args.id, + rest, +) files = [os.path.split(fr[1])[-1] for fr in inspect.stack()] if 'pdb.py' in files: - ceph_rest_api.app.run(host=ceph_rest_api.addr, port=ceph_rest_api.port, - debug=True, use_reloader=False, use_debugger=False) + app.run(host=app.ceph_addr, port=app.ceph_port, + debug=True, use_reloader=False, use_debugger=False) else: - ceph_rest_api.app.run(host=ceph_rest_api.addr, port=ceph_rest_api.port, - debug=True) + app.run(host=app.ceph_addr, port=app.ceph_port) diff --git a/src/ceph.in b/src/ceph.in index adaeb68d6fbe9..22c54a486891d 100755 --- a/src/ceph.in +++ b/src/ceph.in @@ -64,6 +64,9 @@ cluster_handle = None def osdids(): ret, outbuf, outs = json_command(cluster_handle, prefix='osd ls') + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, cmd=['osd', 'ls']) if ret: raise RuntimeError('Can\'t contact mon for osd list') return [i for i in outbuf.split('\n') if i != ''] @@ -71,6 +74,10 @@ def osdids(): def monids(): ret, outbuf, outs = json_command(cluster_handle, prefix='mon dump', argdict={'format':'json'}) + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, + cmd=['mon', 'dump', '--format=json']) if ret: raise RuntimeError('Can\'t contact mon for mon list') d = json.loads(outbuf) @@ -79,6 +86,10 @@ def monids(): def mdsids(): ret, outbuf, outs = json_command(cluster_handle, prefix='mds dump', argdict={'format':'json'}) + if ret == -errno.EINVAL: + # try old mon + ret, outbuf, outs = send_command(cluster_handle, + cmd=['mds', 'dump', '--format=json']) if ret: raise RuntimeError('Can\'t contact mon for mds list') d = json.loads(outbuf) @@ -100,8 +111,6 @@ def parse_cmdargs(args=None, target=''): parser.add_argument('-h', '--help', help='request mon help', action='store_true') - parser.add_argument('--help-all', help='request help for all daemons', - action='store_true') parser.add_argument('-c', '--conf', dest='cephconf', help='ceph configuration file') @@ -150,14 +159,16 @@ def parse_cmdargs(args=None, target=''): return parser, parsed_args, extras -def do_help(parser, args, help_all = False): +def do_help(parser, args): """ Print basic parser help - If the cluster is available: - get and print monitor help; - if help_all, print help for daemon commands as well + If the cluster is available, get and print monitor help """ + def help_for_sigs(sigs, partial=None): + sys.stdout.write(format_help(parse_json_funcsigs(sigs, 'cli'), + partial=partial)) + def help_for_target(target, partial=None): ret, outbuf, outs = json_command(cluster_handle, target=target, prefix='get_command_descriptions', @@ -167,40 +178,19 @@ def do_help(parser, args, help_all = False): "couldn't get command descriptions for {0}: {1}".\ format(target, outs) else: - sys.stdout.write(format_help(parse_json_funcsigs(outbuf, 'cli'), - partial)) + help_for_sigs(outbuf, partial) - parser.print_help() - print '\n' - if (cluster_handle): - help_for_target(target=('mon', ''), partial=' '.join(args)) - - if help_all and cluster_handle: - # try/except in case there are no daemons of that type - try: - firstosd = osdids()[0] - print '\nOSD.{0} tell commands and pg pgid commands:\n\n'.\ - format(firstosd) - help_for_target(target=('osd', osdids()[0])) - print '\nOSD daemon commands:\n\n' - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'osd.' + firstosd), ['get_command_descriptions']), 'cli'))) - except: - pass + def hdr(s): + print '\n', s, '\n', '=' * len(s) - try: - firstmon = monids()[0] - print '\nmon.{0} daemon commands:\n\n'.format(firstmon) - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'mon.' + firstmon), ['get_command_descriptions']), 'cli'))) - except: - pass + hdr('Monitor commands:') + partial = ' '.join(args) + parser.print_help() + print '\n' - try: - firstmds = mdsids()[0] - print '\nmds.{0} daemon commands:\n\n'.format(firstmds) - sys.stdout.write(format_help(parse_json_funcsigs(admin_socket(ceph_conf('admin_socket', 'mds.' + firstmds), ['get_command_descriptions']), 'cli'))) - except: - pass + if (cluster_handle): + help_for_target(target=('mon', ''), partial=partial) return 0 @@ -285,26 +275,57 @@ def format_help(cmddict, partial=None): return fullusage -def admin_socket(asok_path, cmd): - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +def admin_socket(asok_path, cmd, format=''): + """ + Send a daemon (--admin-daemon) command 'cmd'. asok_path is the + path to the admin socket; cmd is a list of strings; format may be + set to one of the formatted forms to get output in that form + (daemon commands don't support 'plain' output). + """ + + def do_sockio(path, cmd): + """ helper: do all the actual low-level stream I/O """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.connect(path) + try: + sock.sendall(cmd + '\0') + len_str = sock.recv(4) + if len(len_str) < 4: + raise RuntimeError("no data returned from admin socket") + l, = struct.unpack(">I", len_str) + ret = '' + + got = 0 + while got < l: + bit = sock.recv(l - got) + ret += bit + got += len(bit) + + except Exception as e: + raise RuntimeError('exception: ' + str(e)) + return ret + try: - sock.connect(asok_path) - sock.sendall(' '.join(cmd) + '\0') + cmd_json = do_sockio(asok_path, + json.dumps({"prefix":"get_command_descriptions"})) + except Exception as e: + raise RuntimeError('exception getting command descriptions: ' + str(e)) + + if cmd == 'get_command_descriptions': + return cmd_json - len_str = sock.recv(4) - if len(len_str) < 4: - raise RuntimeError("no data returned from admin socket") - l, = struct.unpack(">I", len_str) - ret = '' + sigdict = parse_json_funcsigs(cmd_json, 'cli') + valid_dict = validate_command(sigdict, cmd) + if not valid_dict: + return -errno.EINVAL - got = 0 - while got < l: - bit = sock.recv(l - got) - ret += bit - got += len(bit) + if format: + valid_dict['format'] = format + try: + ret = do_sockio(asok_path, json.dumps(valid_dict)) except Exception as e: - raise RuntimeError('exception: {0}'.format(e)) + raise RuntimeError('exception: ' + str(e)) return ret @@ -344,10 +365,11 @@ def new_style_command(parsed_args, cmdargs, target, sigdict, inbuf, verbose): if not got_command: if cmdargs: # Validate input args against list of sigs - valid_dict = validate_command(parsed_args, sigdict, cmdargs, - verbose) + valid_dict = validate_command(sigdict, cmdargs, verbose) if valid_dict: got_command = True + if parsed_args.output_format: + valid_dict['format'] = parsed_args.output_format else: return -errno.EINVAL, '', 'invalid command' else: @@ -360,8 +382,10 @@ def new_style_command(parsed_args, cmdargs, target, sigdict, inbuf, verbose): return 0, '', '' cmdargs = parse_cmdargs(interactive_input.split())[2] target = find_cmd_target(cmdargs) - valid_dict = validate_command(parsed_args, sigdict, cmdargs) + valid_dict = validate_command(sigdict, cmdargs, verbose) if valid_dict: + if parsed_args.output_format: + valid_dict['format'] = parsed_args.output_format if verbose: print >> sys.stderr, "Submitting command ", valid_dict ret, outbuf, outs = json_command(cluster_handle, @@ -470,9 +494,12 @@ def main(): conffile = parsed_args.cephconf # For now, --admin-daemon is handled as usual. Try it # first in case we can't connect() to the cluster + + format = parsed_args.output_format + if parsed_args.admin_socket: try: - print admin_socket(parsed_args.admin_socket, childargs) + print admin_socket(parsed_args.admin_socket, childargs, format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 @@ -481,7 +508,7 @@ def main(): if len(childargs) > 2: if childargs[1].find('/') >= 0: try: - print admin_socket(childargs[1], childargs[2:]) + print admin_socket(childargs[1], childargs[2:], format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 @@ -489,7 +516,7 @@ def main(): # try resolve daemon name path = ceph_conf('admin_socket', childargs[1]) try: - print admin_socket(path, childargs[2:]) + print admin_socket(path, childargs[2:], format) except Exception as e: print >> sys.stderr, 'admin_socket: {0}'.format(e) return 0 @@ -544,8 +571,8 @@ def main(): format(e.__class__.__name__) return 1 - if parsed_args.help or parsed_args.help_all: - return do_help(parser, childargs, parsed_args.help_all) + if parsed_args.help: + return do_help(parser, childargs) # implement -w/--watch_* # This is ugly, but Namespace() isn't quite rich enough. diff --git a/src/ceph_authtool.cc b/src/ceph_authtool.cc index 3075d9c69a780..f66a3c66eee9a 100644 --- a/src/ceph_authtool.cc +++ b/src/ceph_authtool.cc @@ -36,9 +36,15 @@ void usage() << " 'mount -o secret=..' argument\n" << " -C, --create-keyring will create a new keyring, overwriting any\n" << " existing keyringfile\n" - << " --gen-key will generate a new secret key for the\n" + << " -g, --gen-key will generate a new secret key for the\n" << " specified entityname\n" - << " --add-key will add an encoded key to the keyring\n" + << " --gen-print-key will generate a new secret key without set it\n" + << " to the keyringfile, prints the secret to stdout\n" + << " --import-keyring will import the content of a given keyring\n" + << " into the keyringfile\n" + << " -u, --set-uid sets the auid (authenticated user id) for the\n" + << " specified entityname\n" + << " -a, --add-key will add an encoded key to the keyring\n" << " --cap subsystem capability will set the capability for given subsystem\n" << " --caps capsfile will set all of capabilities associated with a\n" << " given key, for all subsystems" diff --git a/src/client/Client.cc b/src/client/Client.cc index ba036ad998036..5a9c5fdafcca2 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -102,22 +102,24 @@ Client::CommandHook::CommandHook(Client *client) : { } -bool Client::CommandHook::call(std::string command, std::string args, bufferlist& out) +bool Client::CommandHook::call(std::string command, std::string args, + std::string format, bufferlist& out) { stringstream ss; - JSONFormatter formatter(true); + Formatter *f = new_formatter(format); m_client->client_lock.Lock(); if (command == "mds_requests") - m_client->dump_mds_requests(&formatter); + m_client->dump_mds_requests(f); else if (command == "mds_sessions") - m_client->dump_mds_sessions(&formatter); + m_client->dump_mds_sessions(f); else if (command == "dump_cache") - m_client->dump_cache(&formatter); + m_client->dump_cache(f); else assert(0 == "bad command registered"); m_client->client_lock.Unlock(); - formatter.flush(ss); + f->flush(ss); out.append(ss); + delete f; return true; } diff --git a/src/client/Client.h b/src/client/Client.h index ade0b8f29c84d..bc1fbc0401b5c 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -196,7 +196,8 @@ class Client : public Dispatcher { Client *m_client; public: CommandHook(Client *client); - bool call(std::string command, std::string args, bufferlist& out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out); }; CommandHook m_command_hook; diff --git a/src/common/Formatter.cc b/src/common/Formatter.cc index 357b287fe3222..c08ea5b9a2051 100644 --- a/src/common/Formatter.cc +++ b/src/common/Formatter.cc @@ -62,15 +62,19 @@ Formatter::~Formatter() } Formatter * -new_formatter(const std::string &type) +new_formatter(const std::string type) { - if (type == "json") + std::string mytype = type; + if (mytype == "") + mytype = "json-pretty"; + + if (mytype == "json") return new JSONFormatter(false); - else if (type == "json-pretty") + else if (mytype == "json-pretty") return new JSONFormatter(true); - else if (type == "xml") + else if (mytype == "xml") return new XMLFormatter(false); - else if (type == "xml-pretty") + else if (mytype == "xml-pretty") return new XMLFormatter(true); else return (Formatter *)NULL; @@ -250,6 +254,18 @@ void JSONFormatter::dump_format(const char *name, const char *fmt, ...) print_quoted_string(buf); } +void JSONFormatter::dump_format_unquoted(const char *name, const char *fmt, ...) +{ + char buf[LARGE_SIZE]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, LARGE_SIZE, fmt, ap); + va_end(ap); + + print_name(name); + m_ss << buf; +} + int JSONFormatter::get_len() const { return m_ss.str().size(); @@ -400,6 +416,21 @@ void XMLFormatter::dump_format(const char *name, const char *fmt, ...) m_ss << "\n"; } +void XMLFormatter::dump_format_unquoted(const char *name, const char *fmt, ...) +{ + char buf[LARGE_SIZE]; + va_list ap; + va_start(ap, fmt); + vsnprintf(buf, LARGE_SIZE, fmt, ap); + va_end(ap); + + std::string e(name); + print_spaces(); + m_ss << "<" << e << ">" << buf << ""; + if (m_pretty) + m_ss << "\n"; +} + int XMLFormatter::get_len() const { return m_ss.str().size(); diff --git a/src/common/Formatter.h b/src/common/Formatter.h index 8775c0cf9df5c..da730103f41c2 100644 --- a/src/common/Formatter.h +++ b/src/common/Formatter.h @@ -45,6 +45,7 @@ class Formatter { virtual void dump_string(const char *name, std::string s) = 0; virtual std::ostream& dump_stream(const char *name) = 0; virtual void dump_format(const char *name, const char *fmt, ...) = 0; + virtual void dump_format_unquoted(const char *name, const char *fmt, ...) = 0; virtual int get_len() const = 0; virtual void write_raw_data(const char *data) = 0; @@ -60,7 +61,7 @@ class Formatter { } }; -Formatter *new_formatter(const std::string &type); +Formatter *new_formatter(const std::string type); class JSONFormatter : public Formatter { public: @@ -79,6 +80,7 @@ class JSONFormatter : public Formatter { void dump_string(const char *name, std::string s); std::ostream& dump_stream(const char *name); void dump_format(const char *name, const char *fmt, ...); + void dump_format_unquoted(const char *name, const char *fmt, ...); int get_len() const; void write_raw_data(const char *data); @@ -119,6 +121,7 @@ class XMLFormatter : public Formatter { void dump_string(const char *name, std::string s); std::ostream& dump_stream(const char *name); void dump_format(const char *name, const char *fmt, ...); + void dump_format_unquoted(const char *name, const char *fmt, ...); int get_len() const; void write_raw_data(const char *data); diff --git a/src/common/admin_socket.cc b/src/common/admin_socket.cc index f7ab3501dfff9..e73f3ce0a0c2a 100644 --- a/src/common/admin_socket.cc +++ b/src/common/admin_socket.cc @@ -310,6 +310,18 @@ bool AdminSocket::do_accept() bool rval = false; + map cmdmap; + string format; + vector cmdvec; + stringstream errss; + cmdvec.push_back(cmd); + if (!cmdmap_from_json(cmdvec, &cmdmap, errss)) { + ldout(m_cct, 0) << "AdminSocket: " << errss << dendl; + return false; + } + cmd_getval(m_cct, cmdmap, "format", format); + cmd_getval(m_cct, cmdmap, "prefix", c); + string firstword; if (c.find(" ") == string::npos) firstword = c; @@ -341,7 +353,7 @@ bool AdminSocket::do_accept() string args; if (match != c) args = c.substr(match.length() + 1); - bool success = p->second->call(match, args, out); + bool success = p->second->call(match, args, format, out); if (!success) { ldout(m_cct, 0) << "AdminSocket: request '" << match << "' args '" << args << "' to " << p->second << " failed" << dendl; @@ -378,8 +390,7 @@ int AdminSocket::register_command(std::string command, std::string cmddesc, Admi ldout(m_cct, 5) << "register_command " << command << " hook " << hook << dendl; m_hooks[command] = hook; m_descs[command] = cmddesc; - if (help.length()) - m_help[command] = help; + m_help[command] = help; ret = 0; } m_lock.Unlock(); @@ -406,7 +417,8 @@ int AdminSocket::unregister_command(std::string command) class VersionHook : public AdminSocketHook { public: - virtual bool call(std::string command, std::string args, bufferlist& out) { + virtual bool call(std::string command, std::string args, std::string format, + bufferlist& out) { if (command == "0") { out.append(CEPH_ADMIN_SOCK_VERSION); } else { @@ -429,18 +441,20 @@ class HelpHook : public AdminSocketHook { AdminSocket *m_as; public: HelpHook(AdminSocket *as) : m_as(as) {} - bool call(string command, string args, bufferlist& out) { - JSONFormatter jf(true); - jf.open_object_section("help"); + bool call(string command, string args, string format, bufferlist& out) { + Formatter *f = new_formatter(format); + f->open_object_section("help"); for (map::iterator p = m_as->m_help.begin(); p != m_as->m_help.end(); ++p) { - jf.dump_string(p->first.c_str(), p->second); + if (p->second.length()) + f->dump_string(p->first.c_str(), p->second); } - jf.close_section(); + f->close_section(); ostringstream ss; - jf.flush(ss); + f->flush(ss); out.append(ss.str()); + delete f; return true; } }; @@ -449,7 +463,7 @@ class GetdescsHook : public AdminSocketHook { AdminSocket *m_as; public: GetdescsHook(AdminSocket *as) : m_as(as) {} - bool call(string command, string args, bufferlist& out) { + bool call(string command, string args, string format, bufferlist& out) { int cmdnum = 0; JSONFormatter jf(false); jf.open_object_section("command_descriptions"); diff --git a/src/common/admin_socket.h b/src/common/admin_socket.h index c390bca0382c3..30c5eb96ab8d6 100644 --- a/src/common/admin_socket.h +++ b/src/common/admin_socket.h @@ -29,7 +29,8 @@ class CephContext; class AdminSocketHook { public: - virtual bool call(std::string command, std::string args, bufferlist& out) = 0; + virtual bool call(std::string command, std::string args, std::string format, + bufferlist& out) = 0; virtual ~AdminSocketHook() {}; }; diff --git a/src/common/ceph_context.cc b/src/common/ceph_context.cc index cad980bb2a6f2..6b227d8689e31 100644 --- a/src/common/ceph_context.cc +++ b/src/common/ceph_context.cc @@ -156,44 +156,46 @@ class CephContextHook : public AdminSocketHook { public: CephContextHook(CephContext *cct) : m_cct(cct) {} - bool call(std::string command, std::string args, bufferlist& out) { - m_cct->do_command(command, args, &out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { + m_cct->do_command(command, args, format, &out); return true; } }; -void CephContext::do_command(std::string command, std::string args, bufferlist *out) +void CephContext::do_command(std::string command, std::string args, + std::string format, bufferlist *out) { + Formatter *f = new_formatter(format); lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "'" << dendl; if (command == "perfcounters_dump" || command == "1" || command == "perf dump") { - _perf_counters_collection->write_json_to_buf(*out, false); + _perf_counters_collection->dump_formatted(f, false); } else if (command == "perfcounters_schema" || command == "2" || command == "perf schema") { - _perf_counters_collection->write_json_to_buf(*out, true); + _perf_counters_collection->dump_formatted(f, true); } else { - JSONFormatter jf(true); - jf.open_object_section(command.c_str()); + f->open_object_section(command.c_str()); if (command == "config show") { - _conf->show_config(&jf); + _conf->show_config(f); } else if (command == "config set") { std::string var = args; size_t pos = var.find(' '); if (pos == string::npos) { - jf.dump_string("error", "syntax error: 'config set '"); + f->dump_string("error", "syntax error: 'config set '"); } else { std::string val = var.substr(pos+1); var.resize(pos); int r = _conf->set_val(var.c_str(), val.c_str()); if (r < 0) { - jf.dump_stream("error") << "error setting '" << var << "' to '" << val << "': " << cpp_strerror(r); + f->dump_stream("error") << "error setting '" << var << "' to '" << val << "': " << cpp_strerror(r); } else { ostringstream ss; _conf->apply_changes(&ss); - jf.dump_string("success", ss.str()); + f->dump_string("success", ss.str()); } } } else if (command == "config get") { @@ -202,9 +204,9 @@ void CephContext::do_command(std::string command, std::string args, bufferlist * char *tmp = buf; int r = _conf->get_val(args.c_str(), &tmp, sizeof(buf)); if (r < 0) { - jf.dump_stream("error") << "error getting '" << args << "': " << cpp_strerror(r); + f->dump_stream("error") << "error getting '" << args << "': " << cpp_strerror(r); } else { - jf.dump_string(args.c_str(), buf); + f->dump_string(args.c_str(), buf); } } else if (command == "log flush") { _log->flush(); @@ -218,11 +220,10 @@ void CephContext::do_command(std::string command, std::string args, bufferlist * else { assert(0 == "registered under wrong command?"); } - ostringstream ss; - jf.close_section(); - jf.flush(ss); - out->append(ss.str()); + f->close_section(); } + f->flush(*out); + delete f; lgeneric_dout(this, 1) << "do_command '" << command << "' '" << args << "' result is " << out->length() << " bytes" << dendl; }; diff --git a/src/common/ceph_context.h b/src/common/ceph_context.h index 1678680fa6db5..85618e3521934 100644 --- a/src/common/ceph_context.h +++ b/src/common/ceph_context.h @@ -97,7 +97,8 @@ class CephContext { /** * process an admin socket command */ - void do_command(std::string command, std::string args, bufferlist *out); + void do_command(std::string command, std::string args, std::string foramt, + bufferlist *out); /** * get a crypto handler diff --git a/src/common/fiemap.cc b/src/common/fiemap.cc deleted file mode 100644 index a1d5fbe9396ad..0000000000000 --- a/src/common/fiemap.cc +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (C) 2010 Canonical - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * - */ - -/* - * Author Colin Ian King, colin.king@canonical.com - */ - -#include -#include -#include -#include -#include - -#include -#include -#include - -#if defined(__linux__) -#include -#endif -#include "include/inttypes.h" -#include "include/fiemap.h" - -struct fiemap *read_fiemap(int fd) -{ - struct fiemap *fiemap; - struct fiemap *_realloc_fiemap = NULL; - int extents_size; - int r; - - if ((fiemap = (struct fiemap*)malloc(sizeof(struct fiemap))) == NULL) { - fprintf(stderr, "Out of memory allocating fiemap\n"); - return NULL; - } - memset(fiemap, 0, sizeof(struct fiemap)); - - fiemap->fm_start = 0; - fiemap->fm_length = ~0; /* Lazy */ - fiemap->fm_flags = 0; - fiemap->fm_extent_count = 0; - fiemap->fm_mapped_extents = 0; - - /* Find out how many extents there are */ - r = ioctl(fd, FS_IOC_FIEMAP, fiemap); - if (r < 0) { - goto done_err; - } - - if (!fiemap->fm_mapped_extents) { - goto done_err; - } - - /* Read in the extents */ - extents_size = sizeof(struct fiemap_extent) * (fiemap->fm_mapped_extents); - - /* Resize fiemap to allow us to read in the extents */ - - if ((_realloc_fiemap = (struct fiemap*)realloc(fiemap,sizeof(struct fiemap) + - extents_size)) == NULL) { - fprintf(stderr, "Out of memory allocating fiemap\n"); - goto done_err; - } else { - fiemap = _realloc_fiemap; - } - - memset(fiemap->fm_extents, 0, extents_size); - fiemap->fm_extent_count = fiemap->fm_mapped_extents; - fiemap->fm_mapped_extents = 0; - - if (ioctl(fd, FS_IOC_FIEMAP, fiemap) < 0) { - fprintf(stderr, "fiemap ioctl() failed\n"); - goto done_err; - } - - return fiemap; -done_err: - free(fiemap); - return NULL; -} - diff --git a/src/common/perf_counters.cc b/src/common/perf_counters.cc index 67a777497b38a..1dd4cdabd9d85 100644 --- a/src/common/perf_counters.cc +++ b/src/common/perf_counters.cc @@ -15,6 +15,7 @@ #include "common/perf_counters.h" #include "common/dout.h" #include "common/errno.h" +#include "common/Formatter.h" #include #include @@ -72,21 +73,20 @@ void PerfCountersCollection::clear() } } -void PerfCountersCollection::write_json_to_buf(bufferlist& bl, bool schema) +void PerfCountersCollection::dump_formatted(Formatter *f, bool schema) { Mutex::Locker lck(m_lock); - bl.append('{'); + f->open_object_section("perfcounter_collection"); perf_counters_set_t::iterator l = m_loggers.begin(); perf_counters_set_t::iterator l_end = m_loggers.end(); if (l != l_end) { while (true) { - (*l)->write_json_to_buf(bl, schema); + (*l)->dump_formatted(f, schema); if (++l == l_end) break; - bl.append(','); } } - bl.append('}'); + f->close_section(); } // --------------------------- @@ -203,34 +203,54 @@ utime_t PerfCounters::tget(int idx) const return utime_t(data.u64 / 1000000000ull, data.u64 % 1000000000ull); } -void PerfCounters::write_json_to_buf(bufferlist& bl, bool schema) +void PerfCounters::dump_formatted(Formatter *f, bool schema) { - char buf[512]; Mutex::Locker lck(m_lock); - snprintf(buf, sizeof(buf), "\"%s\":{", m_name.c_str()); - bl.append(buf); - + f->open_object_section(m_name.c_str()); perf_counter_data_vec_t::const_iterator d = m_data.begin(); perf_counter_data_vec_t::const_iterator d_end = m_data.end(); if (d == d_end) { - bl.append('}'); + f->close_section(); return; } while (true) { - const perf_counter_data_any_d &data(*d); - buf[0] = '\0'; - if (schema) - data.write_schema_json(buf, sizeof(buf)); - else - data.write_json(buf, sizeof(buf)); - - bl.append(buf); + if (schema) { + f->open_object_section(d->name); + f->dump_int("type", d->type); + f->close_section(); + } else { + if (d->type & PERFCOUNTER_LONGRUNAVG) { + f->open_object_section(d->name); + if (d->type & PERFCOUNTER_U64) { + f->dump_unsigned("avgcount", d->avgcount); + f->dump_unsigned("sum", d->u64); + } else if (d->type & PERFCOUNTER_TIME) { + f->dump_unsigned("avgcount", d->avgcount); + f->dump_format_unquoted("sum", "%"PRId64".%09"PRId64, + d->u64 / 1000000000ull, + d->u64 % 1000000000ull); + } else { + assert(0); + } + f->close_section(); + } else { + if (d->type & PERFCOUNTER_U64) { + f->dump_unsigned(d->name, d->u64); + } else if (d->type & PERFCOUNTER_TIME) { + f->dump_format_unquoted(d->name, "%"PRId64".%09"PRId64, + d->u64 / 1000000000ull, + d->u64 % 1000000000ull); + } else { + assert(0); + } + } + } + if (++d == d_end) break; - bl.append(','); } - bl.append('}'); + f->close_section(); } const std::string &PerfCounters::get_name() const @@ -258,42 +278,6 @@ PerfCounters::perf_counter_data_any_d::perf_counter_data_any_d() { } -void PerfCounters::perf_counter_data_any_d::write_schema_json(char *buf, size_t buf_sz) const -{ - snprintf(buf, buf_sz, "\"%s\":{\"type\":%d}", name, type); -} - -void PerfCounters::perf_counter_data_any_d::write_json(char *buf, size_t buf_sz) const -{ - if (type & PERFCOUNTER_LONGRUNAVG) { - if (type & PERFCOUNTER_U64) { - snprintf(buf, buf_sz, "\"%s\":{\"avgcount\":%" PRId64 "," - "\"sum\":%" PRId64 "}", - name, avgcount, u64); - } - else if (type & PERFCOUNTER_TIME) { - snprintf(buf, buf_sz, "\"%s\":{\"avgcount\":%" PRId64 "," - "\"sum\":%llu.%09llu}", - name, avgcount, u64 / 1000000000ull, u64 % 1000000000ull); - } - else { - assert(0); - } - } - else { - if (type & PERFCOUNTER_U64) { - snprintf(buf, buf_sz, "\"%s\":%" PRId64, - name, u64); - } - else if (type & PERFCOUNTER_TIME) { - snprintf(buf, buf_sz, "\"%s\":%llu.%09llu", name, u64 / 1000000000ull, u64 % 1000000000ull); - } - else { - assert(0); - } - } -} - PerfCountersBuilder::PerfCountersBuilder(CephContext *cct, const std::string &name, int first, int last) : m_perf_counters(new PerfCounters(cct, name, first, last)) diff --git a/src/common/perf_counters.h b/src/common/perf_counters.h index 269a32f2c46f8..ec10f9a928286 100644 --- a/src/common/perf_counters.h +++ b/src/common/perf_counters.h @@ -76,7 +76,7 @@ class PerfCounters void tinc(int idx, utime_t v); utime_t tget(int idx) const; - void write_json_to_buf(ceph::bufferlist& bl, bool schema); + void dump_formatted(ceph::Formatter *f, bool schema); const std::string& get_name() const; void set_name(std::string s) { @@ -136,7 +136,7 @@ class PerfCountersCollection void add(class PerfCounters *l); void remove(class PerfCounters *l); void clear(); - void write_json_to_buf(ceph::bufferlist& bl, bool schema); + void dump_formatted(ceph::Formatter *f, bool schema); private: CephContext *m_cct; diff --git a/src/include/fiemap.h b/src/include/fiemap.h deleted file mode 100644 index 846adb155ff6a..0000000000000 --- a/src/include/fiemap.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef __CEPH_FIEMAP_H -#define __CEPH_FIEMAP_H - -#include "acconfig.h" - -/* - * the header is missing on most systems. for the time being at - * least, include our own copy in the repo. - */ -#ifdef HAVE_FIEMAP_H -# include -#else -# include "linux_fiemap.h" -#endif - -#if defined(__linux__) -#include -#elif defined(__FreeBSD__) -#include -#endif -#ifndef FS_IOC_FIEMAP -# define FS_IOC_FIEMAP _IOWR('f', 11, struct fiemap) -#endif - -extern "C" struct fiemap *read_fiemap(int fd); - -#endif diff --git a/src/librados/RadosClient.cc b/src/librados/RadosClient.cc index f68125fb8c071..e8dd019af3a28 100644 --- a/src/librados/RadosClient.cc +++ b/src/librados/RadosClient.cc @@ -642,6 +642,10 @@ int librados::RadosClient::osd_command(int osd, vector& cmd, bool done; int ret; tid_t tid; + + if (osd < 0) + return -EINVAL; + lock.Lock(); // XXX do anything with tid? int r = objecter->osd_command(osd, cmd, inbl, &tid, poutbl, prs, diff --git a/src/messages/MMonScrub.h b/src/messages/MMonScrub.h index ab4588f4a76e2..b16728bcdd555 100644 --- a/src/messages/MMonScrub.h +++ b/src/messages/MMonScrub.h @@ -31,7 +31,7 @@ class MMonScrub : public Message switch (op) { case OP_SCRUB: return "scrub"; case OP_RESULT: return "result"; - default: assert("unknown op type"); return NULL; + default: assert(0 == "unknown op type"); return NULL; } } diff --git a/src/messages/MMonSync.h b/src/messages/MMonSync.h index a5415a8f45157..48229d15bcc0b 100644 --- a/src/messages/MMonSync.h +++ b/src/messages/MMonSync.h @@ -49,7 +49,7 @@ class MMonSync : public Message case OP_CHUNK: return "chunk"; case OP_LAST_CHUNK: return "last_chunk"; case OP_NO_COOKIE: return "no_cookie"; - default: assert("unknown op type"); return NULL; + default: assert(0 == "unknown op type"); return NULL; } } diff --git a/src/mon/DataHealthService.cc b/src/mon/DataHealthService.cc index a55e8c392e22e..6e8aa313a3616 100644 --- a/src/mon/DataHealthService.cc +++ b/src/mon/DataHealthService.cc @@ -93,7 +93,7 @@ health_status_t DataHealthService::get_health( } if (f) { - f->open_object_section(mon_name.c_str()); + f->open_object_section("mon"); f->dump_string("name", mon_name.c_str()); f->dump_int("kb_total", stats.kb_total); f->dump_int("kb_used", stats.kb_used); diff --git a/src/mon/MonCap.cc b/src/mon/MonCap.cc index 7ac8d142d87af..b03873ad7ddc6 100644 --- a/src/mon/MonCap.cc +++ b/src/mon/MonCap.cc @@ -133,6 +133,7 @@ void MonCapGrant::expand_profile(entity_name_t name) const if (profile == "mds") { profile_grants.push_back(MonCapGrant("mds", MON_CAP_ALL)); profile_grants.push_back(MonCapGrant("mon", MON_CAP_R)); + profile_grants.push_back(MonCapGrant("osd", MON_CAP_R)); profile_grants.push_back(MonCapGrant("log", MON_CAP_W)); } if (profile == "osd" || profile == "mds" || profile == "mon") { diff --git a/src/mon/Monitor.cc b/src/mon/Monitor.cc index bf500dff218fc..119ef740aa834 100644 --- a/src/mon/Monitor.cc +++ b/src/mon/Monitor.cc @@ -225,21 +225,20 @@ class AdminHook : public AdminSocketHook { Monitor *mon; public: AdminHook(Monitor *m) : mon(m) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; - mon->do_admin_command(command, args, ss); + mon->do_admin_command(command, args, format, ss); out.append(ss); return true; } }; -void Monitor::do_admin_command(string command, string args, ostream& ss) +void Monitor::do_admin_command(string command, string args, string format, + ostream& ss) { Mutex::Locker l(lock); - map cmdmap; - string format; - cmd_getval(g_ceph_context, cmdmap, "format", format, string("plain")); boost::scoped_ptr f(new_formatter(format)); if (command == "mon_status") diff --git a/src/mon/Monitor.h b/src/mon/Monitor.h index 82b08816702b0..bed48ecee3436 100644 --- a/src/mon/Monitor.h +++ b/src/mon/Monitor.h @@ -745,7 +745,8 @@ class Monitor : public Dispatcher { int write_fsid(); int write_fsid(MonitorDBStore::Transaction &t); - void do_admin_command(std::string command, std::string args, ostream& ss); + void do_admin_command(std::string command, std::string args, + std::string format, ostream& ss); private: // don't allow copying diff --git a/src/mon/PGMonitor.cc b/src/mon/PGMonitor.cc index 648a8fe238466..d86cbe70c19fd 100644 --- a/src/mon/PGMonitor.cc +++ b/src/mon/PGMonitor.cc @@ -255,6 +255,12 @@ void PGMonitor::update_from_paxos(bool *need_bootstrap) update_logger(); } +void PGMonitor::on_upgrade() +{ + dout(1) << __func__ << " discarding in-core PGMap" << dendl; + pg_map = PGMap(); +} + void PGMonitor::upgrade_format() { unsigned current = 1; diff --git a/src/mon/PGMonitor.h b/src/mon/PGMonitor.h index e8e1b4210aaaa..44015395e943e 100644 --- a/src/mon/PGMonitor.h +++ b/src/mon/PGMonitor.h @@ -60,6 +60,7 @@ class PGMonitor : public PaxosService { void create_initial(); void update_from_paxos(bool *need_bootstrap); void upgrade_format(); + void on_upgrade(); void post_paxos_update(); void handle_osd_timeouts(); void create_pending(); // prepare a new pending diff --git a/src/mon/PaxosService.cc b/src/mon/PaxosService.cc index d6e67a1c4b4c0..1b21689863bc6 100644 --- a/src/mon/PaxosService.cc +++ b/src/mon/PaxosService.cc @@ -114,7 +114,13 @@ void PaxosService::refresh(bool *need_bootstrap) // update cached versions cached_first_committed = mon->store->get(get_service_name(), first_committed_name); cached_last_committed = mon->store->get(get_service_name(), last_committed_name); - format_version = get_value("format_version"); + + version_t new_format = get_value("format_version"); + if (new_format != format_version) { + dout(1) << __func__ << " upgraded, format " << format_version << " -> " << new_format << dendl; + on_upgrade(); + } + format_version = new_format; dout(10) << __func__ << dendl; diff --git a/src/mon/PaxosService.h b/src/mon/PaxosService.h index 74d5a90494c87..5321bebcacefc 100644 --- a/src/mon/PaxosService.h +++ b/src/mon/PaxosService.h @@ -458,6 +458,11 @@ class PaxosService { */ virtual void upgrade_format() { } + /** + * this is called when we detect the store has just upgraded underneath us + */ + virtual void on_upgrade() {} + /** * Called when the Paxos system enters a Leader election. * diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index 28f81b7547fd0..108a857ab9fc9 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -36,7 +36,7 @@ #endif #include "include/compat.h" -#include "include/fiemap.h" +#include "include/linux_fiemap.h" #include "common/xattr.h" #include "chain_xattr.h" diff --git a/src/osd/ClassHandler.cc b/src/osd/ClassHandler.cc index 3cc319efabeef..a9a920ba078e9 100644 --- a/src/osd/ClassHandler.cc +++ b/src/osd/ClassHandler.cc @@ -52,7 +52,7 @@ int ClassHandler::open_all_classes() if (strlen(pde->d_name) > sizeof(CLS_PREFIX) - 1 + sizeof(CLS_SUFFIX) - 1 && strncmp(pde->d_name, CLS_PREFIX, sizeof(CLS_PREFIX) - 1) == 0 && strcmp(pde->d_name + strlen(pde->d_name) - (sizeof(CLS_SUFFIX) - 1), CLS_SUFFIX) == 0) { - char cname[strlen(pde->d_name)]; + char cname[PATH_MAX + 1]; strcpy(cname, pde->d_name + sizeof(CLS_PREFIX) - 1); cname[strlen(cname) - (sizeof(CLS_SUFFIX) - 1)] = '\0'; dout(10) << __func__ << " found " << cname << dendl; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 58020d0c3226e..e3a7c227e1553 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -997,44 +997,44 @@ class OSDSocketHook : public AdminSocketHook { OSD *osd; public: OSDSocketHook(OSD *o) : osd(o) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; - bool r = osd->asok_command(command, args, ss); + bool r = osd->asok_command(command, args, format, ss); out.append(ss); return r; } }; -bool OSD::asok_command(string command, string args, ostream& ss) +bool OSD::asok_command(string command, string args, string format, ostream& ss) { + if (format == "") + format = "json-pretty"; + Formatter *f = new_formatter(format); if (command == "dump_ops_in_flight") { - op_tracker.dump_ops_in_flight(ss); + op_tracker.dump_ops_in_flight(f); } else if (command == "dump_historic_ops") { - op_tracker.dump_historic_ops(ss); + op_tracker.dump_historic_ops(f); } else if (command == "dump_op_pq_state") { - JSONFormatter f(true); - f.open_object_section("pq"); - op_wq.dump(&f); - f.close_section(); - f.flush(ss); + f->open_object_section("pq"); + op_wq.dump(f); + f->close_section(); } else if (command == "dump_blacklist") { list > bl; OSDMapRef curmap = service.get_osdmap(); - JSONFormatter f(true); - f.open_array_section("blacklist"); + f->open_array_section("blacklist"); curmap->get_blacklist(&bl); for (list >::iterator it = bl.begin(); it != bl.end(); ++it) { - f.open_array_section("entry"); - f.open_object_section("entity_addr_t"); - it->first.dump(&f); - f.close_section(); //entity_addr_t - it->second.localtime(f.dump_stream("expire_time")); - f.close_section(); //entry - } - f.close_section(); //blacklist - f.flush(ss); + f->open_array_section("entry"); + f->open_object_section("entity_addr_t"); + it->first.dump(f); + f->close_section(); //entity_addr_t + it->second.localtime(f->dump_stream("expire_time")); + f->close_section(); //entry + } + f->close_section(); //blacklist } else if (command == "dump_watchers") { list watchers; osd_lock.Lock(); @@ -1052,35 +1052,35 @@ bool OSD::asok_command(string command, string args, ostream& ss) } osd_lock.Unlock(); - JSONFormatter f(true); - f.open_array_section("watchers"); + f->open_array_section("watchers"); for (list::iterator it = watchers.begin(); it != watchers.end(); ++it) { - f.open_array_section("watch"); + f->open_array_section("watch"); - f.dump_string("namespace", it->obj.nspace); - f.dump_string("object", it->obj.oid.name); + f->dump_string("namespace", it->obj.nspace); + f->dump_string("object", it->obj.oid.name); - f.open_object_section("entity_name"); - it->wi.name.dump(&f); - f.close_section(); //entity_name_t + f->open_object_section("entity_name"); + it->wi.name.dump(f); + f->close_section(); //entity_name_t - f.dump_int("cookie", it->wi.cookie); - f.dump_int("timeout", it->wi.timeout_seconds); + f->dump_int("cookie", it->wi.cookie); + f->dump_int("timeout", it->wi.timeout_seconds); - f.open_object_section("entity_addr_t"); - it->wi.addr.dump(&f); - f.close_section(); //entity_addr_t + f->open_object_section("entity_addr_t"); + it->wi.addr.dump(f); + f->close_section(); //entity_addr_t - f.close_section(); //watch + f->close_section(); //watch } - f.close_section(); //watches - f.flush(ss); + f->close_section(); //watches } else { assert(0 == "broken asok registration"); } + f->flush(ss); + delete f; return true; } @@ -1089,7 +1089,8 @@ class TestOpsSocketHook : public AdminSocketHook { ObjectStore *store; public: TestOpsSocketHook(OSDService *s, ObjectStore *st) : service(s), store(st) {} - bool call(std::string command, std::string args, bufferlist& out) { + bool call(std::string command, std::string args, std::string format, + bufferlist& out) { stringstream ss; test_ops(service, store, command, args, ss); out.append(ss); @@ -1166,8 +1167,11 @@ int OSD::init() class_handler = new ClassHandler(); cls_initialize(class_handler); - if (g_conf->osd_open_classes_on_start) - class_handler->open_all_classes(); + if (g_conf->osd_open_classes_on_start) { + int r = class_handler->open_all_classes(); + if (r) + dout(1) << "warning: got an error loading one or more classes: " << cpp_strerror(r) << dendl; + } // load up "current" osdmap assert_warn(!osdmap); @@ -3815,51 +3819,90 @@ void OSD::handle_command(MCommand *m) struct OSDCommand { string cmdstring; string helpstring; + string module; + string perm; + string availability; } osd_commands[] = { -#define COMMAND(parsesig, helptext) \ - {parsesig, helptext}, +#define COMMAND(parsesig, helptext, module, perm, availability) \ + {parsesig, helptext, module, perm, availability}, // yes, these are really pg commands, but there's a limit to how -// much work it's worth. The OSD returns all of them. +// much work it's worth. The OSD returns all of them. Make this +// form (pg ) valid only for the cli. +// Rest uses "tell " COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=query", \ - "show details of a specific pg") + "show details of a specific pg", "osd", "r", "cli") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=mark_unfound_lost " \ "name=mulcmd,type=CephChoices,strings=revert", \ - "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available") + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", + "osd", "rw", "cli") COMMAND("pg " \ "name=pgid,type=CephPgid " \ "name=cmd,type=CephChoices,strings=list_missing " \ "name=offset,type=CephString,req=false", - "list missing objects on this pg, perhaps starting at an offset given in JSON") + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "r", "cli") + +// new form: tell for both cli and rest + +COMMAND("query", + "show details of a specific pg", "osd", "r", "cli,rest") +COMMAND("mark_unfound_lost " \ + "name=mulcmd,type=CephChoices,strings=revert", \ + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", + "osd", "rw", "cli,rest") +COMMAND("list_missing " \ + "name=offset,type=CephString,req=false", + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "r", "cli,rest") -COMMAND("version", "report version of OSD") +// tell commands. Validation of osd.n must be special-cased in client + +// tell commands. Validation of osd.n must be special-cased in client +COMMAND("version", "report version of OSD", "osd", "r", "cli,rest") COMMAND("injectargs " \ "name=injected_args,type=CephString,n=N", - "inject configuration arguments into running OSD") + "inject configuration arguments into running OSD", + "osd", "rw", "cli,rest") COMMAND("bench " \ "name=count,type=CephInt,req=false " \ "name=size,type=CephInt,req=false ", \ "OSD benchmark: write -byte objects, " \ - "(default 1G size 4MB). Results in log.") -COMMAND("flush_pg_stats", "flush pg stats") -COMMAND("debug dump_missing " \ + "(default 1G size 4MB). Results in log.", + "osd", "rw", "cli,rest") +COMMAND("flush_pg_stats", "flush pg stats", "osd", "rw", "cli,rest") +COMMAND("debug_dump_missing " \ "name=filename,type=CephFilepath", - "dump missing objects to a named file") + "dump missing objects to a named file", "osd", "r", "cli,rest") COMMAND("debug kick_recovery_wq " \ "name=delay,type=CephInt,range=0", - "set osd_recovery_delay_start to ") + "set osd_recovery_delay_start to ", "osd", "rw", "cli,rest") COMMAND("cpu_profiler " \ "name=arg,type=CephChoices,strings=status|flush", - "run cpu profiling on daemon") -COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics") -COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics") - + "run cpu profiling on daemon", "osd", "rw", "cli,rest") +COMMAND("dump_pg_recovery_stats", "dump pg recovery statistics", + "osd", "r", "cli,rest") +COMMAND("reset_pg_recovery_stats", "reset pg recovery statistics", + "osd", "rw", "cli,rest") + +// experiment: restate pg commands as "tell ". Validation of +// pgid must be special-cased in client. +COMMAND("query", + "show details of a specific pg", "osd", "r", "cli,rest") +COMMAND("mark_unfound_lost revert " \ + "name=mulcmd,type=CephChoices,strings=revert", \ + "mark all unfound objects in this pg as lost, either removing or reverting to a prior version if one is available", + "osd", "rw", "cli,rest") +COMMAND("list_missing " \ + "name=offset,type=CephString,req=false", + "list missing objects on this pg, perhaps starting at an offset given in JSON", + "osd", "rw", "cli,rest") }; void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist& data) @@ -3873,6 +3916,9 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist map cmdmap; string prefix; + string format; + string pgidstr; + boost::scoped_ptr f; if (cmd.empty()) { ss << "no command given"; @@ -3895,8 +3941,8 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist ostringstream secname; secname << "cmd" << setfill('0') << std::setw(3) << cmdnum; - dump_cmd_and_help_to_json(f, secname.str(), - cp->cmdstring, cp->helpstring); + dump_cmddesc_to_json(f, secname.str(), cp->cmdstring, cp->helpstring, + cp->module, cp->perm, cp->availability); cmdnum++; } f->close_section(); // command_descriptions @@ -3907,8 +3953,18 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist goto out; } + cmd_getval(g_ceph_context, cmdmap, "format", format); + f.reset(new_formatter(format)); + if (prefix == "version") { - ds << pretty_version_to_str(); + if (f) { + f->open_object_section("version"); + f->dump_string("version", pretty_version_to_str()); + f->close_section(); + f->flush(ds); + } else { + ds << pretty_version_to_str(); + } goto out; } else if (prefix == "injectargs") { @@ -3928,9 +3984,16 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist osd_lock.Lock(); } - else if (prefix == "pg") { + // either 'pg ' or + // 'tell ' (which comes in without any of that prefix)? + + else if (prefix == "pg" || + (cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr) && + (prefix == "query" || + prefix == "mark_unfound_lost" || + prefix == "list_missing") + )) { pg_t pgid; - string pgidstr; if (!cmd_getval(g_ceph_context, cmdmap, "pgid", pgidstr)) { ss << "no pgid specified"; @@ -3939,14 +4002,15 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist ss << "couldn't parse pgid '" << pgidstr << "'"; r = -EINVAL; } else { - vector args; - cmd_getval(g_ceph_context, cmdmap, "args", args); PG *pg = _lookup_lock_pg(pgid); if (!pg) { ss << "i don't have pgid " << pgid; r = -ENOENT; } else { - r = pg->do_command(cmd, ss, data, odata); + // simulate pg cmd= for pg->do-command + if (prefix != "pg") + cmd_putval(g_ceph_context, cmdmap, "cmd", prefix); + r = pg->do_command(cmdmap, ss, data, odata); pg->unlock(); } } @@ -3985,9 +4049,18 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist store->queue_transaction(NULL, cleanupt); uint64_t rate = (double)count / (end - start); - ss << "bench: wrote " << prettybyte_t(count) - << " in blocks of " << prettybyte_t(bsize) << " in " - << (end-start) << " sec at " << prettybyte_t(rate) << "/sec"; + if (f) { + f->open_object_section("osd_bench_results"); + f->dump_int("bytes_written", count); + f->dump_int("blocksize", bsize); + f->dump_float("bytes_per_sec", rate); + f->close_section(); + f->flush(ss); + } else { + ss << "bench: wrote " << prettybyte_t(count) + << " in blocks of " << prettybyte_t(bsize) << " in " + << (end-start) << " sec at " << prettybyte_t(rate) << "/sec"; + } } else if (prefix == "flush_pg_stats") { @@ -4084,8 +4157,13 @@ void OSD::do_command(Connection *con, tid_t tid, vector& cmd, bufferlist else if (prefix == "dump_pg_recovery_stats") { stringstream s; - pg_recovery_stats.dump(s); - ds << "dump pg recovery stats: " << s.str(); + if (f) { + pg_recovery_stats.dump_formatted(f.get()); + f->flush(ds); + } else { + pg_recovery_stats.dump(s); + ds << "dump pg recovery stats: " << s.str(); + } } else if (prefix == "reset_pg_recovery_stats") { diff --git a/src/osd/OSD.h b/src/osd/OSD.h index f9ceaf81bf391..5bcff7442d7a6 100644 --- a/src/osd/OSD.h +++ b/src/osd/OSD.h @@ -622,7 +622,7 @@ class OSD : public Dispatcher, // asok friend class OSDSocketHook; class OSDSocketHook *asok_hook; - bool asok_command(string command, string args, ostream& ss); + bool asok_command(string command, string args, string format, ostream& ss); public: ClassHandler *class_handler; diff --git a/src/osd/OpRequest.cc b/src/osd/OpRequest.cc index 3b8a8714d9221..a6cdc9ecffb5e 100644 --- a/src/osd/OpRequest.cc +++ b/src/osd/OpRequest.cc @@ -76,31 +76,27 @@ void OpHistory::dump_ops(utime_t now, Formatter *f) f->close_section(); } -void OpTracker::dump_historic_ops(ostream &ss) +void OpTracker::dump_historic_ops(Formatter *f) { - JSONFormatter jf(true); Mutex::Locker locker(ops_in_flight_lock); utime_t now = ceph_clock_now(g_ceph_context); - history.dump_ops(now, &jf); - jf.flush(ss); + history.dump_ops(now, f); } -void OpTracker::dump_ops_in_flight(ostream &ss) +void OpTracker::dump_ops_in_flight(Formatter *f) { - JSONFormatter jf(true); Mutex::Locker locker(ops_in_flight_lock); - jf.open_object_section("ops_in_flight"); // overall dump - jf.dump_int("num_ops", ops_in_flight.size()); - jf.open_array_section("ops"); // list of OpRequests + f->open_object_section("ops_in_flight"); // overall dump + f->dump_int("num_ops", ops_in_flight.size()); + f->open_array_section("ops"); // list of OpRequests utime_t now = ceph_clock_now(g_ceph_context); for (xlist::iterator p = ops_in_flight.begin(); !p.end(); ++p) { - jf.open_object_section("op"); - (*p)->dump(now, &jf); - jf.close_section(); // this OpRequest + f->open_object_section("op"); + (*p)->dump(now, f); + f->close_section(); // this OpRequest } - jf.close_section(); // list of OpRequests - jf.close_section(); // overall dump - jf.flush(ss); + f->close_section(); // list of OpRequests + f->close_section(); // overall dump } void OpTracker::register_inflight_op(xlist::item *i) diff --git a/src/osd/OpRequest.h b/src/osd/OpRequest.h index 47b050b853815..a2014472432da 100644 --- a/src/osd/OpRequest.h +++ b/src/osd/OpRequest.h @@ -59,8 +59,8 @@ class OpTracker { public: OpTracker() : seq(0), ops_in_flight_lock("OpTracker mutex") {} - void dump_ops_in_flight(std::ostream& ss); - void dump_historic_ops(std::ostream& ss); + void dump_ops_in_flight(Formatter *f); + void dump_historic_ops(Formatter *f); void register_inflight_op(xlist::item *i); void unregister_inflight_op(OpRequest *i); diff --git a/src/osd/PG.h b/src/osd/PG.h index 819c9c62f62e3..10e9a2544a9b0 100644 --- a/src/osd/PG.h +++ b/src/osd/PG.h @@ -43,8 +43,10 @@ #include "msg/Messenger.h" #include "messages/MOSDRepScrub.h" #include "messages/MOSDPGLog.h" +#include "common/cmdparse.h" #include "common/tracked_int_ptr.hpp" #include "common/WorkQueue.h" +#include "include/str_list.h" #include #include @@ -108,10 +110,36 @@ struct PGRecoveryStats { << i.total_time << "\t" << i.min_time << "\t" << i.max_time << "\t" << p->first << "\n"; - } } + void dump_formatted(Formatter *f) { + Mutex::Locker l(lock); + f->open_array_section("pg_recovery_stats"); + for (map::iterator p = info.begin(); + p != info.end(); ++p) { + per_state_info& i = p->second; + f->open_object_section("recovery_state"); + f->dump_int("enter", i.enter); + f->dump_int("exit", i.exit); + f->dump_int("events", i.events); + f->dump_stream("event_time") << i.event_time; + f->dump_stream("total_time") << i.total_time; + f->dump_stream("min_time") << i.min_time; + f->dump_stream("max_time") << i.max_time; + vector states; + get_str_vec(p->first, "/", states); + f->open_array_section("nested_states"); + for (vector::iterator st = states.begin(); + st != states.end(); ++st) { + f->dump_string("state", *st); + } + f->close_section(); + f->close_section(); + } + f->close_section(); + } + void log_enter(const char *s) { Mutex::Locker l(lock); info[s].enter++; @@ -1786,7 +1814,7 @@ class PG { virtual void do_push_reply(OpRequestRef op) = 0; virtual void snap_trimmer() = 0; - virtual int do_command(vector& cmd, ostream& ss, + virtual int do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, bufferlist& odata) = 0; virtual bool same_for_read_since(epoch_t e) = 0; diff --git a/src/osd/ReplicatedPG.cc b/src/osd/ReplicatedPG.cc index 298d38d6ace64..658ea7cb74633 100644 --- a/src/osd/ReplicatedPG.cc +++ b/src/osd/ReplicatedPG.cc @@ -268,57 +268,43 @@ int ReplicatedPG::get_pgls_filter(bufferlist::iterator& iter, PGLSFilter **pfilt // ========================================================== -int ReplicatedPG::do_command(vector& cmd, ostream& ss, +int ReplicatedPG::do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, bufferlist& odata) { const pg_missing_t &missing = pg_log.get_missing(); - map cmdmap; string prefix; + string format; - if (cmd.empty()) { - ss << "no command given"; - return -EINVAL; - } - - stringstream ss2; - if (!cmdmap_from_json(cmd, &cmdmap, ss2)) { - ss << ss2.str(); - return -EINVAL; - } - - cmd_getval(g_ceph_context, cmdmap, "prefix", prefix); - if (prefix != "pg") { - ss << "ReplicatedPG::do_command: not pg command"; - return -EINVAL; - } + cmd_getval(g_ceph_context, cmdmap, "format", format); + boost::scoped_ptr f(new_formatter(format)); + // demand that we have a formatter + if (!f) + f.reset(new_formatter("json")); string command; cmd_getval(g_ceph_context, cmdmap, "cmd", command); if (command == "query") { - JSONFormatter jsf(true); - jsf.open_object_section("pg"); - jsf.dump_string("state", pg_state_string(get_state())); - jsf.dump_unsigned("epoch", get_osdmap()->get_epoch()); - jsf.open_array_section("up"); + f->open_object_section("pg"); + f->dump_string("state", pg_state_string(get_state())); + f->dump_unsigned("epoch", get_osdmap()->get_epoch()); + f->open_array_section("up"); for (vector::iterator p = up.begin(); p != up.end(); ++p) - jsf.dump_unsigned("osd", *p); - jsf.close_section(); - jsf.open_array_section("acting"); + f->dump_unsigned("osd", *p); + f->close_section(); + f->open_array_section("acting"); for (vector::iterator p = acting.begin(); p != acting.end(); ++p) - jsf.dump_unsigned("osd", *p); - jsf.close_section(); - jsf.open_object_section("info"); - info.dump(&jsf); - jsf.close_section(); - - jsf.open_array_section("recovery_state"); - handle_query_state(&jsf); - jsf.close_section(); - - jsf.close_section(); - stringstream dss; - jsf.flush(dss); - odata.append(dss); + f->dump_unsigned("osd", *p); + f->close_section(); + f->open_object_section("info"); + info.dump(f.get()); + f->close_section(); + + f->open_array_section("recovery_state"); + handle_query_state(f.get()); + f->close_section(); + + f->close_section(); + f->flush(odata); return 0; } else if (command == "mark_unfound_lost") { @@ -352,7 +338,6 @@ int ReplicatedPG::do_command(vector& cmd, ostream& ss, return 0; } else if (command == "list_missing") { - JSONFormatter jf(true); hobject_t offset; string offset_json; if (cmd_getval(g_ceph_context, cmdmap, "offset", offset_json)) { @@ -366,50 +351,48 @@ int ReplicatedPG::do_command(vector& cmd, ostream& ss, return -EINVAL; } } - jf.open_object_section("missing"); + f->open_object_section("missing"); { - jf.open_object_section("offset"); - offset.dump(&jf); - jf.close_section(); + f->open_object_section("offset"); + offset.dump(f.get()); + f->close_section(); } - jf.dump_int("num_missing", missing.num_missing()); - jf.dump_int("num_unfound", get_num_unfound()); + f->dump_int("num_missing", missing.num_missing()); + f->dump_int("num_unfound", get_num_unfound()); map::const_iterator p = missing.missing.upper_bound(offset); { - jf.open_array_section("objects"); + f->open_array_section("objects"); int32_t num = 0; bufferlist bl; while (p != missing.missing.end() && num < g_conf->osd_command_max_records) { - jf.open_object_section("object"); + f->open_object_section("object"); { - jf.open_object_section("oid"); - p->first.dump(&jf); - jf.close_section(); + f->open_object_section("oid"); + p->first.dump(f.get()); + f->close_section(); } - p->second.dump(&jf); // have, need keys + p->second.dump(f.get()); // have, need keys { - jf.open_array_section("locations"); + f->open_array_section("locations"); map >::iterator q = missing_loc.find(p->first); if (q != missing_loc.end()) for (set::iterator r = q->second.begin(); r != q->second.end(); ++r) - jf.dump_int("osd", *r); - jf.close_section(); + f->dump_int("osd", *r); + f->close_section(); } - jf.close_section(); + f->close_section(); ++p; num++; } - jf.close_section(); + f->close_section(); } - jf.dump_int("more", p != missing.missing.end()); - jf.close_section(); - stringstream jss; - jf.flush(jss); - odata.append(jss); + f->dump_int("more", p != missing.missing.end()); + f->close_section(); + f->flush(odata); return 0; }; - ss << "unknown command " << cmd; + ss << "unknown pg command " << prefix; return -EINVAL; } diff --git a/src/osd/ReplicatedPG.h b/src/osd/ReplicatedPG.h index 9dafe23faa1fb..7b70b4381ea4f 100644 --- a/src/osd/ReplicatedPG.h +++ b/src/osd/ReplicatedPG.h @@ -17,6 +17,7 @@ #include #include "include/assert.h" +#include "common/cmdparse.h" #include "PG.h" #include "OSD.h" @@ -930,7 +931,8 @@ class ReplicatedPG : public PG { const hobject_t& ioid); ~ReplicatedPG() {} - int do_command(vector& cmd, ostream& ss, bufferlist& idata, bufferlist& odata); + int do_command(cmdmap_t cmdmap, ostream& ss, bufferlist& idata, + bufferlist& odata); void do_op(OpRequestRef op); bool pg_op_must_wait(MOSDOp *op); diff --git a/src/osdc/Objecter.cc b/src/osdc/Objecter.cc index 9933f853f8f73..aefe74088e933 100644 --- a/src/osdc/Objecter.cc +++ b/src/osdc/Objecter.cc @@ -2166,154 +2166,154 @@ void Objecter::dump_active() } } -void Objecter::dump_requests(Formatter& fmt) const +void Objecter::dump_requests(Formatter *fmt) const { assert(client_lock.is_locked()); - fmt.open_object_section("requests"); + fmt->open_object_section("requests"); dump_ops(fmt); dump_linger_ops(fmt); dump_pool_ops(fmt); dump_pool_stat_ops(fmt); dump_statfs_ops(fmt); dump_command_ops(fmt); - fmt.close_section(); // requests object + fmt->close_section(); // requests object } -void Objecter::dump_ops(Formatter& fmt) const +void Objecter::dump_ops(Formatter *fmt) const { - fmt.open_array_section("ops"); + fmt->open_array_section("ops"); for (map::const_iterator p = ops.begin(); p != ops.end(); ++p) { Op *op = p->second; - fmt.open_object_section("op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("pg") << op->pgid; - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.dump_stream("last_sent") << op->stamp; - fmt.dump_int("attempts", op->attempts); - fmt.dump_stream("object_id") << op->oid; - fmt.dump_stream("object_locator") << op->oloc; - fmt.dump_stream("snapid") << op->snapid; - fmt.dump_stream("snap_context") << op->snapc; - fmt.dump_stream("mtime") << op->mtime; - - fmt.open_array_section("osd_ops"); + fmt->open_object_section("op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("pg") << op->pgid; + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->dump_stream("last_sent") << op->stamp; + fmt->dump_int("attempts", op->attempts); + fmt->dump_stream("object_id") << op->oid; + fmt->dump_stream("object_locator") << op->oloc; + fmt->dump_stream("snapid") << op->snapid; + fmt->dump_stream("snap_context") << op->snapc; + fmt->dump_stream("mtime") << op->mtime; + + fmt->open_array_section("osd_ops"); for (vector::const_iterator it = op->ops.begin(); it != op->ops.end(); ++it) { - fmt.dump_stream("osd_op") << *it; + fmt->dump_stream("osd_op") << *it; } - fmt.close_section(); // osd_ops array + fmt->close_section(); // osd_ops array - fmt.close_section(); // op object + fmt->close_section(); // op object } - fmt.close_section(); // ops array + fmt->close_section(); // ops array } -void Objecter::dump_linger_ops(Formatter& fmt) const +void Objecter::dump_linger_ops(Formatter *fmt) const { - fmt.open_array_section("linger_ops"); + fmt->open_array_section("linger_ops"); for (map::const_iterator p = linger_ops.begin(); p != linger_ops.end(); ++p) { LingerOp *op = p->second; - fmt.open_object_section("linger_op"); - fmt.dump_unsigned("linger_id", op->linger_id); - fmt.dump_stream("pg") << op->pgid; - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.dump_stream("object_id") << op->oid; - fmt.dump_stream("object_locator") << op->oloc; - fmt.dump_stream("snapid") << op->snap; - fmt.dump_stream("registering") << op->snap; - fmt.dump_stream("registered") << op->snap; - fmt.close_section(); // linger_op object + fmt->open_object_section("linger_op"); + fmt->dump_unsigned("linger_id", op->linger_id); + fmt->dump_stream("pg") << op->pgid; + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->dump_stream("object_id") << op->oid; + fmt->dump_stream("object_locator") << op->oloc; + fmt->dump_stream("snapid") << op->snap; + fmt->dump_stream("registering") << op->snap; + fmt->dump_stream("registered") << op->snap; + fmt->close_section(); // linger_op object } - fmt.close_section(); // linger_ops array + fmt->close_section(); // linger_ops array } -void Objecter::dump_command_ops(Formatter& fmt) const +void Objecter::dump_command_ops(Formatter *fmt) const { - fmt.open_array_section("command_ops"); + fmt->open_array_section("command_ops"); for (map::const_iterator p = command_ops.begin(); p != command_ops.end(); ++p) { CommandOp *op = p->second; - fmt.open_object_section("command_op"); - fmt.dump_unsigned("command_id", op->tid); - fmt.dump_int("osd", op->session ? op->session->osd : -1); - fmt.open_array_section("command"); + fmt->open_object_section("command_op"); + fmt->dump_unsigned("command_id", op->tid); + fmt->dump_int("osd", op->session ? op->session->osd : -1); + fmt->open_array_section("command"); for (vector::const_iterator q = op->cmd.begin(); q != op->cmd.end(); ++q) - fmt.dump_string("word", *q); - fmt.close_section(); + fmt->dump_string("word", *q); + fmt->close_section(); if (op->target_osd >= 0) - fmt.dump_int("target_osd", op->target_osd); + fmt->dump_int("target_osd", op->target_osd); else - fmt.dump_stream("target_pg") << op->target_pg; - fmt.close_section(); // command_op object + fmt->dump_stream("target_pg") << op->target_pg; + fmt->close_section(); // command_op object } - fmt.close_section(); // command_ops array + fmt->close_section(); // command_ops array } -void Objecter::dump_pool_ops(Formatter& fmt) const +void Objecter::dump_pool_ops(Formatter *fmt) const { - fmt.open_array_section("pool_ops"); + fmt->open_array_section("pool_ops"); for (map::const_iterator p = pool_ops.begin(); p != pool_ops.end(); ++p) { PoolOp *op = p->second; - fmt.open_object_section("pool_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_int("pool", op->pool); - fmt.dump_string("name", op->name); - fmt.dump_int("operation_type", op->pool_op); - fmt.dump_unsigned("auid", op->auid); - fmt.dump_unsigned("crush_rule", op->crush_rule); - fmt.dump_stream("snapid") << op->snapid; - fmt.dump_stream("last_sent") << op->last_submit; - fmt.close_section(); // pool_op object + fmt->open_object_section("pool_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_int("pool", op->pool); + fmt->dump_string("name", op->name); + fmt->dump_int("operation_type", op->pool_op); + fmt->dump_unsigned("auid", op->auid); + fmt->dump_unsigned("crush_rule", op->crush_rule); + fmt->dump_stream("snapid") << op->snapid; + fmt->dump_stream("last_sent") << op->last_submit; + fmt->close_section(); // pool_op object } - fmt.close_section(); // pool_ops array + fmt->close_section(); // pool_ops array } -void Objecter::dump_pool_stat_ops(Formatter& fmt) const +void Objecter::dump_pool_stat_ops(Formatter *fmt) const { - fmt.open_array_section("pool_stat_ops"); + fmt->open_array_section("pool_stat_ops"); for (map::const_iterator p = poolstat_ops.begin(); p != poolstat_ops.end(); ++p) { PoolStatOp *op = p->second; - fmt.open_object_section("pool_stat_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("last_sent") << op->last_submit; + fmt->open_object_section("pool_stat_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("last_sent") << op->last_submit; - fmt.open_array_section("pools"); + fmt->open_array_section("pools"); for (list::const_iterator it = op->pools.begin(); it != op->pools.end(); ++it) { - fmt.dump_string("pool", *it); + fmt->dump_string("pool", *it); } - fmt.close_section(); // pool_op object + fmt->close_section(); // pool_op object - fmt.close_section(); // pool_stat_op object + fmt->close_section(); // pool_stat_op object } - fmt.close_section(); // pool_stat_ops array + fmt->close_section(); // pool_stat_ops array } -void Objecter::dump_statfs_ops(Formatter& fmt) const +void Objecter::dump_statfs_ops(Formatter *fmt) const { - fmt.open_array_section("statfs_ops"); + fmt->open_array_section("statfs_ops"); for (map::const_iterator p = statfs_ops.begin(); p != statfs_ops.end(); ++p) { StatfsOp *op = p->second; - fmt.open_object_section("statfs_op"); - fmt.dump_unsigned("tid", op->tid); - fmt.dump_stream("last_sent") << op->last_submit; - fmt.close_section(); // pool_stat_op object + fmt->open_object_section("statfs_op"); + fmt->dump_unsigned("tid", op->tid); + fmt->dump_stream("last_sent") << op->last_submit; + fmt->close_section(); // pool_stat_op object } - fmt.close_section(); // pool_stat_ops array + fmt->close_section(); // pool_stat_ops array } Objecter::RequestStateHook::RequestStateHook(Objecter *objecter) : @@ -2321,14 +2321,16 @@ Objecter::RequestStateHook::RequestStateHook(Objecter *objecter) : { } -bool Objecter::RequestStateHook::call(std::string command, std::string args, bufferlist& out) +bool Objecter::RequestStateHook::call(std::string command, std::string args, + std::string format, bufferlist& out) { stringstream ss; - JSONFormatter formatter(true); + Formatter *f = new_formatter(format); m_objecter->client_lock.Lock(); - m_objecter->dump_requests(formatter); + m_objecter->dump_requests(f); m_objecter->client_lock.Unlock(); - formatter.flush(ss); + f->flush(ss); + delete f; out.append(ss); return true; } diff --git a/src/osdc/Objecter.h b/src/osdc/Objecter.h index b593bef69d974..da5487f3a8458 100644 --- a/src/osdc/Objecter.h +++ b/src/osdc/Objecter.h @@ -726,7 +726,8 @@ class Objecter { Objecter *m_objecter; public: RequestStateHook(Objecter *objecter); - bool call(std::string command, std::string args, bufferlist& out); + bool call(std::string command, std::string args, std::string format, + bufferlist& out); }; RequestStateHook *m_request_state_hook; @@ -1233,13 +1234,13 @@ class Objecter { * Output in-flight requests */ void dump_active(); - void dump_requests(Formatter& fmt) const; - void dump_ops(Formatter& fmt) const; - void dump_linger_ops(Formatter& fmt) const; - void dump_command_ops(Formatter& fmt) const; - void dump_pool_ops(Formatter& fmt) const; - void dump_pool_stat_ops(Formatter& fmt) const; - void dump_statfs_ops(Formatter& fmt) const; + void dump_requests(Formatter *fmt) const; + void dump_ops(Formatter *fmt) const; + void dump_linger_ops(Formatter *fmt) const; + void dump_command_ops(Formatter *fmt) const; + void dump_pool_ops(Formatter *fmt) const; + void dump_pool_stat_ops(Formatter *fmt) const; + void dump_statfs_ops(Formatter *fmt) const; int get_client_incarnation() const { return client_inc; } void set_client_incarnation(int inc) { client_inc = inc; } diff --git a/src/pybind/ceph_argparse.py b/src/pybind/ceph_argparse.py index d72e2c593a375..9a9db6758a599 100644 --- a/src/pybind/ceph_argparse.py +++ b/src/pybind/ceph_argparse.py @@ -263,6 +263,8 @@ def valid(self, s, partial=False): if p is not None and long(p) > 65535: raise ArgumentValid("{0} not a valid port number".format(p)) self.val = s + self.addr = a + self.port = p def __str__(self): return '' @@ -274,6 +276,7 @@ class CephEntityAddr(CephIPAddr): def valid(self, s, partial=False): ip, nonce = s.split('/') super(self.__class__, self).valid(ip) + self.nonce = nonce self.val = s def __str__(self): @@ -824,11 +827,10 @@ def validate(args, signature, partial=False): raise ArgumentError("unused arguments: " + str(myargs)) return d -def validate_command(parsed_args, sigdict, args, verbose=False): +def validate_command(sigdict, args, verbose=False): """ turn args into a valid dictionary ready to be sent off as JSON, validated against sigdict. - parsed_args is the namespace back from argparse """ found = [] valid_dict = {} @@ -886,9 +888,6 @@ def validate_command(parsed_args, sigdict, args, verbose=False): print >> sys.stderr, concise_sig(cmd['sig']) return None - if parsed_args.output_format: - valid_dict['format'] = parsed_args.output_format - return valid_dict def find_cmd_target(childargs): @@ -899,23 +898,38 @@ def find_cmd_target(childargs): right daemon. Returns ('osd', osdid), ('pg', pgid), or ('mon', '') """ - sig = parse_funcsig(['tell', {'name':'target','type':'CephName'}]) + sig = parse_funcsig(['tell', {'name':'target', 'type':'CephName'}]) try: - valid_dict = validate(childargs, sig, partial=True); + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: if len(valid_dict) == 2: + # revalidate to isolate type and id name = CephName() + # if this fails, something is horribly wrong, as it just + # validated successfully above name.valid(valid_dict['target']) return name.nametype, name.nameid - except ArgumentError: - pass - sig = parse_funcsig(['pg', {'name':'pgid','type':'CephPgid'}]) + sig = parse_funcsig(['tell', {'name':'pgid', 'type':'CephPgid'}]) try: - valid_dict = validate(childargs, sig, partial=True); + valid_dict = validate(childargs, sig, partial=True) + except ArgumentError: + pass + else: if len(valid_dict) == 2: + # pg doesn't need revalidation; the string is fine return 'pg', valid_dict['pgid'] + + sig = parse_funcsig(['pg', {'name':'pgid', 'type':'CephPgid'}]) + try: + valid_dict = validate(childargs, sig, partial=True) except ArgumentError: pass + else: + if len(valid_dict) == 2: + return 'pg', valid_dict['pgid'] return 'mon', '' @@ -944,8 +958,15 @@ def send_command(cluster, target=('mon', ''), cmd=None, inbuf='', timeout=0, cluster.osd_command(osdid, cmd, inbuf, timeout) elif target[0] == 'pg': - # leave it in cmddict for the OSD to use too pgid = target[1] + # pgid will already be in the command for the pg + # form, but for tell , we need to put it in + if cmd: + cmddict = json.loads(cmd[0]) + cmddict['pgid'] = pgid + else: + cmddict = dict(pgid=pgid) + cmd = [json.dumps(cmddict)] if verbose: print >> sys.stderr, 'submit {0} for pgid {1}'.\ format(cmd, pgid) diff --git a/src/pybind/ceph_rest_api.py b/src/pybind/ceph_rest_api.py index 28a0419c33c21..59e3f60a3a731 100755 --- a/src/pybind/ceph_rest_api.py +++ b/src/pybind/ceph_rest_api.py @@ -1,12 +1,13 @@ #!/usr/bin/python # vim: ts=4 sw=4 smarttab expandtab -import os import collections -import ConfigParser +import contextlib +import errno import json import logging import logging.handlers +import os import rados import textwrap import xml.etree.ElementTree @@ -16,16 +17,19 @@ from ceph_argparse import * # -# Globals +# Globals and defaults # -APPNAME = '__main__' +DEFAULT_ADDR = '0.0.0.0' +DEFAULT_PORT = '5000' +DEFAULT_ID = 'restapi' + DEFAULT_BASEURL = '/api/v0.1' -DEFAULT_ADDR = '0.0.0.0:5000' DEFAULT_LOG_LEVEL = 'warning' -DEFAULT_CLIENTNAME = 'client.restapi' -DEFAULT_LOG_FILE = '/var/log/ceph/' + DEFAULT_CLIENTNAME + '.log' +# default client name will be 'client.' +# 'app' must be global for decorators, etc. +APPNAME = '__main__' app = flask.Flask(APPNAME) LOGLEVELS = { @@ -36,133 +40,109 @@ 'debug':logging.DEBUG, } -# my globals, in a named tuple for usage clarity - -glob = collections.namedtuple('gvars', 'cluster urls sigdict baseurl') -glob.cluster = None -glob.urls = {} -glob.sigdict = {} -glob.baseurl = '' - -def load_conf(clustername='ceph', conffile=None): - import contextlib - - - class _TrimIndentFile(object): - def __init__(self, fp): - self.fp = fp - - def readline(self): - line = self.fp.readline() - return line.lstrip(' \t') - - - def _optionxform(s): - s = s.replace('_', ' ') - s = '_'.join(s.split()) - return s - - - def parse(fp): - cfg = ConfigParser.RawConfigParser() - cfg.optionxform = _optionxform - ifp = _TrimIndentFile(fp) - cfg.readfp(ifp) - return cfg - +def find_up_osd(app): + ''' + Find an up OSD. Return the last one that's up. + Returns id as an int. + ''' + ret, outbuf, outs = json_command(app.ceph_cluster, prefix="osd dump", + argdict=dict(format='json')) + if ret: + raise EnvironmentError(ret, 'Can\'t get osd dump output') + try: + osddump = json.loads(outbuf) + except: + raise EnvironmentError(errno.EINVAL, 'Invalid JSON back from osd dump') + osds = [osd['osd'] for osd in osddump['osds'] if osd['up']] + if not osds: + raise EnvironmentError(errno.ENOENT, 'No up OSDs found') + return int(osds[-1]) - def load(path): - f = file(path) - with contextlib.closing(f): - return parse(f) - if conffile: - # from CEPH_CONF - return load(conffile) - else: - for path in [ - '/etc/ceph/{0}.conf'.format(clustername), - os.path.expanduser('~/.ceph/{0}.conf'.format(clustername)), - '{0}.conf'.format(clustername), - ]: - if os.path.exists(path): - return load(path) - - raise EnvironmentError('No conf file found for "{0}"'.format(clustername)) - -def get_conf(cfg, clientname, key): - fullkey = 'restapi_' + key - for sectionname in clientname, 'client', 'global': - try: - return cfg.get(sectionname, fullkey) - except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): - pass - return None +METHOD_DICT = {'r':['GET'], 'w':['PUT', 'DELETE']} -# XXX this is done globally, and cluster connection kept open; there -# are facilities to pass around global info to requests and to -# tear down connections between requests if it becomes important +def api_setup(app, conf, cluster, clientname, clientid, args): + ''' + This is done globally, and cluster connection kept open for + the lifetime of the daemon. librados should assure that even + if the cluster goes away and comes back, our connection remains. -def api_setup(): - """ Initialize the running instance. Open the cluster, get the command - signatures, module, perms, and help; stuff them away in the glob.urls - dict. - """ + signatures, module, perms, and help; stuff them away in the app.ceph_urls + dict. Also save app.ceph_sigdict for help() handling. + ''' + def get_command_descriptions(cluster, target=('mon','')): + ret, outbuf, outs = json_command(cluster, target, + prefix='get_command_descriptions', + timeout=30) + if ret: + err = "Can't get command descriptions: {0}".format(outs) + app.logger.error(err) + raise EnvironmentError(ret, err) - conffile = os.environ.get('CEPH_CONF', '') - clustername = os.environ.get('CEPH_CLUSTER_NAME', 'ceph') - clientname = os.environ.get('CEPH_NAME', DEFAULT_CLIENTNAME) - try: - err = '' - cfg = load_conf(clustername, conffile) - except Exception as e: - err = "Can't load Ceph conf file: " + str(e) - app.logger.critical(err) - app.logger.critical("CEPH_CONF: %s", conffile) - app.logger.critical("CEPH_CLUSTER_NAME: %s", clustername) - raise EnvironmentError(err) - - client_logfile = '/var/log/ceph' + clientname + '.log' - - glob.cluster = rados.Rados(name=clientname, conffile=conffile) - glob.cluster.connect() - - glob.baseurl = get_conf(cfg, clientname, 'base_url') or DEFAULT_BASEURL - if glob.baseurl.endswith('/'): - glob.baseurl = glob.baseurl[:-1] - addr = get_conf(cfg, clientname, 'public_addr') or DEFAULT_ADDR - addrport = addr.rsplit(':', 1) - addr = addrport[0] - if len(addrport) > 1: - port = addrport[1] - else: - port = DEFAULT_ADDR.rsplit(':', 1) + try: + sigdict = parse_json_funcsigs(outbuf, 'rest') + except Exception as e: + err = "Can't parse command descriptions: {}".format(e) + app.logger.error(err) + raise EnvironmentError(err) + return sigdict + + app.ceph_cluster = cluster or 'ceph' + app.ceph_urls = {} + app.ceph_sigdict = {} + app.ceph_baseurl = '' + + conf = conf or '' + cluster = cluster or 'ceph' + clientid = clientid or DEFAULT_ID + clientname = clientname or 'client.' + clientid + + app.ceph_cluster = rados.Rados(name=clientname, conffile=conf) + app.ceph_cluster.conf_parse_argv(args) + app.ceph_cluster.connect() + + app.ceph_baseurl = app.ceph_cluster.conf_get('restapi_base_url') \ + or DEFAULT_BASEURL + if app.ceph_baseurl.endswith('/'): + app.ceph_baseurl = app.ceph_baseurl[:-1] + addr = app.ceph_cluster.conf_get('public_addr') or DEFAULT_ADDR + + # remove any nonce from the conf value + addr = addr.split('/')[0] + addr, port = addr.rsplit(':', 1) + addr = addr or DEFAULT_ADDR + port = port or DEFAULT_PORT port = int(port) - loglevel = get_conf(cfg, clientname, 'log_level') or DEFAULT_LOG_LEVEL - logfile = get_conf(cfg, clientname, 'log_file') or client_logfile + loglevel = app.ceph_cluster.conf_get('restapi_log_level') \ + or DEFAULT_LOG_LEVEL + logfile = app.ceph_cluster.conf_get('log_file') app.logger.addHandler(logging.handlers.WatchedFileHandler(logfile)) app.logger.setLevel(LOGLEVELS[loglevel.lower()]) for h in app.logger.handlers: h.setFormatter(logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s')) - ret, outbuf, outs = json_command(glob.cluster, - prefix='get_command_descriptions') - if ret: - err = "Can't contact cluster for command descriptions: {0}".format(outs) - app.logger.error(err) - raise EnvironmentError(ret, err) - - try: - glob.sigdict = parse_json_funcsigs(outbuf, 'rest') - except Exception as e: - err = "Can't parse command descriptions: {}".format(e) - app.logger.error(err) - raise EnvironmentError(err) - - # glob.sigdict maps "cmdNNN" to a dict containing: + app.ceph_sigdict = get_command_descriptions(app.ceph_cluster) + + osdid = find_up_osd(app) + if osdid: + osd_sigdict = get_command_descriptions(app.ceph_cluster, + target=('osd', int(osdid))) + + # shift osd_sigdict keys up to fit at the end of the mon's app.ceph_sigdict + maxkey = sorted(app.ceph_sigdict.keys())[-1] + maxkey = int(maxkey.replace('cmd', '')) + osdkey = maxkey + 1 + for k, v in osd_sigdict.iteritems(): + newv = v + newv['flavor'] = 'tell' + globk = 'cmd' + str(osdkey) + app.ceph_sigdict[globk] = newv + osdkey += 1 + + # app.ceph_sigdict maps "cmdNNN" to a dict containing: # 'sig', an array of argdescs # 'help', the helptext # 'module', the Ceph module this command relates to @@ -170,94 +150,130 @@ def api_setup(): # a hint as to whether this is a GET or POST/PUT operation # 'avail', a comma-separated list of strings of consumers that should # display this command (filtered by parse_json_funcsigs() above) - glob.urls = {} - for cmdnum, cmddict in glob.sigdict.iteritems(): + app.ceph_urls = {} + for cmdnum, cmddict in app.ceph_sigdict.iteritems(): cmdsig = cmddict['sig'] - url, params = generate_url_and_params(cmdsig) - if url in glob.urls: - continue + flavor = cmddict.get('flavor', 'mon') + url, params = generate_url_and_params(app, cmdsig, flavor) + perm = cmddict['perm'] + for k in METHOD_DICT.iterkeys(): + if k in perm: + methods = METHOD_DICT[k] + urldict = {'paramsig':params, + 'help':cmddict['help'], + 'module':cmddict['module'], + 'perm':perm, + 'flavor':flavor, + 'methods':methods, + } + + # app.ceph_urls contains a list of urldicts (usually only one long) + if url not in app.ceph_urls: + app.ceph_urls[url] = [urldict] else: - perm = cmddict['perm'] - urldict = {'paramsig':params, - 'help':cmddict['help'], - 'module':cmddict['module'], - 'perm':perm, - } - method_dict = {'r':['GET'], - 'w':['PUT', 'DELETE']} - for k in method_dict.iterkeys(): - if k in perm: - methods = method_dict[k] - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict - - url += '.' - app.add_url_rule(url, url, handler, methods=methods) - glob.urls[url] = urldict - app.logger.debug("urls added: %d", len(glob.urls)) + # If more than one, need to make union of methods of all. + # Method must be checked in handler + methodset = set(methods) + for old_urldict in app.ceph_urls[url]: + methodset |= set(old_urldict['methods']) + methods = list(methodset) + app.ceph_urls[url].append(urldict) + + # add, or re-add, rule with all methods and urldicts + app.add_url_rule(url, url, handler, methods=methods) + url += '.' + app.add_url_rule(url, url, handler, methods=methods) + + app.logger.debug("urls added: %d", len(app.ceph_urls)) app.add_url_rule('/', '/', handler, methods=['GET', 'PUT']) return addr, port -def generate_url_and_params(sig): - """ +def generate_url_and_params(app, sig, flavor): + ''' Digest command signature from cluster; generate an absolute - (including glob.baseurl) endpoint from all the prefix words, - and a dictionary of non-prefix parameters - """ + (including app.ceph_baseurl) endpoint from all the prefix words, + and a list of non-prefix param descs + ''' url = '' params = [] + # the OSD command descriptors don't include the 'tell ', so + # tack it onto the front of sig + if flavor == 'tell': + tellsig = parse_funcsig(['tell', + {'name':'target', 'type':'CephOsdName'}]) + sig = tellsig + sig + for desc in sig: + # prefixes go in the URL path if desc.t == CephPrefix: url += '/' + desc.instance.prefix + # CephChoices with 1 required string (not --) do too, unless + # we've already started collecting params, in which case they + # too are params elif desc.t == CephChoices and \ len(desc.instance.strings) == 1 and \ desc.req and \ - not str(desc.instance).startswith('--'): - url += '/' + str(desc.instance) + not str(desc.instance).startswith('--') and \ + not params: + url += '/' + str(desc.instance) else: - params.append(desc) - return glob.baseurl + url, params + # tell/ is a weird case; the URL includes what + # would everywhere else be a parameter + if flavor == 'tell' and \ + (desc.t, desc.name) == (CephOsdName, 'target'): + url += '/' + else: + params.append(desc) + + return app.ceph_baseurl + url, params -def concise_sig_for_uri(sig): - """ +# +# end setup (import-time) functions, begin request-time functions +# + +def concise_sig_for_uri(sig, flavor): + ''' Return a generic description of how one would send a REST request for sig - """ + ''' prefix = [] args = [] + ret = '' + if flavor == 'tell': + ret = 'tell//' for d in sig: if d.t == CephPrefix: prefix.append(d.instance.prefix) else: args.append(d.name + '=' + str(d)) - sig = '/'.join(prefix) + ret += '/'.join(prefix) if args: - sig += '?' + '&'.join(args) - return sig + ret += '?' + '&'.join(args) + return ret def show_human_help(prefix): - """ + ''' Dump table showing commands matching prefix - """ - # XXX this really needs to be a template - #s = '' - #s += '' - #s += '' - # XXX the above mucking with css doesn't cause sensible columns. + ''' + # XXX There ought to be a better discovery mechanism than an HTML table s = '
Possible commands:
' - possible = [] permmap = {'r':'GET', 'rw':'PUT'} line = '' - for cmdsig in sorted(glob.sigdict.itervalues(), cmp=descsort): + for cmdsig in sorted(app.ceph_sigdict.itervalues(), cmp=descsort): concise = concise_sig(cmdsig['sig']) + flavor = cmdsig.get('flavor', 'mon') + if flavor == 'tell': + concise = 'tell//' + concise if concise.startswith(prefix): line = ['
Possible commands:MethodDescription
'] - wrapped_sig = textwrap.wrap(concise_sig_for_uri(cmdsig['sig']), 40) + wrapped_sig = textwrap.wrap( + concise_sig_for_uri(cmdsig['sig'], flavor), 40 + ) for sigline in wrapped_sig: line.append(flask.escape(sigline) + '\n') line.append('') @@ -275,23 +291,22 @@ def show_human_help(prefix): @app.before_request def log_request(): - """ + ''' For every request, log it. XXX Probably overkill for production - """ + ''' app.logger.info(flask.request.url + " from " + flask.request.remote_addr + " " + flask.request.user_agent.string) app.logger.debug("Accept: %s", flask.request.accept_mimetypes.values()) - @app.route('/') def root_redir(): - return flask.redirect(glob.baseurl) + return flask.redirect(app.ceph_baseurl) def make_response(fmt, output, statusmsg, errorcode): - """ + ''' If formatted output, cobble up a response object that contains the output and status wrapped in enclosing objects; if nonformatted, just - use output. Return HTTP status errorcode in any event. - """ + use output+status. Return HTTP status errorcode in any event. + ''' response = output if fmt: if 'json' in fmt: @@ -303,6 +318,7 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response("Error decoding JSON from " + output, 500) elif 'xml' in fmt: + # XXX # one is tempted to do this with xml.etree, but figuring out how # to 'un-XML' the XML-dumped output so it can be reassembled into # a piece of the tree here is beyond me right now. @@ -328,19 +344,26 @@ def make_response(fmt, output, statusmsg, errorcode): return flask.make_response(response, errorcode) -def handler(catchall_path=None, fmt=None): - """ - Main endpoint handler; generic for every endpoint - """ +def handler(catchall_path=None, fmt=None, target=None): + ''' + Main endpoint handler; generic for every endpoint, including catchall. + Handles the catchall, anything with <.fmt>, anything with embedded + . Partial match or ?help cause the HTML-table + "show_human_help" output. + ''' - if (catchall_path): - ep = catchall_path.replace('.', '') - else: - ep = flask.request.endpoint.replace('.', '') + ep = catchall_path or flask.request.endpoint + ep = ep.replace('.', '') if ep[0] != '/': ep = '/' + ep + # demand that endpoint begin with app.ceph_baseurl + if not ep.startswith(app.ceph_baseurl): + return make_response(fmt, '', 'Page not found', 404) + + rel_ep = ep[len(app.ceph_baseurl)+1:] + # Extensions override Accept: headers override defaults if not fmt: if 'application/json' in flask.request.accept_mimetypes.values(): @@ -348,15 +371,38 @@ def handler(catchall_path=None, fmt=None): elif 'application/xml' in flask.request.accept_mimetypes.values(): fmt = 'xml' - # demand that endpoint begin with glob.baseurl - if not ep.startswith(glob.baseurl): - return make_response(fmt, '', 'Page not found', 404) + prefix = '' + pgid = None + cmdtarget = 'mon', '' + + if target: + # got tell/; validate osdid or pgid + name = CephOsdName() + pgidobj = CephPgid() + try: + name.valid(target) + except ArgumentError: + # try pgid + try: + pgidobj.valid(target) + except ArgumentError: + return flask.make_response("invalid osdid or pgid", 400) + else: + # it's a pgid + pgid = pgidobj.val + cmdtarget = 'pg', pgid + else: + # it's an osd + cmdtarget = name.nametype, name.nameid - relative_endpoint = ep[len(glob.baseurl)+1:] - prefix = ' '.join(relative_endpoint.split('/')).strip() + # prefix does not include tell// + prefix = ' '.join(rel_ep.split('/')[2:]).strip() + else: + # non-target command: prefix is entire path + prefix = ' '.join(rel_ep.split('/')).strip() # show "match as much as you gave me" help for unknown endpoints - if not ep in glob.urls: + if not ep in app.ceph_urls: helptext = show_human_help(prefix) if helptext: resp = flask.make_response(helptext, 400) @@ -365,43 +411,59 @@ def handler(catchall_path=None, fmt=None): else: return make_response(fmt, '', 'Invalid endpoint ' + ep, 400) - urldict = glob.urls[ep] - paramsig = urldict['paramsig'] - - # allow '?help' for any specifically-known endpoint - if 'help' in flask.request.args: - response = flask.make_response('{0}: {1}'.\ - format(prefix + concise_sig(paramsig), urldict['help'])) - response.headers['Content-Type'] = 'text/plain' - return response - - # if there are parameters for this endpoint, process them - if paramsig: - args = {} - for k, l in flask.request.args.iterlists(): - if len(l) == 1: - args[k] = l[0] - else: - args[k] = l - - # is this a valid set of params? - try: - argdict = validate(args, paramsig) - except Exception as e: - return make_response(fmt, '', str(e) + '\n', 400) - else: - # no parameters for this endpoint; complain if args are supplied - if flask.request.args: - return make_response(fmt, '', ep + 'takes no params', 400) - argdict = {} + found = None + exc = '' + for urldict in app.ceph_urls[ep]: + if flask.request.method not in urldict['methods']: + continue + paramsig = urldict['paramsig'] + + # allow '?help' for any specifically-known endpoint + if 'help' in flask.request.args: + response = flask.make_response('{0}: {1}'.\ + format(prefix + concise_sig(paramsig), urldict['help'])) + response.headers['Content-Type'] = 'text/plain' + return response + + # if there are parameters for this endpoint, process them + if paramsig: + args = {} + for k, l in flask.request.args.iterlists(): + if len(l) == 1: + args[k] = l[0] + else: + args[k] = l + + # is this a valid set of params? + try: + argdict = validate(args, paramsig) + found = urldict + break + except Exception as e: + exc += str(e) + continue + else: + if flask.request.args: + continue + found = urldict + argdict = {} + break + if not found: + return make_response(fmt, '', exc + '\n', 400) argdict['format'] = fmt or 'plain' - argdict['module'] = urldict['module'] - argdict['perm'] = urldict['perm'] + argdict['module'] = found['module'] + argdict['perm'] = found['perm'] + if pgid: + argdict['pgid'] = pgid + + if not cmdtarget: + cmdtarget = ('mon', '') app.logger.debug('sending command prefix %s argdict %s', prefix, argdict) - ret, outbuf, outs = json_command(glob.cluster, prefix=prefix, + ret, outbuf, outs = json_command(app.ceph_cluster, prefix=prefix, + target=cmdtarget, inbuf=flask.request.data, argdict=argdict) if ret: return make_response(fmt, '', 'Error: {0} ({1})'.format(outs, ret), 400) @@ -414,4 +476,12 @@ def handler(catchall_path=None, fmt=None): response.headers['Content-Type'] = contenttype return response -addr, port = api_setup() +# +# Main entry point from wrapper/WSGI server: call with cmdline args, +# get back the WSGI app entry point +# +def generate_app(conf, cluster, clientname, clientid, args): + addr, port = api_setup(app, conf, cluster, clientname, clientid, args) + app.ceph_addr = addr + app.ceph_port = port + return app diff --git a/src/pybind/cephfs.py b/src/pybind/cephfs.py index f89f53fb194fc..80b7e4b773ff4 100644 --- a/src/pybind/cephfs.py +++ b/src/pybind/cephfs.py @@ -2,9 +2,7 @@ This module is a thin wrapper around libcephfs. """ from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_int, c_long, c_uint, c_ulong, \ - create_string_buffer, byref, Structure, c_uint64, c_ubyte, pointer, \ - CFUNCTYPE -import ctypes + create_string_buffer, byref, Structure import errno class Error(Exception): diff --git a/src/pybind/rados.py b/src/pybind/rados.py index 13ce006f8ced9..e543ff7930505 100644 --- a/src/pybind/rados.py +++ b/src/pybind/rados.py @@ -278,7 +278,7 @@ def conf_parse_argv(self, args): ret = run_in_thread(self.librados.rados_conf_parse_argv_remainder, (self.cluster, len(args), cargs, cretargs)) if ret: - raise make_ex("error calling conf_parse_argv_remainder") + raise make_ex(ret, "error calling conf_parse_argv_remainder") # cretargs was allocated with fixed length; collapse return # list to eliminate any missing args diff --git a/src/pybind/rbd.py b/src/pybind/rbd.py index 9d71738e7284f..6e9ca8a22525d 100644 --- a/src/pybind/rbd.py +++ b/src/pybind/rbd.py @@ -17,7 +17,7 @@ # Copyright 2011 Josh Durgin from ctypes import CDLL, c_char, c_char_p, c_size_t, c_void_p, c_int, \ create_string_buffer, byref, Structure, c_uint64, c_int64, c_uint8, \ - CFUNCTYPE, pointer + CFUNCTYPE import ctypes import errno diff --git a/src/rbd.cc b/src/rbd.cc index c9b2f0a272ca3..7f90c1f118eaf 100644 --- a/src/rbd.cc +++ b/src/rbd.cc @@ -60,8 +60,6 @@ #include #endif -#include "include/fiemap.h" - #define MAX_SECRET_LEN 1000 #define MAX_POOL_NAME_SIZE 128 diff --git a/src/rgw/rgw_formats.h b/src/rgw/rgw_formats.h index 0ae917fe7d1a5..3653ef4f48079 100644 --- a/src/rgw/rgw_formats.h +++ b/src/rgw/rgw_formats.h @@ -36,6 +36,9 @@ class RGWFormatter_Plain : public Formatter { virtual void dump_string(const char *name, std::string s); virtual std::ostream& dump_stream(const char *name); virtual void dump_format(const char *name, const char *fmt, ...); + virtual void dump_format_unquoted(const char *name, const char *fmt, ...) { + assert(0 == "not implemented"); + } virtual int get_len() const; virtual void write_raw_data(const char *data); diff --git a/src/test/admin_socket.cc b/src/test/admin_socket.cc index 0dbcb6d2f756a..8f67918e644e4 100644 --- a/src/test/admin_socket.cc +++ b/src/test/admin_socket.cc @@ -64,13 +64,13 @@ TEST(AdminSocket, SendNoOp) { ASSERT_EQ(true, asoct.init(get_rand_socket_path())); AdminSocketClient client(get_rand_socket_path()); string version; - ASSERT_EQ("", client.do_request("0", &version)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"0\"}", &version)); ASSERT_EQ(CEPH_ADMIN_SOCK_VERSION, version); ASSERT_EQ(true, asoct.shutdown()); } class MyTest : public AdminSocketHook { - bool call(std::string command, std::string args, bufferlist& result) { + bool call(std::string command, std::string args, std::string format, bufferlist& result) { result.append(command); result.append("|"); result.append(args); @@ -87,13 +87,13 @@ TEST(AdminSocket, RegisterCommand) { AdminSocketClient client(get_rand_socket_path()); ASSERT_EQ(0, asoct.m_asokc->register_command("test", "test", new MyTest(), "")); string result; - ASSERT_EQ("", client.do_request("test", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test\"}", &result)); ASSERT_EQ("test|", result); ASSERT_EQ(true, asoct.shutdown()); } class MyTest2 : public AdminSocketHook { - bool call(std::string command, std::string args, bufferlist& result) { + bool call(std::string command, std::string args, std::string format, bufferlist& result) { result.append(command); result.append("|"); result.append(args); @@ -111,20 +111,20 @@ TEST(AdminSocket, RegisterCommandPrefixes) { ASSERT_EQ(0, asoct.m_asokc->register_command("test", "test", new MyTest(), "")); ASSERT_EQ(0, asoct.m_asokc->register_command("test command", "test command", new MyTest2(), "")); string result; - ASSERT_EQ("", client.do_request("test", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test\"}", &result)); ASSERT_EQ("test|", result); - ASSERT_EQ("", client.do_request("test command", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test command\"}", &result)); ASSERT_EQ("test command|", result); - ASSERT_EQ("", client.do_request("test command post", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test command post\"}", &result)); ASSERT_EQ("test command|post", result); - ASSERT_EQ("", client.do_request("test command post", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test command post\"}", &result)); ASSERT_EQ("test command| post", result); - ASSERT_EQ("", client.do_request("test this thing", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test this thing\"}", &result)); ASSERT_EQ("test|this thing", result); - ASSERT_EQ("", client.do_request("test command post", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test command post\"}", &result)); ASSERT_EQ("test| command post", result); - ASSERT_EQ("", client.do_request("test this thing", &result)); + ASSERT_EQ("", client.do_request("{\"prefix\":\"test this thing\"}", &result)); ASSERT_EQ("test| this thing", result); ASSERT_EQ(true, asoct.shutdown()); } diff --git a/src/test/bench/small_io_bench_fs.cc b/src/test/bench/small_io_bench_fs.cc index 61fbacc5570ca..a37a7e71153a6 100644 --- a/src/test/bench/small_io_bench_fs.cc +++ b/src/test/bench/small_io_bench_fs.cc @@ -32,7 +32,10 @@ struct MorePrinting : public DetailedStatCollector::AdditionalPrinting { MorePrinting(CephContext *cct) : cct(cct) {} void operator()(std::ostream *out) { bufferlist bl; - cct->get_perfcounters_collection()->write_json_to_buf(bl, 0); + Formatter *f = new_formatter("json-pretty"); + cct->get_perfcounters_collection()->dump_formatted(f, 0); + f->flush(bl); + delete f; bl.append('\0'); *out << bl.c_str() << std::endl; } diff --git a/src/test/cli/ceph-authtool/help.t b/src/test/cli/ceph-authtool/help.t index 7434ec4a7c00d..062c967a1548a 100644 --- a/src/test/cli/ceph-authtool/help.t +++ b/src/test/cli/ceph-authtool/help.t @@ -10,9 +10,15 @@ 'mount -o secret=..' argument -C, --create-keyring will create a new keyring, overwriting any existing keyringfile - --gen-key will generate a new secret key for the + -g, --gen-key will generate a new secret key for the specified entityname - --add-key will add an encoded key to the keyring + --gen-print-key will generate a new secret key without set it + to the keyringfile, prints the secret to stdout + --import-keyring will import the content of a given keyring + into the keyringfile + -u, --set-uid sets the auid (authenticated user id) for the + specified entityname + -a, --add-key will add an encoded key to the keyring --cap subsystem capability will set the capability for given subsystem --caps capsfile will set all of capabilities associated with a given key, for all subsystems diff --git a/src/test/cli/ceph-authtool/manpage.t b/src/test/cli/ceph-authtool/manpage.t index 69e15fa51c357..a9e1408d7163e 100644 --- a/src/test/cli/ceph-authtool/manpage.t +++ b/src/test/cli/ceph-authtool/manpage.t @@ -9,9 +9,15 @@ 'mount -o secret=..' argument -C, --create-keyring will create a new keyring, overwriting any existing keyringfile - --gen-key will generate a new secret key for the + -g, --gen-key will generate a new secret key for the specified entityname - --add-key will add an encoded key to the keyring + --gen-print-key will generate a new secret key without set it + to the keyringfile, prints the secret to stdout + --import-keyring will import the content of a given keyring + into the keyringfile + -u, --set-uid sets the auid (authenticated user id) for the + specified entityname + -a, --add-key will add an encoded key to the keyring --cap subsystem capability will set the capability for given subsystem --caps capsfile will set all of capabilities associated with a given key, for all subsystems diff --git a/src/test/cli/ceph-authtool/simple.t b/src/test/cli/ceph-authtool/simple.t index 69d027c13421d..b86476a5ad0f5 100644 --- a/src/test/cli/ceph-authtool/simple.t +++ b/src/test/cli/ceph-authtool/simple.t @@ -9,9 +9,15 @@ 'mount -o secret=..' argument -C, --create-keyring will create a new keyring, overwriting any existing keyringfile - --gen-key will generate a new secret key for the + -g, --gen-key will generate a new secret key for the specified entityname - --add-key will add an encoded key to the keyring + --gen-print-key will generate a new secret key without set it + to the keyringfile, prints the secret to stdout + --import-keyring will import the content of a given keyring + into the keyringfile + -u, --set-uid sets the auid (authenticated user id) for the + specified entityname + -a, --add-key will add an encoded key to the keyring --cap subsystem capability will set the capability for given subsystem --caps capsfile will set all of capabilities associated with a given key, for all subsystems diff --git a/src/test/osd/TestRados.cc b/src/test/osd/TestRados.cc index 43530f008284c..6ac661c06294a 100644 --- a/src/test/osd/TestRados.cc +++ b/src/test/osd/TestRados.cc @@ -2,6 +2,7 @@ #include "common/Mutex.h" #include "common/Cond.h" #include "common/errno.h" +#include "common/version.h" #include #include @@ -14,6 +15,7 @@ #include "test/osd/RadosModel.h" + using namespace std; class WeightedTestGenerator : public TestOpGenerator @@ -250,6 +252,7 @@ int main(int argc, char **argv) if (max_stride_size < 0) max_stride_size = size / 5; + cout << pretty_version_to_str() << std::endl; cout << "Configuration:" << std::endl << "\tNumber of operations: " << ops << std::endl << "\tNumber of objects: " << objects << std::endl diff --git a/src/test/perf_counters.cc b/src/test/perf_counters.cc index d8f04ca7d10b5..d0b05f9f04945 100644 --- a/src/test/perf_counters.cc +++ b/src/test/perf_counters.cc @@ -60,7 +60,7 @@ int main(int argc, char **argv) { TEST(PerfCounters, SimpleTest) { AdminSocketClient client(get_rand_socket_path()); std::string message; - ASSERT_EQ("", client.do_request("perfcounters_dump", &message)); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\" }", &message)); ASSERT_EQ("{}", message); } @@ -100,20 +100,20 @@ TEST(PerfCounters, SinglePerfCounters) { coll->add(fake_pf); AdminSocketClient client(get_rand_socket_path()); std::string msg; - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':0," - "'element2':0.000000000,'element3':{'avgcount':0,'sum':0.000000000}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":0," + "\"element2\":0.000000000,\"element3\":{\"avgcount\":0,\"sum\":0.000000000}}}"), msg); fake_pf->inc(TEST_PERFCOUNTERS1_ELEMENT_1); fake_pf->tset(TEST_PERFCOUNTERS1_ELEMENT_2, utime_t(0, 500000000)); fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(100, 0)); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':1," - "'element2':0.500000000,'element3':{'avgcount':1,'sum':100.000000000}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1," + "\"element2\":0.500000000,\"element3\":{\"avgcount\":1,\"sum\":100.000000000}}}"), msg); fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t()); fake_pf->tinc(TEST_PERFCOUNTERS1_ELEMENT_3, utime_t(25,0)); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':1,'element2':0.500000000," - "'element3':{'avgcount':3,'sum':125.000000000}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":1,\"element2\":0.500000000," + "\"element3\":{\"avgcount\":3,\"sum\":125.000000000}}}"), msg); } enum { @@ -142,24 +142,24 @@ TEST(PerfCounters, MultiplePerfCounters) { AdminSocketClient client(get_rand_socket_path()); std::string msg; - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':0,'element2':0.000000000,'element3':" - "{'avgcount':0,'sum':0.000000000}},'test_perfcounter_2':{'foo':0,'bar':0.000000000}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":0,\"element2\":0.000000000,\"element3\":" + "{\"avgcount\":0,\"sum\":0.000000000}},\"test_perfcounter_2\":{\"foo\":0,\"bar\":0.000000000}}"), msg); fake_pf1->inc(TEST_PERFCOUNTERS1_ELEMENT_1); fake_pf1->inc(TEST_PERFCOUNTERS1_ELEMENT_1, 5); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':6,'element2':0.000000000,'element3':" - "{'avgcount':0,'sum':0.000000000}},'test_perfcounter_2':{'foo':0,'bar':0.000000000}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":6,\"element2\":0.000000000,\"element3\":" + "{\"avgcount\":0,\"sum\":0.000000000}},\"test_perfcounter_2\":{\"foo\":0,\"bar\":0.000000000}}"), msg); coll->remove(fake_pf2); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':6,'element2':0.000000000," - "'element3':{'avgcount':0,'sum':0.000000000}}}"), msg); - ASSERT_EQ("", client.do_request("perfcounters_schema", &msg)); - ASSERT_EQ(sd("{'test_perfcounter_1':{'element1':{'type':2}," - "'element2':{'type':1},'element3':{'type':5}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":6,\"element2\":0.000000000," + "\"element3\":{\"avgcount\":0,\"sum\":0.000000000}}}"), msg); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perf schema\", \"format\": \"json\" }", &msg)); + ASSERT_EQ(sd("{\"test_perfcounter_1\":{\"element1\":{\"type\":2}," + "\"element2\":{\"type\":1},\"element3\":{\"type\":5}}}"), msg); coll->clear(); - ASSERT_EQ("", client.do_request("perfcounters_dump", &msg)); + ASSERT_EQ("", client.do_request("{ \"prefix\": \"perfcounters_dump\", \"format\": \"json\" }", &msg)); ASSERT_EQ("{}", msg); } diff --git a/src/test/system/rados_list_parallel.cc b/src/test/system/rados_list_parallel.cc index a1c6e270265dc..d530c83441c6b 100644 --- a/src/test/system/rados_list_parallel.cc +++ b/src/test/system/rados_list_parallel.cc @@ -62,6 +62,7 @@ class RadosDeleteObjectsR : public SysTestRunnable RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); std::string log_name = SysTestSettings::inst().get_log_name(get_id_str()); if (!log_name.empty()) rados_conf_set(cl, "log_file", log_name.c_str()); @@ -142,6 +143,7 @@ class RadosAddObjectsR : public SysTestRunnable RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); std::string log_name = SysTestSettings::inst().get_log_name(get_id_str()); if (!log_name.empty()) rados_conf_set(cl, "log_file", log_name.c_str()); diff --git a/src/test/system/rados_open_pools_parallel.cc b/src/test/system/rados_open_pools_parallel.cc index 445f2ebc485d9..82c712077f272 100644 --- a/src/test/system/rados_open_pools_parallel.cc +++ b/src/test/system/rados_open_pools_parallel.cc @@ -68,6 +68,7 @@ class StRadosOpenPool : public SysTestRunnable if (!log_name.empty()) rados_conf_set(cl, "log_file", log_name.c_str()); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); RETURN1_IF_NONZERO(rados_connect(cl)); if (m_pool_setup_sem) m_pool_setup_sem->wait(); diff --git a/src/test/system/st_rados_create_pool.cc b/src/test/system/st_rados_create_pool.cc index dcae15375afa7..29b8d14b7e7ea 100644 --- a/src/test/system/st_rados_create_pool.cc +++ b/src/test/system/st_rados_create_pool.cc @@ -72,6 +72,7 @@ run() std::string log_name = SysTestSettings::inst().get_log_name(get_id_str()); if (!log_name.empty()) rados_conf_set(cl, "log_file", log_name.c_str()); + rados_conf_parse_env(cl, NULL); if (m_setup_sem) { m_setup_sem->wait(); diff --git a/src/test/system/st_rados_delete_objs.cc b/src/test/system/st_rados_delete_objs.cc index 38dc47a35579b..b43ffdda6aeb5 100644 --- a/src/test/system/st_rados_delete_objs.cc +++ b/src/test/system/st_rados_delete_objs.cc @@ -45,6 +45,7 @@ int StRadosDeleteObjs::run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); RETURN1_IF_NONZERO(rados_connect(cl)); m_setup_sem->wait(); m_setup_sem->post(); diff --git a/src/test/system/st_rados_delete_pool.cc b/src/test/system/st_rados_delete_pool.cc index d954bf46c231a..de553e98e13a3 100644 --- a/src/test/system/st_rados_delete_pool.cc +++ b/src/test/system/st_rados_delete_pool.cc @@ -41,6 +41,7 @@ int StRadosDeletePool::run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); RETURN1_IF_NONZERO(rados_connect(cl)); m_pool_setup_sem->wait(); m_pool_setup_sem->post(); diff --git a/src/test/system/st_rados_list_objects.cc b/src/test/system/st_rados_list_objects.cc index bb153affeb836..be6ead6498745 100644 --- a/src/test/system/st_rados_list_objects.cc +++ b/src/test/system/st_rados_list_objects.cc @@ -56,6 +56,7 @@ run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); RETURN1_IF_NONZERO(rados_connect(cl)); m_pool_setup_sem->wait(); m_pool_setup_sem->post(); diff --git a/src/test/system/st_rados_notify.cc b/src/test/system/st_rados_notify.cc index 10e3e4bfbc39e..e74711cb6410c 100644 --- a/src/test/system/st_rados_notify.cc +++ b/src/test/system/st_rados_notify.cc @@ -44,6 +44,8 @@ int StRadosNotify::run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); + if (m_setup_sem) { m_setup_sem->wait(); m_setup_sem->post(); diff --git a/src/test/system/st_rados_watch.cc b/src/test/system/st_rados_watch.cc index 991dde8ea77e9..696b0867e8893 100644 --- a/src/test/system/st_rados_watch.cc +++ b/src/test/system/st_rados_watch.cc @@ -52,6 +52,8 @@ run() RETURN1_IF_NONZERO(rados_create(&cl, NULL)); rados_conf_parse_argv(cl, m_argc, m_argv); RETURN1_IF_NONZERO(rados_conf_read_file(cl, NULL)); + rados_conf_parse_env(cl, NULL); + if (m_setup_sem) { m_setup_sem->wait(); m_setup_sem->post();