do a bunch of stuff

.travis.yml

@@ -8,27 +8,31 @@ services:
 
 before_install:
 - pyenv versions
+- pyenv version-name
+- env
 
 install:
 - pip install tox
 
+python:
+- '2.7'
+- '3.4'
+- '3.5'
+- '3.6'
+- '3.7-dev'
+
+env:
+- SALT=-v2018.3 BACKEND=-cherrypy CODECOV=py
+- SALT=-v2018.3 BACKEND=-tornado CODECOV=py
+- SALT=-v2019.2 BACKEND=-cherrypy CODECOV=py
+- SALT=-v2019.2 BACKEND=-tornado CODECOV=py
+
 matrix:
-  include:
-  - env: TOXENV=27,coverage CODECOV=py
-    python: 2.7
-  - env: TOXENV=34,coverage CODECOV=py
-    python: 3.4
-  - env: TOXENV=35,coverage CODECOV=py
-    python: 3.5
-  - env: TOXENV=36,coverage CODECOV=py
-    python: 3.6
-  - env: TOXENV=37,coverage CODECOV=py
-    python: 3.7-dev
-  - env: TOXENV=flake8
-    python: 3.6
+  env:
 
 script:
-- docker run -v $PWD:/pepper -ti --rm gtmanfred/pepper:latest tox -c /pepper/tox.ini -e "${CODECOV}${TOXENV}"
+- PYTHON="${TRAVIS_PYTHON_VERSION%-dev}"
+- docker run -v $PWD:/pepper -ti --rm gtmanfred/pepper:latest tox -c /pepper/tox.ini -e "${TRAVIS_PYTHON_VERSION%%.*}flake8,${CODECOV}${PYTHON//./}${BACKEND}${SALT}"
 
 after_success:
 - sudo chown $USER .tox/

pepper/cli.py

@@ -468,7 +468,7 @@ def parse_login(self):
 
         return ret
 
-    def parse_cmd(self):
+    def parse_cmd(self, api):
         '''
         Extract the low data for a command from the passed CLI params
         '''
@@ -505,26 +505,37 @@ def parse_cmd(self):
             low['arg'] = args
         elif client.startswith('runner'):
             low['fun'] = args.pop(0)
-            for arg in args:
-                if '=' in arg:
-                    key, value = arg.split('=', 1)
-                    try:
-                        low[key] = json.loads(value)
-                    except JSONDecodeError:
-                        low[key] = value
-                else:
-                    low.setdefault('arg', []).append(arg)
+            # post https://github.com/saltstack/salt/pull/50124, kwargs can be
+            # passed as is in foo=bar form, splitting and deserializing will
+            # happen in salt-api. additionally, the presence of salt-version header
+            # means we are neon or newer, so don't need a finer grained check
+            if api.salt_version:
+                low['arg'] = args
+            else:
+                for arg in args:
+                    if '=' in arg:
+                        key, value = arg.split('=', 1)
+                        try:
+                            low[key] = json.loads(value)
+                        except JSONDecodeError:
+                            low[key] = value
+                    else:
+                        low.setdefault('arg', []).append(arg)
         elif client.startswith('wheel'):
             low['fun'] = args.pop(0)
-            for arg in args:
-                if '=' in arg:
-                    key, value = arg.split('=', 1)
-                    try:
-                        low[key] = json.loads(value)
-                    except JSONDecodeError:
-                        low[key] = value
-                else:
-                    low.setdefault('arg', []).append(arg)
+            # see above comment in runner arg handling
+            if api.salt_version:
+                low['arg'] = args
+            else:
+                for arg in args:
+                    if '=' in arg:
+                        key, value = arg.split('=', 1)
+                        try:
+                            low[key] = json.loads(value)
+                        except JSONDecodeError:
+                            low[key] = value
+                    else:
+                        low.setdefault('arg', []).append(arg)
         elif client.startswith('ssh'):
             if len(args) < 2:
                 self.parser.error("Command or target not specified")
@@ -569,12 +580,16 @@ def poll_for_returns(self, api, load):
                 },
             }])
 
-            responded = set(jid_ret['return'][0].keys()) ^ set(ret_nodes)
+            inner_ret = jid_ret['return'][0]
+            # sometimes ret is nested in data
+            if 'data' in inner_ret:
+                inner_ret = inner_ret['data']
+
+            responded = set(inner_ret.keys()) ^ set(ret_nodes)
+
             for node in responded:
-                yield None, "{{{}: {}}}".format(
-                    node,
-                    jid_ret['return'][0][node])
-            ret_nodes = list(jid_ret['return'][0].keys())
+                yield None, [{node: inner_ret[node]}]
+            ret_nodes = list(inner_ret.keys())
 
             if set(ret_nodes) == set(nodes):
                 exit_code = 0
@@ -583,8 +598,9 @@ def poll_for_returns(self, api, load):
                 time.sleep(self.seconds_to_wait)
 
         exit_code = exit_code if self.options.fail_if_minions_dont_respond else 0
-        yield exit_code, "{{Failed: {}}}".format(
-            list(set(ret_nodes) ^ set(nodes)))
+        failed = list(set(ret_nodes) ^ set(nodes))
+        if failed:
+            yield exit_code, [{'Failed': failed}]
 
     def login(self, api):
         login = api.token if self.options.userun else api.login
@@ -626,21 +642,23 @@ def low(self, api, load):
             for i in load:
                 i['token'] = self.auth['token']
 
+        # having a defined salt_version means changes from https://github.com/saltstack/salt/pull/51979
+        # are available if backend is tornado, so safe to supply timeout
+        if self.options.timeout and api.salt_version:
+            for i in load:
+                if not i.get('client', '').startswith('wheel'):
+                    i['timeout'] = self.options.timeout
+
         return api.low(load, path=path)
 
     def run(self):
         '''
         Parse all arguments and call salt-api
         '''
-        # move logger instantiation to method?
-        logger.addHandler(logging.StreamHandler())
-        logger.setLevel(max(logging.ERROR - (self.options.verbose * 10), 1))
-
-        load = self.parse_cmd()
-
-        for entry in load:
-            if entry.get('client', '').startswith('local'):
-                entry['full_return'] = True
+        # set up logging
+        rootLogger = logging.getLogger(name=None)
+        rootLogger.addHandler(logging.StreamHandler())
+        rootLogger.setLevel(max(logging.ERROR - (self.options.verbose * 10), 1))
 
         api = pepper.Pepper(
             self.parse_url(),
@@ -649,6 +667,12 @@ def run(self):
 
         self.login(api)
 
+        load = self.parse_cmd(api)
+
+        for entry in load:
+            if not entry.get('client', '').startswith('wheel'):
+                entry['full_return'] = True
+
         if self.options.fail_if_minions_dont_respond:
             for exit_code, ret in self.poll_for_returns(api, load):  # pragma: no cover
                 yield exit_code, json.dumps(ret, sort_keys=True, indent=4)

pepper/libpepper.py

@@ -6,6 +6,7 @@
 '''
 import json
 import logging
+import re
 import ssl
 
 from pepper.exceptions import PepperException
@@ -79,6 +80,7 @@ def __init__(self, api_url='https://localhost:8000', debug_http=False, ignore_ss
         self.debug_http = int(debug_http)
         self._ssl_verify = not ignore_ssl_errors
         self.auth = {}
+        self.salt_version = None
 
     def req_stream(self, path):
         '''
@@ -217,7 +219,7 @@ def req(self, path, data=None):
             req.add_header('Content-Length', clen)
 
         # Add auth header to request
-        if self.auth and 'token' in self.auth and self.auth['token']:
+        if path != '/run' and self.auth and 'token' in self.auth and self.auth['token']:
             req.add_header('X-Auth-Token', self.auth['token'])
 
         # Send request
@@ -231,6 +233,10 @@ def req(self, path, data=None):
             if (self.debug_http):
                 logger.debug('Response: %s', content)
             ret = json.loads(content)
+
+            if not self.salt_version and 'x-salt-version' in f.headers:
+                self._parse_salt_version(f.headers['x-salt-version'])
+
         except (HTTPError, URLError) as exc:
             logger.debug('Error with request', exc_info=True)
             status = getattr(exc, 'code', None)
@@ -285,6 +291,10 @@ def req_requests(self, path, data=None):
         if resp.status_code == 500:
             # TODO should be resp.raise_from_status
             raise PepperException('Server error.')
+
+        if not self.salt_version and 'x-salt-version' in resp.headers:
+            self._parse_salt_version(resp.headers['x-salt-version'])
+
         return resp.json()
 
     def low(self, lowstate, path='/'):
@@ -479,3 +489,17 @@ def _construct_url(self, path):
 
         relative_path = path.lstrip('/')
         return urlparse.urljoin(self.api_url, relative_path)
+
+    def _parse_salt_version(self, version):
+        # borrow from salt.version
+        git_describe_regex = re.compile(
+            r'(?:[^\d]+)?(?P<major>[\d]{1,4})'
+            r'\.(?P<minor>[\d]{1,2})'
+            r'(?:\.(?P<bugfix>[\d]{0,2}))?'
+            r'(?:\.(?P<mbugfix>[\d]{0,2}))?'
+            r'(?:(?P<pre_type>rc|a|b|alpha|beta|nb)(?P<pre_num>[\d]{1}))?'
+            r'(?:(?:.*)-(?P<noc>(?:[\d]+|n/a))-(?P<sha>[a-z0-9]{8}))?'
+        )
+        match = git_describe_regex.match(version)
+        if match:
+            self.salt_version = match.groups()

pepper/script.py

@@ -50,13 +50,29 @@ def output(self):
     def __call__(self):
         try:
             for exit_code, result in self.cli.run():
-                if HAS_SALT and not self.cli.options.userun and self.opts:
-                    logger.info('Use Salt outputters')
-                    for ret in json.loads(result)['return']:
+                if HAS_SALT and self.opts:
+                    logger.debug('Use Salt outputters')
+                    result = json.loads(result)
+
+                    # unwrap ret in some cases
+                    if 'return' in result:
+                        result = result['return']
+
+                    for ret in result:
                         if isinstance(ret, dict):
-                            if self.cli.options.client == 'local':
+                            if self.cli.options.client.startswith('local'):
                                 for minionid, minionret in ret.items():
-                                    if isinstance(minionret, dict) and 'ret' in minionret:
+                                    # rest_tornado doesnt return full_return directly
+                                    # it will always be from get_event, so the output differs slightly
+                                    if isinstance(minionret, dict) and 'return' in minionret:
+                                        # version >= 2017.7
+                                        salt.output.display_output(
+                                            {minionid: minionret['return']},
+                                            self.cli.options.output or minionret.get('out', None) or 'nested',
+                                            opts=self.opts
+                                        )
+                                    # cherrypy returns with ret via full_return
+                                    elif isinstance(minionret, dict) and 'ret' in minionret:
                                         # version >= 2017.7
                                         salt.output.display_output(
                                             {minionid: minionret['ret']},
@@ -70,9 +86,13 @@ def __call__(self):
                                             opts=self.opts
                                         )
                             elif 'data' in ret:
+                                # unfold runners
+                                outputter = ret.get('outputter', 'nested')
+                                if isinstance(ret['data'], dict) and 'return' in ret['data']:
+                                    ret = ret['data']['return']
                                 salt.output.display_output(
-                                    ret['data'],
-                                    self.cli.options.output or ret.get('outputter', 'nested'),
+                                    ret,
+                                    self.cli.options.output or outputter,
                                     opts=self.opts
                                 )
                             else:
@@ -84,7 +104,7 @@ def __call__(self):
                         else:
                             salt.output.display_output(
                                 {self.cli.options.client: ret},
-                                'nested',
+                                self.cli.options.output or 'nested',
                                 opts=self.opts,
                             )
                 else:
@@ -95,7 +115,7 @@ def __call__(self):
                         print(result)
                 if exit_code is not None:
                     if exit_code == 0:
-                        return PepperRetcode().validate(self.cli.options, json.loads(result)['return'])
+                        return PepperRetcode().validate(self.cli.options, result)
                     return exit_code
         except (PepperException, PepperAuthException, PepperArgumentsException) as exc:
             print('Pepper error: {0}'.format(exc), file=sys.stderr)