Skip to content

Commit

Permalink
Merge pull request #185 from bloomberg/develop
Browse files Browse the repository at this point in the history 
do a bunch of stuff
  • Loading branch information
@gtmanfred
gtmanfred authored Mar 27, 2019
2 parents 386c08c + 9332567 commit 7413e54
Show file tree
Hide file tree
Showing 11 changed files with 233 additions and 112 deletions.
32 changes: 18 additions & 14 deletions .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,27 +8,31 @@ services:

before_install:
- pyenv versions
- pyenv version-name
- env

install:
- pip install tox

python:
- '2.7'
- '3.4'
- '3.5'
- '3.6'
- '3.7-dev'

env:
- SALT=-v2018.3 BACKEND=-cherrypy CODECOV=py
- SALT=-v2018.3 BACKEND=-tornado CODECOV=py
- SALT=-v2019.2 BACKEND=-cherrypy CODECOV=py
- SALT=-v2019.2 BACKEND=-tornado CODECOV=py

matrix:
include:
- env: TOXENV=27,coverage CODECOV=py
python: 2.7
- env: TOXENV=34,coverage CODECOV=py
python: 3.4
- env: TOXENV=35,coverage CODECOV=py
python: 3.5
- env: TOXENV=36,coverage CODECOV=py
python: 3.6
- env: TOXENV=37,coverage CODECOV=py
python: 3.7-dev
- env: TOXENV=flake8
python: 3.6
env:

script:
- docker run -v $PWD:/pepper -ti --rm gtmanfred/pepper:latest tox -c /pepper/tox.ini -e "${CODECOV}${TOXENV}"
- PYTHON="${TRAVIS_PYTHON_VERSION%-dev}"
- docker run -v $PWD:/pepper -ti --rm gtmanfred/pepper:latest tox -c /pepper/tox.ini -e "${TRAVIS_PYTHON_VERSION%%.*}flake8,${CODECOV}${PYTHON//./}${BACKEND}${SALT}"

after_success:
- sudo chown $USER .tox/
Expand Down
94 changes: 59 additions & 35 deletions pepper/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ def parse_login(self):

return ret

def parse_cmd(self):
def parse_cmd(self, api):
'''
Extract the low data for a command from the passed CLI params
'''
Expand Down Expand Up @@ -505,26 +505,37 @@ def parse_cmd(self):
low['arg'] = args
elif client.startswith('runner'):
low['fun'] = args.pop(0)
for arg in args:
if '=' in arg:
key, value = arg.split('=', 1)
try:
low[key] = json.loads(value)
except JSONDecodeError:
low[key] = value
else:
low.setdefault('arg', []).append(arg)
# post https://github.com/saltstack/salt/pull/50124, kwargs can be
# passed as is in foo=bar form, splitting and deserializing will
# happen in salt-api. additionally, the presence of salt-version header
# means we are neon or newer, so don't need a finer grained check
if api.salt_version:
low['arg'] = args
else:
for arg in args:
if '=' in arg:
key, value = arg.split('=', 1)
try:
low[key] = json.loads(value)
except JSONDecodeError:
low[key] = value
else:
low.setdefault('arg', []).append(arg)
elif client.startswith('wheel'):
low['fun'] = args.pop(0)
for arg in args:
if '=' in arg:
key, value = arg.split('=', 1)
try:
low[key] = json.loads(value)
except JSONDecodeError:
low[key] = value
else:
low.setdefault('arg', []).append(arg)
# see above comment in runner arg handling
if api.salt_version:
low['arg'] = args
else:
for arg in args:
if '=' in arg:
key, value = arg.split('=', 1)
try:
low[key] = json.loads(value)
except JSONDecodeError:
low[key] = value
else:
low.setdefault('arg', []).append(arg)
elif client.startswith('ssh'):
if len(args) < 2:
self.parser.error("Command or target not specified")
Expand Down Expand Up @@ -569,12 +580,16 @@ def poll_for_returns(self, api, load):
},
}])

responded = set(jid_ret['return'][0].keys()) ^ set(ret_nodes)
inner_ret = jid_ret['return'][0]
# sometimes ret is nested in data
if 'data' in inner_ret:
inner_ret = inner_ret['data']

responded = set(inner_ret.keys()) ^ set(ret_nodes)

for node in responded:
yield None, "{{{}: {}}}".format(
node,
jid_ret['return'][0][node])
ret_nodes = list(jid_ret['return'][0].keys())
yield None, [{node: inner_ret[node]}]
ret_nodes = list(inner_ret.keys())

if set(ret_nodes) == set(nodes):
exit_code = 0
Expand All @@ -583,8 +598,9 @@ def poll_for_returns(self, api, load):
time.sleep(self.seconds_to_wait)

exit_code = exit_code if self.options.fail_if_minions_dont_respond else 0
yield exit_code, "{{Failed: {}}}".format(
list(set(ret_nodes) ^ set(nodes)))
failed = list(set(ret_nodes) ^ set(nodes))
if failed:
yield exit_code, [{'Failed': failed}]

def login(self, api):
login = api.token if self.options.userun else api.login
Expand Down Expand Up @@ -626,21 +642,23 @@ def low(self, api, load):
for i in load:
i['token'] = self.auth['token']

# having a defined salt_version means changes from https://github.com/saltstack/salt/pull/51979
# are available if backend is tornado, so safe to supply timeout
if self.options.timeout and api.salt_version:
for i in load:
if not i.get('client', '').startswith('wheel'):
i['timeout'] = self.options.timeout

return api.low(load, path=path)

def run(self):
'''
Parse all arguments and call salt-api
'''
# move logger instantiation to method?
logger.addHandler(logging.StreamHandler())
logger.setLevel(max(logging.ERROR - (self.options.verbose * 10), 1))

load = self.parse_cmd()

for entry in load:
if entry.get('client', '').startswith('local'):
entry['full_return'] = True
# set up logging
rootLogger = logging.getLogger(name=None)
rootLogger.addHandler(logging.StreamHandler())
rootLogger.setLevel(max(logging.ERROR - (self.options.verbose * 10), 1))

api = pepper.Pepper(
self.parse_url(),
Expand All @@ -649,6 +667,12 @@ def run(self):

self.login(api)

load = self.parse_cmd(api)

for entry in load:
if not entry.get('client', '').startswith('wheel'):
entry['full_return'] = True

if self.options.fail_if_minions_dont_respond:
for exit_code, ret in self.poll_for_returns(api, load): # pragma: no cover
yield exit_code, json.dumps(ret, sort_keys=True, indent=4)
Expand Down
26 changes: 25 additions & 1 deletion pepper/libpepper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
'''
import json
import logging
import re
import ssl

from pepper.exceptions import PepperException
Expand Down Expand Up @@ -79,6 +80,7 @@ def __init__(self, api_url='https://localhost:8000', debug_http=False, ignore_ss
self.debug_http = int(debug_http)
self._ssl_verify = not ignore_ssl_errors
self.auth = {}
self.salt_version = None

def req_stream(self, path):
'''
Expand Down Expand Up @@ -217,7 +219,7 @@ def req(self, path, data=None):
req.add_header('Content-Length', clen)

# Add auth header to request
if self.auth and 'token' in self.auth and self.auth['token']:
if path != '/run' and self.auth and 'token' in self.auth and self.auth['token']:
req.add_header('X-Auth-Token', self.auth['token'])

# Send request
Expand All @@ -231,6 +233,10 @@ def req(self, path, data=None):
if (self.debug_http):
logger.debug('Response: %s', content)
ret = json.loads(content)

if not self.salt_version and 'x-salt-version' in f.headers:
self._parse_salt_version(f.headers['x-salt-version'])

except (HTTPError, URLError) as exc:
logger.debug('Error with request', exc_info=True)
status = getattr(exc, 'code', None)
Expand Down Expand Up @@ -285,6 +291,10 @@ def req_requests(self, path, data=None):
if resp.status_code == 500:
# TODO should be resp.raise_from_status
raise PepperException('Server error.')

if not self.salt_version and 'x-salt-version' in resp.headers:
self._parse_salt_version(resp.headers['x-salt-version'])

return resp.json()

def low(self, lowstate, path='/'):
Expand Down Expand Up @@ -479,3 +489,17 @@ def _construct_url(self, path):

relative_path = path.lstrip('/')
return urlparse.urljoin(self.api_url, relative_path)

def _parse_salt_version(self, version):
# borrow from salt.version
git_describe_regex = re.compile(
r'(?:[^\d]+)?(?P<major>[\d]{1,4})'
r'\.(?P<minor>[\d]{1,2})'
r'(?:\.(?P<bugfix>[\d]{0,2}))?'
r'(?:\.(?P<mbugfix>[\d]{0,2}))?'
r'(?:(?P<pre_type>rc|a|b|alpha|beta|nb)(?P<pre_num>[\d]{1}))?'
r'(?:(?:.*)-(?P<noc>(?:[\d]+|n/a))-(?P<sha>[a-z0-9]{8}))?'
)
match = git_describe_regex.match(version)
if match:
self.salt_version = match.groups()
38 changes: 29 additions & 9 deletions pepper/script.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,29 @@ def output(self):
def __call__(self):
try:
for exit_code, result in self.cli.run():
if HAS_SALT and not self.cli.options.userun and self.opts:
logger.info('Use Salt outputters')
for ret in json.loads(result)['return']:
if HAS_SALT and self.opts:
logger.debug('Use Salt outputters')
result = json.loads(result)

# unwrap ret in some cases
if 'return' in result:
result = result['return']

for ret in result:
if isinstance(ret, dict):
if self.cli.options.client == 'local':
if self.cli.options.client.startswith('local'):
for minionid, minionret in ret.items():
if isinstance(minionret, dict) and 'ret' in minionret:
# rest_tornado doesnt return full_return directly
# it will always be from get_event, so the output differs slightly
if isinstance(minionret, dict) and 'return' in minionret:
# version >= 2017.7
salt.output.display_output(
{minionid: minionret['return']},
self.cli.options.output or minionret.get('out', None) or 'nested',
opts=self.opts
)
# cherrypy returns with ret via full_return
elif isinstance(minionret, dict) and 'ret' in minionret:
# version >= 2017.7
salt.output.display_output(
{minionid: minionret['ret']},
Expand All @@ -70,9 +86,13 @@ def __call__(self):
opts=self.opts
)
elif 'data' in ret:
# unfold runners
outputter = ret.get('outputter', 'nested')
if isinstance(ret['data'], dict) and 'return' in ret['data']:
ret = ret['data']['return']
salt.output.display_output(
ret['data'],
self.cli.options.output or ret.get('outputter', 'nested'),
ret,
self.cli.options.output or outputter,
opts=self.opts
)
else:
Expand All @@ -84,7 +104,7 @@ def __call__(self):
else:
salt.output.display_output(
{self.cli.options.client: ret},
'nested',
self.cli.options.output or 'nested',
opts=self.opts,
)
else:
Expand All @@ -95,7 +115,7 @@ def __call__(self):
print(result)
if exit_code is not None:
if exit_code == 0:
return PepperRetcode().validate(self.cli.options, json.loads(result)['return'])
return PepperRetcode().validate(self.cli.options, result)
return exit_code
except (PepperException, PepperAuthException, PepperArgumentsException) as exc:
print('Pepper error: {0}'.format(exc), file=sys.stderr)
Expand Down
Loading

0 comments on commit 7413e54

Please sign in to comment.