From 97c2b09ef25f5a0a93fbb933d47cd8ed1c871647 Mon Sep 17 00:00:00 2001 From: Carlos Date: Mon, 17 Jul 2023 17:23:39 +0200 Subject: [PATCH] Update client auth for 3004+ and support TLS certificate verification. --- CHANGES.md | 13 +++++ README.md | 12 ++-- actions/client.py | 4 +- actions/lib/base.py | 91 +++++++++++++++++-------------- actions/lib/meta.py | 127 ++++++++++++++++++++++++------------------- actions/lib/utils.py | 53 +++++++----------- actions/local.py | 65 ++++++++++------------ actions/runner.py | 30 ++++------ config.schema.yaml | 10 ++-- pack.yaml | 2 +- requirements.txt | 1 - 11 files changed, 205 insertions(+), 203 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1b6a669..a6ea6e2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,18 @@ # Change Log +## Unreleased; 3.0.0 + +### Add +- Added TLS connection verification. +- Special handling for test.ping and test.version with empty lists. + +### Change +- Formatted Python code with black. +- Updated client authentication to work with Salt 3004 to 3006 (maybe higher) + +### Removed +- Removed salt-pepper as a python dependency. + ## 2.0.1 * Drop Python 2.7 support diff --git a/README.md b/README.md index 7ae4835..e3183b5 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ be running on and where the StackStorm packs are installed. ## Usage Options -### Scenario 1: ST2 Installed on a Salt Master +### Scenario 1: StackStorm Installed on a Salt Master #### Configuration -If ST2 is installed on the master, no local configuration is required. +If StackStorm is installed on the master, no local configuration is required. #### Examples @@ -30,7 +30,7 @@ If ST2 is installed on the master, no local configuration is required. st2 run salt.bootstrap instance_id= provider=my-nova name=web.example.com ``` -### Scenario 2: ST2 Using Salt NetAPI +### Scenario 2: StackStorm using Salt NetAPI #### Configuration @@ -47,7 +47,7 @@ password: clams ``` **Note** : When modifying the configuration in `/opt/stackstorm/configs/` please - remember to tell StackStorm to load these new values by running + remember reload StackStorm to load these new values with the command: `st2ctl reload --register-configs` #### Examples @@ -68,9 +68,9 @@ One can also use the generic "runner" action to execute arbitrary runners and ex ### Actions -Saltstack runner/execution module function calls are represented as Stackstorm actions. Considering Saltstack's [`archive` execution module](http://docs.saltstack.com/en/2014.7/ref/modules/all/salt.modules.archive.html#module-salt.modules.archive), every function would be exposed as an Stackstorm action. +Saltstack runner/execution module function calls are represented as StackStorm actions. Considering Saltstack's [`archive` execution module](http://docs.saltstack.com/en/2014.7/ref/modules/all/salt.modules.archive.html#module-salt.modules.archive), every function would be exposed as an StackStorm action. -Stackstorm actions for this pack are namespaced relative to their Saltstack NetAPI client name and module name. Thus having the form: +StackStorm actions for this pack are namespaced relative to their Saltstack NetAPI client name and module name. Thus having the form: `[NetAPI client name]_[module name].[function name]` diff --git a/actions/client.py b/actions/client.py index a54ca21..7113b1c 100644 --- a/actions/client.py +++ b/actions/client.py @@ -5,13 +5,13 @@ class SaltClientAction(Action): def run(self, matches, module, args=[], kwargs={}): - ''' + """ CLI Examples: st2 run salt.client matches='web*' module=test.ping st2 run salt.client module=pkg.install \ kwargs='{"pkgs":["git","httpd"]}' - ''' + """ cli = salt.client.LocalClient() if args is None: ret = cli.cmd(matches, module, kwarg=kwargs) diff --git a/actions/lib/base.py b/actions/lib/base.py index 9590c96..4ba588d 100644 --- a/actions/lib/base.py +++ b/actions/lib/base.py @@ -1,28 +1,16 @@ # pylint: disable=no-member from st2common.runners.base_action import Action -from requests import Request +import requests from lib.utils import sanitize_payload class SaltPackage(object): - _expression_forms = [ - 'glob', - 'grain', - 'pillar', - 'nodegroup', - 'list', - 'compound' - ] + _expression_forms = ["glob", "grain", "pillar", "nodegroup", "list", "compound"] - def __init__(self, client='local'): - self._data = {"eauth": "", - "username": "", - "password": "", - "client": "", - "fun": ""} - - self._data['client'] = client + def __init__(self, client="local"): + self._data = {"client": "", "fun": ""} + self._data["client"] = client @property def data(self): @@ -35,36 +23,55 @@ def data(self, key_value=[]): class SaltAction(Action): + sensitive_keys = ["eauth", "password"] def __init__(self, config): - super(SaltAction, self).__init__(config=config) - self.url = self.config.get('api_url', None) - self.eauth = self.config.get('eauth', None) - self.username = self.config.get('username', None) - self.password = self.config.get('password', None) + super().__init__(config=config) + self.url = self.config.get("api_url", None) + self.eauth = self.config.get("eauth", None) + self.username = self.config.get("username", None) + self.password = self.config.get("password", None) + self.verify_tls = self.config.get("verify_tls", self.config.get("verify_ssl", True)) + + def login(self): + """ + Authenticate with Salt API to receive an authentication token. + """ + resp = requests.request( + "POST", + f"{self.url}/login", + json={"eauth": self.eauth, "username": self.username, "password": self.password}, + verify=self.verify_tls, + ) + token = resp.headers.get("X-Auth-Token", "failed-login") + return token - def generate_package(self, client='local', cmd=None, - **kwargs): + def generate_package(self, client="local", cmd=None, **kwargs): self.data = SaltPackage(client).data - self.data['eauth'] = self.eauth - self.data['username'] = self.username - self.data['password'] = self.password + if cmd: - self.data['fun'] = cmd - if client == 'local': - self.data['tgt'] = kwargs.get('target', '*') - self.data['tgt_type'] = kwargs.get('tgt_type', 'glob') - if isinstance(kwargs.get('args', []), list) and len(kwargs.get('args', [])) > 0: - self.data['arg'] = kwargs['args'] - if len(kwargs.get('data', {})) > 0: - if kwargs['data'].get('kwargs', None) is not None: - self.data['kwarg'] = kwargs['data']['kwargs'] - clean_payload = sanitize_payload(('username', 'password'), self.data) - self.logger.info("[salt] Payload to be sent: {0}".format(clean_payload)) + self.data["fun"] = cmd + if client == "local": + self.data["tgt"] = kwargs.get("target", "*") + self.data["tgt_type"] = kwargs.get("tgt_type", "glob") + if isinstance(kwargs.get("args", []), list) and len(kwargs.get("args", [])) > 0: + self.data["arg"] = kwargs["args"] + if len(kwargs.get("data", {})) > 0: + if kwargs["data"].get("kwargs", None) is not None: + self.data["kwarg"] = kwargs["data"]["kwargs"] + clean_payload = sanitize_payload(SaltAction.sensitive_keys, self.data) + self.logger.info("[salt] Payload to be sent: %s", clean_payload) def generate_request(self): - req = Request('POST', - "{0}/run".format(self.url), - headers={'content-type': 'application/json', - 'charset': 'utf-8'}) + req = requests.Request( + "POST", + self.url, + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "charset": "utf-8", + "x-auth-token": self.login(), + "User-Agent": "St2 Salt pack", + }, + ) return req.prepare() diff --git a/actions/lib/meta.py b/actions/lib/meta.py index 325921a..fbc2475 100644 --- a/actions/lib/meta.py +++ b/actions/lib/meta.py @@ -3,76 +3,89 @@ runner_action_meta = { "name": "", "parameters": { - "action": { - "type": "string", - "immutable": True, - "default": "" - }, - "kwargs": { - "type": "object", - "required": False - } + "action": {"type": "string", "immutable": True, "default": ""}, + "kwargs": {"type": "object", "required": False}, }, "runner_type": "python-script", "description": "Run Salt Runner functions through Salt API", "enabled": True, - "entry_point": "runner.py"} + "entry_point": "runner.py", +} local_action_meta = { "name": "", "parameters": { - "action": { - "type": "string", - "immutable": True, - "default": "" - }, - "kwargs": { - "type": "object", - "required": False - } + "action": {"type": "string", "immutable": True, "default": ""}, + "kwargs": {"type": "object", "required": False}, }, "runner_type": "python-script", "description": "Run Salt Runner functions through Salt API", "enabled": True, - "entry_point": "runner.py"} + "entry_point": "runner.py", +} actions = { - 'archive': ['gunzip', 'gzip', 'rar', 'tar', 'unrar', 'unzip', 'zip_'], - 'cloud': ['action', 'create', 'destroy', 'network_create', 'profile_', - 'virtual_interface_create', 'volume_attach', 'volume_create', - 'volume_delete', 'volume_detach'], - 'cmdmod': ['run', 'run_chroot', 'script'], - 'cp': ['get_file', 'get_url', 'push', 'push_dir'], - 'cron': ['ls', 'rm_job', 'set_job', 'set_env', 'rm_env'], - 'data': ['cas', 'getval', 'update', 'dump'], - 'event': ['fire', 'fire_master', 'send'], - 'file': ['access', 'chgrp', 'chown', 'directory_exists', 'file_exists', - 'find', 'manage_file', 'mkdir', 'remove', 'replace', 'search', - 'symlink', 'touch', 'truncate'], - 'grains': ['append', 'delval', 'get', 'remove', 'setval'], - 'hosts': ['add_hosts', 'get_alias', 'get_ip', 'rm_host', 'set_host'], - 'htpasswd': ['useradd', 'userdel'], - 'mine': ['delete', 'get', 'send', 'update'], - 'network': ['connect', 'ipaddrs', 'interface_ip', 'ping', 'subnets'], - 'pillar': ['get'], - 'pip': ['install', 'freeze', 'uninstall'], - 'pkg': ['install', 'refresh_db', 'remove'], - 'puppet': ['enable', 'disable', 'fact', - 'noop', 'status', 'run', 'summary'], - 'ret': ['get_fun', 'get_jid', 'get_jids', 'get_minions'], - 'saltutil': ['sync_all', 'sync_modules', 'sync_grains', 'sync_outputters', - 'sync_renderers', 'sync_returners', - 'sync_states', 'sync_utils'], - 'schedule': ['add', 'delete', 'enable_job', 'disable_job', 'run_job'], - 'service': ['available', 'start', 'restart', 'status', 'stop'], - 'shadow': ['del_password', 'gen_password', 'set_expire', ''], - 'state': ['highstate', 'single', 'sls'], - 'supervisord': ['add', 'remove', 'restart', - 'reread', 'start', 'stop', 'custom'], - 'systemd': ['available', 'disable', 'restart', 'enable', - 'stop', 'start', 'systemctl_reload'], - 'test': ['ping', 'cross_test', 'echo'], - 'useradd': ['add', 'delete', 'chshell'], - + "archive": ["gunzip", "gzip", "rar", "tar", "unrar", "unzip", "zip_"], + "cloud": [ + "action", + "create", + "destroy", + "network_create", + "profile_", + "virtual_interface_create", + "volume_attach", + "volume_create", + "volume_delete", + "volume_detach", + ], + "cmdmod": ["run", "run_chroot", "script"], + "cp": ["get_file", "get_url", "push", "push_dir"], + "cron": ["ls", "rm_job", "set_job", "set_env", "rm_env"], + "data": ["cas", "getval", "update", "dump"], + "event": ["fire", "fire_master", "send"], + "file": [ + "access", + "chgrp", + "chown", + "directory_exists", + "file_exists", + "find", + "manage_file", + "mkdir", + "remove", + "replace", + "search", + "symlink", + "touch", + "truncate", + ], + "grains": ["append", "delval", "get", "remove", "setval"], + "hosts": ["add_hosts", "get_alias", "get_ip", "rm_host", "set_host"], + "htpasswd": ["useradd", "userdel"], + "mine": ["delete", "get", "send", "update"], + "network": ["connect", "ipaddrs", "interface_ip", "ping", "subnets"], + "pillar": ["get"], + "pip": ["install", "freeze", "uninstall"], + "pkg": ["install", "refresh_db", "remove"], + "puppet": ["enable", "disable", "fact", "noop", "status", "run", "summary"], + "ret": ["get_fun", "get_jid", "get_jids", "get_minions"], + "saltutil": [ + "sync_all", + "sync_modules", + "sync_grains", + "sync_outputters", + "sync_renderers", + "sync_returners", + "sync_states", + "sync_utils", + ], + "schedule": ["add", "delete", "enable_job", "disable_job", "run_job"], + "service": ["available", "start", "restart", "status", "stop"], + "shadow": ["del_password", "gen_password", "set_expire", ""], + "state": ["highstate", "single", "sls"], + "supervisord": ["add", "remove", "restart", "reread", "start", "stop", "custom"], + "systemd": ["available", "disable", "restart", "enable", "stop", "start", "systemctl_reload"], + "test": ["ping", "cross_test", "echo"], + "useradd": ["add", "delete", "chshell"], } diff --git a/actions/lib/utils.py b/actions/lib/utils.py index 3140e28..02fc489 100644 --- a/actions/lib/utils.py +++ b/actions/lib/utils.py @@ -6,63 +6,48 @@ runner_action_meta = { "name": "", "parameters": { - "action": { - "type": "string", - "immutable": True, - "default": "" - }, - "kwargs": { - "type": "object", - "required": False - } + "action": {"type": "string", "immutable": True, "default": ""}, + "kwargs": {"type": "object", "required": False}, }, "runner_type": "python-script", "description": "Run Salt Runner functions through Salt API", "enabled": True, - "entry_point": "runner.py"} + "entry_point": "runner.py", +} local_action_meta = { "name": "", "parameters": { - "action": { - "type": "string", - "immutable": True, - "default": "" - }, - "args": { - "type": "array", - "required": False - }, - "kwargs": { - "type": "object", - "required": False - } + "action": {"type": "string", "immutable": True, "default": ""}, + "args": {"type": "array", "required": False}, + "kwargs": {"type": "object", "required": False}, }, "runner_type": "python-script", "description": "Run Salt Execution modules through Salt API", "enabled": True, - "entry_point": "local.py"} + "entry_point": "local.py", +} def generate_actions(): def create_file(mt, m, a): manifest = local_action_meta - manifest['name'] = "{0}_{1}.{2}".format(mt, m, a) - manifest['parameters']['action']['default'] = "{0}.{1}".format(m, a) + manifest["name"] = f"{mt}_{m}.{a}" + manifest["parameters"]["action"]["default"] = f"{m}.{a}" - fh = open('{0}_{1}.{2}.yaml'.format(mt, m, a), 'w') - fh.write('---\n') + fh = open(f"{mt}_{m}.{a}.yaml", "w") + fh.write("---\n") fh.write(yaml.dump(manifest, default_flow_style=False)) fh.close() + for key in actions: - map(lambda l: create_file('local', key, l), actions[key]) + map(lambda l: create_file("local", key, l), actions[key]) def sanitize_payload(keys_to_sanitize, payload): - ''' - Removes sensitive data from payloads before - publishing to the logs - ''' + """ + Removes sensitive data from payloads before publishing to the logs + """ data = payload.copy() for k in keys_to_sanitize: @@ -70,7 +55,7 @@ def sanitize_payload(keys_to_sanitize, payload): if not val: continue else: - val = '*' * 8 + val = "*" * 8 data[k] = val return data diff --git a/actions/local.py b/actions/local.py index a42daee..0dd9564 100644 --- a/actions/local.py +++ b/actions/local.py @@ -6,50 +6,41 @@ class SaltLocal(SaltAction): __explicit__ = [ - 'cmdmod', - 'event', - 'file', - 'grains', - 'pillar', - 'pkg', - 'saltcloudmod', - 'schedule', - 'service', - 'state', - 'status' + "cmdmod", + "event", + "file", + "grains", + "pillar", + "pkg", + "saltcloudmod", + "schedule", + "service", + "state", + "status", ] def run(self, module, target, tgt_type, args, **kwargs): - self.verify_ssl = self.config.get('verify_ssl', True) - ''' + """ CLI Examples: st2 run salt.local module=test.ping matches='web*' st2 run salt.local module=test.ping tgt_type=grain target='os:Ubuntu' - ''' - - # ChatOps alias and newer ST2 versions set default args=[] - # This breaks test.ping & test.version - - if module not in ['test.ping', 'test.version']: - self.generate_package('local', - cmd=module, - target=target, - tgt_type=tgt_type, - args=args, - data=kwargs) - else: - self.generate_package('local', - cmd=module, - target=target, - tgt_type=tgt_type, - args=None, - data=kwargs) + """ + + # ChatOps alias and newer St2 versions set default args=[] which + # breaks test.ping & test.version + if args == [] and module in ["test.ping", "test.version"]: + args = None + + self.generate_package( + "local", cmd=module, target=target, tgt_type=tgt_type, args=args, data=kwargs + ) request = self.generate_request() - self.logger.info('[salt] Request generated') request.prepare_body(json.dumps(self.data), None) - self.logger.info('[salt] Preparing to send') - resp = Session().send(request, verify=self.verify_ssl) - self.logger.debug('[salt] Response http code: %s', resp.status_code) - return resp.json() + resp = Session().send(request, verify=self.verify_tls) + try: + retval = resp.json() + except Exception as exc: + retval = (False, f"Failed to decode json! {str(exc)}") + return retval diff --git a/actions/runner.py b/actions/runner.py index 3d08727..7bf15c1 100644 --- a/actions/runner.py +++ b/actions/runner.py @@ -6,30 +6,24 @@ class SaltRunner(SaltAction): - __explicit__ = [ - 'jobs', - 'manage', - 'pillar', - 'mine', - 'network' - ] + __explicit__ = ["jobs", "manage", "pillar", "mine", "network"] def run(self, module, **kwargs): - self.verify_ssl = self.config.get('verify_ssl', True) - _cmd = module - ''' + """ CLI Examples: st2 run salt.runner_jobs.active st2 run salt.runner_jobs.list_jobs - ''' - self.generate_package('runner', cmd=_cmd) - if kwargs.get('kwargs', None) is not None: - self.data.update(kwargs['kwargs']) + """ + _cmd = module + + self.generate_package("runner", cmd=_cmd) + if kwargs.get("kwargs", None) is not None: + self.data.update(kwargs["kwargs"]) request = self.generate_request() - self.logger.info('[salt] Request generated') + self.logger.info("[salt] Request generated") request.prepare_body(json.dumps(self.data), None) - self.logger.info('[salt] Preparing to send') - resp = Session().send(request, verify=self.verify_ssl) - self.logger.debug('[salt] Response http code: %s', resp.status_code) + self.logger.info("[salt] Preparing to send") + resp = Session().send(request, verify=self.verify_tls) + self.logger.debug("[salt] Response http code: %s", resp.status_code) return resp.json() diff --git a/config.schema.yaml b/config.schema.yaml index 7c29fad..7197f62 100644 --- a/config.schema.yaml +++ b/config.schema.yaml @@ -1,27 +1,27 @@ --- api_url: - description: "Salt NetAPI URL if using remote server" + description: "Salt NetAPI URL if using remote server." type: "string" secret: false required: false username: - description: "Username for Salt NetAPI integration" + description: "Username for Salt NetAPI integration." type: "string" secret: false required: false password: - description: "Password for Salt NetAPI integration" + description: "Password for Salt NetAPI integration." type: "string" secret: true required: false eauth: - description: "Salt External Authentication System" + description: "Salt external authentication system." type: "string" secret: false required: false default: pam verify_ssl: - description: "Whether ST2 should verify the Salt NetAPI SSL certificate or not" + description: "True: verify the Salt API certificate is valid." type: "boolean" secret: false required: false diff --git a/pack.yaml b/pack.yaml index 51c9a5b..6708b84 100644 --- a/pack.yaml +++ b/pack.yaml @@ -6,7 +6,7 @@ keywords: - salt - cfg management - configuration management -version: 2.0.1 +version: 3.0.0 author : jcockhren email : jurnell@sophicware.com python_versions: diff --git a/requirements.txt b/requirements.txt index f97a9d7..1fa27ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ salt -salt-pepper requests