From d2f425c177e148d7ce5e933f39ba666a669e576f Mon Sep 17 00:00:00 2001 From: Ricardo Amaro Date: Wed, 21 Mar 2018 16:16:37 +0000 Subject: [PATCH 1/2] fixed secret encoding and data revealing (py3) --- kapitan/secrets.py | 3 ++- kapitan/targets.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kapitan/secrets.py b/kapitan/secrets.py index 7089add4..ec6b493f 100644 --- a/kapitan/secrets.py +++ b/kapitan/secrets.py @@ -205,7 +205,8 @@ def reveal_gpg_replace(gpg_obj, secrets_path, match_obj, verify=True, **kwargs): if verify: _, token_path, token_hash = secret_token_compiled_attributes(token) secret_raw_obj = secret_gpg_raw_read(secrets_path, token) - secret_hash = hashlib.sha256("%s%s".encode("UTF-8") % (token_path, secret_raw_obj["data"])).hexdigest() + secret_tag = "%s%s" % (token_path, secret_raw_obj["data"]) + secret_hash = hashlib.sha256(secret_tag.encode("UTF-8")).hexdigest() secret_hash = secret_hash[:8] logger.debug("Attempting to reveal token %s with secret hash %s", token, token_hash) if secret_hash != token_hash: diff --git a/kapitan/targets.py b/kapitan/targets.py index 887b0dce..fefb745f 100644 --- a/kapitan/targets.py +++ b/kapitan/targets.py @@ -361,7 +361,7 @@ def sub_token_reveal_obj(self, obj): for k, v in obj.items(): obj[k] = self.sub_token_reveal_obj(v) elif isinstance(obj, list): - obj = map(self.sub_token_reveal_obj, obj) + obj = list(map(self.sub_token_reveal_obj, obj)) elif isinstance(obj, string_types): obj = self.sub_token_reveal_data(obj) From b4ea1815afc9bb4381a3665812937f13d74d919c Mon Sep 17 00:00:00 2001 From: Ricardo Amaro Date: Thu, 22 Mar 2018 14:49:03 +0000 Subject: [PATCH 2/2] add support to update, validate target secrets. tests too --- .../inventory/targets/minikube-es.yml | 3 +- .../inventory/targets/minikube-mysql.yml | 3 +- kapitan/cli.py | 60 +++++++++++-- kapitan/secrets.py | 87 +++++++++++++++---- kapitan/version.py | 3 +- tests/test_secrets.py | 37 +++++++- 6 files changed, 166 insertions(+), 27 deletions(-) diff --git a/examples/kubernetes/inventory/targets/minikube-es.yml b/examples/kubernetes/inventory/targets/minikube-es.yml index 0d64d585..28dae6c6 100644 --- a/examples/kubernetes/inventory/targets/minikube-es.yml +++ b/examples/kubernetes/inventory/targets/minikube-es.yml @@ -25,7 +25,8 @@ parameters: secrets: recipients: - - dummy@recipient + - name: example@kapitan.dev + fingerprint: D9234C61F58BEB3ED8552A57E28DC07A3CBFAE7C namespace: ${target_name} elasticsearch: diff --git a/examples/kubernetes/inventory/targets/minikube-mysql.yml b/examples/kubernetes/inventory/targets/minikube-mysql.yml index ed260507..28d238d3 100644 --- a/examples/kubernetes/inventory/targets/minikube-mysql.yml +++ b/examples/kubernetes/inventory/targets/minikube-mysql.yml @@ -30,7 +30,8 @@ parameters: - docs/mysql/README.md secrets: recipients: - - example@kapitan.dev + - name: example@kapitan.dev + fingerprint: D9234C61F58BEB3ED8552A57E28DC07A3CBFAE7C namespace: ${target_name} mysql: diff --git a/kapitan/cli.py b/kapitan/cli.py index 64687cf7..dab3237e 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -14,10 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - "command line module" +from __future__ import print_function + import argparse import json import logging @@ -31,7 +31,9 @@ from kapitan.resources import search_imports, resource_callbacks, inventory_reclass from kapitan.version import PROJECT_NAME, DESCRIPTION, VERSION from kapitan.secrets import secret_gpg_backend, secret_gpg_write, secret_gpg_reveal_file -from kapitan.secrets import secret_gpg_reveal_dir, secret_gpg_reveal_raw +from kapitan.secrets import secret_gpg_reveal_dir, secret_gpg_reveal_raw, secret_gpg_update_recipients +from kapitan.secrets import search_target_token_paths, secret_gpg_raw_read_fingerprints +from kapitan.secrets import lookup_fingerprints from kapitan.errors import KapitanError logger = logging.getLogger(__name__) @@ -101,12 +103,18 @@ def main(): secrets_parser = subparser.add_parser('secrets', help='manage secrets') secrets_parser.add_argument('--write', '-w', help='write secret token', metavar='TOKENNAME',) + secrets_parser.add_argument('--update', help='update recipients for secret token', + metavar='TOKENNAME',) + secrets_parser.add_argument('--update-targets', action='store_true', default=False, + help='update target secrets') + secrets_parser.add_argument('--validate-targets', action='store_true', default=False, + help='validate target secrets') secrets_parser.add_argument('--base64', '-b64', help='base64 encode file content', action='store_true', default=False) secrets_parser.add_argument('--reveal', '-r', help='reveal secrets', action='store_true', default=False) secrets_parser.add_argument('--file', '-f', help='read file or directory, set "-" for stdin', - required=True, metavar='FILENAME') + metavar='FILENAME') secrets_parser.add_argument('--target-name', '-t', help='grab recipients from target name') secrets_parser.add_argument('--inventory-path', default='./inventory', help='set inventory path, default is "./inventory"') @@ -126,7 +134,11 @@ def main(): logger.debug('Running with args: %s', args) - cmd = sys.argv[1] + try: + cmd = sys.argv[1] + except IndexError: + parser.print_help() + sys.exit(1) if cmd == 'eval': file_path = args.jsonnet_file @@ -189,8 +201,10 @@ def main(): logging.basicConfig(level=logging.INFO, format="%(message)s") gpg_obj = secret_gpg_backend() if args.write is not None: + if args.file is None: + parser.error('--file is required with --write') data = None - recipients = args.recipients + recipients = [dict((("name", name),)) for name in args.recipients] if args.target_name: inv = inventory_reclass(args.inventory_path) recipients = inv['nodes'][args.target_name]['parameters']['kapitan']['secrets']['recipients'] @@ -203,6 +217,8 @@ def main(): data = fp.read() secret_gpg_write(gpg_obj, args.secrets_path, args.write, data, args.base64, recipients) elif args.reveal: + if args.file is None: + parser.error('--file is required with --reveal') if args.file == '-': secret_gpg_reveal_raw(gpg_obj, args.secrets_path, None, verify=(not args.no_verify)) elif args.file: @@ -213,3 +229,35 @@ def main(): elif os.path.isdir(args.file): secret_gpg_reveal_dir(gpg_obj, args.secrets_path, args.file, verify=(not args.no_verify)) + elif args.update: + # update recipients for secret tag + recipients = [dict([("name", name),]) for name in args.recipients] + if args.target_name: + inv = inventory_reclass(args.inventory_path) + recipients = inv['nodes'][args.target_name]['parameters']['kapitan']['secrets']['recipients'] + secret_gpg_update_recipients(gpg_obj, args.secrets_path, args.update, recipients) + elif args.update_targets or args.validate_targets: + # update recipients for all secrets in secrets_path + # use --secrets-path to set scanning path + inv = inventory_reclass(args.inventory_path) + targets = set(inv['nodes'].keys()) + secrets_path = os.path.abspath(args.secrets_path) + target_token_paths = search_target_token_paths(secrets_path, targets) + ret_code = 0 + for target_name, token_paths in target_token_paths.items(): + try: + recipients = inv['nodes'][target_name]['parameters']['kapitan']['secrets']['recipients'] + for token_path in token_paths: + target_fingerprints = set(lookup_fingerprints(gpg_obj, recipients)) + secret_fingerprints = set(secret_gpg_raw_read_fingerprints(secrets_path, token_path)) + if target_fingerprints != secret_fingerprints: + if args.validate_targets: + logger.info("%s recipient mismatch", token_path) + ret_code = 1 + else: + new_recipients = [dict([("fingerprint", f),]) for f in target_fingerprints] + secret_gpg_update_recipients(gpg_obj, secrets_path, token_path, new_recipients) + except KeyError: + logger.debug("secret_gpg_update_target: target: %s has no inventory recipients, skipping", + target_name) + sys.exit(ret_code) diff --git a/kapitan/secrets.py b/kapitan/secrets.py index ec6b493f..426d9853 100644 --- a/kapitan/secrets.py +++ b/kapitan/secrets.py @@ -16,9 +16,9 @@ "secrets module" -from six import string_types import base64 +from collections import defaultdict import errno from functools import partial import hashlib @@ -31,6 +31,7 @@ import gnupg import yaml +from six import string_types from kapitan.utils import PrettyDumper logger = logging.getLogger(__name__) @@ -55,10 +56,10 @@ def secret_gpg_backend(): return gnupg.GPG() -def secret_gpg_encrypt(gpg_obj, data, recipients, **kwargs): - "encrypt data with recipients keys" - assert isinstance(recipients, list) - return gpg_obj.encrypt(data, recipients, sign=True, armor=False, **kwargs) +def secret_gpg_encrypt(gpg_obj, data, fingerprints, **kwargs): + "encrypt data with fingerprints keys" + assert isinstance(fingerprints, list) + return gpg_obj.encrypt(data, fingerprints, sign=True, armor=False, **kwargs) def secret_gpg_decrypt(gpg_obj, data, **kwargs): @@ -132,21 +133,21 @@ def secret_token_compiled_attributes(token): raise ValueError('Token not valid: %s' % token) -def gpg_fingerprint(gpg_obj, recipient): - "returns first non-expired key fingerprint for recipient" +def gpg_fingerprint_non_expired(gpg_obj, recipient_name): + "returns first non-expired key fingerprint for recipient_name" try: - keys = gpg_obj.list_keys(keys=(recipient,)) + keys = gpg_obj.list_keys(keys=(recipient_name,)) for key in keys: # if 'expires' key is set and time in the future, return - if key['expires'] and (time.time() < int(key['expires'])): + if key.get('expires') and (time.time() < int(key['expires'])): return key['fingerprint'] # if 'expires' key not set, return - elif not key['expires']: + elif key.get('expires') is None: return key['fingerprint'] else: - logger.info("Key for recipient: %s with fingerprint: %s is expired, skipping", - recipient, key['fingerprint']) - raise GPGError("Could not find valid key for recipient: %s" % recipient) + logger.debug("Key for recipient: %s with fingerprint: %s is expired, skipping", + recipient_name, key['fingerprint']) + raise GPGError("Could not find valid key for recipient: %s" % recipient_name) except IndexError as iexp: raise iexp @@ -155,6 +156,9 @@ def secret_gpg_write(gpg_obj, secrets_path, token, data, encode_base64, recipien """ encrypt and write data for token in secrets_path set encode_base64 to True to base64 encode data before writing + recipients is a list of dictionaries with keys: name(required) fingerprint(optional) + if fingerprint key is not set in recipients, the first non-expired fingerprint will be used + if fingerprint is set, there will be no name based lookup """ _, token_path = secret_token_attributes("gpg:%s" % token) full_secret_path = os.path.join(secrets_path, token_path) @@ -170,10 +174,10 @@ def secret_gpg_write(gpg_obj, secrets_path, token, data, encode_base64, recipien if encode_base64: _data = base64.b64encode(data.encode("UTF-8")) encoding = "base64" - enc = secret_gpg_encrypt(gpg_obj, _data, recipients, **kwargs) + fingerprints = lookup_fingerprints(gpg_obj, recipients) + enc = secret_gpg_encrypt(gpg_obj, _data, fingerprints, **kwargs) if enc.ok: - b64data = base64.b64encode(enc.data) - fingerprints = [gpg_fingerprint(gpg_obj, r) for r in recipients] + b64data = base64.b64encode(enc.data).decode("UTF-8") secret_obj = {"data": b64data, "encoding": encoding, "recipients": [{'fingerprint': f} for f in fingerprints]} @@ -215,6 +219,19 @@ def reveal_gpg_replace(gpg_obj, secrets_path, match_obj, verify=True, **kwargs): logger.debug("Revealing %s", token_tag) return secret_gpg_read(gpg_obj, secrets_path, token, **kwargs) +def secret_gpg_update_recipients(gpg_obj, secrets_path, token_path, recipients, **kwargs): + "updates the recipient list for secret in token_path" + token = "gpg:%s" % token_path + secret_raw_obj = secret_gpg_raw_read(secrets_path, token) + data_dec = secret_gpg_read(gpg_obj, secrets_path, token, **kwargs) + encoding = secret_raw_obj.get('encoding', None) + encode_base64 = (encoding == 'base64') + + if encode_base64: + data_dec = base64.b64decode(data_dec).decode('UTF-8') + + secret_gpg_write(gpg_obj, secrets_path, token_path, data_dec, encode_base64, + recipients, **kwargs) def secret_gpg_reveal_raw(gpg_obj, secrets_path, filename, verify=True, output=None, **kwargs): """ @@ -317,3 +334,41 @@ def secret_gpg_reveal_file(gpg_obj, secrets_path, filename, verify=True, **kwarg out = secret_gpg_reveal_raw(gpg_obj, secrets_path, filename, output=devnull, verify=verify, **kwargs) return out + +def search_target_token_paths(target_secrets_path, targets): + """ + returns dict of target and their secret token paths in target_secrets_path + targets is a set of target names used to lookup targets in target_secrets_path + """ + target_files = defaultdict(list) + for root, _, files in os.walk(target_secrets_path): + for f in files: + full_path = os.path.join(root, f) + full_path = full_path[len(target_secrets_path)+1:] + target_name = full_path.split("/")[0] + if target_name in targets: + logger.debug('search_target_token_paths: found %s', full_path) + target_files[target_name].append(full_path) + return target_files + +def lookup_fingerprints(gpg_obj, recipients): + "returns a list of fingerprints for recipients obj" + lookedup = [] + for recipient in recipients: + fingerprint = recipient.get('fingerprint') + name = recipient.get('name') + if fingerprint is None: + lookedup_fingerprint = gpg_fingerprint_non_expired(gpg_obj, name) + lookedup.append(lookedup_fingerprint) + else: + # If fingerprint already set, don't lookup and reuse + lookedup.append(fingerprint) + + return lookedup + +def secret_gpg_raw_read_fingerprints(secrets_path, token_path): + "returns fingerprint list in raw secret for token_path" + token = "gpg:%s" % token_path + secret_raw_obj = secret_gpg_raw_read(secrets_path, token) + secret_raw_obj_fingerprints = [r['fingerprint'] for r in secret_raw_obj['recipients']] + return secret_raw_obj_fingerprints diff --git a/kapitan/version.py b/kapitan/version.py index 063f293b..df18a1c4 100644 --- a/kapitan/version.py +++ b/kapitan/version.py @@ -18,7 +18,8 @@ PROJECT_NAME = 'kapitan' VERSION = '0.12.0' -DESCRIPTION = 'Kapitan is a tool to manage kubernetes configuration using jsonnet templates' +DESCRIPTION = ('Generic templated configuration management for Kubernetes,' + 'Terraform and other things') AUTHOR = 'Ricardo Amaro' AUTHOR_EMAIL = 'ramaro@google.com' LICENCE = 'Apache License 2.0' diff --git a/tests/test_secrets.py b/tests/test_secrets.py index 0e691fc3..9f66d92d 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -22,6 +22,7 @@ import gnupg from kapitan.secrets import secret_token_attributes, SECRET_TOKEN_TAG_PATTERN from kapitan.secrets import secret_gpg_write, secret_gpg_reveal_raw +from kapitan.secrets import secret_gpg_update_recipients, secret_gpg_raw_read_fingerprints GPG_HOME = tempfile.mkdtemp() GPG_OBJ = gnupg.GPG(gnupghome=GPG_HOME) @@ -29,6 +30,9 @@ KEY = GPG_OBJ.gen_key(GPG_OBJ.gen_key_input(key_type="RSA", key_length=2048, passphrase="testphrase")) +KEY2 = GPG_OBJ.gen_key(GPG_OBJ.gen_key_input(key_type="RSA", + key_length=2048, + passphrase="testphrase")) class SecretsTest(unittest.TestCase): "Test secrets" def test_secret_token_attributes(self): @@ -44,7 +48,8 @@ def test_gpg_secret_write_reveal(self): "write secret, confirm secret file exists, reveal and compare content" token = 'secret/sauce' secret_gpg_write(GPG_OBJ, SECRETS_HOME, token, "super secret value", - False, [KEY.fingerprint], passphrase="testphrase") + False, [{'fingerprint': KEY.fingerprint}], + passphrase="testphrase") self.assertTrue(os.path.isfile(os.path.join(SECRETS_HOME, token))) file_with_secret_tags = tempfile.mktemp() @@ -64,7 +69,8 @@ def test_gpg_secret_base64_write_reveal(self): """ token = 'secret/sauce' secret_gpg_write(GPG_OBJ, SECRETS_HOME, token, "super secret value", - True, [KEY.fingerprint], passphrase="testphrase") + True, [{'fingerprint': KEY.fingerprint}], + passphrase="testphrase") self.assertTrue(os.path.isfile(os.path.join(SECRETS_HOME, token))) file_with_secret_tags = tempfile.mktemp() @@ -76,3 +82,30 @@ def test_gpg_secret_base64_write_reveal(self): verify=False, output=fp, passphrase="testphrase") with open(file_revealed) as fp: self.assertEqual("I am a file with a c3VwZXIgc2VjcmV0IHZhbHVl", fp.read()) + + def test_gpg_secret_update_recipients(self): + """ + update existing secret with another recipient, confirm content is the same + """ + token = 'secret/sauce' + secret_gpg_write(GPG_OBJ, SECRETS_HOME, token, "super secret value", + True, [{'fingerprint': KEY.fingerprint}], + passphrase="testphrase") + self.assertTrue(os.path.isfile(os.path.join(SECRETS_HOME, token))) + self.assertTrue(len(secret_gpg_raw_read_fingerprints(SECRETS_HOME, token)), 1) + + file_with_secret_tags = tempfile.mktemp() + file_revealed = tempfile.mktemp() + with open(file_with_secret_tags, 'w') as fp: + fp.write('I am a file with a ?{gpg:secret/sauce:deadbeef}') + + new_recipients = [{'fingerprint': KEY.fingerprint}, + {'fingerprint': KEY2.fingerprint}] + secret_gpg_update_recipients(GPG_OBJ, SECRETS_HOME, token, new_recipients, + passphrase="testphrase") + self.assertTrue(len(secret_gpg_raw_read_fingerprints(SECRETS_HOME, token)), 2) + with open(file_revealed, 'w') as fp: + secret_gpg_reveal_raw(GPG_OBJ, SECRETS_HOME, file_with_secret_tags, + verify=False, output=fp, passphrase="testphrase") + with open(file_revealed) as fp: + self.assertEqual("I am a file with a c3VwZXIgc2VjcmV0IHZhbHVl", fp.read())