Merge pull request salt-formulas#36 from ramaro/target_secrets

Target secrets support

examples/kubernetes/inventory/targets/minikube-es.yml

@@ -25,7 +25,8 @@ parameters:
 
     secrets:
       recipients:
-        - dummy@recipient
+        - name: [email protected]
+          fingerprint: D9234C61F58BEB3ED8552A57E28DC07A3CBFAE7C
   namespace: ${target_name}
 
   elasticsearch:

examples/kubernetes/inventory/targets/minikube-mysql.yml

@@ -30,7 +30,8 @@ parameters:
         - docs/mysql/README.md
     secrets:
       recipients:
-        - [email protected]
+        - name: [email protected]
+          fingerprint: D9234C61F58BEB3ED8552A57E28DC07A3CBFAE7C
   namespace: ${target_name}
 
   mysql:

kapitan/cli.py

@@ -14,10 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 "command line module"
 
+from __future__ import print_function
+
 import argparse
 import json
 import logging
@@ -31,7 +31,9 @@
 from kapitan.resources import search_imports, resource_callbacks, inventory_reclass
 from kapitan.version import PROJECT_NAME, DESCRIPTION, VERSION
 from kapitan.secrets import secret_gpg_backend, secret_gpg_write, secret_gpg_reveal_file
-from kapitan.secrets import secret_gpg_reveal_dir, secret_gpg_reveal_raw
+from kapitan.secrets import secret_gpg_reveal_dir, secret_gpg_reveal_raw, secret_gpg_update_recipients
+from kapitan.secrets import search_target_token_paths, secret_gpg_raw_read_fingerprints
+from kapitan.secrets import lookup_fingerprints
 from kapitan.errors import KapitanError
 
 logger = logging.getLogger(__name__)
@@ -101,12 +103,18 @@ def main():
     secrets_parser = subparser.add_parser('secrets', help='manage secrets')
     secrets_parser.add_argument('--write', '-w', help='write secret token',
                                 metavar='TOKENNAME',)
+    secrets_parser.add_argument('--update', help='update recipients for secret token',
+                                metavar='TOKENNAME',)
+    secrets_parser.add_argument('--update-targets', action='store_true', default=False,
+                                help='update target secrets')
+    secrets_parser.add_argument('--validate-targets', action='store_true', default=False,
+                                help='validate target secrets')
     secrets_parser.add_argument('--base64', '-b64', help='base64 encode file content',
                                 action='store_true', default=False)
     secrets_parser.add_argument('--reveal', '-r', help='reveal secrets',
                                 action='store_true', default=False)
     secrets_parser.add_argument('--file', '-f', help='read file or directory, set "-" for stdin',
-                                required=True, metavar='FILENAME')
+                                metavar='FILENAME')
     secrets_parser.add_argument('--target-name', '-t', help='grab recipients from target name')
     secrets_parser.add_argument('--inventory-path', default='./inventory',
                                 help='set inventory path, default is "./inventory"')
@@ -126,7 +134,11 @@ def main():
 
     logger.debug('Running with args: %s', args)
 
-    cmd = sys.argv[1]
+    try:
+        cmd = sys.argv[1]
+    except IndexError:
+        parser.print_help()
+        sys.exit(1)
 
     if cmd == 'eval':
         file_path = args.jsonnet_file
@@ -189,8 +201,10 @@ def main():
             logging.basicConfig(level=logging.INFO, format="%(message)s")
         gpg_obj = secret_gpg_backend()
         if args.write is not None:
+            if args.file is None:
+                parser.error('--file is required with --write')
             data = None
-            recipients = args.recipients
+            recipients = [dict((("name", name),)) for name in args.recipients]
             if args.target_name:
                 inv = inventory_reclass(args.inventory_path)
                 recipients = inv['nodes'][args.target_name]['parameters']['kapitan']['secrets']['recipients']
@@ -203,6 +217,8 @@ def main():
                     data = fp.read()
             secret_gpg_write(gpg_obj, args.secrets_path, args.write, data, args.base64, recipients)
         elif args.reveal:
+            if args.file is None:
+                parser.error('--file is required with --reveal')
             if args.file == '-':
                 secret_gpg_reveal_raw(gpg_obj, args.secrets_path, None, verify=(not args.no_verify))
             elif args.file:
@@ -213,3 +229,35 @@ def main():
                 elif os.path.isdir(args.file):
                     secret_gpg_reveal_dir(gpg_obj, args.secrets_path, args.file,
                                           verify=(not args.no_verify))
+        elif args.update:
+            # update recipients for secret tag
+            recipients = [dict([("name", name),]) for name in args.recipients]
+            if args.target_name:
+                inv = inventory_reclass(args.inventory_path)
+                recipients = inv['nodes'][args.target_name]['parameters']['kapitan']['secrets']['recipients']
+            secret_gpg_update_recipients(gpg_obj, args.secrets_path, args.update, recipients)
+        elif args.update_targets or args.validate_targets:
+            # update recipients for all secrets in secrets_path
+            # use --secrets-path to set scanning path
+            inv = inventory_reclass(args.inventory_path)
+            targets = set(inv['nodes'].keys())
+            secrets_path = os.path.abspath(args.secrets_path)
+            target_token_paths = search_target_token_paths(secrets_path, targets)
+            ret_code = 0
+            for target_name, token_paths in target_token_paths.items():
+                try:
+                    recipients = inv['nodes'][target_name]['parameters']['kapitan']['secrets']['recipients']
+                    for token_path in token_paths:
+                        target_fingerprints = set(lookup_fingerprints(gpg_obj, recipients))
+                        secret_fingerprints = set(secret_gpg_raw_read_fingerprints(secrets_path, token_path))
+                        if target_fingerprints != secret_fingerprints:
+                            if args.validate_targets:
+                                logger.info("%s recipient mismatch", token_path)
+                                ret_code = 1
+                            else:
+                                new_recipients = [dict([("fingerprint", f),]) for f in target_fingerprints]
+                                secret_gpg_update_recipients(gpg_obj, secrets_path, token_path, new_recipients)
+                except KeyError:
+                    logger.debug("secret_gpg_update_target: target: %s has no inventory recipients, skipping",
+                                 target_name)
+            sys.exit(ret_code)

kapitan/secrets.py

@@ -16,9 +16,9 @@
 
 "secrets module"
 
-from six import string_types
 
 import base64
+from collections import defaultdict
 import errno
 from functools import partial
 import hashlib
@@ -31,6 +31,7 @@
 import gnupg
 import yaml
 
+from six import string_types
 from kapitan.utils import PrettyDumper
 
 logger = logging.getLogger(__name__)
@@ -55,10 +56,10 @@ def secret_gpg_backend():
     return gnupg.GPG()
 
 
-def secret_gpg_encrypt(gpg_obj, data, recipients, **kwargs):
-    "encrypt data with recipients keys"
-    assert isinstance(recipients, list)
-    return gpg_obj.encrypt(data, recipients, sign=True, armor=False, **kwargs)
+def secret_gpg_encrypt(gpg_obj, data, fingerprints, **kwargs):
+    "encrypt data with fingerprints keys"
+    assert isinstance(fingerprints, list)
+    return gpg_obj.encrypt(data, fingerprints, sign=True, armor=False, **kwargs)
 
 
 def secret_gpg_decrypt(gpg_obj, data, **kwargs):
@@ -132,21 +133,21 @@ def secret_token_compiled_attributes(token):
         raise ValueError('Token not valid: %s' % token)
 
 
-def gpg_fingerprint(gpg_obj, recipient):
-    "returns first non-expired key fingerprint for recipient"
+def gpg_fingerprint_non_expired(gpg_obj, recipient_name):
+    "returns first non-expired key fingerprint for recipient_name"
     try:
-        keys = gpg_obj.list_keys(keys=(recipient,))
+        keys = gpg_obj.list_keys(keys=(recipient_name,))
         for key in keys:
             # if 'expires' key is set and time in the future, return
-            if key['expires'] and (time.time() < int(key['expires'])):
+            if key.get('expires') and (time.time() < int(key['expires'])):
                 return key['fingerprint']
             # if 'expires' key not set, return
-            elif not key['expires']:
+            elif key.get('expires') is None:
                 return key['fingerprint']
             else:
-                logger.info("Key for recipient: %s with fingerprint: %s is expired, skipping",
-                            recipient, key['fingerprint'])
-        raise GPGError("Could not find valid key for recipient: %s" % recipient)
+                logger.debug("Key for recipient: %s with fingerprint: %s is expired, skipping",
+                             recipient_name, key['fingerprint'])
+        raise GPGError("Could not find valid key for recipient: %s" % recipient_name)
     except IndexError as iexp:
         raise iexp
 
@@ -155,6 +156,9 @@ def secret_gpg_write(gpg_obj, secrets_path, token, data, encode_base64, recipien
     """
     encrypt and write data for token in secrets_path
     set encode_base64 to True to base64 encode data before writing
+    recipients is a list of dictionaries with keys: name(required) fingerprint(optional)
+    if fingerprint key is not set in recipients, the first non-expired fingerprint will be used
+    if fingerprint is set, there will be no name based lookup
     """
     _, token_path = secret_token_attributes("gpg:%s" % token)
     full_secret_path = os.path.join(secrets_path, token_path)
@@ -170,10 +174,10 @@ def secret_gpg_write(gpg_obj, secrets_path, token, data, encode_base64, recipien
         if encode_base64:
             _data = base64.b64encode(data.encode("UTF-8"))
             encoding = "base64"
-        enc = secret_gpg_encrypt(gpg_obj, _data, recipients, **kwargs)
+        fingerprints = lookup_fingerprints(gpg_obj, recipients)
+        enc = secret_gpg_encrypt(gpg_obj, _data, fingerprints, **kwargs)
         if enc.ok:
-            b64data = base64.b64encode(enc.data)
-            fingerprints = [gpg_fingerprint(gpg_obj, r) for r in recipients]
+            b64data = base64.b64encode(enc.data).decode("UTF-8")
             secret_obj = {"data": b64data,
                           "encoding": encoding,
                           "recipients": [{'fingerprint': f} for f in fingerprints]}
@@ -205,7 +209,8 @@ def reveal_gpg_replace(gpg_obj, secrets_path, match_obj, verify=True, **kwargs):
     if verify:
         _, token_path, token_hash = secret_token_compiled_attributes(token)
         secret_raw_obj = secret_gpg_raw_read(secrets_path, token)
-        secret_hash = hashlib.sha256("%s%s".encode("UTF-8") % (token_path, secret_raw_obj["data"])).hexdigest()
+        secret_tag = "%s%s" % (token_path, secret_raw_obj["data"])
+        secret_hash = hashlib.sha256(secret_tag.encode("UTF-8")).hexdigest()
         secret_hash = secret_hash[:8]
         logger.debug("Attempting to reveal token %s with secret hash %s", token, token_hash)
         if secret_hash != token_hash:
@@ -214,6 +219,19 @@ def reveal_gpg_replace(gpg_obj, secrets_path, match_obj, verify=True, **kwargs):
     logger.debug("Revealing %s", token_tag)
     return secret_gpg_read(gpg_obj, secrets_path, token, **kwargs)
 
+def secret_gpg_update_recipients(gpg_obj, secrets_path, token_path, recipients, **kwargs):
+    "updates the recipient list for secret in token_path"
+    token = "gpg:%s" % token_path
+    secret_raw_obj = secret_gpg_raw_read(secrets_path, token)
+    data_dec = secret_gpg_read(gpg_obj, secrets_path, token, **kwargs)
+    encoding = secret_raw_obj.get('encoding', None)
+    encode_base64 = (encoding == 'base64')
+
+    if encode_base64:
+        data_dec = base64.b64decode(data_dec).decode('UTF-8')
+
+    secret_gpg_write(gpg_obj, secrets_path, token_path, data_dec, encode_base64,
+                     recipients, **kwargs)
 
 def secret_gpg_reveal_raw(gpg_obj, secrets_path, filename, verify=True, output=None, **kwargs):
     """
@@ -316,3 +334,41 @@ def secret_gpg_reveal_file(gpg_obj, secrets_path, filename, verify=True, **kwarg
         out = secret_gpg_reveal_raw(gpg_obj, secrets_path, filename, output=devnull,
                                     verify=verify, **kwargs)
     return out
+
+def search_target_token_paths(target_secrets_path, targets):
+    """
+    returns dict of target and their secret token paths in target_secrets_path
+    targets is a set of target names used to lookup targets in target_secrets_path
+    """
+    target_files = defaultdict(list)
+    for root, _, files in os.walk(target_secrets_path):
+        for f in files:
+            full_path = os.path.join(root, f)
+            full_path = full_path[len(target_secrets_path)+1:]
+            target_name = full_path.split("/")[0]
+            if target_name in targets:
+                logger.debug('search_target_token_paths: found %s', full_path)
+                target_files[target_name].append(full_path)
+    return target_files
+
+def lookup_fingerprints(gpg_obj, recipients):
+    "returns a list of fingerprints for recipients obj"
+    lookedup = []
+    for recipient in recipients:
+        fingerprint = recipient.get('fingerprint')
+        name = recipient.get('name')
+        if fingerprint is None:
+            lookedup_fingerprint = gpg_fingerprint_non_expired(gpg_obj, name)
+            lookedup.append(lookedup_fingerprint)
+        else:
+            # If fingerprint already set, don't lookup and reuse
+            lookedup.append(fingerprint)
+
+    return lookedup
+
+def secret_gpg_raw_read_fingerprints(secrets_path, token_path):
+    "returns fingerprint list in raw secret for token_path"
+    token = "gpg:%s" % token_path
+    secret_raw_obj = secret_gpg_raw_read(secrets_path, token)
+    secret_raw_obj_fingerprints = [r['fingerprint'] for r in secret_raw_obj['recipients']]
+    return secret_raw_obj_fingerprints

kapitan/targets.py

@@ -361,7 +361,7 @@ def sub_token_reveal_obj(self, obj):
             for k, v in obj.items():
                 obj[k] = self.sub_token_reveal_obj(v)
         elif isinstance(obj, list):
-            obj = map(self.sub_token_reveal_obj, obj)
+            obj = list(map(self.sub_token_reveal_obj, obj))
         elif isinstance(obj, string_types):
             obj = self.sub_token_reveal_data(obj)
 

kapitan/version.py

@@ -18,7 +18,8 @@
 
 PROJECT_NAME = 'kapitan'
 VERSION = '0.12.0'
-DESCRIPTION = 'Kapitan is a tool to manage kubernetes configuration using jsonnet templates'
+DESCRIPTION = ('Generic templated configuration management for Kubernetes,'
+               'Terraform and other things')
 AUTHOR = 'Ricardo Amaro'
 AUTHOR_EMAIL = '[email protected]'
 LICENCE = 'Apache License 2.0'

tests/test_secrets.py

@@ -22,13 +22,17 @@
 import gnupg
 from kapitan.secrets import secret_token_attributes, SECRET_TOKEN_TAG_PATTERN
 from kapitan.secrets import secret_gpg_write, secret_gpg_reveal_raw
+from kapitan.secrets import secret_gpg_update_recipients, secret_gpg_raw_read_fingerprints
 
 GPG_HOME = tempfile.mkdtemp()
 GPG_OBJ = gnupg.GPG(gnupghome=GPG_HOME)
 SECRETS_HOME = tempfile.mkdtemp()
 KEY = GPG_OBJ.gen_key(GPG_OBJ.gen_key_input(key_type="RSA",
                                             key_length=2048,
                                             passphrase="testphrase"))
+KEY2 = GPG_OBJ.gen_key(GPG_OBJ.gen_key_input(key_type="RSA",
+                                            key_length=2048,
+                                            passphrase="testphrase"))
 class SecretsTest(unittest.TestCase):
     "Test secrets"
     def test_secret_token_attributes(self):
@@ -44,7 +48,8 @@ def test_gpg_secret_write_reveal(self):
         "write secret, confirm secret file exists, reveal and compare content"
         token = 'secret/sauce'
         secret_gpg_write(GPG_OBJ, SECRETS_HOME, token, "super secret value",
-                         False, [KEY.fingerprint], passphrase="testphrase")
+                         False, [{'fingerprint': KEY.fingerprint}],
+                         passphrase="testphrase")
         self.assertTrue(os.path.isfile(os.path.join(SECRETS_HOME, token)))
 
         file_with_secret_tags = tempfile.mktemp()
@@ -64,7 +69,8 @@ def test_gpg_secret_base64_write_reveal(self):
         """
         token = 'secret/sauce'
         secret_gpg_write(GPG_OBJ, SECRETS_HOME, token, "super secret value",
-                         True, [KEY.fingerprint], passphrase="testphrase")
+                         True, [{'fingerprint': KEY.fingerprint}],
+                         passphrase="testphrase")
         self.assertTrue(os.path.isfile(os.path.join(SECRETS_HOME, token)))
 
         file_with_secret_tags = tempfile.mktemp()
@@ -76,3 +82,30 @@ def test_gpg_secret_base64_write_reveal(self):
                               verify=False, output=fp, passphrase="testphrase")
         with open(file_revealed) as fp:
             self.assertEqual("I am a file with a c3VwZXIgc2VjcmV0IHZhbHVl", fp.read())
+
+    def test_gpg_secret_update_recipients(self):
+        """
+        update existing secret with another recipient, confirm content is the same
+        """
+        token = 'secret/sauce'
+        secret_gpg_write(GPG_OBJ, SECRETS_HOME, token, "super secret value",
+                         True, [{'fingerprint': KEY.fingerprint}],
+                         passphrase="testphrase")
+        self.assertTrue(os.path.isfile(os.path.join(SECRETS_HOME, token)))
+        self.assertTrue(len(secret_gpg_raw_read_fingerprints(SECRETS_HOME, token)), 1)
+
+        file_with_secret_tags = tempfile.mktemp()
+        file_revealed = tempfile.mktemp()
+        with open(file_with_secret_tags, 'w') as fp:
+            fp.write('I am a file with a ?{gpg:secret/sauce:deadbeef}')
+
+        new_recipients = [{'fingerprint': KEY.fingerprint},
+                          {'fingerprint': KEY2.fingerprint}]
+        secret_gpg_update_recipients(GPG_OBJ, SECRETS_HOME, token, new_recipients,
+                                     passphrase="testphrase")
+        self.assertTrue(len(secret_gpg_raw_read_fingerprints(SECRETS_HOME, token)), 2)
+        with open(file_revealed, 'w') as fp:
+            secret_gpg_reveal_raw(GPG_OBJ, SECRETS_HOME, file_with_secret_tags,
+                              verify=False, output=fp, passphrase="testphrase")
+        with open(file_revealed) as fp:
+            self.assertEqual("I am a file with a c3VwZXIgc2VjcmV0IHZhbHVl", fp.read())