Merge branch 'dev' into fix/vault-server

kapitan/cli.py

@@ -548,6 +548,16 @@ def build_parser():
         help='set refs path, default is "./refs"',
         default=from_dot_kapitan("refs", "refs-path", "./refs"),
     )
+    refs_parser.add_argument(
+        "--yaml-multiline-string-style",
+        "-L",
+        type=str,
+        choices=["literal", "folded", "double-quotes"],
+        metavar="STYLE",
+        action="store",
+        default=from_dot_kapitan("refs", "yaml-multiline-string-style", "double-quotes"),
+        help="set multiline string style to STYLE, default is 'double-quotes'",
+    )
 
     lint_parser = subparser.add_parser(
         "lint", aliases=["l"], help="linter for inventory and refs", parents=[logger_parser]

kapitan/inventory/resolvers.py

@@ -112,15 +112,6 @@ def relpath(path: str, _node_):
     return relative_interpolation
 
 
-def from_file(path: str):
-    if os.path.isfile(path):
-        with open(path, "r") as f:
-            return f.read()
-    else:
-        logger.error(f"from_file: file {path} does not exist")
-        raise
-
-
 def write_to_key(destination: str, origin: str, _root_):
     """
     resolver function to write any content to different place in the inventory
@@ -156,6 +147,31 @@ def write_to_key(destination: str, origin: str, _root_):
     return "DONE"
 
 
+def from_file(path: str):
+    if os.path.isfile(path):
+        with open(path, "r") as f:
+            return f.read()
+    else:
+        logger.error(f"from_file: file {path} does not exist")
+        raise
+
+
+def filename(_node_: Node):
+    return _node_._get_flag("filename")
+
+
+def parent_filename(_parent_: Node):
+    return _parent_._get_flag("filename")
+
+
+def path(_node_: Node):
+    return _node_._get_flag("path")
+
+
+def parent_path(_parent_: Node):
+    return _parent_._get_flag("path")
+
+
 def condition_if(condition: str, config: dict):
     if bool(condition):
         return config
@@ -186,6 +202,7 @@ def condition_equal(*configs):
     return all(config == configs[0] for config in configs)
 
 
+# TODO: nexenio only
 def helm_dep(name: str, source: str):
     """kapitan template for a helm chart dependency"""
     return {
@@ -197,6 +214,7 @@ def helm_dep(name: str, source: str):
     }
 
 
+# TODO: nexenio only
 def helm_input(name: str):
     """kapitan template for a helm input type configuration"""
     return {
@@ -214,6 +232,10 @@ def helm_input(name: str):
     }
 
 
+# TODO: load / import user modules as resolver-libraries
+# * connect with flag
+# * merge as dict
+# * register resolvers  from dict
 def register_resolvers(inventory_path: str) -> None:
     """register pre-defined and user-defined resolvers"""
     replace = True
@@ -233,6 +255,10 @@ def register_resolvers(inventory_path: str) -> None:
     OmegaConf.register_new_resolver("default", default, replace=replace)
     OmegaConf.register_new_resolver("write", write_to_key, replace=replace)
     OmegaConf.register_new_resolver("from_file", from_file, replace=replace)
+    OmegaConf.register_new_resolver("filename", filename, replace=replace)
+    OmegaConf.register_new_resolver("parent_filename", parent_filename, replace=replace)
+    OmegaConf.register_new_resolver("path", path, replace=replace)
+    OmegaConf.register_new_resolver("parent_path", parent_path, replace=replace)
 
     # boolean algebra
     OmegaConf.register_new_resolver("if", condition_if, replace=replace)

kapitan/refs/env.py

@@ -5,41 +5,48 @@
 
 "environment refs module"
 
-import base64
-import hashlib
 import os
 
-from kapitan.refs.base import PlainRef, PlainRefBackend
+from kapitan.errors import KapitanError
+from kapitan.refs.base64 import Base64Ref, Base64RefBackend
 
-DEFAULT_ENV_REF_VAR_PREFIX = "KAPITAN_VAR_"
+DEFAULT_ENV_REF_VAR_PREFIX = ""  # "KAPITAN_ENV_" (testing only)
 
 
-class EnvRef(PlainRef):
+class EnvError(KapitanError):
+    """Generic Env errors"""
+
+    pass
+
+
+class EnvRef(Base64Ref):
     def __init__(self, data, **kwargs):
         """
-        writes plain data, which is the "default" value if the ref cannot be located in the KAPITAN_VAR_*
-        environment vars prefix during the reveal phase.
+        looks up KAPITAN_ENV_* when revealing
         """
-        super().__init__(data, kwargs=kwargs)
+        super().__init__(data, from_base64=True, **kwargs)
         self.type_name = "env"
 
     def reveal(self):
         """
         Attempt to locate the variable in the environment w/ the suffix.
         """
-        path_part = self.path.split("/")[-1]
-        var_key = "{}{}".format(DEFAULT_ENV_REF_VAR_PREFIX, path_part)
-        return os.getenv(var_key, default=os.getenv(var_key.upper(), default=self.data))
+        env_var_key = self.path
+        env_var = f"{DEFAULT_ENV_REF_VAR_PREFIX}{env_var_key}"
+        value = os.getenv(env_var, default=os.getenv(env_var.upper()))
+        if value is None:
+            raise EnvError(f"env: variable {env_var} is not defined")
+        return value
 
     def compile(self):
-        """
-        Override the way an EnvRef is compiled, since we want to reveal it via the env later.
-        """
-        compiled = f"?{{{self.type_name}:{self.path}:{self.hash[:8]}}}"
-        return compiled
+        return f"?{{env:{self.path}}}"
+
+    @classmethod
+    def from_path(cls, ref_full_path, **kwargs):
+        return cls(ref_full_path, **kwargs)
 
 
-class EnvRefBackend(PlainRefBackend):
+class EnvRefBackend(Base64RefBackend):
     def __init__(self, path, ref_type=EnvRef, **ref_kwargs):
         "Get and create EnvRefs"
         super().__init__(path, ref_type, ref_kwargs=ref_kwargs)

kapitan/utils.py

@@ -223,16 +223,21 @@ def multiline_str_presenter(dumper, data):
     Ref: https://github.com/yaml/pyyaml/issues/240#issuecomment-1018712495
     """
     # get parsed args from cached.py
-    compile_args = cached.args.get("compile", None)
+    compile_args = cached.args.get("compile")
     style = None
     if compile_args:
         style = compile_args.yaml_multiline_string_style
 
     # check for inventory args too
-    inventory_args = cached.args.get("inventory", None)
+    inventory_args = cached.args.get("inventory")
     if inventory_args:
         style = inventory_args.multiline_string_style
 
+    # check for refs args too
+    refs_args = cached.args.get("refs")
+    if refs_args:
+        style = refs_args.yaml_multiline_string_style
+
     if style == "literal":
         style = "|"
     elif style == "folded":

tests/test_refs.py

@@ -16,7 +16,7 @@
 from kapitan.errors import RefError, RefFromFuncError, RefHashMismatchError
 from kapitan.refs.base import PlainRef, RefController, RefParams, Revealer
 from kapitan.refs.base64 import Base64Ref
-from kapitan.refs.env import DEFAULT_ENV_REF_VAR_PREFIX, EnvRef
+from kapitan.refs.env import DEFAULT_ENV_REF_VAR_PREFIX, EnvError, EnvRef
 from kapitan.utils import get_entropy
 
 REFS_HOME = tempfile.mkdtemp()
@@ -47,26 +47,26 @@ def test_plain_ref_reveal(self):
 
     def test_env_ref_compile(self):
         "check env ref compile() output is valid"
-        tag = "?{env:my/ref1_env}"
+        tag = "?{env:ref1_env}"
         REF_CONTROLLER[tag] = EnvRef(b"ref 1 env data")
         ref_obj = REF_CONTROLLER[tag]
         revealed = ref_obj.compile()
-        self.assertEqual(revealed, "?{env:my/ref1_env:877234e3}")
+        self.assertEqual(revealed, "?{env:ref1_env}")
 
     def test_env_ref_reveal_default(self):
         "check env ref reveal() output is original plain env data when environment var is not set"
-        tag = "?{env:my/ref1_env}"
+        tag = "?{env:ref1_env}"
         REF_CONTROLLER[tag] = EnvRef(b"ref 1 env data")
         ref_obj = REF_CONTROLLER[tag]
-        revealed = ref_obj.reveal()
-        self.assertEqual(revealed, "ref 1 env data")
+        with self.assertRaises(EnvError):
+            revealed = ref_obj.reveal()
 
     def test_env_ref_reveal_from_env(self):
         "check env ref reveal() output is value of environment var data when environment var is set"
-        tag = "?{env:my/ref1_env}"
+        tag = "?{env:ref1_env}"
         REF_CONTROLLER[tag] = EnvRef(b"ref 1 env data")
         ref_obj = REF_CONTROLLER[tag]
-        os.environ["{}{}".format(DEFAULT_ENV_REF_VAR_PREFIX, "ref1_env")] = "ref 1 env data from EV"
+        os.environ[f"{DEFAULT_ENV_REF_VAR_PREFIX}ref1_env"] = "ref 1 env data from EV"
         revealed = ref_obj.reveal()
         self.assertEqual(revealed, "ref 1 env data from EV")