From c027542d8fb7ddbc58be21f7569b9cdec0bdc2a6 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Fri, 22 Sep 2023 16:31:59 +0200 Subject: [PATCH 1/6] feat: embedded env engine --- kapitan/refs/base64.py | 2 +- kapitan/refs/env.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/kapitan/refs/base64.py b/kapitan/refs/base64.py index 20795c2ce..5b0023f96 100644 --- a/kapitan/refs/base64.py +++ b/kapitan/refs/base64.py @@ -31,7 +31,7 @@ def __init__(self, data, from_base64=False, **kwargs): super().__init__(data, **kwargs) self.type_name = "base64" self.encoding = kwargs.get("encoding", "original") - self.embed_refs = kwargs.get("embed_refs", False) + self.embed_refs = kwargs.get("embed_refs", True) # TODO data should be bytes only if from_base64: diff --git a/kapitan/refs/env.py b/kapitan/refs/env.py index bb7dbe13c..c5f5c4ff8 100644 --- a/kapitan/refs/env.py +++ b/kapitan/refs/env.py @@ -9,18 +9,18 @@ import hashlib import os -from kapitan.refs.base import PlainRef, PlainRefBackend +from kapitan.refs.base64 import Base64Ref, Base64RefBackend DEFAULT_ENV_REF_VAR_PREFIX = "KAPITAN_VAR_" -class EnvRef(PlainRef): +class EnvRef(Base64Ref): def __init__(self, data, **kwargs): """ writes plain data, which is the "default" value if the ref cannot be located in the KAPITAN_VAR_* environment vars prefix during the reveal phase. """ - super().__init__(data, kwargs=kwargs) + super().__init__(data, from_base64=True, kwargs=kwargs) self.type_name = "env" def reveal(self): @@ -31,15 +31,15 @@ def reveal(self): var_key = "{}{}".format(DEFAULT_ENV_REF_VAR_PREFIX, path_part) return os.getenv(var_key, default=os.getenv(var_key.upper(), default=self.data)) - def compile(self): - """ - Override the way an EnvRef is compiled, since we want to reveal it via the env later. - """ - compiled = f"?{{{self.type_name}:{self.path}:{self.hash[:8]}}}" - return compiled + # def compile(self): + # """ + # Override the way an EnvRef is compiled, since we want to reveal it via the env later. + # """ + # compiled = f"?{{{self.type_name}:{self.path}:{self.hash[:8]}}}" + # return compiled -class EnvRefBackend(PlainRefBackend): +class EnvRefBackend(Base64RefBackend): def __init__(self, path, ref_type=EnvRef, **ref_kwargs): "Get and create EnvRefs" super().__init__(path, ref_type, ref_kwargs=ref_kwargs) From 03e52f722c2cacf2f32253928c7571d286f3c0e5 Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Mon, 25 Sep 2023 10:58:55 +0200 Subject: [PATCH 2/6] feat(refs): add multiline option (#20) --- kapitan/cli.py | 10 ++++++++++ kapitan/utils.py | 9 +++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/kapitan/cli.py b/kapitan/cli.py index 64a9ffaaa..088531de1 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -548,6 +548,16 @@ def build_parser(): help='set refs path, default is "./refs"', default=from_dot_kapitan("refs", "refs-path", "./refs"), ) + refs_parser.add_argument( + "--yaml-multiline-string-style", + "-L", + type=str, + choices=["literal", "folded", "double-quotes"], + metavar="STYLE", + action="store", + default=from_dot_kapitan("refs", "yaml-multiline-string-style", "double-quotes"), + help="set multiline string style to STYLE, default is 'double-quotes'", + ) lint_parser = subparser.add_parser( "lint", aliases=["l"], help="linter for inventory and refs", parents=[logger_parser] diff --git a/kapitan/utils.py b/kapitan/utils.py index 377fa2b9b..1d9940ba2 100644 --- a/kapitan/utils.py +++ b/kapitan/utils.py @@ -223,15 +223,20 @@ def multiline_str_presenter(dumper, data): Ref: https://github.com/yaml/pyyaml/issues/240#issuecomment-1018712495 """ # get parsed args from cached.py - compile_args = cached.args.get("compile", None) + compile_args = cached.args.get("compile") style = None if compile_args: style = compile_args.yaml_multiline_string_style # check for inventory args too - inventory_args = cached.args.get("inventory", None) + inventory_args = cached.args.get("inventory") if inventory_args: style = inventory_args.multiline_string_style + + # check for refs args too + refs_args = cached.args.get("refs") + if refs_args: + style = refs_args.yaml_multiline_string_style if style == "literal": style = "|" From 9a7e32c6082009a46a6595f8f85d0449067dfda6 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Mon, 25 Sep 2023 12:59:05 +0200 Subject: [PATCH 3/6] lint: apply black --- kapitan/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kapitan/utils.py b/kapitan/utils.py index 1d9940ba2..87c0de18f 100644 --- a/kapitan/utils.py +++ b/kapitan/utils.py @@ -232,7 +232,7 @@ def multiline_str_presenter(dumper, data): inventory_args = cached.args.get("inventory") if inventory_args: style = inventory_args.multiline_string_style - + # check for refs args too refs_args = cached.args.get("refs") if refs_args: From 5e2f9f3e1a9a5967dac112d46530ecaa1140a18e Mon Sep 17 00:00:00 2001 From: Matteo Voges <98756476+MatteoVoges@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:47:03 +0200 Subject: [PATCH 4/6] feat(refs): add new env engine (#21) --- kapitan/refs/base64.py | 2 +- kapitan/refs/env.py | 39 +++++++++++++++++++++++---------------- tests/test_refs.py | 16 ++++++++-------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/kapitan/refs/base64.py b/kapitan/refs/base64.py index 5b0023f96..20795c2ce 100644 --- a/kapitan/refs/base64.py +++ b/kapitan/refs/base64.py @@ -31,7 +31,7 @@ def __init__(self, data, from_base64=False, **kwargs): super().__init__(data, **kwargs) self.type_name = "base64" self.encoding = kwargs.get("encoding", "original") - self.embed_refs = kwargs.get("embed_refs", True) + self.embed_refs = kwargs.get("embed_refs", False) # TODO data should be bytes only if from_base64: diff --git a/kapitan/refs/env.py b/kapitan/refs/env.py index c5f5c4ff8..4c1fb2424 100644 --- a/kapitan/refs/env.py +++ b/kapitan/refs/env.py @@ -5,38 +5,45 @@ "environment refs module" -import base64 -import hashlib import os +from kapitan.errors import KapitanError from kapitan.refs.base64 import Base64Ref, Base64RefBackend -DEFAULT_ENV_REF_VAR_PREFIX = "KAPITAN_VAR_" +DEFAULT_ENV_REF_VAR_PREFIX = "KAPITAN_ENV_" + + +class EnvError(KapitanError): + """Generic Env errors""" + + pass class EnvRef(Base64Ref): def __init__(self, data, **kwargs): """ - writes plain data, which is the "default" value if the ref cannot be located in the KAPITAN_VAR_* - environment vars prefix during the reveal phase. + looks up KAPITAN_ENV_* when revealing """ - super().__init__(data, from_base64=True, kwargs=kwargs) + super().__init__(data, from_base64=True, **kwargs) self.type_name = "env" def reveal(self): """ Attempt to locate the variable in the environment w/ the suffix. """ - path_part = self.path.split("/")[-1] - var_key = "{}{}".format(DEFAULT_ENV_REF_VAR_PREFIX, path_part) - return os.getenv(var_key, default=os.getenv(var_key.upper(), default=self.data)) - - # def compile(self): - # """ - # Override the way an EnvRef is compiled, since we want to reveal it via the env later. - # """ - # compiled = f"?{{{self.type_name}:{self.path}:{self.hash[:8]}}}" - # return compiled + env_var_key = self.path + env_var = f"{DEFAULT_ENV_REF_VAR_PREFIX}{env_var_key}" + value = os.getenv(env_var, default=os.getenv(env_var.upper())) + if value is None: + raise EnvError(f"env: variable {env_var} is not defined") + return value + + def compile(self): + return f"?{{env:{self.path}}}" + + @classmethod + def from_path(cls, ref_full_path, **kwargs): + return cls(ref_full_path, **kwargs) class EnvRefBackend(Base64RefBackend): diff --git a/tests/test_refs.py b/tests/test_refs.py index 1fc998a3e..66f3da83b 100644 --- a/tests/test_refs.py +++ b/tests/test_refs.py @@ -16,7 +16,7 @@ from kapitan.errors import RefError, RefFromFuncError, RefHashMismatchError from kapitan.refs.base import PlainRef, RefController, RefParams, Revealer from kapitan.refs.base64 import Base64Ref -from kapitan.refs.env import DEFAULT_ENV_REF_VAR_PREFIX, EnvRef +from kapitan.refs.env import DEFAULT_ENV_REF_VAR_PREFIX, EnvError, EnvRef from kapitan.utils import get_entropy REFS_HOME = tempfile.mkdtemp() @@ -47,26 +47,26 @@ def test_plain_ref_reveal(self): def test_env_ref_compile(self): "check env ref compile() output is valid" - tag = "?{env:my/ref1_env}" + tag = "?{env:ref1_env}" REF_CONTROLLER[tag] = EnvRef(b"ref 1 env data") ref_obj = REF_CONTROLLER[tag] revealed = ref_obj.compile() - self.assertEqual(revealed, "?{env:my/ref1_env:877234e3}") + self.assertEqual(revealed, "?{env:ref1_env}") def test_env_ref_reveal_default(self): "check env ref reveal() output is original plain env data when environment var is not set" - tag = "?{env:my/ref1_env}" + tag = "?{env:ref1_env}" REF_CONTROLLER[tag] = EnvRef(b"ref 1 env data") ref_obj = REF_CONTROLLER[tag] - revealed = ref_obj.reveal() - self.assertEqual(revealed, "ref 1 env data") + with self.assertRaises(EnvError): + revealed = ref_obj.reveal() def test_env_ref_reveal_from_env(self): "check env ref reveal() output is value of environment var data when environment var is set" - tag = "?{env:my/ref1_env}" + tag = "?{env:ref1_env}" REF_CONTROLLER[tag] = EnvRef(b"ref 1 env data") ref_obj = REF_CONTROLLER[tag] - os.environ["{}{}".format(DEFAULT_ENV_REF_VAR_PREFIX, "ref1_env")] = "ref 1 env data from EV" + os.environ[f"{DEFAULT_ENV_REF_VAR_PREFIX}ref1_env"] = "ref 1 env data from EV" revealed = ref_obj.reveal() self.assertEqual(revealed, "ref 1 env data from EV") From 669f071db708b379cb592c3ee05c7608ae355aa1 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Mon, 25 Sep 2023 16:28:08 +0200 Subject: [PATCH 5/6] fix(env): set prefix to empty --- kapitan/refs/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kapitan/refs/env.py b/kapitan/refs/env.py index 4c1fb2424..00ee22591 100644 --- a/kapitan/refs/env.py +++ b/kapitan/refs/env.py @@ -10,7 +10,7 @@ from kapitan.errors import KapitanError from kapitan.refs.base64 import Base64Ref, Base64RefBackend -DEFAULT_ENV_REF_VAR_PREFIX = "KAPITAN_ENV_" +DEFAULT_ENV_REF_VAR_PREFIX = "" # "KAPITAN_ENV_" (testing only) class EnvError(KapitanError): From 135d072d989636c8fa738037f7206e267b3da1d4 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Mon, 25 Sep 2023 18:46:23 +0200 Subject: [PATCH 6/6] feat(resolver): add `filename` resolver family --- kapitan/inventory/resolvers.py | 44 +++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/kapitan/inventory/resolvers.py b/kapitan/inventory/resolvers.py index 4e2474e75..abb610848 100644 --- a/kapitan/inventory/resolvers.py +++ b/kapitan/inventory/resolvers.py @@ -112,15 +112,6 @@ def relpath(path: str, _node_): return relative_interpolation -def from_file(path: str): - if os.path.isfile(path): - with open(path, "r") as f: - return f.read() - else: - logger.error(f"from_file: file {path} does not exist") - raise - - def write_to_key(destination: str, origin: str, _root_): """ resolver function to write any content to different place in the inventory @@ -156,6 +147,31 @@ def write_to_key(destination: str, origin: str, _root_): return "DONE" +def from_file(path: str): + if os.path.isfile(path): + with open(path, "r") as f: + return f.read() + else: + logger.error(f"from_file: file {path} does not exist") + raise + + +def filename(_node_: Node): + return _node_._get_flag("filename") + + +def parent_filename(_parent_: Node): + return _parent_._get_flag("filename") + + +def path(_node_: Node): + return _node_._get_flag("path") + + +def parent_path(_parent_: Node): + return _parent_._get_flag("path") + + def condition_if(condition: str, config: dict): if bool(condition): return config @@ -186,6 +202,7 @@ def condition_equal(*configs): return all(config == configs[0] for config in configs) +# TODO: nexenio only def helm_dep(name: str, source: str): """kapitan template for a helm chart dependency""" return { @@ -197,6 +214,7 @@ def helm_dep(name: str, source: str): } +# TODO: nexenio only def helm_input(name: str): """kapitan template for a helm input type configuration""" return { @@ -214,6 +232,10 @@ def helm_input(name: str): } +# TODO: load / import user modules as resolver-libraries +# * connect with flag +# * merge as dict +# * register resolvers from dict def register_resolvers(inventory_path: str) -> None: """register pre-defined and user-defined resolvers""" replace = True @@ -233,6 +255,10 @@ def register_resolvers(inventory_path: str) -> None: OmegaConf.register_new_resolver("default", default, replace=replace) OmegaConf.register_new_resolver("write", write_to_key, replace=replace) OmegaConf.register_new_resolver("from_file", from_file, replace=replace) + OmegaConf.register_new_resolver("filename", filename, replace=replace) + OmegaConf.register_new_resolver("parent_filename", parent_filename, replace=replace) + OmegaConf.register_new_resolver("path", path, replace=replace) + OmegaConf.register_new_resolver("parent_path", parent_path, replace=replace) # boolean algebra OmegaConf.register_new_resolver("if", condition_if, replace=replace)