From eb4bcd33c18f58e9a5598e295c4143453775a56f Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Fri, 2 Feb 2024 14:03:59 +0100 Subject: [PATCH 1/3] chore: update dependencies and dockerfile for omegaconf --- Dockerfile | 4 +- poetry.lock | 209 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 2 + 3 files changed, 211 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2835ec0d3..80c9d116c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,9 @@ WORKDIR /kapitan RUN apt-get update \ && apt-get install --no-install-recommends -y \ curl \ - build-essential + build-essential \ + git \ + default-jre ENV POETRY_VERSION=1.7.1 ENV VIRTUAL_ENV=/opt/venv diff --git a/poetry.lock b/poetry.lock index eed4c2b34..8dc5063fe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "addict" version = "2.4.0" description = "Addict is a dictionary whose items can be set using both attribute and item syntax." +category = "main" optional = false python-versions = "*" files = [ @@ -11,10 +12,22 @@ files = [ {file = "addict-2.4.0.tar.gz", hash = "sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494"}, ] +[[package]] +name = "antlr4-python3-runtime" +version = "4.9.3" +description = "ANTLR 4.9.3 runtime for Python 3.7" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "antlr4-python3-runtime-4.9.3.tar.gz", hash = "sha256:f224469b4168294902bb1efa80a8bf7855f24c99aef99cbefc1bcd3cce77881b"}, +] + [[package]] name = "anyio" version = "4.2.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -37,6 +50,7 @@ trio = ["trio (>=0.23)"] name = "attrs" version = "23.2.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -56,6 +70,7 @@ tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "p name = "azure-common" version = "1.1.28" description = "Microsoft Azure Client Library for Python (Common)" +category = "main" optional = false python-versions = "*" files = [ @@ -67,6 +82,7 @@ files = [ name = "azure-core" version = "1.29.6" description = "Microsoft Azure Core Library for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -87,6 +103,7 @@ aio = ["aiohttp (>=3.0)"] name = "azure-identity" version = "1.15.0" description = "Microsoft Azure Identity Library for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -104,6 +121,7 @@ msal-extensions = ">=0.3.0,<2.0.0" name = "azure-keyvault-keys" version = "4.8.0" description = "Microsoft Azure Key Vault Keys Client Library for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -122,6 +140,7 @@ typing-extensions = ">=4.0.1" name = "boto3" version = "1.34.21" description = "The AWS SDK for Python" +category = "main" optional = false python-versions = ">= 3.8" files = [ @@ -141,6 +160,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.34.21" description = "Low-level, data-driven core of boto 3." +category = "main" optional = false python-versions = ">= 3.8" files = [ @@ -160,6 +180,7 @@ crt = ["awscrt (==0.19.19)"] name = "cachetools" version = "5.3.2" description = "Extensible memoizing collections and decorators" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -171,6 +192,7 @@ files = [ name = "certifi" version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -182,6 +204,7 @@ files = [ name = "cffi" version = "1.16.0" description = "Foreign Function Interface for Python calling C code." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -246,6 +269,7 @@ pycparser = "*" name = "charset-normalizer" version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -345,6 +369,7 @@ files = [ name = "cryptography" version = "41.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -390,6 +415,7 @@ test-randomorder = ["pytest-randomly"] name = "docker" version = "6.1.3" description = "A Python library for the Docker Engine API." +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -411,6 +437,7 @@ ssh = ["paramiko (>=2.4.3)"] name = "exceptiongroup" version = "1.2.0" description = "Backport of PEP 654 (exception groups)" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -425,6 +452,7 @@ test = ["pytest (>=6)"] name = "gitdb" version = "4.0.11" description = "Git Object Database" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -439,6 +467,7 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.41" description = "GitPython is a Python library used to interact with Git repositories" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -456,6 +485,7 @@ test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre name = "gojsonnet" version = "0.20.0" description = "Python bindings for Jsonnet - The data templating language" +category = "main" optional = true python-versions = "*" files = [ @@ -466,6 +496,7 @@ files = [ name = "google-api-core" version = "2.15.0" description = "Google API client core library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -488,6 +519,7 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] name = "google-api-python-client" version = "2.114.0" description = "Google API Client Library for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -496,7 +528,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0.dev0" +google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0.dev0" google-auth = ">=1.19.0,<3.0.0.dev0" google-auth-httplib2 = ">=0.1.0" httplib2 = ">=0.15.0,<1.dev0" @@ -506,6 +538,7 @@ uritemplate = ">=3.0.1,<5" name = "google-auth" version = "2.26.2" description = "Google Authentication Library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -529,6 +562,7 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] name = "google-auth-httplib2" version = "0.2.0" description = "Google Authentication Library: httplib2 transport" +category = "main" optional = false python-versions = "*" files = [ @@ -544,6 +578,7 @@ httplib2 = ">=0.19.0" name = "googleapis-common-protos" version = "1.62.0" description = "Common protobufs used in Google APIs" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -561,6 +596,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -575,6 +611,7 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 name = "hvac" version = "1.2.1" description = "HashiCorp Vault API client" +category = "main" optional = false python-versions = ">=3.6.2,<4.0.0" files = [ @@ -590,6 +627,7 @@ requests = ">=2.27.1,<3.0.0" name = "idna" version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -601,6 +639,7 @@ files = [ name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" optional = false python-versions = "*" files = [ @@ -615,6 +654,7 @@ six = "*" name = "jinja2" version = "3.1.3" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -632,6 +672,7 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -643,6 +684,7 @@ files = [ name = "jsonnet" version = "0.20.0" description = "Python bindings for Jsonnet - The data templating language" +category = "main" optional = false python-versions = "*" files = [ @@ -653,6 +695,7 @@ files = [ name = "jsonschema" version = "4.21.0" description = "An implementation of JSON Schema validation for Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -674,6 +717,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "jsonschema-specifications" version = "2023.12.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -688,6 +732,7 @@ referencing = ">=0.31.0" name = "kadet" version = "0.2.2" description = "Easily define and reuse complex Python objects that serialize into JSON or YAML." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -705,6 +750,7 @@ typeguard = ">=2.12.1" name = "markupsafe" version = "2.1.3" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -764,6 +810,7 @@ files = [ name = "msal" version = "1.26.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +category = "main" optional = false python-versions = ">=2.7" files = [ @@ -783,6 +830,7 @@ broker = ["pymsalruntime (>=0.13.2,<0.14)"] name = "msal-extensions" version = "1.1.0" description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -798,10 +846,31 @@ portalocker = [ {version = ">=1.6,<3", markers = "platform_system == \"Windows\""}, ] +[[package]] +name = "omegaconf" +version = "2.4.0.dev1" +description = "A flexible configuration library" +category = "main" +optional = false +python-versions = ">=3.8" +files = [] +develop = false + +[package.dependencies] +antlr4-python3-runtime = ">=4.9.0,<4.10.0" +PyYAML = ">=5.1.0" + +[package.source] +type = "git" +url = "https://github.com/neXenio/omegaconf.git" +reference = "dev" +resolved_reference = "118b46bcbf79de435e6ca4ec60d19c49be84f634" + [[package]] name = "packaging" version = "23.2" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -813,6 +882,7 @@ files = [ name = "pathspec" version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -824,6 +894,7 @@ files = [ name = "portalocker" version = "2.8.2" description = "Wraps the portalocker recipe for easy usage" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -843,6 +914,7 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p name = "protobuf" version = "4.25.2" description = "" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -863,6 +935,7 @@ files = [ name = "pyasn1" version = "0.5.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -874,6 +947,7 @@ files = [ name = "pyasn1-modules" version = "0.3.0" description = "A collection of ASN.1-based protocols modules" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -888,6 +962,7 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "pycparser" version = "2.21" description = "C parser in Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -899,6 +974,7 @@ files = [ name = "pydantic" version = "1.10.13" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -951,6 +1027,7 @@ email = ["email-validator (>=1.0.3)"] name = "pyhcl" version = "0.4.5" description = "HCL configuration parser for python" +category = "main" optional = false python-versions = "*" files = [ @@ -962,6 +1039,7 @@ files = [ name = "pyjwt" version = "2.8.0" description = "JSON Web Token implementation in Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -982,6 +1060,7 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pyparsing" version = "3.1.1" description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -996,6 +1075,7 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "python-box" version = "6.0.2" description = "Advanced Python dictionaries with dot notation access" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1027,6 +1107,7 @@ yaml = ["ruamel.yaml (>=0.17)"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1041,6 +1122,7 @@ six = ">=1.5" name = "python-gnupg" version = "0.5.2" description = "A wrapper for the Gnu Privacy Guard (GPG or GnuPG)" +category = "main" optional = false python-versions = "*" files = [ @@ -1052,6 +1134,7 @@ files = [ name = "python-magic" version = "0.4.27" description = "File type identification using libmagic" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1063,6 +1146,7 @@ files = [ name = "pywin32" version = "306" description = "Python for Window Extensions" +category = "main" optional = true python-versions = "*" files = [ @@ -1086,6 +1170,7 @@ files = [ name = "pyyaml" version = "6.0.1" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1135,6 +1220,7 @@ files = [ name = "referencing" version = "0.32.1" description = "JSON Referencing + Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1146,10 +1232,114 @@ files = [ attrs = ">=22.2.0" rpds-py = ">=0.7.0" +[[package]] +name = "regex" +version = "2023.12.25" +description = "Alternative regular expression module, to replace re." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0694219a1d54336fd0445ea382d49d36882415c0134ee1e8332afd1529f0baa5"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b014333bd0217ad3d54c143de9d4b9a3ca1c5a29a6d0d554952ea071cff0f1f8"}, + {file = "regex-2023.12.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d865984b3f71f6d0af64d0d88f5733521698f6c16f445bb09ce746c92c97c586"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e0eabac536b4cc7f57a5f3d095bfa557860ab912f25965e08fe1545e2ed8b4c"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c25a8ad70e716f96e13a637802813f65d8a6760ef48672aa3502f4c24ea8b400"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9b6d73353f777630626f403b0652055ebfe8ff142a44ec2cf18ae470395766e"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9cc99d6946d750eb75827cb53c4371b8b0fe89c733a94b1573c9dd16ea6c9e4"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88d1f7bef20c721359d8675f7d9f8e414ec5003d8f642fdfd8087777ff7f94b5"}, + {file = "regex-2023.12.25-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cb3fe77aec8f1995611f966d0c656fdce398317f850d0e6e7aebdfe61f40e1cd"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7aa47c2e9ea33a4a2a05f40fcd3ea36d73853a2aae7b4feab6fc85f8bf2c9704"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:df26481f0c7a3f8739fecb3e81bc9da3fcfae34d6c094563b9d4670b047312e1"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c40281f7d70baf6e0db0c2f7472b31609f5bc2748fe7275ea65a0b4601d9b392"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:d94a1db462d5690ebf6ae86d11c5e420042b9898af5dcf278bd97d6bda065423"}, + {file = "regex-2023.12.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ba1b30765a55acf15dce3f364e4928b80858fa8f979ad41f862358939bdd1f2f"}, + {file = "regex-2023.12.25-cp310-cp310-win32.whl", hash = "sha256:150c39f5b964e4d7dba46a7962a088fbc91f06e606f023ce57bb347a3b2d4630"}, + {file = "regex-2023.12.25-cp310-cp310-win_amd64.whl", hash = "sha256:09da66917262d9481c719599116c7dc0c321ffcec4b1f510c4f8a066f8768105"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:1b9d811f72210fa9306aeb88385b8f8bcef0dfbf3873410413c00aa94c56c2b6"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d902a43085a308cef32c0d3aea962524b725403fd9373dea18110904003bac97"}, + {file = "regex-2023.12.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d166eafc19f4718df38887b2bbe1467a4f74a9830e8605089ea7a30dd4da8887"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c7ad32824b7f02bb3c9f80306d405a1d9b7bb89362d68b3c5a9be53836caebdb"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:636ba0a77de609d6510235b7f0e77ec494d2657108f777e8765efc060094c98c"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fda75704357805eb953a3ee15a2b240694a9a514548cd49b3c5124b4e2ad01b"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f72cbae7f6b01591f90814250e636065850c5926751af02bb48da94dfced7baa"}, + {file = "regex-2023.12.25-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db2a0b1857f18b11e3b0e54ddfefc96af46b0896fb678c85f63fb8c37518b3e7"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7502534e55c7c36c0978c91ba6f61703faf7ce733715ca48f499d3dbbd7657e0"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:e8c7e08bb566de4faaf11984af13f6bcf6a08f327b13631d41d62592681d24fe"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:283fc8eed679758de38fe493b7d7d84a198b558942b03f017b1f94dda8efae80"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:f44dd4d68697559d007462b0a3a1d9acd61d97072b71f6d1968daef26bc744bd"}, + {file = "regex-2023.12.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:67d3ccfc590e5e7197750fcb3a2915b416a53e2de847a728cfa60141054123d4"}, + {file = "regex-2023.12.25-cp311-cp311-win32.whl", hash = "sha256:68191f80a9bad283432385961d9efe09d783bcd36ed35a60fb1ff3f1ec2efe87"}, + {file = "regex-2023.12.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d2af3f6b8419661a0c421584cfe8aaec1c0e435ce7e47ee2a97e344b98f794f"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8a0ccf52bb37d1a700375a6b395bff5dd15c50acb745f7db30415bae3c2b0715"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c3c4a78615b7762740531c27cf46e2f388d8d727d0c0c739e72048beb26c8a9d"}, + {file = "regex-2023.12.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad83e7545b4ab69216cef4cc47e344d19622e28aabec61574b20257c65466d6a"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7a635871143661feccce3979e1727c4e094f2bdfd3ec4b90dfd4f16f571a87a"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d498eea3f581fbe1b34b59c697512a8baef88212f92e4c7830fcc1499f5b45a5"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:43f7cd5754d02a56ae4ebb91b33461dc67be8e3e0153f593c509e21d219c5060"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51f4b32f793812714fd5307222a7f77e739b9bc566dc94a18126aba3b92b98a3"}, + {file = "regex-2023.12.25-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba99d8077424501b9616b43a2d208095746fb1284fc5ba490139651f971d39d9"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4bfc2b16e3ba8850e0e262467275dd4d62f0d045e0e9eda2bc65078c0110a11f"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8c2c19dae8a3eb0ea45a8448356ed561be843b13cbc34b840922ddf565498c1c"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:60080bb3d8617d96f0fb7e19796384cc2467447ef1c491694850ebd3670bc457"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b77e27b79448e34c2c51c09836033056a0547aa360c45eeeb67803da7b0eedaf"}, + {file = "regex-2023.12.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:518440c991f514331f4850a63560321f833979d145d7d81186dbe2f19e27ae3d"}, + {file = "regex-2023.12.25-cp312-cp312-win32.whl", hash = "sha256:e2610e9406d3b0073636a3a2e80db05a02f0c3169b5632022b4e81c0364bcda5"}, + {file = "regex-2023.12.25-cp312-cp312-win_amd64.whl", hash = "sha256:cc37b9aeebab425f11f27e5e9e6cf580be7206c6582a64467a14dda211abc232"}, + {file = "regex-2023.12.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:da695d75ac97cb1cd725adac136d25ca687da4536154cdc2815f576e4da11c69"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d126361607b33c4eb7b36debc173bf25d7805847346dd4d99b5499e1fef52bc7"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4719bb05094d7d8563a450cf8738d2e1061420f79cfcc1fa7f0a44744c4d8f73"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5dd58946bce44b53b06d94aa95560d0b243eb2fe64227cba50017a8d8b3cd3e2"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22a86d9fff2009302c440b9d799ef2fe322416d2d58fc124b926aa89365ec482"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2aae8101919e8aa05ecfe6322b278f41ce2994c4a430303c4cd163fef746e04f"}, + {file = "regex-2023.12.25-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e692296c4cc2873967771345a876bcfc1c547e8dd695c6b89342488b0ea55cd8"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:263ef5cc10979837f243950637fffb06e8daed7f1ac1e39d5910fd29929e489a"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:d6f7e255e5fa94642a0724e35406e6cb7001c09d476ab5fce002f652b36d0c39"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:88ad44e220e22b63b0f8f81f007e8abbb92874d8ced66f32571ef8beb0643b2b"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:3a17d3ede18f9cedcbe23d2daa8a2cd6f59fe2bf082c567e43083bba3fb00347"}, + {file = "regex-2023.12.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d15b274f9e15b1a0b7a45d2ac86d1f634d983ca40d6b886721626c47a400bf39"}, + {file = "regex-2023.12.25-cp37-cp37m-win32.whl", hash = "sha256:ed19b3a05ae0c97dd8f75a5d8f21f7723a8c33bbc555da6bbe1f96c470139d3c"}, + {file = "regex-2023.12.25-cp37-cp37m-win_amd64.whl", hash = "sha256:a6d1047952c0b8104a1d371f88f4ab62e6275567d4458c1e26e9627ad489b445"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b43523d7bc2abd757119dbfb38af91b5735eea45537ec6ec3a5ec3f9562a1c53"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:efb2d82f33b2212898f1659fb1c2e9ac30493ac41e4d53123da374c3b5541e64"}, + {file = "regex-2023.12.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b7fca9205b59c1a3d5031f7e64ed627a1074730a51c2a80e97653e3e9fa0d415"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086dd15e9435b393ae06f96ab69ab2d333f5d65cbe65ca5a3ef0ec9564dfe770"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e81469f7d01efed9b53740aedd26085f20d49da65f9c1f41e822a33992cb1590"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:34e4af5b27232f68042aa40a91c3b9bb4da0eeb31b7632e0091afc4310afe6cb"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9852b76ab558e45b20bf1893b59af64a28bd3820b0c2efc80e0a70a4a3ea51c1"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ff100b203092af77d1a5a7abe085b3506b7eaaf9abf65b73b7d6905b6cb76988"}, + {file = "regex-2023.12.25-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cc038b2d8b1470364b1888a98fd22d616fba2b6309c5b5f181ad4483e0017861"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:094ba386bb5c01e54e14434d4caabf6583334090865b23ef58e0424a6286d3dc"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5cd05d0f57846d8ba4b71d9c00f6f37d6b97d5e5ef8b3c3840426a475c8f70f4"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:9aa1a67bbf0f957bbe096375887b2505f5d8ae16bf04488e8b0f334c36e31360"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:98a2636994f943b871786c9e82bfe7883ecdaba2ef5df54e1450fa9869d1f756"}, + {file = "regex-2023.12.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37f8e93a81fc5e5bd8db7e10e62dc64261bcd88f8d7e6640aaebe9bc180d9ce2"}, + {file = "regex-2023.12.25-cp38-cp38-win32.whl", hash = "sha256:d78bd484930c1da2b9679290a41cdb25cc127d783768a0369d6b449e72f88beb"}, + {file = "regex-2023.12.25-cp38-cp38-win_amd64.whl", hash = "sha256:b521dcecebc5b978b447f0f69b5b7f3840eac454862270406a39837ffae4e697"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f7bc09bc9c29ebead055bcba136a67378f03d66bf359e87d0f7c759d6d4ffa31"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e14b73607d6231f3cc4622809c196b540a6a44e903bcfad940779c80dffa7be7"}, + {file = "regex-2023.12.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9eda5f7a50141291beda3edd00abc2d4a5b16c29c92daf8d5bd76934150f3edc"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc6bb9aa69aacf0f6032c307da718f61a40cf970849e471254e0e91c56ffca95"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:298dc6354d414bc921581be85695d18912bea163a8b23cac9a2562bbcd5088b1"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f4e475a80ecbd15896a976aa0b386c5525d0ed34d5c600b6d3ebac0a67c7ddf"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:531ac6cf22b53e0696f8e1d56ce2396311254eb806111ddd3922c9d937151dae"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22f3470f7524b6da61e2020672df2f3063676aff444db1daa283c2ea4ed259d6"}, + {file = "regex-2023.12.25-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:89723d2112697feaa320c9d351e5f5e7b841e83f8b143dba8e2d2b5f04e10923"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0ecf44ddf9171cd7566ef1768047f6e66975788258b1c6c6ca78098b95cf9a3d"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:905466ad1702ed4acfd67a902af50b8db1feeb9781436372261808df7a2a7bca"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:4558410b7a5607a645e9804a3e9dd509af12fb72b9825b13791a37cd417d73a5"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:7e316026cc1095f2a3e8cc012822c99f413b702eaa2ca5408a513609488cb62f"}, + {file = "regex-2023.12.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3b1de218d5375cd6ac4b5493e0b9f3df2be331e86520f23382f216c137913d20"}, + {file = "regex-2023.12.25-cp39-cp39-win32.whl", hash = "sha256:11a963f8e25ab5c61348d090bf1b07f1953929c13bd2309a0662e9ff680763c9"}, + {file = "regex-2023.12.25-cp39-cp39-win_amd64.whl", hash = "sha256:e693e233ac92ba83a87024e1d32b5f9ab15ca55ddd916d878146f4e3406b5c91"}, + {file = "regex-2023.12.25.tar.gz", hash = "sha256:29171aa128da69afdf4bde412d5bedc335f2ca8fcfe4489038577d05f16181e5"}, +] + [[package]] name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1171,6 +1361,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rpds-py" version = "0.17.1" description = "Python bindings to Rust's persistent data structures (rpds)" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1279,6 +1470,7 @@ files = [ name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" +category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -1293,6 +1485,7 @@ pyasn1 = ">=0.1.3" name = "s3transfer" version = "0.10.0" description = "An Amazon S3 Transfer Manager" +category = "main" optional = false python-versions = ">= 3.8" files = [ @@ -1310,6 +1503,7 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1321,6 +1515,7 @@ files = [ name = "smmap" version = "5.0.1" description = "A pure Python implementation of a sliding window memory map manager" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1332,6 +1527,7 @@ files = [ name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1343,6 +1539,7 @@ files = [ name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1354,6 +1551,7 @@ files = [ name = "typeguard" version = "4.1.5" description = "Run-time type checker for Python" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1372,6 +1570,7 @@ test = ["coverage[toml] (>=7)", "mypy (>=1.2.0)", "pytest (>=7)"] name = "typing-extensions" version = "4.9.0" description = "Backported and Experimental Type Hints for Python 3.8+" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1383,6 +1582,7 @@ files = [ name = "uritemplate" version = "4.1.1" description = "Implementation of RFC 6570 URI Templates" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1394,6 +1594,7 @@ files = [ name = "urllib3" version = "2.0.7" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1411,6 +1612,7 @@ zstd = ["zstandard (>=0.18.0)"] name = "websocket-client" version = "1.7.0" description = "WebSocket client for Python with low level API options" +category = "main" optional = true python-versions = ">=3.8" files = [ @@ -1427,6 +1629,7 @@ test = ["websockets"] name = "yamllint" version = "1.33.0" description = "A linter for YAML files." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -1448,4 +1651,4 @@ test = ["docker"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "ac3fdfcfc318acd09a1e0113a1b50fe7059aa20afd0673b24c6a874bf9144ac4" +content-hash = "717f1fec5bfc1a4d44ddd2c4400e8688f07cd7621e6c1684daa63f37767836c9" diff --git a/pyproject.toml b/pyproject.toml index 0ac05cc67..cf4b819d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,8 +49,10 @@ jinja2 = "^3.0.1" jsonnet = "^0.20.0" jsonschema = "^4.17.3" kadet = "^0.2.2" +omegaconf = { git = "https://github.com/neXenio/omegaconf.git", branch = "dev" } python-gnupg = ">=0.4.7,<0.6.0" pyyaml = "^6.0" +regex = "^2023.5.5" requests = "^2.28.2" six = "^1.16.0" toml = "^0.10.2" From 221f25997d2f5c5ffc2ce1c5326858ea4fd1e2ae Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Fri, 2 Feb 2024 14:05:41 +0100 Subject: [PATCH 2/3] feat: add arg `migrate` to `inventory_backend` parser --- kapitan/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kapitan/cli.py b/kapitan/cli.py index f0bda931b..db4ade728 100644 --- a/kapitan/cli.py +++ b/kapitan/cli.py @@ -111,6 +111,12 @@ def build_parser(): choices=AVAILABLE_BACKENDS.keys(), help="Select the inventory backend to use (default=reclass)", ) + inventory_backend_parser.add_argument( + "--migrate", + action="store_true", + default=from_dot_kapitan("inventory_backend", "migrate", False), + help="Migrate your inventory to your selected inventory backend.", + ) eval_parser = subparser.add_parser("eval", aliases=["e"], help="evaluate jsonnet file") eval_parser.add_argument("jsonnet_file", type=str) From 49f2147ac790a7faa2e919ea28f969ff43027d5f Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Fri, 2 Feb 2024 14:06:23 +0100 Subject: [PATCH 3/3] feat: add omegaconf inventory core --- kapitan/inventory/__init__.py | 2 + kapitan/inventory/inv_omegaconf/__init__.py | 0 .../inventory/inv_omegaconf/inv_omegaconf.py | 274 ++++++++++++++++++ kapitan/inventory/inv_omegaconf/migrate.py | 77 +++++ kapitan/inventory/inv_omegaconf/resolvers.py | 272 +++++++++++++++++ kapitan/inventory/inventory.py | 6 + kapitan/resources.py | 4 + 7 files changed, 635 insertions(+) create mode 100644 kapitan/inventory/inv_omegaconf/__init__.py create mode 100644 kapitan/inventory/inv_omegaconf/inv_omegaconf.py create mode 100644 kapitan/inventory/inv_omegaconf/migrate.py create mode 100644 kapitan/inventory/inv_omegaconf/resolvers.py diff --git a/kapitan/inventory/__init__.py b/kapitan/inventory/__init__.py index c3c0f662d..cd16cb7f2 100644 --- a/kapitan/inventory/__init__.py +++ b/kapitan/inventory/__init__.py @@ -1,10 +1,12 @@ from typing import Type from .inv_reclass import ReclassInventory +from kapitan.inventory.inv_omegaconf.inv_omegaconf import OmegaConfInventory from .inventory import Inventory # Dict mapping values for command line flag `--inventory-backend` to the # associated `Inventory` subclass. AVAILABLE_BACKENDS: dict[str, Type[Inventory]] = { "reclass": ReclassInventory, + "omegaconf": OmegaConfInventory, } diff --git a/kapitan/inventory/inv_omegaconf/__init__.py b/kapitan/inventory/inv_omegaconf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kapitan/inventory/inv_omegaconf/inv_omegaconf.py b/kapitan/inventory/inv_omegaconf/inv_omegaconf.py new file mode 100644 index 000000000..146ece4ae --- /dev/null +++ b/kapitan/inventory/inv_omegaconf/inv_omegaconf.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The Kapitan Authors +# SPDX-FileCopyrightText: 2020 The Kapitan Authors +# +# SPDX-License-Identifier: Apache-2.0 + +import logging +import multiprocessing as mp +import os +from copy import deepcopy +from time import time + +import yaml +from omegaconf import ListMergeMode, OmegaConf + +from kapitan import cached +from .migrate import migrate +from ..inventory import InventoryError, Inventory +from .resolvers import register_resolvers + +logger = logging.getLogger(__name__) + + +class InventoryTarget: + targets_path: str + logfile: str + + def __init__(self, target_name: str, target_path: str) -> None: + self.path = target_path + self.name = target_name + + # compose node name + self.composed_name = ( + os.path.splitext(target_path)[0].replace(self.targets_path + os.sep, "").replace("/", ".") + ) + + self.classes: list = [] + self.parameters: dict = {} + self.classes_redundancy_check: set = set() + + def _merge(self, class_parameters): + if not self.parameters: + self.parameters = class_parameters + else: + merged_parameters = OmegaConf.unsafe_merge( + class_parameters, + self.parameters, + list_merge_mode=ListMergeMode.EXTEND, + ) + + self.parameters = merged_parameters + + def _resolve(self): + escape_interpolation_strings = False + OmegaConf.resolve(self.parameters, escape_interpolation_strings) + + # remove specified keys + remove_location = "omegaconf.remove" + removed_keys = OmegaConf.select(self.parameters, remove_location, default=[]) + for key in removed_keys: + OmegaConf.update(self.parameters, key, {}, merge=False) + + # resolve second time and convert to object + # add throw_on_missing = True when resolving second time (--> wait for to_object support) + # reference: https://github.com/omry/omegaconf/pull/1113 + OmegaConf.resolve(self.parameters, escape_interpolation_strings) + self.parameters = OmegaConf.to_container(self.parameters) + + def add_metadata(self): + # append meta data (legacy: _reclass_) + _meta_ = { + "name": { + "full": self.name, + "parts": self.name.split("."), + "path": self.name.replace(".", "/"), + "short": self.name, + } + } + self.parameters["_meta_"] = _meta_ + self.parameters["_reclass_"] = _meta_ # legacy + + +class InventoryClass: + classes_path: str = "./inventory/classes" + + def __init__(self, class_path: str) -> None: + self.path = class_path + self.name = os.path.splitext(class_path)[0].replace(self.classes_path + os.sep, "").replace("/", ".") + self.parameters = {} + self.dependents = [] + + +class OmegaConfInventory(Inventory): + classes_cache: dict = {} + + # InventoryTarget.targets_path = self.targets_searchpath + # InventoryClass.classes_path = self.classes_searchpath + + def inventory(self): + register_resolvers(self.inventory_path) + selected_targets = self.get_selected_targets() + + # FEAT: add flag for multiprocessing + use_mp = True + + if not use_mp: + nodes = {} + # load targets one by one + for target in selected_targets: + try: + self.load_target(target) + nodes[target.name] = {"parameters": target.parameters} + except Exception as e: + raise InventoryError(f"{target.name}: {e}") + else: + # load targets parallel + manager = mp.Manager() # perf: bottleneck --> 90 % of the inventory time + + nodes = manager.dict() + mp.set_start_method("spawn", True) # platform independent + with mp.Pool(len(selected_targets)) as pool: + r = pool.map_async( + self.inventory_worker, [(self, target, nodes) for target in selected_targets] + ) + r.wait() + + # using nodes for reclass legacy code + nodes = dict(nodes) + + # using nodes for reclass legacy code + return {"nodes": nodes} + + @staticmethod + def inventory_worker(zipped_args): + start = time() + self, target, nodes = zipped_args + + try: + register_resolvers(self.inventory_path) + self.load_target(target) + nodes[target.name] = {"parameters": target.parameters} + except Exception as e: + logger.error(f"{target.name}: {e}") + return + + logger.info(f"Rendered {target.name} ({time()-start:.2f}s)") + + def migrate(self): + migrate(self.inventory_path) + + # ---------- + # private + # ---------- + def get_selected_targets(self): + selected_targets = [] + + # loop through targets searchpath and load all targets + for root, dirs, files in os.walk(self.targets_searchpath): + for target_file in files: + # split file extension and check if yml/yaml + target_path = os.path.join(root, target_file) + target_name, ext = os.path.splitext(target_file) + if ext not in (".yml", ".yaml"): + logger.debug(f"{target_file}: targets have to be .yml or .yaml files.") + continue + + # skip targets if they are not specified with -t flag + if self.targets and target_name not in self.targets: + continue + + # initialize target + target = InventoryTarget(target_name, target_path) + if self.compose_node_name: + target.name = target.composed_name + selected_targets.append(target) + + return selected_targets + + def load_target(self, target: InventoryTarget): + """ + load only one target with all its classes + """ + + # load the target parameters + target.classes, target.parameters = self.load_config(target.path) + + # load classes for targets + for class_name in target.classes: + inv_class = self.load_class(target, class_name) + if not inv_class: + # either redundantly defined or not found (with ignore_not_found: true) + continue + + params = deepcopy(inv_class.parameters) + target._merge(params) + target.classes += inv_class.dependents + + if not target.parameters: + # improve error msg + raise InventoryError("empty target") + + # resolve interpolations + target.add_metadata() + target._resolve() + + # obtain target name to insert in inv dict + vars_target_name = target.parameters.get("kapitan", {}).get("vars", {}).get("target") + if not vars_target_name: + # add hint to kapitan.vars.target + logger.warning(f"Could not resolve target name on target {target.name}") + + def load_class(self, target: InventoryTarget, class_name: str): + # resolve class path (has to be absolute) + class_path = os.path.join(self.classes_searchpath, *class_name.split(".")) + if class_path in target.classes_redundancy_check: + logger.debug(f"{class_path}: class {class_name} is redundantly defined") + return None + target.classes_redundancy_check.add(class_path) + + # search in inventory classes cache, otherwise load class + if class_name in self.classes_cache.keys(): + return self.classes_cache[class_name] + + # check if file exists + if os.path.isfile(class_path + ".yml"): + class_path += ".yml" + elif os.path.isdir(class_path): + # search for init file + init_path = os.path.join(self.classes_searchpath, *class_name.split("."), "init") + ".yml" + if os.path.isfile(init_path): + class_path = init_path + elif self.ignore_class_notfound: + logger.debug(f"Could not find {class_path}") + return None + else: + raise InventoryError(f"Class {class_name} not found.") + + # load classes recursively + classes, parameters = self.load_config(class_path) + + if not classes and not parameters: + return None + + # initialize inventory class + inv_class = InventoryClass(class_path) + inv_class.parameters = parameters + # resolve relative class names for new classes + for c in classes: + if c.startswith("."): + c = ".".join(class_name.split(".")[0:-1]) + c + inv_class.dependents.append(c) + + # add class to cache + self.classes_cache[class_name] = inv_class + + return inv_class + + def load_config(self, path: str): + with open(path, "r") as f: + f.seek(0) + config = yaml.load(f, yaml.SafeLoader) + + if not config: + logger.debug(f"{path}: file is empty") + return [], {} + classes = OmegaConf.create(config.get("classes", [])) + parameters = OmegaConf.create(config.get("parameters", {})) + + # add metadata to nodes + filename = os.path.splitext(os.path.split(path)[1])[0] + parameters._set_flag(["filename", "path"], [filename, path], recursive=True) + + return classes, parameters \ No newline at end of file diff --git a/kapitan/inventory/inv_omegaconf/migrate.py b/kapitan/inventory/inv_omegaconf/migrate.py new file mode 100644 index 000000000..1282b8b9c --- /dev/null +++ b/kapitan/inventory/inv_omegaconf/migrate.py @@ -0,0 +1,77 @@ +import os +import sys + +from regex import regex + +from kapitan.inventory.inventory import InventoryError + + +def migrate(inventory_path: str): + # FEAT: write migrations to temp dir and copy only if succeeded + + if os.path.exists(inventory_path): + if os.path.isdir(inventory_path): + migrate_dir(inventory_path) + elif os.path.isfile(inventory_path): + migrate_file(inventory_path) + else: + print(f"Error while migrating: inventory path at {inventory_path} does not exist") + + +def migrate_dir(path: str): + """migrates all .yml/.yaml files in the given path to omegaconfs syntax""" + + for root, _, files in os.walk(path): + for file in files: + file = os.path.join(root, file) + name, ext = os.path.splitext(file) + + if ext not in (".yml", ".yaml"): + continue + + try: + migrate_file(file) + except Exception as e: + InventoryError(f"{file}: error with migration: {e}") + + +def migrate_file(file: str): + with open(file, "r") as fp: + content = fp.read() + + updated_content = migrate_str(content) + + with open(file, "w") as fp: + fp.write(updated_content) + + +def migrate_str(content: str): + # FEAT: don't migrate custom resolvers + # FEAT: migrate interpolations with '.' in the keyname + + # search for interpolation pattern + # migrate path delimiter + # migrate meta data name + updated_content = regex.sub( + r"(?") + print(f"Migrating all .yml/.yaml files in {sys.argv[1]}") + migrate(sys.argv[1]) diff --git a/kapitan/inventory/inv_omegaconf/resolvers.py b/kapitan/inventory/inv_omegaconf/resolvers.py new file mode 100644 index 000000000..d2cb20ed6 --- /dev/null +++ b/kapitan/inventory/inv_omegaconf/resolvers.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 + +# Copyright 2019 The Kapitan Authors +# SPDX-FileCopyrightText: 2020 The Kapitan Authors +# +# SPDX-License-Identifier: Apache-2.0 + +import copy +import logging +import os +import sys +from typing import Any + +from omegaconf import Container, ListMergeMode, Node, OmegaConf + +logger = logging.getLogger(__name__) + + +def key(_node_: Node): + """resolver function, that returns the name of its key""" + return _node_._key() + + +def parentkey(_parent_: Node): + """resolver function, that returns the name of its parent key""" + return _parent_._key() + + +def fullkey(_node_: Node): + """resolver function, that returns the full name of its key""" + return _node_._get_full_key("") + + +def access_key_with_dots(*key: str, _root_: Container): + """resolver function, that accesses a key with dots in it""" + value = _root_ + for part in key: + value = value[part] + + return value + + +def escape_interpolation(content: str): + """resolver function that escapes an interpolation for the next resolving step""" + return f"\\${{{content}}}" + + +def merge(*args): + """resolver function, that merges omegaconf objects""" + return OmegaConf.merge(*args, list_merge_mode=ListMergeMode.EXTEND) + + +def to_dict(input_list: list): + """resolver function, that converts an object to a dict""" + if type(input_list) is not list or OmegaConf.is_list(input_list): + # warning: input is not list + return input_list + + if not all([type(item) is list or OmegaConf.is_dict(item) for item in input_list]): + # warning: some items are not dict + return input_list + + return {dict_key: item[dict_key] for item in input_list for dict_key in item} + + +def to_list(input_obj: Any): + """resolver function, that converts an object to a list""" + if type(input_obj) is dict or OmegaConf.is_dict(input_obj): + return [{k: v} for k, v in input_obj.items()] + + return list(input_obj) + + +def default(*args): + output = "" + for arg in args[:-1]: + output += "${oc.select:" + str(arg) + "," + + output += str(args[-1]) + output += "}" * (len(args) - 1) + return output + + +def relpath(absolute_path: str, _node_: Node): + """ + resolver function, that translates an absolute yaml-path to its relative path + """ + + node_parts = [] + path_parts = absolute_path.split(".") + relative_path = "" + + i = 0 + node = _node_ + while node._key() is not None: + node_parts.append(node._key()) + node = node._get_parent() + i += 1 + + node_parts.reverse() + + for idx, (path_part, node_path) in enumerate(zip(path_parts, node_parts)): + if not path_part == node_path: + rel_prefix = "." * (i - idx) if idx != 0 else "" + relative_path = rel_prefix + ".".join(path_parts[idx:]) + break + + if not relative_path: + # warning: self reference + return "SELF REFERENCE DETECTED" + + relative_interpolation = "${" + relative_path + "}" + + return relative_interpolation + + +def write_to_key(destination: str, origin: str, _root_): + """ + resolver function to write any content to different place in the inventory + NOTE: Behavior for lists is not well-defined + """ + # fetch and resolve content + try: + content = OmegaConf.select(_root_, origin) + if not content: + # warning: origin could not be found / empty content + return "NOT FOUND" + + # resolve relative interpolations + try: + # replace with OC.to_object(), when it supports escaped interpolations (wip) + # reference: https://github.com/omry/omegaconf/pull/1113 + config = copy.deepcopy(content) + OmegaConf.resolve(config, True) + except Exception as e: + # resolver error + logger.warning(e) + return "ERROR WHILE RESOLVING" + + # remove when issue above is resolved + OmegaConf.set_readonly(config, False, recursive=True) + + # write resolved content back to _root_ + OmegaConf.update(_root_, destination, config, merge=True, force_add=True) + except Exception as e: + raise e + return "DONE" + + +def from_file(file_path: str): + if os.path.isfile(file_path): + with open(file_path, "r") as f: + return f.read() + else: + logger.error(f"from_file: file {file_path} does not exist") + return "FILE NOT EXISTS" + + +def filename(_node_: Node): + return _node_._get_flag("filename") + + +def parent_filename(_parent_: Node): + return _parent_._get_flag("filename") + + +def path(_node_: Node): + return _node_._get_flag("path") + + +def parent_path(_parent_: Node): + return _parent_._get_flag("path") + + +def condition_if(condition: str, config: dict): + if bool(condition): + return config + else: + return {} + + +def condition_if_else(condition: str, config_if: dict, config_else: dict): + if bool(condition): + return config_if + else: + return config_else + + +def condition_not(condition: str): + return not bool(condition) + + +def condition_and(*conditions: str): + return all(conditions) + + +def condition_or(*conditions: str): + return any(conditions) + + +def condition_equal(*configs): + return all(config == configs[0] for config in configs) + + +def register_resolvers(inventory_path: str) -> None: + """register pre-defined and user-defined resolvers""" + replace = True + + # yaml key utility functions + OmegaConf.register_new_resolver("key", key, replace=replace) + OmegaConf.register_new_resolver("parentkey", parentkey, replace=replace) + OmegaConf.register_new_resolver("fullkey", fullkey, replace=replace) + OmegaConf.register_new_resolver("relpath", relpath, replace=replace) + + # yaml object utility functions + OmegaConf.register_new_resolver("access", access_key_with_dots, replace=replace) + OmegaConf.register_new_resolver("escape", escape_interpolation, replace=replace) + OmegaConf.register_new_resolver("merge", merge, replace=replace) + OmegaConf.register_new_resolver("dict", to_dict, replace=replace) + OmegaConf.register_new_resolver("list", to_list, replace=replace) + OmegaConf.register_new_resolver("add", lambda x, y: x + y, replace=replace) + OmegaConf.register_new_resolver("default", default, replace=replace) + OmegaConf.register_new_resolver("write", write_to_key, replace=replace) + OmegaConf.register_new_resolver("from_file", from_file, replace=replace) + OmegaConf.register_new_resolver("filename", filename, replace=replace) + OmegaConf.register_new_resolver("parent_filename", parent_filename, replace=replace) + OmegaConf.register_new_resolver("path", path, replace=replace) + OmegaConf.register_new_resolver("parent_path", parent_path, replace=replace) + + # boolean algebra + OmegaConf.register_new_resolver("if", condition_if, replace=replace) + OmegaConf.register_new_resolver("ifelse", condition_if_else, replace=replace) + OmegaConf.register_new_resolver("and", condition_and, replace=replace) + OmegaConf.register_new_resolver("or", condition_or, replace=replace) + OmegaConf.register_new_resolver("not", condition_not, replace=replace) + OmegaConf.register_new_resolver("equal", condition_equal, replace=replace) + + # user defined resolvers + user_resolver_file = os.path.join(inventory_path, "resolvers.py") + if os.path.exists(user_resolver_file): + try: + register_user_resolvers(inventory_path) + except: + logger.warning(f"Couldn't import {os.path.join(inventory_path, 'resolvers.py')}") + + +def register_user_resolvers(inventory_path: str) -> None: + """import user resolvers specified in inventory/resolvers.py""" + try: + import_path = os.path.join(os.getcwd(), inventory_path) + sys.path.append(import_path) + from resolvers import pass_resolvers + + funcs = pass_resolvers() + except ImportError: + logger.warning("resolvers.py must contain function 'pass_resolvers()'") + return + except Exception as e: + logger.error(f"resolvers.py: {e}") + return + + if not isinstance(funcs, dict): + logger.warning("pass_resolvers() should return a dict") + return + + import resolvers + + for name, func in funcs.items(): + try: + OmegaConf.register_new_resolver(name, func, replace=True) + except: + logger.warning(f"Could not load resolver {name}") diff --git a/kapitan/inventory/inventory.py b/kapitan/inventory/inventory.py index c97d4d3e3..64d58411d 100644 --- a/kapitan/inventory/inventory.py +++ b/kapitan/inventory/inventory.py @@ -132,6 +132,12 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal """ raise NotImplementedError + def migrate(self): + """ + migrate the inventory, e.g. change interpolation syntax to new syntax + """ + pass + def __getitem__(self, key): return self.inventory[key] diff --git a/kapitan/resources.py b/kapitan/resources.py index 7079fa34d..ab4289ef0 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -327,6 +327,10 @@ def get_inventory(inventory_path) -> Inventory: logger.debug(f"Backend {backend_id} is unknown, falling back to reclass as inventory backend") inventory_backend = ReclassInventory(inventory_path) + # migrate inventory to selected inventory backend + if cached.args.get("migrate"): + inventory_backend.migrate() + inventory_backend.search_targets() cached.inv = inventory_backend